server/controllers/api/videos/upload.ts

   1 import express from 'express'
   2 import { move } from 'fs-extra'
   3 import { basename } from 'path'
   4 import { getResumableUploadPath } from '@server/helpers/upload'
   5 import { getLocalVideoActivityPubUrl } from '@server/lib/activitypub/url'
   6 import { CreateJobArgument, CreateJobOptions, JobQueue } from '@server/lib/job-queue'
   7 import { Redis } from '@server/lib/redis'
   8 import { uploadx } from '@server/lib/uploadx'
   9 import { buildLocalVideoFromReq, buildMoveToObjectStorageJob, buildVideoThumbnailsFromReq, setVideoTags } from '@server/lib/video'
  10 import { buildNewFile } from '@server/lib/video-file'
  11 import { VideoPathManager } from '@server/lib/video-path-manager'
  12 import { buildNextVideoState } from '@server/lib/video-state'
  13 import { openapiOperationDoc } from '@server/middlewares/doc'
  14 import { VideoSourceModel } from '@server/models/video/video-source'
  15 import { MUserId, MVideoFile, MVideoFullLight } from '@server/types/models'
  16 import { uuidToShort } from '@shared/extra-utils'
  17 import { HttpStatusCode, VideoCreate, VideoState } from '@shared/models'
  18 import { auditLoggerFactory, getAuditIdFromRes, VideoAuditView } from '../../../helpers/audit-logger'
  19 import { createReqFiles } from '../../../helpers/express-utils'
  20 import { logger, loggerTagsFactory } from '../../../helpers/logger'
  21 import { MIMETYPES } from '../../../initializers/constants'
  22 import { sequelizeTypescript } from '../../../initializers/database'
  23 import { Hooks } from '../../../lib/plugins/hooks'
  24 import { generateVideoMiniature } from '../../../lib/thumbnail'
  25 import { autoBlacklistVideoIfNeeded } from '../../../lib/video-blacklist'
  26 import {
  27   asyncMiddleware,
  28   asyncRetryTransactionMiddleware,
  29   authenticate,
  30   videosAddLegacyValidator,
  31   videosAddResumableInitValidator,
  32   videosAddResumableValidator
  33 } from '../../../middlewares'
  34 import { ScheduleVideoUpdateModel } from '../../../models/video/schedule-video-update'
  35 import { VideoModel } from '../../../models/video/video'
  36
  37 const lTags = loggerTagsFactory('api', 'video')
  38 const auditLogger = auditLoggerFactory('videos')
  39 const uploadRouter = express.Router()
  40
  41 const reqVideoFileAdd = createReqFiles(
  42   [ 'videofile', 'thumbnailfile', 'previewfile' ],
  43   { ...MIMETYPES.VIDEO.MIMETYPE_EXT, ...MIMETYPES.IMAGE.MIMETYPE_EXT }
  44 )
  45
  46 const reqVideoFileAddResumable = createReqFiles(
  47   [ 'thumbnailfile', 'previewfile' ],
  48   MIMETYPES.IMAGE.MIMETYPE_EXT,
  49   getResumableUploadPath()
  50 )
  51
  52 uploadRouter.post('/upload',
  53   openapiOperationDoc({ operationId: 'uploadLegacy' }),
  54   authenticate,
  55   reqVideoFileAdd,
  56   asyncMiddleware(videosAddLegacyValidator),
  57   asyncRetryTransactionMiddleware(addVideoLegacy)
  58 )
  59
  60 uploadRouter.post('/upload-resumable',
  61   openapiOperationDoc({ operationId: 'uploadResumableInit' }),
  62   authenticate,
  63   reqVideoFileAddResumable,
  64   asyncMiddleware(videosAddResumableInitValidator),
  65   uploadx.upload
  66 )
  67
  68 uploadRouter.delete('/upload-resumable',
  69   authenticate,
  70   asyncMiddleware(deleteUploadResumableCache),
  71   uploadx.upload
  72 )
  73
  74 uploadRouter.put('/upload-resumable',
  75   openapiOperationDoc({ operationId: 'uploadResumable' }),
  76   authenticate,
  77   uploadx.upload, // uploadx doesn't next() before the file upload completes
  78   asyncMiddleware(videosAddResumableValidator),
  79   asyncMiddleware(addVideoResumable)
  80 )
  81
  82 // ---------------------------------------------------------------------------
  83
  84 export {
  85   uploadRouter
  86 }
  87
  88 // ---------------------------------------------------------------------------
  89
  90 async function addVideoLegacy (req: express.Request, res: express.Response) {
  91   // Uploading the video could be long
  92   // Set timeout to 10 minutes, as Express's default is 2 minutes
  93   req.setTimeout(1000 * 60 * 10, () => {
  94     logger.error('Video upload has timed out.')
  95     return res.fail({
  96       status: HttpStatusCode.REQUEST_TIMEOUT_408,
  97       message: 'Video upload has timed out.'
  98     })
  99   })
 100
 101   const videoPhysicalFile = req.files['videofile'][0]
 102   const videoInfo: VideoCreate = req.body
 103   const files = req.files
 104
 105   const response = await addVideo({ req, res, videoPhysicalFile, videoInfo, files })
 106
 107   return res.json(response)
 108 }
 109
 110 async function addVideoResumable (req: express.Request, res: express.Response) {
 111   const videoPhysicalFile = res.locals.videoFileResumable
 112   const videoInfo = videoPhysicalFile.metadata
 113   const files = { previewfile: videoInfo.previewfile }
 114
 115   const response = await addVideo({ req, res, videoPhysicalFile, videoInfo, files })
 116   await Redis.Instance.setUploadSession(req.query.upload_id, response)
 117
 118   return res.json(response)
 119 }
 120
 121 async function addVideo (options: {
 122   req: express.Request
 123   res: express.Response
 124   videoPhysicalFile: express.VideoUploadFile
 125   videoInfo: VideoCreate
 126   files: express.UploadFiles
 127 }) {
 128   const { req, res, videoPhysicalFile, videoInfo, files } = options
 129   const videoChannel = res.locals.videoChannel
 130   const user = res.locals.oauth.token.User
 131
 132   let videoData = buildLocalVideoFromReq(videoInfo, videoChannel.id)
 133   videoData = await Hooks.wrapObject(videoData, 'filter:api.video.upload.video-attribute.result')
 134
 135   videoData.state = buildNextVideoState()
 136   videoData.duration = videoPhysicalFile.duration // duration was added by a previous middleware
 137
 138   const video = new VideoModel(videoData) as MVideoFullLight
 139   video.VideoChannel = videoChannel
 140   video.url = getLocalVideoActivityPubUrl(video) // We use the UUID, so set the URL after building the object
 141
 142   const videoFile = await buildNewFile({ path: videoPhysicalFile.path, mode: 'web-video' })
 143   const originalFilename = videoPhysicalFile.originalname
 144
 145   // Move physical file
 146   const destination = VideoPathManager.Instance.getFSVideoFileOutputPath(video, videoFile)
 147   await move(videoPhysicalFile.path, destination)
 148   // This is important in case if there is another attempt in the retry process
 149   videoPhysicalFile.filename = basename(destination)
 150   videoPhysicalFile.path = destination
 151
 152   const [ thumbnailModel, previewModel ] = await buildVideoThumbnailsFromReq({
 153     video,
 154     files,
 155     fallback: type => generateVideoMiniature({ video, videoFile, type })
 156   })
 157
 158   const { videoCreated } = await sequelizeTypescript.transaction(async t => {
 159     const sequelizeOptions = { transaction: t }
 160
 161     const videoCreated = await video.save(sequelizeOptions) as MVideoFullLight
 162
 163     await videoCreated.addAndSaveThumbnail(thumbnailModel, t)
 164     await videoCreated.addAndSaveThumbnail(previewModel, t)
 165
 166     // Do not forget to add video channel information to the created video
 167     videoCreated.VideoChannel = res.locals.videoChannel
 168
 169     videoFile.videoId = video.id
 170     await videoFile.save(sequelizeOptions)
 171
 172     video.VideoFiles = [ videoFile ]
 173
 174     await VideoSourceModel.create({
 175       filename: originalFilename,
 176       videoId: video.id
 177     }, { transaction: t })
 178
 179     await setVideoTags({ video, tags: videoInfo.tags, transaction: t })
 180
 181     // Schedule an update in the future?
 182     if (videoInfo.scheduleUpdate) {
 183       await ScheduleVideoUpdateModel.create({
 184         videoId: video.id,
 185         updateAt: new Date(videoInfo.scheduleUpdate.updateAt),
 186         privacy: videoInfo.scheduleUpdate.privacy || null
 187       }, sequelizeOptions)
 188     }
 189
 190     await autoBlacklistVideoIfNeeded({
 191       video,
 192       user,
 193       isRemote: false,
 194       isNew: true,
 195       transaction: t
 196     })
 197
 198     auditLogger.create(getAuditIdFromRes(res), new VideoAuditView(videoCreated.toFormattedDetailsJSON()))
 199     logger.info('Video with name %s and uuid %s created.', videoInfo.name, videoCreated.uuid, lTags(videoCreated.uuid))
 200
 201     return { videoCreated }
 202   })
 203
 204   // Channel has a new content, set as updated
 205   await videoCreated.VideoChannel.setAsUpdated()
 206
 207   addVideoJobsAfterUpload(videoCreated, videoFile, user)
 208     .catch(err => logger.error('Cannot build new video jobs of %s.', videoCreated.uuid, { err, ...lTags(videoCreated.uuid) }))
 209
 210   Hooks.runAction('action:api.video.uploaded', { video: videoCreated, req, res })
 211
 212   return {
 213     video: {
 214       id: videoCreated.id,
 215       shortUUID: uuidToShort(videoCreated.uuid),
 216       uuid: videoCreated.uuid
 217     }
 218   }
 219 }
 220
 221 async function addVideoJobsAfterUpload (video: MVideoFullLight, videoFile: MVideoFile, user: MUserId) {
 222   const jobs: (CreateJobArgument & CreateJobOptions)[] = [
 223     {
 224       type: 'manage-video-torrent' as 'manage-video-torrent',
 225       payload: {
 226         videoId: video.id,
 227         videoFileId: videoFile.id,
 228         action: 'create'
 229       }
 230     },
 231
 232     {
 233       type: 'notify',
 234       payload: {
 235         action: 'new-video',
 236         videoUUID: video.uuid
 237       }
 238     },
 239
 240     {
 241       type: 'federate-video' as 'federate-video',
 242       payload: {
 243         videoUUID: video.uuid,
 244         isNewVideo: true
 245       }
 246     }
 247   ]
 248
 249   if (video.state === VideoState.TO_MOVE_TO_EXTERNAL_STORAGE) {
 250     jobs.push(await buildMoveToObjectStorageJob({ video, previousVideoState: undefined }))
 251   }
 252
 253   if (video.state === VideoState.TO_TRANSCODE) {
 254     jobs.push({
 255       type: 'transcoding-job-builder' as 'transcoding-job-builder',
 256       payload: {
 257         videoUUID: video.uuid,
 258         optimizeJob: {
 259           isNewVideo: true
 260         }
 261       }
 262     })
 263   }
 264
 265   return JobQueue.Instance.createSequentialJobFlow(...jobs)
 266 }
 267
 268 async function deleteUploadResumableCache (req: express.Request, res: express.Response, next: express.NextFunction) {
 269   await Redis.Instance.deleteUploadSession(req.query.upload_id)
 270
 271   return next()
 272 }