Resumable video uploads (#3933)

* WIP: resumable video uploads

relates to #324

* fix review comments

* video upload: error handling

* fix audio upload

* fixes after self review

* Update server/controllers/api/videos/index.ts

Co-authored-by: Rigel Kent <par@rigelk.eu>

* Update server/middlewares/validators/videos/videos.ts

Co-authored-by: Rigel Kent <par@rigelk.eu>

* Update server/controllers/api/videos/index.ts

Co-authored-by: Rigel Kent <par@rigelk.eu>

* update after code review

* refactor upload route

- restore multipart upload route
- move resumable to dedicated upload-resumable route
- move checks to middleware
- do not leak internal fs structure in response

* fix yarn.lock upon rebase

* factorize addVideo for reuse in both endpoints

* add resumable upload API to openapi spec

* add initial test and test helper for resumable upload

* typings for videoAddResumable middleware

* avoid including aws and google packages via node-uploadx, by only including uploadx/core

* rename ex-isAudioBg to more explicit name mentioning it is a preview file for audio

* add video-upload-tmp-folder-cleaner job

* stronger typing of video upload middleware

* reduce dependency to @uploadx/core

* add audio upload test

* refactor resumable uploads cleanup from job to scheduler

* refactor resumable uploads scheduler to compare to last execution time

* make resumable upload validator to always cleanup on failure

* move legacy upload request building outside of uploadVideo test helper

* filter upload-resumable middlewares down to POST, PUT, DELETE

also begin to type metadata

* merge add duration functions

* stronger typings and documentation for uploadx behaviour, move init validator up

* refactor(client/video-edit): options > uploadxOptions

* refactor(client/video-edit): remove obsolete else

* scheduler/remove-dangling-resum: rename tag

* refactor(server/video): add UploadVideoFiles type

* refactor(mw/validators): restructure eslint disable

* refactor(mw/validators/videos): rename import

* refactor(client/vid-upload): rename html elem id

* refactor(sched/remove-dangl): move fn to method

* refactor(mw/async): add method typing

* refactor(mw/vali/video): double quote > single

* refactor(server/upload-resum): express use > all

* proper http methud enum server/middlewares/async.ts

* properly type http methods

* factorize common video upload validation steps

* add check for maximum partially uploaded file size

* fix audioBg use

* fix extname(filename) in addVideo

* document parameters for uploadx's resumable protocol

* clear META files in scheduler

* last audio refactor before cramming preview in the initial POST form data

* refactor as mulitpart/form-data initial post request

this allows preview/thumbnail uploads alongside the initial request,
and cleans up the upload form

* Add more tests for resumable uploads

* Refactor remove dangling resumable uploads

* Prepare changelog

* Add more resumable upload tests

* Remove user quota check for resumable uploads

* Fix upload error handler

* Update nginx template for upload-resumable

* Cleanup comment

* Remove unused express methods

* Prefer to use got instead of raw http

* Don't retry on error 500

Co-authored-by: Rigel Kent <par@rigelk.eu>
Co-authored-by: Rigel Kent <sendmemail@rigelk.eu>
Co-authored-by: Chocobozzz <me@florianbigard.com>

2 files changed, 103 insertions, 20 deletions

diff --git a/server/controllers/api/server/debug.ts b/server/controllers/api/server/debug.ts
index 7787186be..ff0d9ca3c 100644
--- a/server/controllers/api/server/debug.ts
+++ b/server/controllers/api/server/debug.ts
@@ -1,4 +1,6 @@
 import { InboxManager } from '@server/lib/activitypub/inbox-manager'
+import { RemoveDanglingResumableUploadsScheduler } from '@server/lib/schedulers/remove-dangling-resumable-uploads-scheduler'
+import { SendDebugCommand } from '@shared/models'
 import * as express from 'express'
 import { UserRight } from '../../../../shared/models/users'
 import { authenticate, ensureUserHasRight } from '../../../middlewares'
@@ -11,6 +13,12 @@ debugRouter.get('/debug',
   getDebug
 )
 
+debugRouter.post('/debug/run-command',
+  authenticate,
+  ensureUserHasRight(UserRight.MANAGE_DEBUG),
+  runCommand
+)
+
 // ---------------------------------------------------------------------------
 
 export {
@@ -25,3 +33,13 @@ function getDebug (req: express.Request, res: express.Response) {
     activityPubMessagesWaiting: InboxManager.Instance.getActivityPubMessagesWaiting()
   })
 }
+
+async function runCommand (req: express.Request, res: express.Response) {
+  const body: SendDebugCommand = req.body
+
+  if (body.command === 'remove-dandling-resumable-uploads') {
+    await RemoveDanglingResumableUploadsScheduler.Instance.execute()
+  }
+
+  return res.sendStatus(204)
+}
diff --git a/server/controllers/api/videos/index.ts b/server/controllers/api/videos/index.ts
index fbdb0f776..c32626d30 100644
--- a/server/controllers/api/videos/index.ts
+++ b/server/controllers/api/videos/index.ts
@@ -2,6 +2,7 @@ import * as express from 'express'
 import { move } from 'fs-extra'
 import { extname } from 'path'
 import toInt from 'validator/lib/toInt'
+import { deleteResumableUploadMetaFile, getResumableUploadPath } from '@server/helpers/upload'
 import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent'
 import { changeVideoChannelShare } from '@server/lib/activitypub/share'
 import { getLocalVideoActivityPubUrl } from '@server/lib/activitypub/url'
@@ -10,8 +11,9 @@ import { addOptimizeOrMergeAudioJob, buildLocalVideoFromReq, buildVideoThumbnail
 import { generateVideoFilename, getVideoFilePath } from '@server/lib/video-paths'
 import { getServerActor } from '@server/models/application/application'
 import { MVideo, MVideoFile, MVideoFullLight } from '@server/types/models'
+import { uploadx } from '@uploadx/core'
 import { VideoCreate, VideosCommonQuery, VideoState, VideoUpdate } from '../../../../shared'
-import { HttpStatusCode } from '../../../../shared/core-utils/miscs/http-error-codes'
+import { HttpStatusCode } from '../../../../shared/core-utils/miscs'
 import { auditLoggerFactory, getAuditIdFromRes, VideoAuditView } from '../../../helpers/audit-logger'
 import { resetSequelizeInstance, retryTransactionWrapper } from '../../../helpers/database-utils'
 import { buildNSFWFilter, createReqFiles, getCountVideos } from '../../../helpers/express-utils'
@@ -47,7 +49,9 @@ import {
   setDefaultPagination,
   setDefaultVideosSort,
   videoFileMetadataGetValidator,
-  videosAddValidator,
+  videosAddLegacyValidator,
+  videosAddResumableInitValidator,
+  videosAddResumableValidator,
   videosCustomGetValidator,
   videosGetValidator,
   videosRemoveValidator,
@@ -69,6 +73,7 @@ import { watchingRouter } from './watching'
 const lTags = loggerTagsFactory('api', 'video')
 const auditLogger = auditLoggerFactory('videos')
 const videosRouter = express.Router()
+const uploadxMiddleware = uploadx.upload({ directory: getResumableUploadPath() })
 
 const reqVideoFileAdd = createReqFiles(
   [ 'videofile', 'thumbnailfile', 'previewfile' ],
@@ -79,6 +84,16 @@ const reqVideoFileAdd = createReqFiles(
     previewfile: CONFIG.STORAGE.TMP_DIR
   }
 )
+
+const reqVideoFileAddResumable = createReqFiles(
+  [ 'thumbnailfile', 'previewfile' ],
+  MIMETYPES.IMAGE.MIMETYPE_EXT,
+  {
+    thumbnailfile: getResumableUploadPath(),
+    previewfile: getResumableUploadPath()
+  }
+)
+
 const reqVideoFileUpdate = createReqFiles(
   [ 'thumbnailfile', 'previewfile' ],
   MIMETYPES.IMAGE.MIMETYPE_EXT,
@@ -111,18 +126,39 @@ videosRouter.get('/',
   commonVideosFiltersValidator,
   asyncMiddleware(listVideos)
 )
+
+videosRouter.post('/upload',
+  authenticate,
+  reqVideoFileAdd,
+  asyncMiddleware(videosAddLegacyValidator),
+  asyncRetryTransactionMiddleware(addVideoLegacy)
+)
+
+videosRouter.post('/upload-resumable',
+  authenticate,
+  reqVideoFileAddResumable,
+  asyncMiddleware(videosAddResumableInitValidator),
+  uploadxMiddleware
+)
+
+videosRouter.delete('/upload-resumable',
+  authenticate,
+  uploadxMiddleware
+)
+
+videosRouter.put('/upload-resumable',
+  authenticate,
+  uploadxMiddleware, // uploadx doesn't use call next() before the file upload completes
+  asyncMiddleware(videosAddResumableValidator),
+  asyncMiddleware(addVideoResumable)
+)
+
 videosRouter.put('/:id',
   authenticate,
   reqVideoFileUpdate,
   asyncMiddleware(videosUpdateValidator),
   asyncRetryTransactionMiddleware(updateVideo)
 )
-videosRouter.post('/upload',
-  authenticate,
-  reqVideoFileAdd,
-  asyncMiddleware(videosAddValidator),
-  asyncRetryTransactionMiddleware(addVideo)
-)
 
 videosRouter.get('/:id/description',
   asyncMiddleware(videosGetValidator),
@@ -157,23 +193,23 @@ export {
 
 // ---------------------------------------------------------------------------
 
-function listVideoCategories (req: express.Request, res: express.Response) {
+function listVideoCategories (_req: express.Request, res: express.Response) {
   res.json(VIDEO_CATEGORIES)
 }
 
-function listVideoLicences (req: express.Request, res: express.Response) {
+function listVideoLicences (_req: express.Request, res: express.Response) {
   res.json(VIDEO_LICENCES)
 }
 
-function listVideoLanguages (req: express.Request, res: express.Response) {
+function listVideoLanguages (_req: express.Request, res: express.Response) {
   res.json(VIDEO_LANGUAGES)
 }
 
-function listVideoPrivacies (req: express.Request, res: express.Response) {
+function listVideoPrivacies (_req: express.Request, res: express.Response) {
   res.json(VIDEO_PRIVACIES)
 }
 
-async function addVideo (req: express.Request, res: express.Response) {
+async function addVideoLegacy (req: express.Request, res: express.Response) {
   // Uploading the video could be long
   // Set timeout to 10 minutes, as Express's default is 2 minutes
   req.setTimeout(1000 * 60 * 10, () => {
@@ -183,13 +219,42 @@ async function addVideo (req: express.Request, res: express.Response) {
 
   const videoPhysicalFile = req.files['videofile'][0]
   const videoInfo: VideoCreate = req.body
+  const files = req.files
+
+  return addVideo({ res, videoPhysicalFile, videoInfo, files })
+}
+
+async function addVideoResumable (_req: express.Request, res: express.Response) {
+  const videoPhysicalFile = res.locals.videoFileResumable
+  const videoInfo = videoPhysicalFile.metadata
+  const files = { previewfile: videoInfo.previewfile }
+
+  // Don't need the meta file anymore
+  await deleteResumableUploadMetaFile(videoPhysicalFile.path)
+
+  return addVideo({ res, videoPhysicalFile, videoInfo, files })
+}
 
-  const videoData = buildLocalVideoFromReq(videoInfo, res.locals.videoChannel.id)
-  videoData.state = CONFIG.TRANSCODING.ENABLED ? VideoState.TO_TRANSCODE : VideoState.PUBLISHED
-  videoData.duration = videoPhysicalFile['duration'] // duration was added by a previous middleware
+async function addVideo (options: {
+  res: express.Response
+  videoPhysicalFile: express.VideoUploadFile
+  videoInfo: VideoCreate
+  files: express.UploadFiles
+}) {
+  const { res, videoPhysicalFile, videoInfo, files } = options
+  const videoChannel = res.locals.videoChannel
+  const user = res.locals.oauth.token.User
+
+  const videoData = buildLocalVideoFromReq(videoInfo, videoChannel.id)
+
+  videoData.state = CONFIG.TRANSCODING.ENABLED
+    ? VideoState.TO_TRANSCODE
+    : VideoState.PUBLISHED
+
+  videoData.duration = videoPhysicalFile.duration // duration was added by a previous middleware
 
   const video = new VideoModel(videoData) as MVideoFullLight
-  video.VideoChannel = res.locals.videoChannel
+  video.VideoChannel = videoChannel
   video.url = getLocalVideoActivityPubUrl(video) // We use the UUID, so set the URL after building the object
 
   const videoFile = new VideoFileModel({
@@ -217,7 +282,7 @@ async function addVideo (req: express.Request, res: express.Response) {
 
   const [ thumbnailModel, previewModel ] = await buildVideoThumbnailsFromReq({
     video,
-    files: req.files,
+    files,
     fallback: type => generateVideoMiniature({ video, videoFile, type })
   })
 
@@ -253,7 +318,7 @@ async function addVideo (req: express.Request, res: express.Response) {
 
     await autoBlacklistVideoIfNeeded({
       video,
-      user: res.locals.oauth.token.User,
+      user,
       isRemote: false,
       isNew: true,
       transaction: t
@@ -282,7 +347,7 @@ async function addVideo (req: express.Request, res: express.Response) {
     .catch(err => logger.error('Cannot federate or notify video creation %s', video.url, { err, ...lTags(video.uuid) }))
 
   if (video.state === VideoState.TO_TRANSCODE) {
-    await addOptimizeOrMergeAudioJob(videoCreated, videoFile, res.locals.oauth.token.User)
+    await addOptimizeOrMergeAudioJob(videoCreated, videoFile, user)
   }
 
   Hooks.runAction('action:api.video.uploaded', { video: videoCreated })