Resumable video uploads (#3933)

* WIP: resumable video uploads

relates to #324

* fix review comments

* video upload: error handling

* fix audio upload

* fixes after self review

* Update server/controllers/api/videos/index.ts

Co-authored-by: Rigel Kent <par@rigelk.eu>

* Update server/middlewares/validators/videos/videos.ts

Co-authored-by: Rigel Kent <par@rigelk.eu>

* Update server/controllers/api/videos/index.ts

Co-authored-by: Rigel Kent <par@rigelk.eu>

* update after code review

* refactor upload route

- restore multipart upload route
- move resumable to dedicated upload-resumable route
- move checks to middleware
- do not leak internal fs structure in response

* fix yarn.lock upon rebase

* factorize addVideo for reuse in both endpoints

* add resumable upload API to openapi spec

* add initial test and test helper for resumable upload

* typings for videoAddResumable middleware

* avoid including aws and google packages via node-uploadx, by only including uploadx/core

* rename ex-isAudioBg to more explicit name mentioning it is a preview file for audio

* add video-upload-tmp-folder-cleaner job

* stronger typing of video upload middleware

* reduce dependency to @uploadx/core

* add audio upload test

* refactor resumable uploads cleanup from job to scheduler

* refactor resumable uploads scheduler to compare to last execution time

* make resumable upload validator to always cleanup on failure

* move legacy upload request building outside of uploadVideo test helper

* filter upload-resumable middlewares down to POST, PUT, DELETE

also begin to type metadata

* merge add duration functions

* stronger typings and documentation for uploadx behaviour, move init validator up

* refactor(client/video-edit): options > uploadxOptions

* refactor(client/video-edit): remove obsolete else

* scheduler/remove-dangling-resum: rename tag

* refactor(server/video): add UploadVideoFiles type

* refactor(mw/validators): restructure eslint disable

* refactor(mw/validators/videos): rename import

* refactor(client/vid-upload): rename html elem id

* refactor(sched/remove-dangl): move fn to method

* refactor(mw/async): add method typing

* refactor(mw/vali/video): double quote > single

* refactor(server/upload-resum): express use > all

* proper http methud enum server/middlewares/async.ts

* properly type http methods

* factorize common video upload validation steps

* add check for maximum partially uploaded file size

* fix audioBg use

* fix extname(filename) in addVideo

* document parameters for uploadx's resumable protocol

* clear META files in scheduler

* last audio refactor before cramming preview in the initial POST form data

* refactor as mulitpart/form-data initial post request

this allows preview/thumbnail uploads alongside the initial request,
and cleans up the upload form

* Add more tests for resumable uploads

* Refactor remove dangling resumable uploads

* Prepare changelog

* Add more resumable upload tests

* Remove user quota check for resumable uploads

* Fix upload error handler

* Update nginx template for upload-resumable

* Cleanup comment

* Remove unused express methods

* Prefer to use got instead of raw http

* Don't retry on error 500

Co-authored-by: Rigel Kent <par@rigelk.eu>
Co-authored-by: Rigel Kent <sendmemail@rigelk.eu>
Co-authored-by: Chocobozzz <me@florianbigard.com>

2 files changed, 153 insertions, 32 deletions

diff --git a/server/middlewares/async.ts b/server/middlewares/async.ts
index 3d6e38809..0faa4fb8c 100644
--- a/server/middlewares/async.ts
+++ b/server/middlewares/async.ts
@@ -3,6 +3,7 @@ import { NextFunction, Request, RequestHandler, Response } from 'express'
 import { ValidationChain } from 'express-validator'
 import { ExpressPromiseHandler } from '@server/types/express'
 import { retryTransactionWrapper } from '../helpers/database-utils'
+import { HttpMethod, HttpStatusCode } from '@shared/core-utils'
 
 // Syntactic sugar to avoid try/catch in express controllers
 // Thanks: https://medium.com/@Abazhenov/using-async-await-in-express-with-node-8-b8af872c0016
diff --git a/server/middlewares/validators/videos/videos.ts b/server/middlewares/validators/videos/videos.ts
index bb617d77c..d26bcd4a6 100644
--- a/server/middlewares/validators/videos/videos.ts
+++ b/server/middlewares/validators/videos/videos.ts
@@ -1,9 +1,10 @@
 import * as express from 'express'
-import { body, param, query, ValidationChain } from 'express-validator'
+import { body, header, param, query, ValidationChain } from 'express-validator'
+import { getResumableUploadPath } from '@server/helpers/upload'
 import { isAbleToUploadVideo } from '@server/lib/user'
 import { getServerActor } from '@server/models/application/application'
 import { ExpressPromiseHandler } from '@server/types/express'
-import { MVideoWithRights } from '@server/types/models'
+import { MUserAccountId, MVideoWithRights } from '@server/types/models'
 import { ServerErrorCode, UserRight, VideoChangeOwnershipStatus, VideoPrivacy } from '../../../../shared'
 import { HttpStatusCode } from '../../../../shared/core-utils/miscs/http-error-codes'
 import { VideoChangeOwnershipAccept } from '../../../../shared/models/videos/video-change-ownership-accept.model'
@@ -47,6 +48,7 @@ import {
   doesVideoExist,
   doesVideoFileOfVideoExist
 } from '../../../helpers/middlewares'
+import { deleteFileAndCatch } from '../../../helpers/utils'
 import { getVideoWithAttributes } from '../../../helpers/video'
 import { CONFIG } from '../../../initializers/config'
 import { CONSTRAINTS_FIELDS, OVERVIEWS } from '../../../initializers/constants'
@@ -57,7 +59,7 @@ import { VideoModel } from '../../../models/video/video'
 import { authenticatePromiseIfNeeded } from '../../auth'
 import { areValidationErrors } from '../utils'
 
-const videosAddValidator = getCommonVideoEditAttributes().concat([
+const videosAddLegacyValidator = getCommonVideoEditAttributes().concat([
   body('videofile')
     .custom((value, { req }) => isFileFieldValid(req.files, 'videofile'))
     .withMessage('Should have a file'),
@@ -73,54 +75,117 @@ const videosAddValidator = getCommonVideoEditAttributes().concat([
     logger.debug('Checking videosAdd parameters', { parameters: req.body, files: req.files })
 
     if (areValidationErrors(req, res)) return cleanUpReqFiles(req)
-    if (areErrorsInScheduleUpdate(req, res)) return cleanUpReqFiles(req)
 
-    const videoFile: Express.Multer.File & { duration?: number } = req.files['videofile'][0]
+    const videoFile: express.VideoUploadFile = req.files['videofile'][0]
     const user = res.locals.oauth.token.User
 
-    if (!await doesVideoChannelOfAccountExist(req.body.channelId, user, res)) return cleanUpReqFiles(req)
-
-    if (!isVideoFileMimeTypeValid(req.files)) {
-      res.status(HttpStatusCode.UNSUPPORTED_MEDIA_TYPE_415)
-         .json({
-           error: 'This file is not supported. Please, make sure it is of the following type: ' +
-                  CONSTRAINTS_FIELDS.VIDEOS.EXTNAME.join(', ')
-         })
-
+    if (!await commonVideoChecksPass({ req, res, user, videoFileSize: videoFile.size, files: req.files })) {
       return cleanUpReqFiles(req)
     }
 
-    if (!isVideoFileSizeValid(videoFile.size.toString())) {
-      res.status(HttpStatusCode.PAYLOAD_TOO_LARGE_413)
-         .json({
-           error: 'This file is too large.'
-         })
+    try {
+      if (!videoFile.duration) await addDurationToVideo(videoFile)
+    } catch (err) {
+      logger.error('Invalid input file in videosAddLegacyValidator.', { err })
+      res.status(HttpStatusCode.UNPROCESSABLE_ENTITY_422)
+         .json({ error: 'Video file unreadable.' })
 
       return cleanUpReqFiles(req)
     }
 
-    if (await isAbleToUploadVideo(user.id, videoFile.size) === false) {
-      res.status(HttpStatusCode.PAYLOAD_TOO_LARGE_413)
-         .json({ error: 'The user video quota is exceeded with this video.' })
+    if (!await isVideoAccepted(req, res, videoFile)) return cleanUpReqFiles(req)
 
-      return cleanUpReqFiles(req)
-    }
+    return next()
+  }
+])
+
+/**
+ * Gets called after the last PUT request
+ */
+const videosAddResumableValidator = [
+  async (req: express.Request, res: express.Response, next: express.NextFunction) => {
+    const user = res.locals.oauth.token.User
+
+    const body: express.CustomUploadXFile<express.UploadXFileMetadata> = req.body
+    const file = { ...body, duration: undefined, path: getResumableUploadPath(body.id), filename: body.metadata.filename }
+
+    const cleanup = () => deleteFileAndCatch(file.path)
 
-    let duration: number
+    if (!await doesVideoChannelOfAccountExist(file.metadata.channelId, user, res)) return cleanup()
 
     try {
-      duration = await getDurationFromVideoFile(videoFile.path)
+      if (!file.duration) await addDurationToVideo(file)
     } catch (err) {
-      logger.error('Invalid input file in videosAddValidator.', { err })
+      logger.error('Invalid input file in videosAddResumableValidator.', { err })
       res.status(HttpStatusCode.UNPROCESSABLE_ENTITY_422)
          .json({ error: 'Video file unreadable.' })
 
-      return cleanUpReqFiles(req)
+      return cleanup()
     }
 
-    videoFile.duration = duration
+    if (!await isVideoAccepted(req, res, file)) return cleanup()
 
-    if (!await isVideoAccepted(req, res, videoFile)) return cleanUpReqFiles(req)
+    res.locals.videoFileResumable = file
+
+    return next()
+  }
+]
+
+/**
+ * File is created in POST initialisation, and its body is saved as a 'metadata' field is saved by uploadx for later use.
+ * see https://github.com/kukhariev/node-uploadx/blob/dc9fb4a8ac5a6f481902588e93062f591ec6ef03/packages/core/src/handlers/uploadx.ts
+ *
+ * Uploadx doesn't use next() until the upload completes, so this middleware has to be placed before uploadx
+ * see https://github.com/kukhariev/node-uploadx/blob/dc9fb4a8ac5a6f481902588e93062f591ec6ef03/packages/core/src/handlers/base-handler.ts
+ *
+ */
+const videosAddResumableInitValidator = getCommonVideoEditAttributes().concat([
+  body('filename')
+    .isString()
+    .exists()
+    .withMessage('Should have a valid filename'),
+  body('name')
+    .trim()
+    .custom(isVideoNameValid)
+    .withMessage('Should have a valid name'),
+  body('channelId')
+    .customSanitizer(toIntOrNull)
+    .custom(isIdValid).withMessage('Should have correct video channel id'),
+
+  header('x-upload-content-length')
+    .isNumeric()
+    .exists()
+    .withMessage('Should specify the file length'),
+  header('x-upload-content-type')
+    .isString()
+    .exists()
+    .withMessage('Should specify the file mimetype'),
+
+  async (req: express.Request, res: express.Response, next: express.NextFunction) => {
+    const videoFileMetadata = {
+      mimetype: req.headers['x-upload-content-type'] as string,
+      size: +req.headers['x-upload-content-length'],
+      originalname: req.body.name
+    }
+
+    const user = res.locals.oauth.token.User
+    const cleanup = () => cleanUpReqFiles(req)
+
+    logger.debug('Checking videosAddResumableInitValidator parameters and headers', {
+      parameters: req.body,
+      headers: req.headers,
+      files: req.files
+    })
+
+    if (areValidationErrors(req, res)) return cleanup()
+
+    const files = { videofile: [ videoFileMetadata ] }
+    if (!await commonVideoChecksPass({ req, res, user, videoFileSize: videoFileMetadata.size, files })) return cleanup()
+
+    // multer required unsetting the Content-Type, now we can set it for node-uploadx
+    req.headers['content-type'] = 'application/json; charset=utf-8'
+    // place previewfile in metadata so that uploadx saves it in .META
+    if (req.files['previewfile']) req.body.previewfile = req.files['previewfile']
 
     return next()
   }
@@ -478,7 +543,10 @@ const commonVideosFiltersValidator = [
 // ---------------------------------------------------------------------------
 
 export {
-  videosAddValidator,
+  videosAddLegacyValidator,
+  videosAddResumableValidator,
+  videosAddResumableInitValidator,
+
   videosUpdateValidator,
   videosGetValidator,
   videoFileMetadataGetValidator,
@@ -515,7 +583,51 @@ function areErrorsInScheduleUpdate (req: express.Request, res: express.Response)
   return false
 }
 
-async function isVideoAccepted (req: express.Request, res: express.Response, videoFile: Express.Multer.File & { duration?: number }) {
+async function commonVideoChecksPass (parameters: {
+  req: express.Request
+  res: express.Response
+  user: MUserAccountId
+  videoFileSize: number
+  files: express.UploadFilesForCheck
+}): Promise<boolean> {
+  const { req, res, user, videoFileSize, files } = parameters
+
+  if (areErrorsInScheduleUpdate(req, res)) return false
+
+  if (!await doesVideoChannelOfAccountExist(req.body.channelId, user, res)) return false
+
+  if (!isVideoFileMimeTypeValid(files)) {
+    res.status(HttpStatusCode.UNSUPPORTED_MEDIA_TYPE_415)
+        .json({
+          error: 'This file is not supported. Please, make sure it is of the following type: ' +
+                CONSTRAINTS_FIELDS.VIDEOS.EXTNAME.join(', ')
+        })
+
+    return false
+  }
+
+  if (!isVideoFileSizeValid(videoFileSize.toString())) {
+    res.status(HttpStatusCode.PAYLOAD_TOO_LARGE_413)
+        .json({ error: 'This file is too large. It exceeds the maximum file size authorized.' })
+
+    return false
+  }
+
+  if (await isAbleToUploadVideo(user.id, videoFileSize) === false) {
+    res.status(HttpStatusCode.PAYLOAD_TOO_LARGE_413)
+        .json({ error: 'The user video quota is exceeded with this video.' })
+
+    return false
+  }
+
+  return true
+}
+
+export async function isVideoAccepted (
+  req: express.Request,
+  res: express.Response,
+  videoFile: express.VideoUploadFile
+) {
   // Check we accept this video
   const acceptParameters = {
     videoBody: req.body,
@@ -538,3 +650,11 @@ async function isVideoAccepted (req: express.Request, res: express.Response, vid
 
   return true
 }
+
+async function addDurationToVideo (videoFile: { path: string, duration?: number }) {
+  const duration: number = await getDurationFromVideoFile(videoFile.path)
+
+  if (isNaN(duration)) throw new Error(`Couldn't get video duration`)
+
+  videoFile.duration = duration
+}