Resumable video uploads (#3933)

* WIP: resumable video uploads

relates to #324

* fix review comments

* video upload: error handling

* fix audio upload

* fixes after self review

* Update server/controllers/api/videos/index.ts

Co-authored-by: Rigel Kent <par@rigelk.eu>

* Update server/middlewares/validators/videos/videos.ts

Co-authored-by: Rigel Kent <par@rigelk.eu>

* Update server/controllers/api/videos/index.ts

Co-authored-by: Rigel Kent <par@rigelk.eu>

* update after code review

* refactor upload route

- restore multipart upload route
- move resumable to dedicated upload-resumable route
- move checks to middleware
- do not leak internal fs structure in response

* fix yarn.lock upon rebase

* factorize addVideo for reuse in both endpoints

* add resumable upload API to openapi spec

* add initial test and test helper for resumable upload

* typings for videoAddResumable middleware

* avoid including aws and google packages via node-uploadx, by only including uploadx/core

* rename ex-isAudioBg to more explicit name mentioning it is a preview file for audio

* add video-upload-tmp-folder-cleaner job

* stronger typing of video upload middleware

* reduce dependency to @uploadx/core

* add audio upload test

* refactor resumable uploads cleanup from job to scheduler

* refactor resumable uploads scheduler to compare to last execution time

* make resumable upload validator to always cleanup on failure

* move legacy upload request building outside of uploadVideo test helper

* filter upload-resumable middlewares down to POST, PUT, DELETE

also begin to type metadata

* merge add duration functions

* stronger typings and documentation for uploadx behaviour, move init validator up

* refactor(client/video-edit): options > uploadxOptions

* refactor(client/video-edit): remove obsolete else

* scheduler/remove-dangling-resum: rename tag

* refactor(server/video): add UploadVideoFiles type

* refactor(mw/validators): restructure eslint disable

* refactor(mw/validators/videos): rename import

* refactor(client/vid-upload): rename html elem id

* refactor(sched/remove-dangl): move fn to method

* refactor(mw/async): add method typing

* refactor(mw/vali/video): double quote > single

* refactor(server/upload-resum): express use > all

* proper http methud enum server/middlewares/async.ts

* properly type http methods

* factorize common video upload validation steps

* add check for maximum partially uploaded file size

* fix audioBg use

* fix extname(filename) in addVideo

* document parameters for uploadx's resumable protocol

* clear META files in scheduler

* last audio refactor before cramming preview in the initial POST form data

* refactor as mulitpart/form-data initial post request

this allows preview/thumbnail uploads alongside the initial request,
and cleans up the upload form

* Add more tests for resumable uploads

* Refactor remove dangling resumable uploads

* Prepare changelog

* Add more resumable upload tests

* Remove user quota check for resumable uploads

* Fix upload error handler

* Update nginx template for upload-resumable

* Cleanup comment

* Remove unused express methods

* Prefer to use got instead of raw http

* Don't retry on error 500

Co-authored-by: Rigel Kent <par@rigelk.eu>
Co-authored-by: Rigel Kent <sendmemail@rigelk.eu>
Co-authored-by: Chocobozzz <me@florianbigard.com>

5 files changed, 35 insertions, 12 deletions

diff --git a/server/helpers/custom-validators/misc.ts b/server/helpers/custom-validators/misc.ts
index effdd98cb..fd3b45804 100644
--- a/server/helpers/custom-validators/misc.ts
+++ b/server/helpers/custom-validators/misc.ts
@@ -1,6 +1,7 @@
 import 'multer'
-import validator from 'validator'
+import { UploadFilesForCheck } from 'express'
 import { sep } from 'path'
+import validator from 'validator'
 
 function exists (value: any) {
   return value !== undefined && value !== null
@@ -108,7 +109,7 @@ function isFileFieldValid (
 }
 
 function isFileMimeTypeValid (
-  files: { [ fieldname: string ]: Express.Multer.File[] } | Express.Multer.File[],
+  files: UploadFilesForCheck,
   mimeTypeRegex: string,
   field: string,
   optional = false
diff --git a/server/helpers/custom-validators/videos.ts b/server/helpers/custom-validators/videos.ts
index 87966798f..b33e088eb 100644
--- a/server/helpers/custom-validators/videos.ts
+++ b/server/helpers/custom-validators/videos.ts
@@ -1,4 +1,6 @@
+import { UploadFilesForCheck } from 'express'
 import { values } from 'lodash'
+import * as magnetUtil from 'magnet-uri'
 import validator from 'validator'
 import { VideoFilter, VideoPrivacy, VideoRateType } from '../../../shared'
 import {
@@ -6,13 +8,12 @@ import {
   MIMETYPES,
   VIDEO_CATEGORIES,
   VIDEO_LICENCES,
+  VIDEO_LIVE,
   VIDEO_PRIVACIES,
   VIDEO_RATE_TYPES,
-  VIDEO_STATES,
-  VIDEO_LIVE
+  VIDEO_STATES
 } from '../../initializers/constants'
 import { exists, isArray, isDateValid, isFileMimeTypeValid, isFileValid } from './misc'
-import * as magnetUtil from 'magnet-uri'
 
 const VIDEOS_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.VIDEOS
 
@@ -81,7 +82,7 @@ function isVideoFileExtnameValid (value: string) {
   return exists(value) && (value === VIDEO_LIVE.EXTENSION || MIMETYPES.VIDEO.EXT_MIMETYPE[value] !== undefined)
 }
 
-function isVideoFileMimeTypeValid (files: { [ fieldname: string ]: Express.Multer.File[] } | Express.Multer.File[]) {
+function isVideoFileMimeTypeValid (files: UploadFilesForCheck) {
   return isFileMimeTypeValid(files, MIMETYPES.VIDEO.MIMETYPES_REGEX, 'videofile')
 }
 
diff --git a/server/helpers/express-utils.ts b/server/helpers/express-utils.ts
index c0d3f8f32..ede22a3cc 100644
--- a/server/helpers/express-utils.ts
+++ b/server/helpers/express-utils.ts
@@ -2,7 +2,7 @@ import * as express from 'express'
 import * as multer from 'multer'
 import { REMOTE_SCHEME } from '../initializers/constants'
 import { logger } from './logger'
-import { deleteFileAsync, generateRandomString } from './utils'
+import { deleteFileAndCatch, generateRandomString } from './utils'
 import { extname } from 'path'
 import { isArray } from './custom-validators/misc'
 import { CONFIG } from '../initializers/config'
@@ -36,15 +36,15 @@ function cleanUpReqFiles (req: { files: { [fieldname: string]: Express.Multer.Fi
   if (!files) return
 
   if (isArray(files)) {
-    (files as Express.Multer.File[]).forEach(f => deleteFileAsync(f.path))
+    (files as Express.Multer.File[]).forEach(f => deleteFileAndCatch(f.path))
     return
   }
 
   for (const key of Object.keys(files)) {
     const file = files[key]
 
-    if (isArray(file)) file.forEach(f => deleteFileAsync(f.path))
-    else deleteFileAsync(file.path)
+    if (isArray(file)) file.forEach(f => deleteFileAndCatch(f.path))
+    else deleteFileAndCatch(file.path)
   }
 }
 
diff --git a/server/helpers/upload.ts b/server/helpers/upload.ts
new file mode 100644
index 000000000..030a6b7d5
--- /dev/null
+++ b/server/helpers/upload.ts
@@ -0,0 +1,21 @@
+import { METAFILE_EXTNAME } from '@uploadx/core'
+import { remove } from 'fs-extra'
+import { join } from 'path'
+import { RESUMABLE_UPLOAD_DIRECTORY } from '../initializers/constants'
+
+function getResumableUploadPath (filename?: string) {
+  if (filename) return join(RESUMABLE_UPLOAD_DIRECTORY, filename)
+
+  return RESUMABLE_UPLOAD_DIRECTORY
+}
+
+function deleteResumableUploadMetaFile (filepath: string) {
+  return remove(filepath + METAFILE_EXTNAME)
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+  getResumableUploadPath,
+  deleteResumableUploadMetaFile
+}
diff --git a/server/helpers/utils.ts b/server/helpers/utils.ts
index 0545e8996..6c95a43b6 100644
--- a/server/helpers/utils.ts
+++ b/server/helpers/utils.ts
@@ -6,7 +6,7 @@ import { CONFIG } from '../initializers/config'
 import { execPromise, execPromise2, randomBytesPromise, sha256 } from './core-utils'
 import { logger } from './logger'
 
-function deleteFileAsync (path: string) {
+function deleteFileAndCatch (path: string) {
   remove(path)
     .catch(err => logger.error('Cannot delete the file %s asynchronously.', path, { err }))
 }
@@ -83,7 +83,7 @@ function getUUIDFromFilename (filename: string) {
 // ---------------------------------------------------------------------------
 
 export {
-  deleteFileAsync,
+  deleteFileAndCatch,
   generateRandomString,
   getFormattedObjects,
   getSecureTorrentName,