Resumable video uploads (#3933)

* WIP: resumable video uploads

relates to #324

* fix review comments

* video upload: error handling

* fix audio upload

* fixes after self review

* Update server/controllers/api/videos/index.ts

Co-authored-by: Rigel Kent <par@rigelk.eu>

* Update server/middlewares/validators/videos/videos.ts

Co-authored-by: Rigel Kent <par@rigelk.eu>

* Update server/controllers/api/videos/index.ts

Co-authored-by: Rigel Kent <par@rigelk.eu>

* update after code review

* refactor upload route

- restore multipart upload route
- move resumable to dedicated upload-resumable route
- move checks to middleware
- do not leak internal fs structure in response

* fix yarn.lock upon rebase

* factorize addVideo for reuse in both endpoints

* add resumable upload API to openapi spec

* add initial test and test helper for resumable upload

* typings for videoAddResumable middleware

* avoid including aws and google packages via node-uploadx, by only including uploadx/core

* rename ex-isAudioBg to more explicit name mentioning it is a preview file for audio

* add video-upload-tmp-folder-cleaner job

* stronger typing of video upload middleware

* reduce dependency to @uploadx/core

* add audio upload test

* refactor resumable uploads cleanup from job to scheduler

* refactor resumable uploads scheduler to compare to last execution time

* make resumable upload validator to always cleanup on failure

* move legacy upload request building outside of uploadVideo test helper

* filter upload-resumable middlewares down to POST, PUT, DELETE

also begin to type metadata

* merge add duration functions

* stronger typings and documentation for uploadx behaviour, move init validator up

* refactor(client/video-edit): options > uploadxOptions

* refactor(client/video-edit): remove obsolete else

* scheduler/remove-dangling-resum: rename tag

* refactor(server/video): add UploadVideoFiles type

* refactor(mw/validators): restructure eslint disable

* refactor(mw/validators/videos): rename import

* refactor(client/vid-upload): rename html elem id

* refactor(sched/remove-dangl): move fn to method

* refactor(mw/async): add method typing

* refactor(mw/vali/video): double quote > single

* refactor(server/upload-resum): express use > all

* proper http methud enum server/middlewares/async.ts

* properly type http methods

* factorize common video upload validation steps

* add check for maximum partially uploaded file size

* fix audioBg use

* fix extname(filename) in addVideo

* document parameters for uploadx's resumable protocol

* clear META files in scheduler

* last audio refactor before cramming preview in the initial POST form data

* refactor as mulitpart/form-data initial post request

this allows preview/thumbnail uploads alongside the initial request,
and cleans up the upload form

* Add more tests for resumable uploads

* Refactor remove dangling resumable uploads

* Prepare changelog

* Add more resumable upload tests

* Remove user quota check for resumable uploads

* Fix upload error handler

* Update nginx template for upload-resumable

* Cleanup comment

* Remove unused express methods

* Prefer to use got instead of raw http

* Don't retry on error 500

Co-authored-by: Rigel Kent <par@rigelk.eu>
Co-authored-by: Rigel Kent <sendmemail@rigelk.eu>
Co-authored-by: Chocobozzz <me@florianbigard.com>

3 files changed, 66 insertions, 3 deletions

diff --git a/server/lib/moderation.ts b/server/lib/moderation.ts
index 5180b3299..925d64902 100644
--- a/server/lib/moderation.ts
+++ b/server/lib/moderation.ts
@@ -1,6 +1,8 @@
+import { VideoUploadFile } from 'express'
 import { PathLike } from 'fs-extra'
 import { Transaction } from 'sequelize/types'
 import { AbuseAuditView, auditLoggerFactory } from '@server/helpers/audit-logger'
+import { afterCommitIfTransaction } from '@server/helpers/database-utils'
 import { logger } from '@server/helpers/logger'
 import { AbuseModel } from '@server/models/abuse/abuse'
 import { VideoAbuseModel } from '@server/models/abuse/video-abuse'
@@ -28,7 +30,6 @@ import { VideoModel } from '../models/video/video'
 import { VideoCommentModel } from '../models/video/video-comment'
 import { sendAbuse } from './activitypub/send/send-flag'
 import { Notifier } from './notifier'
-import { afterCommitIfTransaction } from '@server/helpers/database-utils'
 
 export type AcceptResult = {
   accepted: boolean
@@ -38,7 +39,7 @@ export type AcceptResult = {
 // Can be filtered by plugins
 function isLocalVideoAccepted (object: {
   videoBody: VideoCreate
-  videoFile: Express.Multer.File & { duration?: number }
+  videoFile: VideoUploadFile
   user: UserModel
 }): AcceptResult {
   return { accepted: true }
diff --git a/server/lib/schedulers/remove-dangling-resumable-uploads-scheduler.ts b/server/lib/schedulers/remove-dangling-resumable-uploads-scheduler.ts
new file mode 100644
index 000000000..1acea7998
--- /dev/null
+++ b/server/lib/schedulers/remove-dangling-resumable-uploads-scheduler.ts
@@ -0,0 +1,61 @@
+import * as bluebird from 'bluebird'
+import { readdir, remove, stat } from 'fs-extra'
+import { logger, loggerTagsFactory } from '@server/helpers/logger'
+import { getResumableUploadPath } from '@server/helpers/upload'
+import { SCHEDULER_INTERVALS_MS } from '@server/initializers/constants'
+import { METAFILE_EXTNAME } from '@uploadx/core'
+import { AbstractScheduler } from './abstract-scheduler'
+
+const lTags = loggerTagsFactory('scheduler', 'resumable-upload', 'cleaner')
+
+export class RemoveDanglingResumableUploadsScheduler extends AbstractScheduler {
+
+  private static instance: AbstractScheduler
+  private lastExecutionTimeMs: number
+
+  protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.removeDanglingResumableUploads
+
+  private constructor () {
+    super()
+
+    this.lastExecutionTimeMs = new Date().getTime()
+  }
+
+  protected async internalExecute () {
+    const path = getResumableUploadPath()
+    const files = await readdir(path)
+
+    const metafiles = files.filter(f => f.endsWith(METAFILE_EXTNAME))
+
+    if (metafiles.length === 0) return
+
+    logger.debug('Reading resumable video upload folder %s with %d files', path, metafiles.length, lTags())
+
+    try {
+      await bluebird.map(metafiles, metafile => {
+        return this.deleteIfOlderThan(metafile, this.lastExecutionTimeMs)
+      }, { concurrency: 5 })
+    } catch (error) {
+      logger.error('Failed to handle file during resumable video upload folder cleanup', { error, ...lTags() })
+    } finally {
+      this.lastExecutionTimeMs = new Date().getTime()
+    }
+  }
+
+  private async deleteIfOlderThan (metafile: string, olderThan: number) {
+    const metafilePath = getResumableUploadPath(metafile)
+    const statResult = await stat(metafilePath)
+
+    // Delete uploads that started since a long time
+    if (statResult.ctimeMs < olderThan) {
+      await remove(metafilePath)
+
+      const datafile = metafilePath.replace(new RegExp(`${METAFILE_EXTNAME}$`), '')
+      await remove(datafile)
+    }
+  }
+
+  static get Instance () {
+    return this.instance || (this.instance = new this())
+  }
+}
diff --git a/server/lib/video.ts b/server/lib/video.ts
index 9469b8178..21e4b7ff2 100644
--- a/server/lib/video.ts
+++ b/server/lib/video.ts
@@ -1,3 +1,4 @@
+import { UploadFiles } from 'express'
 import { Transaction } from 'sequelize/types'
 import { DEFAULT_AUDIO_RESOLUTION, JOB_PRIORITY } from '@server/initializers/constants'
 import { sequelizeTypescript } from '@server/initializers/database'
@@ -32,7 +33,7 @@ function buildLocalVideoFromReq (videoInfo: VideoCreate, channelId: number): Fil
 
 async function buildVideoThumbnailsFromReq (options: {
   video: MVideoThumbnail
-  files: { [fieldname: string]: Express.Multer.File[] } | Express.Multer.File[]
+  files: UploadFiles
   fallback: (type: ThumbnailType) => Promise<MThumbnail>
   automaticallyGenerated?: boolean
 }) {