Resumable video uploads (#3933)

* WIP: resumable video uploads

relates to #324

* fix review comments

* video upload: error handling

* fix audio upload

* fixes after self review

* Update server/controllers/api/videos/index.ts

Co-authored-by: Rigel Kent <par@rigelk.eu>

* Update server/middlewares/validators/videos/videos.ts

Co-authored-by: Rigel Kent <par@rigelk.eu>

* Update server/controllers/api/videos/index.ts

Co-authored-by: Rigel Kent <par@rigelk.eu>

* update after code review

* refactor upload route

- restore multipart upload route
- move resumable to dedicated upload-resumable route
- move checks to middleware
- do not leak internal fs structure in response

* fix yarn.lock upon rebase

* factorize addVideo for reuse in both endpoints

* add resumable upload API to openapi spec

* add initial test and test helper for resumable upload

* typings for videoAddResumable middleware

* avoid including aws and google packages via node-uploadx, by only including uploadx/core

* rename ex-isAudioBg to more explicit name mentioning it is a preview file for audio

* add video-upload-tmp-folder-cleaner job

* stronger typing of video upload middleware

* reduce dependency to @uploadx/core

* add audio upload test

* refactor resumable uploads cleanup from job to scheduler

* refactor resumable uploads scheduler to compare to last execution time

* make resumable upload validator to always cleanup on failure

* move legacy upload request building outside of uploadVideo test helper

* filter upload-resumable middlewares down to POST, PUT, DELETE

also begin to type metadata

* merge add duration functions

* stronger typings and documentation for uploadx behaviour, move init validator up

* refactor(client/video-edit): options > uploadxOptions

* refactor(client/video-edit): remove obsolete else

* scheduler/remove-dangling-resum: rename tag

* refactor(server/video): add UploadVideoFiles type

* refactor(mw/validators): restructure eslint disable

* refactor(mw/validators/videos): rename import

* refactor(client/vid-upload): rename html elem id

* refactor(sched/remove-dangl): move fn to method

* refactor(mw/async): add method typing

* refactor(mw/vali/video): double quote > single

* refactor(server/upload-resum): express use > all

* proper http methud enum server/middlewares/async.ts

* properly type http methods

* factorize common video upload validation steps

* add check for maximum partially uploaded file size

* fix audioBg use

* fix extname(filename) in addVideo

* document parameters for uploadx's resumable protocol

* clear META files in scheduler

* last audio refactor before cramming preview in the initial POST form data

* refactor as mulitpart/form-data initial post request

this allows preview/thumbnail uploads alongside the initial request,
and cleans up the upload form

* Add more tests for resumable uploads

* Refactor remove dangling resumable uploads

* Prepare changelog

* Add more resumable upload tests

* Remove user quota check for resumable uploads

* Fix upload error handler

* Update nginx template for upload-resumable

* Cleanup comment

* Remove unused express methods

* Prefer to use got instead of raw http

* Don't retry on error 500

Co-authored-by: Rigel Kent <par@rigelk.eu>
Co-authored-by: Rigel Kent <sendmemail@rigelk.eu>
Co-authored-by: Chocobozzz <me@florianbigard.com>

1 files changed, 91 insertions, 49 deletions

diff --git a/server/typings/express/index.d.ts b/server/typings/express/index.d.ts
index cf3e7ae34..55b6e0039 100644
--- a/server/typings/express/index.d.ts
+++ b/server/typings/express/index.d.ts
@@ -19,6 +19,9 @@ import { MPlugin, MServer, MServerBlocklist } from '@server/types/models/server'
 import { MVideoImportDefault } from '@server/types/models/video/video-import'
 import { MVideoPlaylistElement, MVideoPlaylistElementVideoUrlPlaylistPrivacy } from '@server/types/models/video/video-playlist-element'
 import { MAccountVideoRateAccountVideo } from '@server/types/models/video/video-rate'
+import { HttpMethod } from '@shared/core-utils/miscs/http-methods'
+import { VideoCreate } from '@shared/models'
+import { File as UploadXFile, Metadata } from '@uploadx/core'
 import { RegisteredPlugin } from '../../lib/plugins/plugin-manager'
 import {
   MAccountDefault,
@@ -37,86 +40,125 @@ import {
   MVideoThumbnail,
   MVideoWithRights
 } from '../../types/models'
-
 declare module 'express' {
   export interface Request {
     query: any
+    method: HttpMethod
   }
-  interface Response {
-    locals: PeerTubeLocals
+
+  // Upload using multer or uploadx middleware
+  export type MulterOrUploadXFile = UploadXFile | Express.Multer.File
+
+  export type UploadFiles = {
+    [fieldname: string]: MulterOrUploadXFile[]
+  } | MulterOrUploadXFile[]
+
+  // Partial object used by some functions to check the file mimetype/extension
+  export type UploadFileForCheck = {
+    originalname: string
+    mimetype: string
   }
-}
 
-interface PeerTubeLocals {
-  videoAll?: MVideoFullLight
-  onlyImmutableVideo?: MVideoImmutable
-  onlyVideo?: MVideoThumbnail
-  onlyVideoWithRights?: MVideoWithRights
-  videoId?: MVideoIdThumbnail
+  export type UploadFilesForCheck = {
+    [fieldname: string]: UploadFileForCheck[]
+  } | UploadFileForCheck[]
 
-  videoLive?: MVideoLive
+  // Upload file with a duration added by our middleware
+  export type VideoUploadFile = Pick<Express.Multer.File, 'path' | 'filename' | 'size'> & {
+    duration: number
+  }
 
-  videoShare?: MVideoShareActor
+  // Extends Metadata property of UploadX object
+  export type UploadXFileMetadata = Metadata & VideoCreate & {
+    previewfile: Express.Multer.File[]
+    thumbnailfile: Express.Multer.File[]
+  }
 
-  videoFile?: MVideoFile
+  // Our custom UploadXFile object using our custom metadata
+  export type CustomUploadXFile <T extends Metadata> = UploadXFile & { metadata: T }
 
-  videoImport?: MVideoImportDefault
+  export type EnhancedUploadXFile = CustomUploadXFile<UploadXFileMetadata> & {
+    duration: number
+    path: string
+    filename: string
+  }
 
-  videoBlacklist?: MVideoBlacklist
+  // Extends locals property from Response
+  interface Response {
+    locals: {
+      videoAll?: MVideoFullLight
+      onlyImmutableVideo?: MVideoImmutable
+      onlyVideo?: MVideoThumbnail
+      onlyVideoWithRights?: MVideoWithRights
+      videoId?: MVideoIdThumbnail
 
-  videoCaption?: MVideoCaptionVideo
+      videoLive?: MVideoLive
 
-  abuse?: MAbuseReporter
-  abuseMessage?: MAbuseMessage
+      videoShare?: MVideoShareActor
 
-  videoStreamingPlaylist?: MStreamingPlaylist
+      videoFile?: MVideoFile
 
-  videoChannel?: MChannelBannerAccountDefault
+      videoFileResumable?: EnhancedUploadXFile
 
-  videoPlaylistFull?: MVideoPlaylistFull
-  videoPlaylistSummary?: MVideoPlaylistFullSummary
+      videoImport?: MVideoImportDefault
 
-  videoPlaylistElement?: MVideoPlaylistElement
-  videoPlaylistElementAP?: MVideoPlaylistElementVideoUrlPlaylistPrivacy
+      videoBlacklist?: MVideoBlacklist
 
-  accountVideoRate?: MAccountVideoRateAccountVideo
+      videoCaption?: MVideoCaptionVideo
 
-  videoCommentFull?: MCommentOwnerVideoReply
-  videoCommentThread?: MComment
+      abuse?: MAbuseReporter
+      abuseMessage?: MAbuseMessage
 
-  follow?: MActorFollowActorsDefault
-  subscription?: MActorFollowActorsDefaultSubscription
+      videoStreamingPlaylist?: MStreamingPlaylist
 
-  nextOwner?: MAccountDefault
-  videoChangeOwnership?: MVideoChangeOwnershipFull
+      videoChannel?: MChannelBannerAccountDefault
 
-  account?: MAccountDefault
+      videoPlaylistFull?: MVideoPlaylistFull
+      videoPlaylistSummary?: MVideoPlaylistFullSummary
 
-  actorUrl?: MActorUrl
-  actorFull?: MActorFull
+      videoPlaylistElement?: MVideoPlaylistElement
+      videoPlaylistElementAP?: MVideoPlaylistElementVideoUrlPlaylistPrivacy
 
-  user?: MUserDefault
+      accountVideoRate?: MAccountVideoRateAccountVideo
 
-  server?: MServer
+      videoCommentFull?: MCommentOwnerVideoReply
+      videoCommentThread?: MComment
 
-  videoRedundancy?: MVideoRedundancyVideo
+      follow?: MActorFollowActorsDefault
+      subscription?: MActorFollowActorsDefaultSubscription
 
-  accountBlock?: MAccountBlocklist
-  serverBlock?: MServerBlocklist
+      nextOwner?: MAccountDefault
+      videoChangeOwnership?: MVideoChangeOwnershipFull
 
-  oauth?: {
-    token: MOAuthTokenUser
-  }
+      account?: MAccountDefault
 
-  signature?: {
-    actor: MActorAccountChannelId
-  }
+      actorUrl?: MActorUrl
+      actorFull?: MActorFull
+
+      user?: MUserDefault
+
+      server?: MServer
+
+      videoRedundancy?: MVideoRedundancyVideo
 
-  authenticated?: boolean
+      accountBlock?: MAccountBlocklist
+      serverBlock?: MServerBlocklist
 
-  registeredPlugin?: RegisteredPlugin
+      oauth?: {
+        token: MOAuthTokenUser
+      }
 
-  externalAuth?: RegisterServerAuthExternalOptions
+      signature?: {
+        actor: MActorAccountChannelId
+      }
 
-  plugin?: MPlugin
+      authenticated?: boolean
+
+      registeredPlugin?: RegisteredPlugin
+
+      externalAuth?: RegisterServerAuthExternalOptions
+
+      plugin?: MPlugin
+    }
+  }
 }