Resumable video uploads (#3933)

* WIP: resumable video uploads

relates to #324

* fix review comments

* video upload: error handling

* fix audio upload

* fixes after self review

* Update server/controllers/api/videos/index.ts

Co-authored-by: Rigel Kent <par@rigelk.eu>

* Update server/middlewares/validators/videos/videos.ts

Co-authored-by: Rigel Kent <par@rigelk.eu>

* Update server/controllers/api/videos/index.ts

Co-authored-by: Rigel Kent <par@rigelk.eu>

* update after code review

* refactor upload route

- restore multipart upload route
- move resumable to dedicated upload-resumable route
- move checks to middleware
- do not leak internal fs structure in response

* fix yarn.lock upon rebase

* factorize addVideo for reuse in both endpoints

* add resumable upload API to openapi spec

* add initial test and test helper for resumable upload

* typings for videoAddResumable middleware

* avoid including aws and google packages via node-uploadx, by only including uploadx/core

* rename ex-isAudioBg to more explicit name mentioning it is a preview file for audio

* add video-upload-tmp-folder-cleaner job

* stronger typing of video upload middleware

* reduce dependency to @uploadx/core

* add audio upload test

* refactor resumable uploads cleanup from job to scheduler

* refactor resumable uploads scheduler to compare to last execution time

* make resumable upload validator to always cleanup on failure

* move legacy upload request building outside of uploadVideo test helper

* filter upload-resumable middlewares down to POST, PUT, DELETE

also begin to type metadata

* merge add duration functions

* stronger typings and documentation for uploadx behaviour, move init validator up

* refactor(client/video-edit): options > uploadxOptions

* refactor(client/video-edit): remove obsolete else

* scheduler/remove-dangling-resum: rename tag

* refactor(server/video): add UploadVideoFiles type

* refactor(mw/validators): restructure eslint disable

* refactor(mw/validators/videos): rename import

* refactor(client/vid-upload): rename html elem id

* refactor(sched/remove-dangl): move fn to method

* refactor(mw/async): add method typing

* refactor(mw/vali/video): double quote > single

* refactor(server/upload-resum): express use > all

* proper http methud enum server/middlewares/async.ts

* properly type http methods

* factorize common video upload validation steps

* add check for maximum partially uploaded file size

* fix audioBg use

* fix extname(filename) in addVideo

* document parameters for uploadx's resumable protocol

* clear META files in scheduler

* last audio refactor before cramming preview in the initial POST form data

* refactor as mulitpart/form-data initial post request

this allows preview/thumbnail uploads alongside the initial request,
and cleans up the upload form

* Add more tests for resumable uploads

* Refactor remove dangling resumable uploads

* Prepare changelog

* Add more resumable upload tests

* Remove user quota check for resumable uploads

* Fix upload error handler

* Update nginx template for upload-resumable

* Cleanup comment

* Remove unused express methods

* Prefer to use got instead of raw http

* Don't retry on error 500

Co-authored-by: Rigel Kent <par@rigelk.eu>
Co-authored-by: Rigel Kent <sendmemail@rigelk.eu>
Co-authored-by: Chocobozzz <me@florianbigard.com>

2 files changed, 9 insertions, 2 deletions

diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts
index f807a1e58..6f388420e 100644
--- a/server/initializers/constants.ts
+++ b/server/initializers/constants.ts
@@ -208,7 +208,8 @@ const SCHEDULER_INTERVALS_MS = {
   autoFollowIndexInstances: 60000 * 60 * 24, // 1 day
   removeOldViews: 60000 * 60 * 24, // 1 day
   removeOldHistory: 60000 * 60 * 24, // 1 day
-  updateInboxStats: 1000 * 60// 1 minute
+  updateInboxStats: 1000 * 60, // 1 minute
+  removeDanglingResumableUploads: 60000 * 60 * 16 // 16 hours
 }
 
 // ---------------------------------------------------------------------------
@@ -285,6 +286,7 @@ const CONSTRAINTS_FIELDS = {
     LIKES: { min: 0 },
     DISLIKES: { min: 0 },
     FILE_SIZE: { min: -1 },
+    PARTIAL_UPLOAD_SIZE: { max: 50 * 1024 * 1024 * 1024 }, // 50GB
     URL: { min: 3, max: 2000 } // Length
   },
   VIDEO_PLAYLISTS: {
@@ -645,6 +647,7 @@ const LRU_CACHE = {
   }
 }
 
+const RESUMABLE_UPLOAD_DIRECTORY = join(CONFIG.STORAGE.TMP_DIR, 'resumable-uploads')
 const HLS_STREAMING_PLAYLIST_DIRECTORY = join(CONFIG.STORAGE.STREAMING_PLAYLISTS_DIR, 'hls')
 const HLS_REDUNDANCY_DIRECTORY = join(CONFIG.STORAGE.REDUNDANCY_DIR, 'hls')
 
@@ -819,6 +822,7 @@ export {
   PEERTUBE_VERSION,
   LAZY_STATIC_PATHS,
   SEARCH_INDEX,
+  RESUMABLE_UPLOAD_DIRECTORY,
   HLS_REDUNDANCY_DIRECTORY,
   P2P_MEDIA_LOADER_PEER_VERSION,
   ACTOR_IMAGES_SIZE,
diff --git a/server/initializers/installer.ts b/server/initializers/installer.ts
index cb58454cb..8dcff64e2 100644
--- a/server/initializers/installer.ts
+++ b/server/initializers/installer.ts
@@ -6,7 +6,7 @@ import { UserModel } from '../models/account/user'
 import { ApplicationModel } from '../models/application/application'
 import { OAuthClientModel } from '../models/oauth/oauth-client'
 import { applicationExist, clientsExist, usersExist } from './checker-after-init'
-import { FILES_CACHE, HLS_STREAMING_PLAYLIST_DIRECTORY, LAST_MIGRATION_VERSION } from './constants'
+import { FILES_CACHE, HLS_STREAMING_PLAYLIST_DIRECTORY, LAST_MIGRATION_VERSION, RESUMABLE_UPLOAD_DIRECTORY } from './constants'
 import { sequelizeTypescript } from './database'
 import { ensureDir, remove } from 'fs-extra'
 import { CONFIG } from './config'
@@ -79,6 +79,9 @@ function createDirectoriesIfNotExist () {
   // Playlist directories
   tasks.push(ensureDir(HLS_STREAMING_PLAYLIST_DIRECTORY))
 
+  // Resumable upload directory
+  tasks.push(ensureDir(RESUMABLE_UPLOAD_DIRECTORY))
+
   return Promise.all(tasks)
 }