Resumable video uploads (#3933)

* WIP: resumable video uploads

relates to #324

* fix review comments

* video upload: error handling

* fix audio upload

* fixes after self review

* Update server/controllers/api/videos/index.ts

Co-authored-by: Rigel Kent <par@rigelk.eu>

* Update server/middlewares/validators/videos/videos.ts

Co-authored-by: Rigel Kent <par@rigelk.eu>

* Update server/controllers/api/videos/index.ts

Co-authored-by: Rigel Kent <par@rigelk.eu>

* update after code review

* refactor upload route

- restore multipart upload route
- move resumable to dedicated upload-resumable route
- move checks to middleware
- do not leak internal fs structure in response

* fix yarn.lock upon rebase

* factorize addVideo for reuse in both endpoints

* add resumable upload API to openapi spec

* add initial test and test helper for resumable upload

* typings for videoAddResumable middleware

* avoid including aws and google packages via node-uploadx, by only including uploadx/core

* rename ex-isAudioBg to more explicit name mentioning it is a preview file for audio

* add video-upload-tmp-folder-cleaner job

* stronger typing of video upload middleware

* reduce dependency to @uploadx/core

* add audio upload test

* refactor resumable uploads cleanup from job to scheduler

* refactor resumable uploads scheduler to compare to last execution time

* make resumable upload validator to always cleanup on failure

* move legacy upload request building outside of uploadVideo test helper

* filter upload-resumable middlewares down to POST, PUT, DELETE

also begin to type metadata

* merge add duration functions

* stronger typings and documentation for uploadx behaviour, move init validator up

* refactor(client/video-edit): options > uploadxOptions

* refactor(client/video-edit): remove obsolete else

* scheduler/remove-dangling-resum: rename tag

* refactor(server/video): add UploadVideoFiles type

* refactor(mw/validators): restructure eslint disable

* refactor(mw/validators/videos): rename import

* refactor(client/vid-upload): rename html elem id

* refactor(sched/remove-dangl): move fn to method

* refactor(mw/async): add method typing

* refactor(mw/vali/video): double quote > single

* refactor(server/upload-resum): express use > all

* proper http methud enum server/middlewares/async.ts

* properly type http methods

* factorize common video upload validation steps

* add check for maximum partially uploaded file size

* fix audioBg use

* fix extname(filename) in addVideo

* document parameters for uploadx's resumable protocol

* clear META files in scheduler

* last audio refactor before cramming preview in the initial POST form data

* refactor as mulitpart/form-data initial post request

this allows preview/thumbnail uploads alongside the initial request,
and cleans up the upload form

* Add more tests for resumable uploads

* Refactor remove dangling resumable uploads

* Prepare changelog

* Add more resumable upload tests

* Remove user quota check for resumable uploads

* Fix upload error handler

* Update nginx template for upload-resumable

* Cleanup comment

* Remove unused express methods

* Prefer to use got instead of raw http

* Don't retry on error 500

Co-authored-by: Rigel Kent <par@rigelk.eu>
Co-authored-by: Rigel Kent <sendmemail@rigelk.eu>
Co-authored-by: Chocobozzz <me@florianbigard.com>

1 files changed, 85 insertions, 74 deletions

diff --git a/server/tests/api/videos/video-transcoder.ts b/server/tests/api/videos/video-transcoder.ts
index 1c99f26df..ea5ffd239 100644
--- a/server/tests/api/videos/video-transcoder.ts
+++ b/server/tests/api/videos/video-transcoder.ts
@@ -361,106 +361,117 @@ describe('Test video transcoding', function () {
 
   describe('Audio upload', function () {
 
-    before(async function () {
-      await updateCustomSubConfig(servers[1].url, servers[1].accessToken, {
-        transcoding: {
-          hls: { enabled: true },
-          webtorrent: { enabled: true },
-          resolutions: {
-            '0p': false,
-            '240p': false,
-            '360p': false,
-            '480p': false,
-            '720p': false,
-            '1080p': false,
-            '1440p': false,
-            '2160p': false
+    function runSuite (mode: 'legacy' | 'resumable') {
+
+      before(async function () {
+        await updateCustomSubConfig(servers[1].url, servers[1].accessToken, {
+          transcoding: {
+            hls: { enabled: true },
+            webtorrent: { enabled: true },
+            resolutions: {
+              '0p': false,
+              '240p': false,
+              '360p': false,
+              '480p': false,
+              '720p': false,
+              '1080p': false,
+              '1440p': false,
+              '2160p': false
+            }
           }
-        }
+        })
       })
-    })
-
-    it('Should merge an audio file with the preview file', async function () {
-      this.timeout(60_000)
-
-      const videoAttributesArg = { name: 'audio_with_preview', previewfile: 'preview.jpg', fixture: 'sample.ogg' }
-      await uploadVideo(servers[1].url, servers[1].accessToken, videoAttributesArg)
 
-      await waitJobs(servers)
+      it('Should merge an audio file with the preview file', async function () {
+        this.timeout(60_000)
 
-      for (const server of servers) {
-        const res = await getVideosList(server.url)
+        const videoAttributesArg = { name: 'audio_with_preview', previewfile: 'preview.jpg', fixture: 'sample.ogg' }
+        await uploadVideo(servers[1].url, servers[1].accessToken, videoAttributesArg, HttpStatusCode.OK_200, mode)
 
-        const video = res.body.data.find(v => v.name === 'audio_with_preview')
-        const res2 = await getVideo(server.url, video.id)
-        const videoDetails: VideoDetails = res2.body
+        await waitJobs(servers)
 
-        expect(videoDetails.files).to.have.lengthOf(1)
+        for (const server of servers) {
+          const res = await getVideosList(server.url)
 
-        await makeGetRequest({ url: server.url, path: videoDetails.thumbnailPath, statusCodeExpected: HttpStatusCode.OK_200 })
-        await makeGetRequest({ url: server.url, path: videoDetails.previewPath, statusCodeExpected: HttpStatusCode.OK_200 })
+          const video = res.body.data.find(v => v.name === 'audio_with_preview')
+          const res2 = await getVideo(server.url, video.id)
+          const videoDetails: VideoDetails = res2.body
 
-        const magnetUri = videoDetails.files[0].magnetUri
-        expect(magnetUri).to.contain('.mp4')
-      }
-    })
+          expect(videoDetails.files).to.have.lengthOf(1)
 
-    it('Should upload an audio file and choose a default background image', async function () {
-      this.timeout(60_000)
+          await makeGetRequest({ url: server.url, path: videoDetails.thumbnailPath, statusCodeExpected: HttpStatusCode.OK_200 })
+          await makeGetRequest({ url: server.url, path: videoDetails.previewPath, statusCodeExpected: HttpStatusCode.OK_200 })
 
-      const videoAttributesArg = { name: 'audio_without_preview', fixture: 'sample.ogg' }
-      await uploadVideo(servers[1].url, servers[1].accessToken, videoAttributesArg)
+          const magnetUri = videoDetails.files[0].magnetUri
+          expect(magnetUri).to.contain('.mp4')
+        }
+      })
 
-      await waitJobs(servers)
+      it('Should upload an audio file and choose a default background image', async function () {
+        this.timeout(60_000)
 
-      for (const server of servers) {
-        const res = await getVideosList(server.url)
+        const videoAttributesArg = { name: 'audio_without_preview', fixture: 'sample.ogg' }
+        await uploadVideo(servers[1].url, servers[1].accessToken, videoAttributesArg, HttpStatusCode.OK_200, mode)
 
-        const video = res.body.data.find(v => v.name === 'audio_without_preview')
-        const res2 = await getVideo(server.url, video.id)
-        const videoDetails = res2.body
+        await waitJobs(servers)
 
-        expect(videoDetails.files).to.have.lengthOf(1)
+        for (const server of servers) {
+          const res = await getVideosList(server.url)
 
-        await makeGetRequest({ url: server.url, path: videoDetails.thumbnailPath, statusCodeExpected: HttpStatusCode.OK_200 })
-        await makeGetRequest({ url: server.url, path: videoDetails.previewPath, statusCodeExpected: HttpStatusCode.OK_200 })
+          const video = res.body.data.find(v => v.name === 'audio_without_preview')
+          const res2 = await getVideo(server.url, video.id)
+          const videoDetails = res2.body
 
-        const magnetUri = videoDetails.files[0].magnetUri
-        expect(magnetUri).to.contain('.mp4')
-      }
-    })
+          expect(videoDetails.files).to.have.lengthOf(1)
 
-    it('Should upload an audio file and create an audio version only', async function () {
-      this.timeout(60_000)
+          await makeGetRequest({ url: server.url, path: videoDetails.thumbnailPath, statusCodeExpected: HttpStatusCode.OK_200 })
+          await makeGetRequest({ url: server.url, path: videoDetails.previewPath, statusCodeExpected: HttpStatusCode.OK_200 })
 
-      await updateCustomSubConfig(servers[1].url, servers[1].accessToken, {
-        transcoding: {
-          hls: { enabled: true },
-          webtorrent: { enabled: true },
-          resolutions: {
-            '0p': true,
-            '240p': false,
-            '360p': false
-          }
+          const magnetUri = videoDetails.files[0].magnetUri
+          expect(magnetUri).to.contain('.mp4')
         }
       })
 
-      const videoAttributesArg = { name: 'audio_with_preview', previewfile: 'preview.jpg', fixture: 'sample.ogg' }
-      const resVideo = await uploadVideo(servers[1].url, servers[1].accessToken, videoAttributesArg)
+      it('Should upload an audio file and create an audio version only', async function () {
+        this.timeout(60_000)
+
+        await updateCustomSubConfig(servers[1].url, servers[1].accessToken, {
+          transcoding: {
+            hls: { enabled: true },
+            webtorrent: { enabled: true },
+            resolutions: {
+              '0p': true,
+              '240p': false,
+              '360p': false
+            }
+          }
+        })
 
-      await waitJobs(servers)
+        const videoAttributesArg = { name: 'audio_with_preview', previewfile: 'preview.jpg', fixture: 'sample.ogg' }
+        const resVideo = await uploadVideo(servers[1].url, servers[1].accessToken, videoAttributesArg, HttpStatusCode.OK_200, mode)
 
-      for (const server of servers) {
-        const res2 = await getVideo(server.url, resVideo.body.video.id)
-        const videoDetails: VideoDetails = res2.body
+        await waitJobs(servers)
+
+        for (const server of servers) {
+          const res2 = await getVideo(server.url, resVideo.body.video.id)
+          const videoDetails: VideoDetails = res2.body
 
-        for (const files of [ videoDetails.files, videoDetails.streamingPlaylists[0].files ]) {
-          expect(files).to.have.lengthOf(2)
-          expect(files.find(f => f.resolution.id === 0)).to.not.be.undefined
+          for (const files of [ videoDetails.files, videoDetails.streamingPlaylists[0].files ]) {
+            expect(files).to.have.lengthOf(2)
+            expect(files.find(f => f.resolution.id === 0)).to.not.be.undefined
+          }
         }
-      }
 
-      await updateConfigForTranscoding(servers[1])
+        await updateConfigForTranscoding(servers[1])
+      })
+    }
+
+    describe('Legacy upload', function () {
+      runSuite('legacy')
+    })
+
+    describe('Resumable upload', function () {
+      runSuite('resumable')
     })
   })