Resumable video uploads (#3933)

* WIP: resumable video uploads

relates to #324

* fix review comments

* video upload: error handling

* fix audio upload

* fixes after self review

* Update server/controllers/api/videos/index.ts

Co-authored-by: Rigel Kent <par@rigelk.eu>

* Update server/middlewares/validators/videos/videos.ts

Co-authored-by: Rigel Kent <par@rigelk.eu>

* Update server/controllers/api/videos/index.ts

Co-authored-by: Rigel Kent <par@rigelk.eu>

* update after code review

* refactor upload route

- restore multipart upload route
- move resumable to dedicated upload-resumable route
- move checks to middleware
- do not leak internal fs structure in response

* fix yarn.lock upon rebase

* factorize addVideo for reuse in both endpoints

* add resumable upload API to openapi spec

* add initial test and test helper for resumable upload

* typings for videoAddResumable middleware

* avoid including aws and google packages via node-uploadx, by only including uploadx/core

* rename ex-isAudioBg to more explicit name mentioning it is a preview file for audio

* add video-upload-tmp-folder-cleaner job

* stronger typing of video upload middleware

* reduce dependency to @uploadx/core

* add audio upload test

* refactor resumable uploads cleanup from job to scheduler

* refactor resumable uploads scheduler to compare to last execution time

* make resumable upload validator to always cleanup on failure

* move legacy upload request building outside of uploadVideo test helper

* filter upload-resumable middlewares down to POST, PUT, DELETE

also begin to type metadata

* merge add duration functions

* stronger typings and documentation for uploadx behaviour, move init validator up

* refactor(client/video-edit): options > uploadxOptions

* refactor(client/video-edit): remove obsolete else

* scheduler/remove-dangling-resum: rename tag

* refactor(server/video): add UploadVideoFiles type

* refactor(mw/validators): restructure eslint disable

* refactor(mw/validators/videos): rename import

* refactor(client/vid-upload): rename html elem id

* refactor(sched/remove-dangl): move fn to method

* refactor(mw/async): add method typing

* refactor(mw/vali/video): double quote > single

* refactor(server/upload-resum): express use > all

* proper http methud enum server/middlewares/async.ts

* properly type http methods

* factorize common video upload validation steps

* add check for maximum partially uploaded file size

* fix audioBg use

* fix extname(filename) in addVideo

* document parameters for uploadx's resumable protocol

* clear META files in scheduler

* last audio refactor before cramming preview in the initial POST form data

* refactor as mulitpart/form-data initial post request

this allows preview/thumbnail uploads alongside the initial request,
and cleans up the upload form

* Add more tests for resumable uploads

* Refactor remove dangling resumable uploads

* Prepare changelog

* Add more resumable upload tests

* Remove user quota check for resumable uploads

* Fix upload error handler

* Update nginx template for upload-resumable

* Cleanup comment

* Remove unused express methods

* Prefer to use got instead of raw http

* Don't retry on error 500

Co-authored-by: Rigel Kent <par@rigelk.eu>
Co-authored-by: Rigel Kent <sendmemail@rigelk.eu>
Co-authored-by: Chocobozzz <me@florianbigard.com>

1 files changed, 187 insertions, 0 deletions

diff --git a/server/tests/api/videos/resumable-upload.ts b/server/tests/api/videos/resumable-upload.ts
new file mode 100644
index 000000000..af9221c43
--- /dev/null
+++ b/server/tests/api/videos/resumable-upload.ts
@@ -0,0 +1,187 @@
+/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
+
+import 'mocha'
+import * as chai from 'chai'
+import { pathExists, readdir, stat } from 'fs-extra'
+import { join } from 'path'
+import { HttpStatusCode } from '@shared/core-utils'
+import {
+  buildAbsoluteFixturePath,
+  buildServerDirectory,
+  flushAndRunServer,
+  getMyUserInformation,
+  prepareResumableUpload,
+  sendDebugCommand,
+  sendResumableChunks,
+  ServerInfo,
+  setAccessTokensToServers,
+  setDefaultVideoChannel,
+  updateUser
+} from '@shared/extra-utils'
+import { MyUser, VideoPrivacy } from '@shared/models'
+
+const expect = chai.expect
+
+// Most classic resumable upload tests are done in other test suites
+
+describe('Test resumable upload', function () {
+  const defaultFixture = 'video_short.mp4'
+  let server: ServerInfo
+  let rootId: number
+
+  async function buildSize (fixture: string, size?: number) {
+    if (size !== undefined) return size
+
+    const baseFixture = buildAbsoluteFixturePath(fixture)
+    return (await stat(baseFixture)).size
+  }
+
+  async function prepareUpload (sizeArg?: number) {
+    const size = await buildSize(defaultFixture, sizeArg)
+
+    const attributes = {
+      name: 'video',
+      channelId: server.videoChannel.id,
+      privacy: VideoPrivacy.PUBLIC,
+      fixture: defaultFixture
+    }
+
+    const mimetype = 'video/mp4'
+
+    const res = await prepareResumableUpload({ url: server.url, token: server.accessToken, attributes, size, mimetype })
+
+    return res.header['location'].split('?')[1]
+  }
+
+  async function sendChunks (options: {
+    pathUploadId: string
+    size?: number
+    expectedStatus?: HttpStatusCode
+    contentLength?: number
+    contentRange?: string
+    contentRangeBuilder?: (start: number, chunk: any) => string
+  }) {
+    const { pathUploadId, expectedStatus, contentLength, contentRangeBuilder } = options
+
+    const size = await buildSize(defaultFixture, options.size)
+    const absoluteFilePath = buildAbsoluteFixturePath(defaultFixture)
+
+    return sendResumableChunks({
+      url: server.url,
+      token: server.accessToken,
+      pathUploadId,
+      videoFilePath: absoluteFilePath,
+      size,
+      contentLength,
+      contentRangeBuilder,
+      specialStatus: expectedStatus
+    })
+  }
+
+  async function checkFileSize (uploadIdArg: string, expectedSize: number | null) {
+    const uploadId = uploadIdArg.replace(/^upload_id=/, '')
+
+    const subPath = join('tmp', 'resumable-uploads', uploadId)
+    const filePath = buildServerDirectory(server, subPath)
+    const exists = await pathExists(filePath)
+
+    if (expectedSize === null) {
+      expect(exists).to.be.false
+      return
+    }
+
+    expect(exists).to.be.true
+
+    expect((await stat(filePath)).size).to.equal(expectedSize)
+  }
+
+  async function countResumableUploads () {
+    const subPath = join('tmp', 'resumable-uploads')
+    const filePath = buildServerDirectory(server, subPath)
+
+    const files = await readdir(filePath)
+    return files.length
+  }
+
+  before(async function () {
+    this.timeout(30000)
+
+    server = await flushAndRunServer(1)
+    await setAccessTokensToServers([ server ])
+    await setDefaultVideoChannel([ server ])
+
+    const res = await getMyUserInformation(server.url, server.accessToken)
+    rootId = (res.body as MyUser).id
+
+    await updateUser({
+      url: server.url,
+      userId: rootId,
+      accessToken: server.accessToken,
+      videoQuota: 10_000_000
+    })
+  })
+
+  describe('Directory cleaning', function () {
+
+    it('Should correctly delete files after an upload', async function () {
+      const uploadId = await prepareUpload()
+      await sendChunks({ pathUploadId: uploadId })
+
+      expect(await countResumableUploads()).to.equal(0)
+    })
+
+    it('Should not delete files after an unfinished upload', async function () {
+      await prepareUpload()
+
+      expect(await countResumableUploads()).to.equal(2)
+    })
+
+    it('Should not delete recent uploads', async function () {
+      await sendDebugCommand(server.url, server.accessToken, { command: 'remove-dandling-resumable-uploads' })
+
+      expect(await countResumableUploads()).to.equal(2)
+    })
+
+    it('Should delete old uploads', async function () {
+      await sendDebugCommand(server.url, server.accessToken, { command: 'remove-dandling-resumable-uploads' })
+
+      expect(await countResumableUploads()).to.equal(0)
+    })
+  })
+
+  describe('Resumable upload and chunks', function () {
+
+    it('Should accept the same amount of chunks', async function () {
+      const uploadId = await prepareUpload()
+      await sendChunks({ pathUploadId: uploadId })
+
+      await checkFileSize(uploadId, null)
+    })
+
+    it('Should not accept more chunks than expected', async function () {
+      const size = 100
+      const uploadId = await prepareUpload(size)
+
+      await sendChunks({ pathUploadId: uploadId, expectedStatus: HttpStatusCode.CONFLICT_409 })
+      await checkFileSize(uploadId, 0)
+    })
+
+    it('Should not accept more chunks than expected with an invalid content length/content range', async function () {
+      const uploadId = await prepareUpload(1500)
+
+      await sendChunks({ pathUploadId: uploadId, expectedStatus: HttpStatusCode.BAD_REQUEST_400, contentLength: 1000 })
+      await checkFileSize(uploadId, 0)
+    })
+
+    it('Should not accept more chunks than expected with an invalid content length', async function () {
+      const uploadId = await prepareUpload(500)
+
+      const size = 1000
+
+      const contentRangeBuilder = start => `bytes ${start}-${start + size - 1}/${size}`
+      await sendChunks({ pathUploadId: uploadId, expectedStatus: HttpStatusCode.BAD_REQUEST_400, contentRangeBuilder, contentLength: size })
+      await checkFileSize(uploadId, 0)
+    })
+  })
+
+})