From f6d6e7f861189a4446f406efb775a29688764b48 Mon Sep 17 00:00:00 2001 From: kontrollanten <6680299+kontrollanten@users.noreply.github.com> Date: Mon, 10 May 2021 11:13:41 +0200 Subject: Resumable video uploads (#3933) * WIP: resumable video uploads relates to #324 * fix review comments * video upload: error handling * fix audio upload * fixes after self review * Update server/controllers/api/videos/index.ts Co-authored-by: Rigel Kent * Update server/middlewares/validators/videos/videos.ts Co-authored-by: Rigel Kent * Update server/controllers/api/videos/index.ts Co-authored-by: Rigel Kent * update after code review * refactor upload route - restore multipart upload route - move resumable to dedicated upload-resumable route - move checks to middleware - do not leak internal fs structure in response * fix yarn.lock upon rebase * factorize addVideo for reuse in both endpoints * add resumable upload API to openapi spec * add initial test and test helper for resumable upload * typings for videoAddResumable middleware * avoid including aws and google packages via node-uploadx, by only including uploadx/core * rename ex-isAudioBg to more explicit name mentioning it is a preview file for audio * add video-upload-tmp-folder-cleaner job * stronger typing of video upload middleware * reduce dependency to @uploadx/core * add audio upload test * refactor resumable uploads cleanup from job to scheduler * refactor resumable uploads scheduler to compare to last execution time * make resumable upload validator to always cleanup on failure * move legacy upload request building outside of uploadVideo test helper * filter upload-resumable middlewares down to POST, PUT, DELETE also begin to type metadata * merge add duration functions * stronger typings and documentation for uploadx behaviour, move init validator up * refactor(client/video-edit): options > uploadxOptions * refactor(client/video-edit): remove obsolete else * scheduler/remove-dangling-resum: rename tag * refactor(server/video): add UploadVideoFiles type * refactor(mw/validators): restructure eslint disable * refactor(mw/validators/videos): rename import * refactor(client/vid-upload): rename html elem id * refactor(sched/remove-dangl): move fn to method * refactor(mw/async): add method typing * refactor(mw/vali/video): double quote > single * refactor(server/upload-resum): express use > all * proper http methud enum server/middlewares/async.ts * properly type http methods * factorize common video upload validation steps * add check for maximum partially uploaded file size * fix audioBg use * fix extname(filename) in addVideo * document parameters for uploadx's resumable protocol * clear META files in scheduler * last audio refactor before cramming preview in the initial POST form data * refactor as mulitpart/form-data initial post request this allows preview/thumbnail uploads alongside the initial request, and cleans up the upload form * Add more tests for resumable uploads * Refactor remove dangling resumable uploads * Prepare changelog * Add more resumable upload tests * Remove user quota check for resumable uploads * Fix upload error handler * Update nginx template for upload-resumable * Cleanup comment * Remove unused express methods * Prefer to use got instead of raw http * Don't retry on error 500 Co-authored-by: Rigel Kent Co-authored-by: Rigel Kent Co-authored-by: Chocobozzz --- server/tests/api/videos/resumable-upload.ts | 187 ++++++++++++++++++++++++++++ 1 file changed, 187 insertions(+) create mode 100644 server/tests/api/videos/resumable-upload.ts (limited to 'server/tests/api/videos/resumable-upload.ts') diff --git a/server/tests/api/videos/resumable-upload.ts b/server/tests/api/videos/resumable-upload.ts new file mode 100644 index 000000000..af9221c43 --- /dev/null +++ b/server/tests/api/videos/resumable-upload.ts @@ -0,0 +1,187 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import 'mocha' +import * as chai from 'chai' +import { pathExists, readdir, stat } from 'fs-extra' +import { join } from 'path' +import { HttpStatusCode } from '@shared/core-utils' +import { + buildAbsoluteFixturePath, + buildServerDirectory, + flushAndRunServer, + getMyUserInformation, + prepareResumableUpload, + sendDebugCommand, + sendResumableChunks, + ServerInfo, + setAccessTokensToServers, + setDefaultVideoChannel, + updateUser +} from '@shared/extra-utils' +import { MyUser, VideoPrivacy } from '@shared/models' + +const expect = chai.expect + +// Most classic resumable upload tests are done in other test suites + +describe('Test resumable upload', function () { + const defaultFixture = 'video_short.mp4' + let server: ServerInfo + let rootId: number + + async function buildSize (fixture: string, size?: number) { + if (size !== undefined) return size + + const baseFixture = buildAbsoluteFixturePath(fixture) + return (await stat(baseFixture)).size + } + + async function prepareUpload (sizeArg?: number) { + const size = await buildSize(defaultFixture, sizeArg) + + const attributes = { + name: 'video', + channelId: server.videoChannel.id, + privacy: VideoPrivacy.PUBLIC, + fixture: defaultFixture + } + + const mimetype = 'video/mp4' + + const res = await prepareResumableUpload({ url: server.url, token: server.accessToken, attributes, size, mimetype }) + + return res.header['location'].split('?')[1] + } + + async function sendChunks (options: { + pathUploadId: string + size?: number + expectedStatus?: HttpStatusCode + contentLength?: number + contentRange?: string + contentRangeBuilder?: (start: number, chunk: any) => string + }) { + const { pathUploadId, expectedStatus, contentLength, contentRangeBuilder } = options + + const size = await buildSize(defaultFixture, options.size) + const absoluteFilePath = buildAbsoluteFixturePath(defaultFixture) + + return sendResumableChunks({ + url: server.url, + token: server.accessToken, + pathUploadId, + videoFilePath: absoluteFilePath, + size, + contentLength, + contentRangeBuilder, + specialStatus: expectedStatus + }) + } + + async function checkFileSize (uploadIdArg: string, expectedSize: number | null) { + const uploadId = uploadIdArg.replace(/^upload_id=/, '') + + const subPath = join('tmp', 'resumable-uploads', uploadId) + const filePath = buildServerDirectory(server, subPath) + const exists = await pathExists(filePath) + + if (expectedSize === null) { + expect(exists).to.be.false + return + } + + expect(exists).to.be.true + + expect((await stat(filePath)).size).to.equal(expectedSize) + } + + async function countResumableUploads () { + const subPath = join('tmp', 'resumable-uploads') + const filePath = buildServerDirectory(server, subPath) + + const files = await readdir(filePath) + return files.length + } + + before(async function () { + this.timeout(30000) + + server = await flushAndRunServer(1) + await setAccessTokensToServers([ server ]) + await setDefaultVideoChannel([ server ]) + + const res = await getMyUserInformation(server.url, server.accessToken) + rootId = (res.body as MyUser).id + + await updateUser({ + url: server.url, + userId: rootId, + accessToken: server.accessToken, + videoQuota: 10_000_000 + }) + }) + + describe('Directory cleaning', function () { + + it('Should correctly delete files after an upload', async function () { + const uploadId = await prepareUpload() + await sendChunks({ pathUploadId: uploadId }) + + expect(await countResumableUploads()).to.equal(0) + }) + + it('Should not delete files after an unfinished upload', async function () { + await prepareUpload() + + expect(await countResumableUploads()).to.equal(2) + }) + + it('Should not delete recent uploads', async function () { + await sendDebugCommand(server.url, server.accessToken, { command: 'remove-dandling-resumable-uploads' }) + + expect(await countResumableUploads()).to.equal(2) + }) + + it('Should delete old uploads', async function () { + await sendDebugCommand(server.url, server.accessToken, { command: 'remove-dandling-resumable-uploads' }) + + expect(await countResumableUploads()).to.equal(0) + }) + }) + + describe('Resumable upload and chunks', function () { + + it('Should accept the same amount of chunks', async function () { + const uploadId = await prepareUpload() + await sendChunks({ pathUploadId: uploadId }) + + await checkFileSize(uploadId, null) + }) + + it('Should not accept more chunks than expected', async function () { + const size = 100 + const uploadId = await prepareUpload(size) + + await sendChunks({ pathUploadId: uploadId, expectedStatus: HttpStatusCode.CONFLICT_409 }) + await checkFileSize(uploadId, 0) + }) + + it('Should not accept more chunks than expected with an invalid content length/content range', async function () { + const uploadId = await prepareUpload(1500) + + await sendChunks({ pathUploadId: uploadId, expectedStatus: HttpStatusCode.BAD_REQUEST_400, contentLength: 1000 }) + await checkFileSize(uploadId, 0) + }) + + it('Should not accept more chunks than expected with an invalid content length', async function () { + const uploadId = await prepareUpload(500) + + const size = 1000 + + const contentRangeBuilder = start => `bytes ${start}-${start + size - 1}/${size}` + await sendChunks({ pathUploadId: uploadId, expectedStatus: HttpStatusCode.BAD_REQUEST_400, contentRangeBuilder, contentLength: size }) + await checkFileSize(uploadId, 0) + }) + }) + +}) -- cgit v1.2.3