From f6d6e7f861189a4446f406efb775a29688764b48 Mon Sep 17 00:00:00 2001 From: kontrollanten <6680299+kontrollanten@users.noreply.github.com> Date: Mon, 10 May 2021 11:13:41 +0200 Subject: Resumable video uploads (#3933) * WIP: resumable video uploads relates to #324 * fix review comments * video upload: error handling * fix audio upload * fixes after self review * Update server/controllers/api/videos/index.ts Co-authored-by: Rigel Kent * Update server/middlewares/validators/videos/videos.ts Co-authored-by: Rigel Kent * Update server/controllers/api/videos/index.ts Co-authored-by: Rigel Kent * update after code review * refactor upload route - restore multipart upload route - move resumable to dedicated upload-resumable route - move checks to middleware - do not leak internal fs structure in response * fix yarn.lock upon rebase * factorize addVideo for reuse in both endpoints * add resumable upload API to openapi spec * add initial test and test helper for resumable upload * typings for videoAddResumable middleware * avoid including aws and google packages via node-uploadx, by only including uploadx/core * rename ex-isAudioBg to more explicit name mentioning it is a preview file for audio * add video-upload-tmp-folder-cleaner job * stronger typing of video upload middleware * reduce dependency to @uploadx/core * add audio upload test * refactor resumable uploads cleanup from job to scheduler * refactor resumable uploads scheduler to compare to last execution time * make resumable upload validator to always cleanup on failure * move legacy upload request building outside of uploadVideo test helper * filter upload-resumable middlewares down to POST, PUT, DELETE also begin to type metadata * merge add duration functions * stronger typings and documentation for uploadx behaviour, move init validator up * refactor(client/video-edit): options > uploadxOptions * refactor(client/video-edit): remove obsolete else * scheduler/remove-dangling-resum: rename tag * refactor(server/video): add UploadVideoFiles type * refactor(mw/validators): restructure eslint disable * refactor(mw/validators/videos): rename import * refactor(client/vid-upload): rename html elem id * refactor(sched/remove-dangl): move fn to method * refactor(mw/async): add method typing * refactor(mw/vali/video): double quote > single * refactor(server/upload-resum): express use > all * proper http methud enum server/middlewares/async.ts * properly type http methods * factorize common video upload validation steps * add check for maximum partially uploaded file size * fix audioBg use * fix extname(filename) in addVideo * document parameters for uploadx's resumable protocol * clear META files in scheduler * last audio refactor before cramming preview in the initial POST form data * refactor as mulitpart/form-data initial post request this allows preview/thumbnail uploads alongside the initial request, and cleans up the upload form * Add more tests for resumable uploads * Refactor remove dangling resumable uploads * Prepare changelog * Add more resumable upload tests * Remove user quota check for resumable uploads * Fix upload error handler * Update nginx template for upload-resumable * Cleanup comment * Remove unused express methods * Prefer to use got instead of raw http * Don't retry on error 500 Co-authored-by: Rigel Kent Co-authored-by: Rigel Kent Co-authored-by: Chocobozzz --- server/tests/api/check-params/videos.ts | 393 ++++++++++++++++---------------- 1 file changed, 191 insertions(+), 202 deletions(-) (limited to 'server/tests/api/check-params/videos.ts') diff --git a/server/tests/api/check-params/videos.ts b/server/tests/api/check-params/videos.ts index 188d1835c..c970c4a15 100644 --- a/server/tests/api/check-params/videos.ts +++ b/server/tests/api/check-params/videos.ts @@ -1,11 +1,12 @@ /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ +import 'mocha' import * as chai from 'chai' import { omit } from 'lodash' -import 'mocha' import { join } from 'path' -import { VideoPrivacy } from '../../../../shared/models/videos/video-privacy.enum' +import { HttpStatusCode } from '../../../../shared/core-utils/miscs/http-error-codes' import { + checkUploadVideoParam, cleanupTests, createUser, flushAndRunServer, @@ -18,17 +19,18 @@ import { makePutBodyRequest, makeUploadRequest, removeVideo, + root, ServerInfo, setAccessTokensToServers, - userLogin, - root + userLogin } from '../../../../shared/extra-utils' import { checkBadCountPagination, checkBadSortPagination, checkBadStartPagination } from '../../../../shared/extra-utils/requests/check-api-params' -import { HttpStatusCode } from '../../../../shared/core-utils/miscs/http-error-codes' +import { VideoPrivacy } from '../../../../shared/models/videos/video-privacy.enum' +import { randomInt } from '@shared/core-utils' const expect = chai.expect @@ -183,7 +185,7 @@ describe('Test videos API validator', function () { describe('When adding a video', function () { let baseCorrectParams const baseCorrectAttaches = { - videofile: join(root(), 'server', 'tests', 'fixtures', 'video_short.webm') + fixture: join(root(), 'server', 'tests', 'fixtures', 'video_short.webm') } before(function () { @@ -206,256 +208,243 @@ describe('Test videos API validator', function () { } }) - it('Should fail with nothing', async function () { - const fields = {} - const attaches = {} - await makeUploadRequest({ url: server.url, path: path + '/upload', token: server.accessToken, fields, attaches }) - }) + function runSuite (mode: 'legacy' | 'resumable') { - it('Should fail without name', async function () { - const fields = omit(baseCorrectParams, 'name') - const attaches = baseCorrectAttaches + it('Should fail with nothing', async function () { + const fields = {} + const attaches = {} + await checkUploadVideoParam(server.url, server.accessToken, { ...fields, ...attaches }, HttpStatusCode.BAD_REQUEST_400, mode) + }) - await makeUploadRequest({ url: server.url, path: path + '/upload', token: server.accessToken, fields, attaches }) - }) + it('Should fail without name', async function () { + const fields = omit(baseCorrectParams, 'name') + const attaches = baseCorrectAttaches - it('Should fail with a long name', async function () { - const fields = immutableAssign(baseCorrectParams, { name: 'super'.repeat(65) }) - const attaches = baseCorrectAttaches + await checkUploadVideoParam(server.url, server.accessToken, { ...fields, ...attaches }, HttpStatusCode.BAD_REQUEST_400, mode) + }) - await makeUploadRequest({ url: server.url, path: path + '/upload', token: server.accessToken, fields, attaches }) - }) + it('Should fail with a long name', async function () { + const fields = immutableAssign(baseCorrectParams, { name: 'super'.repeat(65) }) + const attaches = baseCorrectAttaches - it('Should fail with a bad category', async function () { - const fields = immutableAssign(baseCorrectParams, { category: 125 }) - const attaches = baseCorrectAttaches + await checkUploadVideoParam(server.url, server.accessToken, { ...fields, ...attaches }, HttpStatusCode.BAD_REQUEST_400, mode) + }) - await makeUploadRequest({ url: server.url, path: path + '/upload', token: server.accessToken, fields, attaches }) - }) + it('Should fail with a bad category', async function () { + const fields = immutableAssign(baseCorrectParams, { category: 125 }) + const attaches = baseCorrectAttaches - it('Should fail with a bad licence', async function () { - const fields = immutableAssign(baseCorrectParams, { licence: 125 }) - const attaches = baseCorrectAttaches + await checkUploadVideoParam(server.url, server.accessToken, { ...fields, ...attaches }, HttpStatusCode.BAD_REQUEST_400, mode) + }) - await makeUploadRequest({ url: server.url, path: path + '/upload', token: server.accessToken, fields, attaches }) - }) + it('Should fail with a bad licence', async function () { + const fields = immutableAssign(baseCorrectParams, { licence: 125 }) + const attaches = baseCorrectAttaches - it('Should fail with a bad language', async function () { - const fields = immutableAssign(baseCorrectParams, { language: 'a'.repeat(15) }) - const attaches = baseCorrectAttaches + await checkUploadVideoParam(server.url, server.accessToken, { ...fields, ...attaches }, HttpStatusCode.BAD_REQUEST_400, mode) + }) - await makeUploadRequest({ url: server.url, path: path + '/upload', token: server.accessToken, fields, attaches }) - }) + it('Should fail with a bad language', async function () { + const fields = immutableAssign(baseCorrectParams, { language: 'a'.repeat(15) }) + const attaches = baseCorrectAttaches - it('Should fail with a long description', async function () { - const fields = immutableAssign(baseCorrectParams, { description: 'super'.repeat(2500) }) - const attaches = baseCorrectAttaches + await checkUploadVideoParam(server.url, server.accessToken, { ...fields, ...attaches }, HttpStatusCode.BAD_REQUEST_400, mode) + }) - await makeUploadRequest({ url: server.url, path: path + '/upload', token: server.accessToken, fields, attaches }) - }) + it('Should fail with a long description', async function () { + const fields = immutableAssign(baseCorrectParams, { description: 'super'.repeat(2500) }) + const attaches = baseCorrectAttaches - it('Should fail with a long support text', async function () { - const fields = immutableAssign(baseCorrectParams, { support: 'super'.repeat(201) }) - const attaches = baseCorrectAttaches + await checkUploadVideoParam(server.url, server.accessToken, { ...fields, ...attaches }, HttpStatusCode.BAD_REQUEST_400, mode) + }) - await makeUploadRequest({ url: server.url, path: path + '/upload', token: server.accessToken, fields, attaches }) - }) + it('Should fail with a long support text', async function () { + const fields = immutableAssign(baseCorrectParams, { support: 'super'.repeat(201) }) + const attaches = baseCorrectAttaches - it('Should fail without a channel', async function () { - const fields = omit(baseCorrectParams, 'channelId') - const attaches = baseCorrectAttaches + await checkUploadVideoParam(server.url, server.accessToken, { ...fields, ...attaches }, HttpStatusCode.BAD_REQUEST_400, mode) + }) - await makeUploadRequest({ url: server.url, path: path + '/upload', token: server.accessToken, fields, attaches }) - }) + it('Should fail without a channel', async function () { + const fields = omit(baseCorrectParams, 'channelId') + const attaches = baseCorrectAttaches - it('Should fail with a bad channel', async function () { - const fields = immutableAssign(baseCorrectParams, { channelId: 545454 }) - const attaches = baseCorrectAttaches + await checkUploadVideoParam(server.url, server.accessToken, { ...fields, ...attaches }, HttpStatusCode.BAD_REQUEST_400, mode) + }) - await makeUploadRequest({ url: server.url, path: path + '/upload', token: server.accessToken, fields, attaches }) - }) + it('Should fail with a bad channel', async function () { + const fields = immutableAssign(baseCorrectParams, { channelId: 545454 }) + const attaches = baseCorrectAttaches - it('Should fail with another user channel', async function () { - const user = { - username: 'fake', - password: 'fake_password' - } - await createUser({ url: server.url, accessToken: server.accessToken, username: user.username, password: user.password }) + await checkUploadVideoParam(server.url, server.accessToken, { ...fields, ...attaches }, HttpStatusCode.BAD_REQUEST_400, mode) + }) - const accessTokenUser = await userLogin(server, user) - const res = await getMyUserInformation(server.url, accessTokenUser) - const customChannelId = res.body.videoChannels[0].id + it('Should fail with another user channel', async function () { + const user = { + username: 'fake' + randomInt(0, 1500), + password: 'fake_password' + } + await createUser({ url: server.url, accessToken: server.accessToken, username: user.username, password: user.password }) - const fields = immutableAssign(baseCorrectParams, { channelId: customChannelId }) - const attaches = baseCorrectAttaches + const accessTokenUser = await userLogin(server, user) + const res = await getMyUserInformation(server.url, accessTokenUser) + const customChannelId = res.body.videoChannels[0].id - await makeUploadRequest({ url: server.url, path: path + '/upload', token: userAccessToken, fields, attaches }) - }) + const fields = immutableAssign(baseCorrectParams, { channelId: customChannelId }) + const attaches = baseCorrectAttaches - it('Should fail with too many tags', async function () { - const fields = immutableAssign(baseCorrectParams, { tags: [ 'tag1', 'tag2', 'tag3', 'tag4', 'tag5', 'tag6' ] }) - const attaches = baseCorrectAttaches + await checkUploadVideoParam(server.url, userAccessToken, { ...fields, ...attaches }, HttpStatusCode.BAD_REQUEST_400, mode) + }) - await makeUploadRequest({ url: server.url, path: path + '/upload', token: server.accessToken, fields, attaches }) - }) + it('Should fail with too many tags', async function () { + const fields = immutableAssign(baseCorrectParams, { tags: [ 'tag1', 'tag2', 'tag3', 'tag4', 'tag5', 'tag6' ] }) + const attaches = baseCorrectAttaches - it('Should fail with a tag length too low', async function () { - const fields = immutableAssign(baseCorrectParams, { tags: [ 'tag1', 't' ] }) - const attaches = baseCorrectAttaches + await checkUploadVideoParam(server.url, server.accessToken, { ...fields, ...attaches }, HttpStatusCode.BAD_REQUEST_400, mode) + }) - await makeUploadRequest({ url: server.url, path: path + '/upload', token: server.accessToken, fields, attaches }) - }) + it('Should fail with a tag length too low', async function () { + const fields = immutableAssign(baseCorrectParams, { tags: [ 'tag1', 't' ] }) + const attaches = baseCorrectAttaches - it('Should fail with a tag length too big', async function () { - const fields = immutableAssign(baseCorrectParams, { tags: [ 'tag1', 'my_super_tag_too_long_long_long_long_long_long' ] }) - const attaches = baseCorrectAttaches + await checkUploadVideoParam(server.url, server.accessToken, { ...fields, ...attaches }, HttpStatusCode.BAD_REQUEST_400, mode) + }) - await makeUploadRequest({ url: server.url, path: path + '/upload', token: server.accessToken, fields, attaches }) - }) + it('Should fail with a tag length too big', async function () { + const fields = immutableAssign(baseCorrectParams, { tags: [ 'tag1', 'my_super_tag_too_long_long_long_long_long_long' ] }) + const attaches = baseCorrectAttaches - it('Should fail with a bad schedule update (miss updateAt)', async function () { - const fields = immutableAssign(baseCorrectParams, { 'scheduleUpdate[privacy]': VideoPrivacy.PUBLIC }) - const attaches = baseCorrectAttaches + await checkUploadVideoParam(server.url, server.accessToken, { ...fields, ...attaches }, HttpStatusCode.BAD_REQUEST_400, mode) + }) - await makeUploadRequest({ url: server.url, path: path + '/upload', token: server.accessToken, fields, attaches }) - }) + it('Should fail with a bad schedule update (miss updateAt)', async function () { + const fields = immutableAssign(baseCorrectParams, { scheduleUpdate: { privacy: VideoPrivacy.PUBLIC } }) + const attaches = baseCorrectAttaches - it('Should fail with a bad schedule update (wrong updateAt)', async function () { - const fields = immutableAssign(baseCorrectParams, { - 'scheduleUpdate[privacy]': VideoPrivacy.PUBLIC, - 'scheduleUpdate[updateAt]': 'toto' + await checkUploadVideoParam(server.url, server.accessToken, { ...fields, ...attaches }, HttpStatusCode.BAD_REQUEST_400, mode) }) - const attaches = baseCorrectAttaches - await makeUploadRequest({ url: server.url, path: path + '/upload', token: server.accessToken, fields, attaches }) - }) + it('Should fail with a bad schedule update (wrong updateAt)', async function () { + const fields = immutableAssign(baseCorrectParams, { + scheduleUpdate: { + privacy: VideoPrivacy.PUBLIC, + updateAt: 'toto' + } + }) + const attaches = baseCorrectAttaches - it('Should fail with a bad originally published at attribute', async function () { - const fields = immutableAssign(baseCorrectParams, { originallyPublishedAt: 'toto' }) - const attaches = baseCorrectAttaches + await checkUploadVideoParam(server.url, server.accessToken, { ...fields, ...attaches }, HttpStatusCode.BAD_REQUEST_400, mode) + }) - await makeUploadRequest({ url: server.url, path: path + '/upload', token: server.accessToken, fields, attaches }) - }) + it('Should fail with a bad originally published at attribute', async function () { + const fields = immutableAssign(baseCorrectParams, { originallyPublishedAt: 'toto' }) + const attaches = baseCorrectAttaches - it('Should fail without an input file', async function () { - const fields = baseCorrectParams - const attaches = {} - await makeUploadRequest({ url: server.url, path: path + '/upload', token: server.accessToken, fields, attaches }) - }) + await checkUploadVideoParam(server.url, server.accessToken, { ...fields, ...attaches }, HttpStatusCode.BAD_REQUEST_400, mode) + }) - it('Should fail with an incorrect input file', async function () { - const fields = baseCorrectParams - let attaches = { - videofile: join(root(), 'server', 'tests', 'fixtures', 'video_short_fake.webm') - } - await makeUploadRequest({ - url: server.url, - path: path + '/upload', - token: server.accessToken, - fields, - attaches, - statusCodeExpected: HttpStatusCode.UNPROCESSABLE_ENTITY_422 + it('Should fail without an input file', async function () { + const fields = baseCorrectParams + const attaches = {} + await checkUploadVideoParam(server.url, server.accessToken, { ...fields, ...attaches }, HttpStatusCode.BAD_REQUEST_400, mode) }) - attaches = { - videofile: join(root(), 'server', 'tests', 'fixtures', 'video_short.mkv') - } - await makeUploadRequest({ - url: server.url, - path: path + '/upload', - token: server.accessToken, - fields, - attaches, - statusCodeExpected: HttpStatusCode.UNSUPPORTED_MEDIA_TYPE_415 + it('Should fail with an incorrect input file', async function () { + const fields = baseCorrectParams + let attaches = { fixture: join(root(), 'server', 'tests', 'fixtures', 'video_short_fake.webm') } + + await checkUploadVideoParam( + server.url, + server.accessToken, + { ...fields, ...attaches }, + HttpStatusCode.UNPROCESSABLE_ENTITY_422, + mode + ) + + attaches = { fixture: join(root(), 'server', 'tests', 'fixtures', 'video_short.mkv') } + await checkUploadVideoParam( + server.url, + server.accessToken, + { ...fields, ...attaches }, + HttpStatusCode.UNSUPPORTED_MEDIA_TYPE_415, + mode + ) }) - }) - it('Should fail with an incorrect thumbnail file', async function () { - const fields = baseCorrectParams - const attaches = { - thumbnailfile: join(root(), 'server', 'tests', 'fixtures', 'video_short.mp4'), - videofile: join(root(), 'server', 'tests', 'fixtures', 'video_short.mp4') - } + it('Should fail with an incorrect thumbnail file', async function () { + const fields = baseCorrectParams + const attaches = { + thumbnailfile: join(root(), 'server', 'tests', 'fixtures', 'video_short.mp4'), + fixture: join(root(), 'server', 'tests', 'fixtures', 'video_short.mp4') + } - await makeUploadRequest({ url: server.url, path: path + '/upload', token: server.accessToken, fields, attaches }) - }) + await checkUploadVideoParam(server.url, server.accessToken, { ...fields, ...attaches }, HttpStatusCode.BAD_REQUEST_400, mode) + }) - it('Should fail with a big thumbnail file', async function () { - const fields = baseCorrectParams - const attaches = { - thumbnailfile: join(root(), 'server', 'tests', 'fixtures', 'preview-big.png'), - videofile: join(root(), 'server', 'tests', 'fixtures', 'video_short.mp4') - } + it('Should fail with a big thumbnail file', async function () { + const fields = baseCorrectParams + const attaches = { + thumbnailfile: join(root(), 'server', 'tests', 'fixtures', 'preview-big.png'), + fixture: join(root(), 'server', 'tests', 'fixtures', 'video_short.mp4') + } - await makeUploadRequest({ url: server.url, path: path + '/upload', token: server.accessToken, fields, attaches }) - }) + await checkUploadVideoParam(server.url, server.accessToken, { ...fields, ...attaches }, HttpStatusCode.BAD_REQUEST_400, mode) + }) - it('Should fail with an incorrect preview file', async function () { - const fields = baseCorrectParams - const attaches = { - previewfile: join(root(), 'server', 'tests', 'fixtures', 'video_short.mp4'), - videofile: join(root(), 'server', 'tests', 'fixtures', 'video_short.mp4') - } + it('Should fail with an incorrect preview file', async function () { + const fields = baseCorrectParams + const attaches = { + previewfile: join(root(), 'server', 'tests', 'fixtures', 'video_short.mp4'), + fixture: join(root(), 'server', 'tests', 'fixtures', 'video_short.mp4') + } - await makeUploadRequest({ url: server.url, path: path + '/upload', token: server.accessToken, fields, attaches }) - }) + await checkUploadVideoParam(server.url, server.accessToken, { ...fields, ...attaches }, HttpStatusCode.BAD_REQUEST_400, mode) + }) - it('Should fail with a big preview file', async function () { - const fields = baseCorrectParams - const attaches = { - previewfile: join(root(), 'server', 'tests', 'fixtures', 'preview-big.png'), - videofile: join(root(), 'server', 'tests', 'fixtures', 'video_short.mp4') - } + it('Should fail with a big preview file', async function () { + const fields = baseCorrectParams + const attaches = { + previewfile: join(root(), 'server', 'tests', 'fixtures', 'preview-big.png'), + fixture: join(root(), 'server', 'tests', 'fixtures', 'video_short.mp4') + } - await makeUploadRequest({ url: server.url, path: path + '/upload', token: server.accessToken, fields, attaches }) - }) + await checkUploadVideoParam(server.url, server.accessToken, { ...fields, ...attaches }, HttpStatusCode.BAD_REQUEST_400, mode) + }) - it('Should succeed with the correct parameters', async function () { - this.timeout(10000) + it('Should succeed with the correct parameters', async function () { + this.timeout(10000) - const fields = baseCorrectParams + const fields = baseCorrectParams - { - const attaches = baseCorrectAttaches - await makeUploadRequest({ - url: server.url, - path: path + '/upload', - token: server.accessToken, - fields, - attaches, - statusCodeExpected: HttpStatusCode.OK_200 - }) - } + { + const attaches = baseCorrectAttaches + await checkUploadVideoParam(server.url, server.accessToken, { ...fields, ...attaches }, HttpStatusCode.OK_200, mode) + } - { - const attaches = immutableAssign(baseCorrectAttaches, { - videofile: join(root(), 'server', 'tests', 'fixtures', 'video_short.mp4') - }) + { + const attaches = immutableAssign(baseCorrectAttaches, { + videofile: join(root(), 'server', 'tests', 'fixtures', 'video_short.mp4') + }) - await makeUploadRequest({ - url: server.url, - path: path + '/upload', - token: server.accessToken, - fields, - attaches, - statusCodeExpected: HttpStatusCode.OK_200 - }) - } + await checkUploadVideoParam(server.url, server.accessToken, { ...fields, ...attaches }, HttpStatusCode.OK_200, mode) + } - { - const attaches = immutableAssign(baseCorrectAttaches, { - videofile: join(root(), 'server', 'tests', 'fixtures', 'video_short.ogv') - }) + { + const attaches = immutableAssign(baseCorrectAttaches, { + videofile: join(root(), 'server', 'tests', 'fixtures', 'video_short.ogv') + }) - await makeUploadRequest({ - url: server.url, - path: path + '/upload', - token: server.accessToken, - fields, - attaches, - statusCodeExpected: HttpStatusCode.OK_200 - }) - } + await checkUploadVideoParam(server.url, server.accessToken, { ...fields, ...attaches }, HttpStatusCode.OK_200, mode) + } + }) + } + + describe('Resumable upload', function () { + runSuite('resumable') + }) + + describe('Legacy upload', function () { + runSuite('legacy') }) }) @@ -678,7 +667,7 @@ describe('Test videos API validator', function () { }) expect(res.body.data).to.be.an('array') - expect(res.body.data.length).to.equal(3) + expect(res.body.data.length).to.equal(6) }) it('Should fail without a correct uuid', async function () { -- cgit v1.2.3