From f6d6e7f861189a4446f406efb775a29688764b48 Mon Sep 17 00:00:00 2001 From: kontrollanten <6680299+kontrollanten@users.noreply.github.com> Date: Mon, 10 May 2021 11:13:41 +0200 Subject: Resumable video uploads (#3933) * WIP: resumable video uploads relates to #324 * fix review comments * video upload: error handling * fix audio upload * fixes after self review * Update server/controllers/api/videos/index.ts Co-authored-by: Rigel Kent * Update server/middlewares/validators/videos/videos.ts Co-authored-by: Rigel Kent * Update server/controllers/api/videos/index.ts Co-authored-by: Rigel Kent * update after code review * refactor upload route - restore multipart upload route - move resumable to dedicated upload-resumable route - move checks to middleware - do not leak internal fs structure in response * fix yarn.lock upon rebase * factorize addVideo for reuse in both endpoints * add resumable upload API to openapi spec * add initial test and test helper for resumable upload * typings for videoAddResumable middleware * avoid including aws and google packages via node-uploadx, by only including uploadx/core * rename ex-isAudioBg to more explicit name mentioning it is a preview file for audio * add video-upload-tmp-folder-cleaner job * stronger typing of video upload middleware * reduce dependency to @uploadx/core * add audio upload test * refactor resumable uploads cleanup from job to scheduler * refactor resumable uploads scheduler to compare to last execution time * make resumable upload validator to always cleanup on failure * move legacy upload request building outside of uploadVideo test helper * filter upload-resumable middlewares down to POST, PUT, DELETE also begin to type metadata * merge add duration functions * stronger typings and documentation for uploadx behaviour, move init validator up * refactor(client/video-edit): options > uploadxOptions * refactor(client/video-edit): remove obsolete else * scheduler/remove-dangling-resum: rename tag * refactor(server/video): add UploadVideoFiles type * refactor(mw/validators): restructure eslint disable * refactor(mw/validators/videos): rename import * refactor(client/vid-upload): rename html elem id * refactor(sched/remove-dangl): move fn to method * refactor(mw/async): add method typing * refactor(mw/vali/video): double quote > single * refactor(server/upload-resum): express use > all * proper http methud enum server/middlewares/async.ts * properly type http methods * factorize common video upload validation steps * add check for maximum partially uploaded file size * fix audioBg use * fix extname(filename) in addVideo * document parameters for uploadx's resumable protocol * clear META files in scheduler * last audio refactor before cramming preview in the initial POST form data * refactor as mulitpart/form-data initial post request this allows preview/thumbnail uploads alongside the initial request, and cleans up the upload form * Add more tests for resumable uploads * Refactor remove dangling resumable uploads * Prepare changelog * Add more resumable upload tests * Remove user quota check for resumable uploads * Fix upload error handler * Update nginx template for upload-resumable * Cleanup comment * Remove unused express methods * Prefer to use got instead of raw http * Don't retry on error 500 Co-authored-by: Rigel Kent Co-authored-by: Rigel Kent Co-authored-by: Chocobozzz --- server/tests/api/check-params/index.ts | 1 + server/tests/api/check-params/upload-quota.ts | 152 ++++++ server/tests/api/check-params/users.ts | 105 +--- server/tests/api/check-params/videos.ts | 393 +++++++------- server/tests/api/videos/index.ts | 1 + server/tests/api/videos/multiple-servers.ts | 2 +- server/tests/api/videos/resumable-upload.ts | 187 +++++++ server/tests/api/videos/single-server.ts | 724 +++++++++++++------------- server/tests/api/videos/video-transcoder.ts | 159 +++--- 9 files changed, 987 insertions(+), 737 deletions(-) create mode 100644 server/tests/api/check-params/upload-quota.ts create mode 100644 server/tests/api/videos/resumable-upload.ts (limited to 'server/tests') diff --git a/server/tests/api/check-params/index.ts b/server/tests/api/check-params/index.ts index d0b0b9c21..143515838 100644 --- a/server/tests/api/check-params/index.ts +++ b/server/tests/api/check-params/index.ts @@ -13,6 +13,7 @@ import './plugins' import './redundancy' import './search' import './services' +import './upload-quota' import './user-notifications' import './user-subscriptions' import './users' diff --git a/server/tests/api/check-params/upload-quota.ts b/server/tests/api/check-params/upload-quota.ts new file mode 100644 index 000000000..d0fbec415 --- /dev/null +++ b/server/tests/api/check-params/upload-quota.ts @@ -0,0 +1,152 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import 'mocha' +import { expect } from 'chai' +import { HttpStatusCode, randomInt } from '@shared/core-utils' +import { getGoodVideoUrl, getMagnetURI, getMyVideoImports, importVideo } from '@shared/extra-utils/videos/video-imports' +import { MyUser, VideoImport, VideoImportState, VideoPrivacy } from '@shared/models' +import { + cleanupTests, + flushAndRunServer, + getMyUserInformation, + immutableAssign, + registerUser, + ServerInfo, + setAccessTokensToServers, + setDefaultVideoChannel, + updateUser, + uploadVideo, + userLogin, + waitJobs +} from '../../../../shared/extra-utils' + +describe('Test upload quota', function () { + let server: ServerInfo + let rootId: number + + // --------------------------------------------------------------- + + before(async function () { + this.timeout(30000) + + server = await flushAndRunServer(1) + await setAccessTokensToServers([ server ]) + await setDefaultVideoChannel([ server ]) + + const res = await getMyUserInformation(server.url, server.accessToken) + rootId = (res.body as MyUser).id + + await updateUser({ + url: server.url, + userId: rootId, + accessToken: server.accessToken, + videoQuota: 42 + }) + }) + + describe('When having a video quota', function () { + + it('Should fail with a registered user having too many videos with legacy upload', async function () { + this.timeout(30000) + + const user = { username: 'registered' + randomInt(1, 1500), password: 'password' } + await registerUser(server.url, user.username, user.password) + const userAccessToken = await userLogin(server, user) + + const videoAttributes = { fixture: 'video_short2.webm' } + for (let i = 0; i < 5; i++) { + await uploadVideo(server.url, userAccessToken, videoAttributes) + } + + await uploadVideo(server.url, userAccessToken, videoAttributes, HttpStatusCode.PAYLOAD_TOO_LARGE_413, 'legacy') + }) + + it('Should fail with a registered user having too many videos with resumable upload', async function () { + this.timeout(30000) + + const user = { username: 'registered' + randomInt(1, 1500), password: 'password' } + await registerUser(server.url, user.username, user.password) + const userAccessToken = await userLogin(server, user) + + const videoAttributes = { fixture: 'video_short2.webm' } + for (let i = 0; i < 5; i++) { + await uploadVideo(server.url, userAccessToken, videoAttributes) + } + + await uploadVideo(server.url, userAccessToken, videoAttributes, HttpStatusCode.PAYLOAD_TOO_LARGE_413, 'resumable') + }) + + it('Should fail to import with HTTP/Torrent/magnet', async function () { + this.timeout(120000) + + const baseAttributes = { + channelId: server.videoChannel.id, + privacy: VideoPrivacy.PUBLIC + } + await importVideo(server.url, server.accessToken, immutableAssign(baseAttributes, { targetUrl: getGoodVideoUrl() })) + await importVideo(server.url, server.accessToken, immutableAssign(baseAttributes, { magnetUri: getMagnetURI() })) + await importVideo(server.url, server.accessToken, immutableAssign(baseAttributes, { torrentfile: 'video-720p.torrent' as any })) + + await waitJobs([ server ]) + + const res = await getMyVideoImports(server.url, server.accessToken) + + expect(res.body.total).to.equal(3) + const videoImports: VideoImport[] = res.body.data + expect(videoImports).to.have.lengthOf(3) + + for (const videoImport of videoImports) { + expect(videoImport.state.id).to.equal(VideoImportState.FAILED) + expect(videoImport.error).not.to.be.undefined + expect(videoImport.error).to.contain('user video quota is exceeded') + } + }) + }) + + describe('When having a daily video quota', function () { + + it('Should fail with a user having too many videos daily', async function () { + await updateUser({ + url: server.url, + userId: rootId, + accessToken: server.accessToken, + videoQuotaDaily: 42 + }) + + await uploadVideo(server.url, server.accessToken, {}, HttpStatusCode.PAYLOAD_TOO_LARGE_413, 'legacy') + await uploadVideo(server.url, server.accessToken, {}, HttpStatusCode.PAYLOAD_TOO_LARGE_413, 'resumable') + }) + }) + + describe('When having an absolute and daily video quota', function () { + it('Should fail if exceeding total quota', async function () { + await updateUser({ + url: server.url, + userId: rootId, + accessToken: server.accessToken, + videoQuota: 42, + videoQuotaDaily: 1024 * 1024 * 1024 + }) + + await uploadVideo(server.url, server.accessToken, {}, HttpStatusCode.PAYLOAD_TOO_LARGE_413, 'legacy') + await uploadVideo(server.url, server.accessToken, {}, HttpStatusCode.PAYLOAD_TOO_LARGE_413, 'resumable') + }) + + it('Should fail if exceeding daily quota', async function () { + await updateUser({ + url: server.url, + userId: rootId, + accessToken: server.accessToken, + videoQuota: 1024 * 1024 * 1024, + videoQuotaDaily: 42 + }) + + await uploadVideo(server.url, server.accessToken, {}, HttpStatusCode.PAYLOAD_TOO_LARGE_413, 'legacy') + await uploadVideo(server.url, server.accessToken, {}, HttpStatusCode.PAYLOAD_TOO_LARGE_413, 'resumable') + }) + }) + + after(async function () { + await cleanupTests([ server ]) + }) +}) diff --git a/server/tests/api/check-params/users.ts b/server/tests/api/check-params/users.ts index 2b03fde2d..dcff0d52b 100644 --- a/server/tests/api/check-params/users.ts +++ b/server/tests/api/check-params/users.ts @@ -1,10 +1,10 @@ /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ import 'mocha' -import { expect } from 'chai' import { omit } from 'lodash' import { join } from 'path' -import { User, UserRole, VideoImport, VideoImportState } from '../../../../shared' +import { User, UserRole } from '../../../../shared' +import { HttpStatusCode } from '../../../../shared/core-utils/miscs/http-error-codes' import { addVideoChannel, blockUser, @@ -29,7 +29,6 @@ import { ServerInfo, setAccessTokensToServers, unblockUser, - updateUser, uploadVideo, userLogin } from '../../../../shared/extra-utils' @@ -39,11 +38,7 @@ import { checkBadSortPagination, checkBadStartPagination } from '../../../../shared/extra-utils/requests/check-api-params' -import { waitJobs } from '../../../../shared/extra-utils/server/jobs' -import { getGoodVideoUrl, getMagnetURI, getMyVideoImports, importVideo } from '../../../../shared/extra-utils/videos/video-imports' import { UserAdminFlag } from '../../../../shared/models/users/user-flag.model' -import { VideoPrivacy } from '../../../../shared/models/videos' -import { HttpStatusCode } from '../../../../shared/core-utils/miscs/http-error-codes' describe('Test users API validators', function () { const path = '/api/v1/users/' @@ -1093,102 +1088,6 @@ describe('Test users API validators', function () { }) }) - describe('When having a video quota', function () { - it('Should fail with a user having too many videos', async function () { - await updateUser({ - url: server.url, - userId: rootId, - accessToken: server.accessToken, - videoQuota: 42 - }) - - await uploadVideo(server.url, server.accessToken, {}, HttpStatusCode.PAYLOAD_TOO_LARGE_413) - }) - - it('Should fail with a registered user having too many videos', async function () { - this.timeout(30000) - - const user = { - username: 'user3', - password: 'my super password' - } - userAccessToken = await userLogin(server, user) - - const videoAttributes = { fixture: 'video_short2.webm' } - await uploadVideo(server.url, userAccessToken, videoAttributes) - await uploadVideo(server.url, userAccessToken, videoAttributes) - await uploadVideo(server.url, userAccessToken, videoAttributes) - await uploadVideo(server.url, userAccessToken, videoAttributes) - await uploadVideo(server.url, userAccessToken, videoAttributes) - await uploadVideo(server.url, userAccessToken, videoAttributes, HttpStatusCode.PAYLOAD_TOO_LARGE_413) - }) - - it('Should fail to import with HTTP/Torrent/magnet', async function () { - this.timeout(120000) - - const baseAttributes = { - channelId: 1, - privacy: VideoPrivacy.PUBLIC - } - await importVideo(server.url, server.accessToken, immutableAssign(baseAttributes, { targetUrl: getGoodVideoUrl() })) - await importVideo(server.url, server.accessToken, immutableAssign(baseAttributes, { magnetUri: getMagnetURI() })) - await importVideo(server.url, server.accessToken, immutableAssign(baseAttributes, { torrentfile: 'video-720p.torrent' as any })) - - await waitJobs([ server ]) - - const res = await getMyVideoImports(server.url, server.accessToken) - - expect(res.body.total).to.equal(3) - const videoImports: VideoImport[] = res.body.data - expect(videoImports).to.have.lengthOf(3) - - for (const videoImport of videoImports) { - expect(videoImport.state.id).to.equal(VideoImportState.FAILED) - expect(videoImport.error).not.to.be.undefined - expect(videoImport.error).to.contain('user video quota is exceeded') - } - }) - }) - - describe('When having a daily video quota', function () { - it('Should fail with a user having too many videos daily', async function () { - await updateUser({ - url: server.url, - userId: rootId, - accessToken: server.accessToken, - videoQuotaDaily: 42 - }) - - await uploadVideo(server.url, server.accessToken, {}, HttpStatusCode.PAYLOAD_TOO_LARGE_413) - }) - }) - - describe('When having an absolute and daily video quota', function () { - it('Should fail if exceeding total quota', async function () { - await updateUser({ - url: server.url, - userId: rootId, - accessToken: server.accessToken, - videoQuota: 42, - videoQuotaDaily: 1024 * 1024 * 1024 - }) - - await uploadVideo(server.url, server.accessToken, {}, HttpStatusCode.PAYLOAD_TOO_LARGE_413) - }) - - it('Should fail if exceeding daily quota', async function () { - await updateUser({ - url: server.url, - userId: rootId, - accessToken: server.accessToken, - videoQuota: 1024 * 1024 * 1024, - videoQuotaDaily: 42 - }) - - await uploadVideo(server.url, server.accessToken, {}, HttpStatusCode.PAYLOAD_TOO_LARGE_413) - }) - }) - describe('When asking a password reset', function () { const path = '/api/v1/users/ask-reset-password' diff --git a/server/tests/api/check-params/videos.ts b/server/tests/api/check-params/videos.ts index 188d1835c..c970c4a15 100644 --- a/server/tests/api/check-params/videos.ts +++ b/server/tests/api/check-params/videos.ts @@ -1,11 +1,12 @@ /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ +import 'mocha' import * as chai from 'chai' import { omit } from 'lodash' -import 'mocha' import { join } from 'path' -import { VideoPrivacy } from '../../../../shared/models/videos/video-privacy.enum' +import { HttpStatusCode } from '../../../../shared/core-utils/miscs/http-error-codes' import { + checkUploadVideoParam, cleanupTests, createUser, flushAndRunServer, @@ -18,17 +19,18 @@ import { makePutBodyRequest, makeUploadRequest, removeVideo, + root, ServerInfo, setAccessTokensToServers, - userLogin, - root + userLogin } from '../../../../shared/extra-utils' import { checkBadCountPagination, checkBadSortPagination, checkBadStartPagination } from '../../../../shared/extra-utils/requests/check-api-params' -import { HttpStatusCode } from '../../../../shared/core-utils/miscs/http-error-codes' +import { VideoPrivacy } from '../../../../shared/models/videos/video-privacy.enum' +import { randomInt } from '@shared/core-utils' const expect = chai.expect @@ -183,7 +185,7 @@ describe('Test videos API validator', function () { describe('When adding a video', function () { let baseCorrectParams const baseCorrectAttaches = { - videofile: join(root(), 'server', 'tests', 'fixtures', 'video_short.webm') + fixture: join(root(), 'server', 'tests', 'fixtures', 'video_short.webm') } before(function () { @@ -206,256 +208,243 @@ describe('Test videos API validator', function () { } }) - it('Should fail with nothing', async function () { - const fields = {} - const attaches = {} - await makeUploadRequest({ url: server.url, path: path + '/upload', token: server.accessToken, fields, attaches }) - }) + function runSuite (mode: 'legacy' | 'resumable') { - it('Should fail without name', async function () { - const fields = omit(baseCorrectParams, 'name') - const attaches = baseCorrectAttaches + it('Should fail with nothing', async function () { + const fields = {} + const attaches = {} + await checkUploadVideoParam(server.url, server.accessToken, { ...fields, ...attaches }, HttpStatusCode.BAD_REQUEST_400, mode) + }) - await makeUploadRequest({ url: server.url, path: path + '/upload', token: server.accessToken, fields, attaches }) - }) + it('Should fail without name', async function () { + const fields = omit(baseCorrectParams, 'name') + const attaches = baseCorrectAttaches - it('Should fail with a long name', async function () { - const fields = immutableAssign(baseCorrectParams, { name: 'super'.repeat(65) }) - const attaches = baseCorrectAttaches + await checkUploadVideoParam(server.url, server.accessToken, { ...fields, ...attaches }, HttpStatusCode.BAD_REQUEST_400, mode) + }) - await makeUploadRequest({ url: server.url, path: path + '/upload', token: server.accessToken, fields, attaches }) - }) + it('Should fail with a long name', async function () { + const fields = immutableAssign(baseCorrectParams, { name: 'super'.repeat(65) }) + const attaches = baseCorrectAttaches - it('Should fail with a bad category', async function () { - const fields = immutableAssign(baseCorrectParams, { category: 125 }) - const attaches = baseCorrectAttaches + await checkUploadVideoParam(server.url, server.accessToken, { ...fields, ...attaches }, HttpStatusCode.BAD_REQUEST_400, mode) + }) - await makeUploadRequest({ url: server.url, path: path + '/upload', token: server.accessToken, fields, attaches }) - }) + it('Should fail with a bad category', async function () { + const fields = immutableAssign(baseCorrectParams, { category: 125 }) + const attaches = baseCorrectAttaches - it('Should fail with a bad licence', async function () { - const fields = immutableAssign(baseCorrectParams, { licence: 125 }) - const attaches = baseCorrectAttaches + await checkUploadVideoParam(server.url, server.accessToken, { ...fields, ...attaches }, HttpStatusCode.BAD_REQUEST_400, mode) + }) - await makeUploadRequest({ url: server.url, path: path + '/upload', token: server.accessToken, fields, attaches }) - }) + it('Should fail with a bad licence', async function () { + const fields = immutableAssign(baseCorrectParams, { licence: 125 }) + const attaches = baseCorrectAttaches - it('Should fail with a bad language', async function () { - const fields = immutableAssign(baseCorrectParams, { language: 'a'.repeat(15) }) - const attaches = baseCorrectAttaches + await checkUploadVideoParam(server.url, server.accessToken, { ...fields, ...attaches }, HttpStatusCode.BAD_REQUEST_400, mode) + }) - await makeUploadRequest({ url: server.url, path: path + '/upload', token: server.accessToken, fields, attaches }) - }) + it('Should fail with a bad language', async function () { + const fields = immutableAssign(baseCorrectParams, { language: 'a'.repeat(15) }) + const attaches = baseCorrectAttaches - it('Should fail with a long description', async function () { - const fields = immutableAssign(baseCorrectParams, { description: 'super'.repeat(2500) }) - const attaches = baseCorrectAttaches + await checkUploadVideoParam(server.url, server.accessToken, { ...fields, ...attaches }, HttpStatusCode.BAD_REQUEST_400, mode) + }) - await makeUploadRequest({ url: server.url, path: path + '/upload', token: server.accessToken, fields, attaches }) - }) + it('Should fail with a long description', async function () { + const fields = immutableAssign(baseCorrectParams, { description: 'super'.repeat(2500) }) + const attaches = baseCorrectAttaches - it('Should fail with a long support text', async function () { - const fields = immutableAssign(baseCorrectParams, { support: 'super'.repeat(201) }) - const attaches = baseCorrectAttaches + await checkUploadVideoParam(server.url, server.accessToken, { ...fields, ...attaches }, HttpStatusCode.BAD_REQUEST_400, mode) + }) - await makeUploadRequest({ url: server.url, path: path + '/upload', token: server.accessToken, fields, attaches }) - }) + it('Should fail with a long support text', async function () { + const fields = immutableAssign(baseCorrectParams, { support: 'super'.repeat(201) }) + const attaches = baseCorrectAttaches - it('Should fail without a channel', async function () { - const fields = omit(baseCorrectParams, 'channelId') - const attaches = baseCorrectAttaches + await checkUploadVideoParam(server.url, server.accessToken, { ...fields, ...attaches }, HttpStatusCode.BAD_REQUEST_400, mode) + }) - await makeUploadRequest({ url: server.url, path: path + '/upload', token: server.accessToken, fields, attaches }) - }) + it('Should fail without a channel', async function () { + const fields = omit(baseCorrectParams, 'channelId') + const attaches = baseCorrectAttaches - it('Should fail with a bad channel', async function () { - const fields = immutableAssign(baseCorrectParams, { channelId: 545454 }) - const attaches = baseCorrectAttaches + await checkUploadVideoParam(server.url, server.accessToken, { ...fields, ...attaches }, HttpStatusCode.BAD_REQUEST_400, mode) + }) - await makeUploadRequest({ url: server.url, path: path + '/upload', token: server.accessToken, fields, attaches }) - }) + it('Should fail with a bad channel', async function () { + const fields = immutableAssign(baseCorrectParams, { channelId: 545454 }) + const attaches = baseCorrectAttaches - it('Should fail with another user channel', async function () { - const user = { - username: 'fake', - password: 'fake_password' - } - await createUser({ url: server.url, accessToken: server.accessToken, username: user.username, password: user.password }) + await checkUploadVideoParam(server.url, server.accessToken, { ...fields, ...attaches }, HttpStatusCode.BAD_REQUEST_400, mode) + }) - const accessTokenUser = await userLogin(server, user) - const res = await getMyUserInformation(server.url, accessTokenUser) - const customChannelId = res.body.videoChannels[0].id + it('Should fail with another user channel', async function () { + const user = { + username: 'fake' + randomInt(0, 1500), + password: 'fake_password' + } + await createUser({ url: server.url, accessToken: server.accessToken, username: user.username, password: user.password }) - const fields = immutableAssign(baseCorrectParams, { channelId: customChannelId }) - const attaches = baseCorrectAttaches + const accessTokenUser = await userLogin(server, user) + const res = await getMyUserInformation(server.url, accessTokenUser) + const customChannelId = res.body.videoChannels[0].id - await makeUploadRequest({ url: server.url, path: path + '/upload', token: userAccessToken, fields, attaches }) - }) + const fields = immutableAssign(baseCorrectParams, { channelId: customChannelId }) + const attaches = baseCorrectAttaches - it('Should fail with too many tags', async function () { - const fields = immutableAssign(baseCorrectParams, { tags: [ 'tag1', 'tag2', 'tag3', 'tag4', 'tag5', 'tag6' ] }) - const attaches = baseCorrectAttaches + await checkUploadVideoParam(server.url, userAccessToken, { ...fields, ...attaches }, HttpStatusCode.BAD_REQUEST_400, mode) + }) - await makeUploadRequest({ url: server.url, path: path + '/upload', token: server.accessToken, fields, attaches }) - }) + it('Should fail with too many tags', async function () { + const fields = immutableAssign(baseCorrectParams, { tags: [ 'tag1', 'tag2', 'tag3', 'tag4', 'tag5', 'tag6' ] }) + const attaches = baseCorrectAttaches - it('Should fail with a tag length too low', async function () { - const fields = immutableAssign(baseCorrectParams, { tags: [ 'tag1', 't' ] }) - const attaches = baseCorrectAttaches + await checkUploadVideoParam(server.url, server.accessToken, { ...fields, ...attaches }, HttpStatusCode.BAD_REQUEST_400, mode) + }) - await makeUploadRequest({ url: server.url, path: path + '/upload', token: server.accessToken, fields, attaches }) - }) + it('Should fail with a tag length too low', async function () { + const fields = immutableAssign(baseCorrectParams, { tags: [ 'tag1', 't' ] }) + const attaches = baseCorrectAttaches - it('Should fail with a tag length too big', async function () { - const fields = immutableAssign(baseCorrectParams, { tags: [ 'tag1', 'my_super_tag_too_long_long_long_long_long_long' ] }) - const attaches = baseCorrectAttaches + await checkUploadVideoParam(server.url, server.accessToken, { ...fields, ...attaches }, HttpStatusCode.BAD_REQUEST_400, mode) + }) - await makeUploadRequest({ url: server.url, path: path + '/upload', token: server.accessToken, fields, attaches }) - }) + it('Should fail with a tag length too big', async function () { + const fields = immutableAssign(baseCorrectParams, { tags: [ 'tag1', 'my_super_tag_too_long_long_long_long_long_long' ] }) + const attaches = baseCorrectAttaches - it('Should fail with a bad schedule update (miss updateAt)', async function () { - const fields = immutableAssign(baseCorrectParams, { 'scheduleUpdate[privacy]': VideoPrivacy.PUBLIC }) - const attaches = baseCorrectAttaches + await checkUploadVideoParam(server.url, server.accessToken, { ...fields, ...attaches }, HttpStatusCode.BAD_REQUEST_400, mode) + }) - await makeUploadRequest({ url: server.url, path: path + '/upload', token: server.accessToken, fields, attaches }) - }) + it('Should fail with a bad schedule update (miss updateAt)', async function () { + const fields = immutableAssign(baseCorrectParams, { scheduleUpdate: { privacy: VideoPrivacy.PUBLIC } }) + const attaches = baseCorrectAttaches - it('Should fail with a bad schedule update (wrong updateAt)', async function () { - const fields = immutableAssign(baseCorrectParams, { - 'scheduleUpdate[privacy]': VideoPrivacy.PUBLIC, - 'scheduleUpdate[updateAt]': 'toto' + await checkUploadVideoParam(server.url, server.accessToken, { ...fields, ...attaches }, HttpStatusCode.BAD_REQUEST_400, mode) }) - const attaches = baseCorrectAttaches - await makeUploadRequest({ url: server.url, path: path + '/upload', token: server.accessToken, fields, attaches }) - }) + it('Should fail with a bad schedule update (wrong updateAt)', async function () { + const fields = immutableAssign(baseCorrectParams, { + scheduleUpdate: { + privacy: VideoPrivacy.PUBLIC, + updateAt: 'toto' + } + }) + const attaches = baseCorrectAttaches - it('Should fail with a bad originally published at attribute', async function () { - const fields = immutableAssign(baseCorrectParams, { originallyPublishedAt: 'toto' }) - const attaches = baseCorrectAttaches + await checkUploadVideoParam(server.url, server.accessToken, { ...fields, ...attaches }, HttpStatusCode.BAD_REQUEST_400, mode) + }) - await makeUploadRequest({ url: server.url, path: path + '/upload', token: server.accessToken, fields, attaches }) - }) + it('Should fail with a bad originally published at attribute', async function () { + const fields = immutableAssign(baseCorrectParams, { originallyPublishedAt: 'toto' }) + const attaches = baseCorrectAttaches - it('Should fail without an input file', async function () { - const fields = baseCorrectParams - const attaches = {} - await makeUploadRequest({ url: server.url, path: path + '/upload', token: server.accessToken, fields, attaches }) - }) + await checkUploadVideoParam(server.url, server.accessToken, { ...fields, ...attaches }, HttpStatusCode.BAD_REQUEST_400, mode) + }) - it('Should fail with an incorrect input file', async function () { - const fields = baseCorrectParams - let attaches = { - videofile: join(root(), 'server', 'tests', 'fixtures', 'video_short_fake.webm') - } - await makeUploadRequest({ - url: server.url, - path: path + '/upload', - token: server.accessToken, - fields, - attaches, - statusCodeExpected: HttpStatusCode.UNPROCESSABLE_ENTITY_422 + it('Should fail without an input file', async function () { + const fields = baseCorrectParams + const attaches = {} + await checkUploadVideoParam(server.url, server.accessToken, { ...fields, ...attaches }, HttpStatusCode.BAD_REQUEST_400, mode) }) - attaches = { - videofile: join(root(), 'server', 'tests', 'fixtures', 'video_short.mkv') - } - await makeUploadRequest({ - url: server.url, - path: path + '/upload', - token: server.accessToken, - fields, - attaches, - statusCodeExpected: HttpStatusCode.UNSUPPORTED_MEDIA_TYPE_415 + it('Should fail with an incorrect input file', async function () { + const fields = baseCorrectParams + let attaches = { fixture: join(root(), 'server', 'tests', 'fixtures', 'video_short_fake.webm') } + + await checkUploadVideoParam( + server.url, + server.accessToken, + { ...fields, ...attaches }, + HttpStatusCode.UNPROCESSABLE_ENTITY_422, + mode + ) + + attaches = { fixture: join(root(), 'server', 'tests', 'fixtures', 'video_short.mkv') } + await checkUploadVideoParam( + server.url, + server.accessToken, + { ...fields, ...attaches }, + HttpStatusCode.UNSUPPORTED_MEDIA_TYPE_415, + mode + ) }) - }) - it('Should fail with an incorrect thumbnail file', async function () { - const fields = baseCorrectParams - const attaches = { - thumbnailfile: join(root(), 'server', 'tests', 'fixtures', 'video_short.mp4'), - videofile: join(root(), 'server', 'tests', 'fixtures', 'video_short.mp4') - } + it('Should fail with an incorrect thumbnail file', async function () { + const fields = baseCorrectParams + const attaches = { + thumbnailfile: join(root(), 'server', 'tests', 'fixtures', 'video_short.mp4'), + fixture: join(root(), 'server', 'tests', 'fixtures', 'video_short.mp4') + } - await makeUploadRequest({ url: server.url, path: path + '/upload', token: server.accessToken, fields, attaches }) - }) + await checkUploadVideoParam(server.url, server.accessToken, { ...fields, ...attaches }, HttpStatusCode.BAD_REQUEST_400, mode) + }) - it('Should fail with a big thumbnail file', async function () { - const fields = baseCorrectParams - const attaches = { - thumbnailfile: join(root(), 'server', 'tests', 'fixtures', 'preview-big.png'), - videofile: join(root(), 'server', 'tests', 'fixtures', 'video_short.mp4') - } + it('Should fail with a big thumbnail file', async function () { + const fields = baseCorrectParams + const attaches = { + thumbnailfile: join(root(), 'server', 'tests', 'fixtures', 'preview-big.png'), + fixture: join(root(), 'server', 'tests', 'fixtures', 'video_short.mp4') + } - await makeUploadRequest({ url: server.url, path: path + '/upload', token: server.accessToken, fields, attaches }) - }) + await checkUploadVideoParam(server.url, server.accessToken, { ...fields, ...attaches }, HttpStatusCode.BAD_REQUEST_400, mode) + }) - it('Should fail with an incorrect preview file', async function () { - const fields = baseCorrectParams - const attaches = { - previewfile: join(root(), 'server', 'tests', 'fixtures', 'video_short.mp4'), - videofile: join(root(), 'server', 'tests', 'fixtures', 'video_short.mp4') - } + it('Should fail with an incorrect preview file', async function () { + const fields = baseCorrectParams + const attaches = { + previewfile: join(root(), 'server', 'tests', 'fixtures', 'video_short.mp4'), + fixture: join(root(), 'server', 'tests', 'fixtures', 'video_short.mp4') + } - await makeUploadRequest({ url: server.url, path: path + '/upload', token: server.accessToken, fields, attaches }) - }) + await checkUploadVideoParam(server.url, server.accessToken, { ...fields, ...attaches }, HttpStatusCode.BAD_REQUEST_400, mode) + }) - it('Should fail with a big preview file', async function () { - const fields = baseCorrectParams - const attaches = { - previewfile: join(root(), 'server', 'tests', 'fixtures', 'preview-big.png'), - videofile: join(root(), 'server', 'tests', 'fixtures', 'video_short.mp4') - } + it('Should fail with a big preview file', async function () { + const fields = baseCorrectParams + const attaches = { + previewfile: join(root(), 'server', 'tests', 'fixtures', 'preview-big.png'), + fixture: join(root(), 'server', 'tests', 'fixtures', 'video_short.mp4') + } - await makeUploadRequest({ url: server.url, path: path + '/upload', token: server.accessToken, fields, attaches }) - }) + await checkUploadVideoParam(server.url, server.accessToken, { ...fields, ...attaches }, HttpStatusCode.BAD_REQUEST_400, mode) + }) - it('Should succeed with the correct parameters', async function () { - this.timeout(10000) + it('Should succeed with the correct parameters', async function () { + this.timeout(10000) - const fields = baseCorrectParams + const fields = baseCorrectParams - { - const attaches = baseCorrectAttaches - await makeUploadRequest({ - url: server.url, - path: path + '/upload', - token: server.accessToken, - fields, - attaches, - statusCodeExpected: HttpStatusCode.OK_200 - }) - } + { + const attaches = baseCorrectAttaches + await checkUploadVideoParam(server.url, server.accessToken, { ...fields, ...attaches }, HttpStatusCode.OK_200, mode) + } - { - const attaches = immutableAssign(baseCorrectAttaches, { - videofile: join(root(), 'server', 'tests', 'fixtures', 'video_short.mp4') - }) + { + const attaches = immutableAssign(baseCorrectAttaches, { + videofile: join(root(), 'server', 'tests', 'fixtures', 'video_short.mp4') + }) - await makeUploadRequest({ - url: server.url, - path: path + '/upload', - token: server.accessToken, - fields, - attaches, - statusCodeExpected: HttpStatusCode.OK_200 - }) - } + await checkUploadVideoParam(server.url, server.accessToken, { ...fields, ...attaches }, HttpStatusCode.OK_200, mode) + } - { - const attaches = immutableAssign(baseCorrectAttaches, { - videofile: join(root(), 'server', 'tests', 'fixtures', 'video_short.ogv') - }) + { + const attaches = immutableAssign(baseCorrectAttaches, { + videofile: join(root(), 'server', 'tests', 'fixtures', 'video_short.ogv') + }) - await makeUploadRequest({ - url: server.url, - path: path + '/upload', - token: server.accessToken, - fields, - attaches, - statusCodeExpected: HttpStatusCode.OK_200 - }) - } + await checkUploadVideoParam(server.url, server.accessToken, { ...fields, ...attaches }, HttpStatusCode.OK_200, mode) + } + }) + } + + describe('Resumable upload', function () { + runSuite('resumable') + }) + + describe('Legacy upload', function () { + runSuite('legacy') }) }) @@ -678,7 +667,7 @@ describe('Test videos API validator', function () { }) expect(res.body.data).to.be.an('array') - expect(res.body.data.length).to.equal(3) + expect(res.body.data.length).to.equal(6) }) it('Should fail without a correct uuid', async function () { diff --git a/server/tests/api/videos/index.ts b/server/tests/api/videos/index.ts index fc8b447b7..5c07f8926 100644 --- a/server/tests/api/videos/index.ts +++ b/server/tests/api/videos/index.ts @@ -1,5 +1,6 @@ import './audio-only' import './multiple-servers' +import './resumable-upload' import './single-server' import './video-captions' import './video-change-ownership' diff --git a/server/tests/api/videos/multiple-servers.ts b/server/tests/api/videos/multiple-servers.ts index 55e280e9f..41cd814e0 100644 --- a/server/tests/api/videos/multiple-servers.ts +++ b/server/tests/api/videos/multiple-servers.ts @@ -181,7 +181,7 @@ describe('Test multiple servers', function () { thumbnailfile: 'thumbnail.jpg', previewfile: 'preview.jpg' } - await uploadVideo(servers[1].url, userAccessToken, videoAttributes) + await uploadVideo(servers[1].url, userAccessToken, videoAttributes, HttpStatusCode.OK_200, 'resumable') // Transcoding await waitJobs(servers) diff --git a/server/tests/api/videos/resumable-upload.ts b/server/tests/api/videos/resumable-upload.ts new file mode 100644 index 000000000..af9221c43 --- /dev/null +++ b/server/tests/api/videos/resumable-upload.ts @@ -0,0 +1,187 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import 'mocha' +import * as chai from 'chai' +import { pathExists, readdir, stat } from 'fs-extra' +import { join } from 'path' +import { HttpStatusCode } from '@shared/core-utils' +import { + buildAbsoluteFixturePath, + buildServerDirectory, + flushAndRunServer, + getMyUserInformation, + prepareResumableUpload, + sendDebugCommand, + sendResumableChunks, + ServerInfo, + setAccessTokensToServers, + setDefaultVideoChannel, + updateUser +} from '@shared/extra-utils' +import { MyUser, VideoPrivacy } from '@shared/models' + +const expect = chai.expect + +// Most classic resumable upload tests are done in other test suites + +describe('Test resumable upload', function () { + const defaultFixture = 'video_short.mp4' + let server: ServerInfo + let rootId: number + + async function buildSize (fixture: string, size?: number) { + if (size !== undefined) return size + + const baseFixture = buildAbsoluteFixturePath(fixture) + return (await stat(baseFixture)).size + } + + async function prepareUpload (sizeArg?: number) { + const size = await buildSize(defaultFixture, sizeArg) + + const attributes = { + name: 'video', + channelId: server.videoChannel.id, + privacy: VideoPrivacy.PUBLIC, + fixture: defaultFixture + } + + const mimetype = 'video/mp4' + + const res = await prepareResumableUpload({ url: server.url, token: server.accessToken, attributes, size, mimetype }) + + return res.header['location'].split('?')[1] + } + + async function sendChunks (options: { + pathUploadId: string + size?: number + expectedStatus?: HttpStatusCode + contentLength?: number + contentRange?: string + contentRangeBuilder?: (start: number, chunk: any) => string + }) { + const { pathUploadId, expectedStatus, contentLength, contentRangeBuilder } = options + + const size = await buildSize(defaultFixture, options.size) + const absoluteFilePath = buildAbsoluteFixturePath(defaultFixture) + + return sendResumableChunks({ + url: server.url, + token: server.accessToken, + pathUploadId, + videoFilePath: absoluteFilePath, + size, + contentLength, + contentRangeBuilder, + specialStatus: expectedStatus + }) + } + + async function checkFileSize (uploadIdArg: string, expectedSize: number | null) { + const uploadId = uploadIdArg.replace(/^upload_id=/, '') + + const subPath = join('tmp', 'resumable-uploads', uploadId) + const filePath = buildServerDirectory(server, subPath) + const exists = await pathExists(filePath) + + if (expectedSize === null) { + expect(exists).to.be.false + return + } + + expect(exists).to.be.true + + expect((await stat(filePath)).size).to.equal(expectedSize) + } + + async function countResumableUploads () { + const subPath = join('tmp', 'resumable-uploads') + const filePath = buildServerDirectory(server, subPath) + + const files = await readdir(filePath) + return files.length + } + + before(async function () { + this.timeout(30000) + + server = await flushAndRunServer(1) + await setAccessTokensToServers([ server ]) + await setDefaultVideoChannel([ server ]) + + const res = await getMyUserInformation(server.url, server.accessToken) + rootId = (res.body as MyUser).id + + await updateUser({ + url: server.url, + userId: rootId, + accessToken: server.accessToken, + videoQuota: 10_000_000 + }) + }) + + describe('Directory cleaning', function () { + + it('Should correctly delete files after an upload', async function () { + const uploadId = await prepareUpload() + await sendChunks({ pathUploadId: uploadId }) + + expect(await countResumableUploads()).to.equal(0) + }) + + it('Should not delete files after an unfinished upload', async function () { + await prepareUpload() + + expect(await countResumableUploads()).to.equal(2) + }) + + it('Should not delete recent uploads', async function () { + await sendDebugCommand(server.url, server.accessToken, { command: 'remove-dandling-resumable-uploads' }) + + expect(await countResumableUploads()).to.equal(2) + }) + + it('Should delete old uploads', async function () { + await sendDebugCommand(server.url, server.accessToken, { command: 'remove-dandling-resumable-uploads' }) + + expect(await countResumableUploads()).to.equal(0) + }) + }) + + describe('Resumable upload and chunks', function () { + + it('Should accept the same amount of chunks', async function () { + const uploadId = await prepareUpload() + await sendChunks({ pathUploadId: uploadId }) + + await checkFileSize(uploadId, null) + }) + + it('Should not accept more chunks than expected', async function () { + const size = 100 + const uploadId = await prepareUpload(size) + + await sendChunks({ pathUploadId: uploadId, expectedStatus: HttpStatusCode.CONFLICT_409 }) + await checkFileSize(uploadId, 0) + }) + + it('Should not accept more chunks than expected with an invalid content length/content range', async function () { + const uploadId = await prepareUpload(1500) + + await sendChunks({ pathUploadId: uploadId, expectedStatus: HttpStatusCode.BAD_REQUEST_400, contentLength: 1000 }) + await checkFileSize(uploadId, 0) + }) + + it('Should not accept more chunks than expected with an invalid content length', async function () { + const uploadId = await prepareUpload(500) + + const size = 1000 + + const contentRangeBuilder = start => `bytes ${start}-${start + size - 1}/${size}` + await sendChunks({ pathUploadId: uploadId, expectedStatus: HttpStatusCode.BAD_REQUEST_400, contentRangeBuilder, contentLength: size }) + await checkFileSize(uploadId, 0) + }) + }) + +}) diff --git a/server/tests/api/videos/single-server.ts b/server/tests/api/videos/single-server.ts index a79648bf7..1058a1e9c 100644 --- a/server/tests/api/videos/single-server.ts +++ b/server/tests/api/videos/single-server.ts @@ -1,9 +1,9 @@ /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ +import 'mocha' import * as chai from 'chai' import { keyBy } from 'lodash' -import 'mocha' -import { VideoPrivacy } from '../../../../shared/models/videos' + import { checkVideoFilesWereRemoved, cleanupTests, @@ -28,430 +28,432 @@ import { viewVideo, wait } from '../../../../shared/extra-utils' +import { VideoPrivacy } from '../../../../shared/models/videos' +import { HttpStatusCode } from '@shared/core-utils' const expect = chai.expect describe('Test a single server', function () { - let server: ServerInfo = null - let videoId = -1 - let videoId2 = -1 - let videoUUID = '' - let videosListBase: any[] = null - - const getCheckAttributes = () => ({ - name: 'my super name', - category: 2, - licence: 6, - language: 'zh', - nsfw: true, - description: 'my super description', - support: 'my super support text', - account: { - name: 'root', - host: 'localhost:' + server.port - }, - isLocal: true, - duration: 5, - tags: [ 'tag1', 'tag2', 'tag3' ], - privacy: VideoPrivacy.PUBLIC, - commentsEnabled: true, - downloadEnabled: true, - channel: { - displayName: 'Main root channel', - name: 'root_channel', - description: '', - isLocal: true - }, - fixture: 'video_short.webm', - files: [ - { - resolution: 720, - size: 218910 - } - ] - }) - - const updateCheckAttributes = () => ({ - name: 'my super video updated', - category: 4, - licence: 2, - language: 'ar', - nsfw: false, - description: 'my super description updated', - support: 'my super support text updated', - account: { - name: 'root', - host: 'localhost:' + server.port - }, - isLocal: true, - tags: [ 'tagup1', 'tagup2' ], - privacy: VideoPrivacy.PUBLIC, - duration: 5, - commentsEnabled: false, - downloadEnabled: false, - channel: { - name: 'root_channel', - displayName: 'Main root channel', - description: '', - isLocal: true - }, - fixture: 'video_short3.webm', - files: [ - { - resolution: 720, - size: 292677 - } - ] - }) - - before(async function () { - this.timeout(30000) - - server = await flushAndRunServer(1) - - await setAccessTokensToServers([ server ]) - }) - - it('Should list video categories', async function () { - const res = await getVideoCategories(server.url) - - const categories = res.body - expect(Object.keys(categories)).to.have.length.above(10) - - expect(categories[11]).to.equal('News & Politics') - }) - - it('Should list video licences', async function () { - const res = await getVideoLicences(server.url) - - const licences = res.body - expect(Object.keys(licences)).to.have.length.above(5) - - expect(licences[3]).to.equal('Attribution - No Derivatives') - }) - - it('Should list video languages', async function () { - const res = await getVideoLanguages(server.url) - - const languages = res.body - expect(Object.keys(languages)).to.have.length.above(5) - - expect(languages['ru']).to.equal('Russian') - }) - - it('Should list video privacies', async function () { - const res = await getVideoPrivacies(server.url) - - const privacies = res.body - expect(Object.keys(privacies)).to.have.length.at.least(3) - - expect(privacies[3]).to.equal('Private') - }) - - it('Should not have videos', async function () { - const res = await getVideosList(server.url) - - expect(res.body.total).to.equal(0) - expect(res.body.data).to.be.an('array') - expect(res.body.data.length).to.equal(0) - }) - it('Should upload the video', async function () { - this.timeout(10000) + function runSuite (mode: 'legacy' | 'resumable') { + let server: ServerInfo = null + let videoId = -1 + let videoId2 = -1 + let videoUUID = '' + let videosListBase: any[] = null - const videoAttributes = { + const getCheckAttributes = () => ({ name: 'my super name', category: 2, - nsfw: true, licence: 6, - tags: [ 'tag1', 'tag2', 'tag3' ] - } - const res = await uploadVideo(server.url, server.accessToken, videoAttributes) - expect(res.body.video).to.not.be.undefined - expect(res.body.video.id).to.equal(1) - expect(res.body.video.uuid).to.have.length.above(5) - - videoId = res.body.video.id - videoUUID = res.body.video.uuid - }) - - it('Should get and seed the uploaded video', async function () { - this.timeout(5000) - - const res = await getVideosList(server.url) - - expect(res.body.total).to.equal(1) - expect(res.body.data).to.be.an('array') - expect(res.body.data.length).to.equal(1) - - const video = res.body.data[0] - await completeVideoCheck(server.url, video, getCheckAttributes()) - }) + language: 'zh', + nsfw: true, + description: 'my super description', + support: 'my super support text', + account: { + name: 'root', + host: 'localhost:' + server.port + }, + isLocal: true, + duration: 5, + tags: [ 'tag1', 'tag2', 'tag3' ], + privacy: VideoPrivacy.PUBLIC, + commentsEnabled: true, + downloadEnabled: true, + channel: { + displayName: 'Main root channel', + name: 'root_channel', + description: '', + isLocal: true + }, + fixture: 'video_short.webm', + files: [ + { + resolution: 720, + size: 218910 + } + ] + }) + + const updateCheckAttributes = () => ({ + name: 'my super video updated', + category: 4, + licence: 2, + language: 'ar', + nsfw: false, + description: 'my super description updated', + support: 'my super support text updated', + account: { + name: 'root', + host: 'localhost:' + server.port + }, + isLocal: true, + tags: [ 'tagup1', 'tagup2' ], + privacy: VideoPrivacy.PUBLIC, + duration: 5, + commentsEnabled: false, + downloadEnabled: false, + channel: { + name: 'root_channel', + displayName: 'Main root channel', + description: '', + isLocal: true + }, + fixture: 'video_short3.webm', + files: [ + { + resolution: 720, + size: 292677 + } + ] + }) - it('Should get the video by UUID', async function () { - this.timeout(5000) + before(async function () { + this.timeout(30000) - const res = await getVideo(server.url, videoUUID) + server = await flushAndRunServer(1) - const video = res.body - await completeVideoCheck(server.url, video, getCheckAttributes()) - }) + await setAccessTokensToServers([ server ]) + }) - it('Should have the views updated', async function () { - this.timeout(20000) + it('Should list video categories', async function () { + const res = await getVideoCategories(server.url) - await viewVideo(server.url, videoId) - await viewVideo(server.url, videoId) - await viewVideo(server.url, videoId) + const categories = res.body + expect(Object.keys(categories)).to.have.length.above(10) - await wait(1500) + expect(categories[11]).to.equal('News & Politics') + }) - await viewVideo(server.url, videoId) - await viewVideo(server.url, videoId) + it('Should list video licences', async function () { + const res = await getVideoLicences(server.url) - await wait(1500) + const licences = res.body + expect(Object.keys(licences)).to.have.length.above(5) - await viewVideo(server.url, videoId) - await viewVideo(server.url, videoId) + expect(licences[3]).to.equal('Attribution - No Derivatives') + }) - // Wait the repeatable job - await wait(8000) + it('Should list video languages', async function () { + const res = await getVideoLanguages(server.url) - const res = await getVideo(server.url, videoId) + const languages = res.body + expect(Object.keys(languages)).to.have.length.above(5) - const video = res.body - expect(video.views).to.equal(3) - }) + expect(languages['ru']).to.equal('Russian') + }) - it('Should remove the video', async function () { - await removeVideo(server.url, server.accessToken, videoId) + it('Should list video privacies', async function () { + const res = await getVideoPrivacies(server.url) - await checkVideoFilesWereRemoved(videoUUID, 1) - }) + const privacies = res.body + expect(Object.keys(privacies)).to.have.length.at.least(3) - it('Should not have videos', async function () { - const res = await getVideosList(server.url) + expect(privacies[3]).to.equal('Private') + }) - expect(res.body.total).to.equal(0) - expect(res.body.data).to.be.an('array') - expect(res.body.data).to.have.lengthOf(0) - }) + it('Should not have videos', async function () { + const res = await getVideosList(server.url) - it('Should upload 6 videos', async function () { - this.timeout(25000) + expect(res.body.total).to.equal(0) + expect(res.body.data).to.be.an('array') + expect(res.body.data.length).to.equal(0) + }) - const videos = new Set([ - 'video_short.mp4', 'video_short.ogv', 'video_short.webm', - 'video_short1.webm', 'video_short2.webm', 'video_short3.webm' - ]) + it('Should upload the video', async function () { + this.timeout(10000) - for (const video of videos) { const videoAttributes = { - name: video + ' name', - description: video + ' description', + name: 'my super name', category: 2, - licence: 1, - language: 'en', nsfw: true, - tags: [ 'tag1', 'tag2', 'tag3' ], - fixture: video + licence: 6, + tags: [ 'tag1', 'tag2', 'tag3' ] } + const res = await uploadVideo(server.url, server.accessToken, videoAttributes, HttpStatusCode.OK_200, mode) + expect(res.body.video).to.not.be.undefined + expect(res.body.video.id).to.equal(1) + expect(res.body.video.uuid).to.have.length.above(5) - await uploadVideo(server.url, server.accessToken, videoAttributes) - } - }) + videoId = res.body.video.id + videoUUID = res.body.video.uuid + }) - it('Should have the correct durations', async function () { - const res = await getVideosList(server.url) - - expect(res.body.total).to.equal(6) - const videos = res.body.data - expect(videos).to.be.an('array') - expect(videos).to.have.lengthOf(6) - - const videosByName = keyBy<{ duration: number }>(videos, 'name') - expect(videosByName['video_short.mp4 name'].duration).to.equal(5) - expect(videosByName['video_short.ogv name'].duration).to.equal(5) - expect(videosByName['video_short.webm name'].duration).to.equal(5) - expect(videosByName['video_short1.webm name'].duration).to.equal(10) - expect(videosByName['video_short2.webm name'].duration).to.equal(5) - expect(videosByName['video_short3.webm name'].duration).to.equal(5) - }) + it('Should get and seed the uploaded video', async function () { + this.timeout(5000) - it('Should have the correct thumbnails', async function () { - const res = await getVideosList(server.url) + const res = await getVideosList(server.url) - const videos = res.body.data - // For the next test - videosListBase = videos + expect(res.body.total).to.equal(1) + expect(res.body.data).to.be.an('array') + expect(res.body.data.length).to.equal(1) - for (const video of videos) { - const videoName = video.name.replace(' name', '') - await testImage(server.url, videoName, video.thumbnailPath) - } - }) + const video = res.body.data[0] + await completeVideoCheck(server.url, video, getCheckAttributes()) + }) - it('Should list only the two first videos', async function () { - const res = await getVideosListPagination(server.url, 0, 2, 'name') + it('Should get the video by UUID', async function () { + this.timeout(5000) - const videos = res.body.data - expect(res.body.total).to.equal(6) - expect(videos.length).to.equal(2) - expect(videos[0].name).to.equal(videosListBase[0].name) - expect(videos[1].name).to.equal(videosListBase[1].name) - }) + const res = await getVideo(server.url, videoUUID) - it('Should list only the next three videos', async function () { - const res = await getVideosListPagination(server.url, 2, 3, 'name') + const video = res.body + await completeVideoCheck(server.url, video, getCheckAttributes()) + }) - const videos = res.body.data - expect(res.body.total).to.equal(6) - expect(videos.length).to.equal(3) - expect(videos[0].name).to.equal(videosListBase[2].name) - expect(videos[1].name).to.equal(videosListBase[3].name) - expect(videos[2].name).to.equal(videosListBase[4].name) - }) + it('Should have the views updated', async function () { + this.timeout(20000) - it('Should list the last video', async function () { - const res = await getVideosListPagination(server.url, 5, 6, 'name') + await viewVideo(server.url, videoId) + await viewVideo(server.url, videoId) + await viewVideo(server.url, videoId) - const videos = res.body.data - expect(res.body.total).to.equal(6) - expect(videos.length).to.equal(1) - expect(videos[0].name).to.equal(videosListBase[5].name) - }) + await wait(1500) - it('Should not have the total field', async function () { - const res = await getVideosListPagination(server.url, 5, 6, 'name', true) + await viewVideo(server.url, videoId) + await viewVideo(server.url, videoId) - const videos = res.body.data - expect(res.body.total).to.not.exist - expect(videos.length).to.equal(1) - expect(videos[0].name).to.equal(videosListBase[5].name) - }) + await wait(1500) - it('Should list and sort by name in descending order', async function () { - const res = await getVideosListSort(server.url, '-name') - - const videos = res.body.data - expect(res.body.total).to.equal(6) - expect(videos.length).to.equal(6) - expect(videos[0].name).to.equal('video_short.webm name') - expect(videos[1].name).to.equal('video_short.ogv name') - expect(videos[2].name).to.equal('video_short.mp4 name') - expect(videos[3].name).to.equal('video_short3.webm name') - expect(videos[4].name).to.equal('video_short2.webm name') - expect(videos[5].name).to.equal('video_short1.webm name') - - videoId = videos[3].uuid - videoId2 = videos[5].uuid - }) + await viewVideo(server.url, videoId) + await viewVideo(server.url, videoId) - it('Should list and sort by trending in descending order', async function () { - const res = await getVideosListPagination(server.url, 0, 2, '-trending') + // Wait the repeatable job + await wait(8000) - const videos = res.body.data - expect(res.body.total).to.equal(6) - expect(videos.length).to.equal(2) - }) + const res = await getVideo(server.url, videoId) - it('Should list and sort by hotness in descending order', async function () { - const res = await getVideosListPagination(server.url, 0, 2, '-hot') + const video = res.body + expect(video.views).to.equal(3) + }) - const videos = res.body.data - expect(res.body.total).to.equal(6) - expect(videos.length).to.equal(2) - }) + it('Should remove the video', async function () { + await removeVideo(server.url, server.accessToken, videoId) - it('Should list and sort by best in descending order', async function () { - const res = await getVideosListPagination(server.url, 0, 2, '-best') + await checkVideoFilesWereRemoved(videoUUID, 1) + }) - const videos = res.body.data - expect(res.body.total).to.equal(6) - expect(videos.length).to.equal(2) - }) + it('Should not have videos', async function () { + const res = await getVideosList(server.url) - it('Should update a video', async function () { - const attributes = { - name: 'my super video updated', - category: 4, - licence: 2, - language: 'ar', - nsfw: false, - description: 'my super description updated', - commentsEnabled: false, - downloadEnabled: false, - tags: [ 'tagup1', 'tagup2' ] - } - await updateVideo(server.url, server.accessToken, videoId, attributes) - }) + expect(res.body.total).to.equal(0) + expect(res.body.data).to.be.an('array') + expect(res.body.data).to.have.lengthOf(0) + }) - it('Should filter by tags and category', async function () { - const res1 = await getVideosWithFilters(server.url, { tagsAllOf: [ 'tagup1', 'tagup2' ], categoryOneOf: [ 4 ] }) - expect(res1.body.total).to.equal(1) - expect(res1.body.data[0].name).to.equal('my super video updated') + it('Should upload 6 videos', async function () { + this.timeout(25000) - const res2 = await getVideosWithFilters(server.url, { tagsAllOf: [ 'tagup1', 'tagup2' ], categoryOneOf: [ 3 ] }) - expect(res2.body.total).to.equal(0) - }) + const videos = new Set([ + 'video_short.mp4', 'video_short.ogv', 'video_short.webm', + 'video_short1.webm', 'video_short2.webm', 'video_short3.webm' + ]) - it('Should have the video updated', async function () { - this.timeout(60000) + for (const video of videos) { + const videoAttributes = { + name: video + ' name', + description: video + ' description', + category: 2, + licence: 1, + language: 'en', + nsfw: true, + tags: [ 'tag1', 'tag2', 'tag3' ], + fixture: video + } - const res = await getVideo(server.url, videoId) - const video = res.body + await uploadVideo(server.url, server.accessToken, videoAttributes, HttpStatusCode.OK_200, mode) + } + }) + + it('Should have the correct durations', async function () { + const res = await getVideosList(server.url) + + expect(res.body.total).to.equal(6) + const videos = res.body.data + expect(videos).to.be.an('array') + expect(videos).to.have.lengthOf(6) + + const videosByName = keyBy<{ duration: number }>(videos, 'name') + expect(videosByName['video_short.mp4 name'].duration).to.equal(5) + expect(videosByName['video_short.ogv name'].duration).to.equal(5) + expect(videosByName['video_short.webm name'].duration).to.equal(5) + expect(videosByName['video_short1.webm name'].duration).to.equal(10) + expect(videosByName['video_short2.webm name'].duration).to.equal(5) + expect(videosByName['video_short3.webm name'].duration).to.equal(5) + }) + + it('Should have the correct thumbnails', async function () { + const res = await getVideosList(server.url) + + const videos = res.body.data + // For the next test + videosListBase = videos + + for (const video of videos) { + const videoName = video.name.replace(' name', '') + await testImage(server.url, videoName, video.thumbnailPath) + } + }) + + it('Should list only the two first videos', async function () { + const res = await getVideosListPagination(server.url, 0, 2, 'name') + + const videos = res.body.data + expect(res.body.total).to.equal(6) + expect(videos.length).to.equal(2) + expect(videos[0].name).to.equal(videosListBase[0].name) + expect(videos[1].name).to.equal(videosListBase[1].name) + }) + + it('Should list only the next three videos', async function () { + const res = await getVideosListPagination(server.url, 2, 3, 'name') + + const videos = res.body.data + expect(res.body.total).to.equal(6) + expect(videos.length).to.equal(3) + expect(videos[0].name).to.equal(videosListBase[2].name) + expect(videos[1].name).to.equal(videosListBase[3].name) + expect(videos[2].name).to.equal(videosListBase[4].name) + }) + + it('Should list the last video', async function () { + const res = await getVideosListPagination(server.url, 5, 6, 'name') + + const videos = res.body.data + expect(res.body.total).to.equal(6) + expect(videos.length).to.equal(1) + expect(videos[0].name).to.equal(videosListBase[5].name) + }) + + it('Should not have the total field', async function () { + const res = await getVideosListPagination(server.url, 5, 6, 'name', true) + + const videos = res.body.data + expect(res.body.total).to.not.exist + expect(videos.length).to.equal(1) + expect(videos[0].name).to.equal(videosListBase[5].name) + }) + + it('Should list and sort by name in descending order', async function () { + const res = await getVideosListSort(server.url, '-name') + + const videos = res.body.data + expect(res.body.total).to.equal(6) + expect(videos.length).to.equal(6) + expect(videos[0].name).to.equal('video_short.webm name') + expect(videos[1].name).to.equal('video_short.ogv name') + expect(videos[2].name).to.equal('video_short.mp4 name') + expect(videos[3].name).to.equal('video_short3.webm name') + expect(videos[4].name).to.equal('video_short2.webm name') + expect(videos[5].name).to.equal('video_short1.webm name') + + videoId = videos[3].uuid + videoId2 = videos[5].uuid + }) + + it('Should list and sort by trending in descending order', async function () { + const res = await getVideosListPagination(server.url, 0, 2, '-trending') + + const videos = res.body.data + expect(res.body.total).to.equal(6) + expect(videos.length).to.equal(2) + }) + + it('Should list and sort by hotness in descending order', async function () { + const res = await getVideosListPagination(server.url, 0, 2, '-hot') + + const videos = res.body.data + expect(res.body.total).to.equal(6) + expect(videos.length).to.equal(2) + }) + + it('Should list and sort by best in descending order', async function () { + const res = await getVideosListPagination(server.url, 0, 2, '-best') + + const videos = res.body.data + expect(res.body.total).to.equal(6) + expect(videos.length).to.equal(2) + }) + + it('Should update a video', async function () { + const attributes = { + name: 'my super video updated', + category: 4, + licence: 2, + language: 'ar', + nsfw: false, + description: 'my super description updated', + commentsEnabled: false, + downloadEnabled: false, + tags: [ 'tagup1', 'tagup2' ] + } + await updateVideo(server.url, server.accessToken, videoId, attributes) + }) - await completeVideoCheck(server.url, video, updateCheckAttributes()) - }) + it('Should filter by tags and category', async function () { + const res1 = await getVideosWithFilters(server.url, { tagsAllOf: [ 'tagup1', 'tagup2' ], categoryOneOf: [ 4 ] }) + expect(res1.body.total).to.equal(1) + expect(res1.body.data[0].name).to.equal('my super video updated') - it('Should update only the tags of a video', async function () { - const attributes = { - tags: [ 'supertag', 'tag1', 'tag2' ] - } - await updateVideo(server.url, server.accessToken, videoId, attributes) + const res2 = await getVideosWithFilters(server.url, { tagsAllOf: [ 'tagup1', 'tagup2' ], categoryOneOf: [ 3 ] }) + expect(res2.body.total).to.equal(0) + }) - const res = await getVideo(server.url, videoId) - const video = res.body + it('Should have the video updated', async function () { + this.timeout(60000) - await completeVideoCheck(server.url, video, Object.assign(updateCheckAttributes(), attributes)) - }) + const res = await getVideo(server.url, videoId) + const video = res.body - it('Should update only the description of a video', async function () { - const attributes = { - description: 'hello everybody' - } - await updateVideo(server.url, server.accessToken, videoId, attributes) + await completeVideoCheck(server.url, video, updateCheckAttributes()) + }) - const res = await getVideo(server.url, videoId) - const video = res.body + it('Should update only the tags of a video', async function () { + const attributes = { + tags: [ 'supertag', 'tag1', 'tag2' ] + } + await updateVideo(server.url, server.accessToken, videoId, attributes) - const expectedAttributes = Object.assign(updateCheckAttributes(), { tags: [ 'supertag', 'tag1', 'tag2' ] }, attributes) - await completeVideoCheck(server.url, video, expectedAttributes) - }) + const res = await getVideo(server.url, videoId) + const video = res.body - it('Should like a video', async function () { - await rateVideo(server.url, server.accessToken, videoId, 'like') + await completeVideoCheck(server.url, video, Object.assign(updateCheckAttributes(), attributes)) + }) - const res = await getVideo(server.url, videoId) - const video = res.body + it('Should update only the description of a video', async function () { + const attributes = { + description: 'hello everybody' + } + await updateVideo(server.url, server.accessToken, videoId, attributes) - expect(video.likes).to.equal(1) - expect(video.dislikes).to.equal(0) - }) + const res = await getVideo(server.url, videoId) + const video = res.body - it('Should dislike the same video', async function () { - await rateVideo(server.url, server.accessToken, videoId, 'dislike') + const expectedAttributes = Object.assign(updateCheckAttributes(), { tags: [ 'supertag', 'tag1', 'tag2' ] }, attributes) + await completeVideoCheck(server.url, video, expectedAttributes) + }) - const res = await getVideo(server.url, videoId) - const video = res.body + it('Should like a video', async function () { + await rateVideo(server.url, server.accessToken, videoId, 'like') - expect(video.likes).to.equal(0) - expect(video.dislikes).to.equal(1) - }) + const res = await getVideo(server.url, videoId) + const video = res.body - it('Should sort by originallyPublishedAt', async function () { - { + expect(video.likes).to.equal(1) + expect(video.dislikes).to.equal(0) + }) + it('Should dislike the same video', async function () { + await rateVideo(server.url, server.accessToken, videoId, 'dislike') + + const res = await getVideo(server.url, videoId) + const video = res.body + + expect(video.likes).to.equal(0) + expect(video.dislikes).to.equal(1) + }) + + it('Should sort by originallyPublishedAt', async function () { { const now = new Date() const attributes = { originallyPublishedAt: now.toISOString() } @@ -483,10 +485,18 @@ describe('Test a single server', function () { expect(names[4]).to.equal('video_short.ogv name') expect(names[5]).to.equal('video_short.mp4 name') } - } + }) + + after(async function () { + await cleanupTests([ server ]) + }) + } + + describe('Legacy upload', function () { + runSuite('legacy') }) - after(async function () { - await cleanupTests([ server ]) + describe('Resumable upload', function () { + runSuite('resumable') }) }) diff --git a/server/tests/api/videos/video-transcoder.ts b/server/tests/api/videos/video-transcoder.ts index 1c99f26df..ea5ffd239 100644 --- a/server/tests/api/videos/video-transcoder.ts +++ b/server/tests/api/videos/video-transcoder.ts @@ -361,106 +361,117 @@ describe('Test video transcoding', function () { describe('Audio upload', function () { - before(async function () { - await updateCustomSubConfig(servers[1].url, servers[1].accessToken, { - transcoding: { - hls: { enabled: true }, - webtorrent: { enabled: true }, - resolutions: { - '0p': false, - '240p': false, - '360p': false, - '480p': false, - '720p': false, - '1080p': false, - '1440p': false, - '2160p': false + function runSuite (mode: 'legacy' | 'resumable') { + + before(async function () { + await updateCustomSubConfig(servers[1].url, servers[1].accessToken, { + transcoding: { + hls: { enabled: true }, + webtorrent: { enabled: true }, + resolutions: { + '0p': false, + '240p': false, + '360p': false, + '480p': false, + '720p': false, + '1080p': false, + '1440p': false, + '2160p': false + } } - } + }) }) - }) - - it('Should merge an audio file with the preview file', async function () { - this.timeout(60_000) - - const videoAttributesArg = { name: 'audio_with_preview', previewfile: 'preview.jpg', fixture: 'sample.ogg' } - await uploadVideo(servers[1].url, servers[1].accessToken, videoAttributesArg) - await waitJobs(servers) + it('Should merge an audio file with the preview file', async function () { + this.timeout(60_000) - for (const server of servers) { - const res = await getVideosList(server.url) + const videoAttributesArg = { name: 'audio_with_preview', previewfile: 'preview.jpg', fixture: 'sample.ogg' } + await uploadVideo(servers[1].url, servers[1].accessToken, videoAttributesArg, HttpStatusCode.OK_200, mode) - const video = res.body.data.find(v => v.name === 'audio_with_preview') - const res2 = await getVideo(server.url, video.id) - const videoDetails: VideoDetails = res2.body + await waitJobs(servers) - expect(videoDetails.files).to.have.lengthOf(1) + for (const server of servers) { + const res = await getVideosList(server.url) - await makeGetRequest({ url: server.url, path: videoDetails.thumbnailPath, statusCodeExpected: HttpStatusCode.OK_200 }) - await makeGetRequest({ url: server.url, path: videoDetails.previewPath, statusCodeExpected: HttpStatusCode.OK_200 }) + const video = res.body.data.find(v => v.name === 'audio_with_preview') + const res2 = await getVideo(server.url, video.id) + const videoDetails: VideoDetails = res2.body - const magnetUri = videoDetails.files[0].magnetUri - expect(magnetUri).to.contain('.mp4') - } - }) + expect(videoDetails.files).to.have.lengthOf(1) - it('Should upload an audio file and choose a default background image', async function () { - this.timeout(60_000) + await makeGetRequest({ url: server.url, path: videoDetails.thumbnailPath, statusCodeExpected: HttpStatusCode.OK_200 }) + await makeGetRequest({ url: server.url, path: videoDetails.previewPath, statusCodeExpected: HttpStatusCode.OK_200 }) - const videoAttributesArg = { name: 'audio_without_preview', fixture: 'sample.ogg' } - await uploadVideo(servers[1].url, servers[1].accessToken, videoAttributesArg) + const magnetUri = videoDetails.files[0].magnetUri + expect(magnetUri).to.contain('.mp4') + } + }) - await waitJobs(servers) + it('Should upload an audio file and choose a default background image', async function () { + this.timeout(60_000) - for (const server of servers) { - const res = await getVideosList(server.url) + const videoAttributesArg = { name: 'audio_without_preview', fixture: 'sample.ogg' } + await uploadVideo(servers[1].url, servers[1].accessToken, videoAttributesArg, HttpStatusCode.OK_200, mode) - const video = res.body.data.find(v => v.name === 'audio_without_preview') - const res2 = await getVideo(server.url, video.id) - const videoDetails = res2.body + await waitJobs(servers) - expect(videoDetails.files).to.have.lengthOf(1) + for (const server of servers) { + const res = await getVideosList(server.url) - await makeGetRequest({ url: server.url, path: videoDetails.thumbnailPath, statusCodeExpected: HttpStatusCode.OK_200 }) - await makeGetRequest({ url: server.url, path: videoDetails.previewPath, statusCodeExpected: HttpStatusCode.OK_200 }) + const video = res.body.data.find(v => v.name === 'audio_without_preview') + const res2 = await getVideo(server.url, video.id) + const videoDetails = res2.body - const magnetUri = videoDetails.files[0].magnetUri - expect(magnetUri).to.contain('.mp4') - } - }) + expect(videoDetails.files).to.have.lengthOf(1) - it('Should upload an audio file and create an audio version only', async function () { - this.timeout(60_000) + await makeGetRequest({ url: server.url, path: videoDetails.thumbnailPath, statusCodeExpected: HttpStatusCode.OK_200 }) + await makeGetRequest({ url: server.url, path: videoDetails.previewPath, statusCodeExpected: HttpStatusCode.OK_200 }) - await updateCustomSubConfig(servers[1].url, servers[1].accessToken, { - transcoding: { - hls: { enabled: true }, - webtorrent: { enabled: true }, - resolutions: { - '0p': true, - '240p': false, - '360p': false - } + const magnetUri = videoDetails.files[0].magnetUri + expect(magnetUri).to.contain('.mp4') } }) - const videoAttributesArg = { name: 'audio_with_preview', previewfile: 'preview.jpg', fixture: 'sample.ogg' } - const resVideo = await uploadVideo(servers[1].url, servers[1].accessToken, videoAttributesArg) + it('Should upload an audio file and create an audio version only', async function () { + this.timeout(60_000) + + await updateCustomSubConfig(servers[1].url, servers[1].accessToken, { + transcoding: { + hls: { enabled: true }, + webtorrent: { enabled: true }, + resolutions: { + '0p': true, + '240p': false, + '360p': false + } + } + }) - await waitJobs(servers) + const videoAttributesArg = { name: 'audio_with_preview', previewfile: 'preview.jpg', fixture: 'sample.ogg' } + const resVideo = await uploadVideo(servers[1].url, servers[1].accessToken, videoAttributesArg, HttpStatusCode.OK_200, mode) - for (const server of servers) { - const res2 = await getVideo(server.url, resVideo.body.video.id) - const videoDetails: VideoDetails = res2.body + await waitJobs(servers) + + for (const server of servers) { + const res2 = await getVideo(server.url, resVideo.body.video.id) + const videoDetails: VideoDetails = res2.body - for (const files of [ videoDetails.files, videoDetails.streamingPlaylists[0].files ]) { - expect(files).to.have.lengthOf(2) - expect(files.find(f => f.resolution.id === 0)).to.not.be.undefined + for (const files of [ videoDetails.files, videoDetails.streamingPlaylists[0].files ]) { + expect(files).to.have.lengthOf(2) + expect(files.find(f => f.resolution.id === 0)).to.not.be.undefined + } } - } - await updateConfigForTranscoding(servers[1]) + await updateConfigForTranscoding(servers[1]) + }) + } + + describe('Legacy upload', function () { + runSuite('legacy') + }) + + describe('Resumable upload', function () { + runSuite('resumable') }) }) -- cgit v1.2.3