From f6d6e7f861189a4446f406efb775a29688764b48 Mon Sep 17 00:00:00 2001 From: kontrollanten <6680299+kontrollanten@users.noreply.github.com> Date: Mon, 10 May 2021 11:13:41 +0200 Subject: Resumable video uploads (#3933) * WIP: resumable video uploads relates to #324 * fix review comments * video upload: error handling * fix audio upload * fixes after self review * Update server/controllers/api/videos/index.ts Co-authored-by: Rigel Kent * Update server/middlewares/validators/videos/videos.ts Co-authored-by: Rigel Kent * Update server/controllers/api/videos/index.ts Co-authored-by: Rigel Kent * update after code review * refactor upload route - restore multipart upload route - move resumable to dedicated upload-resumable route - move checks to middleware - do not leak internal fs structure in response * fix yarn.lock upon rebase * factorize addVideo for reuse in both endpoints * add resumable upload API to openapi spec * add initial test and test helper for resumable upload * typings for videoAddResumable middleware * avoid including aws and google packages via node-uploadx, by only including uploadx/core * rename ex-isAudioBg to more explicit name mentioning it is a preview file for audio * add video-upload-tmp-folder-cleaner job * stronger typing of video upload middleware * reduce dependency to @uploadx/core * add audio upload test * refactor resumable uploads cleanup from job to scheduler * refactor resumable uploads scheduler to compare to last execution time * make resumable upload validator to always cleanup on failure * move legacy upload request building outside of uploadVideo test helper * filter upload-resumable middlewares down to POST, PUT, DELETE also begin to type metadata * merge add duration functions * stronger typings and documentation for uploadx behaviour, move init validator up * refactor(client/video-edit): options > uploadxOptions * refactor(client/video-edit): remove obsolete else * scheduler/remove-dangling-resum: rename tag * refactor(server/video): add UploadVideoFiles type * refactor(mw/validators): restructure eslint disable * refactor(mw/validators/videos): rename import * refactor(client/vid-upload): rename html elem id * refactor(sched/remove-dangl): move fn to method * refactor(mw/async): add method typing * refactor(mw/vali/video): double quote > single * refactor(server/upload-resum): express use > all * proper http methud enum server/middlewares/async.ts * properly type http methods * factorize common video upload validation steps * add check for maximum partially uploaded file size * fix audioBg use * fix extname(filename) in addVideo * document parameters for uploadx's resumable protocol * clear META files in scheduler * last audio refactor before cramming preview in the initial POST form data * refactor as mulitpart/form-data initial post request this allows preview/thumbnail uploads alongside the initial request, and cleans up the upload form * Add more tests for resumable uploads * Refactor remove dangling resumable uploads * Prepare changelog * Add more resumable upload tests * Remove user quota check for resumable uploads * Fix upload error handler * Update nginx template for upload-resumable * Cleanup comment * Remove unused express methods * Prefer to use got instead of raw http * Don't retry on error 500 Co-authored-by: Rigel Kent Co-authored-by: Rigel Kent Co-authored-by: Chocobozzz --- server/controllers/api/server/debug.ts | 18 + server/controllers/api/videos/index.ts | 105 ++- server/helpers/custom-validators/misc.ts | 5 +- server/helpers/custom-validators/videos.ts | 9 +- server/helpers/express-utils.ts | 8 +- server/helpers/upload.ts | 21 + server/helpers/utils.ts | 4 +- server/initializers/constants.ts | 6 +- server/initializers/installer.ts | 5 +- server/lib/moderation.ts | 5 +- .../remove-dangling-resumable-uploads-scheduler.ts | 61 ++ server/lib/video.ts | 3 +- server/middlewares/async.ts | 1 + server/middlewares/validators/videos/videos.ts | 184 +++++- server/tests/api/check-params/index.ts | 1 + server/tests/api/check-params/upload-quota.ts | 152 +++++ server/tests/api/check-params/users.ts | 105 +-- server/tests/api/check-params/videos.ts | 393 ++++++----- server/tests/api/videos/index.ts | 1 + server/tests/api/videos/multiple-servers.ts | 2 +- server/tests/api/videos/resumable-upload.ts | 187 ++++++ server/tests/api/videos/single-server.ts | 724 +++++++++++---------- server/tests/api/videos/video-transcoder.ts | 159 ++--- server/typings/express/index.d.ts | 140 ++-- 24 files changed, 1444 insertions(+), 855 deletions(-) create mode 100644 server/helpers/upload.ts create mode 100644 server/lib/schedulers/remove-dangling-resumable-uploads-scheduler.ts create mode 100644 server/tests/api/check-params/upload-quota.ts create mode 100644 server/tests/api/videos/resumable-upload.ts (limited to 'server') diff --git a/server/controllers/api/server/debug.ts b/server/controllers/api/server/debug.ts index 7787186be..ff0d9ca3c 100644 --- a/server/controllers/api/server/debug.ts +++ b/server/controllers/api/server/debug.ts @@ -1,4 +1,6 @@ import { InboxManager } from '@server/lib/activitypub/inbox-manager' +import { RemoveDanglingResumableUploadsScheduler } from '@server/lib/schedulers/remove-dangling-resumable-uploads-scheduler' +import { SendDebugCommand } from '@shared/models' import * as express from 'express' import { UserRight } from '../../../../shared/models/users' import { authenticate, ensureUserHasRight } from '../../../middlewares' @@ -11,6 +13,12 @@ debugRouter.get('/debug', getDebug ) +debugRouter.post('/debug/run-command', + authenticate, + ensureUserHasRight(UserRight.MANAGE_DEBUG), + runCommand +) + // --------------------------------------------------------------------------- export { @@ -25,3 +33,13 @@ function getDebug (req: express.Request, res: express.Response) { activityPubMessagesWaiting: InboxManager.Instance.getActivityPubMessagesWaiting() }) } + +async function runCommand (req: express.Request, res: express.Response) { + const body: SendDebugCommand = req.body + + if (body.command === 'remove-dandling-resumable-uploads') { + await RemoveDanglingResumableUploadsScheduler.Instance.execute() + } + + return res.sendStatus(204) +} diff --git a/server/controllers/api/videos/index.ts b/server/controllers/api/videos/index.ts index fbdb0f776..c32626d30 100644 --- a/server/controllers/api/videos/index.ts +++ b/server/controllers/api/videos/index.ts @@ -2,6 +2,7 @@ import * as express from 'express' import { move } from 'fs-extra' import { extname } from 'path' import toInt from 'validator/lib/toInt' +import { deleteResumableUploadMetaFile, getResumableUploadPath } from '@server/helpers/upload' import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent' import { changeVideoChannelShare } from '@server/lib/activitypub/share' import { getLocalVideoActivityPubUrl } from '@server/lib/activitypub/url' @@ -10,8 +11,9 @@ import { addOptimizeOrMergeAudioJob, buildLocalVideoFromReq, buildVideoThumbnail import { generateVideoFilename, getVideoFilePath } from '@server/lib/video-paths' import { getServerActor } from '@server/models/application/application' import { MVideo, MVideoFile, MVideoFullLight } from '@server/types/models' +import { uploadx } from '@uploadx/core' import { VideoCreate, VideosCommonQuery, VideoState, VideoUpdate } from '../../../../shared' -import { HttpStatusCode } from '../../../../shared/core-utils/miscs/http-error-codes' +import { HttpStatusCode } from '../../../../shared/core-utils/miscs' import { auditLoggerFactory, getAuditIdFromRes, VideoAuditView } from '../../../helpers/audit-logger' import { resetSequelizeInstance, retryTransactionWrapper } from '../../../helpers/database-utils' import { buildNSFWFilter, createReqFiles, getCountVideos } from '../../../helpers/express-utils' @@ -47,7 +49,9 @@ import { setDefaultPagination, setDefaultVideosSort, videoFileMetadataGetValidator, - videosAddValidator, + videosAddLegacyValidator, + videosAddResumableInitValidator, + videosAddResumableValidator, videosCustomGetValidator, videosGetValidator, videosRemoveValidator, @@ -69,6 +73,7 @@ import { watchingRouter } from './watching' const lTags = loggerTagsFactory('api', 'video') const auditLogger = auditLoggerFactory('videos') const videosRouter = express.Router() +const uploadxMiddleware = uploadx.upload({ directory: getResumableUploadPath() }) const reqVideoFileAdd = createReqFiles( [ 'videofile', 'thumbnailfile', 'previewfile' ], @@ -79,6 +84,16 @@ const reqVideoFileAdd = createReqFiles( previewfile: CONFIG.STORAGE.TMP_DIR } ) + +const reqVideoFileAddResumable = createReqFiles( + [ 'thumbnailfile', 'previewfile' ], + MIMETYPES.IMAGE.MIMETYPE_EXT, + { + thumbnailfile: getResumableUploadPath(), + previewfile: getResumableUploadPath() + } +) + const reqVideoFileUpdate = createReqFiles( [ 'thumbnailfile', 'previewfile' ], MIMETYPES.IMAGE.MIMETYPE_EXT, @@ -111,18 +126,39 @@ videosRouter.get('/', commonVideosFiltersValidator, asyncMiddleware(listVideos) ) + +videosRouter.post('/upload', + authenticate, + reqVideoFileAdd, + asyncMiddleware(videosAddLegacyValidator), + asyncRetryTransactionMiddleware(addVideoLegacy) +) + +videosRouter.post('/upload-resumable', + authenticate, + reqVideoFileAddResumable, + asyncMiddleware(videosAddResumableInitValidator), + uploadxMiddleware +) + +videosRouter.delete('/upload-resumable', + authenticate, + uploadxMiddleware +) + +videosRouter.put('/upload-resumable', + authenticate, + uploadxMiddleware, // uploadx doesn't use call next() before the file upload completes + asyncMiddleware(videosAddResumableValidator), + asyncMiddleware(addVideoResumable) +) + videosRouter.put('/:id', authenticate, reqVideoFileUpdate, asyncMiddleware(videosUpdateValidator), asyncRetryTransactionMiddleware(updateVideo) ) -videosRouter.post('/upload', - authenticate, - reqVideoFileAdd, - asyncMiddleware(videosAddValidator), - asyncRetryTransactionMiddleware(addVideo) -) videosRouter.get('/:id/description', asyncMiddleware(videosGetValidator), @@ -157,23 +193,23 @@ export { // --------------------------------------------------------------------------- -function listVideoCategories (req: express.Request, res: express.Response) { +function listVideoCategories (_req: express.Request, res: express.Response) { res.json(VIDEO_CATEGORIES) } -function listVideoLicences (req: express.Request, res: express.Response) { +function listVideoLicences (_req: express.Request, res: express.Response) { res.json(VIDEO_LICENCES) } -function listVideoLanguages (req: express.Request, res: express.Response) { +function listVideoLanguages (_req: express.Request, res: express.Response) { res.json(VIDEO_LANGUAGES) } -function listVideoPrivacies (req: express.Request, res: express.Response) { +function listVideoPrivacies (_req: express.Request, res: express.Response) { res.json(VIDEO_PRIVACIES) } -async function addVideo (req: express.Request, res: express.Response) { +async function addVideoLegacy (req: express.Request, res: express.Response) { // Uploading the video could be long // Set timeout to 10 minutes, as Express's default is 2 minutes req.setTimeout(1000 * 60 * 10, () => { @@ -183,13 +219,42 @@ async function addVideo (req: express.Request, res: express.Response) { const videoPhysicalFile = req.files['videofile'][0] const videoInfo: VideoCreate = req.body + const files = req.files + + return addVideo({ res, videoPhysicalFile, videoInfo, files }) +} + +async function addVideoResumable (_req: express.Request, res: express.Response) { + const videoPhysicalFile = res.locals.videoFileResumable + const videoInfo = videoPhysicalFile.metadata + const files = { previewfile: videoInfo.previewfile } + + // Don't need the meta file anymore + await deleteResumableUploadMetaFile(videoPhysicalFile.path) + + return addVideo({ res, videoPhysicalFile, videoInfo, files }) +} - const videoData = buildLocalVideoFromReq(videoInfo, res.locals.videoChannel.id) - videoData.state = CONFIG.TRANSCODING.ENABLED ? VideoState.TO_TRANSCODE : VideoState.PUBLISHED - videoData.duration = videoPhysicalFile['duration'] // duration was added by a previous middleware +async function addVideo (options: { + res: express.Response + videoPhysicalFile: express.VideoUploadFile + videoInfo: VideoCreate + files: express.UploadFiles +}) { + const { res, videoPhysicalFile, videoInfo, files } = options + const videoChannel = res.locals.videoChannel + const user = res.locals.oauth.token.User + + const videoData = buildLocalVideoFromReq(videoInfo, videoChannel.id) + + videoData.state = CONFIG.TRANSCODING.ENABLED + ? VideoState.TO_TRANSCODE + : VideoState.PUBLISHED + + videoData.duration = videoPhysicalFile.duration // duration was added by a previous middleware const video = new VideoModel(videoData) as MVideoFullLight - video.VideoChannel = res.locals.videoChannel + video.VideoChannel = videoChannel video.url = getLocalVideoActivityPubUrl(video) // We use the UUID, so set the URL after building the object const videoFile = new VideoFileModel({ @@ -217,7 +282,7 @@ async function addVideo (req: express.Request, res: express.Response) { const [ thumbnailModel, previewModel ] = await buildVideoThumbnailsFromReq({ video, - files: req.files, + files, fallback: type => generateVideoMiniature({ video, videoFile, type }) }) @@ -253,7 +318,7 @@ async function addVideo (req: express.Request, res: express.Response) { await autoBlacklistVideoIfNeeded({ video, - user: res.locals.oauth.token.User, + user, isRemote: false, isNew: true, transaction: t @@ -282,7 +347,7 @@ async function addVideo (req: express.Request, res: express.Response) { .catch(err => logger.error('Cannot federate or notify video creation %s', video.url, { err, ...lTags(video.uuid) })) if (video.state === VideoState.TO_TRANSCODE) { - await addOptimizeOrMergeAudioJob(videoCreated, videoFile, res.locals.oauth.token.User) + await addOptimizeOrMergeAudioJob(videoCreated, videoFile, user) } Hooks.runAction('action:api.video.uploaded', { video: videoCreated }) diff --git a/server/helpers/custom-validators/misc.ts b/server/helpers/custom-validators/misc.ts index effdd98cb..fd3b45804 100644 --- a/server/helpers/custom-validators/misc.ts +++ b/server/helpers/custom-validators/misc.ts @@ -1,6 +1,7 @@ import 'multer' -import validator from 'validator' +import { UploadFilesForCheck } from 'express' import { sep } from 'path' +import validator from 'validator' function exists (value: any) { return value !== undefined && value !== null @@ -108,7 +109,7 @@ function isFileFieldValid ( } function isFileMimeTypeValid ( - files: { [ fieldname: string ]: Express.Multer.File[] } | Express.Multer.File[], + files: UploadFilesForCheck, mimeTypeRegex: string, field: string, optional = false diff --git a/server/helpers/custom-validators/videos.ts b/server/helpers/custom-validators/videos.ts index 87966798f..b33e088eb 100644 --- a/server/helpers/custom-validators/videos.ts +++ b/server/helpers/custom-validators/videos.ts @@ -1,4 +1,6 @@ +import { UploadFilesForCheck } from 'express' import { values } from 'lodash' +import * as magnetUtil from 'magnet-uri' import validator from 'validator' import { VideoFilter, VideoPrivacy, VideoRateType } from '../../../shared' import { @@ -6,13 +8,12 @@ import { MIMETYPES, VIDEO_CATEGORIES, VIDEO_LICENCES, + VIDEO_LIVE, VIDEO_PRIVACIES, VIDEO_RATE_TYPES, - VIDEO_STATES, - VIDEO_LIVE + VIDEO_STATES } from '../../initializers/constants' import { exists, isArray, isDateValid, isFileMimeTypeValid, isFileValid } from './misc' -import * as magnetUtil from 'magnet-uri' const VIDEOS_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.VIDEOS @@ -81,7 +82,7 @@ function isVideoFileExtnameValid (value: string) { return exists(value) && (value === VIDEO_LIVE.EXTENSION || MIMETYPES.VIDEO.EXT_MIMETYPE[value] !== undefined) } -function isVideoFileMimeTypeValid (files: { [ fieldname: string ]: Express.Multer.File[] } | Express.Multer.File[]) { +function isVideoFileMimeTypeValid (files: UploadFilesForCheck) { return isFileMimeTypeValid(files, MIMETYPES.VIDEO.MIMETYPES_REGEX, 'videofile') } diff --git a/server/helpers/express-utils.ts b/server/helpers/express-utils.ts index c0d3f8f32..ede22a3cc 100644 --- a/server/helpers/express-utils.ts +++ b/server/helpers/express-utils.ts @@ -2,7 +2,7 @@ import * as express from 'express' import * as multer from 'multer' import { REMOTE_SCHEME } from '../initializers/constants' import { logger } from './logger' -import { deleteFileAsync, generateRandomString } from './utils' +import { deleteFileAndCatch, generateRandomString } from './utils' import { extname } from 'path' import { isArray } from './custom-validators/misc' import { CONFIG } from '../initializers/config' @@ -36,15 +36,15 @@ function cleanUpReqFiles (req: { files: { [fieldname: string]: Express.Multer.Fi if (!files) return if (isArray(files)) { - (files as Express.Multer.File[]).forEach(f => deleteFileAsync(f.path)) + (files as Express.Multer.File[]).forEach(f => deleteFileAndCatch(f.path)) return } for (const key of Object.keys(files)) { const file = files[key] - if (isArray(file)) file.forEach(f => deleteFileAsync(f.path)) - else deleteFileAsync(file.path) + if (isArray(file)) file.forEach(f => deleteFileAndCatch(f.path)) + else deleteFileAndCatch(file.path) } } diff --git a/server/helpers/upload.ts b/server/helpers/upload.ts new file mode 100644 index 000000000..030a6b7d5 --- /dev/null +++ b/server/helpers/upload.ts @@ -0,0 +1,21 @@ +import { METAFILE_EXTNAME } from '@uploadx/core' +import { remove } from 'fs-extra' +import { join } from 'path' +import { RESUMABLE_UPLOAD_DIRECTORY } from '../initializers/constants' + +function getResumableUploadPath (filename?: string) { + if (filename) return join(RESUMABLE_UPLOAD_DIRECTORY, filename) + + return RESUMABLE_UPLOAD_DIRECTORY +} + +function deleteResumableUploadMetaFile (filepath: string) { + return remove(filepath + METAFILE_EXTNAME) +} + +// --------------------------------------------------------------------------- + +export { + getResumableUploadPath, + deleteResumableUploadMetaFile +} diff --git a/server/helpers/utils.ts b/server/helpers/utils.ts index 0545e8996..6c95a43b6 100644 --- a/server/helpers/utils.ts +++ b/server/helpers/utils.ts @@ -6,7 +6,7 @@ import { CONFIG } from '../initializers/config' import { execPromise, execPromise2, randomBytesPromise, sha256 } from './core-utils' import { logger } from './logger' -function deleteFileAsync (path: string) { +function deleteFileAndCatch (path: string) { remove(path) .catch(err => logger.error('Cannot delete the file %s asynchronously.', path, { err })) } @@ -83,7 +83,7 @@ function getUUIDFromFilename (filename: string) { // --------------------------------------------------------------------------- export { - deleteFileAsync, + deleteFileAndCatch, generateRandomString, getFormattedObjects, getSecureTorrentName, diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index f807a1e58..6f388420e 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts @@ -208,7 +208,8 @@ const SCHEDULER_INTERVALS_MS = { autoFollowIndexInstances: 60000 * 60 * 24, // 1 day removeOldViews: 60000 * 60 * 24, // 1 day removeOldHistory: 60000 * 60 * 24, // 1 day - updateInboxStats: 1000 * 60// 1 minute + updateInboxStats: 1000 * 60, // 1 minute + removeDanglingResumableUploads: 60000 * 60 * 16 // 16 hours } // --------------------------------------------------------------------------- @@ -285,6 +286,7 @@ const CONSTRAINTS_FIELDS = { LIKES: { min: 0 }, DISLIKES: { min: 0 }, FILE_SIZE: { min: -1 }, + PARTIAL_UPLOAD_SIZE: { max: 50 * 1024 * 1024 * 1024 }, // 50GB URL: { min: 3, max: 2000 } // Length }, VIDEO_PLAYLISTS: { @@ -645,6 +647,7 @@ const LRU_CACHE = { } } +const RESUMABLE_UPLOAD_DIRECTORY = join(CONFIG.STORAGE.TMP_DIR, 'resumable-uploads') const HLS_STREAMING_PLAYLIST_DIRECTORY = join(CONFIG.STORAGE.STREAMING_PLAYLISTS_DIR, 'hls') const HLS_REDUNDANCY_DIRECTORY = join(CONFIG.STORAGE.REDUNDANCY_DIR, 'hls') @@ -819,6 +822,7 @@ export { PEERTUBE_VERSION, LAZY_STATIC_PATHS, SEARCH_INDEX, + RESUMABLE_UPLOAD_DIRECTORY, HLS_REDUNDANCY_DIRECTORY, P2P_MEDIA_LOADER_PEER_VERSION, ACTOR_IMAGES_SIZE, diff --git a/server/initializers/installer.ts b/server/initializers/installer.ts index cb58454cb..8dcff64e2 100644 --- a/server/initializers/installer.ts +++ b/server/initializers/installer.ts @@ -6,7 +6,7 @@ import { UserModel } from '../models/account/user' import { ApplicationModel } from '../models/application/application' import { OAuthClientModel } from '../models/oauth/oauth-client' import { applicationExist, clientsExist, usersExist } from './checker-after-init' -import { FILES_CACHE, HLS_STREAMING_PLAYLIST_DIRECTORY, LAST_MIGRATION_VERSION } from './constants' +import { FILES_CACHE, HLS_STREAMING_PLAYLIST_DIRECTORY, LAST_MIGRATION_VERSION, RESUMABLE_UPLOAD_DIRECTORY } from './constants' import { sequelizeTypescript } from './database' import { ensureDir, remove } from 'fs-extra' import { CONFIG } from './config' @@ -79,6 +79,9 @@ function createDirectoriesIfNotExist () { // Playlist directories tasks.push(ensureDir(HLS_STREAMING_PLAYLIST_DIRECTORY)) + // Resumable upload directory + tasks.push(ensureDir(RESUMABLE_UPLOAD_DIRECTORY)) + return Promise.all(tasks) } diff --git a/server/lib/moderation.ts b/server/lib/moderation.ts index 5180b3299..925d64902 100644 --- a/server/lib/moderation.ts +++ b/server/lib/moderation.ts @@ -1,6 +1,8 @@ +import { VideoUploadFile } from 'express' import { PathLike } from 'fs-extra' import { Transaction } from 'sequelize/types' import { AbuseAuditView, auditLoggerFactory } from '@server/helpers/audit-logger' +import { afterCommitIfTransaction } from '@server/helpers/database-utils' import { logger } from '@server/helpers/logger' import { AbuseModel } from '@server/models/abuse/abuse' import { VideoAbuseModel } from '@server/models/abuse/video-abuse' @@ -28,7 +30,6 @@ import { VideoModel } from '../models/video/video' import { VideoCommentModel } from '../models/video/video-comment' import { sendAbuse } from './activitypub/send/send-flag' import { Notifier } from './notifier' -import { afterCommitIfTransaction } from '@server/helpers/database-utils' export type AcceptResult = { accepted: boolean @@ -38,7 +39,7 @@ export type AcceptResult = { // Can be filtered by plugins function isLocalVideoAccepted (object: { videoBody: VideoCreate - videoFile: Express.Multer.File & { duration?: number } + videoFile: VideoUploadFile user: UserModel }): AcceptResult { return { accepted: true } diff --git a/server/lib/schedulers/remove-dangling-resumable-uploads-scheduler.ts b/server/lib/schedulers/remove-dangling-resumable-uploads-scheduler.ts new file mode 100644 index 000000000..1acea7998 --- /dev/null +++ b/server/lib/schedulers/remove-dangling-resumable-uploads-scheduler.ts @@ -0,0 +1,61 @@ +import * as bluebird from 'bluebird' +import { readdir, remove, stat } from 'fs-extra' +import { logger, loggerTagsFactory } from '@server/helpers/logger' +import { getResumableUploadPath } from '@server/helpers/upload' +import { SCHEDULER_INTERVALS_MS } from '@server/initializers/constants' +import { METAFILE_EXTNAME } from '@uploadx/core' +import { AbstractScheduler } from './abstract-scheduler' + +const lTags = loggerTagsFactory('scheduler', 'resumable-upload', 'cleaner') + +export class RemoveDanglingResumableUploadsScheduler extends AbstractScheduler { + + private static instance: AbstractScheduler + private lastExecutionTimeMs: number + + protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.removeDanglingResumableUploads + + private constructor () { + super() + + this.lastExecutionTimeMs = new Date().getTime() + } + + protected async internalExecute () { + const path = getResumableUploadPath() + const files = await readdir(path) + + const metafiles = files.filter(f => f.endsWith(METAFILE_EXTNAME)) + + if (metafiles.length === 0) return + + logger.debug('Reading resumable video upload folder %s with %d files', path, metafiles.length, lTags()) + + try { + await bluebird.map(metafiles, metafile => { + return this.deleteIfOlderThan(metafile, this.lastExecutionTimeMs) + }, { concurrency: 5 }) + } catch (error) { + logger.error('Failed to handle file during resumable video upload folder cleanup', { error, ...lTags() }) + } finally { + this.lastExecutionTimeMs = new Date().getTime() + } + } + + private async deleteIfOlderThan (metafile: string, olderThan: number) { + const metafilePath = getResumableUploadPath(metafile) + const statResult = await stat(metafilePath) + + // Delete uploads that started since a long time + if (statResult.ctimeMs < olderThan) { + await remove(metafilePath) + + const datafile = metafilePath.replace(new RegExp(`${METAFILE_EXTNAME}$`), '') + await remove(datafile) + } + } + + static get Instance () { + return this.instance || (this.instance = new this()) + } +} diff --git a/server/lib/video.ts b/server/lib/video.ts index 9469b8178..21e4b7ff2 100644 --- a/server/lib/video.ts +++ b/server/lib/video.ts @@ -1,3 +1,4 @@ +import { UploadFiles } from 'express' import { Transaction } from 'sequelize/types' import { DEFAULT_AUDIO_RESOLUTION, JOB_PRIORITY } from '@server/initializers/constants' import { sequelizeTypescript } from '@server/initializers/database' @@ -32,7 +33,7 @@ function buildLocalVideoFromReq (videoInfo: VideoCreate, channelId: number): Fil async function buildVideoThumbnailsFromReq (options: { video: MVideoThumbnail - files: { [fieldname: string]: Express.Multer.File[] } | Express.Multer.File[] + files: UploadFiles fallback: (type: ThumbnailType) => Promise automaticallyGenerated?: boolean }) { diff --git a/server/middlewares/async.ts b/server/middlewares/async.ts index 3d6e38809..0faa4fb8c 100644 --- a/server/middlewares/async.ts +++ b/server/middlewares/async.ts @@ -3,6 +3,7 @@ import { NextFunction, Request, RequestHandler, Response } from 'express' import { ValidationChain } from 'express-validator' import { ExpressPromiseHandler } from '@server/types/express' import { retryTransactionWrapper } from '../helpers/database-utils' +import { HttpMethod, HttpStatusCode } from '@shared/core-utils' // Syntactic sugar to avoid try/catch in express controllers // Thanks: https://medium.com/@Abazhenov/using-async-await-in-express-with-node-8-b8af872c0016 diff --git a/server/middlewares/validators/videos/videos.ts b/server/middlewares/validators/videos/videos.ts index bb617d77c..d26bcd4a6 100644 --- a/server/middlewares/validators/videos/videos.ts +++ b/server/middlewares/validators/videos/videos.ts @@ -1,9 +1,10 @@ import * as express from 'express' -import { body, param, query, ValidationChain } from 'express-validator' +import { body, header, param, query, ValidationChain } from 'express-validator' +import { getResumableUploadPath } from '@server/helpers/upload' import { isAbleToUploadVideo } from '@server/lib/user' import { getServerActor } from '@server/models/application/application' import { ExpressPromiseHandler } from '@server/types/express' -import { MVideoWithRights } from '@server/types/models' +import { MUserAccountId, MVideoWithRights } from '@server/types/models' import { ServerErrorCode, UserRight, VideoChangeOwnershipStatus, VideoPrivacy } from '../../../../shared' import { HttpStatusCode } from '../../../../shared/core-utils/miscs/http-error-codes' import { VideoChangeOwnershipAccept } from '../../../../shared/models/videos/video-change-ownership-accept.model' @@ -47,6 +48,7 @@ import { doesVideoExist, doesVideoFileOfVideoExist } from '../../../helpers/middlewares' +import { deleteFileAndCatch } from '../../../helpers/utils' import { getVideoWithAttributes } from '../../../helpers/video' import { CONFIG } from '../../../initializers/config' import { CONSTRAINTS_FIELDS, OVERVIEWS } from '../../../initializers/constants' @@ -57,7 +59,7 @@ import { VideoModel } from '../../../models/video/video' import { authenticatePromiseIfNeeded } from '../../auth' import { areValidationErrors } from '../utils' -const videosAddValidator = getCommonVideoEditAttributes().concat([ +const videosAddLegacyValidator = getCommonVideoEditAttributes().concat([ body('videofile') .custom((value, { req }) => isFileFieldValid(req.files, 'videofile')) .withMessage('Should have a file'), @@ -73,54 +75,117 @@ const videosAddValidator = getCommonVideoEditAttributes().concat([ logger.debug('Checking videosAdd parameters', { parameters: req.body, files: req.files }) if (areValidationErrors(req, res)) return cleanUpReqFiles(req) - if (areErrorsInScheduleUpdate(req, res)) return cleanUpReqFiles(req) - const videoFile: Express.Multer.File & { duration?: number } = req.files['videofile'][0] + const videoFile: express.VideoUploadFile = req.files['videofile'][0] const user = res.locals.oauth.token.User - if (!await doesVideoChannelOfAccountExist(req.body.channelId, user, res)) return cleanUpReqFiles(req) - - if (!isVideoFileMimeTypeValid(req.files)) { - res.status(HttpStatusCode.UNSUPPORTED_MEDIA_TYPE_415) - .json({ - error: 'This file is not supported. Please, make sure it is of the following type: ' + - CONSTRAINTS_FIELDS.VIDEOS.EXTNAME.join(', ') - }) - + if (!await commonVideoChecksPass({ req, res, user, videoFileSize: videoFile.size, files: req.files })) { return cleanUpReqFiles(req) } - if (!isVideoFileSizeValid(videoFile.size.toString())) { - res.status(HttpStatusCode.PAYLOAD_TOO_LARGE_413) - .json({ - error: 'This file is too large.' - }) + try { + if (!videoFile.duration) await addDurationToVideo(videoFile) + } catch (err) { + logger.error('Invalid input file in videosAddLegacyValidator.', { err }) + res.status(HttpStatusCode.UNPROCESSABLE_ENTITY_422) + .json({ error: 'Video file unreadable.' }) return cleanUpReqFiles(req) } - if (await isAbleToUploadVideo(user.id, videoFile.size) === false) { - res.status(HttpStatusCode.PAYLOAD_TOO_LARGE_413) - .json({ error: 'The user video quota is exceeded with this video.' }) + if (!await isVideoAccepted(req, res, videoFile)) return cleanUpReqFiles(req) - return cleanUpReqFiles(req) - } + return next() + } +]) + +/** + * Gets called after the last PUT request + */ +const videosAddResumableValidator = [ + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + const user = res.locals.oauth.token.User + + const body: express.CustomUploadXFile = req.body + const file = { ...body, duration: undefined, path: getResumableUploadPath(body.id), filename: body.metadata.filename } + + const cleanup = () => deleteFileAndCatch(file.path) - let duration: number + if (!await doesVideoChannelOfAccountExist(file.metadata.channelId, user, res)) return cleanup() try { - duration = await getDurationFromVideoFile(videoFile.path) + if (!file.duration) await addDurationToVideo(file) } catch (err) { - logger.error('Invalid input file in videosAddValidator.', { err }) + logger.error('Invalid input file in videosAddResumableValidator.', { err }) res.status(HttpStatusCode.UNPROCESSABLE_ENTITY_422) .json({ error: 'Video file unreadable.' }) - return cleanUpReqFiles(req) + return cleanup() } - videoFile.duration = duration + if (!await isVideoAccepted(req, res, file)) return cleanup() - if (!await isVideoAccepted(req, res, videoFile)) return cleanUpReqFiles(req) + res.locals.videoFileResumable = file + + return next() + } +] + +/** + * File is created in POST initialisation, and its body is saved as a 'metadata' field is saved by uploadx for later use. + * see https://github.com/kukhariev/node-uploadx/blob/dc9fb4a8ac5a6f481902588e93062f591ec6ef03/packages/core/src/handlers/uploadx.ts + * + * Uploadx doesn't use next() until the upload completes, so this middleware has to be placed before uploadx + * see https://github.com/kukhariev/node-uploadx/blob/dc9fb4a8ac5a6f481902588e93062f591ec6ef03/packages/core/src/handlers/base-handler.ts + * + */ +const videosAddResumableInitValidator = getCommonVideoEditAttributes().concat([ + body('filename') + .isString() + .exists() + .withMessage('Should have a valid filename'), + body('name') + .trim() + .custom(isVideoNameValid) + .withMessage('Should have a valid name'), + body('channelId') + .customSanitizer(toIntOrNull) + .custom(isIdValid).withMessage('Should have correct video channel id'), + + header('x-upload-content-length') + .isNumeric() + .exists() + .withMessage('Should specify the file length'), + header('x-upload-content-type') + .isString() + .exists() + .withMessage('Should specify the file mimetype'), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + const videoFileMetadata = { + mimetype: req.headers['x-upload-content-type'] as string, + size: +req.headers['x-upload-content-length'], + originalname: req.body.name + } + + const user = res.locals.oauth.token.User + const cleanup = () => cleanUpReqFiles(req) + + logger.debug('Checking videosAddResumableInitValidator parameters and headers', { + parameters: req.body, + headers: req.headers, + files: req.files + }) + + if (areValidationErrors(req, res)) return cleanup() + + const files = { videofile: [ videoFileMetadata ] } + if (!await commonVideoChecksPass({ req, res, user, videoFileSize: videoFileMetadata.size, files })) return cleanup() + + // multer required unsetting the Content-Type, now we can set it for node-uploadx + req.headers['content-type'] = 'application/json; charset=utf-8' + // place previewfile in metadata so that uploadx saves it in .META + if (req.files['previewfile']) req.body.previewfile = req.files['previewfile'] return next() } @@ -478,7 +543,10 @@ const commonVideosFiltersValidator = [ // --------------------------------------------------------------------------- export { - videosAddValidator, + videosAddLegacyValidator, + videosAddResumableValidator, + videosAddResumableInitValidator, + videosUpdateValidator, videosGetValidator, videoFileMetadataGetValidator, @@ -515,7 +583,51 @@ function areErrorsInScheduleUpdate (req: express.Request, res: express.Response) return false } -async function isVideoAccepted (req: express.Request, res: express.Response, videoFile: Express.Multer.File & { duration?: number }) { +async function commonVideoChecksPass (parameters: { + req: express.Request + res: express.Response + user: MUserAccountId + videoFileSize: number + files: express.UploadFilesForCheck +}): Promise { + const { req, res, user, videoFileSize, files } = parameters + + if (areErrorsInScheduleUpdate(req, res)) return false + + if (!await doesVideoChannelOfAccountExist(req.body.channelId, user, res)) return false + + if (!isVideoFileMimeTypeValid(files)) { + res.status(HttpStatusCode.UNSUPPORTED_MEDIA_TYPE_415) + .json({ + error: 'This file is not supported. Please, make sure it is of the following type: ' + + CONSTRAINTS_FIELDS.VIDEOS.EXTNAME.join(', ') + }) + + return false + } + + if (!isVideoFileSizeValid(videoFileSize.toString())) { + res.status(HttpStatusCode.PAYLOAD_TOO_LARGE_413) + .json({ error: 'This file is too large. It exceeds the maximum file size authorized.' }) + + return false + } + + if (await isAbleToUploadVideo(user.id, videoFileSize) === false) { + res.status(HttpStatusCode.PAYLOAD_TOO_LARGE_413) + .json({ error: 'The user video quota is exceeded with this video.' }) + + return false + } + + return true +} + +export async function isVideoAccepted ( + req: express.Request, + res: express.Response, + videoFile: express.VideoUploadFile +) { // Check we accept this video const acceptParameters = { videoBody: req.body, @@ -538,3 +650,11 @@ async function isVideoAccepted (req: express.Request, res: express.Response, vid return true } + +async function addDurationToVideo (videoFile: { path: string, duration?: number }) { + const duration: number = await getDurationFromVideoFile(videoFile.path) + + if (isNaN(duration)) throw new Error(`Couldn't get video duration`) + + videoFile.duration = duration +} diff --git a/server/tests/api/check-params/index.ts b/server/tests/api/check-params/index.ts index d0b0b9c21..143515838 100644 --- a/server/tests/api/check-params/index.ts +++ b/server/tests/api/check-params/index.ts @@ -13,6 +13,7 @@ import './plugins' import './redundancy' import './search' import './services' +import './upload-quota' import './user-notifications' import './user-subscriptions' import './users' diff --git a/server/tests/api/check-params/upload-quota.ts b/server/tests/api/check-params/upload-quota.ts new file mode 100644 index 000000000..d0fbec415 --- /dev/null +++ b/server/tests/api/check-params/upload-quota.ts @@ -0,0 +1,152 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import 'mocha' +import { expect } from 'chai' +import { HttpStatusCode, randomInt } from '@shared/core-utils' +import { getGoodVideoUrl, getMagnetURI, getMyVideoImports, importVideo } from '@shared/extra-utils/videos/video-imports' +import { MyUser, VideoImport, VideoImportState, VideoPrivacy } from '@shared/models' +import { + cleanupTests, + flushAndRunServer, + getMyUserInformation, + immutableAssign, + registerUser, + ServerInfo, + setAccessTokensToServers, + setDefaultVideoChannel, + updateUser, + uploadVideo, + userLogin, + waitJobs +} from '../../../../shared/extra-utils' + +describe('Test upload quota', function () { + let server: ServerInfo + let rootId: number + + // --------------------------------------------------------------- + + before(async function () { + this.timeout(30000) + + server = await flushAndRunServer(1) + await setAccessTokensToServers([ server ]) + await setDefaultVideoChannel([ server ]) + + const res = await getMyUserInformation(server.url, server.accessToken) + rootId = (res.body as MyUser).id + + await updateUser({ + url: server.url, + userId: rootId, + accessToken: server.accessToken, + videoQuota: 42 + }) + }) + + describe('When having a video quota', function () { + + it('Should fail with a registered user having too many videos with legacy upload', async function () { + this.timeout(30000) + + const user = { username: 'registered' + randomInt(1, 1500), password: 'password' } + await registerUser(server.url, user.username, user.password) + const userAccessToken = await userLogin(server, user) + + const videoAttributes = { fixture: 'video_short2.webm' } + for (let i = 0; i < 5; i++) { + await uploadVideo(server.url, userAccessToken, videoAttributes) + } + + await uploadVideo(server.url, userAccessToken, videoAttributes, HttpStatusCode.PAYLOAD_TOO_LARGE_413, 'legacy') + }) + + it('Should fail with a registered user having too many videos with resumable upload', async function () { + this.timeout(30000) + + const user = { username: 'registered' + randomInt(1, 1500), password: 'password' } + await registerUser(server.url, user.username, user.password) + const userAccessToken = await userLogin(server, user) + + const videoAttributes = { fixture: 'video_short2.webm' } + for (let i = 0; i < 5; i++) { + await uploadVideo(server.url, userAccessToken, videoAttributes) + } + + await uploadVideo(server.url, userAccessToken, videoAttributes, HttpStatusCode.PAYLOAD_TOO_LARGE_413, 'resumable') + }) + + it('Should fail to import with HTTP/Torrent/magnet', async function () { + this.timeout(120000) + + const baseAttributes = { + channelId: server.videoChannel.id, + privacy: VideoPrivacy.PUBLIC + } + await importVideo(server.url, server.accessToken, immutableAssign(baseAttributes, { targetUrl: getGoodVideoUrl() })) + await importVideo(server.url, server.accessToken, immutableAssign(baseAttributes, { magnetUri: getMagnetURI() })) + await importVideo(server.url, server.accessToken, immutableAssign(baseAttributes, { torrentfile: 'video-720p.torrent' as any })) + + await waitJobs([ server ]) + + const res = await getMyVideoImports(server.url, server.accessToken) + + expect(res.body.total).to.equal(3) + const videoImports: VideoImport[] = res.body.data + expect(videoImports).to.have.lengthOf(3) + + for (const videoImport of videoImports) { + expect(videoImport.state.id).to.equal(VideoImportState.FAILED) + expect(videoImport.error).not.to.be.undefined + expect(videoImport.error).to.contain('user video quota is exceeded') + } + }) + }) + + describe('When having a daily video quota', function () { + + it('Should fail with a user having too many videos daily', async function () { + await updateUser({ + url: server.url, + userId: rootId, + accessToken: server.accessToken, + videoQuotaDaily: 42 + }) + + await uploadVideo(server.url, server.accessToken, {}, HttpStatusCode.PAYLOAD_TOO_LARGE_413, 'legacy') + await uploadVideo(server.url, server.accessToken, {}, HttpStatusCode.PAYLOAD_TOO_LARGE_413, 'resumable') + }) + }) + + describe('When having an absolute and daily video quota', function () { + it('Should fail if exceeding total quota', async function () { + await updateUser({ + url: server.url, + userId: rootId, + accessToken: server.accessToken, + videoQuota: 42, + videoQuotaDaily: 1024 * 1024 * 1024 + }) + + await uploadVideo(server.url, server.accessToken, {}, HttpStatusCode.PAYLOAD_TOO_LARGE_413, 'legacy') + await uploadVideo(server.url, server.accessToken, {}, HttpStatusCode.PAYLOAD_TOO_LARGE_413, 'resumable') + }) + + it('Should fail if exceeding daily quota', async function () { + await updateUser({ + url: server.url, + userId: rootId, + accessToken: server.accessToken, + videoQuota: 1024 * 1024 * 1024, + videoQuotaDaily: 42 + }) + + await uploadVideo(server.url, server.accessToken, {}, HttpStatusCode.PAYLOAD_TOO_LARGE_413, 'legacy') + await uploadVideo(server.url, server.accessToken, {}, HttpStatusCode.PAYLOAD_TOO_LARGE_413, 'resumable') + }) + }) + + after(async function () { + await cleanupTests([ server ]) + }) +}) diff --git a/server/tests/api/check-params/users.ts b/server/tests/api/check-params/users.ts index 2b03fde2d..dcff0d52b 100644 --- a/server/tests/api/check-params/users.ts +++ b/server/tests/api/check-params/users.ts @@ -1,10 +1,10 @@ /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ import 'mocha' -import { expect } from 'chai' import { omit } from 'lodash' import { join } from 'path' -import { User, UserRole, VideoImport, VideoImportState } from '../../../../shared' +import { User, UserRole } from '../../../../shared' +import { HttpStatusCode } from '../../../../shared/core-utils/miscs/http-error-codes' import { addVideoChannel, blockUser, @@ -29,7 +29,6 @@ import { ServerInfo, setAccessTokensToServers, unblockUser, - updateUser, uploadVideo, userLogin } from '../../../../shared/extra-utils' @@ -39,11 +38,7 @@ import { checkBadSortPagination, checkBadStartPagination } from '../../../../shared/extra-utils/requests/check-api-params' -import { waitJobs } from '../../../../shared/extra-utils/server/jobs' -import { getGoodVideoUrl, getMagnetURI, getMyVideoImports, importVideo } from '../../../../shared/extra-utils/videos/video-imports' import { UserAdminFlag } from '../../../../shared/models/users/user-flag.model' -import { VideoPrivacy } from '../../../../shared/models/videos' -import { HttpStatusCode } from '../../../../shared/core-utils/miscs/http-error-codes' describe('Test users API validators', function () { const path = '/api/v1/users/' @@ -1093,102 +1088,6 @@ describe('Test users API validators', function () { }) }) - describe('When having a video quota', function () { - it('Should fail with a user having too many videos', async function () { - await updateUser({ - url: server.url, - userId: rootId, - accessToken: server.accessToken, - videoQuota: 42 - }) - - await uploadVideo(server.url, server.accessToken, {}, HttpStatusCode.PAYLOAD_TOO_LARGE_413) - }) - - it('Should fail with a registered user having too many videos', async function () { - this.timeout(30000) - - const user = { - username: 'user3', - password: 'my super password' - } - userAccessToken = await userLogin(server, user) - - const videoAttributes = { fixture: 'video_short2.webm' } - await uploadVideo(server.url, userAccessToken, videoAttributes) - await uploadVideo(server.url, userAccessToken, videoAttributes) - await uploadVideo(server.url, userAccessToken, videoAttributes) - await uploadVideo(server.url, userAccessToken, videoAttributes) - await uploadVideo(server.url, userAccessToken, videoAttributes) - await uploadVideo(server.url, userAccessToken, videoAttributes, HttpStatusCode.PAYLOAD_TOO_LARGE_413) - }) - - it('Should fail to import with HTTP/Torrent/magnet', async function () { - this.timeout(120000) - - const baseAttributes = { - channelId: 1, - privacy: VideoPrivacy.PUBLIC - } - await importVideo(server.url, server.accessToken, immutableAssign(baseAttributes, { targetUrl: getGoodVideoUrl() })) - await importVideo(server.url, server.accessToken, immutableAssign(baseAttributes, { magnetUri: getMagnetURI() })) - await importVideo(server.url, server.accessToken, immutableAssign(baseAttributes, { torrentfile: 'video-720p.torrent' as any })) - - await waitJobs([ server ]) - - const res = await getMyVideoImports(server.url, server.accessToken) - - expect(res.body.total).to.equal(3) - const videoImports: VideoImport[] = res.body.data - expect(videoImports).to.have.lengthOf(3) - - for (const videoImport of videoImports) { - expect(videoImport.state.id).to.equal(VideoImportState.FAILED) - expect(videoImport.error).not.to.be.undefined - expect(videoImport.error).to.contain('user video quota is exceeded') - } - }) - }) - - describe('When having a daily video quota', function () { - it('Should fail with a user having too many videos daily', async function () { - await updateUser({ - url: server.url, - userId: rootId, - accessToken: server.accessToken, - videoQuotaDaily: 42 - }) - - await uploadVideo(server.url, server.accessToken, {}, HttpStatusCode.PAYLOAD_TOO_LARGE_413) - }) - }) - - describe('When having an absolute and daily video quota', function () { - it('Should fail if exceeding total quota', async function () { - await updateUser({ - url: server.url, - userId: rootId, - accessToken: server.accessToken, - videoQuota: 42, - videoQuotaDaily: 1024 * 1024 * 1024 - }) - - await uploadVideo(server.url, server.accessToken, {}, HttpStatusCode.PAYLOAD_TOO_LARGE_413) - }) - - it('Should fail if exceeding daily quota', async function () { - await updateUser({ - url: server.url, - userId: rootId, - accessToken: server.accessToken, - videoQuota: 1024 * 1024 * 1024, - videoQuotaDaily: 42 - }) - - await uploadVideo(server.url, server.accessToken, {}, HttpStatusCode.PAYLOAD_TOO_LARGE_413) - }) - }) - describe('When asking a password reset', function () { const path = '/api/v1/users/ask-reset-password' diff --git a/server/tests/api/check-params/videos.ts b/server/tests/api/check-params/videos.ts index 188d1835c..c970c4a15 100644 --- a/server/tests/api/check-params/videos.ts +++ b/server/tests/api/check-params/videos.ts @@ -1,11 +1,12 @@ /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ +import 'mocha' import * as chai from 'chai' import { omit } from 'lodash' -import 'mocha' import { join } from 'path' -import { VideoPrivacy } from '../../../../shared/models/videos/video-privacy.enum' +import { HttpStatusCode } from '../../../../shared/core-utils/miscs/http-error-codes' import { + checkUploadVideoParam, cleanupTests, createUser, flushAndRunServer, @@ -18,17 +19,18 @@ import { makePutBodyRequest, makeUploadRequest, removeVideo, + root, ServerInfo, setAccessTokensToServers, - userLogin, - root + userLogin } from '../../../../shared/extra-utils' import { checkBadCountPagination, checkBadSortPagination, checkBadStartPagination } from '../../../../shared/extra-utils/requests/check-api-params' -import { HttpStatusCode } from '../../../../shared/core-utils/miscs/http-error-codes' +import { VideoPrivacy } from '../../../../shared/models/videos/video-privacy.enum' +import { randomInt } from '@shared/core-utils' const expect = chai.expect @@ -183,7 +185,7 @@ describe('Test videos API validator', function () { describe('When adding a video', function () { let baseCorrectParams const baseCorrectAttaches = { - videofile: join(root(), 'server', 'tests', 'fixtures', 'video_short.webm') + fixture: join(root(), 'server', 'tests', 'fixtures', 'video_short.webm') } before(function () { @@ -206,256 +208,243 @@ describe('Test videos API validator', function () { } }) - it('Should fail with nothing', async function () { - const fields = {} - const attaches = {} - await makeUploadRequest({ url: server.url, path: path + '/upload', token: server.accessToken, fields, attaches }) - }) + function runSuite (mode: 'legacy' | 'resumable') { - it('Should fail without name', async function () { - const fields = omit(baseCorrectParams, 'name') - const attaches = baseCorrectAttaches + it('Should fail with nothing', async function () { + const fields = {} + const attaches = {} + await checkUploadVideoParam(server.url, server.accessToken, { ...fields, ...attaches }, HttpStatusCode.BAD_REQUEST_400, mode) + }) - await makeUploadRequest({ url: server.url, path: path + '/upload', token: server.accessToken, fields, attaches }) - }) + it('Should fail without name', async function () { + const fields = omit(baseCorrectParams, 'name') + const attaches = baseCorrectAttaches - it('Should fail with a long name', async function () { - const fields = immutableAssign(baseCorrectParams, { name: 'super'.repeat(65) }) - const attaches = baseCorrectAttaches + await checkUploadVideoParam(server.url, server.accessToken, { ...fields, ...attaches }, HttpStatusCode.BAD_REQUEST_400, mode) + }) - await makeUploadRequest({ url: server.url, path: path + '/upload', token: server.accessToken, fields, attaches }) - }) + it('Should fail with a long name', async function () { + const fields = immutableAssign(baseCorrectParams, { name: 'super'.repeat(65) }) + const attaches = baseCorrectAttaches - it('Should fail with a bad category', async function () { - const fields = immutableAssign(baseCorrectParams, { category: 125 }) - const attaches = baseCorrectAttaches + await checkUploadVideoParam(server.url, server.accessToken, { ...fields, ...attaches }, HttpStatusCode.BAD_REQUEST_400, mode) + }) - await makeUploadRequest({ url: server.url, path: path + '/upload', token: server.accessToken, fields, attaches }) - }) + it('Should fail with a bad category', async function () { + const fields = immutableAssign(baseCorrectParams, { category: 125 }) + const attaches = baseCorrectAttaches - it('Should fail with a bad licence', async function () { - const fields = immutableAssign(baseCorrectParams, { licence: 125 }) - const attaches = baseCorrectAttaches + await checkUploadVideoParam(server.url, server.accessToken, { ...fields, ...attaches }, HttpStatusCode.BAD_REQUEST_400, mode) + }) - await makeUploadRequest({ url: server.url, path: path + '/upload', token: server.accessToken, fields, attaches }) - }) + it('Should fail with a bad licence', async function () { + const fields = immutableAssign(baseCorrectParams, { licence: 125 }) + const attaches = baseCorrectAttaches - it('Should fail with a bad language', async function () { - const fields = immutableAssign(baseCorrectParams, { language: 'a'.repeat(15) }) - const attaches = baseCorrectAttaches + await checkUploadVideoParam(server.url, server.accessToken, { ...fields, ...attaches }, HttpStatusCode.BAD_REQUEST_400, mode) + }) - await makeUploadRequest({ url: server.url, path: path + '/upload', token: server.accessToken, fields, attaches }) - }) + it('Should fail with a bad language', async function () { + const fields = immutableAssign(baseCorrectParams, { language: 'a'.repeat(15) }) + const attaches = baseCorrectAttaches - it('Should fail with a long description', async function () { - const fields = immutableAssign(baseCorrectParams, { description: 'super'.repeat(2500) }) - const attaches = baseCorrectAttaches + await checkUploadVideoParam(server.url, server.accessToken, { ...fields, ...attaches }, HttpStatusCode.BAD_REQUEST_400, mode) + }) - await makeUploadRequest({ url: server.url, path: path + '/upload', token: server.accessToken, fields, attaches }) - }) + it('Should fail with a long description', async function () { + const fields = immutableAssign(baseCorrectParams, { description: 'super'.repeat(2500) }) + const attaches = baseCorrectAttaches - it('Should fail with a long support text', async function () { - const fields = immutableAssign(baseCorrectParams, { support: 'super'.repeat(201) }) - const attaches = baseCorrectAttaches + await checkUploadVideoParam(server.url, server.accessToken, { ...fields, ...attaches }, HttpStatusCode.BAD_REQUEST_400, mode) + }) - await makeUploadRequest({ url: server.url, path: path + '/upload', token: server.accessToken, fields, attaches }) - }) + it('Should fail with a long support text', async function () { + const fields = immutableAssign(baseCorrectParams, { support: 'super'.repeat(201) }) + const attaches = baseCorrectAttaches - it('Should fail without a channel', async function () { - const fields = omit(baseCorrectParams, 'channelId') - const attaches = baseCorrectAttaches + await checkUploadVideoParam(server.url, server.accessToken, { ...fields, ...attaches }, HttpStatusCode.BAD_REQUEST_400, mode) + }) - await makeUploadRequest({ url: server.url, path: path + '/upload', token: server.accessToken, fields, attaches }) - }) + it('Should fail without a channel', async function () { + const fields = omit(baseCorrectParams, 'channelId') + const attaches = baseCorrectAttaches - it('Should fail with a bad channel', async function () { - const fields = immutableAssign(baseCorrectParams, { channelId: 545454 }) - const attaches = baseCorrectAttaches + await checkUploadVideoParam(server.url, server.accessToken, { ...fields, ...attaches }, HttpStatusCode.BAD_REQUEST_400, mode) + }) - await makeUploadRequest({ url: server.url, path: path + '/upload', token: server.accessToken, fields, attaches }) - }) + it('Should fail with a bad channel', async function () { + const fields = immutableAssign(baseCorrectParams, { channelId: 545454 }) + const attaches = baseCorrectAttaches - it('Should fail with another user channel', async function () { - const user = { - username: 'fake', - password: 'fake_password' - } - await createUser({ url: server.url, accessToken: server.accessToken, username: user.username, password: user.password }) + await checkUploadVideoParam(server.url, server.accessToken, { ...fields, ...attaches }, HttpStatusCode.BAD_REQUEST_400, mode) + }) - const accessTokenUser = await userLogin(server, user) - const res = await getMyUserInformation(server.url, accessTokenUser) - const customChannelId = res.body.videoChannels[0].id + it('Should fail with another user channel', async function () { + const user = { + username: 'fake' + randomInt(0, 1500), + password: 'fake_password' + } + await createUser({ url: server.url, accessToken: server.accessToken, username: user.username, password: user.password }) - const fields = immutableAssign(baseCorrectParams, { channelId: customChannelId }) - const attaches = baseCorrectAttaches + const accessTokenUser = await userLogin(server, user) + const res = await getMyUserInformation(server.url, accessTokenUser) + const customChannelId = res.body.videoChannels[0].id - await makeUploadRequest({ url: server.url, path: path + '/upload', token: userAccessToken, fields, attaches }) - }) + const fields = immutableAssign(baseCorrectParams, { channelId: customChannelId }) + const attaches = baseCorrectAttaches - it('Should fail with too many tags', async function () { - const fields = immutableAssign(baseCorrectParams, { tags: [ 'tag1', 'tag2', 'tag3', 'tag4', 'tag5', 'tag6' ] }) - const attaches = baseCorrectAttaches + await checkUploadVideoParam(server.url, userAccessToken, { ...fields, ...attaches }, HttpStatusCode.BAD_REQUEST_400, mode) + }) - await makeUploadRequest({ url: server.url, path: path + '/upload', token: server.accessToken, fields, attaches }) - }) + it('Should fail with too many tags', async function () { + const fields = immutableAssign(baseCorrectParams, { tags: [ 'tag1', 'tag2', 'tag3', 'tag4', 'tag5', 'tag6' ] }) + const attaches = baseCorrectAttaches - it('Should fail with a tag length too low', async function () { - const fields = immutableAssign(baseCorrectParams, { tags: [ 'tag1', 't' ] }) - const attaches = baseCorrectAttaches + await checkUploadVideoParam(server.url, server.accessToken, { ...fields, ...attaches }, HttpStatusCode.BAD_REQUEST_400, mode) + }) - await makeUploadRequest({ url: server.url, path: path + '/upload', token: server.accessToken, fields, attaches }) - }) + it('Should fail with a tag length too low', async function () { + const fields = immutableAssign(baseCorrectParams, { tags: [ 'tag1', 't' ] }) + const attaches = baseCorrectAttaches - it('Should fail with a tag length too big', async function () { - const fields = immutableAssign(baseCorrectParams, { tags: [ 'tag1', 'my_super_tag_too_long_long_long_long_long_long' ] }) - const attaches = baseCorrectAttaches + await checkUploadVideoParam(server.url, server.accessToken, { ...fields, ...attaches }, HttpStatusCode.BAD_REQUEST_400, mode) + }) - await makeUploadRequest({ url: server.url, path: path + '/upload', token: server.accessToken, fields, attaches }) - }) + it('Should fail with a tag length too big', async function () { + const fields = immutableAssign(baseCorrectParams, { tags: [ 'tag1', 'my_super_tag_too_long_long_long_long_long_long' ] }) + const attaches = baseCorrectAttaches - it('Should fail with a bad schedule update (miss updateAt)', async function () { - const fields = immutableAssign(baseCorrectParams, { 'scheduleUpdate[privacy]': VideoPrivacy.PUBLIC }) - const attaches = baseCorrectAttaches + await checkUploadVideoParam(server.url, server.accessToken, { ...fields, ...attaches }, HttpStatusCode.BAD_REQUEST_400, mode) + }) - await makeUploadRequest({ url: server.url, path: path + '/upload', token: server.accessToken, fields, attaches }) - }) + it('Should fail with a bad schedule update (miss updateAt)', async function () { + const fields = immutableAssign(baseCorrectParams, { scheduleUpdate: { privacy: VideoPrivacy.PUBLIC } }) + const attaches = baseCorrectAttaches - it('Should fail with a bad schedule update (wrong updateAt)', async function () { - const fields = immutableAssign(baseCorrectParams, { - 'scheduleUpdate[privacy]': VideoPrivacy.PUBLIC, - 'scheduleUpdate[updateAt]': 'toto' + await checkUploadVideoParam(server.url, server.accessToken, { ...fields, ...attaches }, HttpStatusCode.BAD_REQUEST_400, mode) }) - const attaches = baseCorrectAttaches - await makeUploadRequest({ url: server.url, path: path + '/upload', token: server.accessToken, fields, attaches }) - }) + it('Should fail with a bad schedule update (wrong updateAt)', async function () { + const fields = immutableAssign(baseCorrectParams, { + scheduleUpdate: { + privacy: VideoPrivacy.PUBLIC, + updateAt: 'toto' + } + }) + const attaches = baseCorrectAttaches - it('Should fail with a bad originally published at attribute', async function () { - const fields = immutableAssign(baseCorrectParams, { originallyPublishedAt: 'toto' }) - const attaches = baseCorrectAttaches + await checkUploadVideoParam(server.url, server.accessToken, { ...fields, ...attaches }, HttpStatusCode.BAD_REQUEST_400, mode) + }) - await makeUploadRequest({ url: server.url, path: path + '/upload', token: server.accessToken, fields, attaches }) - }) + it('Should fail with a bad originally published at attribute', async function () { + const fields = immutableAssign(baseCorrectParams, { originallyPublishedAt: 'toto' }) + const attaches = baseCorrectAttaches - it('Should fail without an input file', async function () { - const fields = baseCorrectParams - const attaches = {} - await makeUploadRequest({ url: server.url, path: path + '/upload', token: server.accessToken, fields, attaches }) - }) + await checkUploadVideoParam(server.url, server.accessToken, { ...fields, ...attaches }, HttpStatusCode.BAD_REQUEST_400, mode) + }) - it('Should fail with an incorrect input file', async function () { - const fields = baseCorrectParams - let attaches = { - videofile: join(root(), 'server', 'tests', 'fixtures', 'video_short_fake.webm') - } - await makeUploadRequest({ - url: server.url, - path: path + '/upload', - token: server.accessToken, - fields, - attaches, - statusCodeExpected: HttpStatusCode.UNPROCESSABLE_ENTITY_422 + it('Should fail without an input file', async function () { + const fields = baseCorrectParams + const attaches = {} + await checkUploadVideoParam(server.url, server.accessToken, { ...fields, ...attaches }, HttpStatusCode.BAD_REQUEST_400, mode) }) - attaches = { - videofile: join(root(), 'server', 'tests', 'fixtures', 'video_short.mkv') - } - await makeUploadRequest({ - url: server.url, - path: path + '/upload', - token: server.accessToken, - fields, - attaches, - statusCodeExpected: HttpStatusCode.UNSUPPORTED_MEDIA_TYPE_415 + it('Should fail with an incorrect input file', async function () { + const fields = baseCorrectParams + let attaches = { fixture: join(root(), 'server', 'tests', 'fixtures', 'video_short_fake.webm') } + + await checkUploadVideoParam( + server.url, + server.accessToken, + { ...fields, ...attaches }, + HttpStatusCode.UNPROCESSABLE_ENTITY_422, + mode + ) + + attaches = { fixture: join(root(), 'server', 'tests', 'fixtures', 'video_short.mkv') } + await checkUploadVideoParam( + server.url, + server.accessToken, + { ...fields, ...attaches }, + HttpStatusCode.UNSUPPORTED_MEDIA_TYPE_415, + mode + ) }) - }) - it('Should fail with an incorrect thumbnail file', async function () { - const fields = baseCorrectParams - const attaches = { - thumbnailfile: join(root(), 'server', 'tests', 'fixtures', 'video_short.mp4'), - videofile: join(root(), 'server', 'tests', 'fixtures', 'video_short.mp4') - } + it('Should fail with an incorrect thumbnail file', async function () { + const fields = baseCorrectParams + const attaches = { + thumbnailfile: join(root(), 'server', 'tests', 'fixtures', 'video_short.mp4'), + fixture: join(root(), 'server', 'tests', 'fixtures', 'video_short.mp4') + } - await makeUploadRequest({ url: server.url, path: path + '/upload', token: server.accessToken, fields, attaches }) - }) + await checkUploadVideoParam(server.url, server.accessToken, { ...fields, ...attaches }, HttpStatusCode.BAD_REQUEST_400, mode) + }) - it('Should fail with a big thumbnail file', async function () { - const fields = baseCorrectParams - const attaches = { - thumbnailfile: join(root(), 'server', 'tests', 'fixtures', 'preview-big.png'), - videofile: join(root(), 'server', 'tests', 'fixtures', 'video_short.mp4') - } + it('Should fail with a big thumbnail file', async function () { + const fields = baseCorrectParams + const attaches = { + thumbnailfile: join(root(), 'server', 'tests', 'fixtures', 'preview-big.png'), + fixture: join(root(), 'server', 'tests', 'fixtures', 'video_short.mp4') + } - await makeUploadRequest({ url: server.url, path: path + '/upload', token: server.accessToken, fields, attaches }) - }) + await checkUploadVideoParam(server.url, server.accessToken, { ...fields, ...attaches }, HttpStatusCode.BAD_REQUEST_400, mode) + }) - it('Should fail with an incorrect preview file', async function () { - const fields = baseCorrectParams - const attaches = { - previewfile: join(root(), 'server', 'tests', 'fixtures', 'video_short.mp4'), - videofile: join(root(), 'server', 'tests', 'fixtures', 'video_short.mp4') - } + it('Should fail with an incorrect preview file', async function () { + const fields = baseCorrectParams + const attaches = { + previewfile: join(root(), 'server', 'tests', 'fixtures', 'video_short.mp4'), + fixture: join(root(), 'server', 'tests', 'fixtures', 'video_short.mp4') + } - await makeUploadRequest({ url: server.url, path: path + '/upload', token: server.accessToken, fields, attaches }) - }) + await checkUploadVideoParam(server.url, server.accessToken, { ...fields, ...attaches }, HttpStatusCode.BAD_REQUEST_400, mode) + }) - it('Should fail with a big preview file', async function () { - const fields = baseCorrectParams - const attaches = { - previewfile: join(root(), 'server', 'tests', 'fixtures', 'preview-big.png'), - videofile: join(root(), 'server', 'tests', 'fixtures', 'video_short.mp4') - } + it('Should fail with a big preview file', async function () { + const fields = baseCorrectParams + const attaches = { + previewfile: join(root(), 'server', 'tests', 'fixtures', 'preview-big.png'), + fixture: join(root(), 'server', 'tests', 'fixtures', 'video_short.mp4') + } - await makeUploadRequest({ url: server.url, path: path + '/upload', token: server.accessToken, fields, attaches }) - }) + await checkUploadVideoParam(server.url, server.accessToken, { ...fields, ...attaches }, HttpStatusCode.BAD_REQUEST_400, mode) + }) - it('Should succeed with the correct parameters', async function () { - this.timeout(10000) + it('Should succeed with the correct parameters', async function () { + this.timeout(10000) - const fields = baseCorrectParams + const fields = baseCorrectParams - { - const attaches = baseCorrectAttaches - await makeUploadRequest({ - url: server.url, - path: path + '/upload', - token: server.accessToken, - fields, - attaches, - statusCodeExpected: HttpStatusCode.OK_200 - }) - } + { + const attaches = baseCorrectAttaches + await checkUploadVideoParam(server.url, server.accessToken, { ...fields, ...attaches }, HttpStatusCode.OK_200, mode) + } - { - const attaches = immutableAssign(baseCorrectAttaches, { - videofile: join(root(), 'server', 'tests', 'fixtures', 'video_short.mp4') - }) + { + const attaches = immutableAssign(baseCorrectAttaches, { + videofile: join(root(), 'server', 'tests', 'fixtures', 'video_short.mp4') + }) - await makeUploadRequest({ - url: server.url, - path: path + '/upload', - token: server.accessToken, - fields, - attaches, - statusCodeExpected: HttpStatusCode.OK_200 - }) - } + await checkUploadVideoParam(server.url, server.accessToken, { ...fields, ...attaches }, HttpStatusCode.OK_200, mode) + } - { - const attaches = immutableAssign(baseCorrectAttaches, { - videofile: join(root(), 'server', 'tests', 'fixtures', 'video_short.ogv') - }) + { + const attaches = immutableAssign(baseCorrectAttaches, { + videofile: join(root(), 'server', 'tests', 'fixtures', 'video_short.ogv') + }) - await makeUploadRequest({ - url: server.url, - path: path + '/upload', - token: server.accessToken, - fields, - attaches, - statusCodeExpected: HttpStatusCode.OK_200 - }) - } + await checkUploadVideoParam(server.url, server.accessToken, { ...fields, ...attaches }, HttpStatusCode.OK_200, mode) + } + }) + } + + describe('Resumable upload', function () { + runSuite('resumable') + }) + + describe('Legacy upload', function () { + runSuite('legacy') }) }) @@ -678,7 +667,7 @@ describe('Test videos API validator', function () { }) expect(res.body.data).to.be.an('array') - expect(res.body.data.length).to.equal(3) + expect(res.body.data.length).to.equal(6) }) it('Should fail without a correct uuid', async function () { diff --git a/server/tests/api/videos/index.ts b/server/tests/api/videos/index.ts index fc8b447b7..5c07f8926 100644 --- a/server/tests/api/videos/index.ts +++ b/server/tests/api/videos/index.ts @@ -1,5 +1,6 @@ import './audio-only' import './multiple-servers' +import './resumable-upload' import './single-server' import './video-captions' import './video-change-ownership' diff --git a/server/tests/api/videos/multiple-servers.ts b/server/tests/api/videos/multiple-servers.ts index 55e280e9f..41cd814e0 100644 --- a/server/tests/api/videos/multiple-servers.ts +++ b/server/tests/api/videos/multiple-servers.ts @@ -181,7 +181,7 @@ describe('Test multiple servers', function () { thumbnailfile: 'thumbnail.jpg', previewfile: 'preview.jpg' } - await uploadVideo(servers[1].url, userAccessToken, videoAttributes) + await uploadVideo(servers[1].url, userAccessToken, videoAttributes, HttpStatusCode.OK_200, 'resumable') // Transcoding await waitJobs(servers) diff --git a/server/tests/api/videos/resumable-upload.ts b/server/tests/api/videos/resumable-upload.ts new file mode 100644 index 000000000..af9221c43 --- /dev/null +++ b/server/tests/api/videos/resumable-upload.ts @@ -0,0 +1,187 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import 'mocha' +import * as chai from 'chai' +import { pathExists, readdir, stat } from 'fs-extra' +import { join } from 'path' +import { HttpStatusCode } from '@shared/core-utils' +import { + buildAbsoluteFixturePath, + buildServerDirectory, + flushAndRunServer, + getMyUserInformation, + prepareResumableUpload, + sendDebugCommand, + sendResumableChunks, + ServerInfo, + setAccessTokensToServers, + setDefaultVideoChannel, + updateUser +} from '@shared/extra-utils' +import { MyUser, VideoPrivacy } from '@shared/models' + +const expect = chai.expect + +// Most classic resumable upload tests are done in other test suites + +describe('Test resumable upload', function () { + const defaultFixture = 'video_short.mp4' + let server: ServerInfo + let rootId: number + + async function buildSize (fixture: string, size?: number) { + if (size !== undefined) return size + + const baseFixture = buildAbsoluteFixturePath(fixture) + return (await stat(baseFixture)).size + } + + async function prepareUpload (sizeArg?: number) { + const size = await buildSize(defaultFixture, sizeArg) + + const attributes = { + name: 'video', + channelId: server.videoChannel.id, + privacy: VideoPrivacy.PUBLIC, + fixture: defaultFixture + } + + const mimetype = 'video/mp4' + + const res = await prepareResumableUpload({ url: server.url, token: server.accessToken, attributes, size, mimetype }) + + return res.header['location'].split('?')[1] + } + + async function sendChunks (options: { + pathUploadId: string + size?: number + expectedStatus?: HttpStatusCode + contentLength?: number + contentRange?: string + contentRangeBuilder?: (start: number, chunk: any) => string + }) { + const { pathUploadId, expectedStatus, contentLength, contentRangeBuilder } = options + + const size = await buildSize(defaultFixture, options.size) + const absoluteFilePath = buildAbsoluteFixturePath(defaultFixture) + + return sendResumableChunks({ + url: server.url, + token: server.accessToken, + pathUploadId, + videoFilePath: absoluteFilePath, + size, + contentLength, + contentRangeBuilder, + specialStatus: expectedStatus + }) + } + + async function checkFileSize (uploadIdArg: string, expectedSize: number | null) { + const uploadId = uploadIdArg.replace(/^upload_id=/, '') + + const subPath = join('tmp', 'resumable-uploads', uploadId) + const filePath = buildServerDirectory(server, subPath) + const exists = await pathExists(filePath) + + if (expectedSize === null) { + expect(exists).to.be.false + return + } + + expect(exists).to.be.true + + expect((await stat(filePath)).size).to.equal(expectedSize) + } + + async function countResumableUploads () { + const subPath = join('tmp', 'resumable-uploads') + const filePath = buildServerDirectory(server, subPath) + + const files = await readdir(filePath) + return files.length + } + + before(async function () { + this.timeout(30000) + + server = await flushAndRunServer(1) + await setAccessTokensToServers([ server ]) + await setDefaultVideoChannel([ server ]) + + const res = await getMyUserInformation(server.url, server.accessToken) + rootId = (res.body as MyUser).id + + await updateUser({ + url: server.url, + userId: rootId, + accessToken: server.accessToken, + videoQuota: 10_000_000 + }) + }) + + describe('Directory cleaning', function () { + + it('Should correctly delete files after an upload', async function () { + const uploadId = await prepareUpload() + await sendChunks({ pathUploadId: uploadId }) + + expect(await countResumableUploads()).to.equal(0) + }) + + it('Should not delete files after an unfinished upload', async function () { + await prepareUpload() + + expect(await countResumableUploads()).to.equal(2) + }) + + it('Should not delete recent uploads', async function () { + await sendDebugCommand(server.url, server.accessToken, { command: 'remove-dandling-resumable-uploads' }) + + expect(await countResumableUploads()).to.equal(2) + }) + + it('Should delete old uploads', async function () { + await sendDebugCommand(server.url, server.accessToken, { command: 'remove-dandling-resumable-uploads' }) + + expect(await countResumableUploads()).to.equal(0) + }) + }) + + describe('Resumable upload and chunks', function () { + + it('Should accept the same amount of chunks', async function () { + const uploadId = await prepareUpload() + await sendChunks({ pathUploadId: uploadId }) + + await checkFileSize(uploadId, null) + }) + + it('Should not accept more chunks than expected', async function () { + const size = 100 + const uploadId = await prepareUpload(size) + + await sendChunks({ pathUploadId: uploadId, expectedStatus: HttpStatusCode.CONFLICT_409 }) + await checkFileSize(uploadId, 0) + }) + + it('Should not accept more chunks than expected with an invalid content length/content range', async function () { + const uploadId = await prepareUpload(1500) + + await sendChunks({ pathUploadId: uploadId, expectedStatus: HttpStatusCode.BAD_REQUEST_400, contentLength: 1000 }) + await checkFileSize(uploadId, 0) + }) + + it('Should not accept more chunks than expected with an invalid content length', async function () { + const uploadId = await prepareUpload(500) + + const size = 1000 + + const contentRangeBuilder = start => `bytes ${start}-${start + size - 1}/${size}` + await sendChunks({ pathUploadId: uploadId, expectedStatus: HttpStatusCode.BAD_REQUEST_400, contentRangeBuilder, contentLength: size }) + await checkFileSize(uploadId, 0) + }) + }) + +}) diff --git a/server/tests/api/videos/single-server.ts b/server/tests/api/videos/single-server.ts index a79648bf7..1058a1e9c 100644 --- a/server/tests/api/videos/single-server.ts +++ b/server/tests/api/videos/single-server.ts @@ -1,9 +1,9 @@ /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ +import 'mocha' import * as chai from 'chai' import { keyBy } from 'lodash' -import 'mocha' -import { VideoPrivacy } from '../../../../shared/models/videos' + import { checkVideoFilesWereRemoved, cleanupTests, @@ -28,430 +28,432 @@ import { viewVideo, wait } from '../../../../shared/extra-utils' +import { VideoPrivacy } from '../../../../shared/models/videos' +import { HttpStatusCode } from '@shared/core-utils' const expect = chai.expect describe('Test a single server', function () { - let server: ServerInfo = null - let videoId = -1 - let videoId2 = -1 - let videoUUID = '' - let videosListBase: any[] = null - - const getCheckAttributes = () => ({ - name: 'my super name', - category: 2, - licence: 6, - language: 'zh', - nsfw: true, - description: 'my super description', - support: 'my super support text', - account: { - name: 'root', - host: 'localhost:' + server.port - }, - isLocal: true, - duration: 5, - tags: [ 'tag1', 'tag2', 'tag3' ], - privacy: VideoPrivacy.PUBLIC, - commentsEnabled: true, - downloadEnabled: true, - channel: { - displayName: 'Main root channel', - name: 'root_channel', - description: '', - isLocal: true - }, - fixture: 'video_short.webm', - files: [ - { - resolution: 720, - size: 218910 - } - ] - }) - - const updateCheckAttributes = () => ({ - name: 'my super video updated', - category: 4, - licence: 2, - language: 'ar', - nsfw: false, - description: 'my super description updated', - support: 'my super support text updated', - account: { - name: 'root', - host: 'localhost:' + server.port - }, - isLocal: true, - tags: [ 'tagup1', 'tagup2' ], - privacy: VideoPrivacy.PUBLIC, - duration: 5, - commentsEnabled: false, - downloadEnabled: false, - channel: { - name: 'root_channel', - displayName: 'Main root channel', - description: '', - isLocal: true - }, - fixture: 'video_short3.webm', - files: [ - { - resolution: 720, - size: 292677 - } - ] - }) - - before(async function () { - this.timeout(30000) - - server = await flushAndRunServer(1) - - await setAccessTokensToServers([ server ]) - }) - - it('Should list video categories', async function () { - const res = await getVideoCategories(server.url) - - const categories = res.body - expect(Object.keys(categories)).to.have.length.above(10) - - expect(categories[11]).to.equal('News & Politics') - }) - - it('Should list video licences', async function () { - const res = await getVideoLicences(server.url) - - const licences = res.body - expect(Object.keys(licences)).to.have.length.above(5) - - expect(licences[3]).to.equal('Attribution - No Derivatives') - }) - - it('Should list video languages', async function () { - const res = await getVideoLanguages(server.url) - - const languages = res.body - expect(Object.keys(languages)).to.have.length.above(5) - - expect(languages['ru']).to.equal('Russian') - }) - - it('Should list video privacies', async function () { - const res = await getVideoPrivacies(server.url) - - const privacies = res.body - expect(Object.keys(privacies)).to.have.length.at.least(3) - - expect(privacies[3]).to.equal('Private') - }) - - it('Should not have videos', async function () { - const res = await getVideosList(server.url) - - expect(res.body.total).to.equal(0) - expect(res.body.data).to.be.an('array') - expect(res.body.data.length).to.equal(0) - }) - it('Should upload the video', async function () { - this.timeout(10000) + function runSuite (mode: 'legacy' | 'resumable') { + let server: ServerInfo = null + let videoId = -1 + let videoId2 = -1 + let videoUUID = '' + let videosListBase: any[] = null - const videoAttributes = { + const getCheckAttributes = () => ({ name: 'my super name', category: 2, - nsfw: true, licence: 6, - tags: [ 'tag1', 'tag2', 'tag3' ] - } - const res = await uploadVideo(server.url, server.accessToken, videoAttributes) - expect(res.body.video).to.not.be.undefined - expect(res.body.video.id).to.equal(1) - expect(res.body.video.uuid).to.have.length.above(5) - - videoId = res.body.video.id - videoUUID = res.body.video.uuid - }) - - it('Should get and seed the uploaded video', async function () { - this.timeout(5000) - - const res = await getVideosList(server.url) - - expect(res.body.total).to.equal(1) - expect(res.body.data).to.be.an('array') - expect(res.body.data.length).to.equal(1) - - const video = res.body.data[0] - await completeVideoCheck(server.url, video, getCheckAttributes()) - }) + language: 'zh', + nsfw: true, + description: 'my super description', + support: 'my super support text', + account: { + name: 'root', + host: 'localhost:' + server.port + }, + isLocal: true, + duration: 5, + tags: [ 'tag1', 'tag2', 'tag3' ], + privacy: VideoPrivacy.PUBLIC, + commentsEnabled: true, + downloadEnabled: true, + channel: { + displayName: 'Main root channel', + name: 'root_channel', + description: '', + isLocal: true + }, + fixture: 'video_short.webm', + files: [ + { + resolution: 720, + size: 218910 + } + ] + }) + + const updateCheckAttributes = () => ({ + name: 'my super video updated', + category: 4, + licence: 2, + language: 'ar', + nsfw: false, + description: 'my super description updated', + support: 'my super support text updated', + account: { + name: 'root', + host: 'localhost:' + server.port + }, + isLocal: true, + tags: [ 'tagup1', 'tagup2' ], + privacy: VideoPrivacy.PUBLIC, + duration: 5, + commentsEnabled: false, + downloadEnabled: false, + channel: { + name: 'root_channel', + displayName: 'Main root channel', + description: '', + isLocal: true + }, + fixture: 'video_short3.webm', + files: [ + { + resolution: 720, + size: 292677 + } + ] + }) - it('Should get the video by UUID', async function () { - this.timeout(5000) + before(async function () { + this.timeout(30000) - const res = await getVideo(server.url, videoUUID) + server = await flushAndRunServer(1) - const video = res.body - await completeVideoCheck(server.url, video, getCheckAttributes()) - }) + await setAccessTokensToServers([ server ]) + }) - it('Should have the views updated', async function () { - this.timeout(20000) + it('Should list video categories', async function () { + const res = await getVideoCategories(server.url) - await viewVideo(server.url, videoId) - await viewVideo(server.url, videoId) - await viewVideo(server.url, videoId) + const categories = res.body + expect(Object.keys(categories)).to.have.length.above(10) - await wait(1500) + expect(categories[11]).to.equal('News & Politics') + }) - await viewVideo(server.url, videoId) - await viewVideo(server.url, videoId) + it('Should list video licences', async function () { + const res = await getVideoLicences(server.url) - await wait(1500) + const licences = res.body + expect(Object.keys(licences)).to.have.length.above(5) - await viewVideo(server.url, videoId) - await viewVideo(server.url, videoId) + expect(licences[3]).to.equal('Attribution - No Derivatives') + }) - // Wait the repeatable job - await wait(8000) + it('Should list video languages', async function () { + const res = await getVideoLanguages(server.url) - const res = await getVideo(server.url, videoId) + const languages = res.body + expect(Object.keys(languages)).to.have.length.above(5) - const video = res.body - expect(video.views).to.equal(3) - }) + expect(languages['ru']).to.equal('Russian') + }) - it('Should remove the video', async function () { - await removeVideo(server.url, server.accessToken, videoId) + it('Should list video privacies', async function () { + const res = await getVideoPrivacies(server.url) - await checkVideoFilesWereRemoved(videoUUID, 1) - }) + const privacies = res.body + expect(Object.keys(privacies)).to.have.length.at.least(3) - it('Should not have videos', async function () { - const res = await getVideosList(server.url) + expect(privacies[3]).to.equal('Private') + }) - expect(res.body.total).to.equal(0) - expect(res.body.data).to.be.an('array') - expect(res.body.data).to.have.lengthOf(0) - }) + it('Should not have videos', async function () { + const res = await getVideosList(server.url) - it('Should upload 6 videos', async function () { - this.timeout(25000) + expect(res.body.total).to.equal(0) + expect(res.body.data).to.be.an('array') + expect(res.body.data.length).to.equal(0) + }) - const videos = new Set([ - 'video_short.mp4', 'video_short.ogv', 'video_short.webm', - 'video_short1.webm', 'video_short2.webm', 'video_short3.webm' - ]) + it('Should upload the video', async function () { + this.timeout(10000) - for (const video of videos) { const videoAttributes = { - name: video + ' name', - description: video + ' description', + name: 'my super name', category: 2, - licence: 1, - language: 'en', nsfw: true, - tags: [ 'tag1', 'tag2', 'tag3' ], - fixture: video + licence: 6, + tags: [ 'tag1', 'tag2', 'tag3' ] } + const res = await uploadVideo(server.url, server.accessToken, videoAttributes, HttpStatusCode.OK_200, mode) + expect(res.body.video).to.not.be.undefined + expect(res.body.video.id).to.equal(1) + expect(res.body.video.uuid).to.have.length.above(5) - await uploadVideo(server.url, server.accessToken, videoAttributes) - } - }) + videoId = res.body.video.id + videoUUID = res.body.video.uuid + }) - it('Should have the correct durations', async function () { - const res = await getVideosList(server.url) - - expect(res.body.total).to.equal(6) - const videos = res.body.data - expect(videos).to.be.an('array') - expect(videos).to.have.lengthOf(6) - - const videosByName = keyBy<{ duration: number }>(videos, 'name') - expect(videosByName['video_short.mp4 name'].duration).to.equal(5) - expect(videosByName['video_short.ogv name'].duration).to.equal(5) - expect(videosByName['video_short.webm name'].duration).to.equal(5) - expect(videosByName['video_short1.webm name'].duration).to.equal(10) - expect(videosByName['video_short2.webm name'].duration).to.equal(5) - expect(videosByName['video_short3.webm name'].duration).to.equal(5) - }) + it('Should get and seed the uploaded video', async function () { + this.timeout(5000) - it('Should have the correct thumbnails', async function () { - const res = await getVideosList(server.url) + const res = await getVideosList(server.url) - const videos = res.body.data - // For the next test - videosListBase = videos + expect(res.body.total).to.equal(1) + expect(res.body.data).to.be.an('array') + expect(res.body.data.length).to.equal(1) - for (const video of videos) { - const videoName = video.name.replace(' name', '') - await testImage(server.url, videoName, video.thumbnailPath) - } - }) + const video = res.body.data[0] + await completeVideoCheck(server.url, video, getCheckAttributes()) + }) - it('Should list only the two first videos', async function () { - const res = await getVideosListPagination(server.url, 0, 2, 'name') + it('Should get the video by UUID', async function () { + this.timeout(5000) - const videos = res.body.data - expect(res.body.total).to.equal(6) - expect(videos.length).to.equal(2) - expect(videos[0].name).to.equal(videosListBase[0].name) - expect(videos[1].name).to.equal(videosListBase[1].name) - }) + const res = await getVideo(server.url, videoUUID) - it('Should list only the next three videos', async function () { - const res = await getVideosListPagination(server.url, 2, 3, 'name') + const video = res.body + await completeVideoCheck(server.url, video, getCheckAttributes()) + }) - const videos = res.body.data - expect(res.body.total).to.equal(6) - expect(videos.length).to.equal(3) - expect(videos[0].name).to.equal(videosListBase[2].name) - expect(videos[1].name).to.equal(videosListBase[3].name) - expect(videos[2].name).to.equal(videosListBase[4].name) - }) + it('Should have the views updated', async function () { + this.timeout(20000) - it('Should list the last video', async function () { - const res = await getVideosListPagination(server.url, 5, 6, 'name') + await viewVideo(server.url, videoId) + await viewVideo(server.url, videoId) + await viewVideo(server.url, videoId) - const videos = res.body.data - expect(res.body.total).to.equal(6) - expect(videos.length).to.equal(1) - expect(videos[0].name).to.equal(videosListBase[5].name) - }) + await wait(1500) - it('Should not have the total field', async function () { - const res = await getVideosListPagination(server.url, 5, 6, 'name', true) + await viewVideo(server.url, videoId) + await viewVideo(server.url, videoId) - const videos = res.body.data - expect(res.body.total).to.not.exist - expect(videos.length).to.equal(1) - expect(videos[0].name).to.equal(videosListBase[5].name) - }) + await wait(1500) - it('Should list and sort by name in descending order', async function () { - const res = await getVideosListSort(server.url, '-name') - - const videos = res.body.data - expect(res.body.total).to.equal(6) - expect(videos.length).to.equal(6) - expect(videos[0].name).to.equal('video_short.webm name') - expect(videos[1].name).to.equal('video_short.ogv name') - expect(videos[2].name).to.equal('video_short.mp4 name') - expect(videos[3].name).to.equal('video_short3.webm name') - expect(videos[4].name).to.equal('video_short2.webm name') - expect(videos[5].name).to.equal('video_short1.webm name') - - videoId = videos[3].uuid - videoId2 = videos[5].uuid - }) + await viewVideo(server.url, videoId) + await viewVideo(server.url, videoId) - it('Should list and sort by trending in descending order', async function () { - const res = await getVideosListPagination(server.url, 0, 2, '-trending') + // Wait the repeatable job + await wait(8000) - const videos = res.body.data - expect(res.body.total).to.equal(6) - expect(videos.length).to.equal(2) - }) + const res = await getVideo(server.url, videoId) - it('Should list and sort by hotness in descending order', async function () { - const res = await getVideosListPagination(server.url, 0, 2, '-hot') + const video = res.body + expect(video.views).to.equal(3) + }) - const videos = res.body.data - expect(res.body.total).to.equal(6) - expect(videos.length).to.equal(2) - }) + it('Should remove the video', async function () { + await removeVideo(server.url, server.accessToken, videoId) - it('Should list and sort by best in descending order', async function () { - const res = await getVideosListPagination(server.url, 0, 2, '-best') + await checkVideoFilesWereRemoved(videoUUID, 1) + }) - const videos = res.body.data - expect(res.body.total).to.equal(6) - expect(videos.length).to.equal(2) - }) + it('Should not have videos', async function () { + const res = await getVideosList(server.url) - it('Should update a video', async function () { - const attributes = { - name: 'my super video updated', - category: 4, - licence: 2, - language: 'ar', - nsfw: false, - description: 'my super description updated', - commentsEnabled: false, - downloadEnabled: false, - tags: [ 'tagup1', 'tagup2' ] - } - await updateVideo(server.url, server.accessToken, videoId, attributes) - }) + expect(res.body.total).to.equal(0) + expect(res.body.data).to.be.an('array') + expect(res.body.data).to.have.lengthOf(0) + }) - it('Should filter by tags and category', async function () { - const res1 = await getVideosWithFilters(server.url, { tagsAllOf: [ 'tagup1', 'tagup2' ], categoryOneOf: [ 4 ] }) - expect(res1.body.total).to.equal(1) - expect(res1.body.data[0].name).to.equal('my super video updated') + it('Should upload 6 videos', async function () { + this.timeout(25000) - const res2 = await getVideosWithFilters(server.url, { tagsAllOf: [ 'tagup1', 'tagup2' ], categoryOneOf: [ 3 ] }) - expect(res2.body.total).to.equal(0) - }) + const videos = new Set([ + 'video_short.mp4', 'video_short.ogv', 'video_short.webm', + 'video_short1.webm', 'video_short2.webm', 'video_short3.webm' + ]) - it('Should have the video updated', async function () { - this.timeout(60000) + for (const video of videos) { + const videoAttributes = { + name: video + ' name', + description: video + ' description', + category: 2, + licence: 1, + language: 'en', + nsfw: true, + tags: [ 'tag1', 'tag2', 'tag3' ], + fixture: video + } - const res = await getVideo(server.url, videoId) - const video = res.body + await uploadVideo(server.url, server.accessToken, videoAttributes, HttpStatusCode.OK_200, mode) + } + }) + + it('Should have the correct durations', async function () { + const res = await getVideosList(server.url) + + expect(res.body.total).to.equal(6) + const videos = res.body.data + expect(videos).to.be.an('array') + expect(videos).to.have.lengthOf(6) + + const videosByName = keyBy<{ duration: number }>(videos, 'name') + expect(videosByName['video_short.mp4 name'].duration).to.equal(5) + expect(videosByName['video_short.ogv name'].duration).to.equal(5) + expect(videosByName['video_short.webm name'].duration).to.equal(5) + expect(videosByName['video_short1.webm name'].duration).to.equal(10) + expect(videosByName['video_short2.webm name'].duration).to.equal(5) + expect(videosByName['video_short3.webm name'].duration).to.equal(5) + }) + + it('Should have the correct thumbnails', async function () { + const res = await getVideosList(server.url) + + const videos = res.body.data + // For the next test + videosListBase = videos + + for (const video of videos) { + const videoName = video.name.replace(' name', '') + await testImage(server.url, videoName, video.thumbnailPath) + } + }) + + it('Should list only the two first videos', async function () { + const res = await getVideosListPagination(server.url, 0, 2, 'name') + + const videos = res.body.data + expect(res.body.total).to.equal(6) + expect(videos.length).to.equal(2) + expect(videos[0].name).to.equal(videosListBase[0].name) + expect(videos[1].name).to.equal(videosListBase[1].name) + }) + + it('Should list only the next three videos', async function () { + const res = await getVideosListPagination(server.url, 2, 3, 'name') + + const videos = res.body.data + expect(res.body.total).to.equal(6) + expect(videos.length).to.equal(3) + expect(videos[0].name).to.equal(videosListBase[2].name) + expect(videos[1].name).to.equal(videosListBase[3].name) + expect(videos[2].name).to.equal(videosListBase[4].name) + }) + + it('Should list the last video', async function () { + const res = await getVideosListPagination(server.url, 5, 6, 'name') + + const videos = res.body.data + expect(res.body.total).to.equal(6) + expect(videos.length).to.equal(1) + expect(videos[0].name).to.equal(videosListBase[5].name) + }) + + it('Should not have the total field', async function () { + const res = await getVideosListPagination(server.url, 5, 6, 'name', true) + + const videos = res.body.data + expect(res.body.total).to.not.exist + expect(videos.length).to.equal(1) + expect(videos[0].name).to.equal(videosListBase[5].name) + }) + + it('Should list and sort by name in descending order', async function () { + const res = await getVideosListSort(server.url, '-name') + + const videos = res.body.data + expect(res.body.total).to.equal(6) + expect(videos.length).to.equal(6) + expect(videos[0].name).to.equal('video_short.webm name') + expect(videos[1].name).to.equal('video_short.ogv name') + expect(videos[2].name).to.equal('video_short.mp4 name') + expect(videos[3].name).to.equal('video_short3.webm name') + expect(videos[4].name).to.equal('video_short2.webm name') + expect(videos[5].name).to.equal('video_short1.webm name') + + videoId = videos[3].uuid + videoId2 = videos[5].uuid + }) + + it('Should list and sort by trending in descending order', async function () { + const res = await getVideosListPagination(server.url, 0, 2, '-trending') + + const videos = res.body.data + expect(res.body.total).to.equal(6) + expect(videos.length).to.equal(2) + }) + + it('Should list and sort by hotness in descending order', async function () { + const res = await getVideosListPagination(server.url, 0, 2, '-hot') + + const videos = res.body.data + expect(res.body.total).to.equal(6) + expect(videos.length).to.equal(2) + }) + + it('Should list and sort by best in descending order', async function () { + const res = await getVideosListPagination(server.url, 0, 2, '-best') + + const videos = res.body.data + expect(res.body.total).to.equal(6) + expect(videos.length).to.equal(2) + }) + + it('Should update a video', async function () { + const attributes = { + name: 'my super video updated', + category: 4, + licence: 2, + language: 'ar', + nsfw: false, + description: 'my super description updated', + commentsEnabled: false, + downloadEnabled: false, + tags: [ 'tagup1', 'tagup2' ] + } + await updateVideo(server.url, server.accessToken, videoId, attributes) + }) - await completeVideoCheck(server.url, video, updateCheckAttributes()) - }) + it('Should filter by tags and category', async function () { + const res1 = await getVideosWithFilters(server.url, { tagsAllOf: [ 'tagup1', 'tagup2' ], categoryOneOf: [ 4 ] }) + expect(res1.body.total).to.equal(1) + expect(res1.body.data[0].name).to.equal('my super video updated') - it('Should update only the tags of a video', async function () { - const attributes = { - tags: [ 'supertag', 'tag1', 'tag2' ] - } - await updateVideo(server.url, server.accessToken, videoId, attributes) + const res2 = await getVideosWithFilters(server.url, { tagsAllOf: [ 'tagup1', 'tagup2' ], categoryOneOf: [ 3 ] }) + expect(res2.body.total).to.equal(0) + }) - const res = await getVideo(server.url, videoId) - const video = res.body + it('Should have the video updated', async function () { + this.timeout(60000) - await completeVideoCheck(server.url, video, Object.assign(updateCheckAttributes(), attributes)) - }) + const res = await getVideo(server.url, videoId) + const video = res.body - it('Should update only the description of a video', async function () { - const attributes = { - description: 'hello everybody' - } - await updateVideo(server.url, server.accessToken, videoId, attributes) + await completeVideoCheck(server.url, video, updateCheckAttributes()) + }) - const res = await getVideo(server.url, videoId) - const video = res.body + it('Should update only the tags of a video', async function () { + const attributes = { + tags: [ 'supertag', 'tag1', 'tag2' ] + } + await updateVideo(server.url, server.accessToken, videoId, attributes) - const expectedAttributes = Object.assign(updateCheckAttributes(), { tags: [ 'supertag', 'tag1', 'tag2' ] }, attributes) - await completeVideoCheck(server.url, video, expectedAttributes) - }) + const res = await getVideo(server.url, videoId) + const video = res.body - it('Should like a video', async function () { - await rateVideo(server.url, server.accessToken, videoId, 'like') + await completeVideoCheck(server.url, video, Object.assign(updateCheckAttributes(), attributes)) + }) - const res = await getVideo(server.url, videoId) - const video = res.body + it('Should update only the description of a video', async function () { + const attributes = { + description: 'hello everybody' + } + await updateVideo(server.url, server.accessToken, videoId, attributes) - expect(video.likes).to.equal(1) - expect(video.dislikes).to.equal(0) - }) + const res = await getVideo(server.url, videoId) + const video = res.body - it('Should dislike the same video', async function () { - await rateVideo(server.url, server.accessToken, videoId, 'dislike') + const expectedAttributes = Object.assign(updateCheckAttributes(), { tags: [ 'supertag', 'tag1', 'tag2' ] }, attributes) + await completeVideoCheck(server.url, video, expectedAttributes) + }) - const res = await getVideo(server.url, videoId) - const video = res.body + it('Should like a video', async function () { + await rateVideo(server.url, server.accessToken, videoId, 'like') - expect(video.likes).to.equal(0) - expect(video.dislikes).to.equal(1) - }) + const res = await getVideo(server.url, videoId) + const video = res.body - it('Should sort by originallyPublishedAt', async function () { - { + expect(video.likes).to.equal(1) + expect(video.dislikes).to.equal(0) + }) + it('Should dislike the same video', async function () { + await rateVideo(server.url, server.accessToken, videoId, 'dislike') + + const res = await getVideo(server.url, videoId) + const video = res.body + + expect(video.likes).to.equal(0) + expect(video.dislikes).to.equal(1) + }) + + it('Should sort by originallyPublishedAt', async function () { { const now = new Date() const attributes = { originallyPublishedAt: now.toISOString() } @@ -483,10 +485,18 @@ describe('Test a single server', function () { expect(names[4]).to.equal('video_short.ogv name') expect(names[5]).to.equal('video_short.mp4 name') } - } + }) + + after(async function () { + await cleanupTests([ server ]) + }) + } + + describe('Legacy upload', function () { + runSuite('legacy') }) - after(async function () { - await cleanupTests([ server ]) + describe('Resumable upload', function () { + runSuite('resumable') }) }) diff --git a/server/tests/api/videos/video-transcoder.ts b/server/tests/api/videos/video-transcoder.ts index 1c99f26df..ea5ffd239 100644 --- a/server/tests/api/videos/video-transcoder.ts +++ b/server/tests/api/videos/video-transcoder.ts @@ -361,106 +361,117 @@ describe('Test video transcoding', function () { describe('Audio upload', function () { - before(async function () { - await updateCustomSubConfig(servers[1].url, servers[1].accessToken, { - transcoding: { - hls: { enabled: true }, - webtorrent: { enabled: true }, - resolutions: { - '0p': false, - '240p': false, - '360p': false, - '480p': false, - '720p': false, - '1080p': false, - '1440p': false, - '2160p': false + function runSuite (mode: 'legacy' | 'resumable') { + + before(async function () { + await updateCustomSubConfig(servers[1].url, servers[1].accessToken, { + transcoding: { + hls: { enabled: true }, + webtorrent: { enabled: true }, + resolutions: { + '0p': false, + '240p': false, + '360p': false, + '480p': false, + '720p': false, + '1080p': false, + '1440p': false, + '2160p': false + } } - } + }) }) - }) - - it('Should merge an audio file with the preview file', async function () { - this.timeout(60_000) - - const videoAttributesArg = { name: 'audio_with_preview', previewfile: 'preview.jpg', fixture: 'sample.ogg' } - await uploadVideo(servers[1].url, servers[1].accessToken, videoAttributesArg) - await waitJobs(servers) + it('Should merge an audio file with the preview file', async function () { + this.timeout(60_000) - for (const server of servers) { - const res = await getVideosList(server.url) + const videoAttributesArg = { name: 'audio_with_preview', previewfile: 'preview.jpg', fixture: 'sample.ogg' } + await uploadVideo(servers[1].url, servers[1].accessToken, videoAttributesArg, HttpStatusCode.OK_200, mode) - const video = res.body.data.find(v => v.name === 'audio_with_preview') - const res2 = await getVideo(server.url, video.id) - const videoDetails: VideoDetails = res2.body + await waitJobs(servers) - expect(videoDetails.files).to.have.lengthOf(1) + for (const server of servers) { + const res = await getVideosList(server.url) - await makeGetRequest({ url: server.url, path: videoDetails.thumbnailPath, statusCodeExpected: HttpStatusCode.OK_200 }) - await makeGetRequest({ url: server.url, path: videoDetails.previewPath, statusCodeExpected: HttpStatusCode.OK_200 }) + const video = res.body.data.find(v => v.name === 'audio_with_preview') + const res2 = await getVideo(server.url, video.id) + const videoDetails: VideoDetails = res2.body - const magnetUri = videoDetails.files[0].magnetUri - expect(magnetUri).to.contain('.mp4') - } - }) + expect(videoDetails.files).to.have.lengthOf(1) - it('Should upload an audio file and choose a default background image', async function () { - this.timeout(60_000) + await makeGetRequest({ url: server.url, path: videoDetails.thumbnailPath, statusCodeExpected: HttpStatusCode.OK_200 }) + await makeGetRequest({ url: server.url, path: videoDetails.previewPath, statusCodeExpected: HttpStatusCode.OK_200 }) - const videoAttributesArg = { name: 'audio_without_preview', fixture: 'sample.ogg' } - await uploadVideo(servers[1].url, servers[1].accessToken, videoAttributesArg) + const magnetUri = videoDetails.files[0].magnetUri + expect(magnetUri).to.contain('.mp4') + } + }) - await waitJobs(servers) + it('Should upload an audio file and choose a default background image', async function () { + this.timeout(60_000) - for (const server of servers) { - const res = await getVideosList(server.url) + const videoAttributesArg = { name: 'audio_without_preview', fixture: 'sample.ogg' } + await uploadVideo(servers[1].url, servers[1].accessToken, videoAttributesArg, HttpStatusCode.OK_200, mode) - const video = res.body.data.find(v => v.name === 'audio_without_preview') - const res2 = await getVideo(server.url, video.id) - const videoDetails = res2.body + await waitJobs(servers) - expect(videoDetails.files).to.have.lengthOf(1) + for (const server of servers) { + const res = await getVideosList(server.url) - await makeGetRequest({ url: server.url, path: videoDetails.thumbnailPath, statusCodeExpected: HttpStatusCode.OK_200 }) - await makeGetRequest({ url: server.url, path: videoDetails.previewPath, statusCodeExpected: HttpStatusCode.OK_200 }) + const video = res.body.data.find(v => v.name === 'audio_without_preview') + const res2 = await getVideo(server.url, video.id) + const videoDetails = res2.body - const magnetUri = videoDetails.files[0].magnetUri - expect(magnetUri).to.contain('.mp4') - } - }) + expect(videoDetails.files).to.have.lengthOf(1) - it('Should upload an audio file and create an audio version only', async function () { - this.timeout(60_000) + await makeGetRequest({ url: server.url, path: videoDetails.thumbnailPath, statusCodeExpected: HttpStatusCode.OK_200 }) + await makeGetRequest({ url: server.url, path: videoDetails.previewPath, statusCodeExpected: HttpStatusCode.OK_200 }) - await updateCustomSubConfig(servers[1].url, servers[1].accessToken, { - transcoding: { - hls: { enabled: true }, - webtorrent: { enabled: true }, - resolutions: { - '0p': true, - '240p': false, - '360p': false - } + const magnetUri = videoDetails.files[0].magnetUri + expect(magnetUri).to.contain('.mp4') } }) - const videoAttributesArg = { name: 'audio_with_preview', previewfile: 'preview.jpg', fixture: 'sample.ogg' } - const resVideo = await uploadVideo(servers[1].url, servers[1].accessToken, videoAttributesArg) + it('Should upload an audio file and create an audio version only', async function () { + this.timeout(60_000) + + await updateCustomSubConfig(servers[1].url, servers[1].accessToken, { + transcoding: { + hls: { enabled: true }, + webtorrent: { enabled: true }, + resolutions: { + '0p': true, + '240p': false, + '360p': false + } + } + }) - await waitJobs(servers) + const videoAttributesArg = { name: 'audio_with_preview', previewfile: 'preview.jpg', fixture: 'sample.ogg' } + const resVideo = await uploadVideo(servers[1].url, servers[1].accessToken, videoAttributesArg, HttpStatusCode.OK_200, mode) - for (const server of servers) { - const res2 = await getVideo(server.url, resVideo.body.video.id) - const videoDetails: VideoDetails = res2.body + await waitJobs(servers) + + for (const server of servers) { + const res2 = await getVideo(server.url, resVideo.body.video.id) + const videoDetails: VideoDetails = res2.body - for (const files of [ videoDetails.files, videoDetails.streamingPlaylists[0].files ]) { - expect(files).to.have.lengthOf(2) - expect(files.find(f => f.resolution.id === 0)).to.not.be.undefined + for (const files of [ videoDetails.files, videoDetails.streamingPlaylists[0].files ]) { + expect(files).to.have.lengthOf(2) + expect(files.find(f => f.resolution.id === 0)).to.not.be.undefined + } } - } - await updateConfigForTranscoding(servers[1]) + await updateConfigForTranscoding(servers[1]) + }) + } + + describe('Legacy upload', function () { + runSuite('legacy') + }) + + describe('Resumable upload', function () { + runSuite('resumable') }) }) diff --git a/server/typings/express/index.d.ts b/server/typings/express/index.d.ts index cf3e7ae34..55b6e0039 100644 --- a/server/typings/express/index.d.ts +++ b/server/typings/express/index.d.ts @@ -19,6 +19,9 @@ import { MPlugin, MServer, MServerBlocklist } from '@server/types/models/server' import { MVideoImportDefault } from '@server/types/models/video/video-import' import { MVideoPlaylistElement, MVideoPlaylistElementVideoUrlPlaylistPrivacy } from '@server/types/models/video/video-playlist-element' import { MAccountVideoRateAccountVideo } from '@server/types/models/video/video-rate' +import { HttpMethod } from '@shared/core-utils/miscs/http-methods' +import { VideoCreate } from '@shared/models' +import { File as UploadXFile, Metadata } from '@uploadx/core' import { RegisteredPlugin } from '../../lib/plugins/plugin-manager' import { MAccountDefault, @@ -37,86 +40,125 @@ import { MVideoThumbnail, MVideoWithRights } from '../../types/models' - declare module 'express' { export interface Request { query: any + method: HttpMethod } - interface Response { - locals: PeerTubeLocals + + // Upload using multer or uploadx middleware + export type MulterOrUploadXFile = UploadXFile | Express.Multer.File + + export type UploadFiles = { + [fieldname: string]: MulterOrUploadXFile[] + } | MulterOrUploadXFile[] + + // Partial object used by some functions to check the file mimetype/extension + export type UploadFileForCheck = { + originalname: string + mimetype: string } -} -interface PeerTubeLocals { - videoAll?: MVideoFullLight - onlyImmutableVideo?: MVideoImmutable - onlyVideo?: MVideoThumbnail - onlyVideoWithRights?: MVideoWithRights - videoId?: MVideoIdThumbnail + export type UploadFilesForCheck = { + [fieldname: string]: UploadFileForCheck[] + } | UploadFileForCheck[] - videoLive?: MVideoLive + // Upload file with a duration added by our middleware + export type VideoUploadFile = Pick & { + duration: number + } - videoShare?: MVideoShareActor + // Extends Metadata property of UploadX object + export type UploadXFileMetadata = Metadata & VideoCreate & { + previewfile: Express.Multer.File[] + thumbnailfile: Express.Multer.File[] + } - videoFile?: MVideoFile + // Our custom UploadXFile object using our custom metadata + export type CustomUploadXFile = UploadXFile & { metadata: T } - videoImport?: MVideoImportDefault + export type EnhancedUploadXFile = CustomUploadXFile & { + duration: number + path: string + filename: string + } - videoBlacklist?: MVideoBlacklist + // Extends locals property from Response + interface Response { + locals: { + videoAll?: MVideoFullLight + onlyImmutableVideo?: MVideoImmutable + onlyVideo?: MVideoThumbnail + onlyVideoWithRights?: MVideoWithRights + videoId?: MVideoIdThumbnail - videoCaption?: MVideoCaptionVideo + videoLive?: MVideoLive - abuse?: MAbuseReporter - abuseMessage?: MAbuseMessage + videoShare?: MVideoShareActor - videoStreamingPlaylist?: MStreamingPlaylist + videoFile?: MVideoFile - videoChannel?: MChannelBannerAccountDefault + videoFileResumable?: EnhancedUploadXFile - videoPlaylistFull?: MVideoPlaylistFull - videoPlaylistSummary?: MVideoPlaylistFullSummary + videoImport?: MVideoImportDefault - videoPlaylistElement?: MVideoPlaylistElement - videoPlaylistElementAP?: MVideoPlaylistElementVideoUrlPlaylistPrivacy + videoBlacklist?: MVideoBlacklist - accountVideoRate?: MAccountVideoRateAccountVideo + videoCaption?: MVideoCaptionVideo - videoCommentFull?: MCommentOwnerVideoReply - videoCommentThread?: MComment + abuse?: MAbuseReporter + abuseMessage?: MAbuseMessage - follow?: MActorFollowActorsDefault - subscription?: MActorFollowActorsDefaultSubscription + videoStreamingPlaylist?: MStreamingPlaylist - nextOwner?: MAccountDefault - videoChangeOwnership?: MVideoChangeOwnershipFull + videoChannel?: MChannelBannerAccountDefault - account?: MAccountDefault + videoPlaylistFull?: MVideoPlaylistFull + videoPlaylistSummary?: MVideoPlaylistFullSummary - actorUrl?: MActorUrl - actorFull?: MActorFull + videoPlaylistElement?: MVideoPlaylistElement + videoPlaylistElementAP?: MVideoPlaylistElementVideoUrlPlaylistPrivacy - user?: MUserDefault + accountVideoRate?: MAccountVideoRateAccountVideo - server?: MServer + videoCommentFull?: MCommentOwnerVideoReply + videoCommentThread?: MComment - videoRedundancy?: MVideoRedundancyVideo + follow?: MActorFollowActorsDefault + subscription?: MActorFollowActorsDefaultSubscription - accountBlock?: MAccountBlocklist - serverBlock?: MServerBlocklist + nextOwner?: MAccountDefault + videoChangeOwnership?: MVideoChangeOwnershipFull - oauth?: { - token: MOAuthTokenUser - } + account?: MAccountDefault - signature?: { - actor: MActorAccountChannelId - } + actorUrl?: MActorUrl + actorFull?: MActorFull + + user?: MUserDefault + + server?: MServer + + videoRedundancy?: MVideoRedundancyVideo - authenticated?: boolean + accountBlock?: MAccountBlocklist + serverBlock?: MServerBlocklist - registeredPlugin?: RegisteredPlugin + oauth?: { + token: MOAuthTokenUser + } - externalAuth?: RegisterServerAuthExternalOptions + signature?: { + actor: MActorAccountChannelId + } - plugin?: MPlugin + authenticated?: boolean + + registeredPlugin?: RegisteredPlugin + + externalAuth?: RegisterServerAuthExternalOptions + + plugin?: MPlugin + } + } } -- cgit v1.2.3