From f6d6e7f861189a4446f406efb775a29688764b48 Mon Sep 17 00:00:00 2001 From: kontrollanten <6680299+kontrollanten@users.noreply.github.com> Date: Mon, 10 May 2021 11:13:41 +0200 Subject: Resumable video uploads (#3933) * WIP: resumable video uploads relates to #324 * fix review comments * video upload: error handling * fix audio upload * fixes after self review * Update server/controllers/api/videos/index.ts Co-authored-by: Rigel Kent * Update server/middlewares/validators/videos/videos.ts Co-authored-by: Rigel Kent * Update server/controllers/api/videos/index.ts Co-authored-by: Rigel Kent * update after code review * refactor upload route - restore multipart upload route - move resumable to dedicated upload-resumable route - move checks to middleware - do not leak internal fs structure in response * fix yarn.lock upon rebase * factorize addVideo for reuse in both endpoints * add resumable upload API to openapi spec * add initial test and test helper for resumable upload * typings for videoAddResumable middleware * avoid including aws and google packages via node-uploadx, by only including uploadx/core * rename ex-isAudioBg to more explicit name mentioning it is a preview file for audio * add video-upload-tmp-folder-cleaner job * stronger typing of video upload middleware * reduce dependency to @uploadx/core * add audio upload test * refactor resumable uploads cleanup from job to scheduler * refactor resumable uploads scheduler to compare to last execution time * make resumable upload validator to always cleanup on failure * move legacy upload request building outside of uploadVideo test helper * filter upload-resumable middlewares down to POST, PUT, DELETE also begin to type metadata * merge add duration functions * stronger typings and documentation for uploadx behaviour, move init validator up * refactor(client/video-edit): options > uploadxOptions * refactor(client/video-edit): remove obsolete else * scheduler/remove-dangling-resum: rename tag * refactor(server/video): add UploadVideoFiles type * refactor(mw/validators): restructure eslint disable * refactor(mw/validators/videos): rename import * refactor(client/vid-upload): rename html elem id * refactor(sched/remove-dangl): move fn to method * refactor(mw/async): add method typing * refactor(mw/vali/video): double quote > single * refactor(server/upload-resum): express use > all * proper http methud enum server/middlewares/async.ts * properly type http methods * factorize common video upload validation steps * add check for maximum partially uploaded file size * fix audioBg use * fix extname(filename) in addVideo * document parameters for uploadx's resumable protocol * clear META files in scheduler * last audio refactor before cramming preview in the initial POST form data * refactor as mulitpart/form-data initial post request this allows preview/thumbnail uploads alongside the initial request, and cleans up the upload form * Add more tests for resumable uploads * Refactor remove dangling resumable uploads * Prepare changelog * Add more resumable upload tests * Remove user quota check for resumable uploads * Fix upload error handler * Update nginx template for upload-resumable * Cleanup comment * Remove unused express methods * Prefer to use got instead of raw http * Don't retry on error 500 Co-authored-by: Rigel Kent Co-authored-by: Rigel Kent Co-authored-by: Chocobozzz --- server/middlewares/validators/videos/videos.ts | 184 ++++++++++++++++++++----- 1 file changed, 152 insertions(+), 32 deletions(-) (limited to 'server/middlewares/validators') diff --git a/server/middlewares/validators/videos/videos.ts b/server/middlewares/validators/videos/videos.ts index bb617d77c..d26bcd4a6 100644 --- a/server/middlewares/validators/videos/videos.ts +++ b/server/middlewares/validators/videos/videos.ts @@ -1,9 +1,10 @@ import * as express from 'express' -import { body, param, query, ValidationChain } from 'express-validator' +import { body, header, param, query, ValidationChain } from 'express-validator' +import { getResumableUploadPath } from '@server/helpers/upload' import { isAbleToUploadVideo } from '@server/lib/user' import { getServerActor } from '@server/models/application/application' import { ExpressPromiseHandler } from '@server/types/express' -import { MVideoWithRights } from '@server/types/models' +import { MUserAccountId, MVideoWithRights } from '@server/types/models' import { ServerErrorCode, UserRight, VideoChangeOwnershipStatus, VideoPrivacy } from '../../../../shared' import { HttpStatusCode } from '../../../../shared/core-utils/miscs/http-error-codes' import { VideoChangeOwnershipAccept } from '../../../../shared/models/videos/video-change-ownership-accept.model' @@ -47,6 +48,7 @@ import { doesVideoExist, doesVideoFileOfVideoExist } from '../../../helpers/middlewares' +import { deleteFileAndCatch } from '../../../helpers/utils' import { getVideoWithAttributes } from '../../../helpers/video' import { CONFIG } from '../../../initializers/config' import { CONSTRAINTS_FIELDS, OVERVIEWS } from '../../../initializers/constants' @@ -57,7 +59,7 @@ import { VideoModel } from '../../../models/video/video' import { authenticatePromiseIfNeeded } from '../../auth' import { areValidationErrors } from '../utils' -const videosAddValidator = getCommonVideoEditAttributes().concat([ +const videosAddLegacyValidator = getCommonVideoEditAttributes().concat([ body('videofile') .custom((value, { req }) => isFileFieldValid(req.files, 'videofile')) .withMessage('Should have a file'), @@ -73,54 +75,117 @@ const videosAddValidator = getCommonVideoEditAttributes().concat([ logger.debug('Checking videosAdd parameters', { parameters: req.body, files: req.files }) if (areValidationErrors(req, res)) return cleanUpReqFiles(req) - if (areErrorsInScheduleUpdate(req, res)) return cleanUpReqFiles(req) - const videoFile: Express.Multer.File & { duration?: number } = req.files['videofile'][0] + const videoFile: express.VideoUploadFile = req.files['videofile'][0] const user = res.locals.oauth.token.User - if (!await doesVideoChannelOfAccountExist(req.body.channelId, user, res)) return cleanUpReqFiles(req) - - if (!isVideoFileMimeTypeValid(req.files)) { - res.status(HttpStatusCode.UNSUPPORTED_MEDIA_TYPE_415) - .json({ - error: 'This file is not supported. Please, make sure it is of the following type: ' + - CONSTRAINTS_FIELDS.VIDEOS.EXTNAME.join(', ') - }) - + if (!await commonVideoChecksPass({ req, res, user, videoFileSize: videoFile.size, files: req.files })) { return cleanUpReqFiles(req) } - if (!isVideoFileSizeValid(videoFile.size.toString())) { - res.status(HttpStatusCode.PAYLOAD_TOO_LARGE_413) - .json({ - error: 'This file is too large.' - }) + try { + if (!videoFile.duration) await addDurationToVideo(videoFile) + } catch (err) { + logger.error('Invalid input file in videosAddLegacyValidator.', { err }) + res.status(HttpStatusCode.UNPROCESSABLE_ENTITY_422) + .json({ error: 'Video file unreadable.' }) return cleanUpReqFiles(req) } - if (await isAbleToUploadVideo(user.id, videoFile.size) === false) { - res.status(HttpStatusCode.PAYLOAD_TOO_LARGE_413) - .json({ error: 'The user video quota is exceeded with this video.' }) + if (!await isVideoAccepted(req, res, videoFile)) return cleanUpReqFiles(req) - return cleanUpReqFiles(req) - } + return next() + } +]) + +/** + * Gets called after the last PUT request + */ +const videosAddResumableValidator = [ + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + const user = res.locals.oauth.token.User + + const body: express.CustomUploadXFile = req.body + const file = { ...body, duration: undefined, path: getResumableUploadPath(body.id), filename: body.metadata.filename } + + const cleanup = () => deleteFileAndCatch(file.path) - let duration: number + if (!await doesVideoChannelOfAccountExist(file.metadata.channelId, user, res)) return cleanup() try { - duration = await getDurationFromVideoFile(videoFile.path) + if (!file.duration) await addDurationToVideo(file) } catch (err) { - logger.error('Invalid input file in videosAddValidator.', { err }) + logger.error('Invalid input file in videosAddResumableValidator.', { err }) res.status(HttpStatusCode.UNPROCESSABLE_ENTITY_422) .json({ error: 'Video file unreadable.' }) - return cleanUpReqFiles(req) + return cleanup() } - videoFile.duration = duration + if (!await isVideoAccepted(req, res, file)) return cleanup() - if (!await isVideoAccepted(req, res, videoFile)) return cleanUpReqFiles(req) + res.locals.videoFileResumable = file + + return next() + } +] + +/** + * File is created in POST initialisation, and its body is saved as a 'metadata' field is saved by uploadx for later use. + * see https://github.com/kukhariev/node-uploadx/blob/dc9fb4a8ac5a6f481902588e93062f591ec6ef03/packages/core/src/handlers/uploadx.ts + * + * Uploadx doesn't use next() until the upload completes, so this middleware has to be placed before uploadx + * see https://github.com/kukhariev/node-uploadx/blob/dc9fb4a8ac5a6f481902588e93062f591ec6ef03/packages/core/src/handlers/base-handler.ts + * + */ +const videosAddResumableInitValidator = getCommonVideoEditAttributes().concat([ + body('filename') + .isString() + .exists() + .withMessage('Should have a valid filename'), + body('name') + .trim() + .custom(isVideoNameValid) + .withMessage('Should have a valid name'), + body('channelId') + .customSanitizer(toIntOrNull) + .custom(isIdValid).withMessage('Should have correct video channel id'), + + header('x-upload-content-length') + .isNumeric() + .exists() + .withMessage('Should specify the file length'), + header('x-upload-content-type') + .isString() + .exists() + .withMessage('Should specify the file mimetype'), + + async (req: express.Request, res: express.Response, next: express.NextFunction) => { + const videoFileMetadata = { + mimetype: req.headers['x-upload-content-type'] as string, + size: +req.headers['x-upload-content-length'], + originalname: req.body.name + } + + const user = res.locals.oauth.token.User + const cleanup = () => cleanUpReqFiles(req) + + logger.debug('Checking videosAddResumableInitValidator parameters and headers', { + parameters: req.body, + headers: req.headers, + files: req.files + }) + + if (areValidationErrors(req, res)) return cleanup() + + const files = { videofile: [ videoFileMetadata ] } + if (!await commonVideoChecksPass({ req, res, user, videoFileSize: videoFileMetadata.size, files })) return cleanup() + + // multer required unsetting the Content-Type, now we can set it for node-uploadx + req.headers['content-type'] = 'application/json; charset=utf-8' + // place previewfile in metadata so that uploadx saves it in .META + if (req.files['previewfile']) req.body.previewfile = req.files['previewfile'] return next() } @@ -478,7 +543,10 @@ const commonVideosFiltersValidator = [ // --------------------------------------------------------------------------- export { - videosAddValidator, + videosAddLegacyValidator, + videosAddResumableValidator, + videosAddResumableInitValidator, + videosUpdateValidator, videosGetValidator, videoFileMetadataGetValidator, @@ -515,7 +583,51 @@ function areErrorsInScheduleUpdate (req: express.Request, res: express.Response) return false } -async function isVideoAccepted (req: express.Request, res: express.Response, videoFile: Express.Multer.File & { duration?: number }) { +async function commonVideoChecksPass (parameters: { + req: express.Request + res: express.Response + user: MUserAccountId + videoFileSize: number + files: express.UploadFilesForCheck +}): Promise { + const { req, res, user, videoFileSize, files } = parameters + + if (areErrorsInScheduleUpdate(req, res)) return false + + if (!await doesVideoChannelOfAccountExist(req.body.channelId, user, res)) return false + + if (!isVideoFileMimeTypeValid(files)) { + res.status(HttpStatusCode.UNSUPPORTED_MEDIA_TYPE_415) + .json({ + error: 'This file is not supported. Please, make sure it is of the following type: ' + + CONSTRAINTS_FIELDS.VIDEOS.EXTNAME.join(', ') + }) + + return false + } + + if (!isVideoFileSizeValid(videoFileSize.toString())) { + res.status(HttpStatusCode.PAYLOAD_TOO_LARGE_413) + .json({ error: 'This file is too large. It exceeds the maximum file size authorized.' }) + + return false + } + + if (await isAbleToUploadVideo(user.id, videoFileSize) === false) { + res.status(HttpStatusCode.PAYLOAD_TOO_LARGE_413) + .json({ error: 'The user video quota is exceeded with this video.' }) + + return false + } + + return true +} + +export async function isVideoAccepted ( + req: express.Request, + res: express.Response, + videoFile: express.VideoUploadFile +) { // Check we accept this video const acceptParameters = { videoBody: req.body, @@ -538,3 +650,11 @@ async function isVideoAccepted (req: express.Request, res: express.Response, vid return true } + +async function addDurationToVideo (videoFile: { path: string, duration?: number }) { + const duration: number = await getDurationFromVideoFile(videoFile.path) + + if (isNaN(duration)) throw new Error(`Couldn't get video duration`) + + videoFile.duration = duration +} -- cgit v1.2.3