From f6d6e7f861189a4446f406efb775a29688764b48 Mon Sep 17 00:00:00 2001 From: kontrollanten <6680299+kontrollanten@users.noreply.github.com> Date: Mon, 10 May 2021 11:13:41 +0200 Subject: Resumable video uploads (#3933) * WIP: resumable video uploads relates to #324 * fix review comments * video upload: error handling * fix audio upload * fixes after self review * Update server/controllers/api/videos/index.ts Co-authored-by: Rigel Kent * Update server/middlewares/validators/videos/videos.ts Co-authored-by: Rigel Kent * Update server/controllers/api/videos/index.ts Co-authored-by: Rigel Kent * update after code review * refactor upload route - restore multipart upload route - move resumable to dedicated upload-resumable route - move checks to middleware - do not leak internal fs structure in response * fix yarn.lock upon rebase * factorize addVideo for reuse in both endpoints * add resumable upload API to openapi spec * add initial test and test helper for resumable upload * typings for videoAddResumable middleware * avoid including aws and google packages via node-uploadx, by only including uploadx/core * rename ex-isAudioBg to more explicit name mentioning it is a preview file for audio * add video-upload-tmp-folder-cleaner job * stronger typing of video upload middleware * reduce dependency to @uploadx/core * add audio upload test * refactor resumable uploads cleanup from job to scheduler * refactor resumable uploads scheduler to compare to last execution time * make resumable upload validator to always cleanup on failure * move legacy upload request building outside of uploadVideo test helper * filter upload-resumable middlewares down to POST, PUT, DELETE also begin to type metadata * merge add duration functions * stronger typings and documentation for uploadx behaviour, move init validator up * refactor(client/video-edit): options > uploadxOptions * refactor(client/video-edit): remove obsolete else * scheduler/remove-dangling-resum: rename tag * refactor(server/video): add UploadVideoFiles type * refactor(mw/validators): restructure eslint disable * refactor(mw/validators/videos): rename import * refactor(client/vid-upload): rename html elem id * refactor(sched/remove-dangl): move fn to method * refactor(mw/async): add method typing * refactor(mw/vali/video): double quote > single * refactor(server/upload-resum): express use > all * proper http methud enum server/middlewares/async.ts * properly type http methods * factorize common video upload validation steps * add check for maximum partially uploaded file size * fix audioBg use * fix extname(filename) in addVideo * document parameters for uploadx's resumable protocol * clear META files in scheduler * last audio refactor before cramming preview in the initial POST form data * refactor as mulitpart/form-data initial post request this allows preview/thumbnail uploads alongside the initial request, and cleans up the upload form * Add more tests for resumable uploads * Refactor remove dangling resumable uploads * Prepare changelog * Add more resumable upload tests * Remove user quota check for resumable uploads * Fix upload error handler * Update nginx template for upload-resumable * Cleanup comment * Remove unused express methods * Prefer to use got instead of raw http * Don't retry on error 500 Co-authored-by: Rigel Kent Co-authored-by: Rigel Kent Co-authored-by: Chocobozzz --- server/controllers/api/videos/index.ts | 105 ++++++++++++++++++++++++++------- 1 file changed, 85 insertions(+), 20 deletions(-) (limited to 'server/controllers/api/videos/index.ts') diff --git a/server/controllers/api/videos/index.ts b/server/controllers/api/videos/index.ts index fbdb0f776..c32626d30 100644 --- a/server/controllers/api/videos/index.ts +++ b/server/controllers/api/videos/index.ts @@ -2,6 +2,7 @@ import * as express from 'express' import { move } from 'fs-extra' import { extname } from 'path' import toInt from 'validator/lib/toInt' +import { deleteResumableUploadMetaFile, getResumableUploadPath } from '@server/helpers/upload' import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent' import { changeVideoChannelShare } from '@server/lib/activitypub/share' import { getLocalVideoActivityPubUrl } from '@server/lib/activitypub/url' @@ -10,8 +11,9 @@ import { addOptimizeOrMergeAudioJob, buildLocalVideoFromReq, buildVideoThumbnail import { generateVideoFilename, getVideoFilePath } from '@server/lib/video-paths' import { getServerActor } from '@server/models/application/application' import { MVideo, MVideoFile, MVideoFullLight } from '@server/types/models' +import { uploadx } from '@uploadx/core' import { VideoCreate, VideosCommonQuery, VideoState, VideoUpdate } from '../../../../shared' -import { HttpStatusCode } from '../../../../shared/core-utils/miscs/http-error-codes' +import { HttpStatusCode } from '../../../../shared/core-utils/miscs' import { auditLoggerFactory, getAuditIdFromRes, VideoAuditView } from '../../../helpers/audit-logger' import { resetSequelizeInstance, retryTransactionWrapper } from '../../../helpers/database-utils' import { buildNSFWFilter, createReqFiles, getCountVideos } from '../../../helpers/express-utils' @@ -47,7 +49,9 @@ import { setDefaultPagination, setDefaultVideosSort, videoFileMetadataGetValidator, - videosAddValidator, + videosAddLegacyValidator, + videosAddResumableInitValidator, + videosAddResumableValidator, videosCustomGetValidator, videosGetValidator, videosRemoveValidator, @@ -69,6 +73,7 @@ import { watchingRouter } from './watching' const lTags = loggerTagsFactory('api', 'video') const auditLogger = auditLoggerFactory('videos') const videosRouter = express.Router() +const uploadxMiddleware = uploadx.upload({ directory: getResumableUploadPath() }) const reqVideoFileAdd = createReqFiles( [ 'videofile', 'thumbnailfile', 'previewfile' ], @@ -79,6 +84,16 @@ const reqVideoFileAdd = createReqFiles( previewfile: CONFIG.STORAGE.TMP_DIR } ) + +const reqVideoFileAddResumable = createReqFiles( + [ 'thumbnailfile', 'previewfile' ], + MIMETYPES.IMAGE.MIMETYPE_EXT, + { + thumbnailfile: getResumableUploadPath(), + previewfile: getResumableUploadPath() + } +) + const reqVideoFileUpdate = createReqFiles( [ 'thumbnailfile', 'previewfile' ], MIMETYPES.IMAGE.MIMETYPE_EXT, @@ -111,18 +126,39 @@ videosRouter.get('/', commonVideosFiltersValidator, asyncMiddleware(listVideos) ) + +videosRouter.post('/upload', + authenticate, + reqVideoFileAdd, + asyncMiddleware(videosAddLegacyValidator), + asyncRetryTransactionMiddleware(addVideoLegacy) +) + +videosRouter.post('/upload-resumable', + authenticate, + reqVideoFileAddResumable, + asyncMiddleware(videosAddResumableInitValidator), + uploadxMiddleware +) + +videosRouter.delete('/upload-resumable', + authenticate, + uploadxMiddleware +) + +videosRouter.put('/upload-resumable', + authenticate, + uploadxMiddleware, // uploadx doesn't use call next() before the file upload completes + asyncMiddleware(videosAddResumableValidator), + asyncMiddleware(addVideoResumable) +) + videosRouter.put('/:id', authenticate, reqVideoFileUpdate, asyncMiddleware(videosUpdateValidator), asyncRetryTransactionMiddleware(updateVideo) ) -videosRouter.post('/upload', - authenticate, - reqVideoFileAdd, - asyncMiddleware(videosAddValidator), - asyncRetryTransactionMiddleware(addVideo) -) videosRouter.get('/:id/description', asyncMiddleware(videosGetValidator), @@ -157,23 +193,23 @@ export { // --------------------------------------------------------------------------- -function listVideoCategories (req: express.Request, res: express.Response) { +function listVideoCategories (_req: express.Request, res: express.Response) { res.json(VIDEO_CATEGORIES) } -function listVideoLicences (req: express.Request, res: express.Response) { +function listVideoLicences (_req: express.Request, res: express.Response) { res.json(VIDEO_LICENCES) } -function listVideoLanguages (req: express.Request, res: express.Response) { +function listVideoLanguages (_req: express.Request, res: express.Response) { res.json(VIDEO_LANGUAGES) } -function listVideoPrivacies (req: express.Request, res: express.Response) { +function listVideoPrivacies (_req: express.Request, res: express.Response) { res.json(VIDEO_PRIVACIES) } -async function addVideo (req: express.Request, res: express.Response) { +async function addVideoLegacy (req: express.Request, res: express.Response) { // Uploading the video could be long // Set timeout to 10 minutes, as Express's default is 2 minutes req.setTimeout(1000 * 60 * 10, () => { @@ -183,13 +219,42 @@ async function addVideo (req: express.Request, res: express.Response) { const videoPhysicalFile = req.files['videofile'][0] const videoInfo: VideoCreate = req.body + const files = req.files + + return addVideo({ res, videoPhysicalFile, videoInfo, files }) +} + +async function addVideoResumable (_req: express.Request, res: express.Response) { + const videoPhysicalFile = res.locals.videoFileResumable + const videoInfo = videoPhysicalFile.metadata + const files = { previewfile: videoInfo.previewfile } + + // Don't need the meta file anymore + await deleteResumableUploadMetaFile(videoPhysicalFile.path) + + return addVideo({ res, videoPhysicalFile, videoInfo, files }) +} - const videoData = buildLocalVideoFromReq(videoInfo, res.locals.videoChannel.id) - videoData.state = CONFIG.TRANSCODING.ENABLED ? VideoState.TO_TRANSCODE : VideoState.PUBLISHED - videoData.duration = videoPhysicalFile['duration'] // duration was added by a previous middleware +async function addVideo (options: { + res: express.Response + videoPhysicalFile: express.VideoUploadFile + videoInfo: VideoCreate + files: express.UploadFiles +}) { + const { res, videoPhysicalFile, videoInfo, files } = options + const videoChannel = res.locals.videoChannel + const user = res.locals.oauth.token.User + + const videoData = buildLocalVideoFromReq(videoInfo, videoChannel.id) + + videoData.state = CONFIG.TRANSCODING.ENABLED + ? VideoState.TO_TRANSCODE + : VideoState.PUBLISHED + + videoData.duration = videoPhysicalFile.duration // duration was added by a previous middleware const video = new VideoModel(videoData) as MVideoFullLight - video.VideoChannel = res.locals.videoChannel + video.VideoChannel = videoChannel video.url = getLocalVideoActivityPubUrl(video) // We use the UUID, so set the URL after building the object const videoFile = new VideoFileModel({ @@ -217,7 +282,7 @@ async function addVideo (req: express.Request, res: express.Response) { const [ thumbnailModel, previewModel ] = await buildVideoThumbnailsFromReq({ video, - files: req.files, + files, fallback: type => generateVideoMiniature({ video, videoFile, type }) }) @@ -253,7 +318,7 @@ async function addVideo (req: express.Request, res: express.Response) { await autoBlacklistVideoIfNeeded({ video, - user: res.locals.oauth.token.User, + user, isRemote: false, isNew: true, transaction: t @@ -282,7 +347,7 @@ async function addVideo (req: express.Request, res: express.Response) { .catch(err => logger.error('Cannot federate or notify video creation %s', video.url, { err, ...lTags(video.uuid) })) if (video.state === VideoState.TO_TRANSCODE) { - await addOptimizeOrMergeAudioJob(videoCreated, videoFile, res.locals.oauth.token.User) + await addOptimizeOrMergeAudioJob(videoCreated, videoFile, user) } Hooks.runAction('action:api.video.uploaded', { video: videoCreated }) -- cgit v1.2.3