aboutsummaryrefslogtreecommitdiffhomepage
path: root/server/controllers
diff options
context:
space:
mode:
authorkontrollanten <6680299+kontrollanten@users.noreply.github.com>2021-05-10 11:13:41 +0200
committerGitHub <noreply@github.com>2021-05-10 11:13:41 +0200
commitf6d6e7f861189a4446f406efb775a29688764b48 (patch)
treec3dda9958c3f189d4c39e8743c738d8c1fef4c2d /server/controllers
parentd29ced1a8582d99b776f664475a157adcf555d98 (diff)
downloadPeerTube-f6d6e7f861189a4446f406efb775a29688764b48.tar.gz
PeerTube-f6d6e7f861189a4446f406efb775a29688764b48.tar.zst
PeerTube-f6d6e7f861189a4446f406efb775a29688764b48.zip
Resumable video uploads (#3933)
* WIP: resumable video uploads relates to #324 * fix review comments * video upload: error handling * fix audio upload * fixes after self review * Update server/controllers/api/videos/index.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * Update server/middlewares/validators/videos/videos.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * Update server/controllers/api/videos/index.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * update after code review * refactor upload route - restore multipart upload route - move resumable to dedicated upload-resumable route - move checks to middleware - do not leak internal fs structure in response * fix yarn.lock upon rebase * factorize addVideo for reuse in both endpoints * add resumable upload API to openapi spec * add initial test and test helper for resumable upload * typings for videoAddResumable middleware * avoid including aws and google packages via node-uploadx, by only including uploadx/core * rename ex-isAudioBg to more explicit name mentioning it is a preview file for audio * add video-upload-tmp-folder-cleaner job * stronger typing of video upload middleware * reduce dependency to @uploadx/core * add audio upload test * refactor resumable uploads cleanup from job to scheduler * refactor resumable uploads scheduler to compare to last execution time * make resumable upload validator to always cleanup on failure * move legacy upload request building outside of uploadVideo test helper * filter upload-resumable middlewares down to POST, PUT, DELETE also begin to type metadata * merge add duration functions * stronger typings and documentation for uploadx behaviour, move init validator up * refactor(client/video-edit): options > uploadxOptions * refactor(client/video-edit): remove obsolete else * scheduler/remove-dangling-resum: rename tag * refactor(server/video): add UploadVideoFiles type * refactor(mw/validators): restructure eslint disable * refactor(mw/validators/videos): rename import * refactor(client/vid-upload): rename html elem id * refactor(sched/remove-dangl): move fn to method * refactor(mw/async): add method typing * refactor(mw/vali/video): double quote > single * refactor(server/upload-resum): express use > all * proper http methud enum server/middlewares/async.ts * properly type http methods * factorize common video upload validation steps * add check for maximum partially uploaded file size * fix audioBg use * fix extname(filename) in addVideo * document parameters for uploadx's resumable protocol * clear META files in scheduler * last audio refactor before cramming preview in the initial POST form data * refactor as mulitpart/form-data initial post request this allows preview/thumbnail uploads alongside the initial request, and cleans up the upload form * Add more tests for resumable uploads * Refactor remove dangling resumable uploads * Prepare changelog * Add more resumable upload tests * Remove user quota check for resumable uploads * Fix upload error handler * Update nginx template for upload-resumable * Cleanup comment * Remove unused express methods * Prefer to use got instead of raw http * Don't retry on error 500 Co-authored-by: Rigel Kent <par@rigelk.eu> Co-authored-by: Rigel Kent <sendmemail@rigelk.eu> Co-authored-by: Chocobozzz <me@florianbigard.com>
Diffstat (limited to 'server/controllers')
-rw-r--r--server/controllers/api/server/debug.ts18
-rw-r--r--server/controllers/api/videos/index.ts105
2 files changed, 103 insertions, 20 deletions
diff --git a/server/controllers/api/server/debug.ts b/server/controllers/api/server/debug.ts
index 7787186be..ff0d9ca3c 100644
--- a/server/controllers/api/server/debug.ts
+++ b/server/controllers/api/server/debug.ts
@@ -1,4 +1,6 @@
1import { InboxManager } from '@server/lib/activitypub/inbox-manager' 1import { InboxManager } from '@server/lib/activitypub/inbox-manager'
2import { RemoveDanglingResumableUploadsScheduler } from '@server/lib/schedulers/remove-dangling-resumable-uploads-scheduler'
3import { SendDebugCommand } from '@shared/models'
2import * as express from 'express' 4import * as express from 'express'
3import { UserRight } from '../../../../shared/models/users' 5import { UserRight } from '../../../../shared/models/users'
4import { authenticate, ensureUserHasRight } from '../../../middlewares' 6import { authenticate, ensureUserHasRight } from '../../../middlewares'
@@ -11,6 +13,12 @@ debugRouter.get('/debug',
11 getDebug 13 getDebug
12) 14)
13 15
16debugRouter.post('/debug/run-command',
17 authenticate,
18 ensureUserHasRight(UserRight.MANAGE_DEBUG),
19 runCommand
20)
21
14// --------------------------------------------------------------------------- 22// ---------------------------------------------------------------------------
15 23
16export { 24export {
@@ -25,3 +33,13 @@ function getDebug (req: express.Request, res: express.Response) {
25 activityPubMessagesWaiting: InboxManager.Instance.getActivityPubMessagesWaiting() 33 activityPubMessagesWaiting: InboxManager.Instance.getActivityPubMessagesWaiting()
26 }) 34 })
27} 35}
36
37async function runCommand (req: express.Request, res: express.Response) {
38 const body: SendDebugCommand = req.body
39
40 if (body.command === 'remove-dandling-resumable-uploads') {
41 await RemoveDanglingResumableUploadsScheduler.Instance.execute()
42 }
43
44 return res.sendStatus(204)
45}
diff --git a/server/controllers/api/videos/index.ts b/server/controllers/api/videos/index.ts
index fbdb0f776..c32626d30 100644
--- a/server/controllers/api/videos/index.ts
+++ b/server/controllers/api/videos/index.ts
@@ -2,6 +2,7 @@ import * as express from 'express'
2import { move } from 'fs-extra' 2import { move } from 'fs-extra'
3import { extname } from 'path' 3import { extname } from 'path'
4import toInt from 'validator/lib/toInt' 4import toInt from 'validator/lib/toInt'
5import { deleteResumableUploadMetaFile, getResumableUploadPath } from '@server/helpers/upload'
5import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent' 6import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent'
6import { changeVideoChannelShare } from '@server/lib/activitypub/share' 7import { changeVideoChannelShare } from '@server/lib/activitypub/share'
7import { getLocalVideoActivityPubUrl } from '@server/lib/activitypub/url' 8import { getLocalVideoActivityPubUrl } from '@server/lib/activitypub/url'
@@ -10,8 +11,9 @@ import { addOptimizeOrMergeAudioJob, buildLocalVideoFromReq, buildVideoThumbnail
10import { generateVideoFilename, getVideoFilePath } from '@server/lib/video-paths' 11import { generateVideoFilename, getVideoFilePath } from '@server/lib/video-paths'
11import { getServerActor } from '@server/models/application/application' 12import { getServerActor } from '@server/models/application/application'
12import { MVideo, MVideoFile, MVideoFullLight } from '@server/types/models' 13import { MVideo, MVideoFile, MVideoFullLight } from '@server/types/models'
14import { uploadx } from '@uploadx/core'
13import { VideoCreate, VideosCommonQuery, VideoState, VideoUpdate } from '../../../../shared' 15import { VideoCreate, VideosCommonQuery, VideoState, VideoUpdate } from '../../../../shared'
14import { HttpStatusCode } from '../../../../shared/core-utils/miscs/http-error-codes' 16import { HttpStatusCode } from '../../../../shared/core-utils/miscs'
15import { auditLoggerFactory, getAuditIdFromRes, VideoAuditView } from '../../../helpers/audit-logger' 17import { auditLoggerFactory, getAuditIdFromRes, VideoAuditView } from '../../../helpers/audit-logger'
16import { resetSequelizeInstance, retryTransactionWrapper } from '../../../helpers/database-utils' 18import { resetSequelizeInstance, retryTransactionWrapper } from '../../../helpers/database-utils'
17import { buildNSFWFilter, createReqFiles, getCountVideos } from '../../../helpers/express-utils' 19import { buildNSFWFilter, createReqFiles, getCountVideos } from '../../../helpers/express-utils'
@@ -47,7 +49,9 @@ import {
47 setDefaultPagination, 49 setDefaultPagination,
48 setDefaultVideosSort, 50 setDefaultVideosSort,
49 videoFileMetadataGetValidator, 51 videoFileMetadataGetValidator,
50 videosAddValidator, 52 videosAddLegacyValidator,
53 videosAddResumableInitValidator,
54 videosAddResumableValidator,
51 videosCustomGetValidator, 55 videosCustomGetValidator,
52 videosGetValidator, 56 videosGetValidator,
53 videosRemoveValidator, 57 videosRemoveValidator,
@@ -69,6 +73,7 @@ import { watchingRouter } from './watching'
69const lTags = loggerTagsFactory('api', 'video') 73const lTags = loggerTagsFactory('api', 'video')
70const auditLogger = auditLoggerFactory('videos') 74const auditLogger = auditLoggerFactory('videos')
71const videosRouter = express.Router() 75const videosRouter = express.Router()
76const uploadxMiddleware = uploadx.upload({ directory: getResumableUploadPath() })
72 77
73const reqVideoFileAdd = createReqFiles( 78const reqVideoFileAdd = createReqFiles(
74 [ 'videofile', 'thumbnailfile', 'previewfile' ], 79 [ 'videofile', 'thumbnailfile', 'previewfile' ],
@@ -79,6 +84,16 @@ const reqVideoFileAdd = createReqFiles(
79 previewfile: CONFIG.STORAGE.TMP_DIR 84 previewfile: CONFIG.STORAGE.TMP_DIR
80 } 85 }
81) 86)
87
88const reqVideoFileAddResumable = createReqFiles(
89 [ 'thumbnailfile', 'previewfile' ],
90 MIMETYPES.IMAGE.MIMETYPE_EXT,
91 {
92 thumbnailfile: getResumableUploadPath(),
93 previewfile: getResumableUploadPath()
94 }
95)
96
82const reqVideoFileUpdate = createReqFiles( 97const reqVideoFileUpdate = createReqFiles(
83 [ 'thumbnailfile', 'previewfile' ], 98 [ 'thumbnailfile', 'previewfile' ],
84 MIMETYPES.IMAGE.MIMETYPE_EXT, 99 MIMETYPES.IMAGE.MIMETYPE_EXT,
@@ -111,18 +126,39 @@ videosRouter.get('/',
111 commonVideosFiltersValidator, 126 commonVideosFiltersValidator,
112 asyncMiddleware(listVideos) 127 asyncMiddleware(listVideos)
113) 128)
129
130videosRouter.post('/upload',
131 authenticate,
132 reqVideoFileAdd,
133 asyncMiddleware(videosAddLegacyValidator),
134 asyncRetryTransactionMiddleware(addVideoLegacy)
135)
136
137videosRouter.post('/upload-resumable',
138 authenticate,
139 reqVideoFileAddResumable,
140 asyncMiddleware(videosAddResumableInitValidator),
141 uploadxMiddleware
142)
143
144videosRouter.delete('/upload-resumable',
145 authenticate,
146 uploadxMiddleware
147)
148
149videosRouter.put('/upload-resumable',
150 authenticate,
151 uploadxMiddleware, // uploadx doesn't use call next() before the file upload completes
152 asyncMiddleware(videosAddResumableValidator),
153 asyncMiddleware(addVideoResumable)
154)
155
114videosRouter.put('/:id', 156videosRouter.put('/:id',
115 authenticate, 157 authenticate,
116 reqVideoFileUpdate, 158 reqVideoFileUpdate,
117 asyncMiddleware(videosUpdateValidator), 159 asyncMiddleware(videosUpdateValidator),
118 asyncRetryTransactionMiddleware(updateVideo) 160 asyncRetryTransactionMiddleware(updateVideo)
119) 161)
120videosRouter.post('/upload',
121 authenticate,
122 reqVideoFileAdd,
123 asyncMiddleware(videosAddValidator),
124 asyncRetryTransactionMiddleware(addVideo)
125)
126 162
127videosRouter.get('/:id/description', 163videosRouter.get('/:id/description',
128 asyncMiddleware(videosGetValidator), 164 asyncMiddleware(videosGetValidator),
@@ -157,23 +193,23 @@ export {
157 193
158// --------------------------------------------------------------------------- 194// ---------------------------------------------------------------------------
159 195
160function listVideoCategories (req: express.Request, res: express.Response) { 196function listVideoCategories (_req: express.Request, res: express.Response) {
161 res.json(VIDEO_CATEGORIES) 197 res.json(VIDEO_CATEGORIES)
162} 198}
163 199
164function listVideoLicences (req: express.Request, res: express.Response) { 200function listVideoLicences (_req: express.Request, res: express.Response) {
165 res.json(VIDEO_LICENCES) 201 res.json(VIDEO_LICENCES)
166} 202}
167 203
168function listVideoLanguages (req: express.Request, res: express.Response) { 204function listVideoLanguages (_req: express.Request, res: express.Response) {
169 res.json(VIDEO_LANGUAGES) 205 res.json(VIDEO_LANGUAGES)
170} 206}
171 207
172function listVideoPrivacies (req: express.Request, res: express.Response) { 208function listVideoPrivacies (_req: express.Request, res: express.Response) {
173 res.json(VIDEO_PRIVACIES) 209 res.json(VIDEO_PRIVACIES)
174} 210}
175 211
176async function addVideo (req: express.Request, res: express.Response) { 212async function addVideoLegacy (req: express.Request, res: express.Response) {
177 // Uploading the video could be long 213 // Uploading the video could be long
178 // Set timeout to 10 minutes, as Express's default is 2 minutes 214 // Set timeout to 10 minutes, as Express's default is 2 minutes
179 req.setTimeout(1000 * 60 * 10, () => { 215 req.setTimeout(1000 * 60 * 10, () => {
@@ -183,13 +219,42 @@ async function addVideo (req: express.Request, res: express.Response) {
183 219
184 const videoPhysicalFile = req.files['videofile'][0] 220 const videoPhysicalFile = req.files['videofile'][0]
185 const videoInfo: VideoCreate = req.body 221 const videoInfo: VideoCreate = req.body
222 const files = req.files
223
224 return addVideo({ res, videoPhysicalFile, videoInfo, files })
225}
226
227async function addVideoResumable (_req: express.Request, res: express.Response) {
228 const videoPhysicalFile = res.locals.videoFileResumable
229 const videoInfo = videoPhysicalFile.metadata
230 const files = { previewfile: videoInfo.previewfile }
231
232 // Don't need the meta file anymore
233 await deleteResumableUploadMetaFile(videoPhysicalFile.path)
234
235 return addVideo({ res, videoPhysicalFile, videoInfo, files })
236}
186 237
187 const videoData = buildLocalVideoFromReq(videoInfo, res.locals.videoChannel.id) 238async function addVideo (options: {
188 videoData.state = CONFIG.TRANSCODING.ENABLED ? VideoState.TO_TRANSCODE : VideoState.PUBLISHED 239 res: express.Response
189 videoData.duration = videoPhysicalFile['duration'] // duration was added by a previous middleware 240 videoPhysicalFile: express.VideoUploadFile
241 videoInfo: VideoCreate
242 files: express.UploadFiles
243}) {
244 const { res, videoPhysicalFile, videoInfo, files } = options
245 const videoChannel = res.locals.videoChannel
246 const user = res.locals.oauth.token.User
247
248 const videoData = buildLocalVideoFromReq(videoInfo, videoChannel.id)
249
250 videoData.state = CONFIG.TRANSCODING.ENABLED
251 ? VideoState.TO_TRANSCODE
252 : VideoState.PUBLISHED
253
254 videoData.duration = videoPhysicalFile.duration // duration was added by a previous middleware
190 255
191 const video = new VideoModel(videoData) as MVideoFullLight 256 const video = new VideoModel(videoData) as MVideoFullLight
192 video.VideoChannel = res.locals.videoChannel 257 video.VideoChannel = videoChannel
193 video.url = getLocalVideoActivityPubUrl(video) // We use the UUID, so set the URL after building the object 258 video.url = getLocalVideoActivityPubUrl(video) // We use the UUID, so set the URL after building the object
194 259
195 const videoFile = new VideoFileModel({ 260 const videoFile = new VideoFileModel({
@@ -217,7 +282,7 @@ async function addVideo (req: express.Request, res: express.Response) {
217 282
218 const [ thumbnailModel, previewModel ] = await buildVideoThumbnailsFromReq({ 283 const [ thumbnailModel, previewModel ] = await buildVideoThumbnailsFromReq({
219 video, 284 video,
220 files: req.files, 285 files,
221 fallback: type => generateVideoMiniature({ video, videoFile, type }) 286 fallback: type => generateVideoMiniature({ video, videoFile, type })
222 }) 287 })
223 288
@@ -253,7 +318,7 @@ async function addVideo (req: express.Request, res: express.Response) {
253 318
254 await autoBlacklistVideoIfNeeded({ 319 await autoBlacklistVideoIfNeeded({
255 video, 320 video,
256 user: res.locals.oauth.token.User, 321 user,
257 isRemote: false, 322 isRemote: false,
258 isNew: true, 323 isNew: true,
259 transaction: t 324 transaction: t
@@ -282,7 +347,7 @@ async function addVideo (req: express.Request, res: express.Response) {
282 .catch(err => logger.error('Cannot federate or notify video creation %s', video.url, { err, ...lTags(video.uuid) })) 347 .catch(err => logger.error('Cannot federate or notify video creation %s', video.url, { err, ...lTags(video.uuid) }))
283 348
284 if (video.state === VideoState.TO_TRANSCODE) { 349 if (video.state === VideoState.TO_TRANSCODE) {
285 await addOptimizeOrMergeAudioJob(videoCreated, videoFile, res.locals.oauth.token.User) 350 await addOptimizeOrMergeAudioJob(videoCreated, videoFile, user)
286 } 351 }
287 352
288 Hooks.runAction('action:api.video.uploaded', { video: videoCreated }) 353 Hooks.runAction('action:api.video.uploaded', { video: videoCreated })