aboutsummaryrefslogtreecommitdiffhomepage
path: root/shared/extra-utils
diff options
context:
space:
mode:
authorkontrollanten <6680299+kontrollanten@users.noreply.github.com>2021-05-10 11:13:41 +0200
committerGitHub <noreply@github.com>2021-05-10 11:13:41 +0200
commitf6d6e7f861189a4446f406efb775a29688764b48 (patch)
treec3dda9958c3f189d4c39e8743c738d8c1fef4c2d /shared/extra-utils
parentd29ced1a8582d99b776f664475a157adcf555d98 (diff)
downloadPeerTube-f6d6e7f861189a4446f406efb775a29688764b48.tar.gz
PeerTube-f6d6e7f861189a4446f406efb775a29688764b48.tar.zst
PeerTube-f6d6e7f861189a4446f406efb775a29688764b48.zip
Resumable video uploads (#3933)
* WIP: resumable video uploads relates to #324 * fix review comments * video upload: error handling * fix audio upload * fixes after self review * Update server/controllers/api/videos/index.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * Update server/middlewares/validators/videos/videos.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * Update server/controllers/api/videos/index.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * update after code review * refactor upload route - restore multipart upload route - move resumable to dedicated upload-resumable route - move checks to middleware - do not leak internal fs structure in response * fix yarn.lock upon rebase * factorize addVideo for reuse in both endpoints * add resumable upload API to openapi spec * add initial test and test helper for resumable upload * typings for videoAddResumable middleware * avoid including aws and google packages via node-uploadx, by only including uploadx/core * rename ex-isAudioBg to more explicit name mentioning it is a preview file for audio * add video-upload-tmp-folder-cleaner job * stronger typing of video upload middleware * reduce dependency to @uploadx/core * add audio upload test * refactor resumable uploads cleanup from job to scheduler * refactor resumable uploads scheduler to compare to last execution time * make resumable upload validator to always cleanup on failure * move legacy upload request building outside of uploadVideo test helper * filter upload-resumable middlewares down to POST, PUT, DELETE also begin to type metadata * merge add duration functions * stronger typings and documentation for uploadx behaviour, move init validator up * refactor(client/video-edit): options > uploadxOptions * refactor(client/video-edit): remove obsolete else * scheduler/remove-dangling-resum: rename tag * refactor(server/video): add UploadVideoFiles type * refactor(mw/validators): restructure eslint disable * refactor(mw/validators/videos): rename import * refactor(client/vid-upload): rename html elem id * refactor(sched/remove-dangl): move fn to method * refactor(mw/async): add method typing * refactor(mw/vali/video): double quote > single * refactor(server/upload-resum): express use > all * proper http methud enum server/middlewares/async.ts * properly type http methods * factorize common video upload validation steps * add check for maximum partially uploaded file size * fix audioBg use * fix extname(filename) in addVideo * document parameters for uploadx's resumable protocol * clear META files in scheduler * last audio refactor before cramming preview in the initial POST form data * refactor as mulitpart/form-data initial post request this allows preview/thumbnail uploads alongside the initial request, and cleans up the upload form * Add more tests for resumable uploads * Refactor remove dangling resumable uploads * Prepare changelog * Add more resumable upload tests * Remove user quota check for resumable uploads * Fix upload error handler * Update nginx template for upload-resumable * Cleanup comment * Remove unused express methods * Prefer to use got instead of raw http * Don't retry on error 500 Co-authored-by: Rigel Kent <par@rigelk.eu> Co-authored-by: Rigel Kent <sendmemail@rigelk.eu> Co-authored-by: Chocobozzz <me@florianbigard.com>
Diffstat (limited to 'shared/extra-utils')
-rw-r--r--shared/extra-utils/server/debug.ts18
-rw-r--r--shared/extra-utils/server/servers.ts2
-rw-r--r--shared/extra-utils/videos/video-channels.ts11
-rw-r--r--shared/extra-utils/videos/videos.ts258
4 files changed, 230 insertions, 59 deletions
diff --git a/shared/extra-utils/server/debug.ts b/shared/extra-utils/server/debug.ts
index 5cf80a5fb..f196812b7 100644
--- a/shared/extra-utils/server/debug.ts
+++ b/shared/extra-utils/server/debug.ts
@@ -1,5 +1,6 @@
1import { makeGetRequest } from '../requests/requests' 1import { makeGetRequest, makePostBodyRequest } from '../requests/requests'
2import { HttpStatusCode } from '../../core-utils/miscs/http-error-codes' 2import { HttpStatusCode } from '../../core-utils/miscs/http-error-codes'
3import { SendDebugCommand } from '@shared/models'
3 4
4function getDebug (url: string, token: string) { 5function getDebug (url: string, token: string) {
5 const path = '/api/v1/server/debug' 6 const path = '/api/v1/server/debug'
@@ -12,8 +13,21 @@ function getDebug (url: string, token: string) {
12 }) 13 })
13} 14}
14 15
16function sendDebugCommand (url: string, token: string, body: SendDebugCommand) {
17 const path = '/api/v1/server/debug/run-command'
18
19 return makePostBodyRequest({
20 url,
21 path,
22 token,
23 fields: body,
24 statusCodeExpected: HttpStatusCode.NO_CONTENT_204
25 })
26}
27
15// --------------------------------------------------------------------------- 28// ---------------------------------------------------------------------------
16 29
17export { 30export {
18 getDebug 31 getDebug,
32 sendDebugCommand
19} 33}
diff --git a/shared/extra-utils/server/servers.ts b/shared/extra-utils/server/servers.ts
index 779a3cc36..479f08e12 100644
--- a/shared/extra-utils/server/servers.ts
+++ b/shared/extra-utils/server/servers.ts
@@ -274,7 +274,7 @@ async function reRunServer (server: ServerInfo, configOverride?: any) {
274} 274}
275 275
276async function checkTmpIsEmpty (server: ServerInfo) { 276async function checkTmpIsEmpty (server: ServerInfo) {
277 await checkDirectoryIsEmpty(server, 'tmp', [ 'plugins-global.css', 'hls' ]) 277 await checkDirectoryIsEmpty(server, 'tmp', [ 'plugins-global.css', 'hls', 'resumable-uploads' ])
278 278
279 if (await pathExists(join('test' + server.internalServerNumber, 'tmp', 'hls'))) { 279 if (await pathExists(join('test' + server.internalServerNumber, 'tmp', 'hls'))) {
280 await checkDirectoryIsEmpty(server, 'tmp/hls') 280 await checkDirectoryIsEmpty(server, 'tmp/hls')
diff --git a/shared/extra-utils/videos/video-channels.ts b/shared/extra-utils/videos/video-channels.ts
index d0dfb5856..0aab93e52 100644
--- a/shared/extra-utils/videos/video-channels.ts
+++ b/shared/extra-utils/videos/video-channels.ts
@@ -5,7 +5,7 @@ import { VideoChannelUpdate } from '../../models/videos/channel/video-channel-up
5import { VideoChannelCreate } from '../../models/videos/channel/video-channel-create.model' 5import { VideoChannelCreate } from '../../models/videos/channel/video-channel-create.model'
6import { makeDeleteRequest, makeGetRequest, updateImageRequest } from '../requests/requests' 6import { makeDeleteRequest, makeGetRequest, updateImageRequest } from '../requests/requests'
7import { ServerInfo } from '../server/servers' 7import { ServerInfo } from '../server/servers'
8import { User } from '../../models/users/user.model' 8import { MyUser, User } from '../../models/users/user.model'
9import { getMyUserInformation } from '../users/users' 9import { getMyUserInformation } from '../users/users'
10import { HttpStatusCode } from '../../../shared/core-utils/miscs/http-error-codes' 10import { HttpStatusCode } from '../../../shared/core-utils/miscs/http-error-codes'
11 11
@@ -170,6 +170,12 @@ function setDefaultVideoChannel (servers: ServerInfo[]) {
170 return Promise.all(tasks) 170 return Promise.all(tasks)
171} 171}
172 172
173async function getDefaultVideoChannel (url: string, token: string) {
174 const res = await getMyUserInformation(url, token)
175
176 return (res.body as MyUser).videoChannels[0].id
177}
178
173// --------------------------------------------------------------------------- 179// ---------------------------------------------------------------------------
174 180
175export { 181export {
@@ -181,5 +187,6 @@ export {
181 deleteVideoChannel, 187 deleteVideoChannel,
182 getVideoChannel, 188 getVideoChannel,
183 setDefaultVideoChannel, 189 setDefaultVideoChannel,
184 deleteVideoChannelImage 190 deleteVideoChannelImage,
191 getDefaultVideoChannel
185} 192}
diff --git a/shared/extra-utils/videos/videos.ts b/shared/extra-utils/videos/videos.ts
index a0143b0ef..e88256ac0 100644
--- a/shared/extra-utils/videos/videos.ts
+++ b/shared/extra-utils/videos/videos.ts
@@ -1,7 +1,8 @@
1/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/no-floating-promises */ 1/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/no-floating-promises */
2 2
3import { expect } from 'chai' 3import { expect } from 'chai'
4import { pathExists, readdir, readFile } from 'fs-extra' 4import { createReadStream, pathExists, readdir, readFile, stat } from 'fs-extra'
5import got, { Response as GotResponse } from 'got/dist/source'
5import * as parseTorrent from 'parse-torrent' 6import * as parseTorrent from 'parse-torrent'
6import { extname, join } from 'path' 7import { extname, join } from 'path'
7import * as request from 'supertest' 8import * as request from 'supertest'
@@ -42,6 +43,7 @@ type VideoAttributes = {
42 channelId?: number 43 channelId?: number
43 privacy?: VideoPrivacy 44 privacy?: VideoPrivacy
44 fixture?: string 45 fixture?: string
46 support?: string
45 thumbnailfile?: string 47 thumbnailfile?: string
46 previewfile?: string 48 previewfile?: string
47 scheduleUpdate?: { 49 scheduleUpdate?: {
@@ -364,8 +366,13 @@ async function checkVideoFilesWereRemoved (
364 } 366 }
365} 367}
366 368
367async function uploadVideo (url: string, accessToken: string, videoAttributesArg: VideoAttributes, specialStatus = HttpStatusCode.OK_200) { 369async function uploadVideo (
368 const path = '/api/v1/videos/upload' 370 url: string,
371 accessToken: string,
372 videoAttributesArg: VideoAttributes,
373 specialStatus = HttpStatusCode.OK_200,
374 mode: 'legacy' | 'resumable' = 'legacy'
375) {
369 let defaultChannelId = '1' 376 let defaultChannelId = '1'
370 377
371 try { 378 try {
@@ -391,74 +398,170 @@ async function uploadVideo (url: string, accessToken: string, videoAttributesArg
391 fixture: 'video_short.webm' 398 fixture: 'video_short.webm'
392 }, videoAttributesArg) 399 }, videoAttributesArg)
393 400
401 const res = mode === 'legacy'
402 ? await buildLegacyUpload(url, accessToken, attributes, specialStatus)
403 : await buildResumeUpload(url, accessToken, attributes, specialStatus)
404
405 // Wait torrent generation
406 if (specialStatus === HttpStatusCode.OK_200) {
407 let video: VideoDetails
408 do {
409 const resVideo = await getVideoWithToken(url, accessToken, res.body.video.uuid)
410 video = resVideo.body
411
412 await wait(50)
413 } while (!video.files[0].torrentUrl)
414 }
415
416 return res
417}
418
419function checkUploadVideoParam (
420 url: string,
421 token: string,
422 attributes: Partial<VideoAttributes>,
423 specialStatus = HttpStatusCode.OK_200,
424 mode: 'legacy' | 'resumable' = 'legacy'
425) {
426 return mode === 'legacy'
427 ? buildLegacyUpload(url, token, attributes, specialStatus)
428 : buildResumeUpload(url, token, attributes, specialStatus)
429}
430
431async function buildLegacyUpload (url: string, token: string, attributes: VideoAttributes, specialStatus = HttpStatusCode.OK_200) {
432 const path = '/api/v1/videos/upload'
394 const req = request(url) 433 const req = request(url)
395 .post(path) 434 .post(path)
396 .set('Accept', 'application/json') 435 .set('Accept', 'application/json')
397 .set('Authorization', 'Bearer ' + accessToken) 436 .set('Authorization', 'Bearer ' + token)
398 .field('name', attributes.name)
399 .field('nsfw', JSON.stringify(attributes.nsfw))
400 .field('commentsEnabled', JSON.stringify(attributes.commentsEnabled))
401 .field('downloadEnabled', JSON.stringify(attributes.downloadEnabled))
402 .field('waitTranscoding', JSON.stringify(attributes.waitTranscoding))
403 .field('privacy', attributes.privacy.toString())
404 .field('channelId', attributes.channelId)
405
406 if (attributes.support !== undefined) {
407 req.field('support', attributes.support)
408 }
409 437
410 if (attributes.description !== undefined) { 438 buildUploadReq(req, attributes)
411 req.field('description', attributes.description)
412 }
413 if (attributes.language !== undefined) {
414 req.field('language', attributes.language.toString())
415 }
416 if (attributes.category !== undefined) {
417 req.field('category', attributes.category.toString())
418 }
419 if (attributes.licence !== undefined) {
420 req.field('licence', attributes.licence.toString())
421 }
422 439
423 const tags = attributes.tags || [] 440 if (attributes.fixture !== undefined) {
424 for (let i = 0; i < tags.length; i++) { 441 req.attach('videofile', buildAbsoluteFixturePath(attributes.fixture))
425 req.field('tags[' + i + ']', attributes.tags[i])
426 } 442 }
427 443
428 if (attributes.thumbnailfile !== undefined) { 444 return req.expect(specialStatus)
429 req.attach('thumbnailfile', buildAbsoluteFixturePath(attributes.thumbnailfile)) 445}
430 }
431 if (attributes.previewfile !== undefined) {
432 req.attach('previewfile', buildAbsoluteFixturePath(attributes.previewfile))
433 }
434 446
435 if (attributes.scheduleUpdate) { 447async function buildResumeUpload (url: string, token: string, attributes: VideoAttributes, specialStatus = HttpStatusCode.OK_200) {
436 req.field('scheduleUpdate[updateAt]', attributes.scheduleUpdate.updateAt) 448 let size = 0
449 let videoFilePath: string
450 let mimetype = 'video/mp4'
437 451
438 if (attributes.scheduleUpdate.privacy) { 452 if (attributes.fixture) {
439 req.field('scheduleUpdate[privacy]', attributes.scheduleUpdate.privacy) 453 videoFilePath = buildAbsoluteFixturePath(attributes.fixture)
454 size = (await stat(videoFilePath)).size
455
456 if (videoFilePath.endsWith('.mkv')) {
457 mimetype = 'video/x-matroska'
458 } else if (videoFilePath.endsWith('.webm')) {
459 mimetype = 'video/webm'
440 } 460 }
441 } 461 }
442 462
443 if (attributes.originallyPublishedAt !== undefined) { 463 const initializeSessionRes = await prepareResumableUpload({ url, token, attributes, size, mimetype })
444 req.field('originallyPublishedAt', attributes.originallyPublishedAt) 464 const initStatus = initializeSessionRes.status
465
466 if (videoFilePath && initStatus === HttpStatusCode.CREATED_201) {
467 const locationHeader = initializeSessionRes.header['location']
468 expect(locationHeader).to.not.be.undefined
469
470 const pathUploadId = locationHeader.split('?')[1]
471
472 return sendResumableChunks({ url, token, pathUploadId, videoFilePath, size, specialStatus })
445 } 473 }
446 474
447 const res = await req.attach('videofile', buildAbsoluteFixturePath(attributes.fixture)) 475 const expectedInitStatus = specialStatus === HttpStatusCode.OK_200
448 .expect(specialStatus) 476 ? HttpStatusCode.CREATED_201
477 : specialStatus
449 478
450 // Wait torrent generation 479 expect(initStatus).to.equal(expectedInitStatus)
451 if (specialStatus === HttpStatusCode.OK_200) {
452 let video: VideoDetails
453 do {
454 const resVideo = await getVideoWithToken(url, accessToken, res.body.video.uuid)
455 video = resVideo.body
456 480
457 await wait(50) 481 return initializeSessionRes
458 } while (!video.files[0].torrentUrl) 482}
483
484async function prepareResumableUpload (options: {
485 url: string
486 token: string
487 attributes: VideoAttributes
488 size: number
489 mimetype: string
490}) {
491 const { url, token, attributes, size, mimetype } = options
492
493 const path = '/api/v1/videos/upload-resumable'
494
495 const req = request(url)
496 .post(path)
497 .set('Authorization', 'Bearer ' + token)
498 .set('X-Upload-Content-Type', mimetype)
499 .set('X-Upload-Content-Length', size.toString())
500
501 buildUploadReq(req, attributes)
502
503 if (attributes.fixture) {
504 req.field('filename', attributes.fixture)
459 } 505 }
460 506
461 return res 507 return req
508}
509
510function sendResumableChunks (options: {
511 url: string
512 token: string
513 pathUploadId: string
514 videoFilePath: string
515 size: number
516 specialStatus?: HttpStatusCode
517 contentLength?: number
518 contentRangeBuilder?: (start: number, chunk: any) => string
519}) {
520 const { url, token, pathUploadId, videoFilePath, size, specialStatus, contentLength, contentRangeBuilder } = options
521
522 const expectedStatus = specialStatus || HttpStatusCode.OK_200
523
524 const path = '/api/v1/videos/upload-resumable'
525 let start = 0
526
527 const readable = createReadStream(videoFilePath, { highWaterMark: 8 * 1024 })
528 return new Promise<GotResponse>((resolve, reject) => {
529 readable.on('data', async function onData (chunk) {
530 readable.pause()
531
532 const headers = {
533 'Authorization': 'Bearer ' + token,
534 'Content-Type': 'application/octet-stream',
535 'Content-Range': contentRangeBuilder
536 ? contentRangeBuilder(start, chunk)
537 : `bytes ${start}-${start + chunk.length - 1}/${size}`,
538 'Content-Length': contentLength ? contentLength + '' : chunk.length + ''
539 }
540
541 const res = await got({
542 url,
543 method: 'put',
544 headers,
545 path: path + '?' + pathUploadId,
546 body: chunk,
547 responseType: 'json',
548 throwHttpErrors: false
549 })
550
551 start += chunk.length
552
553 if (res.statusCode === expectedStatus) {
554 return resolve(res)
555 }
556
557 if (res.statusCode !== HttpStatusCode.PERMANENT_REDIRECT_308) {
558 readable.off('data', onData)
559 return reject(new Error('Incorrect transient behaviour sending intermediary chunks'))
560 }
561
562 readable.resume()
563 })
564 })
462} 565}
463 566
464function updateVideo ( 567function updateVideo (
@@ -749,11 +852,13 @@ export {
749 getVideoWithToken, 852 getVideoWithToken,
750 getVideosList, 853 getVideosList,
751 removeAllVideos, 854 removeAllVideos,
855 checkUploadVideoParam,
752 getVideosListPagination, 856 getVideosListPagination,
753 getVideosListSort, 857 getVideosListSort,
754 removeVideo, 858 removeVideo,
755 getVideosListWithToken, 859 getVideosListWithToken,
756 uploadVideo, 860 uploadVideo,
861 sendResumableChunks,
757 getVideosWithFilters, 862 getVideosWithFilters,
758 uploadRandomVideoOnServers, 863 uploadRandomVideoOnServers,
759 updateVideo, 864 updateVideo,
@@ -767,5 +872,50 @@ export {
767 getMyVideosWithFilter, 872 getMyVideosWithFilter,
768 uploadVideoAndGetId, 873 uploadVideoAndGetId,
769 getLocalIdByUUID, 874 getLocalIdByUUID,
770 getVideoIdFromUUID 875 getVideoIdFromUUID,
876 prepareResumableUpload
877}
878
879// ---------------------------------------------------------------------------
880
881function buildUploadReq (req: request.Test, attributes: VideoAttributes) {
882
883 for (const key of [ 'name', 'support', 'channelId', 'description', 'originallyPublishedAt' ]) {
884 if (attributes[key] !== undefined) {
885 req.field(key, attributes[key])
886 }
887 }
888
889 for (const key of [ 'nsfw', 'commentsEnabled', 'downloadEnabled', 'waitTranscoding' ]) {
890 if (attributes[key] !== undefined) {
891 req.field(key, JSON.stringify(attributes[key]))
892 }
893 }
894
895 for (const key of [ 'language', 'privacy', 'category', 'licence' ]) {
896 if (attributes[key] !== undefined) {
897 req.field(key, attributes[key].toString())
898 }
899 }
900
901 const tags = attributes.tags || []
902 for (let i = 0; i < tags.length; i++) {
903 req.field('tags[' + i + ']', attributes.tags[i])
904 }
905
906 for (const key of [ 'thumbnailfile', 'previewfile' ]) {
907 if (attributes[key] !== undefined) {
908 req.attach(key, buildAbsoluteFixturePath(attributes[key]))
909 }
910 }
911
912 if (attributes.scheduleUpdate) {
913 if (attributes.scheduleUpdate.updateAt) {
914 req.field('scheduleUpdate[updateAt]', attributes.scheduleUpdate.updateAt)
915 }
916
917 if (attributes.scheduleUpdate.privacy) {
918 req.field('scheduleUpdate[privacy]', attributes.scheduleUpdate.privacy)
919 }
920 }
771} 921}