]> git.immae.eu Git - github/Chocobozzz/PeerTube.git/blame - shared/server-commands/videos/videos-command.ts
Correctly cleanup files from object storage
[github/Chocobozzz/PeerTube.git] / shared / server-commands / videos / videos-command.ts
CommitLineData
d23dd9fb
C
1/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/no-floating-promises */
2
3import { expect } from 'chai'
4import { createReadStream, stat } from 'fs-extra'
5import got, { Response as GotResponse } from 'got'
d23dd9fb 6import validator from 'validator'
508c1b1e 7import { buildAbsoluteFixturePath, getAllPrivacies, omit, pick, wait } from '@shared/core-utils'
0628157f 8import { buildUUID } from '@shared/extra-utils'
d23dd9fb 9import {
4c7e60bc 10 HttpStatusCode,
d23dd9fb
C
11 ResultList,
12 UserVideoRateType,
13 Video,
14 VideoCreate,
15 VideoCreateResult,
16 VideoDetails,
17 VideoFileMetadata,
508c1b1e 18 VideoInclude,
d23dd9fb 19 VideoPrivacy,
ad5db104
C
20 VideosCommonQuery,
21 VideoTranscodingCreate
d23dd9fb 22} from '@shared/models'
1bb4c9ab 23import { VideoSource } from '@shared/models/videos/video-source'
d23dd9fb 24import { unwrapBody } from '../requests'
06aad801 25import { waitJobs } from '../server'
d23dd9fb
C
26import { AbstractCommand, OverrideCommandOptions } from '../shared'
27
28export type VideoEdit = Partial<Omit<VideoCreate, 'thumbnailfile' | 'previewfile'>> & {
29 fixture?: string
30 thumbnailfile?: string
31 previewfile?: string
32}
33
34export class VideosCommand extends AbstractCommand {
d23dd9fb
C
35 getCategories (options: OverrideCommandOptions = {}) {
36 const path = '/api/v1/videos/categories'
37
38 return this.getRequestBody<{ [id: number]: string }>({
39 ...options,
40 path,
41
42 implicitToken: false,
43 defaultExpectedStatus: HttpStatusCode.OK_200
44 })
45 }
46
47 getLicences (options: OverrideCommandOptions = {}) {
48 const path = '/api/v1/videos/licences'
49
50 return this.getRequestBody<{ [id: number]: string }>({
51 ...options,
52 path,
53
54 implicitToken: false,
55 defaultExpectedStatus: HttpStatusCode.OK_200
56 })
57 }
58
59 getLanguages (options: OverrideCommandOptions = {}) {
60 const path = '/api/v1/videos/languages'
61
62 return this.getRequestBody<{ [id: string]: string }>({
63 ...options,
64 path,
65
66 implicitToken: false,
67 defaultExpectedStatus: HttpStatusCode.OK_200
68 })
69 }
70
71 getPrivacies (options: OverrideCommandOptions = {}) {
72 const path = '/api/v1/videos/privacies'
73
74 return this.getRequestBody<{ [id in VideoPrivacy]: string }>({
75 ...options,
76 path,
77
78 implicitToken: false,
79 defaultExpectedStatus: HttpStatusCode.OK_200
80 })
81 }
82
83 // ---------------------------------------------------------------------------
84
85 getDescription (options: OverrideCommandOptions & {
86 descriptionPath: string
87 }) {
88 return this.getRequestBody<{ description: string }>({
89 ...options,
90 path: options.descriptionPath,
91
92 implicitToken: false,
93 defaultExpectedStatus: HttpStatusCode.OK_200
94 })
95 }
96
97 getFileMetadata (options: OverrideCommandOptions & {
98 url: string
99 }) {
100 return unwrapBody<VideoFileMetadata>(this.getRawRequest({
101 ...options,
102
103 url: options.url,
104 implicitToken: false,
105 defaultExpectedStatus: HttpStatusCode.OK_200
106 }))
107 }
108
109 // ---------------------------------------------------------------------------
110
d23dd9fb
C
111 rate (options: OverrideCommandOptions & {
112 id: number | string
113 rating: UserVideoRateType
114 }) {
115 const { id, rating } = options
116 const path = '/api/v1/videos/' + id + '/rate'
117
118 return this.putBodyRequest({
119 ...options,
120
121 path,
122 fields: { rating },
123 implicitToken: true,
124 defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204
125 })
126 }
127
128 // ---------------------------------------------------------------------------
129
130 get (options: OverrideCommandOptions & {
131 id: number | string
132 }) {
133 const path = '/api/v1/videos/' + options.id
134
135 return this.getRequestBody<VideoDetails>({
136 ...options,
137
138 path,
139 implicitToken: false,
140 defaultExpectedStatus: HttpStatusCode.OK_200
141 })
142 }
143
144 getWithToken (options: OverrideCommandOptions & {
145 id: number | string
146 }) {
147 return this.get({
148 ...options,
149
150 token: this.buildCommonRequestToken({ ...options, implicitToken: true })
151 })
152 }
153
2e401e85 154 getSource (options: OverrideCommandOptions & {
155 id: number | string
156 }) {
157 const path = '/api/v1/videos/' + options.id + '/source'
158
159 return this.getRequestBody<VideoSource>({
160 ...options,
161
162 path,
163 implicitToken: true,
164 defaultExpectedStatus: HttpStatusCode.OK_200
165 })
166 }
167
d23dd9fb
C
168 async getId (options: OverrideCommandOptions & {
169 uuid: number | string
170 }) {
171 const { uuid } = options
172
173 if (validator.isUUID('' + uuid) === false) return uuid as number
174
175 const { id } = await this.get({ ...options, id: uuid })
176
177 return id
178 }
179
0305db28
JB
180 async listFiles (options: OverrideCommandOptions & {
181 id: number | string
182 }) {
183 const video = await this.get(options)
184
185 const files = video.files || []
186 const hlsFiles = video.streamingPlaylists[0]?.files || []
187
188 return files.concat(hlsFiles)
189 }
190
d23dd9fb
C
191 // ---------------------------------------------------------------------------
192
193 listMyVideos (options: OverrideCommandOptions & {
194 start?: number
195 count?: number
196 sort?: string
197 search?: string
198 isLive?: boolean
978c87e7 199 channelId?: number
d23dd9fb
C
200 } = {}) {
201 const path = '/api/v1/users/me/videos'
202
203 return this.getRequestBody<ResultList<Video>>({
204 ...options,
205
206 path,
978c87e7 207 query: pick(options, [ 'start', 'count', 'sort', 'search', 'isLive', 'channelId' ]),
d23dd9fb
C
208 implicitToken: true,
209 defaultExpectedStatus: HttpStatusCode.OK_200
210 })
211 }
212
213 // ---------------------------------------------------------------------------
214
215 list (options: OverrideCommandOptions & VideosCommonQuery = {}) {
216 const path = '/api/v1/videos'
217
218 const query = this.buildListQuery(options)
219
220 return this.getRequestBody<ResultList<Video>>({
221 ...options,
222
223 path,
224 query: { sort: 'name', ...query },
225 implicitToken: false,
226 defaultExpectedStatus: HttpStatusCode.OK_200
227 })
228 }
229
230 listWithToken (options: OverrideCommandOptions & VideosCommonQuery = {}) {
231 return this.list({
232 ...options,
233
234 token: this.buildCommonRequestToken({ ...options, implicitToken: true })
235 })
236 }
237
508c1b1e
C
238 listAllForAdmin (options: OverrideCommandOptions & VideosCommonQuery = {}) {
239 const include = VideoInclude.NOT_PUBLISHED_STATE | VideoInclude.BLACKLISTED | VideoInclude.BLOCKED_OWNER
240 const nsfw = 'both'
241 const privacyOneOf = getAllPrivacies()
242
243 return this.list({
244 ...options,
245
246 include,
247 nsfw,
248 privacyOneOf,
249
250 token: this.buildCommonRequestToken({ ...options, implicitToken: true })
251 })
252 }
253
2760b454 254 listByAccount (options: OverrideCommandOptions & VideosCommonQuery & {
c0e8b12e 255 handle: string
d23dd9fb 256 }) {
c0e8b12e
C
257 const { handle, search } = options
258 const path = '/api/v1/accounts/' + handle + '/videos'
d23dd9fb
C
259
260 return this.getRequestBody<ResultList<Video>>({
261 ...options,
262
263 path,
264 query: { search, ...this.buildListQuery(options) },
265 implicitToken: true,
266 defaultExpectedStatus: HttpStatusCode.OK_200
267 })
268 }
269
2760b454 270 listByChannel (options: OverrideCommandOptions & VideosCommonQuery & {
c0e8b12e 271 handle: string
d23dd9fb 272 }) {
c0e8b12e
C
273 const { handle } = options
274 const path = '/api/v1/video-channels/' + handle + '/videos'
d23dd9fb
C
275
276 return this.getRequestBody<ResultList<Video>>({
277 ...options,
278
279 path,
280 query: this.buildListQuery(options),
281 implicitToken: true,
282 defaultExpectedStatus: HttpStatusCode.OK_200
283 })
284 }
285
286 // ---------------------------------------------------------------------------
287
4d029ef8
C
288 async find (options: OverrideCommandOptions & {
289 name: string
290 }) {
291 const { data } = await this.list(options)
292
293 return data.find(v => v.name === options.name)
294 }
295
296 // ---------------------------------------------------------------------------
297
d23dd9fb
C
298 update (options: OverrideCommandOptions & {
299 id: number | string
300 attributes?: VideoEdit
301 }) {
302 const { id, attributes = {} } = options
303 const path = '/api/v1/videos/' + id
304
305 // Upload request
306 if (attributes.thumbnailfile || attributes.previewfile) {
307 const attaches: any = {}
308 if (attributes.thumbnailfile) attaches.thumbnailfile = attributes.thumbnailfile
309 if (attributes.previewfile) attaches.previewfile = attributes.previewfile
310
311 return this.putUploadRequest({
312 ...options,
313
314 path,
315 fields: options.attributes,
316 attaches: {
317 thumbnailfile: attributes.thumbnailfile,
318 previewfile: attributes.previewfile
319 },
320 implicitToken: true,
321 defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204
322 })
323 }
324
325 return this.putBodyRequest({
326 ...options,
327
328 path,
329 fields: options.attributes,
330 implicitToken: true,
331 defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204
332 })
333 }
334
335 remove (options: OverrideCommandOptions & {
336 id: number | string
337 }) {
338 const path = '/api/v1/videos/' + options.id
339
c0e8b12e 340 return unwrapBody(this.deleteRequest({
d23dd9fb
C
341 ...options,
342
343 path,
344 implicitToken: true,
345 defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204
c0e8b12e 346 }))
d23dd9fb
C
347 }
348
349 async removeAll () {
350 const { data } = await this.list()
351
352 for (const v of data) {
353 await this.remove({ id: v.id })
354 }
355 }
356
357 // ---------------------------------------------------------------------------
358
359 async upload (options: OverrideCommandOptions & {
360 attributes?: VideoEdit
361 mode?: 'legacy' | 'resumable' // default legacy
3545e72c 362 waitTorrentGeneration?: boolean // default true
d23dd9fb 363 } = {}) {
01771012 364 const { mode = 'legacy', waitTorrentGeneration = true } = options
d23dd9fb
C
365 let defaultChannelId = 1
366
367 try {
89d241a7 368 const { videoChannels } = await this.server.users.getMyInfo({ token: options.token })
d23dd9fb
C
369 defaultChannelId = videoChannels[0].id
370 } catch (e) { /* empty */ }
371
372 // Override default attributes
373 const attributes = {
374 name: 'my super video',
375 category: 5,
376 licence: 4,
377 language: 'zh',
378 channelId: defaultChannelId,
379 nsfw: true,
380 waitTranscoding: false,
381 description: 'my super description',
382 support: 'my super support text',
383 tags: [ 'tag' ],
384 privacy: VideoPrivacy.PUBLIC,
385 commentsEnabled: true,
386 downloadEnabled: true,
387 fixture: 'video_short.webm',
388
389 ...options.attributes
390 }
391
4c7e60bc 392 const created = mode === 'legacy'
d23dd9fb
C
393 ? await this.buildLegacyUpload({ ...options, attributes })
394 : await this.buildResumeUpload({ ...options, attributes })
395
396 // Wait torrent generation
4c7e60bc 397 const expectedStatus = this.buildExpectedStatus({ ...options, defaultExpectedStatus: HttpStatusCode.OK_200 })
3545e72c 398 if (expectedStatus === HttpStatusCode.OK_200 && waitTorrentGeneration) {
d23dd9fb
C
399 let video: VideoDetails
400
401 do {
4c7e60bc 402 video = await this.getWithToken({ ...options, id: created.uuid })
d23dd9fb
C
403
404 await wait(50)
405 } while (!video.files[0].torrentUrl)
406 }
407
4c7e60bc 408 return created
d23dd9fb
C
409 }
410
411 async buildLegacyUpload (options: OverrideCommandOptions & {
412 attributes: VideoEdit
413 }): Promise<VideoCreateResult> {
414 const path = '/api/v1/videos/upload'
415
416 return unwrapBody<{ video: VideoCreateResult }>(this.postUploadRequest({
417 ...options,
418
419 path,
420 fields: this.buildUploadFields(options.attributes),
421 attaches: this.buildUploadAttaches(options.attributes),
422 implicitToken: true,
423 defaultExpectedStatus: HttpStatusCode.OK_200
424 })).then(body => body.video || body as any)
425 }
426
427 async buildResumeUpload (options: OverrideCommandOptions & {
428 attributes: VideoEdit
c0e8b12e 429 }): Promise<VideoCreateResult> {
d23dd9fb
C
430 const { attributes, expectedStatus } = options
431
432 let size = 0
433 let videoFilePath: string
434 let mimetype = 'video/mp4'
435
436 if (attributes.fixture) {
437 videoFilePath = buildAbsoluteFixturePath(attributes.fixture)
438 size = (await stat(videoFilePath)).size
439
440 if (videoFilePath.endsWith('.mkv')) {
441 mimetype = 'video/x-matroska'
442 } else if (videoFilePath.endsWith('.webm')) {
443 mimetype = 'video/webm'
444 }
445 }
446
c0e8b12e
C
447 // Do not check status automatically, we'll check it manually
448 const initializeSessionRes = await this.prepareResumableUpload({ ...options, expectedStatus: null, attributes, size, mimetype })
d23dd9fb
C
449 const initStatus = initializeSessionRes.status
450
451 if (videoFilePath && initStatus === HttpStatusCode.CREATED_201) {
452 const locationHeader = initializeSessionRes.header['location']
453 expect(locationHeader).to.not.be.undefined
454
455 const pathUploadId = locationHeader.split('?')[1]
456
457 const result = await this.sendResumableChunks({ ...options, pathUploadId, videoFilePath, size })
458
8de589b9
C
459 if (result.statusCode === HttpStatusCode.OK_200) {
460 await this.endResumableUpload({ ...options, expectedStatus: HttpStatusCode.NO_CONTENT_204, pathUploadId })
461 }
790c2837 462
c0e8b12e 463 return result.body?.video || result.body as any
d23dd9fb
C
464 }
465
466 const expectedInitStatus = expectedStatus === HttpStatusCode.OK_200
467 ? HttpStatusCode.CREATED_201
468 : expectedStatus
469
470 expect(initStatus).to.equal(expectedInitStatus)
471
c0e8b12e 472 return initializeSessionRes.body.video || initializeSessionRes.body
d23dd9fb
C
473 }
474
475 async prepareResumableUpload (options: OverrideCommandOptions & {
476 attributes: VideoEdit
477 size: number
478 mimetype: string
020d3d3d
C
479
480 originalName?: string
481 lastModified?: number
d23dd9fb 482 }) {
020d3d3d 483 const { attributes, originalName, lastModified, size, mimetype } = options
d23dd9fb
C
484
485 const path = '/api/v1/videos/upload-resumable'
486
487 return this.postUploadRequest({
488 ...options,
489
490 path,
491 headers: {
492 'X-Upload-Content-Type': mimetype,
493 'X-Upload-Content-Length': size.toString()
494 },
020d3d3d
C
495 fields: {
496 filename: attributes.fixture,
497 originalName,
498 lastModified,
499
500 ...this.buildUploadFields(options.attributes)
501 },
502
c0e8b12e 503 // Fixture will be sent later
bbd5aa7e 504 attaches: this.buildUploadAttaches(omit(options.attributes, [ 'fixture' ])),
d23dd9fb 505 implicitToken: true,
c0e8b12e 506
d23dd9fb
C
507 defaultExpectedStatus: null
508 })
509 }
510
511 sendResumableChunks (options: OverrideCommandOptions & {
512 pathUploadId: string
513 videoFilePath: string
514 size: number
515 contentLength?: number
516 contentRangeBuilder?: (start: number, chunk: any) => string
33ac85bf 517 digestBuilder?: (chunk: any) => string
d23dd9fb 518 }) {
33ac85bf
C
519 const {
520 pathUploadId,
521 videoFilePath,
522 size,
523 contentLength,
524 contentRangeBuilder,
525 digestBuilder,
526 expectedStatus = HttpStatusCode.OK_200
527 } = options
d23dd9fb
C
528
529 const path = '/api/v1/videos/upload-resumable'
530 let start = 0
531
532 const token = this.buildCommonRequestToken({ ...options, implicitToken: true })
533 const url = this.server.url
534
535 const readable = createReadStream(videoFilePath, { highWaterMark: 8 * 1024 })
536 return new Promise<GotResponse<{ video: VideoCreateResult }>>((resolve, reject) => {
537 readable.on('data', async function onData (chunk) {
538 readable.pause()
539
540 const headers = {
541 'Authorization': 'Bearer ' + token,
542 'Content-Type': 'application/octet-stream',
543 'Content-Range': contentRangeBuilder
544 ? contentRangeBuilder(start, chunk)
545 : `bytes ${start}-${start + chunk.length - 1}/${size}`,
546 'Content-Length': contentLength ? contentLength + '' : chunk.length + ''
547 }
548
33ac85bf
C
549 if (digestBuilder) {
550 Object.assign(headers, { digest: digestBuilder(chunk) })
551 }
552
d23dd9fb
C
553 const res = await got<{ video: VideoCreateResult }>({
554 url,
555 method: 'put',
556 headers,
557 path: path + '?' + pathUploadId,
558 body: chunk,
559 responseType: 'json',
560 throwHttpErrors: false
561 })
562
563 start += chunk.length
564
565 if (res.statusCode === expectedStatus) {
566 return resolve(res)
567 }
568
569 if (res.statusCode !== HttpStatusCode.PERMANENT_REDIRECT_308) {
570 readable.off('data', onData)
571 return reject(new Error('Incorrect transient behaviour sending intermediary chunks'))
572 }
573
574 readable.resume()
575 })
576 })
577 }
578
790c2837
C
579 endResumableUpload (options: OverrideCommandOptions & {
580 pathUploadId: string
581 }) {
582 return this.deleteRequest({
583 ...options,
584
585 path: '/api/v1/videos/upload-resumable',
586 rawQuery: options.pathUploadId,
587 implicitToken: true,
588 defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204
589 })
590 }
591
d23dd9fb
C
592 quickUpload (options: OverrideCommandOptions & {
593 name: string
594 nsfw?: boolean
595 privacy?: VideoPrivacy
596 fixture?: string
597 }) {
598 const attributes: VideoEdit = { name: options.name }
599 if (options.nsfw) attributes.nsfw = options.nsfw
600 if (options.privacy) attributes.privacy = options.privacy
601 if (options.fixture) attributes.fixture = options.fixture
602
603 return this.upload({ ...options, attributes })
604 }
605
606 async randomUpload (options: OverrideCommandOptions & {
607 wait?: boolean // default true
4c7e60bc 608 additionalParams?: VideoEdit & { prefixName?: string }
d23dd9fb
C
609 } = {}) {
610 const { wait = true, additionalParams } = options
611 const prefixName = additionalParams?.prefixName || ''
612 const name = prefixName + buildUUID()
613
4c7e60bc 614 const attributes = { name, ...additionalParams }
d23dd9fb 615
d23dd9fb
C
616 const result = await this.upload({ ...options, attributes })
617
c0e8b12e
C
618 if (wait) await waitJobs([ this.server ])
619
d23dd9fb
C
620 return { ...result, name }
621 }
622
623 // ---------------------------------------------------------------------------
624
1bb4c9ab 625 removeHLSPlaylist (options: OverrideCommandOptions & {
b46cf4b9
C
626 videoId: number | string
627 }) {
628 const path = '/api/v1/videos/' + options.videoId + '/hls'
629
630 return this.deleteRequest({
631 ...options,
632
633 path,
634 implicitToken: true,
635 defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204
636 })
637 }
638
1bb4c9ab
C
639 removeHLSFile (options: OverrideCommandOptions & {
640 videoId: number | string
641 fileId: number
642 }) {
643 const path = '/api/v1/videos/' + options.videoId + '/hls/' + options.fileId
644
645 return this.deleteRequest({
646 ...options,
647
648 path,
649 implicitToken: true,
650 defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204
651 })
652 }
653
654 removeAllWebTorrentFiles (options: OverrideCommandOptions & {
b46cf4b9
C
655 videoId: number | string
656 }) {
657 const path = '/api/v1/videos/' + options.videoId + '/webtorrent'
658
659 return this.deleteRequest({
660 ...options,
661
662 path,
663 implicitToken: true,
ad5db104
C
664 defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204
665 })
666 }
667
1bb4c9ab
C
668 removeWebTorrentFile (options: OverrideCommandOptions & {
669 videoId: number | string
670 fileId: number
671 }) {
672 const path = '/api/v1/videos/' + options.videoId + '/webtorrent/' + options.fileId
673
674 return this.deleteRequest({
675 ...options,
676
677 path,
678 implicitToken: true,
679 defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204
680 })
681 }
682
ad5db104
C
683 runTranscoding (options: OverrideCommandOptions & {
684 videoId: number | string
685 transcodingType: 'hls' | 'webtorrent'
686 }) {
687 const path = '/api/v1/videos/' + options.videoId + '/transcoding'
688
689 const fields: VideoTranscodingCreate = pick(options, [ 'transcodingType' ])
690
691 return this.postBodyRequest({
692 ...options,
693
694 path,
695 fields,
696 implicitToken: true,
b46cf4b9
C
697 defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204
698 })
699 }
700
701 // ---------------------------------------------------------------------------
702
d23dd9fb
C
703 private buildListQuery (options: VideosCommonQuery) {
704 return pick(options, [
705 'start',
706 'count',
707 'sort',
708 'nsfw',
709 'isLive',
710 'categoryOneOf',
711 'licenceOneOf',
712 'languageOneOf',
3545e72c 713 'privacyOneOf',
d23dd9fb
C
714 'tagsOneOf',
715 'tagsAllOf',
2760b454
C
716 'isLocal',
717 'include',
d23dd9fb
C
718 'skipCount'
719 ])
720 }
721
722 private buildUploadFields (attributes: VideoEdit) {
c0e8b12e 723 return omit(attributes, [ 'fixture', 'thumbnailfile', 'previewfile' ])
d23dd9fb
C
724 }
725
726 private buildUploadAttaches (attributes: VideoEdit) {
727 const attaches: { [ name: string ]: string } = {}
728
729 for (const key of [ 'thumbnailfile', 'previewfile' ]) {
730 if (attributes[key]) attaches[key] = buildAbsoluteFixturePath(attributes[key])
731 }
732
733 if (attributes.fixture) attaches.videofile = buildAbsoluteFixturePath(attributes.fixture)
734
735 return attaches
736 }
737}