]> git.immae.eu Git - github/Chocobozzz/PeerTube.git/blob - server/tests/api/videos/video-imports.ts
Bumped to version v5.2.1
[github/Chocobozzz/PeerTube.git] / server / tests / api / videos / video-imports.ts
1 /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
2
3 import { expect } from 'chai'
4 import { pathExists, readdir, remove } from 'fs-extra'
5 import { join } from 'path'
6 import { FIXTURE_URLS, testCaptionFile, testImage } from '@server/tests/shared'
7 import { areHttpImportTestsDisabled } from '@shared/core-utils'
8 import { CustomConfig, HttpStatusCode, Video, VideoImportState, VideoPrivacy, VideoResolution, VideoState } from '@shared/models'
9 import {
10 cleanupTests,
11 createMultipleServers,
12 createSingleServer,
13 doubleFollow,
14 getServerImportConfig,
15 PeerTubeServer,
16 setAccessTokensToServers,
17 setDefaultVideoChannel,
18 waitJobs
19 } from '@shared/server-commands'
20 import { DeepPartial } from '@shared/typescript-utils'
21
22 async function checkVideosServer1 (server: PeerTubeServer, idHttp: string, idMagnet: string, idTorrent: string) {
23 const videoHttp = await server.videos.get({ id: idHttp })
24
25 expect(videoHttp.name).to.equal('small video - youtube')
26 expect(videoHttp.category.label).to.equal('News & Politics')
27 expect(videoHttp.licence.label).to.equal('Attribution')
28 expect(videoHttp.language.label).to.equal('Unknown')
29 expect(videoHttp.nsfw).to.be.false
30 expect(videoHttp.description).to.equal('this is a super description')
31 expect(videoHttp.tags).to.deep.equal([ 'tag1', 'tag2' ])
32 expect(videoHttp.files).to.have.lengthOf(1)
33
34 const originallyPublishedAt = new Date(videoHttp.originallyPublishedAt)
35 expect(originallyPublishedAt.getDate()).to.equal(14)
36 expect(originallyPublishedAt.getMonth()).to.equal(0)
37 expect(originallyPublishedAt.getFullYear()).to.equal(2019)
38
39 const videoMagnet = await server.videos.get({ id: idMagnet })
40 const videoTorrent = await server.videos.get({ id: idTorrent })
41
42 for (const video of [ videoMagnet, videoTorrent ]) {
43 expect(video.category.label).to.equal('Unknown')
44 expect(video.licence.label).to.equal('Unknown')
45 expect(video.language.label).to.equal('Unknown')
46 expect(video.nsfw).to.be.false
47 expect(video.description).to.equal('this is a super torrent description')
48 expect(video.tags).to.deep.equal([ 'tag_torrent1', 'tag_torrent2' ])
49 expect(video.files).to.have.lengthOf(1)
50 }
51
52 expect(videoTorrent.name).to.contain('你好 世界 720p.mp4')
53 expect(videoMagnet.name).to.contain('super peertube2 video')
54
55 const bodyCaptions = await server.captions.list({ videoId: idHttp })
56 expect(bodyCaptions.total).to.equal(2)
57 }
58
59 async function checkVideoServer2 (server: PeerTubeServer, id: number | string) {
60 const video = await server.videos.get({ id })
61
62 expect(video.name).to.equal('my super name')
63 expect(video.category.label).to.equal('Entertainment')
64 expect(video.licence.label).to.equal('Public Domain Dedication')
65 expect(video.language.label).to.equal('English')
66 expect(video.nsfw).to.be.false
67 expect(video.description).to.equal('my super description')
68 expect(video.tags).to.deep.equal([ 'supertag1', 'supertag2' ])
69
70 await testImage(server.url, 'thumbnail', video.thumbnailPath)
71
72 expect(video.files).to.have.lengthOf(1)
73
74 const bodyCaptions = await server.captions.list({ videoId: id })
75 expect(bodyCaptions.total).to.equal(2)
76 }
77
78 describe('Test video imports', function () {
79
80 if (areHttpImportTestsDisabled()) return
81
82 function runSuite (mode: 'youtube-dl' | 'yt-dlp') {
83
84 describe('Import ' + mode, function () {
85 let servers: PeerTubeServer[] = []
86
87 before(async function () {
88 this.timeout(60_000)
89
90 servers = await createMultipleServers(2, getServerImportConfig(mode))
91
92 await setAccessTokensToServers(servers)
93 await setDefaultVideoChannel(servers)
94
95 for (const server of servers) {
96 await server.config.updateExistingSubConfig({
97 newConfig: {
98 transcoding: {
99 alwaysTranscodeOriginalResolution: false
100 }
101 }
102 })
103 }
104
105 await doubleFollow(servers[0], servers[1])
106 })
107
108 it('Should import videos on server 1', async function () {
109 this.timeout(60_000)
110
111 const baseAttributes = {
112 channelId: servers[0].store.channel.id,
113 privacy: VideoPrivacy.PUBLIC
114 }
115
116 {
117 const attributes = { ...baseAttributes, targetUrl: FIXTURE_URLS.youtube }
118 const { video } = await servers[0].imports.importVideo({ attributes })
119 expect(video.name).to.equal('small video - youtube')
120
121 {
122 expect(video.thumbnailPath).to.match(new RegExp(`^/static/thumbnails/.+.jpg$`))
123 expect(video.previewPath).to.match(new RegExp(`^/lazy-static/previews/.+.jpg$`))
124
125 const suffix = mode === 'yt-dlp'
126 ? '_yt_dlp'
127 : ''
128
129 await testImage(servers[0].url, 'video_import_thumbnail' + suffix, video.thumbnailPath)
130 await testImage(servers[0].url, 'video_import_preview' + suffix, video.previewPath)
131 }
132
133 const bodyCaptions = await servers[0].captions.list({ videoId: video.id })
134 const videoCaptions = bodyCaptions.data
135 expect(videoCaptions).to.have.lengthOf(2)
136
137 {
138 const enCaption = videoCaptions.find(caption => caption.language.id === 'en')
139 expect(enCaption).to.exist
140 expect(enCaption.language.label).to.equal('English')
141 expect(enCaption.captionPath).to.match(new RegExp(`^/lazy-static/video-captions/.+-en.vtt$`))
142
143 const regex = `WEBVTT[ \n]+Kind: captions[ \n]+` +
144 `(Language: en[ \n]+)?` +
145 `00:00:01.600 --> 00:00:04.200( position:\\d+% line:\\d+%)?[ \n]+English \\(US\\)[ \n]+` +
146 `00:00:05.900 --> 00:00:07.999( position:\\d+% line:\\d+%)?[ \n]+This is a subtitle in American English[ \n]+` +
147 `00:00:10.000 --> 00:00:14.000( position:\\d+% line:\\d+%)?[ \n]+Adding subtitles is very easy to do`
148 await testCaptionFile(servers[0].url, enCaption.captionPath, new RegExp(regex))
149 }
150
151 {
152 const frCaption = videoCaptions.find(caption => caption.language.id === 'fr')
153 expect(frCaption).to.exist
154 expect(frCaption.language.label).to.equal('French')
155 expect(frCaption.captionPath).to.match(new RegExp(`^/lazy-static/video-captions/.+-fr.vtt`))
156
157 const regex = `WEBVTT[ \n]+Kind: captions[ \n]+` +
158 `(Language: fr[ \n]+)?` +
159 `00:00:01.600 --> 00:00:04.200( position:\\d+% line:\\d+%)?[ \n]+Français \\(FR\\)[ \n]+` +
160 `00:00:05.900 --> 00:00:07.999( position:\\d+% line:\\d+%)?[ \n]+C'est un sous-titre français[ \n]+` +
161 `00:00:10.000 --> 00:00:14.000( position:\\d+% line:\\d+%)?[ \n]+Ajouter un sous-titre est vraiment facile`
162
163 await testCaptionFile(servers[0].url, frCaption.captionPath, new RegExp(regex))
164 }
165 }
166
167 {
168 const attributes = {
169 ...baseAttributes,
170 magnetUri: FIXTURE_URLS.magnet,
171 description: 'this is a super torrent description',
172 tags: [ 'tag_torrent1', 'tag_torrent2' ]
173 }
174 const { video } = await servers[0].imports.importVideo({ attributes })
175 expect(video.name).to.equal('super peertube2 video')
176 }
177
178 {
179 const attributes = {
180 ...baseAttributes,
181 torrentfile: 'video-720p.torrent' as any,
182 description: 'this is a super torrent description',
183 tags: [ 'tag_torrent1', 'tag_torrent2' ]
184 }
185 const { video } = await servers[0].imports.importVideo({ attributes })
186 expect(video.name).to.equal('你好 世界 720p.mp4')
187 }
188 })
189
190 it('Should list the videos to import in my videos on server 1', async function () {
191 const { total, data } = await servers[0].videos.listMyVideos({ sort: 'createdAt' })
192
193 expect(total).to.equal(3)
194
195 expect(data).to.have.lengthOf(3)
196 expect(data[0].name).to.equal('small video - youtube')
197 expect(data[1].name).to.equal('super peertube2 video')
198 expect(data[2].name).to.equal('你好 世界 720p.mp4')
199 })
200
201 it('Should list the videos to import in my imports on server 1', async function () {
202 const { total, data: videoImports } = await servers[0].imports.getMyVideoImports({ sort: '-createdAt' })
203 expect(total).to.equal(3)
204
205 expect(videoImports).to.have.lengthOf(3)
206
207 expect(videoImports[2].targetUrl).to.equal(FIXTURE_URLS.youtube)
208 expect(videoImports[2].magnetUri).to.be.null
209 expect(videoImports[2].torrentName).to.be.null
210 expect(videoImports[2].video.name).to.equal('small video - youtube')
211
212 expect(videoImports[1].targetUrl).to.be.null
213 expect(videoImports[1].magnetUri).to.equal(FIXTURE_URLS.magnet)
214 expect(videoImports[1].torrentName).to.be.null
215 expect(videoImports[1].video.name).to.equal('super peertube2 video')
216
217 expect(videoImports[0].targetUrl).to.be.null
218 expect(videoImports[0].magnetUri).to.be.null
219 expect(videoImports[0].torrentName).to.equal('video-720p.torrent')
220 expect(videoImports[0].video.name).to.equal('你好 世界 720p.mp4')
221 })
222
223 it('Should filter my imports on target URL', async function () {
224 const { total, data: videoImports } = await servers[0].imports.getMyVideoImports({ targetUrl: FIXTURE_URLS.youtube })
225 expect(total).to.equal(1)
226 expect(videoImports).to.have.lengthOf(1)
227
228 expect(videoImports[0].targetUrl).to.equal(FIXTURE_URLS.youtube)
229 })
230
231 it('Should search in my imports', async function () {
232 const { total, data: videoImports } = await servers[0].imports.getMyVideoImports({ search: 'peertube2' })
233 expect(total).to.equal(1)
234 expect(videoImports).to.have.lengthOf(1)
235
236 expect(videoImports[0].magnetUri).to.equal(FIXTURE_URLS.magnet)
237 expect(videoImports[0].video.name).to.equal('super peertube2 video')
238 })
239
240 it('Should have the video listed on the two instances', async function () {
241 this.timeout(120_000)
242
243 await waitJobs(servers)
244
245 for (const server of servers) {
246 const { total, data } = await server.videos.list()
247 expect(total).to.equal(3)
248 expect(data).to.have.lengthOf(3)
249
250 const [ videoHttp, videoMagnet, videoTorrent ] = data
251 await checkVideosServer1(server, videoHttp.uuid, videoMagnet.uuid, videoTorrent.uuid)
252 }
253 })
254
255 it('Should import a video on server 2 with some fields', async function () {
256 this.timeout(60_000)
257
258 const { video } = await servers[1].imports.importVideo({
259 attributes: {
260 targetUrl: FIXTURE_URLS.youtube,
261 channelId: servers[1].store.channel.id,
262 privacy: VideoPrivacy.PUBLIC,
263 category: 10,
264 licence: 7,
265 language: 'en',
266 name: 'my super name',
267 description: 'my super description',
268 tags: [ 'supertag1', 'supertag2' ],
269 thumbnailfile: 'thumbnail.jpg'
270 }
271 })
272 expect(video.name).to.equal('my super name')
273 })
274
275 it('Should have the videos listed on the two instances', async function () {
276 this.timeout(120_000)
277
278 await waitJobs(servers)
279
280 for (const server of servers) {
281 const { total, data } = await server.videos.list()
282 expect(total).to.equal(4)
283 expect(data).to.have.lengthOf(4)
284
285 await checkVideoServer2(server, data[0].uuid)
286
287 const [ , videoHttp, videoMagnet, videoTorrent ] = data
288 await checkVideosServer1(server, videoHttp.uuid, videoMagnet.uuid, videoTorrent.uuid)
289 }
290 })
291
292 it('Should import a video that will be transcoded', async function () {
293 this.timeout(240_000)
294
295 const attributes = {
296 name: 'transcoded video',
297 magnetUri: FIXTURE_URLS.magnet,
298 channelId: servers[1].store.channel.id,
299 privacy: VideoPrivacy.PUBLIC
300 }
301 const { video } = await servers[1].imports.importVideo({ attributes })
302 const videoUUID = video.uuid
303
304 await waitJobs(servers)
305
306 for (const server of servers) {
307 const video = await server.videos.get({ id: videoUUID })
308
309 expect(video.name).to.equal('transcoded video')
310 expect(video.files).to.have.lengthOf(4)
311 }
312 })
313
314 it('Should import no HDR version on a HDR video', async function () {
315 this.timeout(300_000)
316
317 const config: DeepPartial<CustomConfig> = {
318 transcoding: {
319 enabled: true,
320 resolutions: {
321 '0p': false,
322 '144p': true,
323 '240p': true,
324 '360p': false,
325 '480p': false,
326 '720p': false,
327 '1080p': false, // the resulting resolution shouldn't be higher than this, and not vp9.2/av01
328 '1440p': false,
329 '2160p': false
330 },
331 webtorrent: { enabled: true },
332 hls: { enabled: false }
333 }
334 }
335 await servers[0].config.updateExistingSubConfig({ newConfig: config })
336
337 const attributes = {
338 name: 'hdr video',
339 targetUrl: FIXTURE_URLS.youtubeHDR,
340 channelId: servers[0].store.channel.id,
341 privacy: VideoPrivacy.PUBLIC
342 }
343 const { video: videoImported } = await servers[0].imports.importVideo({ attributes })
344 const videoUUID = videoImported.uuid
345
346 await waitJobs(servers)
347
348 // test resolution
349 const video = await servers[0].videos.get({ id: videoUUID })
350 expect(video.name).to.equal('hdr video')
351 const maxResolution = Math.max.apply(Math, video.files.map(function (o) { return o.resolution.id }))
352 expect(maxResolution, 'expected max resolution not met').to.equals(VideoResolution.H_240P)
353 })
354
355 it('Should not import resolution higher than enabled transcoding resolution', async function () {
356 this.timeout(300_000)
357
358 const config: DeepPartial<CustomConfig> = {
359 transcoding: {
360 enabled: true,
361 resolutions: {
362 '0p': false,
363 '144p': true,
364 '240p': false,
365 '360p': false,
366 '480p': false,
367 '720p': false,
368 '1080p': false,
369 '1440p': false,
370 '2160p': false
371 },
372 alwaysTranscodeOriginalResolution: false
373 }
374 }
375 await servers[0].config.updateExistingSubConfig({ newConfig: config })
376
377 const attributes = {
378 name: 'small resolution video',
379 targetUrl: FIXTURE_URLS.youtube,
380 channelId: servers[0].store.channel.id,
381 privacy: VideoPrivacy.PUBLIC
382 }
383 const { video: videoImported } = await servers[0].imports.importVideo({ attributes })
384 const videoUUID = videoImported.uuid
385
386 await waitJobs(servers)
387
388 // test resolution
389 const video = await servers[0].videos.get({ id: videoUUID })
390 expect(video.name).to.equal('small resolution video')
391 expect(video.files).to.have.lengthOf(1)
392 expect(video.files[0].resolution.id).to.equal(144)
393 })
394
395 it('Should import resolution higher than enabled transcoding resolution', async function () {
396 this.timeout(300_000)
397
398 const config: DeepPartial<CustomConfig> = {
399 transcoding: {
400 alwaysTranscodeOriginalResolution: true
401 }
402 }
403 await servers[0].config.updateExistingSubConfig({ newConfig: config })
404
405 const attributes = {
406 name: 'bigger resolution video',
407 targetUrl: FIXTURE_URLS.youtube,
408 channelId: servers[0].store.channel.id,
409 privacy: VideoPrivacy.PUBLIC
410 }
411 const { video: videoImported } = await servers[0].imports.importVideo({ attributes })
412 const videoUUID = videoImported.uuid
413
414 await waitJobs(servers)
415
416 // test resolution
417 const video = await servers[0].videos.get({ id: videoUUID })
418 expect(video.name).to.equal('bigger resolution video')
419
420 expect(video.files).to.have.lengthOf(2)
421 expect(video.files.find(f => f.resolution.id === 240)).to.exist
422 expect(video.files.find(f => f.resolution.id === 144)).to.exist
423 })
424
425 it('Should import a peertube video', async function () {
426 this.timeout(120_000)
427
428 const toTest = [ FIXTURE_URLS.peertube_long ]
429
430 // TODO: include peertube_short when https://github.com/ytdl-org/youtube-dl/pull/29475 is merged
431 if (mode === 'yt-dlp') {
432 toTest.push(FIXTURE_URLS.peertube_short)
433 }
434
435 for (const targetUrl of toTest) {
436 await servers[0].config.disableTranscoding()
437
438 const attributes = {
439 targetUrl,
440 channelId: servers[0].store.channel.id,
441 privacy: VideoPrivacy.PUBLIC
442 }
443 const { video } = await servers[0].imports.importVideo({ attributes })
444 const videoUUID = video.uuid
445
446 await waitJobs(servers)
447
448 for (const server of servers) {
449 const video = await server.videos.get({ id: videoUUID })
450
451 expect(video.name).to.equal('E2E tests')
452
453 const { data: captions } = await server.captions.list({ videoId: videoUUID })
454 expect(captions).to.have.lengthOf(1)
455 expect(captions[0].language.id).to.equal('fr')
456
457 const str = `WEBVTT FILE\r?\n\r?\n` +
458 `1\r?\n` +
459 `00:00:04.000 --> 00:00:09.000\r?\n` +
460 `January 1, 1994. The North American`
461 await testCaptionFile(server.url, captions[0].captionPath, new RegExp(str))
462 }
463 }
464 })
465
466 after(async function () {
467 await cleanupTests(servers)
468 })
469 })
470 }
471
472 // FIXME: youtube-dl seems broken
473 // runSuite('youtube-dl')
474
475 runSuite('yt-dlp')
476
477 describe('Delete/cancel an import', function () {
478 let server: PeerTubeServer
479
480 let finishedImportId: number
481 let finishedVideo: Video
482 let pendingImportId: number
483
484 async function importVideo (name: string) {
485 const attributes = { name, channelId: server.store.channel.id, targetUrl: FIXTURE_URLS.goodVideo }
486 const res = await server.imports.importVideo({ attributes })
487
488 return res.id
489 }
490
491 before(async function () {
492 this.timeout(120_000)
493
494 server = await createSingleServer(1)
495
496 await setAccessTokensToServers([ server ])
497 await setDefaultVideoChannel([ server ])
498
499 finishedImportId = await importVideo('finished')
500 await waitJobs([ server ])
501
502 await server.jobs.pauseJobQueue()
503 pendingImportId = await importVideo('pending')
504
505 const { data } = await server.imports.getMyVideoImports()
506 expect(data).to.have.lengthOf(2)
507
508 finishedVideo = data.find(i => i.id === finishedImportId).video
509 })
510
511 it('Should delete a video import', async function () {
512 await server.imports.delete({ importId: finishedImportId })
513
514 const { data } = await server.imports.getMyVideoImports()
515 expect(data).to.have.lengthOf(1)
516 expect(data[0].id).to.equal(pendingImportId)
517 expect(data[0].state.id).to.equal(VideoImportState.PENDING)
518 })
519
520 it('Should not have deleted the associated video', async function () {
521 const video = await server.videos.get({ id: finishedVideo.id, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 })
522 expect(video.name).to.equal('finished')
523 expect(video.state.id).to.equal(VideoState.PUBLISHED)
524 })
525
526 it('Should cancel a video import', async function () {
527 await server.imports.cancel({ importId: pendingImportId })
528
529 const { data } = await server.imports.getMyVideoImports()
530 expect(data).to.have.lengthOf(1)
531 expect(data[0].id).to.equal(pendingImportId)
532 expect(data[0].state.id).to.equal(VideoImportState.CANCELLED)
533 })
534
535 it('Should not have processed the cancelled video import', async function () {
536 this.timeout(60_000)
537
538 await server.jobs.resumeJobQueue()
539
540 await waitJobs([ server ])
541
542 const { data } = await server.imports.getMyVideoImports()
543 expect(data).to.have.lengthOf(1)
544 expect(data[0].id).to.equal(pendingImportId)
545 expect(data[0].state.id).to.equal(VideoImportState.CANCELLED)
546 expect(data[0].video.state.id).to.equal(VideoState.TO_IMPORT)
547 })
548
549 it('Should delete the cancelled video import', async function () {
550 await server.imports.delete({ importId: pendingImportId })
551 const { data } = await server.imports.getMyVideoImports()
552 expect(data).to.have.lengthOf(0)
553 })
554
555 after(async function () {
556 await cleanupTests([ server ])
557 })
558 })
559
560 describe('Auto update', function () {
561 let server: PeerTubeServer
562
563 function quickPeerTubeImport () {
564 const attributes = {
565 targetUrl: FIXTURE_URLS.peertube_long,
566 channelId: server.store.channel.id,
567 privacy: VideoPrivacy.PUBLIC
568 }
569
570 return server.imports.importVideo({ attributes })
571 }
572
573 async function testBinaryUpdate (releaseUrl: string, releaseName: string) {
574 await remove(join(server.servers.buildDirectory('bin'), releaseName))
575
576 await server.kill()
577 await server.run({
578 import: {
579 videos: {
580 http: {
581 youtube_dl_release: {
582 url: releaseUrl,
583 name: releaseName
584 }
585 }
586 }
587 }
588 })
589
590 await quickPeerTubeImport()
591
592 const base = server.servers.buildDirectory('bin')
593 const content = await readdir(base)
594 const binaryPath = join(base, releaseName)
595
596 expect(await pathExists(binaryPath), `${binaryPath} does not exist in ${base} (${content.join(', ')})`).to.be.true
597 }
598
599 before(async function () {
600 this.timeout(30_000)
601
602 // Run servers
603 server = await createSingleServer(1)
604
605 await setAccessTokensToServers([ server ])
606 await setDefaultVideoChannel([ server ])
607 })
608
609 it('Should update youtube-dl from github URL', async function () {
610 this.timeout(120_000)
611
612 await testBinaryUpdate('https://api.github.com/repos/ytdl-org/youtube-dl/releases', 'youtube-dl')
613 })
614
615 it('Should update youtube-dl from raw URL', async function () {
616 this.timeout(120_000)
617
618 await testBinaryUpdate('https://yt-dl.org/downloads/latest/youtube-dl', 'youtube-dl')
619 })
620
621 it('Should update youtube-dl from youtube-dl fork', async function () {
622 this.timeout(120_000)
623
624 await testBinaryUpdate('https://api.github.com/repos/yt-dlp/yt-dlp/releases', 'yt-dlp')
625 })
626
627 after(async function () {
628 await cleanupTests([ server ])
629 })
630 })
631 })