1 /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
4 import { expect } from 'chai'
5 import { pathExists, readdir, remove } from 'fs-extra'
6 import { join } from 'path'
7 import { FIXTURE_URLS, testCaptionFile, testImage } from '@server/tests/shared'
8 import { areHttpImportTestsDisabled } from '@shared/core-utils'
9 import { CustomConfig, HttpStatusCode, Video, VideoImportState, VideoPrivacy, VideoResolution, VideoState } from '@shared/models'
12 createMultipleServers,
15 getServerImportConfig,
17 setAccessTokensToServers,
18 setDefaultVideoChannel,
20 } from '@shared/server-commands'
21 import { DeepPartial } from '@shared/typescript-utils'
23 async function checkVideosServer1 (server: PeerTubeServer, idHttp: string, idMagnet: string, idTorrent: string) {
24 const videoHttp = await server.videos.get({ id: idHttp })
26 expect(videoHttp.name).to.equal('small video - youtube')
27 // FIXME: youtube-dl seems broken
28 // expect(videoHttp.category.label).to.equal('News & Politics')
29 // expect(videoHttp.licence.label).to.equal('Attribution')
30 expect(videoHttp.language.label).to.equal('Unknown')
31 expect(videoHttp.nsfw).to.be.false
32 expect(videoHttp.description).to.equal('this is a super description')
33 expect(videoHttp.tags).to.deep.equal([ 'tag1', 'tag2' ])
34 expect(videoHttp.files).to.have.lengthOf(1)
36 const originallyPublishedAt = new Date(videoHttp.originallyPublishedAt)
37 expect(originallyPublishedAt.getDate()).to.equal(14)
38 expect(originallyPublishedAt.getMonth()).to.equal(0)
39 expect(originallyPublishedAt.getFullYear()).to.equal(2019)
41 const videoMagnet = await server.videos.get({ id: idMagnet })
42 const videoTorrent = await server.videos.get({ id: idTorrent })
44 for (const video of [ videoMagnet, videoTorrent ]) {
45 expect(video.category.label).to.equal('Misc')
46 expect(video.licence.label).to.equal('Unknown')
47 expect(video.language.label).to.equal('Unknown')
48 expect(video.nsfw).to.be.false
49 expect(video.description).to.equal('this is a super torrent description')
50 expect(video.tags).to.deep.equal([ 'tag_torrent1', 'tag_torrent2' ])
51 expect(video.files).to.have.lengthOf(1)
54 expect(videoTorrent.name).to.contain('你好 世界 720p.mp4')
55 expect(videoMagnet.name).to.contain('super peertube2 video')
57 const bodyCaptions = await server.captions.list({ videoId: idHttp })
58 expect(bodyCaptions.total).to.equal(2)
61 async function checkVideoServer2 (server: PeerTubeServer, id: number | string) {
62 const video = await server.videos.get({ id })
64 expect(video.name).to.equal('my super name')
65 expect(video.category.label).to.equal('Entertainment')
66 expect(video.licence.label).to.equal('Public Domain Dedication')
67 expect(video.language.label).to.equal('English')
68 expect(video.nsfw).to.be.false
69 expect(video.description).to.equal('my super description')
70 expect(video.tags).to.deep.equal([ 'supertag1', 'supertag2' ])
72 expect(video.files).to.have.lengthOf(1)
74 const bodyCaptions = await server.captions.list({ videoId: id })
75 expect(bodyCaptions.total).to.equal(2)
78 describe('Test video imports', function () {
80 if (areHttpImportTestsDisabled()) return
82 function runSuite (mode: 'youtube-dl' | 'yt-dlp') {
84 describe('Import ' + mode, function () {
85 let servers: PeerTubeServer[] = []
87 before(async function () {
90 servers = await createMultipleServers(2, getServerImportConfig(mode))
92 await setAccessTokensToServers(servers)
93 await setDefaultVideoChannel(servers)
95 for (const server of servers) {
96 await server.config.updateExistingSubConfig({
99 alwaysTranscodeOriginalResolution: false
105 await doubleFollow(servers[0], servers[1])
108 it('Should import videos on server 1', async function () {
111 const baseAttributes = {
112 channelId: servers[0].store.channel.id,
113 privacy: VideoPrivacy.PUBLIC
117 const attributes = { ...baseAttributes, targetUrl: FIXTURE_URLS.youtube }
118 const { video } = await servers[0].imports.importVideo({ attributes })
119 expect(video.name).to.equal('small video - youtube')
122 expect(video.thumbnailPath).to.match(new RegExp(`^/static/thumbnails/.+.jpg$`))
123 expect(video.previewPath).to.match(new RegExp(`^/lazy-static/previews/.+.jpg$`))
125 const suffix = mode === 'yt-dlp'
129 await testImage(servers[0].url, 'video_import_thumbnail' + suffix, video.thumbnailPath)
130 await testImage(servers[0].url, 'video_import_preview' + suffix, video.previewPath)
133 const bodyCaptions = await servers[0].captions.list({ videoId: video.id })
134 const videoCaptions = bodyCaptions.data
135 expect(videoCaptions).to.have.lengthOf(2)
138 const enCaption = videoCaptions.find(caption => caption.language.id === 'en')
139 expect(enCaption).to.exist
140 expect(enCaption.language.label).to.equal('English')
141 expect(enCaption.captionPath).to.match(new RegExp(`^/lazy-static/video-captions/.+-en.vtt$`))
143 const regex = `WEBVTT[ \n]+Kind: captions[ \n]+` +
144 `(Language: en[ \n]+)?` +
145 `00:00:01.600 --> 00:00:04.200( position:\\d+% line:\\d+%)?[ \n]+English \\(US\\)[ \n]+` +
146 `00:00:05.900 --> 00:00:07.999( position:\\d+% line:\\d+%)?[ \n]+This is a subtitle in American English[ \n]+` +
147 `00:00:10.000 --> 00:00:14.000( position:\\d+% line:\\d+%)?[ \n]+Adding subtitles is very easy to do`
148 await testCaptionFile(servers[0].url, enCaption.captionPath, new RegExp(regex))
152 const frCaption = videoCaptions.find(caption => caption.language.id === 'fr')
153 expect(frCaption).to.exist
154 expect(frCaption.language.label).to.equal('French')
155 expect(frCaption.captionPath).to.match(new RegExp(`^/lazy-static/video-captions/.+-fr.vtt`))
157 const regex = `WEBVTT[ \n]+Kind: captions[ \n]+` +
158 `(Language: fr[ \n]+)?` +
159 `00:00:01.600 --> 00:00:04.200( position:\\d+% line:\\d+%)?[ \n]+Français \\(FR\\)[ \n]+` +
160 `00:00:05.900 --> 00:00:07.999( position:\\d+% line:\\d+%)?[ \n]+C'est un sous-titre français[ \n]+` +
161 `00:00:10.000 --> 00:00:14.000( position:\\d+% line:\\d+%)?[ \n]+Ajouter un sous-titre est vraiment facile`
163 await testCaptionFile(servers[0].url, frCaption.captionPath, new RegExp(regex))
170 magnetUri: FIXTURE_URLS.magnet,
171 description: 'this is a super torrent description',
172 tags: [ 'tag_torrent1', 'tag_torrent2' ]
174 const { video } = await servers[0].imports.importVideo({ attributes })
175 expect(video.name).to.equal('super peertube2 video')
181 torrentfile: 'video-720p.torrent' as any,
182 description: 'this is a super torrent description',
183 tags: [ 'tag_torrent1', 'tag_torrent2' ]
185 const { video } = await servers[0].imports.importVideo({ attributes })
186 expect(video.name).to.equal('你好 世界 720p.mp4')
190 it('Should list the videos to import in my videos on server 1', async function () {
191 const { total, data } = await servers[0].videos.listMyVideos({ sort: 'createdAt' })
193 expect(total).to.equal(3)
195 expect(data).to.have.lengthOf(3)
196 expect(data[0].name).to.equal('small video - youtube')
197 expect(data[1].name).to.equal('super peertube2 video')
198 expect(data[2].name).to.equal('你好 世界 720p.mp4')
201 it('Should list the videos to import in my imports on server 1', async function () {
202 const { total, data: videoImports } = await servers[0].imports.getMyVideoImports({ sort: '-createdAt' })
203 expect(total).to.equal(3)
205 expect(videoImports).to.have.lengthOf(3)
207 expect(videoImports[2].targetUrl).to.equal(FIXTURE_URLS.youtube)
208 expect(videoImports[2].magnetUri).to.be.null
209 expect(videoImports[2].torrentName).to.be.null
210 expect(videoImports[2].video.name).to.equal('small video - youtube')
212 expect(videoImports[1].targetUrl).to.be.null
213 expect(videoImports[1].magnetUri).to.equal(FIXTURE_URLS.magnet)
214 expect(videoImports[1].torrentName).to.be.null
215 expect(videoImports[1].video.name).to.equal('super peertube2 video')
217 expect(videoImports[0].targetUrl).to.be.null
218 expect(videoImports[0].magnetUri).to.be.null
219 expect(videoImports[0].torrentName).to.equal('video-720p.torrent')
220 expect(videoImports[0].video.name).to.equal('你好 世界 720p.mp4')
223 it('Should filter my imports on target URL', async function () {
224 const { total, data: videoImports } = await servers[0].imports.getMyVideoImports({ targetUrl: FIXTURE_URLS.youtube })
225 expect(total).to.equal(1)
226 expect(videoImports).to.have.lengthOf(1)
228 expect(videoImports[0].targetUrl).to.equal(FIXTURE_URLS.youtube)
231 it('Should search in my imports', async function () {
232 const { total, data: videoImports } = await servers[0].imports.getMyVideoImports({ search: 'peertube2' })
233 expect(total).to.equal(1)
234 expect(videoImports).to.have.lengthOf(1)
236 expect(videoImports[0].magnetUri).to.equal(FIXTURE_URLS.magnet)
237 expect(videoImports[0].video.name).to.equal('super peertube2 video')
240 it('Should have the video listed on the two instances', async function () {
241 this.timeout(120_000)
243 await waitJobs(servers)
245 for (const server of servers) {
246 const { total, data } = await server.videos.list()
247 expect(total).to.equal(3)
248 expect(data).to.have.lengthOf(3)
250 const [ videoHttp, videoMagnet, videoTorrent ] = data
251 await checkVideosServer1(server, videoHttp.uuid, videoMagnet.uuid, videoTorrent.uuid)
255 it('Should import a video on server 2 with some fields', async function () {
259 targetUrl: FIXTURE_URLS.youtube,
260 channelId: servers[1].store.channel.id,
261 privacy: VideoPrivacy.PUBLIC,
265 name: 'my super name',
266 description: 'my super description',
267 tags: [ 'supertag1', 'supertag2' ]
269 const { video } = await servers[1].imports.importVideo({ attributes })
270 expect(video.name).to.equal('my super name')
273 it('Should have the videos listed on the two instances', async function () {
274 this.timeout(120_000)
276 await waitJobs(servers)
278 for (const server of servers) {
279 const { total, data } = await server.videos.list()
280 expect(total).to.equal(4)
281 expect(data).to.have.lengthOf(4)
283 await checkVideoServer2(server, data[0].uuid)
285 const [ , videoHttp, videoMagnet, videoTorrent ] = data
286 await checkVideosServer1(server, videoHttp.uuid, videoMagnet.uuid, videoTorrent.uuid)
290 it('Should import a video that will be transcoded', async function () {
291 this.timeout(240_000)
294 name: 'transcoded video',
295 magnetUri: FIXTURE_URLS.magnet,
296 channelId: servers[1].store.channel.id,
297 privacy: VideoPrivacy.PUBLIC
299 const { video } = await servers[1].imports.importVideo({ attributes })
300 const videoUUID = video.uuid
302 await waitJobs(servers)
304 for (const server of servers) {
305 const video = await server.videos.get({ id: videoUUID })
307 expect(video.name).to.equal('transcoded video')
308 expect(video.files).to.have.lengthOf(4)
312 it('Should import no HDR version on a HDR video', async function () {
313 this.timeout(300_000)
315 const config: DeepPartial<CustomConfig> = {
325 '1080p': false, // the resulting resolution shouldn't be higher than this, and not vp9.2/av01
329 webtorrent: { enabled: true },
330 hls: { enabled: false }
333 await servers[0].config.updateExistingSubConfig({ newConfig: config })
337 targetUrl: FIXTURE_URLS.youtubeHDR,
338 channelId: servers[0].store.channel.id,
339 privacy: VideoPrivacy.PUBLIC
341 const { video: videoImported } = await servers[0].imports.importVideo({ attributes })
342 const videoUUID = videoImported.uuid
344 await waitJobs(servers)
347 const video = await servers[0].videos.get({ id: videoUUID })
348 expect(video.name).to.equal('hdr video')
349 const maxResolution = Math.max.apply(Math, video.files.map(function (o) { return o.resolution.id }))
350 expect(maxResolution, 'expected max resolution not met').to.equals(VideoResolution.H_240P)
353 it('Should not import resolution higher than enabled transcoding resolution', async function () {
354 this.timeout(300_000)
356 const config: DeepPartial<CustomConfig> = {
370 alwaysTranscodeOriginalResolution: false
373 await servers[0].config.updateExistingSubConfig({ newConfig: config })
376 name: 'small resolution video',
377 targetUrl: FIXTURE_URLS.youtube,
378 channelId: servers[0].store.channel.id,
379 privacy: VideoPrivacy.PUBLIC
381 const { video: videoImported } = await servers[0].imports.importVideo({ attributes })
382 const videoUUID = videoImported.uuid
384 await waitJobs(servers)
387 const video = await servers[0].videos.get({ id: videoUUID })
388 expect(video.name).to.equal('small resolution video')
389 expect(video.files).to.have.lengthOf(1)
390 expect(video.files[0].resolution.id).to.equal(144)
393 it('Should import resolution higher than enabled transcoding resolution', async function () {
394 this.timeout(300_000)
396 const config: DeepPartial<CustomConfig> = {
398 alwaysTranscodeOriginalResolution: true
401 await servers[0].config.updateExistingSubConfig({ newConfig: config })
404 name: 'bigger resolution video',
405 targetUrl: FIXTURE_URLS.youtube,
406 channelId: servers[0].store.channel.id,
407 privacy: VideoPrivacy.PUBLIC
409 const { video: videoImported } = await servers[0].imports.importVideo({ attributes })
410 const videoUUID = videoImported.uuid
412 await waitJobs(servers)
415 const video = await servers[0].videos.get({ id: videoUUID })
416 expect(video.name).to.equal('bigger resolution video')
418 expect(video.files).to.have.lengthOf(2)
419 expect(video.files.find(f => f.resolution.id === 240)).to.exist
420 expect(video.files.find(f => f.resolution.id === 144)).to.exist
423 it('Should import a peertube video', async function () {
424 this.timeout(120_000)
426 const toTest = [ FIXTURE_URLS.peertube_long ]
428 // TODO: include peertube_short when https://github.com/ytdl-org/youtube-dl/pull/29475 is merged
429 if (mode === 'yt-dlp') {
430 toTest.push(FIXTURE_URLS.peertube_short)
433 for (const targetUrl of toTest) {
434 await servers[0].config.disableTranscoding()
438 channelId: servers[0].store.channel.id,
439 privacy: VideoPrivacy.PUBLIC
441 const { video } = await servers[0].imports.importVideo({ attributes })
442 const videoUUID = video.uuid
444 await waitJobs(servers)
446 for (const server of servers) {
447 const video = await server.videos.get({ id: videoUUID })
449 expect(video.name).to.equal('E2E tests')
454 after(async function () {
455 await cleanupTests(servers)
460 runSuite('youtube-dl')
464 describe('Delete/cancel an import', function () {
465 let server: PeerTubeServer
467 let finishedImportId: number
468 let finishedVideo: Video
469 let pendingImportId: number
471 async function importVideo (name: string) {
472 const attributes = { name, channelId: server.store.channel.id, targetUrl: FIXTURE_URLS.goodVideo }
473 const res = await server.imports.importVideo({ attributes })
478 before(async function () {
479 this.timeout(120_000)
481 server = await createSingleServer(1)
483 await setAccessTokensToServers([ server ])
484 await setDefaultVideoChannel([ server ])
486 finishedImportId = await importVideo('finished')
487 await waitJobs([ server ])
489 await server.jobs.pauseJobQueue()
490 pendingImportId = await importVideo('pending')
492 const { data } = await server.imports.getMyVideoImports()
493 expect(data).to.have.lengthOf(2)
495 finishedVideo = data.find(i => i.id === finishedImportId).video
498 it('Should delete a video import', async function () {
499 await server.imports.delete({ importId: finishedImportId })
501 const { data } = await server.imports.getMyVideoImports()
502 expect(data).to.have.lengthOf(1)
503 expect(data[0].id).to.equal(pendingImportId)
504 expect(data[0].state.id).to.equal(VideoImportState.PENDING)
507 it('Should not have deleted the associated video', async function () {
508 const video = await server.videos.get({ id: finishedVideo.id, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 })
509 expect(video.name).to.equal('finished')
510 expect(video.state.id).to.equal(VideoState.PUBLISHED)
513 it('Should cancel a video import', async function () {
514 await server.imports.cancel({ importId: pendingImportId })
516 const { data } = await server.imports.getMyVideoImports()
517 expect(data).to.have.lengthOf(1)
518 expect(data[0].id).to.equal(pendingImportId)
519 expect(data[0].state.id).to.equal(VideoImportState.CANCELLED)
522 it('Should not have processed the cancelled video import', async function () {
525 await server.jobs.resumeJobQueue()
527 await waitJobs([ server ])
529 const { data } = await server.imports.getMyVideoImports()
530 expect(data).to.have.lengthOf(1)
531 expect(data[0].id).to.equal(pendingImportId)
532 expect(data[0].state.id).to.equal(VideoImportState.CANCELLED)
533 expect(data[0].video.state.id).to.equal(VideoState.TO_IMPORT)
536 it('Should delete the cancelled video import', async function () {
537 await server.imports.delete({ importId: pendingImportId })
538 const { data } = await server.imports.getMyVideoImports()
539 expect(data).to.have.lengthOf(0)
542 after(async function () {
543 await cleanupTests([ server ])
547 describe('Auto update', function () {
548 let server: PeerTubeServer
550 function quickPeerTubeImport () {
552 targetUrl: FIXTURE_URLS.peertube_long,
553 channelId: server.store.channel.id,
554 privacy: VideoPrivacy.PUBLIC
557 return server.imports.importVideo({ attributes })
560 async function testBinaryUpdate (releaseUrl: string, releaseName: string) {
561 await remove(join(server.servers.buildDirectory('bin'), releaseName))
568 youtube_dl_release: {
577 await quickPeerTubeImport()
579 const base = server.servers.buildDirectory('bin')
580 const content = await readdir(base)
581 const binaryPath = join(base, releaseName)
583 expect(await pathExists(binaryPath), `${binaryPath} does not exist in ${base} (${content.join(', ')})`).to.be.true
586 before(async function () {
590 server = await createSingleServer(1)
592 await setAccessTokensToServers([ server ])
593 await setDefaultVideoChannel([ server ])
596 it('Should update youtube-dl from github URL', async function () {
597 this.timeout(120_000)
599 await testBinaryUpdate('https://api.github.com/repos/ytdl-org/youtube-dl/releases', 'youtube-dl')
602 it('Should update youtube-dl from raw URL', async function () {
603 this.timeout(120_000)
605 await testBinaryUpdate('https://yt-dl.org/downloads/latest/youtube-dl', 'youtube-dl')
608 it('Should update youtube-dl from youtube-dl fork', async function () {
609 this.timeout(120_000)
611 await testBinaryUpdate('https://api.github.com/repos/yt-dlp/yt-dlp/releases', 'yt-dlp')
614 after(async function () {
615 await cleanupTests([ server ])