server/tests/api/videos/video-imports.ts

   1 /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
   2
   3 import 'mocha'
   4 import { expect } from 'chai'
   5 import { pathExists, readdir, remove } from 'fs-extra'
   6 import { join } from 'path'
   7 import { FIXTURE_URLS, testCaptionFile, testImage } from '@server/tests/shared'
   8 import { areHttpImportTestsDisabled } from '@shared/core-utils'
   9 import { CustomConfig, HttpStatusCode, Video, VideoImportState, VideoPrivacy, VideoResolution, VideoState } from '@shared/models'
  10 import {
  11   cleanupTests,
  12   createMultipleServers,
  13   createSingleServer,
  14   doubleFollow,
  15   PeerTubeServer,
  16   setAccessTokensToServers,
  17   setDefaultVideoChannel,
  18   waitJobs
  19 } from '@shared/server-commands'
  20 import { DeepPartial } from '@shared/typescript-utils'
  21
  22 async function checkVideosServer1 (server: PeerTubeServer, idHttp: string, idMagnet: string, idTorrent: string) {
  23   const videoHttp = await server.videos.get({ id: idHttp })
  24
  25   expect(videoHttp.name).to.equal('small video - youtube')
  26   // FIXME: youtube-dl seems broken
  27   // expect(videoHttp.category.label).to.equal('News & Politics')
  28   // expect(videoHttp.licence.label).to.equal('Attribution')
  29   expect(videoHttp.language.label).to.equal('Unknown')
  30   expect(videoHttp.nsfw).to.be.false
  31   expect(videoHttp.description).to.equal('this is a super description')
  32   expect(videoHttp.tags).to.deep.equal([ 'tag1', 'tag2' ])
  33   expect(videoHttp.files).to.have.lengthOf(1)
  34
  35   const originallyPublishedAt = new Date(videoHttp.originallyPublishedAt)
  36   expect(originallyPublishedAt.getDate()).to.equal(14)
  37   expect(originallyPublishedAt.getMonth()).to.equal(0)
  38   expect(originallyPublishedAt.getFullYear()).to.equal(2019)
  39
  40   const videoMagnet = await server.videos.get({ id: idMagnet })
  41   const videoTorrent = await server.videos.get({ id: idTorrent })
  42
  43   for (const video of [ videoMagnet, videoTorrent ]) {
  44     expect(video.category.label).to.equal('Misc')
  45     expect(video.licence.label).to.equal('Unknown')
  46     expect(video.language.label).to.equal('Unknown')
  47     expect(video.nsfw).to.be.false
  48     expect(video.description).to.equal('this is a super torrent description')
  49     expect(video.tags).to.deep.equal([ 'tag_torrent1', 'tag_torrent2' ])
  50     expect(video.files).to.have.lengthOf(1)
  51   }
  52
  53   expect(videoTorrent.name).to.contain('你好 世界 720p.mp4')
  54   expect(videoMagnet.name).to.contain('super peertube2 video')
  55
  56   const bodyCaptions = await server.captions.list({ videoId: idHttp })
  57   expect(bodyCaptions.total).to.equal(2)
  58 }
  59
  60 async function checkVideoServer2 (server: PeerTubeServer, id: number | string) {
  61   const video = await server.videos.get({ id })
  62
  63   expect(video.name).to.equal('my super name')
  64   expect(video.category.label).to.equal('Entertainment')
  65   expect(video.licence.label).to.equal('Public Domain Dedication')
  66   expect(video.language.label).to.equal('English')
  67   expect(video.nsfw).to.be.false
  68   expect(video.description).to.equal('my super description')
  69   expect(video.tags).to.deep.equal([ 'supertag1', 'supertag2' ])
  70
  71   expect(video.files).to.have.lengthOf(1)
  72
  73   const bodyCaptions = await server.captions.list({ videoId: id })
  74   expect(bodyCaptions.total).to.equal(2)
  75 }
  76
  77 describe('Test video imports', function () {
  78
  79   if (areHttpImportTestsDisabled()) return
  80
  81   function runSuite (mode: 'youtube-dl' | 'yt-dlp') {
  82
  83     describe('Import ' + mode, function () {
  84       let servers: PeerTubeServer[] = []
  85
  86       before(async function () {
  87         this.timeout(30_000)
  88
  89         // Run servers
  90         servers = await createMultipleServers(2, {
  91           import: {
  92             videos: {
  93               http: {
  94                 youtube_dl_release: {
  95                   url: mode === 'youtube-dl'
  96                     ? 'https://yt-dl.org/downloads/latest/youtube-dl'
  97                     : 'https://api.github.com/repos/yt-dlp/yt-dlp/releases',
  98
  99                   name: mode
 100                 }
 101               }
 102             }
 103           }
 104         })
 105
 106         await setAccessTokensToServers(servers)
 107         await setDefaultVideoChannel(servers)
 108
 109         for (const server of servers) {
 110           await server.config.updateExistingSubConfig({
 111             newConfig: {
 112               transcoding: {
 113                 alwaysTranscodeOriginalResolution: false
 114               }
 115             }
 116           })
 117         }
 118
 119         await doubleFollow(servers[0], servers[1])
 120       })
 121
 122       it('Should import videos on server 1', async function () {
 123         this.timeout(60_000)
 124
 125         const baseAttributes = {
 126           channelId: servers[0].store.channel.id,
 127           privacy: VideoPrivacy.PUBLIC
 128         }
 129
 130         {
 131           const attributes = { ...baseAttributes, targetUrl: FIXTURE_URLS.youtube }
 132           const { video } = await servers[0].imports.importVideo({ attributes })
 133           expect(video.name).to.equal('small video - youtube')
 134
 135           {
 136             expect(video.thumbnailPath).to.match(new RegExp(`^/static/thumbnails/.+.jpg$`))
 137             expect(video.previewPath).to.match(new RegExp(`^/lazy-static/previews/.+.jpg$`))
 138
 139             const suffix = mode === 'yt-dlp'
 140               ? '_yt_dlp'
 141               : ''
 142
 143             await testImage(servers[0].url, 'video_import_thumbnail' + suffix, video.thumbnailPath)
 144             await testImage(servers[0].url, 'video_import_preview' + suffix, video.previewPath)
 145           }
 146
 147           const bodyCaptions = await servers[0].captions.list({ videoId: video.id })
 148           const videoCaptions = bodyCaptions.data
 149           expect(videoCaptions).to.have.lengthOf(2)
 150
 151           {
 152             const enCaption = videoCaptions.find(caption => caption.language.id === 'en')
 153             expect(enCaption).to.exist
 154             expect(enCaption.language.label).to.equal('English')
 155             expect(enCaption.captionPath).to.match(new RegExp(`^/lazy-static/video-captions/.+-en.vtt$`))
 156
 157             const regex = `WEBVTT[ \n]+Kind: captions[ \n]+` +
 158               `(Language: en[ \n]+)?` +
 159               `00:00:01.600 --> 00:00:04.200( position:\\d+% line:\\d+%)?[ \n]+English \\(US\\)[ \n]+` +
 160               `00:00:05.900 --> 00:00:07.999( position:\\d+% line:\\d+%)?[ \n]+This is a subtitle in American English[ \n]+` +
 161               `00:00:10.000 --> 00:00:14.000( position:\\d+% line:\\d+%)?[ \n]+Adding subtitles is very easy to do`
 162             await testCaptionFile(servers[0].url, enCaption.captionPath, new RegExp(regex))
 163           }
 164
 165           {
 166             const frCaption = videoCaptions.find(caption => caption.language.id === 'fr')
 167             expect(frCaption).to.exist
 168             expect(frCaption.language.label).to.equal('French')
 169             expect(frCaption.captionPath).to.match(new RegExp(`^/lazy-static/video-captions/.+-fr.vtt`))
 170
 171             const regex = `WEBVTT[ \n]+Kind: captions[ \n]+` +
 172               `(Language: fr[ \n]+)?` +
 173               `00:00:01.600 --> 00:00:04.200( position:\\d+% line:\\d+%)?[ \n]+Français \\(FR\\)[ \n]+` +
 174               `00:00:05.900 --> 00:00:07.999( position:\\d+% line:\\d+%)?[ \n]+C'est un sous-titre français[ \n]+` +
 175               `00:00:10.000 --> 00:00:14.000( position:\\d+% line:\\d+%)?[ \n]+Ajouter un sous-titre est vraiment facile`
 176
 177             await testCaptionFile(servers[0].url, frCaption.captionPath, new RegExp(regex))
 178           }
 179         }
 180
 181         {
 182           const attributes = {
 183             ...baseAttributes,
 184             magnetUri: FIXTURE_URLS.magnet,
 185             description: 'this is a super torrent description',
 186             tags: [ 'tag_torrent1', 'tag_torrent2' ]
 187           }
 188           const { video } = await servers[0].imports.importVideo({ attributes })
 189           expect(video.name).to.equal('super peertube2 video')
 190         }
 191
 192         {
 193           const attributes = {
 194             ...baseAttributes,
 195             torrentfile: 'video-720p.torrent' as any,
 196             description: 'this is a super torrent description',
 197             tags: [ 'tag_torrent1', 'tag_torrent2' ]
 198           }
 199           const { video } = await servers[0].imports.importVideo({ attributes })
 200           expect(video.name).to.equal('你好 世界 720p.mp4')
 201         }
 202       })
 203
 204       it('Should list the videos to import in my videos on server 1', async function () {
 205         const { total, data } = await servers[0].videos.listMyVideos({ sort: 'createdAt' })
 206
 207         expect(total).to.equal(3)
 208
 209         expect(data).to.have.lengthOf(3)
 210         expect(data[0].name).to.equal('small video - youtube')
 211         expect(data[1].name).to.equal('super peertube2 video')
 212         expect(data[2].name).to.equal('你好 世界 720p.mp4')
 213       })
 214
 215       it('Should list the videos to import in my imports on server 1', async function () {
 216         const { total, data: videoImports } = await servers[0].imports.getMyVideoImports({ sort: '-createdAt' })
 217         expect(total).to.equal(3)
 218
 219         expect(videoImports).to.have.lengthOf(3)
 220
 221         expect(videoImports[2].targetUrl).to.equal(FIXTURE_URLS.youtube)
 222         expect(videoImports[2].magnetUri).to.be.null
 223         expect(videoImports[2].torrentName).to.be.null
 224         expect(videoImports[2].video.name).to.equal('small video - youtube')
 225
 226         expect(videoImports[1].targetUrl).to.be.null
 227         expect(videoImports[1].magnetUri).to.equal(FIXTURE_URLS.magnet)
 228         expect(videoImports[1].torrentName).to.be.null
 229         expect(videoImports[1].video.name).to.equal('super peertube2 video')
 230
 231         expect(videoImports[0].targetUrl).to.be.null
 232         expect(videoImports[0].magnetUri).to.be.null
 233         expect(videoImports[0].torrentName).to.equal('video-720p.torrent')
 234         expect(videoImports[0].video.name).to.equal('你好 世界 720p.mp4')
 235       })
 236
 237       it('Should filter my imports on target URL', async function () {
 238         const { total, data: videoImports } = await servers[0].imports.getMyVideoImports({ targetUrl: FIXTURE_URLS.youtube })
 239         expect(total).to.equal(1)
 240         expect(videoImports).to.have.lengthOf(1)
 241
 242         expect(videoImports[0].targetUrl).to.equal(FIXTURE_URLS.youtube)
 243       })
 244
 245       it('Should have the video listed on the two instances', async function () {
 246         this.timeout(120_000)
 247
 248         await waitJobs(servers)
 249
 250         for (const server of servers) {
 251           const { total, data } = await server.videos.list()
 252           expect(total).to.equal(3)
 253           expect(data).to.have.lengthOf(3)
 254
 255           const [ videoHttp, videoMagnet, videoTorrent ] = data
 256           await checkVideosServer1(server, videoHttp.uuid, videoMagnet.uuid, videoTorrent.uuid)
 257         }
 258       })
 259
 260       it('Should import a video on server 2 with some fields', async function () {
 261         this.timeout(60_000)
 262
 263         const attributes = {
 264           targetUrl: FIXTURE_URLS.youtube,
 265           channelId: servers[1].store.channel.id,
 266           privacy: VideoPrivacy.PUBLIC,
 267           category: 10,
 268           licence: 7,
 269           language: 'en',
 270           name: 'my super name',
 271           description: 'my super description',
 272           tags: [ 'supertag1', 'supertag2' ]
 273         }
 274         const { video } = await servers[1].imports.importVideo({ attributes })
 275         expect(video.name).to.equal('my super name')
 276       })
 277
 278       it('Should have the videos listed on the two instances', async function () {
 279         this.timeout(120_000)
 280
 281         await waitJobs(servers)
 282
 283         for (const server of servers) {
 284           const { total, data } = await server.videos.list()
 285           expect(total).to.equal(4)
 286           expect(data).to.have.lengthOf(4)
 287
 288           await checkVideoServer2(server, data[0].uuid)
 289
 290           const [ , videoHttp, videoMagnet, videoTorrent ] = data
 291           await checkVideosServer1(server, videoHttp.uuid, videoMagnet.uuid, videoTorrent.uuid)
 292         }
 293       })
 294
 295       it('Should import a video that will be transcoded', async function () {
 296         this.timeout(240_000)
 297
 298         const attributes = {
 299           name: 'transcoded video',
 300           magnetUri: FIXTURE_URLS.magnet,
 301           channelId: servers[1].store.channel.id,
 302           privacy: VideoPrivacy.PUBLIC
 303         }
 304         const { video } = await servers[1].imports.importVideo({ attributes })
 305         const videoUUID = video.uuid
 306
 307         await waitJobs(servers)
 308
 309         for (const server of servers) {
 310           const video = await server.videos.get({ id: videoUUID })
 311
 312           expect(video.name).to.equal('transcoded video')
 313           expect(video.files).to.have.lengthOf(4)
 314         }
 315       })
 316
 317       it('Should import no HDR version on a HDR video', async function () {
 318         this.timeout(300_000)
 319
 320         const config: DeepPartial<CustomConfig> = {
 321           transcoding: {
 322             enabled: true,
 323             resolutions: {
 324               '0p': false,
 325               '144p': true,
 326               '240p': true,
 327               '360p': false,
 328               '480p': false,
 329               '720p': false,
 330               '1080p': false, // the resulting resolution shouldn't be higher than this, and not vp9.2/av01
 331               '1440p': false,
 332               '2160p': false
 333             },
 334             webtorrent: { enabled: true },
 335             hls: { enabled: false }
 336           }
 337         }
 338         await servers[0].config.updateExistingSubConfig({ newConfig: config })
 339
 340         const attributes = {
 341           name: 'hdr video',
 342           targetUrl: FIXTURE_URLS.youtubeHDR,
 343           channelId: servers[0].store.channel.id,
 344           privacy: VideoPrivacy.PUBLIC
 345         }
 346         const { video: videoImported } = await servers[0].imports.importVideo({ attributes })
 347         const videoUUID = videoImported.uuid
 348
 349         await waitJobs(servers)
 350
 351         // test resolution
 352         const video = await servers[0].videos.get({ id: videoUUID })
 353         expect(video.name).to.equal('hdr video')
 354         const maxResolution = Math.max.apply(Math, video.files.map(function (o) { return o.resolution.id }))
 355         expect(maxResolution, 'expected max resolution not met').to.equals(VideoResolution.H_240P)
 356       })
 357
 358       it('Should not import resolution higher than enabled transcoding resolution', async function () {
 359         this.timeout(300_000)
 360
 361         const config: DeepPartial<CustomConfig> = {
 362           transcoding: {
 363             enabled: true,
 364             resolutions: {
 365               '0p': false,
 366               '144p': true,
 367               '240p': false,
 368               '360p': false,
 369               '480p': false,
 370               '720p': false,
 371               '1080p': false,
 372               '1440p': false,
 373               '2160p': false
 374             },
 375             alwaysTranscodeOriginalResolution: false
 376           }
 377         }
 378         await servers[0].config.updateExistingSubConfig({ newConfig: config })
 379
 380         const attributes = {
 381           name: 'small resolution video',
 382           targetUrl: FIXTURE_URLS.youtube,
 383           channelId: servers[0].store.channel.id,
 384           privacy: VideoPrivacy.PUBLIC
 385         }
 386         const { video: videoImported } = await servers[0].imports.importVideo({ attributes })
 387         const videoUUID = videoImported.uuid
 388
 389         await waitJobs(servers)
 390
 391         // test resolution
 392         const video = await servers[0].videos.get({ id: videoUUID })
 393         expect(video.name).to.equal('small resolution video')
 394         expect(video.files).to.have.lengthOf(1)
 395         expect(video.files[0].resolution.id).to.equal(144)
 396       })
 397
 398       it('Should import resolution higher than enabled transcoding resolution', async function () {
 399         this.timeout(300_000)
 400
 401         const config: DeepPartial<CustomConfig> = {
 402           transcoding: {
 403             alwaysTranscodeOriginalResolution: true
 404           }
 405         }
 406         await servers[0].config.updateExistingSubConfig({ newConfig: config })
 407
 408         const attributes = {
 409           name: 'bigger resolution video',
 410           targetUrl: FIXTURE_URLS.youtube,
 411           channelId: servers[0].store.channel.id,
 412           privacy: VideoPrivacy.PUBLIC
 413         }
 414         const { video: videoImported } = await servers[0].imports.importVideo({ attributes })
 415         const videoUUID = videoImported.uuid
 416
 417         await waitJobs(servers)
 418
 419         // test resolution
 420         const video = await servers[0].videos.get({ id: videoUUID })
 421         expect(video.name).to.equal('bigger resolution video')
 422
 423         expect(video.files).to.have.lengthOf(2)
 424         expect(video.files.find(f => f.resolution.id === 240)).to.exist
 425         expect(video.files.find(f => f.resolution.id === 144)).to.exist
 426       })
 427
 428       it('Should import a peertube video', async function () {
 429         this.timeout(120_000)
 430
 431         const toTest = [ FIXTURE_URLS.peertube_long ]
 432
 433         // TODO: include peertube_short when https://github.com/ytdl-org/youtube-dl/pull/29475 is merged
 434         if (mode === 'yt-dlp') {
 435           toTest.push(FIXTURE_URLS.peertube_short)
 436         }
 437
 438         for (const targetUrl of toTest) {
 439           await servers[0].config.disableTranscoding()
 440
 441           const attributes = {
 442             targetUrl,
 443             channelId: servers[0].store.channel.id,
 444             privacy: VideoPrivacy.PUBLIC
 445           }
 446           const { video } = await servers[0].imports.importVideo({ attributes })
 447           const videoUUID = video.uuid
 448
 449           await waitJobs(servers)
 450
 451           for (const server of servers) {
 452             const video = await server.videos.get({ id: videoUUID })
 453
 454             expect(video.name).to.equal('E2E tests')
 455           }
 456         }
 457       })
 458
 459       after(async function () {
 460         await cleanupTests(servers)
 461       })
 462     })
 463   }
 464
 465   runSuite('youtube-dl')
 466
 467   runSuite('yt-dlp')
 468
 469   describe('Delete/cancel an import', function () {
 470     let server: PeerTubeServer
 471
 472     let finishedImportId: number
 473     let finishedVideo: Video
 474     let pendingImportId: number
 475
 476     async function importVideo (name: string) {
 477       const attributes = { name, channelId: server.store.channel.id, targetUrl: FIXTURE_URLS.goodVideo }
 478       const res = await server.imports.importVideo({ attributes })
 479
 480       return res.id
 481     }
 482
 483     before(async function () {
 484       this.timeout(120_000)
 485
 486       server = await createSingleServer(1)
 487
 488       await setAccessTokensToServers([ server ])
 489       await setDefaultVideoChannel([ server ])
 490
 491       finishedImportId = await importVideo('finished')
 492       await waitJobs([ server ])
 493
 494       await server.jobs.pauseJobQueue()
 495       pendingImportId = await importVideo('pending')
 496
 497       const { data } = await server.imports.getMyVideoImports()
 498       expect(data).to.have.lengthOf(2)
 499
 500       finishedVideo = data.find(i => i.id === finishedImportId).video
 501     })
 502
 503     it('Should delete a video import', async function () {
 504       await server.imports.delete({ importId: finishedImportId })
 505
 506       const { data } = await server.imports.getMyVideoImports()
 507       expect(data).to.have.lengthOf(1)
 508       expect(data[0].id).to.equal(pendingImportId)
 509       expect(data[0].state.id).to.equal(VideoImportState.PENDING)
 510     })
 511
 512     it('Should not have deleted the associated video', async function () {
 513       const video = await server.videos.get({ id: finishedVideo.id, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 })
 514       expect(video.name).to.equal('finished')
 515       expect(video.state.id).to.equal(VideoState.PUBLISHED)
 516     })
 517
 518     it('Should cancel a video import', async function () {
 519       await server.imports.cancel({ importId: pendingImportId })
 520
 521       const { data } = await server.imports.getMyVideoImports()
 522       expect(data).to.have.lengthOf(1)
 523       expect(data[0].id).to.equal(pendingImportId)
 524       expect(data[0].state.id).to.equal(VideoImportState.CANCELLED)
 525     })
 526
 527     it('Should not have processed the cancelled video import', async function () {
 528       this.timeout(60_000)
 529
 530       await server.jobs.resumeJobQueue()
 531
 532       await waitJobs([ server ])
 533
 534       const { data } = await server.imports.getMyVideoImports()
 535       expect(data).to.have.lengthOf(1)
 536       expect(data[0].id).to.equal(pendingImportId)
 537       expect(data[0].state.id).to.equal(VideoImportState.CANCELLED)
 538       expect(data[0].video.state.id).to.equal(VideoState.TO_IMPORT)
 539     })
 540
 541     it('Should delete the cancelled video import', async function () {
 542       await server.imports.delete({ importId: pendingImportId })
 543       const { data } = await server.imports.getMyVideoImports()
 544       expect(data).to.have.lengthOf(0)
 545     })
 546
 547     after(async function () {
 548       await cleanupTests([ server ])
 549     })
 550   })
 551
 552   describe('Auto update', function () {
 553     let server: PeerTubeServer
 554
 555     function quickPeerTubeImport () {
 556       const attributes = {
 557         targetUrl: FIXTURE_URLS.peertube_long,
 558         channelId: server.store.channel.id,
 559         privacy: VideoPrivacy.PUBLIC
 560       }
 561
 562       return server.imports.importVideo({ attributes })
 563     }
 564
 565     async function testBinaryUpdate (releaseUrl: string, releaseName: string) {
 566       await remove(join(server.servers.buildDirectory('bin'), releaseName))
 567
 568       await server.kill()
 569       await server.run({
 570         import: {
 571           videos: {
 572             http: {
 573               youtube_dl_release: {
 574                 url: releaseUrl,
 575                 name: releaseName
 576               }
 577             }
 578           }
 579         }
 580       })
 581
 582       await quickPeerTubeImport()
 583
 584       const base = server.servers.buildDirectory('bin')
 585       const content = await readdir(base)
 586       const binaryPath = join(base, releaseName)
 587
 588       expect(await pathExists(binaryPath), `${binaryPath} does not exist in ${base} (${content.join(', ')})`).to.be.true
 589     }
 590
 591     before(async function () {
 592       this.timeout(30_000)
 593
 594       // Run servers
 595       server = await createSingleServer(1)
 596
 597       await setAccessTokensToServers([ server ])
 598       await setDefaultVideoChannel([ server ])
 599     })
 600
 601     it('Should update youtube-dl from github URL', async function () {
 602       this.timeout(120_000)
 603
 604       await testBinaryUpdate('https://api.github.com/repos/ytdl-org/youtube-dl/releases', 'youtube-dl')
 605     })
 606
 607     it('Should update youtube-dl from raw URL', async function () {
 608       this.timeout(120_000)
 609
 610       await testBinaryUpdate('https://yt-dl.org/downloads/latest/youtube-dl', 'youtube-dl')
 611     })
 612
 613     it('Should update youtube-dl from youtube-dl fork', async function () {
 614       this.timeout(120_000)
 615
 616       await testBinaryUpdate('https://api.github.com/repos/yt-dlp/yt-dlp/releases', 'yt-dlp')
 617     })
 618
 619     after(async function () {
 620       await cleanupTests([ server ])
 621     })
 622   })
 623 })