server/tests/api/videos/video-imports.ts

   1 /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
   2
   3 import { expect } from 'chai'
   4 import { pathExists, readdir, remove } from 'fs-extra'
   5 import { join } from 'path'
   6 import { FIXTURE_URLS, testCaptionFile, testImage } from '@server/tests/shared'
   7 import { areHttpImportTestsDisabled } from '@shared/core-utils'
   8 import { CustomConfig, HttpStatusCode, Video, VideoImportState, VideoPrivacy, VideoResolution, VideoState } from '@shared/models'
   9 import {
  10   cleanupTests,
  11   createMultipleServers,
  12   createSingleServer,
  13   doubleFollow,
  14   getServerImportConfig,
  15   PeerTubeServer,
  16   setAccessTokensToServers,
  17   setDefaultVideoChannel,
  18   waitJobs
  19 } from '@shared/server-commands'
  20 import { DeepPartial } from '@shared/typescript-utils'
  21
  22 async function checkVideosServer1 (server: PeerTubeServer, idHttp: string, idMagnet: string, idTorrent: string) {
  23   const videoHttp = await server.videos.get({ id: idHttp })
  24
  25   expect(videoHttp.name).to.equal('small video - youtube')
  26   // FIXME: youtube-dl seems broken
  27   // expect(videoHttp.category.label).to.equal('News & Politics')
  28   // expect(videoHttp.licence.label).to.equal('Attribution')
  29   expect(videoHttp.language.label).to.equal('Unknown')
  30   expect(videoHttp.nsfw).to.be.false
  31   expect(videoHttp.description).to.equal('this is a super description')
  32   expect(videoHttp.tags).to.deep.equal([ 'tag1', 'tag2' ])
  33   expect(videoHttp.files).to.have.lengthOf(1)
  34
  35   const originallyPublishedAt = new Date(videoHttp.originallyPublishedAt)
  36   expect(originallyPublishedAt.getDate()).to.equal(14)
  37   expect(originallyPublishedAt.getMonth()).to.equal(0)
  38   expect(originallyPublishedAt.getFullYear()).to.equal(2019)
  39
  40   const videoMagnet = await server.videos.get({ id: idMagnet })
  41   const videoTorrent = await server.videos.get({ id: idTorrent })
  42
  43   for (const video of [ videoMagnet, videoTorrent ]) {
  44     expect(video.category.label).to.equal('Unknown')
  45     expect(video.licence.label).to.equal('Unknown')
  46     expect(video.language.label).to.equal('Unknown')
  47     expect(video.nsfw).to.be.false
  48     expect(video.description).to.equal('this is a super torrent description')
  49     expect(video.tags).to.deep.equal([ 'tag_torrent1', 'tag_torrent2' ])
  50     expect(video.files).to.have.lengthOf(1)
  51   }
  52
  53   expect(videoTorrent.name).to.contain('你好 世界 720p.mp4')
  54   expect(videoMagnet.name).to.contain('super peertube2 video')
  55
  56   const bodyCaptions = await server.captions.list({ videoId: idHttp })
  57   expect(bodyCaptions.total).to.equal(2)
  58 }
  59
  60 async function checkVideoServer2 (server: PeerTubeServer, id: number | string) {
  61   const video = await server.videos.get({ id })
  62
  63   expect(video.name).to.equal('my super name')
  64   expect(video.category.label).to.equal('Entertainment')
  65   expect(video.licence.label).to.equal('Public Domain Dedication')
  66   expect(video.language.label).to.equal('English')
  67   expect(video.nsfw).to.be.false
  68   expect(video.description).to.equal('my super description')
  69   expect(video.tags).to.deep.equal([ 'supertag1', 'supertag2' ])
  70
  71   expect(video.files).to.have.lengthOf(1)
  72
  73   const bodyCaptions = await server.captions.list({ videoId: id })
  74   expect(bodyCaptions.total).to.equal(2)
  75 }
  76
  77 describe('Test video imports', function () {
  78
  79   if (areHttpImportTestsDisabled()) return
  80
  81   function runSuite (mode: 'youtube-dl' | 'yt-dlp') {
  82
  83     describe('Import ' + mode, function () {
  84       let servers: PeerTubeServer[] = []
  85
  86       before(async function () {
  87         this.timeout(60_000)
  88
  89         servers = await createMultipleServers(2, getServerImportConfig(mode))
  90
  91         await setAccessTokensToServers(servers)
  92         await setDefaultVideoChannel(servers)
  93
  94         for (const server of servers) {
  95           await server.config.updateExistingSubConfig({
  96             newConfig: {
  97               transcoding: {
  98                 alwaysTranscodeOriginalResolution: false
  99               }
 100             }
 101           })
 102         }
 103
 104         await doubleFollow(servers[0], servers[1])
 105       })
 106
 107       it('Should import videos on server 1', async function () {
 108         this.timeout(60_000)
 109
 110         const baseAttributes = {
 111           channelId: servers[0].store.channel.id,
 112           privacy: VideoPrivacy.PUBLIC
 113         }
 114
 115         {
 116           const attributes = { ...baseAttributes, targetUrl: FIXTURE_URLS.youtube }
 117           const { video } = await servers[0].imports.importVideo({ attributes })
 118           expect(video.name).to.equal('small video - youtube')
 119
 120           {
 121             expect(video.thumbnailPath).to.match(new RegExp(`^/static/thumbnails/.+.jpg$`))
 122             expect(video.previewPath).to.match(new RegExp(`^/lazy-static/previews/.+.jpg$`))
 123
 124             const suffix = mode === 'yt-dlp'
 125               ? '_yt_dlp'
 126               : ''
 127
 128             await testImage(servers[0].url, 'video_import_thumbnail' + suffix, video.thumbnailPath)
 129             await testImage(servers[0].url, 'video_import_preview' + suffix, video.previewPath)
 130           }
 131
 132           const bodyCaptions = await servers[0].captions.list({ videoId: video.id })
 133           const videoCaptions = bodyCaptions.data
 134           expect(videoCaptions).to.have.lengthOf(2)
 135
 136           {
 137             const enCaption = videoCaptions.find(caption => caption.language.id === 'en')
 138             expect(enCaption).to.exist
 139             expect(enCaption.language.label).to.equal('English')
 140             expect(enCaption.captionPath).to.match(new RegExp(`^/lazy-static/video-captions/.+-en.vtt$`))
 141
 142             const regex = `WEBVTT[ \n]+Kind: captions[ \n]+` +
 143               `(Language: en[ \n]+)?` +
 144               `00:00:01.600 --> 00:00:04.200( position:\\d+% line:\\d+%)?[ \n]+English \\(US\\)[ \n]+` +
 145               `00:00:05.900 --> 00:00:07.999( position:\\d+% line:\\d+%)?[ \n]+This is a subtitle in American English[ \n]+` +
 146               `00:00:10.000 --> 00:00:14.000( position:\\d+% line:\\d+%)?[ \n]+Adding subtitles is very easy to do`
 147             await testCaptionFile(servers[0].url, enCaption.captionPath, new RegExp(regex))
 148           }
 149
 150           {
 151             const frCaption = videoCaptions.find(caption => caption.language.id === 'fr')
 152             expect(frCaption).to.exist
 153             expect(frCaption.language.label).to.equal('French')
 154             expect(frCaption.captionPath).to.match(new RegExp(`^/lazy-static/video-captions/.+-fr.vtt`))
 155
 156             const regex = `WEBVTT[ \n]+Kind: captions[ \n]+` +
 157               `(Language: fr[ \n]+)?` +
 158               `00:00:01.600 --> 00:00:04.200( position:\\d+% line:\\d+%)?[ \n]+Français \\(FR\\)[ \n]+` +
 159               `00:00:05.900 --> 00:00:07.999( position:\\d+% line:\\d+%)?[ \n]+C'est un sous-titre français[ \n]+` +
 160               `00:00:10.000 --> 00:00:14.000( position:\\d+% line:\\d+%)?[ \n]+Ajouter un sous-titre est vraiment facile`
 161
 162             await testCaptionFile(servers[0].url, frCaption.captionPath, new RegExp(regex))
 163           }
 164         }
 165
 166         {
 167           const attributes = {
 168             ...baseAttributes,
 169             magnetUri: FIXTURE_URLS.magnet,
 170             description: 'this is a super torrent description',
 171             tags: [ 'tag_torrent1', 'tag_torrent2' ]
 172           }
 173           const { video } = await servers[0].imports.importVideo({ attributes })
 174           expect(video.name).to.equal('super peertube2 video')
 175         }
 176
 177         {
 178           const attributes = {
 179             ...baseAttributes,
 180             torrentfile: 'video-720p.torrent' as any,
 181             description: 'this is a super torrent description',
 182             tags: [ 'tag_torrent1', 'tag_torrent2' ]
 183           }
 184           const { video } = await servers[0].imports.importVideo({ attributes })
 185           expect(video.name).to.equal('你好 世界 720p.mp4')
 186         }
 187       })
 188
 189       it('Should list the videos to import in my videos on server 1', async function () {
 190         const { total, data } = await servers[0].videos.listMyVideos({ sort: 'createdAt' })
 191
 192         expect(total).to.equal(3)
 193
 194         expect(data).to.have.lengthOf(3)
 195         expect(data[0].name).to.equal('small video - youtube')
 196         expect(data[1].name).to.equal('super peertube2 video')
 197         expect(data[2].name).to.equal('你好 世界 720p.mp4')
 198       })
 199
 200       it('Should list the videos to import in my imports on server 1', async function () {
 201         const { total, data: videoImports } = await servers[0].imports.getMyVideoImports({ sort: '-createdAt' })
 202         expect(total).to.equal(3)
 203
 204         expect(videoImports).to.have.lengthOf(3)
 205
 206         expect(videoImports[2].targetUrl).to.equal(FIXTURE_URLS.youtube)
 207         expect(videoImports[2].magnetUri).to.be.null
 208         expect(videoImports[2].torrentName).to.be.null
 209         expect(videoImports[2].video.name).to.equal('small video - youtube')
 210
 211         expect(videoImports[1].targetUrl).to.be.null
 212         expect(videoImports[1].magnetUri).to.equal(FIXTURE_URLS.magnet)
 213         expect(videoImports[1].torrentName).to.be.null
 214         expect(videoImports[1].video.name).to.equal('super peertube2 video')
 215
 216         expect(videoImports[0].targetUrl).to.be.null
 217         expect(videoImports[0].magnetUri).to.be.null
 218         expect(videoImports[0].torrentName).to.equal('video-720p.torrent')
 219         expect(videoImports[0].video.name).to.equal('你好 世界 720p.mp4')
 220       })
 221
 222       it('Should filter my imports on target URL', async function () {
 223         const { total, data: videoImports } = await servers[0].imports.getMyVideoImports({ targetUrl: FIXTURE_URLS.youtube })
 224         expect(total).to.equal(1)
 225         expect(videoImports).to.have.lengthOf(1)
 226
 227         expect(videoImports[0].targetUrl).to.equal(FIXTURE_URLS.youtube)
 228       })
 229
 230       it('Should search in my imports', async function () {
 231         const { total, data: videoImports } = await servers[0].imports.getMyVideoImports({ search: 'peertube2' })
 232         expect(total).to.equal(1)
 233         expect(videoImports).to.have.lengthOf(1)
 234
 235         expect(videoImports[0].magnetUri).to.equal(FIXTURE_URLS.magnet)
 236         expect(videoImports[0].video.name).to.equal('super peertube2 video')
 237       })
 238
 239       it('Should have the video listed on the two instances', async function () {
 240         this.timeout(120_000)
 241
 242         await waitJobs(servers)
 243
 244         for (const server of servers) {
 245           const { total, data } = await server.videos.list()
 246           expect(total).to.equal(3)
 247           expect(data).to.have.lengthOf(3)
 248
 249           const [ videoHttp, videoMagnet, videoTorrent ] = data
 250           await checkVideosServer1(server, videoHttp.uuid, videoMagnet.uuid, videoTorrent.uuid)
 251         }
 252       })
 253
 254       it('Should import a video on server 2 with some fields', async function () {
 255         this.timeout(60_000)
 256
 257         const attributes = {
 258           targetUrl: FIXTURE_URLS.youtube,
 259           channelId: servers[1].store.channel.id,
 260           privacy: VideoPrivacy.PUBLIC,
 261           category: 10,
 262           licence: 7,
 263           language: 'en',
 264           name: 'my super name',
 265           description: 'my super description',
 266           tags: [ 'supertag1', 'supertag2' ]
 267         }
 268         const { video } = await servers[1].imports.importVideo({ attributes })
 269         expect(video.name).to.equal('my super name')
 270       })
 271
 272       it('Should have the videos listed on the two instances', async function () {
 273         this.timeout(120_000)
 274
 275         await waitJobs(servers)
 276
 277         for (const server of servers) {
 278           const { total, data } = await server.videos.list()
 279           expect(total).to.equal(4)
 280           expect(data).to.have.lengthOf(4)
 281
 282           await checkVideoServer2(server, data[0].uuid)
 283
 284           const [ , videoHttp, videoMagnet, videoTorrent ] = data
 285           await checkVideosServer1(server, videoHttp.uuid, videoMagnet.uuid, videoTorrent.uuid)
 286         }
 287       })
 288
 289       it('Should import a video that will be transcoded', async function () {
 290         this.timeout(240_000)
 291
 292         const attributes = {
 293           name: 'transcoded video',
 294           magnetUri: FIXTURE_URLS.magnet,
 295           channelId: servers[1].store.channel.id,
 296           privacy: VideoPrivacy.PUBLIC
 297         }
 298         const { video } = await servers[1].imports.importVideo({ attributes })
 299         const videoUUID = video.uuid
 300
 301         await waitJobs(servers)
 302
 303         for (const server of servers) {
 304           const video = await server.videos.get({ id: videoUUID })
 305
 306           expect(video.name).to.equal('transcoded video')
 307           expect(video.files).to.have.lengthOf(4)
 308         }
 309       })
 310
 311       it('Should import no HDR version on a HDR video', async function () {
 312         this.timeout(300_000)
 313
 314         const config: DeepPartial<CustomConfig> = {
 315           transcoding: {
 316             enabled: true,
 317             resolutions: {
 318               '0p': false,
 319               '144p': true,
 320               '240p': true,
 321               '360p': false,
 322               '480p': false,
 323               '720p': false,
 324               '1080p': false, // the resulting resolution shouldn't be higher than this, and not vp9.2/av01
 325               '1440p': false,
 326               '2160p': false
 327             },
 328             webtorrent: { enabled: true },
 329             hls: { enabled: false }
 330           }
 331         }
 332         await servers[0].config.updateExistingSubConfig({ newConfig: config })
 333
 334         const attributes = {
 335           name: 'hdr video',
 336           targetUrl: FIXTURE_URLS.youtubeHDR,
 337           channelId: servers[0].store.channel.id,
 338           privacy: VideoPrivacy.PUBLIC
 339         }
 340         const { video: videoImported } = await servers[0].imports.importVideo({ attributes })
 341         const videoUUID = videoImported.uuid
 342
 343         await waitJobs(servers)
 344
 345         // test resolution
 346         const video = await servers[0].videos.get({ id: videoUUID })
 347         expect(video.name).to.equal('hdr video')
 348         const maxResolution = Math.max.apply(Math, video.files.map(function (o) { return o.resolution.id }))
 349         expect(maxResolution, 'expected max resolution not met').to.equals(VideoResolution.H_240P)
 350       })
 351
 352       it('Should not import resolution higher than enabled transcoding resolution', async function () {
 353         this.timeout(300_000)
 354
 355         const config: DeepPartial<CustomConfig> = {
 356           transcoding: {
 357             enabled: true,
 358             resolutions: {
 359               '0p': false,
 360               '144p': true,
 361               '240p': false,
 362               '360p': false,
 363               '480p': false,
 364               '720p': false,
 365               '1080p': false,
 366               '1440p': false,
 367               '2160p': false
 368             },
 369             alwaysTranscodeOriginalResolution: false
 370           }
 371         }
 372         await servers[0].config.updateExistingSubConfig({ newConfig: config })
 373
 374         const attributes = {
 375           name: 'small resolution video',
 376           targetUrl: FIXTURE_URLS.youtube,
 377           channelId: servers[0].store.channel.id,
 378           privacy: VideoPrivacy.PUBLIC
 379         }
 380         const { video: videoImported } = await servers[0].imports.importVideo({ attributes })
 381         const videoUUID = videoImported.uuid
 382
 383         await waitJobs(servers)
 384
 385         // test resolution
 386         const video = await servers[0].videos.get({ id: videoUUID })
 387         expect(video.name).to.equal('small resolution video')
 388         expect(video.files).to.have.lengthOf(1)
 389         expect(video.files[0].resolution.id).to.equal(144)
 390       })
 391
 392       it('Should import resolution higher than enabled transcoding resolution', async function () {
 393         this.timeout(300_000)
 394
 395         const config: DeepPartial<CustomConfig> = {
 396           transcoding: {
 397             alwaysTranscodeOriginalResolution: true
 398           }
 399         }
 400         await servers[0].config.updateExistingSubConfig({ newConfig: config })
 401
 402         const attributes = {
 403           name: 'bigger resolution video',
 404           targetUrl: FIXTURE_URLS.youtube,
 405           channelId: servers[0].store.channel.id,
 406           privacy: VideoPrivacy.PUBLIC
 407         }
 408         const { video: videoImported } = await servers[0].imports.importVideo({ attributes })
 409         const videoUUID = videoImported.uuid
 410
 411         await waitJobs(servers)
 412
 413         // test resolution
 414         const video = await servers[0].videos.get({ id: videoUUID })
 415         expect(video.name).to.equal('bigger resolution video')
 416
 417         expect(video.files).to.have.lengthOf(2)
 418         expect(video.files.find(f => f.resolution.id === 240)).to.exist
 419         expect(video.files.find(f => f.resolution.id === 144)).to.exist
 420       })
 421
 422       it('Should import a peertube video', async function () {
 423         this.timeout(120_000)
 424
 425         const toTest = [ FIXTURE_URLS.peertube_long ]
 426
 427         // TODO: include peertube_short when https://github.com/ytdl-org/youtube-dl/pull/29475 is merged
 428         if (mode === 'yt-dlp') {
 429           toTest.push(FIXTURE_URLS.peertube_short)
 430         }
 431
 432         for (const targetUrl of toTest) {
 433           await servers[0].config.disableTranscoding()
 434
 435           const attributes = {
 436             targetUrl,
 437             channelId: servers[0].store.channel.id,
 438             privacy: VideoPrivacy.PUBLIC
 439           }
 440           const { video } = await servers[0].imports.importVideo({ attributes })
 441           const videoUUID = video.uuid
 442
 443           await waitJobs(servers)
 444
 445           for (const server of servers) {
 446             const video = await server.videos.get({ id: videoUUID })
 447
 448             expect(video.name).to.equal('E2E tests')
 449           }
 450         }
 451       })
 452
 453       after(async function () {
 454         await cleanupTests(servers)
 455       })
 456     })
 457   }
 458
 459   runSuite('youtube-dl')
 460
 461   runSuite('yt-dlp')
 462
 463   describe('Delete/cancel an import', function () {
 464     let server: PeerTubeServer
 465
 466     let finishedImportId: number
 467     let finishedVideo: Video
 468     let pendingImportId: number
 469
 470     async function importVideo (name: string) {
 471       const attributes = { name, channelId: server.store.channel.id, targetUrl: FIXTURE_URLS.goodVideo }
 472       const res = await server.imports.importVideo({ attributes })
 473
 474       return res.id
 475     }
 476
 477     before(async function () {
 478       this.timeout(120_000)
 479
 480       server = await createSingleServer(1)
 481
 482       await setAccessTokensToServers([ server ])
 483       await setDefaultVideoChannel([ server ])
 484
 485       finishedImportId = await importVideo('finished')
 486       await waitJobs([ server ])
 487
 488       await server.jobs.pauseJobQueue()
 489       pendingImportId = await importVideo('pending')
 490
 491       const { data } = await server.imports.getMyVideoImports()
 492       expect(data).to.have.lengthOf(2)
 493
 494       finishedVideo = data.find(i => i.id === finishedImportId).video
 495     })
 496
 497     it('Should delete a video import', async function () {
 498       await server.imports.delete({ importId: finishedImportId })
 499
 500       const { data } = await server.imports.getMyVideoImports()
 501       expect(data).to.have.lengthOf(1)
 502       expect(data[0].id).to.equal(pendingImportId)
 503       expect(data[0].state.id).to.equal(VideoImportState.PENDING)
 504     })
 505
 506     it('Should not have deleted the associated video', async function () {
 507       const video = await server.videos.get({ id: finishedVideo.id, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 })
 508       expect(video.name).to.equal('finished')
 509       expect(video.state.id).to.equal(VideoState.PUBLISHED)
 510     })
 511
 512     it('Should cancel a video import', async function () {
 513       await server.imports.cancel({ importId: pendingImportId })
 514
 515       const { data } = await server.imports.getMyVideoImports()
 516       expect(data).to.have.lengthOf(1)
 517       expect(data[0].id).to.equal(pendingImportId)
 518       expect(data[0].state.id).to.equal(VideoImportState.CANCELLED)
 519     })
 520
 521     it('Should not have processed the cancelled video import', async function () {
 522       this.timeout(60_000)
 523
 524       await server.jobs.resumeJobQueue()
 525
 526       await waitJobs([ server ])
 527
 528       const { data } = await server.imports.getMyVideoImports()
 529       expect(data).to.have.lengthOf(1)
 530       expect(data[0].id).to.equal(pendingImportId)
 531       expect(data[0].state.id).to.equal(VideoImportState.CANCELLED)
 532       expect(data[0].video.state.id).to.equal(VideoState.TO_IMPORT)
 533     })
 534
 535     it('Should delete the cancelled video import', async function () {
 536       await server.imports.delete({ importId: pendingImportId })
 537       const { data } = await server.imports.getMyVideoImports()
 538       expect(data).to.have.lengthOf(0)
 539     })
 540
 541     after(async function () {
 542       await cleanupTests([ server ])
 543     })
 544   })
 545
 546   describe('Auto update', function () {
 547     let server: PeerTubeServer
 548
 549     function quickPeerTubeImport () {
 550       const attributes = {
 551         targetUrl: FIXTURE_URLS.peertube_long,
 552         channelId: server.store.channel.id,
 553         privacy: VideoPrivacy.PUBLIC
 554       }
 555
 556       return server.imports.importVideo({ attributes })
 557     }
 558
 559     async function testBinaryUpdate (releaseUrl: string, releaseName: string) {
 560       await remove(join(server.servers.buildDirectory('bin'), releaseName))
 561
 562       await server.kill()
 563       await server.run({
 564         import: {
 565           videos: {
 566             http: {
 567               youtube_dl_release: {
 568                 url: releaseUrl,
 569                 name: releaseName
 570               }
 571             }
 572           }
 573         }
 574       })
 575
 576       await quickPeerTubeImport()
 577
 578       const base = server.servers.buildDirectory('bin')
 579       const content = await readdir(base)
 580       const binaryPath = join(base, releaseName)
 581
 582       expect(await pathExists(binaryPath), `${binaryPath} does not exist in ${base} (${content.join(', ')})`).to.be.true
 583     }
 584
 585     before(async function () {
 586       this.timeout(30_000)
 587
 588       // Run servers
 589       server = await createSingleServer(1)
 590
 591       await setAccessTokensToServers([ server ])
 592       await setDefaultVideoChannel([ server ])
 593     })
 594
 595     it('Should update youtube-dl from github URL', async function () {
 596       this.timeout(120_000)
 597
 598       await testBinaryUpdate('https://api.github.com/repos/ytdl-org/youtube-dl/releases', 'youtube-dl')
 599     })
 600
 601     it('Should update youtube-dl from raw URL', async function () {
 602       this.timeout(120_000)
 603
 604       await testBinaryUpdate('https://yt-dl.org/downloads/latest/youtube-dl', 'youtube-dl')
 605     })
 606
 607     it('Should update youtube-dl from youtube-dl fork', async function () {
 608       this.timeout(120_000)
 609
 610       await testBinaryUpdate('https://api.github.com/repos/yt-dlp/yt-dlp/releases', 'yt-dlp')
 611     })
 612
 613     after(async function () {
 614       await cleanupTests([ server ])
 615     })
 616   })
 617 })