server/tests/api/videos/video-imports.ts

   1 /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
   2
   3 import 'mocha'
   4 import { expect } from 'chai'
   5 import { pathExists, readdir, remove } from 'fs-extra'
   6 import { join } from 'path'
   7 import { FIXTURE_URLS, testCaptionFile, testImage } from '@server/tests/shared'
   8 import { areHttpImportTestsDisabled } from '@shared/core-utils'
   9 import { CustomConfig, HttpStatusCode, Video, VideoImportState, VideoPrivacy, VideoResolution, VideoState } from '@shared/models'
  10 import {
  11   cleanupTests,
  12   createMultipleServers,
  13   createSingleServer,
  14   doubleFollow,
  15   getServerImportConfig,
  16   PeerTubeServer,
  17   setAccessTokensToServers,
  18   setDefaultVideoChannel,
  19   waitJobs
  20 } from '@shared/server-commands'
  21 import { DeepPartial } from '@shared/typescript-utils'
  22
  23 async function checkVideosServer1 (server: PeerTubeServer, idHttp: string, idMagnet: string, idTorrent: string) {
  24   const videoHttp = await server.videos.get({ id: idHttp })
  25
  26   expect(videoHttp.name).to.equal('small video - youtube')
  27   // FIXME: youtube-dl seems broken
  28   // expect(videoHttp.category.label).to.equal('News & Politics')
  29   // expect(videoHttp.licence.label).to.equal('Attribution')
  30   expect(videoHttp.language.label).to.equal('Unknown')
  31   expect(videoHttp.nsfw).to.be.false
  32   expect(videoHttp.description).to.equal('this is a super description')
  33   expect(videoHttp.tags).to.deep.equal([ 'tag1', 'tag2' ])
  34   expect(videoHttp.files).to.have.lengthOf(1)
  35
  36   const originallyPublishedAt = new Date(videoHttp.originallyPublishedAt)
  37   expect(originallyPublishedAt.getDate()).to.equal(14)
  38   expect(originallyPublishedAt.getMonth()).to.equal(0)
  39   expect(originallyPublishedAt.getFullYear()).to.equal(2019)
  40
  41   const videoMagnet = await server.videos.get({ id: idMagnet })
  42   const videoTorrent = await server.videos.get({ id: idTorrent })
  43
  44   for (const video of [ videoMagnet, videoTorrent ]) {
  45     expect(video.category.label).to.equal('Misc')
  46     expect(video.licence.label).to.equal('Unknown')
  47     expect(video.language.label).to.equal('Unknown')
  48     expect(video.nsfw).to.be.false
  49     expect(video.description).to.equal('this is a super torrent description')
  50     expect(video.tags).to.deep.equal([ 'tag_torrent1', 'tag_torrent2' ])
  51     expect(video.files).to.have.lengthOf(1)
  52   }
  53
  54   expect(videoTorrent.name).to.contain('你好 世界 720p.mp4')
  55   expect(videoMagnet.name).to.contain('super peertube2 video')
  56
  57   const bodyCaptions = await server.captions.list({ videoId: idHttp })
  58   expect(bodyCaptions.total).to.equal(2)
  59 }
  60
  61 async function checkVideoServer2 (server: PeerTubeServer, id: number | string) {
  62   const video = await server.videos.get({ id })
  63
  64   expect(video.name).to.equal('my super name')
  65   expect(video.category.label).to.equal('Entertainment')
  66   expect(video.licence.label).to.equal('Public Domain Dedication')
  67   expect(video.language.label).to.equal('English')
  68   expect(video.nsfw).to.be.false
  69   expect(video.description).to.equal('my super description')
  70   expect(video.tags).to.deep.equal([ 'supertag1', 'supertag2' ])
  71
  72   expect(video.files).to.have.lengthOf(1)
  73
  74   const bodyCaptions = await server.captions.list({ videoId: id })
  75   expect(bodyCaptions.total).to.equal(2)
  76 }
  77
  78 describe('Test video imports', function () {
  79
  80   if (areHttpImportTestsDisabled()) return
  81
  82   function runSuite (mode: 'youtube-dl' | 'yt-dlp') {
  83
  84     describe('Import ' + mode, function () {
  85       let servers: PeerTubeServer[] = []
  86
  87       before(async function () {
  88         this.timeout(60_000)
  89
  90         servers = await createMultipleServers(2, getServerImportConfig(mode))
  91
  92         await setAccessTokensToServers(servers)
  93         await setDefaultVideoChannel(servers)
  94
  95         for (const server of servers) {
  96           await server.config.updateExistingSubConfig({
  97             newConfig: {
  98               transcoding: {
  99                 alwaysTranscodeOriginalResolution: false
 100               }
 101             }
 102           })
 103         }
 104
 105         await doubleFollow(servers[0], servers[1])
 106       })
 107
 108       it('Should import videos on server 1', async function () {
 109         this.timeout(60_000)
 110
 111         const baseAttributes = {
 112           channelId: servers[0].store.channel.id,
 113           privacy: VideoPrivacy.PUBLIC
 114         }
 115
 116         {
 117           const attributes = { ...baseAttributes, targetUrl: FIXTURE_URLS.youtube }
 118           const { video } = await servers[0].imports.importVideo({ attributes })
 119           expect(video.name).to.equal('small video - youtube')
 120
 121           {
 122             expect(video.thumbnailPath).to.match(new RegExp(`^/static/thumbnails/.+.jpg$`))
 123             expect(video.previewPath).to.match(new RegExp(`^/lazy-static/previews/.+.jpg$`))
 124
 125             const suffix = mode === 'yt-dlp'
 126               ? '_yt_dlp'
 127               : ''
 128
 129             await testImage(servers[0].url, 'video_import_thumbnail' + suffix, video.thumbnailPath)
 130             await testImage(servers[0].url, 'video_import_preview' + suffix, video.previewPath)
 131           }
 132
 133           const bodyCaptions = await servers[0].captions.list({ videoId: video.id })
 134           const videoCaptions = bodyCaptions.data
 135           expect(videoCaptions).to.have.lengthOf(2)
 136
 137           {
 138             const enCaption = videoCaptions.find(caption => caption.language.id === 'en')
 139             expect(enCaption).to.exist
 140             expect(enCaption.language.label).to.equal('English')
 141             expect(enCaption.captionPath).to.match(new RegExp(`^/lazy-static/video-captions/.+-en.vtt$`))
 142
 143             const regex = `WEBVTT[ \n]+Kind: captions[ \n]+` +
 144               `(Language: en[ \n]+)?` +
 145               `00:00:01.600 --> 00:00:04.200( position:\\d+% line:\\d+%)?[ \n]+English \\(US\\)[ \n]+` +
 146               `00:00:05.900 --> 00:00:07.999( position:\\d+% line:\\d+%)?[ \n]+This is a subtitle in American English[ \n]+` +
 147               `00:00:10.000 --> 00:00:14.000( position:\\d+% line:\\d+%)?[ \n]+Adding subtitles is very easy to do`
 148             await testCaptionFile(servers[0].url, enCaption.captionPath, new RegExp(regex))
 149           }
 150
 151           {
 152             const frCaption = videoCaptions.find(caption => caption.language.id === 'fr')
 153             expect(frCaption).to.exist
 154             expect(frCaption.language.label).to.equal('French')
 155             expect(frCaption.captionPath).to.match(new RegExp(`^/lazy-static/video-captions/.+-fr.vtt`))
 156
 157             const regex = `WEBVTT[ \n]+Kind: captions[ \n]+` +
 158               `(Language: fr[ \n]+)?` +
 159               `00:00:01.600 --> 00:00:04.200( position:\\d+% line:\\d+%)?[ \n]+Français \\(FR\\)[ \n]+` +
 160               `00:00:05.900 --> 00:00:07.999( position:\\d+% line:\\d+%)?[ \n]+C'est un sous-titre français[ \n]+` +
 161               `00:00:10.000 --> 00:00:14.000( position:\\d+% line:\\d+%)?[ \n]+Ajouter un sous-titre est vraiment facile`
 162
 163             await testCaptionFile(servers[0].url, frCaption.captionPath, new RegExp(regex))
 164           }
 165         }
 166
 167         {
 168           const attributes = {
 169             ...baseAttributes,
 170             magnetUri: FIXTURE_URLS.magnet,
 171             description: 'this is a super torrent description',
 172             tags: [ 'tag_torrent1', 'tag_torrent2' ]
 173           }
 174           const { video } = await servers[0].imports.importVideo({ attributes })
 175           expect(video.name).to.equal('super peertube2 video')
 176         }
 177
 178         {
 179           const attributes = {
 180             ...baseAttributes,
 181             torrentfile: 'video-720p.torrent' as any,
 182             description: 'this is a super torrent description',
 183             tags: [ 'tag_torrent1', 'tag_torrent2' ]
 184           }
 185           const { video } = await servers[0].imports.importVideo({ attributes })
 186           expect(video.name).to.equal('你好 世界 720p.mp4')
 187         }
 188       })
 189
 190       it('Should list the videos to import in my videos on server 1', async function () {
 191         const { total, data } = await servers[0].videos.listMyVideos({ sort: 'createdAt' })
 192
 193         expect(total).to.equal(3)
 194
 195         expect(data).to.have.lengthOf(3)
 196         expect(data[0].name).to.equal('small video - youtube')
 197         expect(data[1].name).to.equal('super peertube2 video')
 198         expect(data[2].name).to.equal('你好 世界 720p.mp4')
 199       })
 200
 201       it('Should list the videos to import in my imports on server 1', async function () {
 202         const { total, data: videoImports } = await servers[0].imports.getMyVideoImports({ sort: '-createdAt' })
 203         expect(total).to.equal(3)
 204
 205         expect(videoImports).to.have.lengthOf(3)
 206
 207         expect(videoImports[2].targetUrl).to.equal(FIXTURE_URLS.youtube)
 208         expect(videoImports[2].magnetUri).to.be.null
 209         expect(videoImports[2].torrentName).to.be.null
 210         expect(videoImports[2].video.name).to.equal('small video - youtube')
 211
 212         expect(videoImports[1].targetUrl).to.be.null
 213         expect(videoImports[1].magnetUri).to.equal(FIXTURE_URLS.magnet)
 214         expect(videoImports[1].torrentName).to.be.null
 215         expect(videoImports[1].video.name).to.equal('super peertube2 video')
 216
 217         expect(videoImports[0].targetUrl).to.be.null
 218         expect(videoImports[0].magnetUri).to.be.null
 219         expect(videoImports[0].torrentName).to.equal('video-720p.torrent')
 220         expect(videoImports[0].video.name).to.equal('你好 世界 720p.mp4')
 221       })
 222
 223       it('Should filter my imports on target URL', async function () {
 224         const { total, data: videoImports } = await servers[0].imports.getMyVideoImports({ targetUrl: FIXTURE_URLS.youtube })
 225         expect(total).to.equal(1)
 226         expect(videoImports).to.have.lengthOf(1)
 227
 228         expect(videoImports[0].targetUrl).to.equal(FIXTURE_URLS.youtube)
 229       })
 230
 231       it('Should have the video listed on the two instances', async function () {
 232         this.timeout(120_000)
 233
 234         await waitJobs(servers)
 235
 236         for (const server of servers) {
 237           const { total, data } = await server.videos.list()
 238           expect(total).to.equal(3)
 239           expect(data).to.have.lengthOf(3)
 240
 241           const [ videoHttp, videoMagnet, videoTorrent ] = data
 242           await checkVideosServer1(server, videoHttp.uuid, videoMagnet.uuid, videoTorrent.uuid)
 243         }
 244       })
 245
 246       it('Should import a video on server 2 with some fields', async function () {
 247         this.timeout(60_000)
 248
 249         const attributes = {
 250           targetUrl: FIXTURE_URLS.youtube,
 251           channelId: servers[1].store.channel.id,
 252           privacy: VideoPrivacy.PUBLIC,
 253           category: 10,
 254           licence: 7,
 255           language: 'en',
 256           name: 'my super name',
 257           description: 'my super description',
 258           tags: [ 'supertag1', 'supertag2' ]
 259         }
 260         const { video } = await servers[1].imports.importVideo({ attributes })
 261         expect(video.name).to.equal('my super name')
 262       })
 263
 264       it('Should have the videos listed on the two instances', async function () {
 265         this.timeout(120_000)
 266
 267         await waitJobs(servers)
 268
 269         for (const server of servers) {
 270           const { total, data } = await server.videos.list()
 271           expect(total).to.equal(4)
 272           expect(data).to.have.lengthOf(4)
 273
 274           await checkVideoServer2(server, data[0].uuid)
 275
 276           const [ , videoHttp, videoMagnet, videoTorrent ] = data
 277           await checkVideosServer1(server, videoHttp.uuid, videoMagnet.uuid, videoTorrent.uuid)
 278         }
 279       })
 280
 281       it('Should import a video that will be transcoded', async function () {
 282         this.timeout(240_000)
 283
 284         const attributes = {
 285           name: 'transcoded video',
 286           magnetUri: FIXTURE_URLS.magnet,
 287           channelId: servers[1].store.channel.id,
 288           privacy: VideoPrivacy.PUBLIC
 289         }
 290         const { video } = await servers[1].imports.importVideo({ attributes })
 291         const videoUUID = video.uuid
 292
 293         await waitJobs(servers)
 294
 295         for (const server of servers) {
 296           const video = await server.videos.get({ id: videoUUID })
 297
 298           expect(video.name).to.equal('transcoded video')
 299           expect(video.files).to.have.lengthOf(4)
 300         }
 301       })
 302
 303       it('Should import no HDR version on a HDR video', async function () {
 304         this.timeout(300_000)
 305
 306         const config: DeepPartial<CustomConfig> = {
 307           transcoding: {
 308             enabled: true,
 309             resolutions: {
 310               '0p': false,
 311               '144p': true,
 312               '240p': true,
 313               '360p': false,
 314               '480p': false,
 315               '720p': false,
 316               '1080p': false, // the resulting resolution shouldn't be higher than this, and not vp9.2/av01
 317               '1440p': false,
 318               '2160p': false
 319             },
 320             webtorrent: { enabled: true },
 321             hls: { enabled: false }
 322           }
 323         }
 324         await servers[0].config.updateExistingSubConfig({ newConfig: config })
 325
 326         const attributes = {
 327           name: 'hdr video',
 328           targetUrl: FIXTURE_URLS.youtubeHDR,
 329           channelId: servers[0].store.channel.id,
 330           privacy: VideoPrivacy.PUBLIC
 331         }
 332         const { video: videoImported } = await servers[0].imports.importVideo({ attributes })
 333         const videoUUID = videoImported.uuid
 334
 335         await waitJobs(servers)
 336
 337         // test resolution
 338         const video = await servers[0].videos.get({ id: videoUUID })
 339         expect(video.name).to.equal('hdr video')
 340         const maxResolution = Math.max.apply(Math, video.files.map(function (o) { return o.resolution.id }))
 341         expect(maxResolution, 'expected max resolution not met').to.equals(VideoResolution.H_240P)
 342       })
 343
 344       it('Should not import resolution higher than enabled transcoding resolution', async function () {
 345         this.timeout(300_000)
 346
 347         const config: DeepPartial<CustomConfig> = {
 348           transcoding: {
 349             enabled: true,
 350             resolutions: {
 351               '0p': false,
 352               '144p': true,
 353               '240p': false,
 354               '360p': false,
 355               '480p': false,
 356               '720p': false,
 357               '1080p': false,
 358               '1440p': false,
 359               '2160p': false
 360             },
 361             alwaysTranscodeOriginalResolution: false
 362           }
 363         }
 364         await servers[0].config.updateExistingSubConfig({ newConfig: config })
 365
 366         const attributes = {
 367           name: 'small resolution video',
 368           targetUrl: FIXTURE_URLS.youtube,
 369           channelId: servers[0].store.channel.id,
 370           privacy: VideoPrivacy.PUBLIC
 371         }
 372         const { video: videoImported } = await servers[0].imports.importVideo({ attributes })
 373         const videoUUID = videoImported.uuid
 374
 375         await waitJobs(servers)
 376
 377         // test resolution
 378         const video = await servers[0].videos.get({ id: videoUUID })
 379         expect(video.name).to.equal('small resolution video')
 380         expect(video.files).to.have.lengthOf(1)
 381         expect(video.files[0].resolution.id).to.equal(144)
 382       })
 383
 384       it('Should import resolution higher than enabled transcoding resolution', async function () {
 385         this.timeout(300_000)
 386
 387         const config: DeepPartial<CustomConfig> = {
 388           transcoding: {
 389             alwaysTranscodeOriginalResolution: true
 390           }
 391         }
 392         await servers[0].config.updateExistingSubConfig({ newConfig: config })
 393
 394         const attributes = {
 395           name: 'bigger resolution video',
 396           targetUrl: FIXTURE_URLS.youtube,
 397           channelId: servers[0].store.channel.id,
 398           privacy: VideoPrivacy.PUBLIC
 399         }
 400         const { video: videoImported } = await servers[0].imports.importVideo({ attributes })
 401         const videoUUID = videoImported.uuid
 402
 403         await waitJobs(servers)
 404
 405         // test resolution
 406         const video = await servers[0].videos.get({ id: videoUUID })
 407         expect(video.name).to.equal('bigger resolution video')
 408
 409         expect(video.files).to.have.lengthOf(2)
 410         expect(video.files.find(f => f.resolution.id === 240)).to.exist
 411         expect(video.files.find(f => f.resolution.id === 144)).to.exist
 412       })
 413
 414       it('Should import a peertube video', async function () {
 415         this.timeout(120_000)
 416
 417         const toTest = [ FIXTURE_URLS.peertube_long ]
 418
 419         // TODO: include peertube_short when https://github.com/ytdl-org/youtube-dl/pull/29475 is merged
 420         if (mode === 'yt-dlp') {
 421           toTest.push(FIXTURE_URLS.peertube_short)
 422         }
 423
 424         for (const targetUrl of toTest) {
 425           await servers[0].config.disableTranscoding()
 426
 427           const attributes = {
 428             targetUrl,
 429             channelId: servers[0].store.channel.id,
 430             privacy: VideoPrivacy.PUBLIC
 431           }
 432           const { video } = await servers[0].imports.importVideo({ attributes })
 433           const videoUUID = video.uuid
 434
 435           await waitJobs(servers)
 436
 437           for (const server of servers) {
 438             const video = await server.videos.get({ id: videoUUID })
 439
 440             expect(video.name).to.equal('E2E tests')
 441           }
 442         }
 443       })
 444
 445       after(async function () {
 446         await cleanupTests(servers)
 447       })
 448     })
 449   }
 450
 451   runSuite('youtube-dl')
 452
 453   runSuite('yt-dlp')
 454
 455   describe('Delete/cancel an import', function () {
 456     let server: PeerTubeServer
 457
 458     let finishedImportId: number
 459     let finishedVideo: Video
 460     let pendingImportId: number
 461
 462     async function importVideo (name: string) {
 463       const attributes = { name, channelId: server.store.channel.id, targetUrl: FIXTURE_URLS.goodVideo }
 464       const res = await server.imports.importVideo({ attributes })
 465
 466       return res.id
 467     }
 468
 469     before(async function () {
 470       this.timeout(120_000)
 471
 472       server = await createSingleServer(1)
 473
 474       await setAccessTokensToServers([ server ])
 475       await setDefaultVideoChannel([ server ])
 476
 477       finishedImportId = await importVideo('finished')
 478       await waitJobs([ server ])
 479
 480       await server.jobs.pauseJobQueue()
 481       pendingImportId = await importVideo('pending')
 482
 483       const { data } = await server.imports.getMyVideoImports()
 484       expect(data).to.have.lengthOf(2)
 485
 486       finishedVideo = data.find(i => i.id === finishedImportId).video
 487     })
 488
 489     it('Should delete a video import', async function () {
 490       await server.imports.delete({ importId: finishedImportId })
 491
 492       const { data } = await server.imports.getMyVideoImports()
 493       expect(data).to.have.lengthOf(1)
 494       expect(data[0].id).to.equal(pendingImportId)
 495       expect(data[0].state.id).to.equal(VideoImportState.PENDING)
 496     })
 497
 498     it('Should not have deleted the associated video', async function () {
 499       const video = await server.videos.get({ id: finishedVideo.id, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 })
 500       expect(video.name).to.equal('finished')
 501       expect(video.state.id).to.equal(VideoState.PUBLISHED)
 502     })
 503
 504     it('Should cancel a video import', async function () {
 505       await server.imports.cancel({ importId: pendingImportId })
 506
 507       const { data } = await server.imports.getMyVideoImports()
 508       expect(data).to.have.lengthOf(1)
 509       expect(data[0].id).to.equal(pendingImportId)
 510       expect(data[0].state.id).to.equal(VideoImportState.CANCELLED)
 511     })
 512
 513     it('Should not have processed the cancelled video import', async function () {
 514       this.timeout(60_000)
 515
 516       await server.jobs.resumeJobQueue()
 517
 518       await waitJobs([ server ])
 519
 520       const { data } = await server.imports.getMyVideoImports()
 521       expect(data).to.have.lengthOf(1)
 522       expect(data[0].id).to.equal(pendingImportId)
 523       expect(data[0].state.id).to.equal(VideoImportState.CANCELLED)
 524       expect(data[0].video.state.id).to.equal(VideoState.TO_IMPORT)
 525     })
 526
 527     it('Should delete the cancelled video import', async function () {
 528       await server.imports.delete({ importId: pendingImportId })
 529       const { data } = await server.imports.getMyVideoImports()
 530       expect(data).to.have.lengthOf(0)
 531     })
 532
 533     after(async function () {
 534       await cleanupTests([ server ])
 535     })
 536   })
 537
 538   describe('Auto update', function () {
 539     let server: PeerTubeServer
 540
 541     function quickPeerTubeImport () {
 542       const attributes = {
 543         targetUrl: FIXTURE_URLS.peertube_long,
 544         channelId: server.store.channel.id,
 545         privacy: VideoPrivacy.PUBLIC
 546       }
 547
 548       return server.imports.importVideo({ attributes })
 549     }
 550
 551     async function testBinaryUpdate (releaseUrl: string, releaseName: string) {
 552       await remove(join(server.servers.buildDirectory('bin'), releaseName))
 553
 554       await server.kill()
 555       await server.run({
 556         import: {
 557           videos: {
 558             http: {
 559               youtube_dl_release: {
 560                 url: releaseUrl,
 561                 name: releaseName
 562               }
 563             }
 564           }
 565         }
 566       })
 567
 568       await quickPeerTubeImport()
 569
 570       const base = server.servers.buildDirectory('bin')
 571       const content = await readdir(base)
 572       const binaryPath = join(base, releaseName)
 573
 574       expect(await pathExists(binaryPath), `${binaryPath} does not exist in ${base} (${content.join(', ')})`).to.be.true
 575     }
 576
 577     before(async function () {
 578       this.timeout(30_000)
 579
 580       // Run servers
 581       server = await createSingleServer(1)
 582
 583       await setAccessTokensToServers([ server ])
 584       await setDefaultVideoChannel([ server ])
 585     })
 586
 587     it('Should update youtube-dl from github URL', async function () {
 588       this.timeout(120_000)
 589
 590       await testBinaryUpdate('https://api.github.com/repos/ytdl-org/youtube-dl/releases', 'youtube-dl')
 591     })
 592
 593     it('Should update youtube-dl from raw URL', async function () {
 594       this.timeout(120_000)
 595
 596       await testBinaryUpdate('https://yt-dl.org/downloads/latest/youtube-dl', 'youtube-dl')
 597     })
 598
 599     it('Should update youtube-dl from youtube-dl fork', async function () {
 600       this.timeout(120_000)
 601
 602       await testBinaryUpdate('https://api.github.com/repos/yt-dlp/yt-dlp/releases', 'yt-dlp')
 603     })
 604
 605     after(async function () {
 606       await cleanupTests([ server ])
 607     })
 608   })
 609 })