server/tests/api/videos/video-imports.ts

   1 /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
   2
   3 import 'mocha'
   4 import { expect } from 'chai'
   5 import { pathExists, readdir, remove } from 'fs-extra'
   6 import { join } from 'path'
   7 import { FIXTURE_URLS, testCaptionFile, testImage } from '@server/tests/shared'
   8 import { areHttpImportTestsDisabled } from '@shared/core-utils'
   9 import { CustomConfig, HttpStatusCode, Video, VideoImportState, VideoPrivacy, VideoResolution, VideoState } from '@shared/models'
  10 import {
  11   cleanupTests,
  12   createMultipleServers,
  13   createSingleServer,
  14   doubleFollow,
  15   getServerImportConfig,
  16   PeerTubeServer,
  17   setAccessTokensToServers,
  18   setDefaultVideoChannel,
  19   waitJobs
  20 } from '@shared/server-commands'
  21 import { DeepPartial } from '@shared/typescript-utils'
  22
  23 async function checkVideosServer1 (server: PeerTubeServer, idHttp: string, idMagnet: string, idTorrent: string) {
  24   const videoHttp = await server.videos.get({ id: idHttp })
  25
  26   expect(videoHttp.name).to.equal('small video - youtube')
  27   // FIXME: youtube-dl seems broken
  28   // expect(videoHttp.category.label).to.equal('News & Politics')
  29   // expect(videoHttp.licence.label).to.equal('Attribution')
  30   expect(videoHttp.language.label).to.equal('Unknown')
  31   expect(videoHttp.nsfw).to.be.false
  32   expect(videoHttp.description).to.equal('this is a super description')
  33   expect(videoHttp.tags).to.deep.equal([ 'tag1', 'tag2' ])
  34   expect(videoHttp.files).to.have.lengthOf(1)
  35
  36   const originallyPublishedAt = new Date(videoHttp.originallyPublishedAt)
  37   expect(originallyPublishedAt.getDate()).to.equal(14)
  38   expect(originallyPublishedAt.getMonth()).to.equal(0)
  39   expect(originallyPublishedAt.getFullYear()).to.equal(2019)
  40
  41   const videoMagnet = await server.videos.get({ id: idMagnet })
  42   const videoTorrent = await server.videos.get({ id: idTorrent })
  43
  44   for (const video of [ videoMagnet, videoTorrent ]) {
  45     expect(video.category.label).to.equal('Misc')
  46     expect(video.licence.label).to.equal('Unknown')
  47     expect(video.language.label).to.equal('Unknown')
  48     expect(video.nsfw).to.be.false
  49     expect(video.description).to.equal('this is a super torrent description')
  50     expect(video.tags).to.deep.equal([ 'tag_torrent1', 'tag_torrent2' ])
  51     expect(video.files).to.have.lengthOf(1)
  52   }
  53
  54   expect(videoTorrent.name).to.contain('你好 世界 720p.mp4')
  55   expect(videoMagnet.name).to.contain('super peertube2 video')
  56
  57   const bodyCaptions = await server.captions.list({ videoId: idHttp })
  58   expect(bodyCaptions.total).to.equal(2)
  59 }
  60
  61 async function checkVideoServer2 (server: PeerTubeServer, id: number | string) {
  62   const video = await server.videos.get({ id })
  63
  64   expect(video.name).to.equal('my super name')
  65   expect(video.category.label).to.equal('Entertainment')
  66   expect(video.licence.label).to.equal('Public Domain Dedication')
  67   expect(video.language.label).to.equal('English')
  68   expect(video.nsfw).to.be.false
  69   expect(video.description).to.equal('my super description')
  70   expect(video.tags).to.deep.equal([ 'supertag1', 'supertag2' ])
  71
  72   expect(video.files).to.have.lengthOf(1)
  73
  74   const bodyCaptions = await server.captions.list({ videoId: id })
  75   expect(bodyCaptions.total).to.equal(2)
  76 }
  77
  78 describe('Test video imports', function () {
  79
  80   if (areHttpImportTestsDisabled()) return
  81
  82   function runSuite (mode: 'youtube-dl' | 'yt-dlp') {
  83
  84     describe('Import ' + mode, function () {
  85       let servers: PeerTubeServer[] = []
  86
  87       before(async function () {
  88         this.timeout(60_000)
  89
  90         servers = await createMultipleServers(2, getServerImportConfig(mode))
  91
  92         await setAccessTokensToServers(servers)
  93         await setDefaultVideoChannel(servers)
  94
  95         for (const server of servers) {
  96           await server.config.updateExistingSubConfig({
  97             newConfig: {
  98               transcoding: {
  99                 alwaysTranscodeOriginalResolution: false
 100               }
 101             }
 102           })
 103         }
 104
 105         await doubleFollow(servers[0], servers[1])
 106       })
 107
 108       it('Should import videos on server 1', async function () {
 109         this.timeout(60_000)
 110
 111         const baseAttributes = {
 112           channelId: servers[0].store.channel.id,
 113           privacy: VideoPrivacy.PUBLIC
 114         }
 115
 116         {
 117           const attributes = { ...baseAttributes, targetUrl: FIXTURE_URLS.youtube }
 118           const { video } = await servers[0].imports.importVideo({ attributes })
 119           expect(video.name).to.equal('small video - youtube')
 120
 121           {
 122             expect(video.thumbnailPath).to.match(new RegExp(`^/static/thumbnails/.+.jpg$`))
 123             expect(video.previewPath).to.match(new RegExp(`^/lazy-static/previews/.+.jpg$`))
 124
 125             const suffix = mode === 'yt-dlp'
 126               ? '_yt_dlp'
 127               : ''
 128
 129             await testImage(servers[0].url, 'video_import_thumbnail' + suffix, video.thumbnailPath)
 130             await testImage(servers[0].url, 'video_import_preview' + suffix, video.previewPath)
 131           }
 132
 133           const bodyCaptions = await servers[0].captions.list({ videoId: video.id })
 134           const videoCaptions = bodyCaptions.data
 135           expect(videoCaptions).to.have.lengthOf(2)
 136
 137           {
 138             const enCaption = videoCaptions.find(caption => caption.language.id === 'en')
 139             expect(enCaption).to.exist
 140             expect(enCaption.language.label).to.equal('English')
 141             expect(enCaption.captionPath).to.match(new RegExp(`^/lazy-static/video-captions/.+-en.vtt$`))
 142
 143             const regex = `WEBVTT[ \n]+Kind: captions[ \n]+` +
 144               `(Language: en[ \n]+)?` +
 145               `00:00:01.600 --> 00:00:04.200( position:\\d+% line:\\d+%)?[ \n]+English \\(US\\)[ \n]+` +
 146               `00:00:05.900 --> 00:00:07.999( position:\\d+% line:\\d+%)?[ \n]+This is a subtitle in American English[ \n]+` +
 147               `00:00:10.000 --> 00:00:14.000( position:\\d+% line:\\d+%)?[ \n]+Adding subtitles is very easy to do`
 148             await testCaptionFile(servers[0].url, enCaption.captionPath, new RegExp(regex))
 149           }
 150
 151           {
 152             const frCaption = videoCaptions.find(caption => caption.language.id === 'fr')
 153             expect(frCaption).to.exist
 154             expect(frCaption.language.label).to.equal('French')
 155             expect(frCaption.captionPath).to.match(new RegExp(`^/lazy-static/video-captions/.+-fr.vtt`))
 156
 157             const regex = `WEBVTT[ \n]+Kind: captions[ \n]+` +
 158               `(Language: fr[ \n]+)?` +
 159               `00:00:01.600 --> 00:00:04.200( position:\\d+% line:\\d+%)?[ \n]+Français \\(FR\\)[ \n]+` +
 160               `00:00:05.900 --> 00:00:07.999( position:\\d+% line:\\d+%)?[ \n]+C'est un sous-titre français[ \n]+` +
 161               `00:00:10.000 --> 00:00:14.000( position:\\d+% line:\\d+%)?[ \n]+Ajouter un sous-titre est vraiment facile`
 162
 163             await testCaptionFile(servers[0].url, frCaption.captionPath, new RegExp(regex))
 164           }
 165         }
 166
 167         {
 168           const attributes = {
 169             ...baseAttributes,
 170             magnetUri: FIXTURE_URLS.magnet,
 171             description: 'this is a super torrent description',
 172             tags: [ 'tag_torrent1', 'tag_torrent2' ]
 173           }
 174           const { video } = await servers[0].imports.importVideo({ attributes })
 175           expect(video.name).to.equal('super peertube2 video')
 176         }
 177
 178         {
 179           const attributes = {
 180             ...baseAttributes,
 181             torrentfile: 'video-720p.torrent' as any,
 182             description: 'this is a super torrent description',
 183             tags: [ 'tag_torrent1', 'tag_torrent2' ]
 184           }
 185           const { video } = await servers[0].imports.importVideo({ attributes })
 186           expect(video.name).to.equal('你好 世界 720p.mp4')
 187         }
 188       })
 189
 190       it('Should list the videos to import in my videos on server 1', async function () {
 191         const { total, data } = await servers[0].videos.listMyVideos({ sort: 'createdAt' })
 192
 193         expect(total).to.equal(3)
 194
 195         expect(data).to.have.lengthOf(3)
 196         expect(data[0].name).to.equal('small video - youtube')
 197         expect(data[1].name).to.equal('super peertube2 video')
 198         expect(data[2].name).to.equal('你好 世界 720p.mp4')
 199       })
 200
 201       it('Should list the videos to import in my imports on server 1', async function () {
 202         const { total, data: videoImports } = await servers[0].imports.getMyVideoImports({ sort: '-createdAt' })
 203         expect(total).to.equal(3)
 204
 205         expect(videoImports).to.have.lengthOf(3)
 206
 207         expect(videoImports[2].targetUrl).to.equal(FIXTURE_URLS.youtube)
 208         expect(videoImports[2].magnetUri).to.be.null
 209         expect(videoImports[2].torrentName).to.be.null
 210         expect(videoImports[2].video.name).to.equal('small video - youtube')
 211
 212         expect(videoImports[1].targetUrl).to.be.null
 213         expect(videoImports[1].magnetUri).to.equal(FIXTURE_URLS.magnet)
 214         expect(videoImports[1].torrentName).to.be.null
 215         expect(videoImports[1].video.name).to.equal('super peertube2 video')
 216
 217         expect(videoImports[0].targetUrl).to.be.null
 218         expect(videoImports[0].magnetUri).to.be.null
 219         expect(videoImports[0].torrentName).to.equal('video-720p.torrent')
 220         expect(videoImports[0].video.name).to.equal('你好 世界 720p.mp4')
 221       })
 222
 223       it('Should filter my imports on target URL', async function () {
 224         const { total, data: videoImports } = await servers[0].imports.getMyVideoImports({ targetUrl: FIXTURE_URLS.youtube })
 225         expect(total).to.equal(1)
 226         expect(videoImports).to.have.lengthOf(1)
 227
 228         expect(videoImports[0].targetUrl).to.equal(FIXTURE_URLS.youtube)
 229       })
 230
 231       it('Should search in my imports', async function () {
 232         const { total, data: videoImports } = await servers[0].imports.getMyVideoImports({ search: 'peertube2' })
 233         expect(total).to.equal(1)
 234         expect(videoImports).to.have.lengthOf(1)
 235
 236         expect(videoImports[0].magnetUri).to.equal(FIXTURE_URLS.magnet)
 237         expect(videoImports[0].video.name).to.equal('super peertube2 video')
 238       })
 239
 240       it('Should have the video listed on the two instances', async function () {
 241         this.timeout(120_000)
 242
 243         await waitJobs(servers)
 244
 245         for (const server of servers) {
 246           const { total, data } = await server.videos.list()
 247           expect(total).to.equal(3)
 248           expect(data).to.have.lengthOf(3)
 249
 250           const [ videoHttp, videoMagnet, videoTorrent ] = data
 251           await checkVideosServer1(server, videoHttp.uuid, videoMagnet.uuid, videoTorrent.uuid)
 252         }
 253       })
 254
 255       it('Should import a video on server 2 with some fields', async function () {
 256         this.timeout(60_000)
 257
 258         const attributes = {
 259           targetUrl: FIXTURE_URLS.youtube,
 260           channelId: servers[1].store.channel.id,
 261           privacy: VideoPrivacy.PUBLIC,
 262           category: 10,
 263           licence: 7,
 264           language: 'en',
 265           name: 'my super name',
 266           description: 'my super description',
 267           tags: [ 'supertag1', 'supertag2' ]
 268         }
 269         const { video } = await servers[1].imports.importVideo({ attributes })
 270         expect(video.name).to.equal('my super name')
 271       })
 272
 273       it('Should have the videos listed on the two instances', async function () {
 274         this.timeout(120_000)
 275
 276         await waitJobs(servers)
 277
 278         for (const server of servers) {
 279           const { total, data } = await server.videos.list()
 280           expect(total).to.equal(4)
 281           expect(data).to.have.lengthOf(4)
 282
 283           await checkVideoServer2(server, data[0].uuid)
 284
 285           const [ , videoHttp, videoMagnet, videoTorrent ] = data
 286           await checkVideosServer1(server, videoHttp.uuid, videoMagnet.uuid, videoTorrent.uuid)
 287         }
 288       })
 289
 290       it('Should import a video that will be transcoded', async function () {
 291         this.timeout(240_000)
 292
 293         const attributes = {
 294           name: 'transcoded video',
 295           magnetUri: FIXTURE_URLS.magnet,
 296           channelId: servers[1].store.channel.id,
 297           privacy: VideoPrivacy.PUBLIC
 298         }
 299         const { video } = await servers[1].imports.importVideo({ attributes })
 300         const videoUUID = video.uuid
 301
 302         await waitJobs(servers)
 303
 304         for (const server of servers) {
 305           const video = await server.videos.get({ id: videoUUID })
 306
 307           expect(video.name).to.equal('transcoded video')
 308           expect(video.files).to.have.lengthOf(4)
 309         }
 310       })
 311
 312       it('Should import no HDR version on a HDR video', async function () {
 313         this.timeout(300_000)
 314
 315         const config: DeepPartial<CustomConfig> = {
 316           transcoding: {
 317             enabled: true,
 318             resolutions: {
 319               '0p': false,
 320               '144p': true,
 321               '240p': true,
 322               '360p': false,
 323               '480p': false,
 324               '720p': false,
 325               '1080p': false, // the resulting resolution shouldn't be higher than this, and not vp9.2/av01
 326               '1440p': false,
 327               '2160p': false
 328             },
 329             webtorrent: { enabled: true },
 330             hls: { enabled: false }
 331           }
 332         }
 333         await servers[0].config.updateExistingSubConfig({ newConfig: config })
 334
 335         const attributes = {
 336           name: 'hdr video',
 337           targetUrl: FIXTURE_URLS.youtubeHDR,
 338           channelId: servers[0].store.channel.id,
 339           privacy: VideoPrivacy.PUBLIC
 340         }
 341         const { video: videoImported } = await servers[0].imports.importVideo({ attributes })
 342         const videoUUID = videoImported.uuid
 343
 344         await waitJobs(servers)
 345
 346         // test resolution
 347         const video = await servers[0].videos.get({ id: videoUUID })
 348         expect(video.name).to.equal('hdr video')
 349         const maxResolution = Math.max.apply(Math, video.files.map(function (o) { return o.resolution.id }))
 350         expect(maxResolution, 'expected max resolution not met').to.equals(VideoResolution.H_240P)
 351       })
 352
 353       it('Should not import resolution higher than enabled transcoding resolution', async function () {
 354         this.timeout(300_000)
 355
 356         const config: DeepPartial<CustomConfig> = {
 357           transcoding: {
 358             enabled: true,
 359             resolutions: {
 360               '0p': false,
 361               '144p': true,
 362               '240p': false,
 363               '360p': false,
 364               '480p': false,
 365               '720p': false,
 366               '1080p': false,
 367               '1440p': false,
 368               '2160p': false
 369             },
 370             alwaysTranscodeOriginalResolution: false
 371           }
 372         }
 373         await servers[0].config.updateExistingSubConfig({ newConfig: config })
 374
 375         const attributes = {
 376           name: 'small resolution video',
 377           targetUrl: FIXTURE_URLS.youtube,
 378           channelId: servers[0].store.channel.id,
 379           privacy: VideoPrivacy.PUBLIC
 380         }
 381         const { video: videoImported } = await servers[0].imports.importVideo({ attributes })
 382         const videoUUID = videoImported.uuid
 383
 384         await waitJobs(servers)
 385
 386         // test resolution
 387         const video = await servers[0].videos.get({ id: videoUUID })
 388         expect(video.name).to.equal('small resolution video')
 389         expect(video.files).to.have.lengthOf(1)
 390         expect(video.files[0].resolution.id).to.equal(144)
 391       })
 392
 393       it('Should import resolution higher than enabled transcoding resolution', async function () {
 394         this.timeout(300_000)
 395
 396         const config: DeepPartial<CustomConfig> = {
 397           transcoding: {
 398             alwaysTranscodeOriginalResolution: true
 399           }
 400         }
 401         await servers[0].config.updateExistingSubConfig({ newConfig: config })
 402
 403         const attributes = {
 404           name: 'bigger resolution video',
 405           targetUrl: FIXTURE_URLS.youtube,
 406           channelId: servers[0].store.channel.id,
 407           privacy: VideoPrivacy.PUBLIC
 408         }
 409         const { video: videoImported } = await servers[0].imports.importVideo({ attributes })
 410         const videoUUID = videoImported.uuid
 411
 412         await waitJobs(servers)
 413
 414         // test resolution
 415         const video = await servers[0].videos.get({ id: videoUUID })
 416         expect(video.name).to.equal('bigger resolution video')
 417
 418         expect(video.files).to.have.lengthOf(2)
 419         expect(video.files.find(f => f.resolution.id === 240)).to.exist
 420         expect(video.files.find(f => f.resolution.id === 144)).to.exist
 421       })
 422
 423       it('Should import a peertube video', async function () {
 424         this.timeout(120_000)
 425
 426         const toTest = [ FIXTURE_URLS.peertube_long ]
 427
 428         // TODO: include peertube_short when https://github.com/ytdl-org/youtube-dl/pull/29475 is merged
 429         if (mode === 'yt-dlp') {
 430           toTest.push(FIXTURE_URLS.peertube_short)
 431         }
 432
 433         for (const targetUrl of toTest) {
 434           await servers[0].config.disableTranscoding()
 435
 436           const attributes = {
 437             targetUrl,
 438             channelId: servers[0].store.channel.id,
 439             privacy: VideoPrivacy.PUBLIC
 440           }
 441           const { video } = await servers[0].imports.importVideo({ attributes })
 442           const videoUUID = video.uuid
 443
 444           await waitJobs(servers)
 445
 446           for (const server of servers) {
 447             const video = await server.videos.get({ id: videoUUID })
 448
 449             expect(video.name).to.equal('E2E tests')
 450           }
 451         }
 452       })
 453
 454       after(async function () {
 455         await cleanupTests(servers)
 456       })
 457     })
 458   }
 459
 460   runSuite('youtube-dl')
 461
 462   runSuite('yt-dlp')
 463
 464   describe('Delete/cancel an import', function () {
 465     let server: PeerTubeServer
 466
 467     let finishedImportId: number
 468     let finishedVideo: Video
 469     let pendingImportId: number
 470
 471     async function importVideo (name: string) {
 472       const attributes = { name, channelId: server.store.channel.id, targetUrl: FIXTURE_URLS.goodVideo }
 473       const res = await server.imports.importVideo({ attributes })
 474
 475       return res.id
 476     }
 477
 478     before(async function () {
 479       this.timeout(120_000)
 480
 481       server = await createSingleServer(1)
 482
 483       await setAccessTokensToServers([ server ])
 484       await setDefaultVideoChannel([ server ])
 485
 486       finishedImportId = await importVideo('finished')
 487       await waitJobs([ server ])
 488
 489       await server.jobs.pauseJobQueue()
 490       pendingImportId = await importVideo('pending')
 491
 492       const { data } = await server.imports.getMyVideoImports()
 493       expect(data).to.have.lengthOf(2)
 494
 495       finishedVideo = data.find(i => i.id === finishedImportId).video
 496     })
 497
 498     it('Should delete a video import', async function () {
 499       await server.imports.delete({ importId: finishedImportId })
 500
 501       const { data } = await server.imports.getMyVideoImports()
 502       expect(data).to.have.lengthOf(1)
 503       expect(data[0].id).to.equal(pendingImportId)
 504       expect(data[0].state.id).to.equal(VideoImportState.PENDING)
 505     })
 506
 507     it('Should not have deleted the associated video', async function () {
 508       const video = await server.videos.get({ id: finishedVideo.id, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 })
 509       expect(video.name).to.equal('finished')
 510       expect(video.state.id).to.equal(VideoState.PUBLISHED)
 511     })
 512
 513     it('Should cancel a video import', async function () {
 514       await server.imports.cancel({ importId: pendingImportId })
 515
 516       const { data } = await server.imports.getMyVideoImports()
 517       expect(data).to.have.lengthOf(1)
 518       expect(data[0].id).to.equal(pendingImportId)
 519       expect(data[0].state.id).to.equal(VideoImportState.CANCELLED)
 520     })
 521
 522     it('Should not have processed the cancelled video import', async function () {
 523       this.timeout(60_000)
 524
 525       await server.jobs.resumeJobQueue()
 526
 527       await waitJobs([ server ])
 528
 529       const { data } = await server.imports.getMyVideoImports()
 530       expect(data).to.have.lengthOf(1)
 531       expect(data[0].id).to.equal(pendingImportId)
 532       expect(data[0].state.id).to.equal(VideoImportState.CANCELLED)
 533       expect(data[0].video.state.id).to.equal(VideoState.TO_IMPORT)
 534     })
 535
 536     it('Should delete the cancelled video import', async function () {
 537       await server.imports.delete({ importId: pendingImportId })
 538       const { data } = await server.imports.getMyVideoImports()
 539       expect(data).to.have.lengthOf(0)
 540     })
 541
 542     after(async function () {
 543       await cleanupTests([ server ])
 544     })
 545   })
 546
 547   describe('Auto update', function () {
 548     let server: PeerTubeServer
 549
 550     function quickPeerTubeImport () {
 551       const attributes = {
 552         targetUrl: FIXTURE_URLS.peertube_long,
 553         channelId: server.store.channel.id,
 554         privacy: VideoPrivacy.PUBLIC
 555       }
 556
 557       return server.imports.importVideo({ attributes })
 558     }
 559
 560     async function testBinaryUpdate (releaseUrl: string, releaseName: string) {
 561       await remove(join(server.servers.buildDirectory('bin'), releaseName))
 562
 563       await server.kill()
 564       await server.run({
 565         import: {
 566           videos: {
 567             http: {
 568               youtube_dl_release: {
 569                 url: releaseUrl,
 570                 name: releaseName
 571               }
 572             }
 573           }
 574         }
 575       })
 576
 577       await quickPeerTubeImport()
 578
 579       const base = server.servers.buildDirectory('bin')
 580       const content = await readdir(base)
 581       const binaryPath = join(base, releaseName)
 582
 583       expect(await pathExists(binaryPath), `${binaryPath} does not exist in ${base} (${content.join(', ')})`).to.be.true
 584     }
 585
 586     before(async function () {
 587       this.timeout(30_000)
 588
 589       // Run servers
 590       server = await createSingleServer(1)
 591
 592       await setAccessTokensToServers([ server ])
 593       await setDefaultVideoChannel([ server ])
 594     })
 595
 596     it('Should update youtube-dl from github URL', async function () {
 597       this.timeout(120_000)
 598
 599       await testBinaryUpdate('https://api.github.com/repos/ytdl-org/youtube-dl/releases', 'youtube-dl')
 600     })
 601
 602     it('Should update youtube-dl from raw URL', async function () {
 603       this.timeout(120_000)
 604
 605       await testBinaryUpdate('https://yt-dl.org/downloads/latest/youtube-dl', 'youtube-dl')
 606     })
 607
 608     it('Should update youtube-dl from youtube-dl fork', async function () {
 609       this.timeout(120_000)
 610
 611       await testBinaryUpdate('https://api.github.com/repos/yt-dlp/yt-dlp/releases', 'yt-dlp')
 612     })
 613
 614     after(async function () {
 615       await cleanupTests([ server ])
 616     })
 617   })
 618 })