server/tests/api/videos/video-imports.ts

   1 /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
   2
   3 import { expect } from 'chai'
   4 import { pathExists, readdir, remove } from 'fs-extra'
   5 import { join } from 'path'
   6 import { FIXTURE_URLS, testCaptionFile, testImage } from '@server/tests/shared'
   7 import { areHttpImportTestsDisabled } from '@shared/core-utils'
   8 import { CustomConfig, HttpStatusCode, Video, VideoImportState, VideoPrivacy, VideoResolution, VideoState } from '@shared/models'
   9 import {
  10   cleanupTests,
  11   createMultipleServers,
  12   createSingleServer,
  13   doubleFollow,
  14   getServerImportConfig,
  15   PeerTubeServer,
  16   setAccessTokensToServers,
  17   setDefaultVideoChannel,
  18   waitJobs
  19 } from '@shared/server-commands'
  20 import { DeepPartial } from '@shared/typescript-utils'
  21
  22 async function checkVideosServer1 (server: PeerTubeServer, idHttp: string, idMagnet: string, idTorrent: string) {
  23   const videoHttp = await server.videos.get({ id: idHttp })
  24
  25   expect(videoHttp.name).to.equal('small video - youtube')
  26   // FIXME: youtube-dl seems broken
  27   // expect(videoHttp.category.label).to.equal('News & Politics')
  28   // expect(videoHttp.licence.label).to.equal('Attribution')
  29   expect(videoHttp.language.label).to.equal('Unknown')
  30   expect(videoHttp.nsfw).to.be.false
  31   expect(videoHttp.description).to.equal('this is a super description')
  32   expect(videoHttp.tags).to.deep.equal([ 'tag1', 'tag2' ])
  33   expect(videoHttp.files).to.have.lengthOf(1)
  34
  35   const originallyPublishedAt = new Date(videoHttp.originallyPublishedAt)
  36   expect(originallyPublishedAt.getDate()).to.equal(14)
  37   expect(originallyPublishedAt.getMonth()).to.equal(0)
  38   expect(originallyPublishedAt.getFullYear()).to.equal(2019)
  39
  40   const videoMagnet = await server.videos.get({ id: idMagnet })
  41   const videoTorrent = await server.videos.get({ id: idTorrent })
  42
  43   for (const video of [ videoMagnet, videoTorrent ]) {
  44     expect(video.category.label).to.equal('Unknown')
  45     expect(video.licence.label).to.equal('Unknown')
  46     expect(video.language.label).to.equal('Unknown')
  47     expect(video.nsfw).to.be.false
  48     expect(video.description).to.equal('this is a super torrent description')
  49     expect(video.tags).to.deep.equal([ 'tag_torrent1', 'tag_torrent2' ])
  50     expect(video.files).to.have.lengthOf(1)
  51   }
  52
  53   expect(videoTorrent.name).to.contain('你好 世界 720p.mp4')
  54   expect(videoMagnet.name).to.contain('super peertube2 video')
  55
  56   const bodyCaptions = await server.captions.list({ videoId: idHttp })
  57   expect(bodyCaptions.total).to.equal(2)
  58 }
  59
  60 async function checkVideoServer2 (server: PeerTubeServer, id: number | string) {
  61   const video = await server.videos.get({ id })
  62
  63   expect(video.name).to.equal('my super name')
  64   expect(video.category.label).to.equal('Entertainment')
  65   expect(video.licence.label).to.equal('Public Domain Dedication')
  66   expect(video.language.label).to.equal('English')
  67   expect(video.nsfw).to.be.false
  68   expect(video.description).to.equal('my super description')
  69   expect(video.tags).to.deep.equal([ 'supertag1', 'supertag2' ])
  70
  71   await testImage(server.url, 'thumbnail', video.thumbnailPath)
  72
  73   expect(video.files).to.have.lengthOf(1)
  74
  75   const bodyCaptions = await server.captions.list({ videoId: id })
  76   expect(bodyCaptions.total).to.equal(2)
  77 }
  78
  79 describe('Test video imports', function () {
  80
  81   if (areHttpImportTestsDisabled()) return
  82
  83   function runSuite (mode: 'youtube-dl' | 'yt-dlp') {
  84
  85     describe('Import ' + mode, function () {
  86       let servers: PeerTubeServer[] = []
  87
  88       before(async function () {
  89         this.timeout(60_000)
  90
  91         servers = await createMultipleServers(2, getServerImportConfig(mode))
  92
  93         await setAccessTokensToServers(servers)
  94         await setDefaultVideoChannel(servers)
  95
  96         for (const server of servers) {
  97           await server.config.updateExistingSubConfig({
  98             newConfig: {
  99               transcoding: {
 100                 alwaysTranscodeOriginalResolution: false
 101               }
 102             }
 103           })
 104         }
 105
 106         await doubleFollow(servers[0], servers[1])
 107       })
 108
 109       it('Should import videos on server 1', async function () {
 110         this.timeout(60_000)
 111
 112         const baseAttributes = {
 113           channelId: servers[0].store.channel.id,
 114           privacy: VideoPrivacy.PUBLIC
 115         }
 116
 117         {
 118           const attributes = { ...baseAttributes, targetUrl: FIXTURE_URLS.youtube }
 119           const { video } = await servers[0].imports.importVideo({ attributes })
 120           expect(video.name).to.equal('small video - youtube')
 121
 122           {
 123             expect(video.thumbnailPath).to.match(new RegExp(`^/static/thumbnails/.+.jpg$`))
 124             expect(video.previewPath).to.match(new RegExp(`^/lazy-static/previews/.+.jpg$`))
 125
 126             const suffix = mode === 'yt-dlp'
 127               ? '_yt_dlp'
 128               : ''
 129
 130             await testImage(servers[0].url, 'video_import_thumbnail' + suffix, video.thumbnailPath)
 131             await testImage(servers[0].url, 'video_import_preview' + suffix, video.previewPath)
 132           }
 133
 134           const bodyCaptions = await servers[0].captions.list({ videoId: video.id })
 135           const videoCaptions = bodyCaptions.data
 136           expect(videoCaptions).to.have.lengthOf(2)
 137
 138           {
 139             const enCaption = videoCaptions.find(caption => caption.language.id === 'en')
 140             expect(enCaption).to.exist
 141             expect(enCaption.language.label).to.equal('English')
 142             expect(enCaption.captionPath).to.match(new RegExp(`^/lazy-static/video-captions/.+-en.vtt$`))
 143
 144             const regex = `WEBVTT[ \n]+Kind: captions[ \n]+` +
 145               `(Language: en[ \n]+)?` +
 146               `00:00:01.600 --> 00:00:04.200( position:\\d+% line:\\d+%)?[ \n]+English \\(US\\)[ \n]+` +
 147               `00:00:05.900 --> 00:00:07.999( position:\\d+% line:\\d+%)?[ \n]+This is a subtitle in American English[ \n]+` +
 148               `00:00:10.000 --> 00:00:14.000( position:\\d+% line:\\d+%)?[ \n]+Adding subtitles is very easy to do`
 149             await testCaptionFile(servers[0].url, enCaption.captionPath, new RegExp(regex))
 150           }
 151
 152           {
 153             const frCaption = videoCaptions.find(caption => caption.language.id === 'fr')
 154             expect(frCaption).to.exist
 155             expect(frCaption.language.label).to.equal('French')
 156             expect(frCaption.captionPath).to.match(new RegExp(`^/lazy-static/video-captions/.+-fr.vtt`))
 157
 158             const regex = `WEBVTT[ \n]+Kind: captions[ \n]+` +
 159               `(Language: fr[ \n]+)?` +
 160               `00:00:01.600 --> 00:00:04.200( position:\\d+% line:\\d+%)?[ \n]+Français \\(FR\\)[ \n]+` +
 161               `00:00:05.900 --> 00:00:07.999( position:\\d+% line:\\d+%)?[ \n]+C'est un sous-titre français[ \n]+` +
 162               `00:00:10.000 --> 00:00:14.000( position:\\d+% line:\\d+%)?[ \n]+Ajouter un sous-titre est vraiment facile`
 163
 164             await testCaptionFile(servers[0].url, frCaption.captionPath, new RegExp(regex))
 165           }
 166         }
 167
 168         {
 169           const attributes = {
 170             ...baseAttributes,
 171             magnetUri: FIXTURE_URLS.magnet,
 172             description: 'this is a super torrent description',
 173             tags: [ 'tag_torrent1', 'tag_torrent2' ]
 174           }
 175           const { video } = await servers[0].imports.importVideo({ attributes })
 176           expect(video.name).to.equal('super peertube2 video')
 177         }
 178
 179         {
 180           const attributes = {
 181             ...baseAttributes,
 182             torrentfile: 'video-720p.torrent' as any,
 183             description: 'this is a super torrent description',
 184             tags: [ 'tag_torrent1', 'tag_torrent2' ]
 185           }
 186           const { video } = await servers[0].imports.importVideo({ attributes })
 187           expect(video.name).to.equal('你好 世界 720p.mp4')
 188         }
 189       })
 190
 191       it('Should list the videos to import in my videos on server 1', async function () {
 192         const { total, data } = await servers[0].videos.listMyVideos({ sort: 'createdAt' })
 193
 194         expect(total).to.equal(3)
 195
 196         expect(data).to.have.lengthOf(3)
 197         expect(data[0].name).to.equal('small video - youtube')
 198         expect(data[1].name).to.equal('super peertube2 video')
 199         expect(data[2].name).to.equal('你好 世界 720p.mp4')
 200       })
 201
 202       it('Should list the videos to import in my imports on server 1', async function () {
 203         const { total, data: videoImports } = await servers[0].imports.getMyVideoImports({ sort: '-createdAt' })
 204         expect(total).to.equal(3)
 205
 206         expect(videoImports).to.have.lengthOf(3)
 207
 208         expect(videoImports[2].targetUrl).to.equal(FIXTURE_URLS.youtube)
 209         expect(videoImports[2].magnetUri).to.be.null
 210         expect(videoImports[2].torrentName).to.be.null
 211         expect(videoImports[2].video.name).to.equal('small video - youtube')
 212
 213         expect(videoImports[1].targetUrl).to.be.null
 214         expect(videoImports[1].magnetUri).to.equal(FIXTURE_URLS.magnet)
 215         expect(videoImports[1].torrentName).to.be.null
 216         expect(videoImports[1].video.name).to.equal('super peertube2 video')
 217
 218         expect(videoImports[0].targetUrl).to.be.null
 219         expect(videoImports[0].magnetUri).to.be.null
 220         expect(videoImports[0].torrentName).to.equal('video-720p.torrent')
 221         expect(videoImports[0].video.name).to.equal('你好 世界 720p.mp4')
 222       })
 223
 224       it('Should filter my imports on target URL', async function () {
 225         const { total, data: videoImports } = await servers[0].imports.getMyVideoImports({ targetUrl: FIXTURE_URLS.youtube })
 226         expect(total).to.equal(1)
 227         expect(videoImports).to.have.lengthOf(1)
 228
 229         expect(videoImports[0].targetUrl).to.equal(FIXTURE_URLS.youtube)
 230       })
 231
 232       it('Should search in my imports', async function () {
 233         const { total, data: videoImports } = await servers[0].imports.getMyVideoImports({ search: 'peertube2' })
 234         expect(total).to.equal(1)
 235         expect(videoImports).to.have.lengthOf(1)
 236
 237         expect(videoImports[0].magnetUri).to.equal(FIXTURE_URLS.magnet)
 238         expect(videoImports[0].video.name).to.equal('super peertube2 video')
 239       })
 240
 241       it('Should have the video listed on the two instances', async function () {
 242         this.timeout(120_000)
 243
 244         await waitJobs(servers)
 245
 246         for (const server of servers) {
 247           const { total, data } = await server.videos.list()
 248           expect(total).to.equal(3)
 249           expect(data).to.have.lengthOf(3)
 250
 251           const [ videoHttp, videoMagnet, videoTorrent ] = data
 252           await checkVideosServer1(server, videoHttp.uuid, videoMagnet.uuid, videoTorrent.uuid)
 253         }
 254       })
 255
 256       it('Should import a video on server 2 with some fields', async function () {
 257         this.timeout(60_000)
 258
 259         const { video } = await servers[1].imports.importVideo({
 260           attributes: {
 261             targetUrl: FIXTURE_URLS.youtube,
 262             channelId: servers[1].store.channel.id,
 263             privacy: VideoPrivacy.PUBLIC,
 264             category: 10,
 265             licence: 7,
 266             language: 'en',
 267             name: 'my super name',
 268             description: 'my super description',
 269             tags: [ 'supertag1', 'supertag2' ],
 270             thumbnailfile: 'thumbnail.jpg'
 271           }
 272         })
 273         expect(video.name).to.equal('my super name')
 274       })
 275
 276       it('Should have the videos listed on the two instances', async function () {
 277         this.timeout(120_000)
 278
 279         await waitJobs(servers)
 280
 281         for (const server of servers) {
 282           const { total, data } = await server.videos.list()
 283           expect(total).to.equal(4)
 284           expect(data).to.have.lengthOf(4)
 285
 286           await checkVideoServer2(server, data[0].uuid)
 287
 288           const [ , videoHttp, videoMagnet, videoTorrent ] = data
 289           await checkVideosServer1(server, videoHttp.uuid, videoMagnet.uuid, videoTorrent.uuid)
 290         }
 291       })
 292
 293       it('Should import a video that will be transcoded', async function () {
 294         this.timeout(240_000)
 295
 296         const attributes = {
 297           name: 'transcoded video',
 298           magnetUri: FIXTURE_URLS.magnet,
 299           channelId: servers[1].store.channel.id,
 300           privacy: VideoPrivacy.PUBLIC
 301         }
 302         const { video } = await servers[1].imports.importVideo({ attributes })
 303         const videoUUID = video.uuid
 304
 305         await waitJobs(servers)
 306
 307         for (const server of servers) {
 308           const video = await server.videos.get({ id: videoUUID })
 309
 310           expect(video.name).to.equal('transcoded video')
 311           expect(video.files).to.have.lengthOf(4)
 312         }
 313       })
 314
 315       it('Should import no HDR version on a HDR video', async function () {
 316         this.timeout(300_000)
 317
 318         const config: DeepPartial<CustomConfig> = {
 319           transcoding: {
 320             enabled: true,
 321             resolutions: {
 322               '0p': false,
 323               '144p': true,
 324               '240p': true,
 325               '360p': false,
 326               '480p': false,
 327               '720p': false,
 328               '1080p': false, // the resulting resolution shouldn't be higher than this, and not vp9.2/av01
 329               '1440p': false,
 330               '2160p': false
 331             },
 332             webtorrent: { enabled: true },
 333             hls: { enabled: false }
 334           }
 335         }
 336         await servers[0].config.updateExistingSubConfig({ newConfig: config })
 337
 338         const attributes = {
 339           name: 'hdr video',
 340           targetUrl: FIXTURE_URLS.youtubeHDR,
 341           channelId: servers[0].store.channel.id,
 342           privacy: VideoPrivacy.PUBLIC
 343         }
 344         const { video: videoImported } = await servers[0].imports.importVideo({ attributes })
 345         const videoUUID = videoImported.uuid
 346
 347         await waitJobs(servers)
 348
 349         // test resolution
 350         const video = await servers[0].videos.get({ id: videoUUID })
 351         expect(video.name).to.equal('hdr video')
 352         const maxResolution = Math.max.apply(Math, video.files.map(function (o) { return o.resolution.id }))
 353         expect(maxResolution, 'expected max resolution not met').to.equals(VideoResolution.H_240P)
 354       })
 355
 356       it('Should not import resolution higher than enabled transcoding resolution', async function () {
 357         this.timeout(300_000)
 358
 359         const config: DeepPartial<CustomConfig> = {
 360           transcoding: {
 361             enabled: true,
 362             resolutions: {
 363               '0p': false,
 364               '144p': true,
 365               '240p': false,
 366               '360p': false,
 367               '480p': false,
 368               '720p': false,
 369               '1080p': false,
 370               '1440p': false,
 371               '2160p': false
 372             },
 373             alwaysTranscodeOriginalResolution: false
 374           }
 375         }
 376         await servers[0].config.updateExistingSubConfig({ newConfig: config })
 377
 378         const attributes = {
 379           name: 'small resolution video',
 380           targetUrl: FIXTURE_URLS.youtube,
 381           channelId: servers[0].store.channel.id,
 382           privacy: VideoPrivacy.PUBLIC
 383         }
 384         const { video: videoImported } = await servers[0].imports.importVideo({ attributes })
 385         const videoUUID = videoImported.uuid
 386
 387         await waitJobs(servers)
 388
 389         // test resolution
 390         const video = await servers[0].videos.get({ id: videoUUID })
 391         expect(video.name).to.equal('small resolution video')
 392         expect(video.files).to.have.lengthOf(1)
 393         expect(video.files[0].resolution.id).to.equal(144)
 394       })
 395
 396       it('Should import resolution higher than enabled transcoding resolution', async function () {
 397         this.timeout(300_000)
 398
 399         const config: DeepPartial<CustomConfig> = {
 400           transcoding: {
 401             alwaysTranscodeOriginalResolution: true
 402           }
 403         }
 404         await servers[0].config.updateExistingSubConfig({ newConfig: config })
 405
 406         const attributes = {
 407           name: 'bigger resolution video',
 408           targetUrl: FIXTURE_URLS.youtube,
 409           channelId: servers[0].store.channel.id,
 410           privacy: VideoPrivacy.PUBLIC
 411         }
 412         const { video: videoImported } = await servers[0].imports.importVideo({ attributes })
 413         const videoUUID = videoImported.uuid
 414
 415         await waitJobs(servers)
 416
 417         // test resolution
 418         const video = await servers[0].videos.get({ id: videoUUID })
 419         expect(video.name).to.equal('bigger resolution video')
 420
 421         expect(video.files).to.have.lengthOf(2)
 422         expect(video.files.find(f => f.resolution.id === 240)).to.exist
 423         expect(video.files.find(f => f.resolution.id === 144)).to.exist
 424       })
 425
 426       it('Should import a peertube video', async function () {
 427         this.timeout(120_000)
 428
 429         const toTest = [ FIXTURE_URLS.peertube_long ]
 430
 431         // TODO: include peertube_short when https://github.com/ytdl-org/youtube-dl/pull/29475 is merged
 432         if (mode === 'yt-dlp') {
 433           toTest.push(FIXTURE_URLS.peertube_short)
 434         }
 435
 436         for (const targetUrl of toTest) {
 437           await servers[0].config.disableTranscoding()
 438
 439           const attributes = {
 440             targetUrl,
 441             channelId: servers[0].store.channel.id,
 442             privacy: VideoPrivacy.PUBLIC
 443           }
 444           const { video } = await servers[0].imports.importVideo({ attributes })
 445           const videoUUID = video.uuid
 446
 447           await waitJobs(servers)
 448
 449           for (const server of servers) {
 450             const video = await server.videos.get({ id: videoUUID })
 451
 452             expect(video.name).to.equal('E2E tests')
 453           }
 454         }
 455       })
 456
 457       after(async function () {
 458         await cleanupTests(servers)
 459       })
 460     })
 461   }
 462
 463   runSuite('youtube-dl')
 464
 465   runSuite('yt-dlp')
 466
 467   describe('Delete/cancel an import', function () {
 468     let server: PeerTubeServer
 469
 470     let finishedImportId: number
 471     let finishedVideo: Video
 472     let pendingImportId: number
 473
 474     async function importVideo (name: string) {
 475       const attributes = { name, channelId: server.store.channel.id, targetUrl: FIXTURE_URLS.goodVideo }
 476       const res = await server.imports.importVideo({ attributes })
 477
 478       return res.id
 479     }
 480
 481     before(async function () {
 482       this.timeout(120_000)
 483
 484       server = await createSingleServer(1)
 485
 486       await setAccessTokensToServers([ server ])
 487       await setDefaultVideoChannel([ server ])
 488
 489       finishedImportId = await importVideo('finished')
 490       await waitJobs([ server ])
 491
 492       await server.jobs.pauseJobQueue()
 493       pendingImportId = await importVideo('pending')
 494
 495       const { data } = await server.imports.getMyVideoImports()
 496       expect(data).to.have.lengthOf(2)
 497
 498       finishedVideo = data.find(i => i.id === finishedImportId).video
 499     })
 500
 501     it('Should delete a video import', async function () {
 502       await server.imports.delete({ importId: finishedImportId })
 503
 504       const { data } = await server.imports.getMyVideoImports()
 505       expect(data).to.have.lengthOf(1)
 506       expect(data[0].id).to.equal(pendingImportId)
 507       expect(data[0].state.id).to.equal(VideoImportState.PENDING)
 508     })
 509
 510     it('Should not have deleted the associated video', async function () {
 511       const video = await server.videos.get({ id: finishedVideo.id, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 })
 512       expect(video.name).to.equal('finished')
 513       expect(video.state.id).to.equal(VideoState.PUBLISHED)
 514     })
 515
 516     it('Should cancel a video import', async function () {
 517       await server.imports.cancel({ importId: pendingImportId })
 518
 519       const { data } = await server.imports.getMyVideoImports()
 520       expect(data).to.have.lengthOf(1)
 521       expect(data[0].id).to.equal(pendingImportId)
 522       expect(data[0].state.id).to.equal(VideoImportState.CANCELLED)
 523     })
 524
 525     it('Should not have processed the cancelled video import', async function () {
 526       this.timeout(60_000)
 527
 528       await server.jobs.resumeJobQueue()
 529
 530       await waitJobs([ server ])
 531
 532       const { data } = await server.imports.getMyVideoImports()
 533       expect(data).to.have.lengthOf(1)
 534       expect(data[0].id).to.equal(pendingImportId)
 535       expect(data[0].state.id).to.equal(VideoImportState.CANCELLED)
 536       expect(data[0].video.state.id).to.equal(VideoState.TO_IMPORT)
 537     })
 538
 539     it('Should delete the cancelled video import', async function () {
 540       await server.imports.delete({ importId: pendingImportId })
 541       const { data } = await server.imports.getMyVideoImports()
 542       expect(data).to.have.lengthOf(0)
 543     })
 544
 545     after(async function () {
 546       await cleanupTests([ server ])
 547     })
 548   })
 549
 550   describe('Auto update', function () {
 551     let server: PeerTubeServer
 552
 553     function quickPeerTubeImport () {
 554       const attributes = {
 555         targetUrl: FIXTURE_URLS.peertube_long,
 556         channelId: server.store.channel.id,
 557         privacy: VideoPrivacy.PUBLIC
 558       }
 559
 560       return server.imports.importVideo({ attributes })
 561     }
 562
 563     async function testBinaryUpdate (releaseUrl: string, releaseName: string) {
 564       await remove(join(server.servers.buildDirectory('bin'), releaseName))
 565
 566       await server.kill()
 567       await server.run({
 568         import: {
 569           videos: {
 570             http: {
 571               youtube_dl_release: {
 572                 url: releaseUrl,
 573                 name: releaseName
 574               }
 575             }
 576           }
 577         }
 578       })
 579
 580       await quickPeerTubeImport()
 581
 582       const base = server.servers.buildDirectory('bin')
 583       const content = await readdir(base)
 584       const binaryPath = join(base, releaseName)
 585
 586       expect(await pathExists(binaryPath), `${binaryPath} does not exist in ${base} (${content.join(', ')})`).to.be.true
 587     }
 588
 589     before(async function () {
 590       this.timeout(30_000)
 591
 592       // Run servers
 593       server = await createSingleServer(1)
 594
 595       await setAccessTokensToServers([ server ])
 596       await setDefaultVideoChannel([ server ])
 597     })
 598
 599     it('Should update youtube-dl from github URL', async function () {
 600       this.timeout(120_000)
 601
 602       await testBinaryUpdate('https://api.github.com/repos/ytdl-org/youtube-dl/releases', 'youtube-dl')
 603     })
 604
 605     it('Should update youtube-dl from raw URL', async function () {
 606       this.timeout(120_000)
 607
 608       await testBinaryUpdate('https://yt-dl.org/downloads/latest/youtube-dl', 'youtube-dl')
 609     })
 610
 611     it('Should update youtube-dl from youtube-dl fork', async function () {
 612       this.timeout(120_000)
 613
 614       await testBinaryUpdate('https://api.github.com/repos/yt-dlp/yt-dlp/releases', 'yt-dlp')
 615     })
 616
 617     after(async function () {
 618       await cleanupTests([ server ])
 619     })
 620   })
 621 })