server/tests/api/videos/video-imports.ts

   1 /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
   2
   3 import 'mocha'
   4 import { expect } from 'chai'
   5 import { pathExists, remove } from 'fs-extra'
   6 import { join } from 'path'
   7 import {
   8   areHttpImportTestsDisabled,
   9   cleanupTests,
  10   createMultipleServers,
  11   createSingleServer,
  12   doubleFollow,
  13   FIXTURE_URLS,
  14   PeerTubeServer,
  15   setAccessTokensToServers,
  16   setDefaultVideoChannel,
  17   testCaptionFile,
  18   testImage,
  19   waitJobs
  20 } from '@shared/extra-utils'
  21 import { VideoPrivacy, VideoResolution } from '@shared/models'
  22
  23 async function checkVideosServer1 (server: PeerTubeServer, idHttp: string, idMagnet: string, idTorrent: string) {
  24   const videoHttp = await server.videos.get({ id: idHttp })
  25
  26   expect(videoHttp.name).to.equal('small video - youtube')
  27   // FIXME: youtube-dl seems broken
  28   // expect(videoHttp.category.label).to.equal('News & Politics')
  29   // expect(videoHttp.licence.label).to.equal('Attribution')
  30   expect(videoHttp.language.label).to.equal('Unknown')
  31   expect(videoHttp.nsfw).to.be.false
  32   expect(videoHttp.description).to.equal('this is a super description')
  33   expect(videoHttp.tags).to.deep.equal([ 'tag1', 'tag2' ])
  34   expect(videoHttp.files).to.have.lengthOf(1)
  35
  36   const originallyPublishedAt = new Date(videoHttp.originallyPublishedAt)
  37   expect(originallyPublishedAt.getDate()).to.equal(14)
  38   expect(originallyPublishedAt.getMonth()).to.equal(0)
  39   expect(originallyPublishedAt.getFullYear()).to.equal(2019)
  40
  41   const videoMagnet = await server.videos.get({ id: idMagnet })
  42   const videoTorrent = await server.videos.get({ id: idTorrent })
  43
  44   for (const video of [ videoMagnet, videoTorrent ]) {
  45     expect(video.category.label).to.equal('Misc')
  46     expect(video.licence.label).to.equal('Unknown')
  47     expect(video.language.label).to.equal('Unknown')
  48     expect(video.nsfw).to.be.false
  49     expect(video.description).to.equal('this is a super torrent description')
  50     expect(video.tags).to.deep.equal([ 'tag_torrent1', 'tag_torrent2' ])
  51     expect(video.files).to.have.lengthOf(1)
  52   }
  53
  54   expect(videoTorrent.name).to.contain('你好 世界 720p.mp4')
  55   expect(videoMagnet.name).to.contain('super peertube2 video')
  56
  57   const bodyCaptions = await server.captions.list({ videoId: idHttp })
  58   expect(bodyCaptions.total).to.equal(2)
  59 }
  60
  61 async function checkVideoServer2 (server: PeerTubeServer, id: number | string) {
  62   const video = await server.videos.get({ id })
  63
  64   expect(video.name).to.equal('my super name')
  65   expect(video.category.label).to.equal('Entertainment')
  66   expect(video.licence.label).to.equal('Public Domain Dedication')
  67   expect(video.language.label).to.equal('English')
  68   expect(video.nsfw).to.be.false
  69   expect(video.description).to.equal('my super description')
  70   expect(video.tags).to.deep.equal([ 'supertag1', 'supertag2' ])
  71
  72   expect(video.files).to.have.lengthOf(1)
  73
  74   const bodyCaptions = await server.captions.list({ videoId: id })
  75   expect(bodyCaptions.total).to.equal(2)
  76 }
  77
  78 describe('Test video imports', function () {
  79
  80   if (areHttpImportTestsDisabled()) return
  81
  82   function runSuite (mode: 'youtube-dl' | 'yt-dlp') {
  83
  84     describe('Import ' + mode, function () {
  85       let servers: PeerTubeServer[] = []
  86
  87       before(async function () {
  88         this.timeout(30_000)
  89
  90         // Run servers
  91         servers = await createMultipleServers(2, {
  92           import: {
  93             videos: {
  94               http: {
  95                 youtube_dl_release: {
  96                   url: mode === 'youtube-dl'
  97                     ? 'https://yt-dl.org/downloads/latest/youtube-dl'
  98                     : 'https://api.github.com/repos/yt-dlp/yt-dlp/releases',
  99
 100                   name: mode
 101                 }
 102               }
 103             }
 104           }
 105         })
 106
 107         await setAccessTokensToServers(servers)
 108         await setDefaultVideoChannel(servers)
 109
 110         await doubleFollow(servers[0], servers[1])
 111       })
 112
 113       it('Should import videos on server 1', async function () {
 114         this.timeout(60_000)
 115
 116         const baseAttributes = {
 117           channelId: servers[0].store.channel.id,
 118           privacy: VideoPrivacy.PUBLIC
 119         }
 120
 121         {
 122           const attributes = { ...baseAttributes, targetUrl: FIXTURE_URLS.youtube }
 123           const { video } = await servers[0].imports.importVideo({ attributes })
 124           expect(video.name).to.equal('small video - youtube')
 125
 126           {
 127             expect(video.thumbnailPath).to.match(new RegExp(`^/static/thumbnails/.+.jpg$`))
 128             expect(video.previewPath).to.match(new RegExp(`^/lazy-static/previews/.+.jpg$`))
 129
 130             const suffix = mode === 'yt-dlp'
 131               ? '_yt_dlp'
 132               : ''
 133
 134             await testImage(servers[0].url, 'video_import_thumbnail' + suffix, video.thumbnailPath)
 135             await testImage(servers[0].url, 'video_import_preview' + suffix, video.previewPath)
 136           }
 137
 138           const bodyCaptions = await servers[0].captions.list({ videoId: video.id })
 139           const videoCaptions = bodyCaptions.data
 140           expect(videoCaptions).to.have.lengthOf(2)
 141
 142           {
 143             const enCaption = videoCaptions.find(caption => caption.language.id === 'en')
 144             expect(enCaption).to.exist
 145             expect(enCaption.language.label).to.equal('English')
 146             expect(enCaption.captionPath).to.match(new RegExp(`^/lazy-static/video-captions/.+-en.vtt$`))
 147
 148             const regex = `WEBVTT[ \n]+Kind: captions[ \n]+Language: en[ \n]+00:00:01.600 --> 00:00:04.200[ \n]+English \\(US\\)[ \n]+` +
 149               `00:00:05.900 --> 00:00:07.999[ \n]+This is a subtitle in American English[ \n]+` +
 150               `00:00:10.000 --> 00:00:14.000[ \n]+Adding subtitles is very easy to do`
 151             await testCaptionFile(servers[0].url, enCaption.captionPath, new RegExp(regex))
 152           }
 153
 154           {
 155             const frCaption = videoCaptions.find(caption => caption.language.id === 'fr')
 156             expect(frCaption).to.exist
 157             expect(frCaption.language.label).to.equal('French')
 158             expect(frCaption.captionPath).to.match(new RegExp(`^/lazy-static/video-captions/.+-fr.vtt`))
 159
 160             const regex = `WEBVTT[ \n]+Kind: captions[ \n]+Language: fr[ \n]+00:00:01.600 --> 00:00:04.200[ \n]+` +
 161               `Français \\(FR\\)[ \n]+00:00:05.900 --> 00:00:07.999[ \n]+C'est un sous-titre français[ \n]+` +
 162               `00:00:10.000 --> 00:00:14.000[ \n]+Ajouter un sous-titre est vraiment facile`
 163
 164             await testCaptionFile(servers[0].url, frCaption.captionPath, new RegExp(regex))
 165           }
 166         }
 167
 168         {
 169           const attributes = {
 170             ...baseAttributes,
 171             magnetUri: FIXTURE_URLS.magnet,
 172             description: 'this is a super torrent description',
 173             tags: [ 'tag_torrent1', 'tag_torrent2' ]
 174           }
 175           const { video } = await servers[0].imports.importVideo({ attributes })
 176           expect(video.name).to.equal('super peertube2 video')
 177         }
 178
 179         {
 180           const attributes = {
 181             ...baseAttributes,
 182             torrentfile: 'video-720p.torrent' as any,
 183             description: 'this is a super torrent description',
 184             tags: [ 'tag_torrent1', 'tag_torrent2' ]
 185           }
 186           const { video } = await servers[0].imports.importVideo({ attributes })
 187           expect(video.name).to.equal('你好 世界 720p.mp4')
 188         }
 189       })
 190
 191       it('Should list the videos to import in my videos on server 1', async function () {
 192         const { total, data } = await servers[0].videos.listMyVideos({ sort: 'createdAt' })
 193
 194         expect(total).to.equal(3)
 195
 196         expect(data).to.have.lengthOf(3)
 197         expect(data[0].name).to.equal('small video - youtube')
 198         expect(data[1].name).to.equal('super peertube2 video')
 199         expect(data[2].name).to.equal('你好 世界 720p.mp4')
 200       })
 201
 202       it('Should list the videos to import in my imports on server 1', async function () {
 203         const { total, data: videoImports } = await servers[0].imports.getMyVideoImports({ sort: '-createdAt' })
 204         expect(total).to.equal(3)
 205
 206         expect(videoImports).to.have.lengthOf(3)
 207
 208         expect(videoImports[2].targetUrl).to.equal(FIXTURE_URLS.youtube)
 209         expect(videoImports[2].magnetUri).to.be.null
 210         expect(videoImports[2].torrentName).to.be.null
 211         expect(videoImports[2].video.name).to.equal('small video - youtube')
 212
 213         expect(videoImports[1].targetUrl).to.be.null
 214         expect(videoImports[1].magnetUri).to.equal(FIXTURE_URLS.magnet)
 215         expect(videoImports[1].torrentName).to.be.null
 216         expect(videoImports[1].video.name).to.equal('super peertube2 video')
 217
 218         expect(videoImports[0].targetUrl).to.be.null
 219         expect(videoImports[0].magnetUri).to.be.null
 220         expect(videoImports[0].torrentName).to.equal('video-720p.torrent')
 221         expect(videoImports[0].video.name).to.equal('你好 世界 720p.mp4')
 222       })
 223
 224       it('Should have the video listed on the two instances', async function () {
 225         this.timeout(120_000)
 226
 227         await waitJobs(servers)
 228
 229         for (const server of servers) {
 230           const { total, data } = await server.videos.list()
 231           expect(total).to.equal(3)
 232           expect(data).to.have.lengthOf(3)
 233
 234           const [ videoHttp, videoMagnet, videoTorrent ] = data
 235           await checkVideosServer1(server, videoHttp.uuid, videoMagnet.uuid, videoTorrent.uuid)
 236         }
 237       })
 238
 239       it('Should import a video on server 2 with some fields', async function () {
 240         this.timeout(60_000)
 241
 242         const attributes = {
 243           targetUrl: FIXTURE_URLS.youtube,
 244           channelId: servers[1].store.channel.id,
 245           privacy: VideoPrivacy.PUBLIC,
 246           category: 10,
 247           licence: 7,
 248           language: 'en',
 249           name: 'my super name',
 250           description: 'my super description',
 251           tags: [ 'supertag1', 'supertag2' ]
 252         }
 253         const { video } = await servers[1].imports.importVideo({ attributes })
 254         expect(video.name).to.equal('my super name')
 255       })
 256
 257       it('Should have the videos listed on the two instances', async function () {
 258         this.timeout(120_000)
 259
 260         await waitJobs(servers)
 261
 262         for (const server of servers) {
 263           const { total, data } = await server.videos.list()
 264           expect(total).to.equal(4)
 265           expect(data).to.have.lengthOf(4)
 266
 267           await checkVideoServer2(server, data[0].uuid)
 268
 269           const [ , videoHttp, videoMagnet, videoTorrent ] = data
 270           await checkVideosServer1(server, videoHttp.uuid, videoMagnet.uuid, videoTorrent.uuid)
 271         }
 272       })
 273
 274       it('Should import a video that will be transcoded', async function () {
 275         this.timeout(240_000)
 276
 277         const attributes = {
 278           name: 'transcoded video',
 279           magnetUri: FIXTURE_URLS.magnet,
 280           channelId: servers[1].store.channel.id,
 281           privacy: VideoPrivacy.PUBLIC
 282         }
 283         const { video } = await servers[1].imports.importVideo({ attributes })
 284         const videoUUID = video.uuid
 285
 286         await waitJobs(servers)
 287
 288         for (const server of servers) {
 289           const video = await server.videos.get({ id: videoUUID })
 290
 291           expect(video.name).to.equal('transcoded video')
 292           expect(video.files).to.have.lengthOf(4)
 293         }
 294       })
 295
 296       it('Should import no HDR version on a HDR video', async function () {
 297         this.timeout(300_000)
 298
 299         const config = {
 300           transcoding: {
 301             enabled: true,
 302             resolutions: {
 303               '240p': true,
 304               '360p': false,
 305               '480p': false,
 306               '720p': false,
 307               '1080p': false, // the resulting resolution shouldn't be higher than this, and not vp9.2/av01
 308               '1440p': false,
 309               '2160p': false
 310             },
 311             webtorrent: { enabled: true },
 312             hls: { enabled: false }
 313           },
 314           import: {
 315             videos: {
 316               http: {
 317                 enabled: true
 318               },
 319               torrent: {
 320                 enabled: true
 321               }
 322             }
 323           }
 324         }
 325         await servers[0].config.updateCustomSubConfig({ newConfig: config })
 326
 327         const attributes = {
 328           name: 'hdr video',
 329           targetUrl: FIXTURE_URLS.youtubeHDR,
 330           channelId: servers[0].store.channel.id,
 331           privacy: VideoPrivacy.PUBLIC
 332         }
 333         const { video: videoImported } = await servers[0].imports.importVideo({ attributes })
 334         const videoUUID = videoImported.uuid
 335
 336         await waitJobs(servers)
 337
 338         // test resolution
 339         const video = await servers[0].videos.get({ id: videoUUID })
 340         expect(video.name).to.equal('hdr video')
 341         const maxResolution = Math.max.apply(Math, video.files.map(function (o) { return o.resolution.id }))
 342         expect(maxResolution, 'expected max resolution not met').to.equals(VideoResolution.H_240P)
 343       })
 344
 345       it('Should import a peertube video', async function () {
 346         this.timeout(120_000)
 347
 348         const toTest = [ FIXTURE_URLS.peertube_long ]
 349
 350         // TODO: include peertube_short when https://github.com/ytdl-org/youtube-dl/pull/29475 is merged
 351         if (mode === 'yt-dlp') {
 352           toTest.push(FIXTURE_URLS.peertube_short)
 353         }
 354
 355         for (const targetUrl of toTest) {
 356           await servers[0].config.disableTranscoding()
 357
 358           const attributes = {
 359             targetUrl,
 360             channelId: servers[0].store.channel.id,
 361             privacy: VideoPrivacy.PUBLIC
 362           }
 363           const { video } = await servers[0].imports.importVideo({ attributes })
 364           const videoUUID = video.uuid
 365
 366           await waitJobs(servers)
 367
 368           for (const server of servers) {
 369             const video = await server.videos.get({ id: videoUUID })
 370
 371             expect(video.name).to.equal('E2E tests')
 372           }
 373         }
 374       })
 375
 376       after(async function () {
 377         await cleanupTests(servers)
 378       })
 379     })
 380   }
 381
 382   runSuite('youtube-dl')
 383
 384   runSuite('yt-dlp')
 385
 386   describe('Auto update', function () {
 387     let server: PeerTubeServer
 388
 389     function quickPeerTubeImport () {
 390       const attributes = {
 391         targetUrl: FIXTURE_URLS.peertube_long,
 392         channelId: server.store.channel.id,
 393         privacy: VideoPrivacy.PUBLIC
 394       }
 395
 396       return server.imports.importVideo({ attributes })
 397     }
 398
 399     async function testBinaryUpdate (releaseUrl: string, releaseName: string) {
 400       await remove(join(server.servers.buildDirectory('bin'), releaseName))
 401
 402       await server.kill()
 403       await server.run({
 404         import: {
 405           videos: {
 406             http: {
 407               youtube_dl_release: {
 408                 url: releaseUrl,
 409                 name: releaseName
 410               }
 411             }
 412           }
 413         }
 414       })
 415
 416       await quickPeerTubeImport()
 417
 418       expect(await pathExists(join(server.servers.buildDirectory('bin'), releaseName))).to.be.true
 419     }
 420
 421     before(async function () {
 422       this.timeout(30_000)
 423
 424       // Run servers
 425       server = await createSingleServer(1)
 426
 427       await setAccessTokensToServers([ server ])
 428       await setDefaultVideoChannel([ server ])
 429     })
 430
 431     it('Should update youtube-dl from github URL', async function () {
 432       this.timeout(120_000)
 433
 434       await testBinaryUpdate('https://api.github.com/repos/ytdl-org/youtube-dl/releases', 'youtube-dl')
 435     })
 436
 437     it('Should update youtube-dl from raw URL', async function () {
 438       this.timeout(120_000)
 439
 440       await testBinaryUpdate('https://yt-dl.org/downloads/latest/youtube-dl', 'youtube-dl')
 441     })
 442
 443     it('Should update youtube-dl from youtube-dl fork', async function () {
 444       this.timeout(120_000)
 445
 446       await testBinaryUpdate('https://api.github.com/repos/yt-dlp/yt-dlp/releases', 'yt-dlp')
 447     })
 448   })
 449 })