server/tests/api/live/live.ts

   1 /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
   2
   3 import 'mocha'
   4 import * as chai from 'chai'
   5 import { basename, join } from 'path'
   6 import { ffprobePromise, getVideoStreamFromFile } from '@server/helpers/ffprobe-utils'
   7 import { checkLiveCleanupAfterSave, checkLiveSegmentHash, checkResolutionsInMasterPlaylist, testImage } from '@server/tests/shared'
   8 import { wait } from '@shared/core-utils'
   9 import {
  10   HttpStatusCode,
  11   LiveVideo,
  12   LiveVideoCreate,
  13   VideoDetails,
  14   VideoPrivacy,
  15   VideoState,
  16   VideoStreamingPlaylistType
  17 } from '@shared/models'
  18 import {
  19   cleanupTests,
  20   createMultipleServers,
  21   doubleFollow,
  22   killallServers,
  23   LiveCommand,
  24   makeRawRequest,
  25   PeerTubeServer,
  26   sendRTMPStream,
  27   setAccessTokensToServers,
  28   setDefaultVideoChannel,
  29   stopFfmpeg,
  30   testFfmpegStreamError,
  31   waitJobs,
  32   waitUntilLivePublishedOnAllServers
  33 } from '@shared/server-commands'
  34
  35 const expect = chai.expect
  36
  37 describe('Test live', function () {
  38   let servers: PeerTubeServer[] = []
  39   let commands: LiveCommand[]
  40
  41   before(async function () {
  42     this.timeout(120000)
  43
  44     servers = await createMultipleServers(2)
  45
  46     // Get the access tokens
  47     await setAccessTokensToServers(servers)
  48     await setDefaultVideoChannel(servers)
  49
  50     await servers[0].config.updateCustomSubConfig({
  51       newConfig: {
  52         live: {
  53           enabled: true,
  54           allowReplay: true,
  55           transcoding: {
  56             enabled: false
  57           }
  58         }
  59       }
  60     })
  61
  62     // Server 1 and server 2 follow each other
  63     await doubleFollow(servers[0], servers[1])
  64
  65     commands = servers.map(s => s.live)
  66   })
  67
  68   describe('Live creation, update and delete', function () {
  69     let liveVideoUUID: string
  70
  71     it('Should create a live with the appropriate parameters', async function () {
  72       this.timeout(20000)
  73
  74       const attributes: LiveVideoCreate = {
  75         category: 1,
  76         licence: 2,
  77         language: 'fr',
  78         description: 'super live description',
  79         support: 'support field',
  80         channelId: servers[0].store.channel.id,
  81         nsfw: false,
  82         waitTranscoding: false,
  83         name: 'my super live',
  84         tags: [ 'tag1', 'tag2' ],
  85         commentsEnabled: false,
  86         downloadEnabled: false,
  87         saveReplay: true,
  88         privacy: VideoPrivacy.PUBLIC,
  89         previewfile: 'video_short1-preview.webm.jpg',
  90         thumbnailfile: 'video_short1.webm.jpg'
  91       }
  92
  93       const live = await commands[0].create({ fields: attributes })
  94       liveVideoUUID = live.uuid
  95
  96       await waitJobs(servers)
  97
  98       for (const server of servers) {
  99         const video = await server.videos.get({ id: liveVideoUUID })
 100
 101         expect(video.category.id).to.equal(1)
 102         expect(video.licence.id).to.equal(2)
 103         expect(video.language.id).to.equal('fr')
 104         expect(video.description).to.equal('super live description')
 105         expect(video.support).to.equal('support field')
 106
 107         expect(video.channel.name).to.equal(servers[0].store.channel.name)
 108         expect(video.channel.host).to.equal(servers[0].store.channel.host)
 109
 110         expect(video.isLive).to.be.true
 111
 112         expect(video.nsfw).to.be.false
 113         expect(video.waitTranscoding).to.be.false
 114         expect(video.name).to.equal('my super live')
 115         expect(video.tags).to.deep.equal([ 'tag1', 'tag2' ])
 116         expect(video.commentsEnabled).to.be.false
 117         expect(video.downloadEnabled).to.be.false
 118         expect(video.privacy.id).to.equal(VideoPrivacy.PUBLIC)
 119
 120         await testImage(server.url, 'video_short1-preview.webm', video.previewPath)
 121         await testImage(server.url, 'video_short1.webm', video.thumbnailPath)
 122
 123         const live = await server.live.get({ videoId: liveVideoUUID })
 124
 125         if (server.url === servers[0].url) {
 126           expect(live.rtmpUrl).to.equal('rtmp://' + server.hostname + ':' + servers[0].rtmpPort + '/live')
 127           expect(live.streamKey).to.not.be.empty
 128         } else {
 129           expect(live.rtmpUrl).to.be.null
 130           expect(live.streamKey).to.be.null
 131         }
 132
 133         expect(live.saveReplay).to.be.true
 134       }
 135     })
 136
 137     it('Should have a default preview and thumbnail', async function () {
 138       this.timeout(20000)
 139
 140       const attributes: LiveVideoCreate = {
 141         name: 'default live thumbnail',
 142         channelId: servers[0].store.channel.id,
 143         privacy: VideoPrivacy.UNLISTED,
 144         nsfw: true
 145       }
 146
 147       const live = await commands[0].create({ fields: attributes })
 148       const videoId = live.uuid
 149
 150       await waitJobs(servers)
 151
 152       for (const server of servers) {
 153         const video = await server.videos.get({ id: videoId })
 154         expect(video.privacy.id).to.equal(VideoPrivacy.UNLISTED)
 155         expect(video.nsfw).to.be.true
 156
 157         await makeRawRequest(server.url + video.thumbnailPath, HttpStatusCode.OK_200)
 158         await makeRawRequest(server.url + video.previewPath, HttpStatusCode.OK_200)
 159       }
 160     })
 161
 162     it('Should not have the live listed since nobody streams into', async function () {
 163       for (const server of servers) {
 164         const { total, data } = await server.videos.list()
 165
 166         expect(total).to.equal(0)
 167         expect(data).to.have.lengthOf(0)
 168       }
 169     })
 170
 171     it('Should not be able to update a live of another server', async function () {
 172       await commands[1].update({ videoId: liveVideoUUID, fields: { saveReplay: false }, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
 173     })
 174
 175     it('Should update the live', async function () {
 176       this.timeout(10000)
 177
 178       await commands[0].update({ videoId: liveVideoUUID, fields: { saveReplay: false } })
 179       await waitJobs(servers)
 180     })
 181
 182     it('Have the live updated', async function () {
 183       for (const server of servers) {
 184         const live = await server.live.get({ videoId: liveVideoUUID })
 185
 186         if (server.url === servers[0].url) {
 187           expect(live.rtmpUrl).to.equal('rtmp://' + server.hostname + ':' + servers[0].rtmpPort + '/live')
 188           expect(live.streamKey).to.not.be.empty
 189         } else {
 190           expect(live.rtmpUrl).to.be.null
 191           expect(live.streamKey).to.be.null
 192         }
 193
 194         expect(live.saveReplay).to.be.false
 195       }
 196     })
 197
 198     it('Delete the live', async function () {
 199       this.timeout(10000)
 200
 201       await servers[0].videos.remove({ id: liveVideoUUID })
 202       await waitJobs(servers)
 203     })
 204
 205     it('Should have the live deleted', async function () {
 206       for (const server of servers) {
 207         await server.videos.get({ id: liveVideoUUID, expectedStatus: HttpStatusCode.NOT_FOUND_404 })
 208         await server.live.get({ videoId: liveVideoUUID, expectedStatus: HttpStatusCode.NOT_FOUND_404 })
 209       }
 210     })
 211   })
 212
 213   describe('Live filters', function () {
 214     let ffmpegCommand: any
 215     let liveVideoId: string
 216     let vodVideoId: string
 217
 218     before(async function () {
 219       this.timeout(120000)
 220
 221       vodVideoId = (await servers[0].videos.quickUpload({ name: 'vod video' })).uuid
 222
 223       const liveOptions = { name: 'live', privacy: VideoPrivacy.PUBLIC, channelId: servers[0].store.channel.id }
 224       const live = await commands[0].create({ fields: liveOptions })
 225       liveVideoId = live.uuid
 226
 227       ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoId })
 228       await waitUntilLivePublishedOnAllServers(servers, liveVideoId)
 229       await waitJobs(servers)
 230     })
 231
 232     it('Should only display lives', async function () {
 233       const { data, total } = await servers[0].videos.list({ isLive: true })
 234
 235       expect(total).to.equal(1)
 236       expect(data).to.have.lengthOf(1)
 237       expect(data[0].name).to.equal('live')
 238     })
 239
 240     it('Should not display lives', async function () {
 241       const { data, total } = await servers[0].videos.list({ isLive: false })
 242
 243       expect(total).to.equal(1)
 244       expect(data).to.have.lengthOf(1)
 245       expect(data[0].name).to.equal('vod video')
 246     })
 247
 248     it('Should display my lives', async function () {
 249       this.timeout(60000)
 250
 251       await stopFfmpeg(ffmpegCommand)
 252       await waitJobs(servers)
 253
 254       const { data } = await servers[0].videos.listMyVideos({ isLive: true })
 255
 256       const result = data.every(v => v.isLive)
 257       expect(result).to.be.true
 258     })
 259
 260     it('Should not display my lives', async function () {
 261       const { data } = await servers[0].videos.listMyVideos({ isLive: false })
 262
 263       const result = data.every(v => !v.isLive)
 264       expect(result).to.be.true
 265     })
 266
 267     after(async function () {
 268       await servers[0].videos.remove({ id: vodVideoId })
 269       await servers[0].videos.remove({ id: liveVideoId })
 270     })
 271   })
 272
 273   describe('Stream checks', function () {
 274     let liveVideo: LiveVideo & VideoDetails
 275     let rtmpUrl: string
 276
 277     before(function () {
 278       rtmpUrl = 'rtmp://' + servers[0].hostname + ':' + servers[0].rtmpPort + ''
 279     })
 280
 281     async function createLiveWrapper () {
 282       const liveAttributes = {
 283         name: 'user live',
 284         channelId: servers[0].store.channel.id,
 285         privacy: VideoPrivacy.PUBLIC,
 286         saveReplay: false
 287       }
 288
 289       const { uuid } = await commands[0].create({ fields: liveAttributes })
 290
 291       const live = await commands[0].get({ videoId: uuid })
 292       const video = await servers[0].videos.get({ id: uuid })
 293
 294       return Object.assign(video, live)
 295     }
 296
 297     it('Should not allow a stream without the appropriate path', async function () {
 298       this.timeout(60000)
 299
 300       liveVideo = await createLiveWrapper()
 301
 302       const command = sendRTMPStream({ rtmpBaseUrl: rtmpUrl + '/bad-live', streamKey: liveVideo.streamKey })
 303       await testFfmpegStreamError(command, true)
 304     })
 305
 306     it('Should not allow a stream without the appropriate stream key', async function () {
 307       this.timeout(60000)
 308
 309       const command = sendRTMPStream({ rtmpBaseUrl: rtmpUrl + '/live', streamKey: 'bad-stream-key' })
 310       await testFfmpegStreamError(command, true)
 311     })
 312
 313     it('Should succeed with the correct params', async function () {
 314       this.timeout(60000)
 315
 316       const command = sendRTMPStream({ rtmpBaseUrl: rtmpUrl + '/live', streamKey: liveVideo.streamKey })
 317       await testFfmpegStreamError(command, false)
 318     })
 319
 320     it('Should list this live now someone stream into it', async function () {
 321       for (const server of servers) {
 322         const { total, data } = await server.videos.list()
 323
 324         expect(total).to.equal(1)
 325         expect(data).to.have.lengthOf(1)
 326
 327         const video = data[0]
 328         expect(video.name).to.equal('user live')
 329         expect(video.isLive).to.be.true
 330       }
 331     })
 332
 333     it('Should not allow a stream on a live that was blacklisted', async function () {
 334       this.timeout(60000)
 335
 336       liveVideo = await createLiveWrapper()
 337
 338       await servers[0].blacklist.add({ videoId: liveVideo.uuid })
 339
 340       const command = sendRTMPStream({ rtmpBaseUrl: rtmpUrl + '/live', streamKey: liveVideo.streamKey })
 341       await testFfmpegStreamError(command, true)
 342     })
 343
 344     it('Should not allow a stream on a live that was deleted', async function () {
 345       this.timeout(60000)
 346
 347       liveVideo = await createLiveWrapper()
 348
 349       await servers[0].videos.remove({ id: liveVideo.uuid })
 350
 351       const command = sendRTMPStream({ rtmpBaseUrl: rtmpUrl + '/live', streamKey: liveVideo.streamKey })
 352       await testFfmpegStreamError(command, true)
 353     })
 354   })
 355
 356   describe('Live transcoding', function () {
 357     let liveVideoId: string
 358
 359     async function createLiveWrapper (saveReplay: boolean) {
 360       const liveAttributes = {
 361         name: 'live video',
 362         channelId: servers[0].store.channel.id,
 363         privacy: VideoPrivacy.PUBLIC,
 364         saveReplay
 365       }
 366
 367       const { uuid } = await commands[0].create({ fields: liveAttributes })
 368       return uuid
 369     }
 370
 371     async function testVideoResolutions (liveVideoId: string, resolutions: number[]) {
 372       for (const server of servers) {
 373         const { data } = await server.videos.list()
 374         expect(data.find(v => v.uuid === liveVideoId)).to.exist
 375
 376         const video = await server.videos.get({ id: liveVideoId })
 377
 378         expect(video.streamingPlaylists).to.have.lengthOf(1)
 379
 380         const hlsPlaylist = video.streamingPlaylists.find(s => s.type === VideoStreamingPlaylistType.HLS)
 381         expect(hlsPlaylist).to.exist
 382
 383         // Only finite files are displayed
 384         expect(hlsPlaylist.files).to.have.lengthOf(0)
 385
 386         await checkResolutionsInMasterPlaylist({ server, playlistUrl: hlsPlaylist.playlistUrl, resolutions })
 387
 388         for (let i = 0; i < resolutions.length; i++) {
 389           const segmentNum = 3
 390           const segmentName = `${i}-00000${segmentNum}.ts`
 391           await commands[0].waitUntilSegmentGeneration({ videoUUID: video.uuid, resolution: i, segment: segmentNum })
 392
 393           const subPlaylist = await servers[0].streamingPlaylists.get({
 394             url: `${servers[0].url}/static/streaming-playlists/hls/${video.uuid}/${i}.m3u8`
 395           })
 396
 397           expect(subPlaylist).to.contain(segmentName)
 398
 399           const baseUrlAndPath = servers[0].url + '/static/streaming-playlists/hls'
 400           await checkLiveSegmentHash({
 401             server,
 402             baseUrlSegment: baseUrlAndPath,
 403             videoUUID: video.uuid,
 404             segmentName,
 405             hlsPlaylist
 406           })
 407         }
 408       }
 409     }
 410
 411     function updateConf (resolutions: number[]) {
 412       return servers[0].config.updateCustomSubConfig({
 413         newConfig: {
 414           live: {
 415             enabled: true,
 416             allowReplay: true,
 417             maxDuration: -1,
 418             transcoding: {
 419               enabled: true,
 420               resolutions: {
 421                 '144p': resolutions.includes(144),
 422                 '240p': resolutions.includes(240),
 423                 '360p': resolutions.includes(360),
 424                 '480p': resolutions.includes(480),
 425                 '720p': resolutions.includes(720),
 426                 '1080p': resolutions.includes(1080),
 427                 '2160p': resolutions.includes(2160)
 428               }
 429             }
 430           }
 431         }
 432       })
 433     }
 434
 435     before(async function () {
 436       await updateConf([])
 437     })
 438
 439     it('Should enable transcoding without additional resolutions', async function () {
 440       this.timeout(120000)
 441
 442       liveVideoId = await createLiveWrapper(false)
 443
 444       const ffmpegCommand = await commands[0].sendRTMPStreamInVideo({ videoId: liveVideoId })
 445       await waitUntilLivePublishedOnAllServers(servers, liveVideoId)
 446       await waitJobs(servers)
 447
 448       await testVideoResolutions(liveVideoId, [ 720 ])
 449
 450       await stopFfmpeg(ffmpegCommand)
 451     })
 452
 453     it('Should enable transcoding with some resolutions', async function () {
 454       this.timeout(120000)
 455
 456       const resolutions = [ 240, 480 ]
 457       await updateConf(resolutions)
 458       liveVideoId = await createLiveWrapper(false)
 459
 460       const ffmpegCommand = await commands[0].sendRTMPStreamInVideo({ videoId: liveVideoId })
 461       await waitUntilLivePublishedOnAllServers(servers, liveVideoId)
 462       await waitJobs(servers)
 463
 464       await testVideoResolutions(liveVideoId, resolutions)
 465
 466       await stopFfmpeg(ffmpegCommand)
 467     })
 468
 469     it('Should correctly set the appropriate bitrate depending on the input', async function () {
 470       this.timeout(120000)
 471
 472       liveVideoId = await createLiveWrapper(false)
 473
 474       const ffmpegCommand = await commands[0].sendRTMPStreamInVideo({
 475         videoId: liveVideoId,
 476         fixtureName: 'video_short.mp4',
 477         copyCodecs: true
 478       })
 479       await waitUntilLivePublishedOnAllServers(servers, liveVideoId)
 480       await waitJobs(servers)
 481
 482       const video = await servers[0].videos.get({ id: liveVideoId })
 483
 484       const masterPlaylist = video.streamingPlaylists[0].playlistUrl
 485       const probe = await ffprobePromise(masterPlaylist)
 486
 487       const bitrates = probe.streams.map(s => parseInt(s.tags.variant_bitrate))
 488       for (const bitrate of bitrates) {
 489         expect(bitrate).to.exist
 490         expect(isNaN(bitrate)).to.be.false
 491         expect(bitrate).to.be.below(61_000_000) // video_short.mp4 bitrate
 492       }
 493
 494       await stopFfmpeg(ffmpegCommand)
 495     })
 496
 497     it('Should enable transcoding with some resolutions and correctly save them', async function () {
 498       this.timeout(400_000)
 499
 500       const resolutions = [ 240, 360, 720 ]
 501
 502       await updateConf(resolutions)
 503       liveVideoId = await createLiveWrapper(true)
 504
 505       const ffmpegCommand = await commands[0].sendRTMPStreamInVideo({ videoId: liveVideoId, fixtureName: 'video_short2.webm' })
 506       await waitUntilLivePublishedOnAllServers(servers, liveVideoId)
 507       await waitJobs(servers)
 508
 509       await testVideoResolutions(liveVideoId, resolutions)
 510
 511       await stopFfmpeg(ffmpegCommand)
 512       await commands[0].waitUntilEnded({ videoId: liveVideoId })
 513
 514       await waitJobs(servers)
 515
 516       await waitUntilLivePublishedOnAllServers(servers, liveVideoId)
 517
 518       const maxBitrateLimits = {
 519         720: 6500 * 1000, // 60FPS
 520         360: 1250 * 1000,
 521         240: 700 * 1000
 522       }
 523
 524       const minBitrateLimits = {
 525         720: 5500 * 1000,
 526         360: 1000 * 1000,
 527         240: 550 * 1000
 528       }
 529
 530       for (const server of servers) {
 531         const video = await server.videos.get({ id: liveVideoId })
 532
 533         expect(video.state.id).to.equal(VideoState.PUBLISHED)
 534         expect(video.duration).to.be.greaterThan(1)
 535         expect(video.files).to.have.lengthOf(0)
 536
 537         const hlsPlaylist = video.streamingPlaylists.find(s => s.type === VideoStreamingPlaylistType.HLS)
 538         await makeRawRequest(hlsPlaylist.playlistUrl, HttpStatusCode.OK_200)
 539         await makeRawRequest(hlsPlaylist.segmentsSha256Url, HttpStatusCode.OK_200)
 540
 541         // We should have generated random filenames
 542         expect(basename(hlsPlaylist.playlistUrl)).to.not.equal('master.m3u8')
 543         expect(basename(hlsPlaylist.segmentsSha256Url)).to.not.equal('segments-sha256.json')
 544
 545         expect(hlsPlaylist.files).to.have.lengthOf(resolutions.length)
 546
 547         for (const resolution of resolutions) {
 548           const file = hlsPlaylist.files.find(f => f.resolution.id === resolution)
 549
 550           expect(file).to.exist
 551           expect(file.size).to.be.greaterThan(1)
 552
 553           if (resolution >= 720) {
 554             expect(file.fps).to.be.approximately(60, 2)
 555           } else {
 556             expect(file.fps).to.be.approximately(30, 2)
 557           }
 558
 559           const filename = basename(file.fileUrl)
 560           expect(filename).to.not.contain(video.uuid)
 561
 562           const segmentPath = servers[0].servers.buildDirectory(join('streaming-playlists', 'hls', video.uuid, filename))
 563
 564           const probe = await ffprobePromise(segmentPath)
 565           const videoStream = await getVideoStreamFromFile(segmentPath, probe)
 566
 567           expect(probe.format.bit_rate).to.be.below(maxBitrateLimits[videoStream.height])
 568           expect(probe.format.bit_rate).to.be.at.least(minBitrateLimits[videoStream.height])
 569
 570           await makeRawRequest(file.torrentUrl, HttpStatusCode.OK_200)
 571           await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200)
 572         }
 573       }
 574     })
 575
 576     it('Should correctly have cleaned up the live files', async function () {
 577       this.timeout(30000)
 578
 579       await checkLiveCleanupAfterSave(servers[0], liveVideoId, [ 240, 360, 720 ])
 580     })
 581   })
 582
 583   describe('After a server restart', function () {
 584     let liveVideoId: string
 585     let liveVideoReplayId: string
 586
 587     async function createLiveWrapper (saveReplay: boolean) {
 588       const liveAttributes = {
 589         name: 'live video',
 590         channelId: servers[0].store.channel.id,
 591         privacy: VideoPrivacy.PUBLIC,
 592         saveReplay
 593       }
 594
 595       const { uuid } = await commands[0].create({ fields: liveAttributes })
 596       return uuid
 597     }
 598
 599     before(async function () {
 600       this.timeout(120000)
 601
 602       liveVideoId = await createLiveWrapper(false)
 603       liveVideoReplayId = await createLiveWrapper(true)
 604
 605       await Promise.all([
 606         commands[0].sendRTMPStreamInVideo({ videoId: liveVideoId }),
 607         commands[0].sendRTMPStreamInVideo({ videoId: liveVideoReplayId })
 608       ])
 609
 610       await Promise.all([
 611         commands[0].waitUntilPublished({ videoId: liveVideoId }),
 612         commands[0].waitUntilPublished({ videoId: liveVideoReplayId })
 613       ])
 614
 615       await commands[0].waitUntilSegmentGeneration({ videoUUID: liveVideoId, resolution: 0, segment: 2 })
 616       await commands[0].waitUntilSegmentGeneration({ videoUUID: liveVideoReplayId, resolution: 0, segment: 2 })
 617
 618       await killallServers([ servers[0] ])
 619       await servers[0].run()
 620
 621       await wait(5000)
 622     })
 623
 624     it('Should cleanup lives', async function () {
 625       this.timeout(60000)
 626
 627       await commands[0].waitUntilEnded({ videoId: liveVideoId })
 628     })
 629
 630     it('Should save a live replay', async function () {
 631       this.timeout(120000)
 632
 633       await commands[0].waitUntilPublished({ videoId: liveVideoReplayId })
 634     })
 635   })
 636
 637   after(async function () {
 638     await cleanupTests(servers)
 639   })
 640 })