server/tests/api/live/live.ts

   1 /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
   2
   3 import 'mocha'
   4 import * as chai from 'chai'
   5 import { basename, join } from 'path'
   6 import { ffprobePromise, getVideoStream } from '@server/helpers/ffmpeg'
   7 import { checkLiveCleanupAfterSave, checkLiveSegmentHash, checkResolutionsInMasterPlaylist, testImage } from '@server/tests/shared'
   8 import { wait } from '@shared/core-utils'
   9 import {
  10   HttpStatusCode,
  11   LiveVideo,
  12   LiveVideoCreate,
  13   LiveVideoLatencyMode,
  14   VideoDetails,
  15   VideoPrivacy,
  16   VideoState,
  17   VideoStreamingPlaylistType
  18 } from '@shared/models'
  19 import {
  20   cleanupTests,
  21   createMultipleServers,
  22   doubleFollow,
  23   killallServers,
  24   LiveCommand,
  25   makeRawRequest,
  26   PeerTubeServer,
  27   sendRTMPStream,
  28   setAccessTokensToServers,
  29   setDefaultVideoChannel,
  30   stopFfmpeg,
  31   testFfmpegStreamError,
  32   waitJobs,
  33   waitUntilLivePublishedOnAllServers
  34 } from '@shared/server-commands'
  35
  36 const expect = chai.expect
  37
  38 describe('Test live', function () {
  39   let servers: PeerTubeServer[] = []
  40   let commands: LiveCommand[]
  41
  42   before(async function () {
  43     this.timeout(120000)
  44
  45     servers = await createMultipleServers(2)
  46
  47     // Get the access tokens
  48     await setAccessTokensToServers(servers)
  49     await setDefaultVideoChannel(servers)
  50
  51     await servers[0].config.updateCustomSubConfig({
  52       newConfig: {
  53         live: {
  54           enabled: true,
  55           allowReplay: true,
  56           latencySetting: {
  57             enabled: true
  58           },
  59           transcoding: {
  60             enabled: false
  61           }
  62         }
  63       }
  64     })
  65
  66     // Server 1 and server 2 follow each other
  67     await doubleFollow(servers[0], servers[1])
  68
  69     commands = servers.map(s => s.live)
  70   })
  71
  72   describe('Live creation, update and delete', function () {
  73     let liveVideoUUID: string
  74
  75     it('Should create a live with the appropriate parameters', async function () {
  76       this.timeout(20000)
  77
  78       const attributes: LiveVideoCreate = {
  79         category: 1,
  80         licence: 2,
  81         language: 'fr',
  82         description: 'super live description',
  83         support: 'support field',
  84         channelId: servers[0].store.channel.id,
  85         nsfw: false,
  86         waitTranscoding: false,
  87         name: 'my super live',
  88         tags: [ 'tag1', 'tag2' ],
  89         commentsEnabled: false,
  90         downloadEnabled: false,
  91         saveReplay: true,
  92         latencyMode: LiveVideoLatencyMode.SMALL_LATENCY,
  93         privacy: VideoPrivacy.PUBLIC,
  94         previewfile: 'video_short1-preview.webm.jpg',
  95         thumbnailfile: 'video_short1.webm.jpg'
  96       }
  97
  98       const live = await commands[0].create({ fields: attributes })
  99       liveVideoUUID = live.uuid
 100
 101       await waitJobs(servers)
 102
 103       for (const server of servers) {
 104         const video = await server.videos.get({ id: liveVideoUUID })
 105
 106         expect(video.category.id).to.equal(1)
 107         expect(video.licence.id).to.equal(2)
 108         expect(video.language.id).to.equal('fr')
 109         expect(video.description).to.equal('super live description')
 110         expect(video.support).to.equal('support field')
 111
 112         expect(video.channel.name).to.equal(servers[0].store.channel.name)
 113         expect(video.channel.host).to.equal(servers[0].store.channel.host)
 114
 115         expect(video.isLive).to.be.true
 116
 117         expect(video.nsfw).to.be.false
 118         expect(video.waitTranscoding).to.be.false
 119         expect(video.name).to.equal('my super live')
 120         expect(video.tags).to.deep.equal([ 'tag1', 'tag2' ])
 121         expect(video.commentsEnabled).to.be.false
 122         expect(video.downloadEnabled).to.be.false
 123         expect(video.privacy.id).to.equal(VideoPrivacy.PUBLIC)
 124
 125         await testImage(server.url, 'video_short1-preview.webm', video.previewPath)
 126         await testImage(server.url, 'video_short1.webm', video.thumbnailPath)
 127
 128         const live = await server.live.get({ videoId: liveVideoUUID })
 129
 130         if (server.url === servers[0].url) {
 131           expect(live.rtmpUrl).to.equal('rtmp://' + server.hostname + ':' + servers[0].rtmpPort + '/live')
 132           expect(live.streamKey).to.not.be.empty
 133         } else {
 134           expect(live.rtmpUrl).to.be.null
 135           expect(live.streamKey).to.be.null
 136         }
 137
 138         expect(live.saveReplay).to.be.true
 139         expect(live.latencyMode).to.equal(LiveVideoLatencyMode.SMALL_LATENCY)
 140       }
 141     })
 142
 143     it('Should have a default preview and thumbnail', async function () {
 144       this.timeout(20000)
 145
 146       const attributes: LiveVideoCreate = {
 147         name: 'default live thumbnail',
 148         channelId: servers[0].store.channel.id,
 149         privacy: VideoPrivacy.UNLISTED,
 150         nsfw: true
 151       }
 152
 153       const live = await commands[0].create({ fields: attributes })
 154       const videoId = live.uuid
 155
 156       await waitJobs(servers)
 157
 158       for (const server of servers) {
 159         const video = await server.videos.get({ id: videoId })
 160         expect(video.privacy.id).to.equal(VideoPrivacy.UNLISTED)
 161         expect(video.nsfw).to.be.true
 162
 163         await makeRawRequest(server.url + video.thumbnailPath, HttpStatusCode.OK_200)
 164         await makeRawRequest(server.url + video.previewPath, HttpStatusCode.OK_200)
 165       }
 166     })
 167
 168     it('Should not have the live listed since nobody streams into', async function () {
 169       for (const server of servers) {
 170         const { total, data } = await server.videos.list()
 171
 172         expect(total).to.equal(0)
 173         expect(data).to.have.lengthOf(0)
 174       }
 175     })
 176
 177     it('Should not be able to update a live of another server', async function () {
 178       await commands[1].update({ videoId: liveVideoUUID, fields: { saveReplay: false }, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
 179     })
 180
 181     it('Should update the live', async function () {
 182       this.timeout(10000)
 183
 184       await commands[0].update({ videoId: liveVideoUUID, fields: { saveReplay: false, latencyMode: LiveVideoLatencyMode.DEFAULT } })
 185       await waitJobs(servers)
 186     })
 187
 188     it('Have the live updated', async function () {
 189       for (const server of servers) {
 190         const live = await server.live.get({ videoId: liveVideoUUID })
 191
 192         if (server.url === servers[0].url) {
 193           expect(live.rtmpUrl).to.equal('rtmp://' + server.hostname + ':' + servers[0].rtmpPort + '/live')
 194           expect(live.streamKey).to.not.be.empty
 195         } else {
 196           expect(live.rtmpUrl).to.be.null
 197           expect(live.streamKey).to.be.null
 198         }
 199
 200         expect(live.saveReplay).to.be.false
 201         expect(live.latencyMode).to.equal(LiveVideoLatencyMode.DEFAULT)
 202       }
 203     })
 204
 205     it('Delete the live', async function () {
 206       this.timeout(10000)
 207
 208       await servers[0].videos.remove({ id: liveVideoUUID })
 209       await waitJobs(servers)
 210     })
 211
 212     it('Should have the live deleted', async function () {
 213       for (const server of servers) {
 214         await server.videos.get({ id: liveVideoUUID, expectedStatus: HttpStatusCode.NOT_FOUND_404 })
 215         await server.live.get({ videoId: liveVideoUUID, expectedStatus: HttpStatusCode.NOT_FOUND_404 })
 216       }
 217     })
 218   })
 219
 220   describe('Live filters', function () {
 221     let ffmpegCommand: any
 222     let liveVideoId: string
 223     let vodVideoId: string
 224
 225     before(async function () {
 226       this.timeout(120000)
 227
 228       vodVideoId = (await servers[0].videos.quickUpload({ name: 'vod video' })).uuid
 229
 230       const liveOptions = { name: 'live', privacy: VideoPrivacy.PUBLIC, channelId: servers[0].store.channel.id }
 231       const live = await commands[0].create({ fields: liveOptions })
 232       liveVideoId = live.uuid
 233
 234       ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoId })
 235       await waitUntilLivePublishedOnAllServers(servers, liveVideoId)
 236       await waitJobs(servers)
 237     })
 238
 239     it('Should only display lives', async function () {
 240       const { data, total } = await servers[0].videos.list({ isLive: true })
 241
 242       expect(total).to.equal(1)
 243       expect(data).to.have.lengthOf(1)
 244       expect(data[0].name).to.equal('live')
 245     })
 246
 247     it('Should not display lives', async function () {
 248       const { data, total } = await servers[0].videos.list({ isLive: false })
 249
 250       expect(total).to.equal(1)
 251       expect(data).to.have.lengthOf(1)
 252       expect(data[0].name).to.equal('vod video')
 253     })
 254
 255     it('Should display my lives', async function () {
 256       this.timeout(60000)
 257
 258       await stopFfmpeg(ffmpegCommand)
 259       await waitJobs(servers)
 260
 261       const { data } = await servers[0].videos.listMyVideos({ isLive: true })
 262
 263       const result = data.every(v => v.isLive)
 264       expect(result).to.be.true
 265     })
 266
 267     it('Should not display my lives', async function () {
 268       const { data } = await servers[0].videos.listMyVideos({ isLive: false })
 269
 270       const result = data.every(v => !v.isLive)
 271       expect(result).to.be.true
 272     })
 273
 274     after(async function () {
 275       await servers[0].videos.remove({ id: vodVideoId })
 276       await servers[0].videos.remove({ id: liveVideoId })
 277     })
 278   })
 279
 280   describe('Stream checks', function () {
 281     let liveVideo: LiveVideo & VideoDetails
 282     let rtmpUrl: string
 283
 284     before(function () {
 285       rtmpUrl = 'rtmp://' + servers[0].hostname + ':' + servers[0].rtmpPort + ''
 286     })
 287
 288     async function createLiveWrapper () {
 289       const liveAttributes = {
 290         name: 'user live',
 291         channelId: servers[0].store.channel.id,
 292         privacy: VideoPrivacy.PUBLIC,
 293         saveReplay: false
 294       }
 295
 296       const { uuid } = await commands[0].create({ fields: liveAttributes })
 297
 298       const live = await commands[0].get({ videoId: uuid })
 299       const video = await servers[0].videos.get({ id: uuid })
 300
 301       return Object.assign(video, live)
 302     }
 303
 304     it('Should not allow a stream without the appropriate path', async function () {
 305       this.timeout(60000)
 306
 307       liveVideo = await createLiveWrapper()
 308
 309       const command = sendRTMPStream({ rtmpBaseUrl: rtmpUrl + '/bad-live', streamKey: liveVideo.streamKey })
 310       await testFfmpegStreamError(command, true)
 311     })
 312
 313     it('Should not allow a stream without the appropriate stream key', async function () {
 314       this.timeout(60000)
 315
 316       const command = sendRTMPStream({ rtmpBaseUrl: rtmpUrl + '/live', streamKey: 'bad-stream-key' })
 317       await testFfmpegStreamError(command, true)
 318     })
 319
 320     it('Should succeed with the correct params', async function () {
 321       this.timeout(60000)
 322
 323       const command = sendRTMPStream({ rtmpBaseUrl: rtmpUrl + '/live', streamKey: liveVideo.streamKey })
 324       await testFfmpegStreamError(command, false)
 325     })
 326
 327     it('Should list this live now someone stream into it', async function () {
 328       for (const server of servers) {
 329         const { total, data } = await server.videos.list()
 330
 331         expect(total).to.equal(1)
 332         expect(data).to.have.lengthOf(1)
 333
 334         const video = data[0]
 335         expect(video.name).to.equal('user live')
 336         expect(video.isLive).to.be.true
 337       }
 338     })
 339
 340     it('Should not allow a stream on a live that was blacklisted', async function () {
 341       this.timeout(60000)
 342
 343       liveVideo = await createLiveWrapper()
 344
 345       await servers[0].blacklist.add({ videoId: liveVideo.uuid })
 346
 347       const command = sendRTMPStream({ rtmpBaseUrl: rtmpUrl + '/live', streamKey: liveVideo.streamKey })
 348       await testFfmpegStreamError(command, true)
 349     })
 350
 351     it('Should not allow a stream on a live that was deleted', async function () {
 352       this.timeout(60000)
 353
 354       liveVideo = await createLiveWrapper()
 355
 356       await servers[0].videos.remove({ id: liveVideo.uuid })
 357
 358       const command = sendRTMPStream({ rtmpBaseUrl: rtmpUrl + '/live', streamKey: liveVideo.streamKey })
 359       await testFfmpegStreamError(command, true)
 360     })
 361   })
 362
 363   describe('Live transcoding', function () {
 364     let liveVideoId: string
 365
 366     async function createLiveWrapper (saveReplay: boolean) {
 367       const liveAttributes = {
 368         name: 'live video',
 369         channelId: servers[0].store.channel.id,
 370         privacy: VideoPrivacy.PUBLIC,
 371         saveReplay
 372       }
 373
 374       const { uuid } = await commands[0].create({ fields: liveAttributes })
 375       return uuid
 376     }
 377
 378     async function testVideoResolutions (liveVideoId: string, resolutions: number[]) {
 379       for (const server of servers) {
 380         const { data } = await server.videos.list()
 381         expect(data.find(v => v.uuid === liveVideoId)).to.exist
 382
 383         const video = await server.videos.get({ id: liveVideoId })
 384
 385         expect(video.streamingPlaylists).to.have.lengthOf(1)
 386
 387         const hlsPlaylist = video.streamingPlaylists.find(s => s.type === VideoStreamingPlaylistType.HLS)
 388         expect(hlsPlaylist).to.exist
 389
 390         // Only finite files are displayed
 391         expect(hlsPlaylist.files).to.have.lengthOf(0)
 392
 393         await checkResolutionsInMasterPlaylist({ server, playlistUrl: hlsPlaylist.playlistUrl, resolutions })
 394
 395         for (let i = 0; i < resolutions.length; i++) {
 396           const segmentNum = 3
 397           const segmentName = `${i}-00000${segmentNum}.ts`
 398           await commands[0].waitUntilSegmentGeneration({ videoUUID: video.uuid, resolution: i, segment: segmentNum })
 399
 400           const subPlaylist = await servers[0].streamingPlaylists.get({
 401             url: `${servers[0].url}/static/streaming-playlists/hls/${video.uuid}/${i}.m3u8`
 402           })
 403
 404           expect(subPlaylist).to.contain(segmentName)
 405
 406           const baseUrlAndPath = servers[0].url + '/static/streaming-playlists/hls'
 407           await checkLiveSegmentHash({
 408             server,
 409             baseUrlSegment: baseUrlAndPath,
 410             videoUUID: video.uuid,
 411             segmentName,
 412             hlsPlaylist
 413           })
 414         }
 415       }
 416     }
 417
 418     function updateConf (resolutions: number[]) {
 419       return servers[0].config.updateCustomSubConfig({
 420         newConfig: {
 421           live: {
 422             enabled: true,
 423             allowReplay: true,
 424             maxDuration: -1,
 425             transcoding: {
 426               enabled: true,
 427               resolutions: {
 428                 '144p': resolutions.includes(144),
 429                 '240p': resolutions.includes(240),
 430                 '360p': resolutions.includes(360),
 431                 '480p': resolutions.includes(480),
 432                 '720p': resolutions.includes(720),
 433                 '1080p': resolutions.includes(1080),
 434                 '2160p': resolutions.includes(2160)
 435               }
 436             }
 437           }
 438         }
 439       })
 440     }
 441
 442     before(async function () {
 443       await updateConf([])
 444     })
 445
 446     it('Should enable transcoding without additional resolutions', async function () {
 447       this.timeout(120000)
 448
 449       liveVideoId = await createLiveWrapper(false)
 450
 451       const ffmpegCommand = await commands[0].sendRTMPStreamInVideo({ videoId: liveVideoId })
 452       await waitUntilLivePublishedOnAllServers(servers, liveVideoId)
 453       await waitJobs(servers)
 454
 455       await testVideoResolutions(liveVideoId, [ 720 ])
 456
 457       await stopFfmpeg(ffmpegCommand)
 458     })
 459
 460     it('Should enable transcoding with some resolutions', async function () {
 461       this.timeout(120000)
 462
 463       const resolutions = [ 240, 480 ]
 464       await updateConf(resolutions)
 465       liveVideoId = await createLiveWrapper(false)
 466
 467       const ffmpegCommand = await commands[0].sendRTMPStreamInVideo({ videoId: liveVideoId })
 468       await waitUntilLivePublishedOnAllServers(servers, liveVideoId)
 469       await waitJobs(servers)
 470
 471       await testVideoResolutions(liveVideoId, resolutions)
 472
 473       await stopFfmpeg(ffmpegCommand)
 474     })
 475
 476     it('Should correctly set the appropriate bitrate depending on the input', async function () {
 477       this.timeout(120000)
 478
 479       liveVideoId = await createLiveWrapper(false)
 480
 481       const ffmpegCommand = await commands[0].sendRTMPStreamInVideo({
 482         videoId: liveVideoId,
 483         fixtureName: 'video_short.mp4',
 484         copyCodecs: true
 485       })
 486       await waitUntilLivePublishedOnAllServers(servers, liveVideoId)
 487       await waitJobs(servers)
 488
 489       const video = await servers[0].videos.get({ id: liveVideoId })
 490
 491       const masterPlaylist = video.streamingPlaylists[0].playlistUrl
 492       const probe = await ffprobePromise(masterPlaylist)
 493
 494       const bitrates = probe.streams.map(s => parseInt(s.tags.variant_bitrate))
 495       for (const bitrate of bitrates) {
 496         expect(bitrate).to.exist
 497         expect(isNaN(bitrate)).to.be.false
 498         expect(bitrate).to.be.below(61_000_000) // video_short.mp4 bitrate
 499       }
 500
 501       await stopFfmpeg(ffmpegCommand)
 502     })
 503
 504     it('Should enable transcoding with some resolutions and correctly save them', async function () {
 505       this.timeout(400_000)
 506
 507       const resolutions = [ 240, 360, 720 ]
 508
 509       await updateConf(resolutions)
 510       liveVideoId = await createLiveWrapper(true)
 511
 512       const ffmpegCommand = await commands[0].sendRTMPStreamInVideo({ videoId: liveVideoId, fixtureName: 'video_short2.webm' })
 513       await waitUntilLivePublishedOnAllServers(servers, liveVideoId)
 514       await waitJobs(servers)
 515
 516       await testVideoResolutions(liveVideoId, resolutions)
 517
 518       await stopFfmpeg(ffmpegCommand)
 519       await commands[0].waitUntilEnded({ videoId: liveVideoId })
 520
 521       await waitJobs(servers)
 522
 523       await waitUntilLivePublishedOnAllServers(servers, liveVideoId)
 524
 525       const maxBitrateLimits = {
 526         720: 6500 * 1000, // 60FPS
 527         360: 1250 * 1000,
 528         240: 700 * 1000
 529       }
 530
 531       const minBitrateLimits = {
 532         720: 5500 * 1000,
 533         360: 1000 * 1000,
 534         240: 550 * 1000
 535       }
 536
 537       for (const server of servers) {
 538         const video = await server.videos.get({ id: liveVideoId })
 539
 540         expect(video.state.id).to.equal(VideoState.PUBLISHED)
 541         expect(video.duration).to.be.greaterThan(1)
 542         expect(video.files).to.have.lengthOf(0)
 543
 544         const hlsPlaylist = video.streamingPlaylists.find(s => s.type === VideoStreamingPlaylistType.HLS)
 545         await makeRawRequest(hlsPlaylist.playlistUrl, HttpStatusCode.OK_200)
 546         await makeRawRequest(hlsPlaylist.segmentsSha256Url, HttpStatusCode.OK_200)
 547
 548         // We should have generated random filenames
 549         expect(basename(hlsPlaylist.playlistUrl)).to.not.equal('master.m3u8')
 550         expect(basename(hlsPlaylist.segmentsSha256Url)).to.not.equal('segments-sha256.json')
 551
 552         expect(hlsPlaylist.files).to.have.lengthOf(resolutions.length)
 553
 554         for (const resolution of resolutions) {
 555           const file = hlsPlaylist.files.find(f => f.resolution.id === resolution)
 556
 557           expect(file).to.exist
 558           expect(file.size).to.be.greaterThan(1)
 559
 560           if (resolution >= 720) {
 561             expect(file.fps).to.be.approximately(60, 2)
 562           } else {
 563             expect(file.fps).to.be.approximately(30, 2)
 564           }
 565
 566           const filename = basename(file.fileUrl)
 567           expect(filename).to.not.contain(video.uuid)
 568
 569           const segmentPath = servers[0].servers.buildDirectory(join('streaming-playlists', 'hls', video.uuid, filename))
 570
 571           const probe = await ffprobePromise(segmentPath)
 572           const videoStream = await getVideoStream(segmentPath, probe)
 573
 574           expect(probe.format.bit_rate).to.be.below(maxBitrateLimits[videoStream.height])
 575           expect(probe.format.bit_rate).to.be.at.least(minBitrateLimits[videoStream.height])
 576
 577           await makeRawRequest(file.torrentUrl, HttpStatusCode.OK_200)
 578           await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200)
 579         }
 580       }
 581     })
 582
 583     it('Should correctly have cleaned up the live files', async function () {
 584       this.timeout(30000)
 585
 586       await checkLiveCleanupAfterSave(servers[0], liveVideoId, [ 240, 360, 720 ])
 587     })
 588   })
 589
 590   describe('After a server restart', function () {
 591     let liveVideoId: string
 592     let liveVideoReplayId: string
 593
 594     async function createLiveWrapper (saveReplay: boolean) {
 595       const liveAttributes = {
 596         name: 'live video',
 597         channelId: servers[0].store.channel.id,
 598         privacy: VideoPrivacy.PUBLIC,
 599         saveReplay
 600       }
 601
 602       const { uuid } = await commands[0].create({ fields: liveAttributes })
 603       return uuid
 604     }
 605
 606     before(async function () {
 607       this.timeout(120000)
 608
 609       liveVideoId = await createLiveWrapper(false)
 610       liveVideoReplayId = await createLiveWrapper(true)
 611
 612       await Promise.all([
 613         commands[0].sendRTMPStreamInVideo({ videoId: liveVideoId }),
 614         commands[0].sendRTMPStreamInVideo({ videoId: liveVideoReplayId })
 615       ])
 616
 617       await Promise.all([
 618         commands[0].waitUntilPublished({ videoId: liveVideoId }),
 619         commands[0].waitUntilPublished({ videoId: liveVideoReplayId })
 620       ])
 621
 622       await commands[0].waitUntilSegmentGeneration({ videoUUID: liveVideoId, resolution: 0, segment: 2 })
 623       await commands[0].waitUntilSegmentGeneration({ videoUUID: liveVideoReplayId, resolution: 0, segment: 2 })
 624
 625       await killallServers([ servers[0] ])
 626       await servers[0].run()
 627
 628       await wait(5000)
 629     })
 630
 631     it('Should cleanup lives', async function () {
 632       this.timeout(60000)
 633
 634       await commands[0].waitUntilEnded({ videoId: liveVideoId })
 635     })
 636
 637     it('Should save a live replay', async function () {
 638       this.timeout(120000)
 639
 640       await commands[0].waitUntilPublished({ videoId: liveVideoReplayId })
 641     })
 642   })
 643
 644   after(async function () {
 645     await cleanupTests(servers)
 646   })
 647 })