server/tests/api/live/live.ts

   1 /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
   2
   3 import * as chai from 'chai'
   4 import { basename, join } from 'path'
   5 import { ffprobePromise, getVideoStream } from '@server/helpers/ffmpeg'
   6 import { checkLiveSegmentHash, checkResolutionsInMasterPlaylist, getAllFiles, testImage } from '@server/tests/shared'
   7 import { wait } from '@shared/core-utils'
   8 import {
   9   HttpStatusCode,
  10   LiveVideo,
  11   LiveVideoCreate,
  12   LiveVideoLatencyMode,
  13   VideoDetails,
  14   VideoPrivacy,
  15   VideoState,
  16   VideoStreamingPlaylistType
  17 } from '@shared/models'
  18 import {
  19   cleanupTests,
  20   createMultipleServers,
  21   doubleFollow,
  22   killallServers,
  23   LiveCommand,
  24   makeRawRequest,
  25   PeerTubeServer,
  26   sendRTMPStream,
  27   setAccessTokensToServers,
  28   setDefaultVideoChannel,
  29   stopFfmpeg,
  30   testFfmpegStreamError,
  31   waitJobs,
  32   waitUntilLivePublishedOnAllServers
  33 } from '@shared/server-commands'
  34
  35 const expect = chai.expect
  36
  37 describe('Test live', function () {
  38   let servers: PeerTubeServer[] = []
  39   let commands: LiveCommand[]
  40
  41   before(async function () {
  42     this.timeout(120000)
  43
  44     servers = await createMultipleServers(2)
  45
  46     // Get the access tokens
  47     await setAccessTokensToServers(servers)
  48     await setDefaultVideoChannel(servers)
  49
  50     await servers[0].config.updateCustomSubConfig({
  51       newConfig: {
  52         live: {
  53           enabled: true,
  54           allowReplay: true,
  55           latencySetting: {
  56             enabled: true
  57           },
  58           transcoding: {
  59             enabled: false
  60           }
  61         }
  62       }
  63     })
  64
  65     // Server 1 and server 2 follow each other
  66     await doubleFollow(servers[0], servers[1])
  67
  68     commands = servers.map(s => s.live)
  69   })
  70
  71   describe('Live creation, update and delete', function () {
  72     let liveVideoUUID: string
  73
  74     it('Should create a live with the appropriate parameters', async function () {
  75       this.timeout(20000)
  76
  77       const attributes: LiveVideoCreate = {
  78         category: 1,
  79         licence: 2,
  80         language: 'fr',
  81         description: 'super live description',
  82         support: 'support field',
  83         channelId: servers[0].store.channel.id,
  84         nsfw: false,
  85         waitTranscoding: false,
  86         name: 'my super live',
  87         tags: [ 'tag1', 'tag2' ],
  88         commentsEnabled: false,
  89         downloadEnabled: false,
  90         saveReplay: true,
  91         latencyMode: LiveVideoLatencyMode.SMALL_LATENCY,
  92         privacy: VideoPrivacy.PUBLIC,
  93         previewfile: 'video_short1-preview.webm.jpg',
  94         thumbnailfile: 'video_short1.webm.jpg'
  95       }
  96
  97       const live = await commands[0].create({ fields: attributes })
  98       liveVideoUUID = live.uuid
  99
 100       await waitJobs(servers)
 101
 102       for (const server of servers) {
 103         const video = await server.videos.get({ id: liveVideoUUID })
 104
 105         expect(video.category.id).to.equal(1)
 106         expect(video.licence.id).to.equal(2)
 107         expect(video.language.id).to.equal('fr')
 108         expect(video.description).to.equal('super live description')
 109         expect(video.support).to.equal('support field')
 110
 111         expect(video.channel.name).to.equal(servers[0].store.channel.name)
 112         expect(video.channel.host).to.equal(servers[0].store.channel.host)
 113
 114         expect(video.isLive).to.be.true
 115
 116         expect(video.nsfw).to.be.false
 117         expect(video.waitTranscoding).to.be.false
 118         expect(video.name).to.equal('my super live')
 119         expect(video.tags).to.deep.equal([ 'tag1', 'tag2' ])
 120         expect(video.commentsEnabled).to.be.false
 121         expect(video.downloadEnabled).to.be.false
 122         expect(video.privacy.id).to.equal(VideoPrivacy.PUBLIC)
 123
 124         await testImage(server.url, 'video_short1-preview.webm', video.previewPath)
 125         await testImage(server.url, 'video_short1.webm', video.thumbnailPath)
 126
 127         const live = await server.live.get({ videoId: liveVideoUUID })
 128
 129         if (server.url === servers[0].url) {
 130           expect(live.rtmpUrl).to.equal('rtmp://' + server.hostname + ':' + servers[0].rtmpPort + '/live')
 131           expect(live.streamKey).to.not.be.empty
 132         } else {
 133           expect(live.rtmpUrl).to.not.exist
 134           expect(live.streamKey).to.not.exist
 135         }
 136
 137         expect(live.saveReplay).to.be.true
 138         expect(live.latencyMode).to.equal(LiveVideoLatencyMode.SMALL_LATENCY)
 139       }
 140     })
 141
 142     it('Should have a default preview and thumbnail', async function () {
 143       this.timeout(20000)
 144
 145       const attributes: LiveVideoCreate = {
 146         name: 'default live thumbnail',
 147         channelId: servers[0].store.channel.id,
 148         privacy: VideoPrivacy.UNLISTED,
 149         nsfw: true
 150       }
 151
 152       const live = await commands[0].create({ fields: attributes })
 153       const videoId = live.uuid
 154
 155       await waitJobs(servers)
 156
 157       for (const server of servers) {
 158         const video = await server.videos.get({ id: videoId })
 159         expect(video.privacy.id).to.equal(VideoPrivacy.UNLISTED)
 160         expect(video.nsfw).to.be.true
 161
 162         await makeRawRequest(server.url + video.thumbnailPath, HttpStatusCode.OK_200)
 163         await makeRawRequest(server.url + video.previewPath, HttpStatusCode.OK_200)
 164       }
 165     })
 166
 167     it('Should not have the live listed since nobody streams into', async function () {
 168       for (const server of servers) {
 169         const { total, data } = await server.videos.list()
 170
 171         expect(total).to.equal(0)
 172         expect(data).to.have.lengthOf(0)
 173       }
 174     })
 175
 176     it('Should not be able to update a live of another server', async function () {
 177       await commands[1].update({ videoId: liveVideoUUID, fields: { saveReplay: false }, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
 178     })
 179
 180     it('Should update the live', async function () {
 181       this.timeout(10000)
 182
 183       await commands[0].update({ videoId: liveVideoUUID, fields: { saveReplay: false, latencyMode: LiveVideoLatencyMode.DEFAULT } })
 184       await waitJobs(servers)
 185     })
 186
 187     it('Have the live updated', async function () {
 188       for (const server of servers) {
 189         const live = await server.live.get({ videoId: liveVideoUUID })
 190
 191         if (server.url === servers[0].url) {
 192           expect(live.rtmpUrl).to.equal('rtmp://' + server.hostname + ':' + servers[0].rtmpPort + '/live')
 193           expect(live.streamKey).to.not.be.empty
 194         } else {
 195           expect(live.rtmpUrl).to.not.exist
 196           expect(live.streamKey).to.not.exist
 197         }
 198
 199         expect(live.saveReplay).to.be.false
 200         expect(live.latencyMode).to.equal(LiveVideoLatencyMode.DEFAULT)
 201       }
 202     })
 203
 204     it('Delete the live', async function () {
 205       this.timeout(10000)
 206
 207       await servers[0].videos.remove({ id: liveVideoUUID })
 208       await waitJobs(servers)
 209     })
 210
 211     it('Should have the live deleted', async function () {
 212       for (const server of servers) {
 213         await server.videos.get({ id: liveVideoUUID, expectedStatus: HttpStatusCode.NOT_FOUND_404 })
 214         await server.live.get({ videoId: liveVideoUUID, expectedStatus: HttpStatusCode.NOT_FOUND_404 })
 215       }
 216     })
 217   })
 218
 219   describe('Live filters', function () {
 220     let ffmpegCommand: any
 221     let liveVideoId: string
 222     let vodVideoId: string
 223
 224     before(async function () {
 225       this.timeout(240000)
 226
 227       vodVideoId = (await servers[0].videos.quickUpload({ name: 'vod video' })).uuid
 228
 229       const liveOptions = { name: 'live', privacy: VideoPrivacy.PUBLIC, channelId: servers[0].store.channel.id }
 230       const live = await commands[0].create({ fields: liveOptions })
 231       liveVideoId = live.uuid
 232
 233       ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoId })
 234       await waitUntilLivePublishedOnAllServers(servers, liveVideoId)
 235       await waitJobs(servers)
 236     })
 237
 238     it('Should only display lives', async function () {
 239       const { data, total } = await servers[0].videos.list({ isLive: true })
 240
 241       expect(total).to.equal(1)
 242       expect(data).to.have.lengthOf(1)
 243       expect(data[0].name).to.equal('live')
 244     })
 245
 246     it('Should not display lives', async function () {
 247       const { data, total } = await servers[0].videos.list({ isLive: false })
 248
 249       expect(total).to.equal(1)
 250       expect(data).to.have.lengthOf(1)
 251       expect(data[0].name).to.equal('vod video')
 252     })
 253
 254     it('Should display my lives', async function () {
 255       this.timeout(60000)
 256
 257       await stopFfmpeg(ffmpegCommand)
 258       await waitJobs(servers)
 259
 260       const { data } = await servers[0].videos.listMyVideos({ isLive: true })
 261
 262       const result = data.every(v => v.isLive)
 263       expect(result).to.be.true
 264     })
 265
 266     it('Should not display my lives', async function () {
 267       const { data } = await servers[0].videos.listMyVideos({ isLive: false })
 268
 269       const result = data.every(v => !v.isLive)
 270       expect(result).to.be.true
 271     })
 272
 273     after(async function () {
 274       await servers[0].videos.remove({ id: vodVideoId })
 275       await servers[0].videos.remove({ id: liveVideoId })
 276     })
 277   })
 278
 279   describe('Stream checks', function () {
 280     let liveVideo: LiveVideo & VideoDetails
 281     let rtmpUrl: string
 282
 283     before(function () {
 284       rtmpUrl = 'rtmp://' + servers[0].hostname + ':' + servers[0].rtmpPort + ''
 285     })
 286
 287     async function createLiveWrapper () {
 288       const liveAttributes = {
 289         name: 'user live',
 290         channelId: servers[0].store.channel.id,
 291         privacy: VideoPrivacy.PUBLIC,
 292         saveReplay: false
 293       }
 294
 295       const { uuid } = await commands[0].create({ fields: liveAttributes })
 296
 297       const live = await commands[0].get({ videoId: uuid })
 298       const video = await servers[0].videos.get({ id: uuid })
 299
 300       return Object.assign(video, live)
 301     }
 302
 303     it('Should not allow a stream without the appropriate path', async function () {
 304       this.timeout(60000)
 305
 306       liveVideo = await createLiveWrapper()
 307
 308       const command = sendRTMPStream({ rtmpBaseUrl: rtmpUrl + '/bad-live', streamKey: liveVideo.streamKey })
 309       await testFfmpegStreamError(command, true)
 310     })
 311
 312     it('Should not allow a stream without the appropriate stream key', async function () {
 313       this.timeout(60000)
 314
 315       const command = sendRTMPStream({ rtmpBaseUrl: rtmpUrl + '/live', streamKey: 'bad-stream-key' })
 316       await testFfmpegStreamError(command, true)
 317     })
 318
 319     it('Should succeed with the correct params', async function () {
 320       this.timeout(60000)
 321
 322       const command = sendRTMPStream({ rtmpBaseUrl: rtmpUrl + '/live', streamKey: liveVideo.streamKey })
 323       await testFfmpegStreamError(command, false)
 324     })
 325
 326     it('Should list this live now someone stream into it', async function () {
 327       for (const server of servers) {
 328         const { total, data } = await server.videos.list()
 329
 330         expect(total).to.equal(1)
 331         expect(data).to.have.lengthOf(1)
 332
 333         const video = data[0]
 334         expect(video.name).to.equal('user live')
 335         expect(video.isLive).to.be.true
 336       }
 337     })
 338
 339     it('Should not allow a stream on a live that was blacklisted', async function () {
 340       this.timeout(60000)
 341
 342       liveVideo = await createLiveWrapper()
 343
 344       await servers[0].blacklist.add({ videoId: liveVideo.uuid })
 345
 346       const command = sendRTMPStream({ rtmpBaseUrl: rtmpUrl + '/live', streamKey: liveVideo.streamKey })
 347       await testFfmpegStreamError(command, true)
 348     })
 349
 350     it('Should not allow a stream on a live that was deleted', async function () {
 351       this.timeout(60000)
 352
 353       liveVideo = await createLiveWrapper()
 354
 355       await servers[0].videos.remove({ id: liveVideo.uuid })
 356
 357       const command = sendRTMPStream({ rtmpBaseUrl: rtmpUrl + '/live', streamKey: liveVideo.streamKey })
 358       await testFfmpegStreamError(command, true)
 359     })
 360   })
 361
 362   describe('Live transcoding', function () {
 363     let liveVideoId: string
 364
 365     async function createLiveWrapper (saveReplay: boolean) {
 366       const liveAttributes = {
 367         name: 'live video',
 368         channelId: servers[0].store.channel.id,
 369         privacy: VideoPrivacy.PUBLIC,
 370         saveReplay
 371       }
 372
 373       const { uuid } = await commands[0].create({ fields: liveAttributes })
 374       return uuid
 375     }
 376
 377     async function testVideoResolutions (liveVideoId: string, resolutions: number[]) {
 378       for (const server of servers) {
 379         const { data } = await server.videos.list()
 380         expect(data.find(v => v.uuid === liveVideoId)).to.exist
 381
 382         const video = await server.videos.get({ id: liveVideoId })
 383
 384         expect(video.streamingPlaylists).to.have.lengthOf(1)
 385
 386         const hlsPlaylist = video.streamingPlaylists.find(s => s.type === VideoStreamingPlaylistType.HLS)
 387         expect(hlsPlaylist).to.exist
 388
 389         // Only finite files are displayed
 390         expect(hlsPlaylist.files).to.have.lengthOf(0)
 391
 392         await checkResolutionsInMasterPlaylist({ server, playlistUrl: hlsPlaylist.playlistUrl, resolutions })
 393
 394         for (let i = 0; i < resolutions.length; i++) {
 395           const segmentNum = 3
 396           const segmentName = `${i}-00000${segmentNum}.ts`
 397           await commands[0].waitUntilSegmentGeneration({ videoUUID: video.uuid, playlistNumber: i, segment: segmentNum })
 398
 399           const subPlaylist = await servers[0].streamingPlaylists.get({
 400             url: `${servers[0].url}/static/streaming-playlists/hls/${video.uuid}/${i}.m3u8`
 401           })
 402
 403           expect(subPlaylist).to.contain(segmentName)
 404
 405           const baseUrlAndPath = servers[0].url + '/static/streaming-playlists/hls'
 406           await checkLiveSegmentHash({
 407             server,
 408             baseUrlSegment: baseUrlAndPath,
 409             videoUUID: video.uuid,
 410             segmentName,
 411             hlsPlaylist
 412           })
 413         }
 414       }
 415     }
 416
 417     function updateConf (resolutions: number[]) {
 418       return servers[0].config.updateCustomSubConfig({
 419         newConfig: {
 420           live: {
 421             enabled: true,
 422             allowReplay: true,
 423             maxDuration: -1,
 424             transcoding: {
 425               enabled: true,
 426               resolutions: {
 427                 '144p': resolutions.includes(144),
 428                 '240p': resolutions.includes(240),
 429                 '360p': resolutions.includes(360),
 430                 '480p': resolutions.includes(480),
 431                 '720p': resolutions.includes(720),
 432                 '1080p': resolutions.includes(1080),
 433                 '2160p': resolutions.includes(2160)
 434               }
 435             }
 436           }
 437         }
 438       })
 439     }
 440
 441     before(async function () {
 442       await updateConf([])
 443     })
 444
 445     it('Should enable transcoding without additional resolutions', async function () {
 446       this.timeout(120000)
 447
 448       liveVideoId = await createLiveWrapper(false)
 449
 450       const ffmpegCommand = await commands[0].sendRTMPStreamInVideo({ videoId: liveVideoId })
 451       await waitUntilLivePublishedOnAllServers(servers, liveVideoId)
 452       await waitJobs(servers)
 453
 454       await testVideoResolutions(liveVideoId, [ 720 ])
 455
 456       await stopFfmpeg(ffmpegCommand)
 457     })
 458
 459     it('Should enable transcoding with some resolutions', async function () {
 460       this.timeout(120000)
 461
 462       const resolutions = [ 240, 480 ]
 463       await updateConf(resolutions)
 464       liveVideoId = await createLiveWrapper(false)
 465
 466       const ffmpegCommand = await commands[0].sendRTMPStreamInVideo({ videoId: liveVideoId })
 467       await waitUntilLivePublishedOnAllServers(servers, liveVideoId)
 468       await waitJobs(servers)
 469
 470       await testVideoResolutions(liveVideoId, resolutions.concat([ 720 ]))
 471
 472       await stopFfmpeg(ffmpegCommand)
 473     })
 474
 475     it('Should correctly set the appropriate bitrate depending on the input', async function () {
 476       this.timeout(120000)
 477
 478       liveVideoId = await createLiveWrapper(false)
 479
 480       const ffmpegCommand = await commands[0].sendRTMPStreamInVideo({
 481         videoId: liveVideoId,
 482         fixtureName: 'video_short.mp4',
 483         copyCodecs: true
 484       })
 485       await waitUntilLivePublishedOnAllServers(servers, liveVideoId)
 486       await waitJobs(servers)
 487
 488       const video = await servers[0].videos.get({ id: liveVideoId })
 489
 490       const masterPlaylist = video.streamingPlaylists[0].playlistUrl
 491       const probe = await ffprobePromise(masterPlaylist)
 492
 493       const bitrates = probe.streams.map(s => parseInt(s.tags.variant_bitrate))
 494       for (const bitrate of bitrates) {
 495         expect(bitrate).to.exist
 496         expect(isNaN(bitrate)).to.be.false
 497         expect(bitrate).to.be.below(61_000_000) // video_short.mp4 bitrate
 498       }
 499
 500       await stopFfmpeg(ffmpegCommand)
 501     })
 502
 503     it('Should enable transcoding with some resolutions and correctly save them', async function () {
 504       this.timeout(400_000)
 505
 506       const resolutions = [ 240, 360, 720 ]
 507
 508       await updateConf(resolutions)
 509       liveVideoId = await createLiveWrapper(true)
 510
 511       const ffmpegCommand = await commands[0].sendRTMPStreamInVideo({ videoId: liveVideoId, fixtureName: 'video_short2.webm' })
 512       await waitUntilLivePublishedOnAllServers(servers, liveVideoId)
 513       await waitJobs(servers)
 514
 515       await testVideoResolutions(liveVideoId, resolutions)
 516
 517       await stopFfmpeg(ffmpegCommand)
 518       await commands[0].waitUntilEnded({ videoId: liveVideoId })
 519
 520       await waitJobs(servers)
 521
 522       await waitUntilLivePublishedOnAllServers(servers, liveVideoId)
 523
 524       const maxBitrateLimits = {
 525         720: 6500 * 1000, // 60FPS
 526         360: 1250 * 1000,
 527         240: 700 * 1000
 528       }
 529
 530       const minBitrateLimits = {
 531         720: 5500 * 1000,
 532         360: 1000 * 1000,
 533         240: 550 * 1000
 534       }
 535
 536       for (const server of servers) {
 537         const video = await server.videos.get({ id: liveVideoId })
 538
 539         expect(video.state.id).to.equal(VideoState.PUBLISHED)
 540         expect(video.duration).to.be.greaterThan(1)
 541         expect(video.files).to.have.lengthOf(0)
 542
 543         const hlsPlaylist = video.streamingPlaylists.find(s => s.type === VideoStreamingPlaylistType.HLS)
 544         await makeRawRequest(hlsPlaylist.playlistUrl, HttpStatusCode.OK_200)
 545         await makeRawRequest(hlsPlaylist.segmentsSha256Url, HttpStatusCode.OK_200)
 546
 547         // We should have generated random filenames
 548         expect(basename(hlsPlaylist.playlistUrl)).to.not.equal('master.m3u8')
 549         expect(basename(hlsPlaylist.segmentsSha256Url)).to.not.equal('segments-sha256.json')
 550
 551         expect(hlsPlaylist.files).to.have.lengthOf(resolutions.length)
 552
 553         for (const resolution of resolutions) {
 554           const file = hlsPlaylist.files.find(f => f.resolution.id === resolution)
 555
 556           expect(file).to.exist
 557           expect(file.size).to.be.greaterThan(1)
 558
 559           if (resolution >= 720) {
 560             expect(file.fps).to.be.approximately(60, 10)
 561           } else {
 562             expect(file.fps).to.be.approximately(30, 2)
 563           }
 564
 565           const filename = basename(file.fileUrl)
 566           expect(filename).to.not.contain(video.uuid)
 567
 568           const segmentPath = servers[0].servers.buildDirectory(join('streaming-playlists', 'hls', video.uuid, filename))
 569
 570           const probe = await ffprobePromise(segmentPath)
 571           const videoStream = await getVideoStream(segmentPath, probe)
 572
 573           expect(probe.format.bit_rate).to.be.below(maxBitrateLimits[videoStream.height])
 574           expect(probe.format.bit_rate).to.be.at.least(minBitrateLimits[videoStream.height])
 575
 576           await makeRawRequest(file.torrentUrl, HttpStatusCode.OK_200)
 577           await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200)
 578         }
 579       }
 580     })
 581
 582     it('Should not generate an upper resolution than original file', async function () {
 583       this.timeout(400_000)
 584
 585       const resolutions = [ 240, 480 ]
 586       await updateConf(resolutions)
 587
 588       await servers[0].config.updateExistingSubConfig({
 589         newConfig: {
 590           live: {
 591             transcoding: {
 592               alwaysTranscodeOriginalResolution: false
 593             }
 594           }
 595         }
 596       })
 597
 598       liveVideoId = await createLiveWrapper(true)
 599
 600       const ffmpegCommand = await commands[0].sendRTMPStreamInVideo({ videoId: liveVideoId, fixtureName: 'video_short2.webm' })
 601       await waitUntilLivePublishedOnAllServers(servers, liveVideoId)
 602       await waitJobs(servers)
 603
 604       await testVideoResolutions(liveVideoId, resolutions)
 605
 606       await stopFfmpeg(ffmpegCommand)
 607       await commands[0].waitUntilEnded({ videoId: liveVideoId })
 608
 609       await waitJobs(servers)
 610
 611       await waitUntilLivePublishedOnAllServers(servers, liveVideoId)
 612
 613       const video = await servers[0].videos.get({ id: liveVideoId })
 614       const hlsFiles = video.streamingPlaylists[0].files
 615
 616       expect(video.files).to.have.lengthOf(0)
 617       expect(hlsFiles).to.have.lengthOf(resolutions.length)
 618
 619       // eslint-disable-next-line @typescript-eslint/require-array-sort-compare
 620       expect(getAllFiles(video).map(f => f.resolution.id).sort()).to.deep.equal(resolutions)
 621     })
 622
 623     it('Should only keep the original resolution if all resolutions are disabled', async function () {
 624       this.timeout(600_000)
 625
 626       await updateConf([])
 627       liveVideoId = await createLiveWrapper(true)
 628
 629       const ffmpegCommand = await commands[0].sendRTMPStreamInVideo({ videoId: liveVideoId, fixtureName: 'video_short2.webm' })
 630       await waitUntilLivePublishedOnAllServers(servers, liveVideoId)
 631       await waitJobs(servers)
 632
 633       await testVideoResolutions(liveVideoId, [ 720 ])
 634
 635       await stopFfmpeg(ffmpegCommand)
 636       await commands[0].waitUntilEnded({ videoId: liveVideoId })
 637
 638       await waitJobs(servers)
 639
 640       await waitUntilLivePublishedOnAllServers(servers, liveVideoId)
 641
 642       const video = await servers[0].videos.get({ id: liveVideoId })
 643       const hlsFiles = video.streamingPlaylists[0].files
 644
 645       expect(video.files).to.have.lengthOf(0)
 646       expect(hlsFiles).to.have.lengthOf(1)
 647
 648       expect(hlsFiles[0].resolution.id).to.equal(720)
 649     })
 650   })
 651
 652   describe('After a server restart', function () {
 653     let liveVideoId: string
 654     let liveVideoReplayId: string
 655     let permanentLiveVideoReplayId: string
 656
 657     let permanentLiveReplayName: string
 658
 659     let beforeServerRestart: Date
 660
 661     async function createLiveWrapper (options: { saveReplay: boolean, permanent: boolean }) {
 662       const liveAttributes: LiveVideoCreate = {
 663         name: 'live video',
 664         channelId: servers[0].store.channel.id,
 665         privacy: VideoPrivacy.PUBLIC,
 666         saveReplay: options.saveReplay,
 667         permanentLive: options.permanent
 668       }
 669
 670       const { uuid } = await commands[0].create({ fields: liveAttributes })
 671       return uuid
 672     }
 673
 674     before(async function () {
 675       this.timeout(600_000)
 676
 677       liveVideoId = await createLiveWrapper({ saveReplay: false, permanent: false })
 678       liveVideoReplayId = await createLiveWrapper({ saveReplay: true, permanent: false })
 679       permanentLiveVideoReplayId = await createLiveWrapper({ saveReplay: true, permanent: true })
 680
 681       await Promise.all([
 682         commands[0].sendRTMPStreamInVideo({ videoId: liveVideoId }),
 683         commands[0].sendRTMPStreamInVideo({ videoId: permanentLiveVideoReplayId }),
 684         commands[0].sendRTMPStreamInVideo({ videoId: liveVideoReplayId })
 685       ])
 686
 687       await Promise.all([
 688         commands[0].waitUntilPublished({ videoId: liveVideoId }),
 689         commands[0].waitUntilPublished({ videoId: permanentLiveVideoReplayId }),
 690         commands[0].waitUntilPublished({ videoId: liveVideoReplayId })
 691       ])
 692
 693       await commands[0].waitUntilSegmentGeneration({ videoUUID: liveVideoId, playlistNumber: 0, segment: 2 })
 694       await commands[0].waitUntilSegmentGeneration({ videoUUID: liveVideoReplayId, playlistNumber: 0, segment: 2 })
 695       await commands[0].waitUntilSegmentGeneration({ videoUUID: permanentLiveVideoReplayId, playlistNumber: 0, segment: 2 })
 696
 697       {
 698         const video = await servers[0].videos.get({ id: permanentLiveVideoReplayId })
 699         permanentLiveReplayName = video.name + ' - ' + new Date(video.publishedAt).toLocaleString()
 700       }
 701
 702       await killallServers([ servers[0] ])
 703
 704       beforeServerRestart = new Date()
 705       await servers[0].run()
 706
 707       await wait(5000)
 708       await waitJobs(servers)
 709     })
 710
 711     it('Should cleanup lives', async function () {
 712       this.timeout(60000)
 713
 714       await commands[0].waitUntilEnded({ videoId: liveVideoId })
 715       await commands[0].waitUntilWaiting({ videoId: permanentLiveVideoReplayId })
 716     })
 717
 718     it('Should save a non permanent live replay', async function () {
 719       this.timeout(240000)
 720
 721       await commands[0].waitUntilPublished({ videoId: liveVideoReplayId })
 722
 723       const session = await commands[0].getReplaySession({ videoId: liveVideoReplayId })
 724       expect(session.endDate).to.exist
 725       expect(new Date(session.endDate)).to.be.above(beforeServerRestart)
 726     })
 727
 728     it('Should have saved a permanent live replay', async function () {
 729       this.timeout(120000)
 730
 731       const { data } = await servers[0].videos.listMyVideos({ sort: '-publishedAt' })
 732       expect(data.find(v => v.name === permanentLiveReplayName)).to.exist
 733     })
 734   })
 735
 736   after(async function () {
 737     await cleanupTests(servers)
 738   })
 739 })