server/tests/api/live/live.ts

   1 /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
   2
   3 import { expect } from 'chai'
   4 import { basename, join } from 'path'
   5 import { ffprobePromise, getVideoStream } from '@server/helpers/ffmpeg'
   6 import { checkLiveSegmentHash, checkResolutionsInMasterPlaylist, testImage } from '@server/tests/shared'
   7 import { getAllFiles, wait } from '@shared/core-utils'
   8 import {
   9   HttpStatusCode,
  10   LiveVideo,
  11   LiveVideoCreate,
  12   LiveVideoLatencyMode,
  13   VideoDetails,
  14   VideoPrivacy,
  15   VideoState,
  16   VideoStreamingPlaylistType
  17 } from '@shared/models'
  18 import {
  19   cleanupTests,
  20   createMultipleServers,
  21   doubleFollow,
  22   killallServers,
  23   LiveCommand,
  24   makeRawRequest,
  25   PeerTubeServer,
  26   sendRTMPStream,
  27   setAccessTokensToServers,
  28   setDefaultVideoChannel,
  29   stopFfmpeg,
  30   testFfmpegStreamError,
  31   waitJobs,
  32   waitUntilLivePublishedOnAllServers
  33 } from '@shared/server-commands'
  34
  35 describe('Test live', function () {
  36   let servers: PeerTubeServer[] = []
  37   let commands: LiveCommand[]
  38
  39   before(async function () {
  40     this.timeout(120000)
  41
  42     servers = await createMultipleServers(2)
  43
  44     // Get the access tokens
  45     await setAccessTokensToServers(servers)
  46     await setDefaultVideoChannel(servers)
  47
  48     await servers[0].config.updateCustomSubConfig({
  49       newConfig: {
  50         live: {
  51           enabled: true,
  52           allowReplay: true,
  53           latencySetting: {
  54             enabled: true
  55           },
  56           transcoding: {
  57             enabled: false
  58           }
  59         }
  60       }
  61     })
  62
  63     // Server 1 and server 2 follow each other
  64     await doubleFollow(servers[0], servers[1])
  65
  66     commands = servers.map(s => s.live)
  67   })
  68
  69   describe('Live creation, update and delete', function () {
  70     let liveVideoUUID: string
  71
  72     it('Should create a live with the appropriate parameters', async function () {
  73       this.timeout(20000)
  74
  75       const attributes: LiveVideoCreate = {
  76         category: 1,
  77         licence: 2,
  78         language: 'fr',
  79         description: 'super live description',
  80         support: 'support field',
  81         channelId: servers[0].store.channel.id,
  82         nsfw: false,
  83         waitTranscoding: false,
  84         name: 'my super live',
  85         tags: [ 'tag1', 'tag2' ],
  86         commentsEnabled: false,
  87         downloadEnabled: false,
  88         saveReplay: true,
  89         latencyMode: LiveVideoLatencyMode.SMALL_LATENCY,
  90         privacy: VideoPrivacy.PUBLIC,
  91         previewfile: 'video_short1-preview.webm.jpg',
  92         thumbnailfile: 'video_short1.webm.jpg'
  93       }
  94
  95       const live = await commands[0].create({ fields: attributes })
  96       liveVideoUUID = live.uuid
  97
  98       await waitJobs(servers)
  99
 100       for (const server of servers) {
 101         const video = await server.videos.get({ id: liveVideoUUID })
 102
 103         expect(video.category.id).to.equal(1)
 104         expect(video.licence.id).to.equal(2)
 105         expect(video.language.id).to.equal('fr')
 106         expect(video.description).to.equal('super live description')
 107         expect(video.support).to.equal('support field')
 108
 109         expect(video.channel.name).to.equal(servers[0].store.channel.name)
 110         expect(video.channel.host).to.equal(servers[0].store.channel.host)
 111
 112         expect(video.isLive).to.be.true
 113
 114         expect(video.nsfw).to.be.false
 115         expect(video.waitTranscoding).to.be.false
 116         expect(video.name).to.equal('my super live')
 117         expect(video.tags).to.deep.equal([ 'tag1', 'tag2' ])
 118         expect(video.commentsEnabled).to.be.false
 119         expect(video.downloadEnabled).to.be.false
 120         expect(video.privacy.id).to.equal(VideoPrivacy.PUBLIC)
 121
 122         await testImage(server.url, 'video_short1-preview.webm', video.previewPath)
 123         await testImage(server.url, 'video_short1.webm', video.thumbnailPath)
 124
 125         const live = await server.live.get({ videoId: liveVideoUUID })
 126
 127         if (server.url === servers[0].url) {
 128           expect(live.rtmpUrl).to.equal('rtmp://' + server.hostname + ':' + servers[0].rtmpPort + '/live')
 129           expect(live.streamKey).to.not.be.empty
 130         } else {
 131           expect(live.rtmpUrl).to.not.exist
 132           expect(live.streamKey).to.not.exist
 133         }
 134
 135         expect(live.saveReplay).to.be.true
 136         expect(live.latencyMode).to.equal(LiveVideoLatencyMode.SMALL_LATENCY)
 137       }
 138     })
 139
 140     it('Should have a default preview and thumbnail', async function () {
 141       this.timeout(20000)
 142
 143       const attributes: LiveVideoCreate = {
 144         name: 'default live thumbnail',
 145         channelId: servers[0].store.channel.id,
 146         privacy: VideoPrivacy.UNLISTED,
 147         nsfw: true
 148       }
 149
 150       const live = await commands[0].create({ fields: attributes })
 151       const videoId = live.uuid
 152
 153       await waitJobs(servers)
 154
 155       for (const server of servers) {
 156         const video = await server.videos.get({ id: videoId })
 157         expect(video.privacy.id).to.equal(VideoPrivacy.UNLISTED)
 158         expect(video.nsfw).to.be.true
 159
 160         await makeRawRequest(server.url + video.thumbnailPath, HttpStatusCode.OK_200)
 161         await makeRawRequest(server.url + video.previewPath, HttpStatusCode.OK_200)
 162       }
 163     })
 164
 165     it('Should not have the live listed since nobody streams into', async function () {
 166       for (const server of servers) {
 167         const { total, data } = await server.videos.list()
 168
 169         expect(total).to.equal(0)
 170         expect(data).to.have.lengthOf(0)
 171       }
 172     })
 173
 174     it('Should not be able to update a live of another server', async function () {
 175       await commands[1].update({ videoId: liveVideoUUID, fields: { saveReplay: false }, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
 176     })
 177
 178     it('Should update the live', async function () {
 179       this.timeout(10000)
 180
 181       await commands[0].update({ videoId: liveVideoUUID, fields: { saveReplay: false, latencyMode: LiveVideoLatencyMode.DEFAULT } })
 182       await waitJobs(servers)
 183     })
 184
 185     it('Have the live updated', async function () {
 186       for (const server of servers) {
 187         const live = await server.live.get({ videoId: liveVideoUUID })
 188
 189         if (server.url === servers[0].url) {
 190           expect(live.rtmpUrl).to.equal('rtmp://' + server.hostname + ':' + servers[0].rtmpPort + '/live')
 191           expect(live.streamKey).to.not.be.empty
 192         } else {
 193           expect(live.rtmpUrl).to.not.exist
 194           expect(live.streamKey).to.not.exist
 195         }
 196
 197         expect(live.saveReplay).to.be.false
 198         expect(live.latencyMode).to.equal(LiveVideoLatencyMode.DEFAULT)
 199       }
 200     })
 201
 202     it('Delete the live', async function () {
 203       this.timeout(10000)
 204
 205       await servers[0].videos.remove({ id: liveVideoUUID })
 206       await waitJobs(servers)
 207     })
 208
 209     it('Should have the live deleted', async function () {
 210       for (const server of servers) {
 211         await server.videos.get({ id: liveVideoUUID, expectedStatus: HttpStatusCode.NOT_FOUND_404 })
 212         await server.live.get({ videoId: liveVideoUUID, expectedStatus: HttpStatusCode.NOT_FOUND_404 })
 213       }
 214     })
 215   })
 216
 217   describe('Live filters', function () {
 218     let ffmpegCommand: any
 219     let liveVideoId: string
 220     let vodVideoId: string
 221
 222     before(async function () {
 223       this.timeout(240000)
 224
 225       vodVideoId = (await servers[0].videos.quickUpload({ name: 'vod video' })).uuid
 226
 227       const liveOptions = { name: 'live', privacy: VideoPrivacy.PUBLIC, channelId: servers[0].store.channel.id }
 228       const live = await commands[0].create({ fields: liveOptions })
 229       liveVideoId = live.uuid
 230
 231       ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoId })
 232       await waitUntilLivePublishedOnAllServers(servers, liveVideoId)
 233       await waitJobs(servers)
 234     })
 235
 236     it('Should only display lives', async function () {
 237       const { data, total } = await servers[0].videos.list({ isLive: true })
 238
 239       expect(total).to.equal(1)
 240       expect(data).to.have.lengthOf(1)
 241       expect(data[0].name).to.equal('live')
 242     })
 243
 244     it('Should not display lives', async function () {
 245       const { data, total } = await servers[0].videos.list({ isLive: false })
 246
 247       expect(total).to.equal(1)
 248       expect(data).to.have.lengthOf(1)
 249       expect(data[0].name).to.equal('vod video')
 250     })
 251
 252     it('Should display my lives', async function () {
 253       this.timeout(60000)
 254
 255       await stopFfmpeg(ffmpegCommand)
 256       await waitJobs(servers)
 257
 258       const { data } = await servers[0].videos.listMyVideos({ isLive: true })
 259
 260       const result = data.every(v => v.isLive)
 261       expect(result).to.be.true
 262     })
 263
 264     it('Should not display my lives', async function () {
 265       const { data } = await servers[0].videos.listMyVideos({ isLive: false })
 266
 267       const result = data.every(v => !v.isLive)
 268       expect(result).to.be.true
 269     })
 270
 271     after(async function () {
 272       await servers[0].videos.remove({ id: vodVideoId })
 273       await servers[0].videos.remove({ id: liveVideoId })
 274     })
 275   })
 276
 277   describe('Stream checks', function () {
 278     let liveVideo: LiveVideo & VideoDetails
 279     let rtmpUrl: string
 280
 281     before(function () {
 282       rtmpUrl = 'rtmp://' + servers[0].hostname + ':' + servers[0].rtmpPort + ''
 283     })
 284
 285     async function createLiveWrapper () {
 286       const liveAttributes = {
 287         name: 'user live',
 288         channelId: servers[0].store.channel.id,
 289         privacy: VideoPrivacy.PUBLIC,
 290         saveReplay: false
 291       }
 292
 293       const { uuid } = await commands[0].create({ fields: liveAttributes })
 294
 295       const live = await commands[0].get({ videoId: uuid })
 296       const video = await servers[0].videos.get({ id: uuid })
 297
 298       return Object.assign(video, live)
 299     }
 300
 301     it('Should not allow a stream without the appropriate path', async function () {
 302       this.timeout(60000)
 303
 304       liveVideo = await createLiveWrapper()
 305
 306       const command = sendRTMPStream({ rtmpBaseUrl: rtmpUrl + '/bad-live', streamKey: liveVideo.streamKey })
 307       await testFfmpegStreamError(command, true)
 308     })
 309
 310     it('Should not allow a stream without the appropriate stream key', async function () {
 311       this.timeout(60000)
 312
 313       const command = sendRTMPStream({ rtmpBaseUrl: rtmpUrl + '/live', streamKey: 'bad-stream-key' })
 314       await testFfmpegStreamError(command, true)
 315     })
 316
 317     it('Should succeed with the correct params', async function () {
 318       this.timeout(60000)
 319
 320       const command = sendRTMPStream({ rtmpBaseUrl: rtmpUrl + '/live', streamKey: liveVideo.streamKey })
 321       await testFfmpegStreamError(command, false)
 322     })
 323
 324     it('Should list this live now someone stream into it', async function () {
 325       for (const server of servers) {
 326         const { total, data } = await server.videos.list()
 327
 328         expect(total).to.equal(1)
 329         expect(data).to.have.lengthOf(1)
 330
 331         const video = data[0]
 332         expect(video.name).to.equal('user live')
 333         expect(video.isLive).to.be.true
 334       }
 335     })
 336
 337     it('Should not allow a stream on a live that was blacklisted', async function () {
 338       this.timeout(60000)
 339
 340       liveVideo = await createLiveWrapper()
 341
 342       await servers[0].blacklist.add({ videoId: liveVideo.uuid })
 343
 344       const command = sendRTMPStream({ rtmpBaseUrl: rtmpUrl + '/live', streamKey: liveVideo.streamKey })
 345       await testFfmpegStreamError(command, true)
 346     })
 347
 348     it('Should not allow a stream on a live that was deleted', async function () {
 349       this.timeout(60000)
 350
 351       liveVideo = await createLiveWrapper()
 352
 353       await servers[0].videos.remove({ id: liveVideo.uuid })
 354
 355       const command = sendRTMPStream({ rtmpBaseUrl: rtmpUrl + '/live', streamKey: liveVideo.streamKey })
 356       await testFfmpegStreamError(command, true)
 357     })
 358   })
 359
 360   describe('Live transcoding', function () {
 361     let liveVideoId: string
 362
 363     async function createLiveWrapper (saveReplay: boolean) {
 364       const liveAttributes = {
 365         name: 'live video',
 366         channelId: servers[0].store.channel.id,
 367         privacy: VideoPrivacy.PUBLIC,
 368         saveReplay
 369       }
 370
 371       const { uuid } = await commands[0].create({ fields: liveAttributes })
 372       return uuid
 373     }
 374
 375     async function testVideoResolutions (liveVideoId: string, resolutions: number[]) {
 376       for (const server of servers) {
 377         const { data } = await server.videos.list()
 378         expect(data.find(v => v.uuid === liveVideoId)).to.exist
 379
 380         const video = await server.videos.get({ id: liveVideoId })
 381
 382         expect(video.streamingPlaylists).to.have.lengthOf(1)
 383
 384         const hlsPlaylist = video.streamingPlaylists.find(s => s.type === VideoStreamingPlaylistType.HLS)
 385         expect(hlsPlaylist).to.exist
 386
 387         // Only finite files are displayed
 388         expect(hlsPlaylist.files).to.have.lengthOf(0)
 389
 390         await checkResolutionsInMasterPlaylist({ server, playlistUrl: hlsPlaylist.playlistUrl, resolutions })
 391
 392         for (let i = 0; i < resolutions.length; i++) {
 393           const segmentNum = 3
 394           const segmentName = `${i}-00000${segmentNum}.ts`
 395           await commands[0].waitUntilSegmentGeneration({ videoUUID: video.uuid, playlistNumber: i, segment: segmentNum })
 396
 397           const subPlaylist = await servers[0].streamingPlaylists.get({
 398             url: `${servers[0].url}/static/streaming-playlists/hls/${video.uuid}/${i}.m3u8`
 399           })
 400
 401           expect(subPlaylist).to.contain(segmentName)
 402
 403           const baseUrlAndPath = servers[0].url + '/static/streaming-playlists/hls'
 404           await checkLiveSegmentHash({
 405             server,
 406             baseUrlSegment: baseUrlAndPath,
 407             videoUUID: video.uuid,
 408             segmentName,
 409             hlsPlaylist
 410           })
 411         }
 412       }
 413     }
 414
 415     function updateConf (resolutions: number[]) {
 416       return servers[0].config.updateCustomSubConfig({
 417         newConfig: {
 418           live: {
 419             enabled: true,
 420             allowReplay: true,
 421             maxDuration: -1,
 422             transcoding: {
 423               enabled: true,
 424               resolutions: {
 425                 '144p': resolutions.includes(144),
 426                 '240p': resolutions.includes(240),
 427                 '360p': resolutions.includes(360),
 428                 '480p': resolutions.includes(480),
 429                 '720p': resolutions.includes(720),
 430                 '1080p': resolutions.includes(1080),
 431                 '2160p': resolutions.includes(2160)
 432               }
 433             }
 434           }
 435         }
 436       })
 437     }
 438
 439     before(async function () {
 440       await updateConf([])
 441     })
 442
 443     it('Should enable transcoding without additional resolutions', async function () {
 444       this.timeout(120000)
 445
 446       liveVideoId = await createLiveWrapper(false)
 447
 448       const ffmpegCommand = await commands[0].sendRTMPStreamInVideo({ videoId: liveVideoId })
 449       await waitUntilLivePublishedOnAllServers(servers, liveVideoId)
 450       await waitJobs(servers)
 451
 452       await testVideoResolutions(liveVideoId, [ 720 ])
 453
 454       await stopFfmpeg(ffmpegCommand)
 455     })
 456
 457     it('Should enable transcoding with some resolutions', async function () {
 458       this.timeout(120000)
 459
 460       const resolutions = [ 240, 480 ]
 461       await updateConf(resolutions)
 462       liveVideoId = await createLiveWrapper(false)
 463
 464       const ffmpegCommand = await commands[0].sendRTMPStreamInVideo({ videoId: liveVideoId })
 465       await waitUntilLivePublishedOnAllServers(servers, liveVideoId)
 466       await waitJobs(servers)
 467
 468       await testVideoResolutions(liveVideoId, resolutions.concat([ 720 ]))
 469
 470       await stopFfmpeg(ffmpegCommand)
 471     })
 472
 473     it('Should correctly set the appropriate bitrate depending on the input', async function () {
 474       this.timeout(120000)
 475
 476       liveVideoId = await createLiveWrapper(false)
 477
 478       const ffmpegCommand = await commands[0].sendRTMPStreamInVideo({
 479         videoId: liveVideoId,
 480         fixtureName: 'video_short.mp4',
 481         copyCodecs: true
 482       })
 483       await waitUntilLivePublishedOnAllServers(servers, liveVideoId)
 484       await waitJobs(servers)
 485
 486       const video = await servers[0].videos.get({ id: liveVideoId })
 487
 488       const masterPlaylist = video.streamingPlaylists[0].playlistUrl
 489       const probe = await ffprobePromise(masterPlaylist)
 490
 491       const bitrates = probe.streams.map(s => parseInt(s.tags.variant_bitrate))
 492       for (const bitrate of bitrates) {
 493         expect(bitrate).to.exist
 494         expect(isNaN(bitrate)).to.be.false
 495         expect(bitrate).to.be.below(61_000_000) // video_short.mp4 bitrate
 496       }
 497
 498       await stopFfmpeg(ffmpegCommand)
 499     })
 500
 501     it('Should enable transcoding with some resolutions and correctly save them', async function () {
 502       this.timeout(400_000)
 503
 504       const resolutions = [ 240, 360, 720 ]
 505
 506       await updateConf(resolutions)
 507       liveVideoId = await createLiveWrapper(true)
 508
 509       const ffmpegCommand = await commands[0].sendRTMPStreamInVideo({ videoId: liveVideoId, fixtureName: 'video_short2.webm' })
 510       await waitUntilLivePublishedOnAllServers(servers, liveVideoId)
 511       await waitJobs(servers)
 512
 513       await testVideoResolutions(liveVideoId, resolutions)
 514
 515       await stopFfmpeg(ffmpegCommand)
 516       await commands[0].waitUntilEnded({ videoId: liveVideoId })
 517
 518       await waitJobs(servers)
 519
 520       await waitUntilLivePublishedOnAllServers(servers, liveVideoId)
 521
 522       const maxBitrateLimits = {
 523         720: 6500 * 1000, // 60FPS
 524         360: 1250 * 1000,
 525         240: 700 * 1000
 526       }
 527
 528       const minBitrateLimits = {
 529         720: 5500 * 1000,
 530         360: 1000 * 1000,
 531         240: 550 * 1000
 532       }
 533
 534       for (const server of servers) {
 535         const video = await server.videos.get({ id: liveVideoId })
 536
 537         expect(video.state.id).to.equal(VideoState.PUBLISHED)
 538         expect(video.duration).to.be.greaterThan(1)
 539         expect(video.files).to.have.lengthOf(0)
 540
 541         const hlsPlaylist = video.streamingPlaylists.find(s => s.type === VideoStreamingPlaylistType.HLS)
 542         await makeRawRequest(hlsPlaylist.playlistUrl, HttpStatusCode.OK_200)
 543         await makeRawRequest(hlsPlaylist.segmentsSha256Url, HttpStatusCode.OK_200)
 544
 545         // We should have generated random filenames
 546         expect(basename(hlsPlaylist.playlistUrl)).to.not.equal('master.m3u8')
 547         expect(basename(hlsPlaylist.segmentsSha256Url)).to.not.equal('segments-sha256.json')
 548
 549         expect(hlsPlaylist.files).to.have.lengthOf(resolutions.length)
 550
 551         for (const resolution of resolutions) {
 552           const file = hlsPlaylist.files.find(f => f.resolution.id === resolution)
 553
 554           expect(file).to.exist
 555           expect(file.size).to.be.greaterThan(1)
 556
 557           if (resolution >= 720) {
 558             expect(file.fps).to.be.approximately(60, 10)
 559           } else {
 560             expect(file.fps).to.be.approximately(30, 2)
 561           }
 562
 563           const filename = basename(file.fileUrl)
 564           expect(filename).to.not.contain(video.uuid)
 565
 566           const segmentPath = servers[0].servers.buildDirectory(join('streaming-playlists', 'hls', video.uuid, filename))
 567
 568           const probe = await ffprobePromise(segmentPath)
 569           const videoStream = await getVideoStream(segmentPath, probe)
 570
 571           expect(probe.format.bit_rate).to.be.below(maxBitrateLimits[videoStream.height])
 572           expect(probe.format.bit_rate).to.be.at.least(minBitrateLimits[videoStream.height])
 573
 574           await makeRawRequest(file.torrentUrl, HttpStatusCode.OK_200)
 575           await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200)
 576         }
 577       }
 578     })
 579
 580     it('Should not generate an upper resolution than original file', async function () {
 581       this.timeout(400_000)
 582
 583       const resolutions = [ 240, 480 ]
 584       await updateConf(resolutions)
 585
 586       await servers[0].config.updateExistingSubConfig({
 587         newConfig: {
 588           live: {
 589             transcoding: {
 590               alwaysTranscodeOriginalResolution: false
 591             }
 592           }
 593         }
 594       })
 595
 596       liveVideoId = await createLiveWrapper(true)
 597
 598       const ffmpegCommand = await commands[0].sendRTMPStreamInVideo({ videoId: liveVideoId, fixtureName: 'video_short2.webm' })
 599       await waitUntilLivePublishedOnAllServers(servers, liveVideoId)
 600       await waitJobs(servers)
 601
 602       await testVideoResolutions(liveVideoId, resolutions)
 603
 604       await stopFfmpeg(ffmpegCommand)
 605       await commands[0].waitUntilEnded({ videoId: liveVideoId })
 606
 607       await waitJobs(servers)
 608
 609       await waitUntilLivePublishedOnAllServers(servers, liveVideoId)
 610
 611       const video = await servers[0].videos.get({ id: liveVideoId })
 612       const hlsFiles = video.streamingPlaylists[0].files
 613
 614       expect(video.files).to.have.lengthOf(0)
 615       expect(hlsFiles).to.have.lengthOf(resolutions.length)
 616
 617       // eslint-disable-next-line @typescript-eslint/require-array-sort-compare
 618       expect(getAllFiles(video).map(f => f.resolution.id).sort()).to.deep.equal(resolutions)
 619     })
 620
 621     it('Should only keep the original resolution if all resolutions are disabled', async function () {
 622       this.timeout(600_000)
 623
 624       await updateConf([])
 625       liveVideoId = await createLiveWrapper(true)
 626
 627       const ffmpegCommand = await commands[0].sendRTMPStreamInVideo({ videoId: liveVideoId, fixtureName: 'video_short2.webm' })
 628       await waitUntilLivePublishedOnAllServers(servers, liveVideoId)
 629       await waitJobs(servers)
 630
 631       await testVideoResolutions(liveVideoId, [ 720 ])
 632
 633       await stopFfmpeg(ffmpegCommand)
 634       await commands[0].waitUntilEnded({ videoId: liveVideoId })
 635
 636       await waitJobs(servers)
 637
 638       await waitUntilLivePublishedOnAllServers(servers, liveVideoId)
 639
 640       const video = await servers[0].videos.get({ id: liveVideoId })
 641       const hlsFiles = video.streamingPlaylists[0].files
 642
 643       expect(video.files).to.have.lengthOf(0)
 644       expect(hlsFiles).to.have.lengthOf(1)
 645
 646       expect(hlsFiles[0].resolution.id).to.equal(720)
 647     })
 648   })
 649
 650   describe('After a server restart', function () {
 651     let liveVideoId: string
 652     let liveVideoReplayId: string
 653     let permanentLiveVideoReplayId: string
 654
 655     let permanentLiveReplayName: string
 656
 657     let beforeServerRestart: Date
 658
 659     async function createLiveWrapper (options: { saveReplay: boolean, permanent: boolean }) {
 660       const liveAttributes: LiveVideoCreate = {
 661         name: 'live video',
 662         channelId: servers[0].store.channel.id,
 663         privacy: VideoPrivacy.PUBLIC,
 664         saveReplay: options.saveReplay,
 665         permanentLive: options.permanent
 666       }
 667
 668       const { uuid } = await commands[0].create({ fields: liveAttributes })
 669       return uuid
 670     }
 671
 672     before(async function () {
 673       this.timeout(600_000)
 674
 675       liveVideoId = await createLiveWrapper({ saveReplay: false, permanent: false })
 676       liveVideoReplayId = await createLiveWrapper({ saveReplay: true, permanent: false })
 677       permanentLiveVideoReplayId = await createLiveWrapper({ saveReplay: true, permanent: true })
 678
 679       await Promise.all([
 680         commands[0].sendRTMPStreamInVideo({ videoId: liveVideoId }),
 681         commands[0].sendRTMPStreamInVideo({ videoId: permanentLiveVideoReplayId }),
 682         commands[0].sendRTMPStreamInVideo({ videoId: liveVideoReplayId })
 683       ])
 684
 685       await Promise.all([
 686         commands[0].waitUntilPublished({ videoId: liveVideoId }),
 687         commands[0].waitUntilPublished({ videoId: permanentLiveVideoReplayId }),
 688         commands[0].waitUntilPublished({ videoId: liveVideoReplayId })
 689       ])
 690
 691       await commands[0].waitUntilSegmentGeneration({ videoUUID: liveVideoId, playlistNumber: 0, segment: 2 })
 692       await commands[0].waitUntilSegmentGeneration({ videoUUID: liveVideoReplayId, playlistNumber: 0, segment: 2 })
 693       await commands[0].waitUntilSegmentGeneration({ videoUUID: permanentLiveVideoReplayId, playlistNumber: 0, segment: 2 })
 694
 695       {
 696         const video = await servers[0].videos.get({ id: permanentLiveVideoReplayId })
 697         permanentLiveReplayName = video.name + ' - ' + new Date(video.publishedAt).toLocaleString()
 698       }
 699
 700       await killallServers([ servers[0] ])
 701
 702       beforeServerRestart = new Date()
 703       await servers[0].run()
 704
 705       await wait(5000)
 706       await waitJobs(servers)
 707     })
 708
 709     it('Should cleanup lives', async function () {
 710       this.timeout(60000)
 711
 712       await commands[0].waitUntilEnded({ videoId: liveVideoId })
 713       await commands[0].waitUntilWaiting({ videoId: permanentLiveVideoReplayId })
 714     })
 715
 716     it('Should save a non permanent live replay', async function () {
 717       this.timeout(240000)
 718
 719       await commands[0].waitUntilPublished({ videoId: liveVideoReplayId })
 720
 721       const session = await commands[0].getReplaySession({ videoId: liveVideoReplayId })
 722       expect(session.endDate).to.exist
 723       expect(new Date(session.endDate)).to.be.above(beforeServerRestart)
 724     })
 725
 726     it('Should have saved a permanent live replay', async function () {
 727       this.timeout(120000)
 728
 729       const { data } = await servers[0].videos.listMyVideos({ sort: '-publishedAt' })
 730       expect(data.find(v => v.name === permanentLiveReplayName)).to.exist
 731     })
 732   })
 733
 734   after(async function () {
 735     await cleanupTests(servers)
 736   })
 737 })