server/tests/api/live/live.ts

   1 /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
   2
   3 import { expect } from 'chai'
   4 import { basename, join } from 'path'
   5 import { ffprobePromise, getVideoStream } from '@server/helpers/ffmpeg'
   6 import { testImage, testVideoResolutions } from '@server/tests/shared'
   7 import { getAllFiles, wait } from '@shared/core-utils'
   8 import {
   9   HttpStatusCode,
  10   LiveVideo,
  11   LiveVideoCreate,
  12   LiveVideoLatencyMode,
  13   VideoDetails,
  14   VideoPrivacy,
  15   VideoState,
  16   VideoStreamingPlaylistType
  17 } from '@shared/models'
  18 import {
  19   cleanupTests,
  20   createMultipleServers,
  21   doubleFollow,
  22   killallServers,
  23   LiveCommand,
  24   makeGetRequest,
  25   makeRawRequest,
  26   PeerTubeServer,
  27   sendRTMPStream,
  28   setAccessTokensToServers,
  29   setDefaultVideoChannel,
  30   stopFfmpeg,
  31   testFfmpegStreamError,
  32   waitJobs,
  33   waitUntilLivePublishedOnAllServers
  34 } from '@shared/server-commands'
  35
  36 describe('Test live', function () {
  37   let servers: PeerTubeServer[] = []
  38   let commands: LiveCommand[]
  39
  40   before(async function () {
  41     this.timeout(120000)
  42
  43     servers = await createMultipleServers(2)
  44
  45     // Get the access tokens
  46     await setAccessTokensToServers(servers)
  47     await setDefaultVideoChannel(servers)
  48
  49     await servers[0].config.updateCustomSubConfig({
  50       newConfig: {
  51         live: {
  52           enabled: true,
  53           allowReplay: true,
  54           latencySetting: {
  55             enabled: true
  56           },
  57           transcoding: {
  58             enabled: false
  59           }
  60         }
  61       }
  62     })
  63
  64     // Server 1 and server 2 follow each other
  65     await doubleFollow(servers[0], servers[1])
  66
  67     commands = servers.map(s => s.live)
  68   })
  69
  70   describe('Live creation, update and delete', function () {
  71     let liveVideoUUID: string
  72
  73     it('Should create a live with the appropriate parameters', async function () {
  74       this.timeout(20000)
  75
  76       const attributes: LiveVideoCreate = {
  77         category: 1,
  78         licence: 2,
  79         language: 'fr',
  80         description: 'super live description',
  81         support: 'support field',
  82         channelId: servers[0].store.channel.id,
  83         nsfw: false,
  84         waitTranscoding: false,
  85         name: 'my super live',
  86         tags: [ 'tag1', 'tag2' ],
  87         commentsEnabled: false,
  88         downloadEnabled: false,
  89         saveReplay: true,
  90         replaySettings: { privacy: VideoPrivacy.PUBLIC },
  91         latencyMode: LiveVideoLatencyMode.SMALL_LATENCY,
  92         privacy: VideoPrivacy.PUBLIC,
  93         previewfile: 'video_short1-preview.webm.jpg',
  94         thumbnailfile: 'video_short1.webm.jpg'
  95       }
  96
  97       const live = await commands[0].create({ fields: attributes })
  98       liveVideoUUID = live.uuid
  99
 100       await waitJobs(servers)
 101
 102       for (const server of servers) {
 103         const video = await server.videos.get({ id: liveVideoUUID })
 104
 105         expect(video.category.id).to.equal(1)
 106         expect(video.licence.id).to.equal(2)
 107         expect(video.language.id).to.equal('fr')
 108         expect(video.description).to.equal('super live description')
 109         expect(video.support).to.equal('support field')
 110
 111         expect(video.channel.name).to.equal(servers[0].store.channel.name)
 112         expect(video.channel.host).to.equal(servers[0].store.channel.host)
 113
 114         expect(video.isLive).to.be.true
 115
 116         expect(video.nsfw).to.be.false
 117         expect(video.waitTranscoding).to.be.false
 118         expect(video.name).to.equal('my super live')
 119         expect(video.tags).to.deep.equal([ 'tag1', 'tag2' ])
 120         expect(video.commentsEnabled).to.be.false
 121         expect(video.downloadEnabled).to.be.false
 122         expect(video.privacy.id).to.equal(VideoPrivacy.PUBLIC)
 123
 124         await testImage(server.url, 'video_short1-preview.webm', video.previewPath)
 125         await testImage(server.url, 'video_short1.webm', video.thumbnailPath)
 126
 127         const live = await server.live.get({ videoId: liveVideoUUID })
 128
 129         if (server.url === servers[0].url) {
 130           expect(live.rtmpUrl).to.equal('rtmp://' + server.hostname + ':' + servers[0].rtmpPort + '/live')
 131           expect(live.streamKey).to.not.be.empty
 132
 133           expect(live.replaySettings).to.exist
 134           expect(live.replaySettings.privacy).to.equal(VideoPrivacy.PUBLIC)
 135         } else {
 136           expect(live.rtmpUrl).to.not.exist
 137           expect(live.streamKey).to.not.exist
 138         }
 139
 140         expect(live.saveReplay).to.be.true
 141         expect(live.latencyMode).to.equal(LiveVideoLatencyMode.SMALL_LATENCY)
 142       }
 143     })
 144
 145     it('Should have a default preview and thumbnail', async function () {
 146       this.timeout(20000)
 147
 148       const attributes: LiveVideoCreate = {
 149         name: 'default live thumbnail',
 150         channelId: servers[0].store.channel.id,
 151         privacy: VideoPrivacy.UNLISTED,
 152         nsfw: true
 153       }
 154
 155       const live = await commands[0].create({ fields: attributes })
 156       const videoId = live.uuid
 157
 158       await waitJobs(servers)
 159
 160       for (const server of servers) {
 161         const video = await server.videos.get({ id: videoId })
 162         expect(video.privacy.id).to.equal(VideoPrivacy.UNLISTED)
 163         expect(video.nsfw).to.be.true
 164
 165         await makeGetRequest({ url: server.url, path: video.thumbnailPath, expectedStatus: HttpStatusCode.OK_200 })
 166         await makeGetRequest({ url: server.url, path: video.previewPath, expectedStatus: HttpStatusCode.OK_200 })
 167       }
 168     })
 169
 170     it('Should not have the live listed since nobody streams into', async function () {
 171       for (const server of servers) {
 172         const { total, data } = await server.videos.list()
 173
 174         expect(total).to.equal(0)
 175         expect(data).to.have.lengthOf(0)
 176       }
 177     })
 178
 179     it('Should not be able to update a live of another server', async function () {
 180       await commands[1].update({ videoId: liveVideoUUID, fields: { saveReplay: false }, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
 181     })
 182
 183     it('Should update the live', async function () {
 184       this.timeout(10000)
 185
 186       await commands[0].update({ videoId: liveVideoUUID, fields: { saveReplay: false, latencyMode: LiveVideoLatencyMode.DEFAULT } })
 187       await waitJobs(servers)
 188     })
 189
 190     it('Have the live updated', async function () {
 191       for (const server of servers) {
 192         const live = await server.live.get({ videoId: liveVideoUUID })
 193
 194         if (server.url === servers[0].url) {
 195           expect(live.rtmpUrl).to.equal('rtmp://' + server.hostname + ':' + servers[0].rtmpPort + '/live')
 196           expect(live.streamKey).to.not.be.empty
 197         } else {
 198           expect(live.rtmpUrl).to.not.exist
 199           expect(live.streamKey).to.not.exist
 200         }
 201
 202         expect(live.saveReplay).to.be.false
 203         expect(live.replaySettings).to.not.exist
 204         expect(live.latencyMode).to.equal(LiveVideoLatencyMode.DEFAULT)
 205       }
 206     })
 207
 208     it('Delete the live', async function () {
 209       this.timeout(10000)
 210
 211       await servers[0].videos.remove({ id: liveVideoUUID })
 212       await waitJobs(servers)
 213     })
 214
 215     it('Should have the live deleted', async function () {
 216       for (const server of servers) {
 217         await server.videos.get({ id: liveVideoUUID, expectedStatus: HttpStatusCode.NOT_FOUND_404 })
 218         await server.live.get({ videoId: liveVideoUUID, expectedStatus: HttpStatusCode.NOT_FOUND_404 })
 219       }
 220     })
 221   })
 222
 223   describe('Live filters', function () {
 224     let ffmpegCommand: any
 225     let liveVideoId: string
 226     let vodVideoId: string
 227
 228     before(async function () {
 229       this.timeout(240000)
 230
 231       vodVideoId = (await servers[0].videos.quickUpload({ name: 'vod video' })).uuid
 232
 233       const liveOptions = { name: 'live', privacy: VideoPrivacy.PUBLIC, channelId: servers[0].store.channel.id }
 234       const live = await commands[0].create({ fields: liveOptions })
 235       liveVideoId = live.uuid
 236
 237       ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoId })
 238       await waitUntilLivePublishedOnAllServers(servers, liveVideoId)
 239       await waitJobs(servers)
 240     })
 241
 242     it('Should only display lives', async function () {
 243       const { data, total } = await servers[0].videos.list({ isLive: true })
 244
 245       expect(total).to.equal(1)
 246       expect(data).to.have.lengthOf(1)
 247       expect(data[0].name).to.equal('live')
 248     })
 249
 250     it('Should not display lives', async function () {
 251       const { data, total } = await servers[0].videos.list({ isLive: false })
 252
 253       expect(total).to.equal(1)
 254       expect(data).to.have.lengthOf(1)
 255       expect(data[0].name).to.equal('vod video')
 256     })
 257
 258     it('Should display my lives', async function () {
 259       this.timeout(60000)
 260
 261       await stopFfmpeg(ffmpegCommand)
 262       await waitJobs(servers)
 263
 264       const { data } = await servers[0].videos.listMyVideos({ isLive: true })
 265
 266       const result = data.every(v => v.isLive)
 267       expect(result).to.be.true
 268     })
 269
 270     it('Should not display my lives', async function () {
 271       const { data } = await servers[0].videos.listMyVideos({ isLive: false })
 272
 273       const result = data.every(v => !v.isLive)
 274       expect(result).to.be.true
 275     })
 276
 277     after(async function () {
 278       await servers[0].videos.remove({ id: vodVideoId })
 279       await servers[0].videos.remove({ id: liveVideoId })
 280     })
 281   })
 282
 283   describe('Stream checks', function () {
 284     let liveVideo: LiveVideo & VideoDetails
 285     let rtmpUrl: string
 286
 287     before(function () {
 288       rtmpUrl = 'rtmp://' + servers[0].hostname + ':' + servers[0].rtmpPort + ''
 289     })
 290
 291     async function createLiveWrapper () {
 292       const liveAttributes = {
 293         name: 'user live',
 294         channelId: servers[0].store.channel.id,
 295         privacy: VideoPrivacy.PUBLIC,
 296         saveReplay: false
 297       }
 298
 299       const { uuid } = await commands[0].create({ fields: liveAttributes })
 300
 301       const live = await commands[0].get({ videoId: uuid })
 302       const video = await servers[0].videos.get({ id: uuid })
 303
 304       return Object.assign(video, live)
 305     }
 306
 307     it('Should not allow a stream without the appropriate path', async function () {
 308       this.timeout(60000)
 309
 310       liveVideo = await createLiveWrapper()
 311
 312       const command = sendRTMPStream({ rtmpBaseUrl: rtmpUrl + '/bad-live', streamKey: liveVideo.streamKey })
 313       await testFfmpegStreamError(command, true)
 314     })
 315
 316     it('Should not allow a stream without the appropriate stream key', async function () {
 317       this.timeout(60000)
 318
 319       const command = sendRTMPStream({ rtmpBaseUrl: rtmpUrl + '/live', streamKey: 'bad-stream-key' })
 320       await testFfmpegStreamError(command, true)
 321     })
 322
 323     it('Should succeed with the correct params', async function () {
 324       this.timeout(60000)
 325
 326       const command = sendRTMPStream({ rtmpBaseUrl: rtmpUrl + '/live', streamKey: liveVideo.streamKey })
 327       await testFfmpegStreamError(command, false)
 328     })
 329
 330     it('Should list this live now someone stream into it', async function () {
 331       for (const server of servers) {
 332         const { total, data } = await server.videos.list()
 333
 334         expect(total).to.equal(1)
 335         expect(data).to.have.lengthOf(1)
 336
 337         const video = data[0]
 338         expect(video.name).to.equal('user live')
 339         expect(video.isLive).to.be.true
 340       }
 341     })
 342
 343     it('Should not allow a stream on a live that was blacklisted', async function () {
 344       this.timeout(60000)
 345
 346       liveVideo = await createLiveWrapper()
 347
 348       await servers[0].blacklist.add({ videoId: liveVideo.uuid })
 349
 350       const command = sendRTMPStream({ rtmpBaseUrl: rtmpUrl + '/live', streamKey: liveVideo.streamKey })
 351       await testFfmpegStreamError(command, true)
 352     })
 353
 354     it('Should not allow a stream on a live that was deleted', async function () {
 355       this.timeout(60000)
 356
 357       liveVideo = await createLiveWrapper()
 358
 359       await servers[0].videos.remove({ id: liveVideo.uuid })
 360
 361       const command = sendRTMPStream({ rtmpBaseUrl: rtmpUrl + '/live', streamKey: liveVideo.streamKey })
 362       await testFfmpegStreamError(command, true)
 363     })
 364   })
 365
 366   describe('Live transcoding', function () {
 367     let liveVideoId: string
 368
 369     async function createLiveWrapper (saveReplay: boolean) {
 370       const liveAttributes = {
 371         name: 'live video',
 372         channelId: servers[0].store.channel.id,
 373         privacy: VideoPrivacy.PUBLIC,
 374         saveReplay,
 375         replaySettings: saveReplay
 376           ? { privacy: VideoPrivacy.PUBLIC }
 377           : undefined
 378       }
 379
 380       const { uuid } = await commands[0].create({ fields: liveAttributes })
 381       return uuid
 382     }
 383
 384     function updateConf (resolutions: number[]) {
 385       return servers[0].config.updateCustomSubConfig({
 386         newConfig: {
 387           live: {
 388             enabled: true,
 389             allowReplay: true,
 390             maxDuration: -1,
 391             transcoding: {
 392               enabled: true,
 393               resolutions: {
 394                 '144p': resolutions.includes(144),
 395                 '240p': resolutions.includes(240),
 396                 '360p': resolutions.includes(360),
 397                 '480p': resolutions.includes(480),
 398                 '720p': resolutions.includes(720),
 399                 '1080p': resolutions.includes(1080),
 400                 '2160p': resolutions.includes(2160)
 401               }
 402             }
 403           }
 404         }
 405       })
 406     }
 407
 408     before(async function () {
 409       await updateConf([])
 410     })
 411
 412     it('Should enable transcoding without additional resolutions', async function () {
 413       this.timeout(120000)
 414
 415       liveVideoId = await createLiveWrapper(false)
 416
 417       const ffmpegCommand = await commands[0].sendRTMPStreamInVideo({ videoId: liveVideoId })
 418       await waitUntilLivePublishedOnAllServers(servers, liveVideoId)
 419       await waitJobs(servers)
 420
 421       await testVideoResolutions({
 422         originServer: servers[0],
 423         servers,
 424         liveVideoId,
 425         resolutions: [ 720 ],
 426         objectStorage: false,
 427         transcoded: true
 428       })
 429
 430       await stopFfmpeg(ffmpegCommand)
 431     })
 432
 433     it('Should transcode audio only RTMP stream', async function () {
 434       this.timeout(120000)
 435
 436       liveVideoId = await createLiveWrapper(false)
 437
 438       const ffmpegCommand = await commands[0].sendRTMPStreamInVideo({ videoId: liveVideoId, fixtureName: 'video_short_no_audio.mp4' })
 439       await waitUntilLivePublishedOnAllServers(servers, liveVideoId)
 440       await waitJobs(servers)
 441
 442       await stopFfmpeg(ffmpegCommand)
 443     })
 444
 445     it('Should enable transcoding with some resolutions', async function () {
 446       this.timeout(120000)
 447
 448       const resolutions = [ 240, 480 ]
 449       await updateConf(resolutions)
 450       liveVideoId = await createLiveWrapper(false)
 451
 452       const ffmpegCommand = await commands[0].sendRTMPStreamInVideo({ videoId: liveVideoId })
 453       await waitUntilLivePublishedOnAllServers(servers, liveVideoId)
 454       await waitJobs(servers)
 455
 456       await testVideoResolutions({
 457         originServer: servers[0],
 458         servers,
 459         liveVideoId,
 460         resolutions: resolutions.concat([ 720 ]),
 461         objectStorage: false,
 462         transcoded: true
 463       })
 464
 465       await stopFfmpeg(ffmpegCommand)
 466     })
 467
 468     it('Should correctly set the appropriate bitrate depending on the input', async function () {
 469       this.timeout(120000)
 470
 471       liveVideoId = await createLiveWrapper(false)
 472
 473       const ffmpegCommand = await commands[0].sendRTMPStreamInVideo({
 474         videoId: liveVideoId,
 475         fixtureName: 'video_short.mp4',
 476         copyCodecs: true
 477       })
 478       await waitUntilLivePublishedOnAllServers(servers, liveVideoId)
 479       await waitJobs(servers)
 480
 481       const video = await servers[0].videos.get({ id: liveVideoId })
 482
 483       const masterPlaylist = video.streamingPlaylists[0].playlistUrl
 484       const probe = await ffprobePromise(masterPlaylist)
 485
 486       const bitrates = probe.streams.map(s => parseInt(s.tags.variant_bitrate))
 487       for (const bitrate of bitrates) {
 488         expect(bitrate).to.exist
 489         expect(isNaN(bitrate)).to.be.false
 490         expect(bitrate).to.be.below(61_000_000) // video_short.mp4 bitrate
 491       }
 492
 493       await stopFfmpeg(ffmpegCommand)
 494     })
 495
 496     it('Should enable transcoding with some resolutions and correctly save them', async function () {
 497       this.timeout(400_000)
 498
 499       const resolutions = [ 240, 360, 720 ]
 500
 501       await updateConf(resolutions)
 502       liveVideoId = await createLiveWrapper(true)
 503
 504       const ffmpegCommand = await commands[0].sendRTMPStreamInVideo({ videoId: liveVideoId, fixtureName: 'video_short2.webm' })
 505       await waitUntilLivePublishedOnAllServers(servers, liveVideoId)
 506       await waitJobs(servers)
 507
 508       await testVideoResolutions({
 509         originServer: servers[0],
 510         servers,
 511         liveVideoId,
 512         resolutions,
 513         objectStorage: false,
 514         transcoded: true
 515       })
 516
 517       await stopFfmpeg(ffmpegCommand)
 518       await commands[0].waitUntilEnded({ videoId: liveVideoId })
 519
 520       await waitJobs(servers)
 521
 522       await waitUntilLivePublishedOnAllServers(servers, liveVideoId)
 523
 524       const maxBitrateLimits = {
 525         720: 6500 * 1000, // 60FPS
 526         360: 1250 * 1000,
 527         240: 700 * 1000
 528       }
 529
 530       const minBitrateLimits = {
 531         720: 4800 * 1000,
 532         360: 1000 * 1000,
 533         240: 550 * 1000
 534       }
 535
 536       for (const server of servers) {
 537         const video = await server.videos.get({ id: liveVideoId })
 538
 539         expect(video.state.id).to.equal(VideoState.PUBLISHED)
 540         expect(video.duration).to.be.greaterThan(1)
 541         expect(video.files).to.have.lengthOf(0)
 542
 543         const hlsPlaylist = video.streamingPlaylists.find(s => s.type === VideoStreamingPlaylistType.HLS)
 544         await makeRawRequest({ url: hlsPlaylist.playlistUrl, expectedStatus: HttpStatusCode.OK_200 })
 545         await makeRawRequest({ url: hlsPlaylist.segmentsSha256Url, expectedStatus: HttpStatusCode.OK_200 })
 546
 547         // We should have generated random filenames
 548         expect(basename(hlsPlaylist.playlistUrl)).to.not.equal('master.m3u8')
 549         expect(basename(hlsPlaylist.segmentsSha256Url)).to.not.equal('segments-sha256.json')
 550
 551         expect(hlsPlaylist.files).to.have.lengthOf(resolutions.length)
 552
 553         for (const resolution of resolutions) {
 554           const file = hlsPlaylist.files.find(f => f.resolution.id === resolution)
 555
 556           expect(file).to.exist
 557           expect(file.size).to.be.greaterThan(1)
 558
 559           if (resolution >= 720) {
 560             expect(file.fps).to.be.approximately(60, 10)
 561           } else {
 562             expect(file.fps).to.be.approximately(30, 3)
 563           }
 564
 565           const filename = basename(file.fileUrl)
 566           expect(filename).to.not.contain(video.uuid)
 567
 568           const segmentPath = servers[0].servers.buildDirectory(join('streaming-playlists', 'hls', video.uuid, filename))
 569
 570           const probe = await ffprobePromise(segmentPath)
 571           const videoStream = await getVideoStream(segmentPath, probe)
 572
 573           expect(probe.format.bit_rate).to.be.below(maxBitrateLimits[videoStream.height])
 574           expect(probe.format.bit_rate).to.be.at.least(minBitrateLimits[videoStream.height])
 575
 576           await makeRawRequest({ url: file.torrentUrl, expectedStatus: HttpStatusCode.OK_200 })
 577           await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 })
 578         }
 579       }
 580     })
 581
 582     it('Should not generate an upper resolution than original file', async function () {
 583       this.timeout(400_000)
 584
 585       const resolutions = [ 240, 480 ]
 586       await updateConf(resolutions)
 587
 588       await servers[0].config.updateExistingSubConfig({
 589         newConfig: {
 590           live: {
 591             transcoding: {
 592               alwaysTranscodeOriginalResolution: false
 593             }
 594           }
 595         }
 596       })
 597
 598       liveVideoId = await createLiveWrapper(true)
 599
 600       const ffmpegCommand = await commands[0].sendRTMPStreamInVideo({ videoId: liveVideoId, fixtureName: 'video_short2.webm' })
 601       await waitUntilLivePublishedOnAllServers(servers, liveVideoId)
 602       await waitJobs(servers)
 603
 604       await testVideoResolutions({
 605         originServer: servers[0],
 606         servers,
 607         liveVideoId,
 608         resolutions,
 609         objectStorage: false,
 610         transcoded: true
 611       })
 612
 613       await stopFfmpeg(ffmpegCommand)
 614       await commands[0].waitUntilEnded({ videoId: liveVideoId })
 615
 616       await waitJobs(servers)
 617
 618       await waitUntilLivePublishedOnAllServers(servers, liveVideoId)
 619
 620       const video = await servers[0].videos.get({ id: liveVideoId })
 621       const hlsFiles = video.streamingPlaylists[0].files
 622
 623       expect(video.files).to.have.lengthOf(0)
 624       expect(hlsFiles).to.have.lengthOf(resolutions.length)
 625
 626       // eslint-disable-next-line @typescript-eslint/require-array-sort-compare
 627       expect(getAllFiles(video).map(f => f.resolution.id).sort()).to.deep.equal(resolutions)
 628     })
 629
 630     it('Should only keep the original resolution if all resolutions are disabled', async function () {
 631       this.timeout(600_000)
 632
 633       await updateConf([])
 634       liveVideoId = await createLiveWrapper(true)
 635
 636       const ffmpegCommand = await commands[0].sendRTMPStreamInVideo({ videoId: liveVideoId, fixtureName: 'video_short2.webm' })
 637       await waitUntilLivePublishedOnAllServers(servers, liveVideoId)
 638       await waitJobs(servers)
 639
 640       await testVideoResolutions({
 641         originServer: servers[0],
 642         servers,
 643         liveVideoId,
 644         resolutions: [ 720 ],
 645         objectStorage: false,
 646         transcoded: true
 647       })
 648
 649       await stopFfmpeg(ffmpegCommand)
 650       await commands[0].waitUntilEnded({ videoId: liveVideoId })
 651
 652       await waitJobs(servers)
 653
 654       await waitUntilLivePublishedOnAllServers(servers, liveVideoId)
 655
 656       const video = await servers[0].videos.get({ id: liveVideoId })
 657       const hlsFiles = video.streamingPlaylists[0].files
 658
 659       expect(video.files).to.have.lengthOf(0)
 660       expect(hlsFiles).to.have.lengthOf(1)
 661
 662       expect(hlsFiles[0].resolution.id).to.equal(720)
 663     })
 664   })
 665
 666   describe('After a server restart', function () {
 667     let liveVideoId: string
 668     let liveVideoReplayId: string
 669     let permanentLiveVideoReplayId: string
 670
 671     let permanentLiveReplayName: string
 672
 673     let beforeServerRestart: Date
 674
 675     async function createLiveWrapper (options: { saveReplay: boolean, permanent: boolean }) {
 676       const liveAttributes: LiveVideoCreate = {
 677         name: 'live video',
 678         channelId: servers[0].store.channel.id,
 679         privacy: VideoPrivacy.PUBLIC,
 680         saveReplay: options.saveReplay,
 681         replaySettings: options.saveReplay
 682           ? { privacy: VideoPrivacy.PUBLIC }
 683           : undefined,
 684         permanentLive: options.permanent
 685       }
 686
 687       const { uuid } = await commands[0].create({ fields: liveAttributes })
 688       return uuid
 689     }
 690
 691     before(async function () {
 692       this.timeout(600_000)
 693
 694       liveVideoId = await createLiveWrapper({ saveReplay: false, permanent: false })
 695       liveVideoReplayId = await createLiveWrapper({ saveReplay: true, permanent: false })
 696       permanentLiveVideoReplayId = await createLiveWrapper({ saveReplay: true, permanent: true })
 697
 698       await Promise.all([
 699         commands[0].sendRTMPStreamInVideo({ videoId: liveVideoId }),
 700         commands[0].sendRTMPStreamInVideo({ videoId: permanentLiveVideoReplayId }),
 701         commands[0].sendRTMPStreamInVideo({ videoId: liveVideoReplayId })
 702       ])
 703
 704       await Promise.all([
 705         commands[0].waitUntilPublished({ videoId: liveVideoId }),
 706         commands[0].waitUntilPublished({ videoId: permanentLiveVideoReplayId }),
 707         commands[0].waitUntilPublished({ videoId: liveVideoReplayId })
 708       ])
 709
 710       for (const videoUUID of [ liveVideoId, liveVideoReplayId, permanentLiveVideoReplayId ]) {
 711         await commands[0].waitUntilSegmentGeneration({
 712           server: servers[0],
 713           videoUUID,
 714           playlistNumber: 0,
 715           segment: 2,
 716           objectStorage: false
 717         })
 718       }
 719
 720       {
 721         const video = await servers[0].videos.get({ id: permanentLiveVideoReplayId })
 722         permanentLiveReplayName = video.name + ' - ' + new Date(video.publishedAt).toLocaleString()
 723       }
 724
 725       await killallServers([ servers[0] ])
 726
 727       beforeServerRestart = new Date()
 728       await servers[0].run()
 729
 730       await wait(5000)
 731       await waitJobs(servers)
 732     })
 733
 734     it('Should cleanup lives', async function () {
 735       this.timeout(60000)
 736
 737       await commands[0].waitUntilEnded({ videoId: liveVideoId })
 738       await commands[0].waitUntilWaiting({ videoId: permanentLiveVideoReplayId })
 739     })
 740
 741     it('Should save a non permanent live replay', async function () {
 742       this.timeout(240000)
 743
 744       await commands[0].waitUntilPublished({ videoId: liveVideoReplayId })
 745
 746       const session = await commands[0].getReplaySession({ videoId: liveVideoReplayId })
 747       expect(session.endDate).to.exist
 748       expect(new Date(session.endDate)).to.be.above(beforeServerRestart)
 749     })
 750
 751     it('Should have saved a permanent live replay', async function () {
 752       this.timeout(120000)
 753
 754       const { data } = await servers[0].videos.listMyVideos({ sort: '-publishedAt' })
 755       expect(data.find(v => v.name === permanentLiveReplayName)).to.exist
 756     })
 757   })
 758
 759   after(async function () {
 760     await cleanupTests(servers)
 761   })
 762 })