server/tests/api/live/live-save-replay.ts

   1 /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
   2
   3 import 'mocha'
   4 import * as chai from 'chai'
   5 import { FfmpegCommand } from 'fluent-ffmpeg'
   6 import { checkLiveCleanup } from '@server/tests/shared'
   7 import { wait } from '@shared/core-utils'
   8 import { HttpStatusCode, LiveVideoCreate, LiveVideoError, VideoPrivacy, VideoState } from '@shared/models'
   9 import {
  10   cleanupTests,
  11   ConfigCommand,
  12   createMultipleServers,
  13   doubleFollow,
  14   findExternalSavedVideo,
  15   makeRawRequest,
  16   PeerTubeServer,
  17   setAccessTokensToServers,
  18   setDefaultVideoChannel,
  19   stopFfmpeg,
  20   testFfmpegStreamError,
  21   waitJobs,
  22   waitUntilLivePublishedOnAllServers,
  23   waitUntilLiveReplacedByReplayOnAllServers,
  24   waitUntilLiveWaitingOnAllServers
  25 } from '@shared/server-commands'
  26
  27 const expect = chai.expect
  28
  29 describe('Save replay setting', function () {
  30   let servers: PeerTubeServer[] = []
  31   let liveVideoUUID: string
  32   let ffmpegCommand: FfmpegCommand
  33
  34   async function createLiveWrapper (options: { permanent: boolean, replay: boolean }) {
  35     if (liveVideoUUID) {
  36       try {
  37         await servers[0].videos.remove({ id: liveVideoUUID })
  38         await waitJobs(servers)
  39       } catch {}
  40     }
  41
  42     const attributes: LiveVideoCreate = {
  43       channelId: servers[0].store.channel.id,
  44       privacy: VideoPrivacy.PUBLIC,
  45       name: 'my super live',
  46       saveReplay: options.replay,
  47       permanentLive: options.permanent
  48     }
  49
  50     const { uuid } = await servers[0].live.create({ fields: attributes })
  51     return uuid
  52   }
  53
  54   async function publishLive (options: { permanent: boolean, replay: boolean }) {
  55     liveVideoUUID = await createLiveWrapper(options)
  56
  57     const ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID })
  58     await waitUntilLivePublishedOnAllServers(servers, liveVideoUUID)
  59
  60     const liveDetails = await servers[0].videos.get({ id: liveVideoUUID })
  61
  62     await waitJobs(servers)
  63     await checkVideosExist(liveVideoUUID, true, HttpStatusCode.OK_200)
  64
  65     return { ffmpegCommand, liveDetails }
  66   }
  67
  68   async function publishLiveAndDelete (options: { permanent: boolean, replay: boolean }) {
  69     const { ffmpegCommand, liveDetails } = await publishLive(options)
  70
  71     await Promise.all([
  72       servers[0].videos.remove({ id: liveVideoUUID }),
  73       testFfmpegStreamError(ffmpegCommand, true)
  74     ])
  75
  76     await waitJobs(servers)
  77     await wait(5000)
  78     await waitJobs(servers)
  79
  80     return { liveDetails }
  81   }
  82
  83   async function publishLiveAndBlacklist (options: { permanent: boolean, replay: boolean }) {
  84     const { ffmpegCommand, liveDetails } = await publishLive(options)
  85
  86     await Promise.all([
  87       servers[0].blacklist.add({ videoId: liveVideoUUID, reason: 'bad live', unfederate: true }),
  88       testFfmpegStreamError(ffmpegCommand, true)
  89     ])
  90
  91     await waitJobs(servers)
  92     await wait(5000)
  93     await waitJobs(servers)
  94
  95     return { liveDetails }
  96   }
  97
  98   async function checkVideosExist (videoId: string, existsInList: boolean, expectedStatus?: number) {
  99     for (const server of servers) {
 100       const length = existsInList ? 1 : 0
 101
 102       const { data, total } = await server.videos.list()
 103       expect(data).to.have.lengthOf(length)
 104       expect(total).to.equal(length)
 105
 106       if (expectedStatus) {
 107         await server.videos.get({ id: videoId, expectedStatus })
 108       }
 109     }
 110   }
 111
 112   async function checkVideoState (videoId: string, state: VideoState) {
 113     for (const server of servers) {
 114       const video = await server.videos.get({ id: videoId })
 115       expect(video.state.id).to.equal(state)
 116     }
 117   }
 118
 119   before(async function () {
 120     this.timeout(120000)
 121
 122     servers = await createMultipleServers(2)
 123
 124     // Get the access tokens
 125     await setAccessTokensToServers(servers)
 126     await setDefaultVideoChannel(servers)
 127
 128     // Server 1 and server 2 follow each other
 129     await doubleFollow(servers[0], servers[1])
 130
 131     await servers[0].config.updateCustomSubConfig({
 132       newConfig: {
 133         live: {
 134           enabled: true,
 135           allowReplay: true,
 136           maxDuration: -1,
 137           transcoding: {
 138             enabled: false,
 139             resolutions: ConfigCommand.getCustomConfigResolutions(true)
 140           }
 141         }
 142       }
 143     })
 144   })
 145
 146   describe('With save replay disabled', function () {
 147     let sessionStartDateMin: Date
 148     let sessionStartDateMax: Date
 149     let sessionEndDateMin: Date
 150
 151     it('Should correctly create and federate the "waiting for stream" live', async function () {
 152       this.timeout(20000)
 153
 154       liveVideoUUID = await createLiveWrapper({ permanent: false, replay: false })
 155
 156       await waitJobs(servers)
 157
 158       await checkVideosExist(liveVideoUUID, false, HttpStatusCode.OK_200)
 159       await checkVideoState(liveVideoUUID, VideoState.WAITING_FOR_LIVE)
 160     })
 161
 162     it('Should correctly have updated the live and federated it when streaming in the live', async function () {
 163       this.timeout(30000)
 164
 165       ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID })
 166
 167       sessionStartDateMin = new Date()
 168       await waitUntilLivePublishedOnAllServers(servers, liveVideoUUID)
 169       sessionStartDateMax = new Date()
 170
 171       await waitJobs(servers)
 172
 173       await checkVideosExist(liveVideoUUID, true, HttpStatusCode.OK_200)
 174       await checkVideoState(liveVideoUUID, VideoState.PUBLISHED)
 175     })
 176
 177     it('Should correctly delete the video files after the stream ended', async function () {
 178       this.timeout(40000)
 179
 180       sessionEndDateMin = new Date()
 181       await stopFfmpeg(ffmpegCommand)
 182
 183       for (const server of servers) {
 184         await server.live.waitUntilEnded({ videoId: liveVideoUUID })
 185       }
 186       await waitJobs(servers)
 187
 188       // Live still exist, but cannot be played anymore
 189       await checkVideosExist(liveVideoUUID, false, HttpStatusCode.OK_200)
 190       await checkVideoState(liveVideoUUID, VideoState.LIVE_ENDED)
 191
 192       // No resolutions saved since we did not save replay
 193       await checkLiveCleanup(servers[0], liveVideoUUID, [])
 194     })
 195
 196     it('Should have appropriate ended session', async function () {
 197       const { data, total } = await servers[0].live.listSessions({ videoId: liveVideoUUID })
 198       expect(total).to.equal(1)
 199       expect(data).to.have.lengthOf(1)
 200
 201       const session = data[0]
 202
 203       const startDate = new Date(session.startDate)
 204       expect(startDate).to.be.above(sessionStartDateMin)
 205       expect(startDate).to.be.below(sessionStartDateMax)
 206
 207       expect(session.endDate).to.exist
 208       expect(new Date(session.endDate)).to.be.above(sessionEndDateMin)
 209
 210       expect(session.error).to.not.exist
 211       expect(session.replayVideo).to.not.exist
 212     })
 213
 214     it('Should correctly terminate the stream on blacklist and delete the live', async function () {
 215       this.timeout(40000)
 216
 217       await publishLiveAndBlacklist({ permanent: false, replay: false })
 218
 219       await checkVideosExist(liveVideoUUID, false)
 220
 221       await servers[0].videos.get({ id: liveVideoUUID, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 })
 222       await servers[1].videos.get({ id: liveVideoUUID, expectedStatus: HttpStatusCode.NOT_FOUND_404 })
 223
 224       await wait(5000)
 225       await waitJobs(servers)
 226       await checkLiveCleanup(servers[0], liveVideoUUID, [])
 227     })
 228
 229     it('Should have blacklisted session error', async function () {
 230       const session = await servers[0].live.findLatestSession({ videoId: liveVideoUUID })
 231       expect(session.startDate).to.exist
 232       expect(session.endDate).to.exist
 233
 234       expect(session.error).to.equal(LiveVideoError.BLACKLISTED)
 235       expect(session.replayVideo).to.not.exist
 236     })
 237
 238     it('Should correctly terminate the stream on delete and delete the video', async function () {
 239       this.timeout(40000)
 240
 241       await publishLiveAndDelete({ permanent: false, replay: false })
 242
 243       await checkVideosExist(liveVideoUUID, false, HttpStatusCode.NOT_FOUND_404)
 244       await checkLiveCleanup(servers[0], liveVideoUUID, [])
 245     })
 246   })
 247
 248   describe('With save replay enabled on non permanent live', function () {
 249
 250     it('Should correctly create and federate the "waiting for stream" live', async function () {
 251       this.timeout(20000)
 252
 253       liveVideoUUID = await createLiveWrapper({ permanent: false, replay: true })
 254
 255       await waitJobs(servers)
 256
 257       await checkVideosExist(liveVideoUUID, false, HttpStatusCode.OK_200)
 258       await checkVideoState(liveVideoUUID, VideoState.WAITING_FOR_LIVE)
 259     })
 260
 261     it('Should correctly have updated the live and federated it when streaming in the live', async function () {
 262       this.timeout(20000)
 263
 264       ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID })
 265       await waitUntilLivePublishedOnAllServers(servers, liveVideoUUID)
 266
 267       await waitJobs(servers)
 268
 269       await checkVideosExist(liveVideoUUID, true, HttpStatusCode.OK_200)
 270       await checkVideoState(liveVideoUUID, VideoState.PUBLISHED)
 271     })
 272
 273     it('Should correctly have saved the live and federated it after the streaming', async function () {
 274       this.timeout(30000)
 275
 276       await stopFfmpeg(ffmpegCommand)
 277
 278       await waitUntilLiveReplacedByReplayOnAllServers(servers, liveVideoUUID)
 279       await waitJobs(servers)
 280
 281       // Live has been transcoded
 282       await checkVideosExist(liveVideoUUID, true, HttpStatusCode.OK_200)
 283       await checkVideoState(liveVideoUUID, VideoState.PUBLISHED)
 284     })
 285
 286     it('Should find the replay live session', async function () {
 287       const session = await servers[0].live.getReplaySession({ videoId: liveVideoUUID })
 288
 289       expect(session).to.exist
 290
 291       expect(session.startDate).to.exist
 292       expect(session.endDate).to.exist
 293
 294       expect(session.error).to.not.exist
 295
 296       expect(session.replayVideo).to.exist
 297       expect(session.replayVideo.id).to.exist
 298       expect(session.replayVideo.shortUUID).to.exist
 299       expect(session.replayVideo.uuid).to.equal(liveVideoUUID)
 300     })
 301
 302     it('Should update the saved live and correctly federate the updated attributes', async function () {
 303       this.timeout(30000)
 304
 305       await servers[0].videos.update({ id: liveVideoUUID, attributes: { name: 'video updated' } })
 306       await waitJobs(servers)
 307
 308       for (const server of servers) {
 309         const video = await server.videos.get({ id: liveVideoUUID })
 310         expect(video.name).to.equal('video updated')
 311         expect(video.isLive).to.be.false
 312       }
 313     })
 314
 315     it('Should have cleaned up the live files', async function () {
 316       await checkLiveCleanup(servers[0], liveVideoUUID, [ 720 ])
 317     })
 318
 319     it('Should correctly terminate the stream on blacklist and blacklist the saved replay video', async function () {
 320       this.timeout(40000)
 321
 322       await publishLiveAndBlacklist({ permanent: false, replay: true })
 323
 324       await checkVideosExist(liveVideoUUID, false)
 325
 326       await servers[0].videos.get({ id: liveVideoUUID, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 })
 327       await servers[1].videos.get({ id: liveVideoUUID, expectedStatus: HttpStatusCode.NOT_FOUND_404 })
 328
 329       await wait(5000)
 330       await waitJobs(servers)
 331       await checkLiveCleanup(servers[0], liveVideoUUID, [ 720 ])
 332     })
 333
 334     it('Should correctly terminate the stream on delete and delete the video', async function () {
 335       this.timeout(40000)
 336
 337       await publishLiveAndDelete({ permanent: false, replay: true })
 338
 339       await checkVideosExist(liveVideoUUID, false, HttpStatusCode.NOT_FOUND_404)
 340       await checkLiveCleanup(servers[0], liveVideoUUID, [])
 341     })
 342   })
 343
 344   describe('With save replay enabled on permanent live', function () {
 345     let lastReplayUUID: string
 346
 347     it('Should correctly create and federate the "waiting for stream" live', async function () {
 348       this.timeout(20000)
 349
 350       liveVideoUUID = await createLiveWrapper({ permanent: true, replay: true })
 351
 352       await waitJobs(servers)
 353
 354       await checkVideosExist(liveVideoUUID, false, HttpStatusCode.OK_200)
 355       await checkVideoState(liveVideoUUID, VideoState.WAITING_FOR_LIVE)
 356     })
 357
 358     it('Should correctly have updated the live and federated it when streaming in the live', async function () {
 359       this.timeout(20000)
 360
 361       ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID })
 362       await waitUntilLivePublishedOnAllServers(servers, liveVideoUUID)
 363
 364       await waitJobs(servers)
 365
 366       await checkVideosExist(liveVideoUUID, true, HttpStatusCode.OK_200)
 367       await checkVideoState(liveVideoUUID, VideoState.PUBLISHED)
 368     })
 369
 370     it('Should correctly have saved the live and federated it after the streaming', async function () {
 371       this.timeout(30000)
 372
 373       const liveDetails = await servers[0].videos.get({ id: liveVideoUUID })
 374
 375       await stopFfmpeg(ffmpegCommand)
 376
 377       await waitUntilLiveWaitingOnAllServers(servers, liveVideoUUID)
 378       await waitJobs(servers)
 379
 380       const video = await findExternalSavedVideo(servers[0], liveDetails)
 381       expect(video).to.exist
 382
 383       for (const server of servers) {
 384         await server.videos.get({ id: video.uuid })
 385       }
 386
 387       lastReplayUUID = video.uuid
 388     })
 389
 390     it('Should have appropriate ended session and replay live session', async function () {
 391       const { data, total } = await servers[0].live.listSessions({ videoId: liveVideoUUID })
 392       expect(total).to.equal(1)
 393       expect(data).to.have.lengthOf(1)
 394
 395       const sessionFromLive = data[0]
 396       const sessionFromReplay = await servers[0].live.getReplaySession({ videoId: lastReplayUUID })
 397
 398       for (const session of [ sessionFromLive, sessionFromReplay ]) {
 399         expect(session.startDate).to.exist
 400         expect(session.endDate).to.exist
 401
 402         expect(session.error).to.not.exist
 403
 404         expect(session.replayVideo).to.exist
 405         expect(session.replayVideo.id).to.exist
 406         expect(session.replayVideo.shortUUID).to.exist
 407         expect(session.replayVideo.uuid).to.equal(lastReplayUUID)
 408       }
 409     })
 410
 411     it('Should have cleaned up the live files', async function () {
 412       await checkLiveCleanup(servers[0], liveVideoUUID, [])
 413     })
 414
 415     it('Should correctly terminate the stream on blacklist and blacklist the saved replay video', async function () {
 416       this.timeout(60000)
 417
 418       await servers[0].videos.remove({ id: lastReplayUUID })
 419       const { liveDetails } = await publishLiveAndBlacklist({ permanent: true, replay: true })
 420
 421       const replay = await findExternalSavedVideo(servers[0], liveDetails)
 422       expect(replay).to.exist
 423
 424       for (const videoId of [ liveVideoUUID, replay.uuid ]) {
 425         await checkVideosExist(videoId, false)
 426
 427         await servers[0].videos.get({ id: videoId, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 })
 428         await servers[1].videos.get({ id: videoId, expectedStatus: HttpStatusCode.NOT_FOUND_404 })
 429       }
 430
 431       await checkLiveCleanup(servers[0], liveVideoUUID, [])
 432     })
 433
 434     it('Should correctly terminate the stream on delete and not save the video', async function () {
 435       this.timeout(40000)
 436
 437       const { liveDetails } = await publishLiveAndDelete({ permanent: true, replay: true })
 438
 439       const replay = await findExternalSavedVideo(servers[0], liveDetails)
 440       expect(replay).to.not.exist
 441
 442       await checkVideosExist(liveVideoUUID, false, HttpStatusCode.NOT_FOUND_404)
 443       await checkLiveCleanup(servers[0], liveVideoUUID, [])
 444     })
 445
 446     it('Should correctly save replays with multiple sessions', async function () {
 447       this.timeout(120000)
 448
 449       liveVideoUUID = await createLiveWrapper({ permanent: true, replay: true })
 450       await waitJobs(servers)
 451
 452       // Streaming session #1
 453       ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID })
 454       await waitUntilLivePublishedOnAllServers(servers, liveVideoUUID)
 455       await stopFfmpeg(ffmpegCommand)
 456       await servers[0].live.waitUntilWaiting({ videoId: liveVideoUUID })
 457
 458       // Streaming session #2
 459       ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID })
 460       await waitUntilLivePublishedOnAllServers(servers, liveVideoUUID)
 461
 462       await wait(5000)
 463       const video = await servers[0].videos.get({ id: liveVideoUUID })
 464       expect(video.streamingPlaylists).to.have.lengthOf(1)
 465       await makeRawRequest(video.streamingPlaylists[0].playlistUrl)
 466
 467       await stopFfmpeg(ffmpegCommand)
 468       await waitUntilLiveWaitingOnAllServers(servers, liveVideoUUID)
 469
 470       // Wait for replays
 471       await waitJobs(servers)
 472
 473       const { total, data: sessions } = await servers[0].live.listSessions({ videoId: liveVideoUUID })
 474
 475       expect(total).to.equal(2)
 476       expect(sessions).to.have.lengthOf(2)
 477
 478       for (const session of sessions) {
 479         expect(session.error).to.be.null
 480         expect(session.replayVideo).to.exist
 481
 482         await servers[0].videos.get({ id: session.replayVideo.uuid })
 483       }
 484     })
 485   })
 486
 487   after(async function () {
 488     await cleanupTests(servers)
 489   })
 490 })