server/tests/api/live/live-save-replay.ts

   1 /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
   2
   3 import { expect } from 'chai'
   4 import { FfmpegCommand } from 'fluent-ffmpeg'
   5 import { checkLiveCleanup } from '@server/tests/shared'
   6 import { wait } from '@shared/core-utils'
   7 import { HttpStatusCode, LiveVideoCreate, LiveVideoError, VideoPrivacy, VideoState } from '@shared/models'
   8 import {
   9   cleanupTests,
  10   ConfigCommand,
  11   createMultipleServers,
  12   doubleFollow,
  13   findExternalSavedVideo,
  14   PeerTubeServer,
  15   setAccessTokensToServers,
  16   setDefaultVideoChannel,
  17   stopFfmpeg,
  18   testFfmpegStreamError,
  19   waitJobs,
  20   waitUntilLivePublishedOnAllServers,
  21   waitUntilLiveReplacedByReplayOnAllServers,
  22   waitUntilLiveWaitingOnAllServers
  23 } from '@shared/server-commands'
  24
  25 describe('Save replay setting', function () {
  26   let servers: PeerTubeServer[] = []
  27   let liveVideoUUID: string
  28   let ffmpegCommand: FfmpegCommand
  29
  30   async function createLiveWrapper (options: { permanent: boolean, replay: boolean, replaySettings?: { privacy: VideoPrivacy } }) {
  31     if (liveVideoUUID) {
  32       try {
  33         await servers[0].videos.remove({ id: liveVideoUUID })
  34         await waitJobs(servers)
  35       } catch {}
  36     }
  37
  38     const attributes: LiveVideoCreate = {
  39       channelId: servers[0].store.channel.id,
  40       privacy: VideoPrivacy.PUBLIC,
  41       name: 'live'.repeat(30),
  42       saveReplay: options.replay,
  43       replaySettings: options.replaySettings,
  44       permanentLive: options.permanent
  45     }
  46
  47     const { uuid } = await servers[0].live.create({ fields: attributes })
  48     return uuid
  49   }
  50
  51   async function publishLive (options: { permanent: boolean, replay: boolean, replaySettings?: { privacy: VideoPrivacy } }) {
  52     liveVideoUUID = await createLiveWrapper(options)
  53
  54     const ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID })
  55     await waitUntilLivePublishedOnAllServers(servers, liveVideoUUID)
  56
  57     const liveDetails = await servers[0].videos.get({ id: liveVideoUUID })
  58
  59     await waitJobs(servers)
  60     await checkVideosExist(liveVideoUUID, true, HttpStatusCode.OK_200)
  61
  62     return { ffmpegCommand, liveDetails }
  63   }
  64
  65   async function publishLiveAndDelete (options: { permanent: boolean, replay: boolean, replaySettings?: { privacy: VideoPrivacy } }) {
  66     const { ffmpegCommand, liveDetails } = await publishLive(options)
  67
  68     await Promise.all([
  69       servers[0].videos.remove({ id: liveVideoUUID }),
  70       testFfmpegStreamError(ffmpegCommand, true)
  71     ])
  72
  73     await waitJobs(servers)
  74     await wait(5000)
  75     await waitJobs(servers)
  76
  77     return { liveDetails }
  78   }
  79
  80   async function publishLiveAndBlacklist (options: { permanent: boolean, replay: boolean, replaySettings?: { privacy: VideoPrivacy } }) {
  81     const { ffmpegCommand, liveDetails } = await publishLive(options)
  82
  83     await Promise.all([
  84       servers[0].blacklist.add({ videoId: liveVideoUUID, reason: 'bad live', unfederate: true }),
  85       testFfmpegStreamError(ffmpegCommand, true)
  86     ])
  87
  88     await waitJobs(servers)
  89     await wait(5000)
  90     await waitJobs(servers)
  91
  92     return { liveDetails }
  93   }
  94
  95   async function checkVideosExist (videoId: string, existsInList: boolean, expectedStatus?: number) {
  96     for (const server of servers) {
  97       const length = existsInList ? 1 : 0
  98
  99       const { data, total } = await server.videos.list()
 100       expect(data).to.have.lengthOf(length)
 101       expect(total).to.equal(length)
 102
 103       if (expectedStatus) {
 104         await server.videos.get({ id: videoId, expectedStatus })
 105       }
 106     }
 107   }
 108
 109   async function checkVideoState (videoId: string, state: VideoState) {
 110     for (const server of servers) {
 111       const video = await server.videos.get({ id: videoId })
 112       expect(video.state.id).to.equal(state)
 113     }
 114   }
 115
 116   async function checkVideoPrivacy (videoId: string, privacy: VideoPrivacy) {
 117     for (const server of servers) {
 118       const video = await server.videos.get({ id: videoId })
 119       expect(video.privacy.id).to.equal(privacy)
 120     }
 121   }
 122
 123   before(async function () {
 124     this.timeout(120000)
 125
 126     servers = await createMultipleServers(2)
 127
 128     // Get the access tokens
 129     await setAccessTokensToServers(servers)
 130     await setDefaultVideoChannel(servers)
 131
 132     // Server 1 and server 2 follow each other
 133     await doubleFollow(servers[0], servers[1])
 134
 135     await servers[0].config.updateCustomSubConfig({
 136       newConfig: {
 137         live: {
 138           enabled: true,
 139           allowReplay: true,
 140           maxDuration: -1,
 141           transcoding: {
 142             enabled: false,
 143             resolutions: ConfigCommand.getCustomConfigResolutions(true)
 144           }
 145         }
 146       }
 147     })
 148   })
 149
 150   describe('With save replay disabled', function () {
 151     let sessionStartDateMin: Date
 152     let sessionStartDateMax: Date
 153     let sessionEndDateMin: Date
 154
 155     it('Should correctly create and federate the "waiting for stream" live', async function () {
 156       this.timeout(40000)
 157
 158       liveVideoUUID = await createLiveWrapper({ permanent: false, replay: false })
 159
 160       await waitJobs(servers)
 161
 162       await checkVideosExist(liveVideoUUID, false, HttpStatusCode.OK_200)
 163       await checkVideoState(liveVideoUUID, VideoState.WAITING_FOR_LIVE)
 164     })
 165
 166     it('Should correctly have updated the live and federated it when streaming in the live', async function () {
 167       this.timeout(120000)
 168
 169       ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID })
 170
 171       sessionStartDateMin = new Date()
 172       await waitUntilLivePublishedOnAllServers(servers, liveVideoUUID)
 173       sessionStartDateMax = new Date()
 174
 175       await waitJobs(servers)
 176
 177       await checkVideosExist(liveVideoUUID, true, HttpStatusCode.OK_200)
 178       await checkVideoState(liveVideoUUID, VideoState.PUBLISHED)
 179     })
 180
 181     it('Should correctly delete the video files after the stream ended', async function () {
 182       this.timeout(120000)
 183
 184       sessionEndDateMin = new Date()
 185       await stopFfmpeg(ffmpegCommand)
 186
 187       for (const server of servers) {
 188         await server.live.waitUntilEnded({ videoId: liveVideoUUID })
 189       }
 190       await waitJobs(servers)
 191
 192       // Live still exist, but cannot be played anymore
 193       await checkVideosExist(liveVideoUUID, false, HttpStatusCode.OK_200)
 194       await checkVideoState(liveVideoUUID, VideoState.LIVE_ENDED)
 195
 196       // No resolutions saved since we did not save replay
 197       await checkLiveCleanup({ server: servers[0], videoUUID: liveVideoUUID, permanent: false })
 198     })
 199
 200     it('Should have appropriate ended session', async function () {
 201       const { data, total } = await servers[0].live.listSessions({ videoId: liveVideoUUID })
 202       expect(total).to.equal(1)
 203       expect(data).to.have.lengthOf(1)
 204
 205       const session = data[0]
 206
 207       const startDate = new Date(session.startDate)
 208       expect(startDate).to.be.above(sessionStartDateMin)
 209       expect(startDate).to.be.below(sessionStartDateMax)
 210
 211       expect(session.endDate).to.exist
 212       expect(new Date(session.endDate)).to.be.above(sessionEndDateMin)
 213
 214       expect(session.saveReplay).to.be.false
 215       expect(session.error).to.not.exist
 216       expect(session.replayVideo).to.not.exist
 217     })
 218
 219     it('Should correctly terminate the stream on blacklist and delete the live', async function () {
 220       this.timeout(120000)
 221
 222       await publishLiveAndBlacklist({ permanent: false, replay: false })
 223
 224       await checkVideosExist(liveVideoUUID, false)
 225
 226       await servers[0].videos.get({ id: liveVideoUUID, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 })
 227       await servers[1].videos.get({ id: liveVideoUUID, expectedStatus: HttpStatusCode.NOT_FOUND_404 })
 228
 229       await wait(5000)
 230       await waitJobs(servers)
 231       await checkLiveCleanup({ server: servers[0], videoUUID: liveVideoUUID, permanent: false })
 232     })
 233
 234     it('Should have blacklisted session error', async function () {
 235       const session = await servers[0].live.findLatestSession({ videoId: liveVideoUUID })
 236       expect(session.startDate).to.exist
 237       expect(session.endDate).to.exist
 238
 239       expect(session.error).to.equal(LiveVideoError.BLACKLISTED)
 240       expect(session.replayVideo).to.not.exist
 241     })
 242
 243     it('Should correctly terminate the stream on delete and delete the video', async function () {
 244       this.timeout(120000)
 245
 246       await publishLiveAndDelete({ permanent: false, replay: false })
 247
 248       await checkVideosExist(liveVideoUUID, false, HttpStatusCode.NOT_FOUND_404)
 249       await checkLiveCleanup({ server: servers[0], videoUUID: liveVideoUUID, permanent: false })
 250     })
 251   })
 252
 253   describe('With save replay enabled on non permanent live', function () {
 254
 255     it('Should correctly create and federate the "waiting for stream" live', async function () {
 256       this.timeout(120000)
 257
 258       liveVideoUUID = await createLiveWrapper({ permanent: false, replay: true, replaySettings: { privacy: VideoPrivacy.UNLISTED } })
 259
 260       await waitJobs(servers)
 261
 262       await checkVideosExist(liveVideoUUID, false, HttpStatusCode.OK_200)
 263       await checkVideoState(liveVideoUUID, VideoState.WAITING_FOR_LIVE)
 264       await checkVideoPrivacy(liveVideoUUID, VideoPrivacy.PUBLIC)
 265     })
 266
 267     it('Should correctly have updated the live and federated it when streaming in the live', async function () {
 268       this.timeout(120000)
 269
 270       ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID })
 271       await waitUntilLivePublishedOnAllServers(servers, liveVideoUUID)
 272
 273       await waitJobs(servers)
 274
 275       await checkVideosExist(liveVideoUUID, true, HttpStatusCode.OK_200)
 276       await checkVideoState(liveVideoUUID, VideoState.PUBLISHED)
 277       await checkVideoPrivacy(liveVideoUUID, VideoPrivacy.PUBLIC)
 278     })
 279
 280     it('Should correctly have saved the live and federated it after the streaming', async function () {
 281       this.timeout(120000)
 282
 283       const session = await servers[0].live.findLatestSession({ videoId: liveVideoUUID })
 284       expect(session.endDate).to.not.exist
 285       expect(session.endingProcessed).to.be.false
 286       expect(session.saveReplay).to.be.true
 287       expect(session.replaySettings).to.exist
 288       expect(session.replaySettings.privacy).to.equal(VideoPrivacy.UNLISTED)
 289
 290       await stopFfmpeg(ffmpegCommand)
 291
 292       await waitUntilLiveReplacedByReplayOnAllServers(servers, liveVideoUUID)
 293       await waitJobs(servers)
 294
 295       // Live has been transcoded
 296       await checkVideosExist(liveVideoUUID, false, HttpStatusCode.OK_200)
 297       await checkVideoState(liveVideoUUID, VideoState.PUBLISHED)
 298       await checkVideoPrivacy(liveVideoUUID, VideoPrivacy.UNLISTED)
 299     })
 300
 301     it('Should find the replay live session', async function () {
 302       const session = await servers[0].live.getReplaySession({ videoId: liveVideoUUID })
 303
 304       expect(session).to.exist
 305
 306       expect(session.startDate).to.exist
 307       expect(session.endDate).to.exist
 308
 309       expect(session.error).to.not.exist
 310       expect(session.saveReplay).to.be.true
 311       expect(session.endingProcessed).to.be.true
 312       expect(session.replaySettings).to.exist
 313       expect(session.replaySettings.privacy).to.equal(VideoPrivacy.UNLISTED)
 314
 315       expect(session.replayVideo).to.exist
 316       expect(session.replayVideo.id).to.exist
 317       expect(session.replayVideo.shortUUID).to.exist
 318       expect(session.replayVideo.uuid).to.equal(liveVideoUUID)
 319     })
 320
 321     it('Should update the saved live and correctly federate the updated attributes', async function () {
 322       this.timeout(120000)
 323
 324       await servers[0].videos.update({ id: liveVideoUUID, attributes: { name: 'video updated', privacy: VideoPrivacy.PUBLIC } })
 325       await waitJobs(servers)
 326
 327       for (const server of servers) {
 328         const video = await server.videos.get({ id: liveVideoUUID })
 329         expect(video.name).to.equal('video updated')
 330         expect(video.isLive).to.be.false
 331         expect(video.privacy.id).to.equal(VideoPrivacy.PUBLIC)
 332       }
 333     })
 334
 335     it('Should have cleaned up the live files', async function () {
 336       await checkLiveCleanup({ server: servers[0], videoUUID: liveVideoUUID, permanent: false, savedResolutions: [ 720 ] })
 337     })
 338
 339     it('Should correctly terminate the stream on blacklist and blacklist the saved replay video', async function () {
 340       this.timeout(120000)
 341
 342       await publishLiveAndBlacklist({ permanent: false, replay: true, replaySettings: { privacy: VideoPrivacy.PUBLIC } })
 343
 344       await checkVideosExist(liveVideoUUID, false)
 345
 346       await servers[0].videos.get({ id: liveVideoUUID, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 })
 347       await servers[1].videos.get({ id: liveVideoUUID, expectedStatus: HttpStatusCode.NOT_FOUND_404 })
 348
 349       await wait(5000)
 350       await waitJobs(servers)
 351       await checkLiveCleanup({ server: servers[0], videoUUID: liveVideoUUID, permanent: false, savedResolutions: [ 720 ] })
 352     })
 353
 354     it('Should correctly terminate the stream on delete and delete the video', async function () {
 355       this.timeout(120000)
 356
 357       await publishLiveAndDelete({ permanent: false, replay: true, replaySettings: { privacy: VideoPrivacy.PUBLIC } })
 358
 359       await checkVideosExist(liveVideoUUID, false, HttpStatusCode.NOT_FOUND_404)
 360       await checkLiveCleanup({ server: servers[0], videoUUID: liveVideoUUID, permanent: false })
 361     })
 362   })
 363
 364   describe('With save replay enabled on permanent live', function () {
 365     let lastReplayUUID: string
 366
 367     describe('With a first live and its replay', function () {
 368
 369       it('Should correctly create and federate the "waiting for stream" live', async function () {
 370         this.timeout(120000)
 371
 372         liveVideoUUID = await createLiveWrapper({ permanent: true, replay: true, replaySettings: { privacy: VideoPrivacy.UNLISTED } })
 373
 374         await waitJobs(servers)
 375
 376         await checkVideosExist(liveVideoUUID, false, HttpStatusCode.OK_200)
 377         await checkVideoState(liveVideoUUID, VideoState.WAITING_FOR_LIVE)
 378         await checkVideoPrivacy(liveVideoUUID, VideoPrivacy.PUBLIC)
 379       })
 380
 381       it('Should correctly have updated the live and federated it when streaming in the live', async function () {
 382         this.timeout(120000)
 383
 384         ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID })
 385         await waitUntilLivePublishedOnAllServers(servers, liveVideoUUID)
 386
 387         await waitJobs(servers)
 388
 389         await checkVideosExist(liveVideoUUID, true, HttpStatusCode.OK_200)
 390         await checkVideoState(liveVideoUUID, VideoState.PUBLISHED)
 391         await checkVideoPrivacy(liveVideoUUID, VideoPrivacy.PUBLIC)
 392       })
 393
 394       it('Should correctly have saved the live and federated it after the streaming', async function () {
 395         this.timeout(120000)
 396
 397         const liveDetails = await servers[0].videos.get({ id: liveVideoUUID })
 398
 399         await stopFfmpeg(ffmpegCommand)
 400
 401         await waitUntilLiveWaitingOnAllServers(servers, liveVideoUUID)
 402         await waitJobs(servers)
 403
 404         const video = await findExternalSavedVideo(servers[0], liveDetails)
 405         expect(video).to.exist
 406
 407         for (const server of servers) {
 408           await server.videos.get({ id: video.uuid })
 409         }
 410
 411         lastReplayUUID = video.uuid
 412       })
 413
 414       it('Should have appropriate ended session and replay live session', async function () {
 415         const { data, total } = await servers[0].live.listSessions({ videoId: liveVideoUUID })
 416         expect(total).to.equal(1)
 417         expect(data).to.have.lengthOf(1)
 418
 419         const sessionFromLive = data[0]
 420         const sessionFromReplay = await servers[0].live.getReplaySession({ videoId: lastReplayUUID })
 421
 422         for (const session of [ sessionFromLive, sessionFromReplay ]) {
 423           expect(session.startDate).to.exist
 424           expect(session.endDate).to.exist
 425
 426           expect(session.replaySettings).to.exist
 427           expect(session.replaySettings.privacy).to.equal(VideoPrivacy.UNLISTED)
 428
 429           expect(session.error).to.not.exist
 430
 431           expect(session.replayVideo).to.exist
 432           expect(session.replayVideo.id).to.exist
 433           expect(session.replayVideo.shortUUID).to.exist
 434           expect(session.replayVideo.uuid).to.equal(lastReplayUUID)
 435         }
 436       })
 437
 438       it('Should have the first live replay with correct settings', async function () {
 439         await checkVideosExist(lastReplayUUID, false, HttpStatusCode.OK_200)
 440         await checkVideoState(lastReplayUUID, VideoState.PUBLISHED)
 441         await checkVideoPrivacy(lastReplayUUID, VideoPrivacy.UNLISTED)
 442       })
 443     })
 444
 445     describe('With a second live and its replay', function () {
 446
 447       it('Should update the replay settings', async function () {
 448         await servers[0].live.update({ videoId: liveVideoUUID, fields: { replaySettings: { privacy: VideoPrivacy.PUBLIC } } })
 449         await waitJobs(servers)
 450
 451         const live = await servers[0].live.get({ videoId: liveVideoUUID })
 452
 453         expect(live.saveReplay).to.be.true
 454         expect(live.replaySettings).to.exist
 455         expect(live.replaySettings.privacy).to.equal(VideoPrivacy.PUBLIC)
 456
 457       })
 458
 459       it('Should correctly have updated the live and federated it when streaming in the live', async function () {
 460         this.timeout(120000)
 461
 462         ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID })
 463         await waitUntilLivePublishedOnAllServers(servers, liveVideoUUID)
 464
 465         await waitJobs(servers)
 466
 467         await checkVideosExist(liveVideoUUID, true, HttpStatusCode.OK_200)
 468         await checkVideoState(liveVideoUUID, VideoState.PUBLISHED)
 469         await checkVideoPrivacy(liveVideoUUID, VideoPrivacy.PUBLIC)
 470       })
 471
 472       it('Should correctly have saved the live and federated it after the streaming', async function () {
 473         this.timeout(120000)
 474
 475         const liveDetails = await servers[0].videos.get({ id: liveVideoUUID })
 476
 477         await stopFfmpeg(ffmpegCommand)
 478
 479         await waitUntilLiveWaitingOnAllServers(servers, liveVideoUUID)
 480         await waitJobs(servers)
 481
 482         const video = await findExternalSavedVideo(servers[0], liveDetails)
 483         expect(video).to.exist
 484
 485         for (const server of servers) {
 486           await server.videos.get({ id: video.uuid })
 487         }
 488
 489         lastReplayUUID = video.uuid
 490       })
 491
 492       it('Should have appropriate ended session and replay live session', async function () {
 493         const { data, total } = await servers[0].live.listSessions({ videoId: liveVideoUUID })
 494         expect(total).to.equal(2)
 495         expect(data).to.have.lengthOf(2)
 496
 497         const sessionFromLive = data[1]
 498         const sessionFromReplay = await servers[0].live.getReplaySession({ videoId: lastReplayUUID })
 499
 500         for (const session of [ sessionFromLive, sessionFromReplay ]) {
 501           expect(session.startDate).to.exist
 502           expect(session.endDate).to.exist
 503
 504           expect(session.replaySettings).to.exist
 505           expect(session.replaySettings.privacy).to.equal(VideoPrivacy.PUBLIC)
 506
 507           expect(session.error).to.not.exist
 508
 509           expect(session.replayVideo).to.exist
 510           expect(session.replayVideo.id).to.exist
 511           expect(session.replayVideo.shortUUID).to.exist
 512           expect(session.replayVideo.uuid).to.equal(lastReplayUUID)
 513         }
 514       })
 515
 516       it('Should have the first live replay with correct settings', async function () {
 517         await checkVideosExist(lastReplayUUID, true, HttpStatusCode.OK_200)
 518         await checkVideoState(lastReplayUUID, VideoState.PUBLISHED)
 519         await checkVideoPrivacy(lastReplayUUID, VideoPrivacy.PUBLIC)
 520       })
 521
 522       it('Should have cleaned up the live files', async function () {
 523         await checkLiveCleanup({ server: servers[0], videoUUID: liveVideoUUID, permanent: false })
 524       })
 525
 526       it('Should correctly terminate the stream on blacklist and blacklist the saved replay video', async function () {
 527         this.timeout(120000)
 528
 529         await servers[0].videos.remove({ id: lastReplayUUID })
 530         const { liveDetails } = await publishLiveAndBlacklist({
 531           permanent: true,
 532           replay: true,
 533           replaySettings: { privacy: VideoPrivacy.PUBLIC }
 534         })
 535
 536         const replay = await findExternalSavedVideo(servers[0], liveDetails)
 537         expect(replay).to.exist
 538
 539         for (const videoId of [ liveVideoUUID, replay.uuid ]) {
 540           await checkVideosExist(videoId, false)
 541
 542           await servers[0].videos.get({ id: videoId, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 })
 543           await servers[1].videos.get({ id: videoId, expectedStatus: HttpStatusCode.NOT_FOUND_404 })
 544         }
 545
 546         await checkLiveCleanup({ server: servers[0], videoUUID: liveVideoUUID, permanent: false })
 547       })
 548
 549       it('Should correctly terminate the stream on delete and not save the video', async function () {
 550         this.timeout(120000)
 551
 552         const { liveDetails } = await publishLiveAndDelete({
 553           permanent: true,
 554           replay: true,
 555           replaySettings: { privacy: VideoPrivacy.PUBLIC }
 556         })
 557
 558         const replay = await findExternalSavedVideo(servers[0], liveDetails)
 559         expect(replay).to.not.exist
 560
 561         await checkVideosExist(liveVideoUUID, false, HttpStatusCode.NOT_FOUND_404)
 562         await checkLiveCleanup({ server: servers[0], videoUUID: liveVideoUUID, permanent: false })
 563       })
 564     })
 565   })
 566
 567   after(async function () {
 568     await cleanupTests(servers)
 569   })
 570 })