server/tests/api/live/live-save-replay.ts

   1 /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
   2
   3 import 'mocha'
   4 import * as chai from 'chai'
   5 import { FfmpegCommand } from 'fluent-ffmpeg'
   6 import { checkLiveCleanup } from '@server/tests/shared'
   7 import { wait } from '@shared/core-utils'
   8 import { HttpStatusCode, LiveVideoCreate, LiveVideoError, VideoPrivacy, VideoState } from '@shared/models'
   9 import {
  10   cleanupTests,
  11   ConfigCommand,
  12   createMultipleServers,
  13   doubleFollow,
  14   findExternalSavedVideo,
  15   PeerTubeServer,
  16   setAccessTokensToServers,
  17   setDefaultVideoChannel,
  18   stopFfmpeg,
  19   testFfmpegStreamError,
  20   waitJobs,
  21   waitUntilLivePublishedOnAllServers,
  22   waitUntilLiveReplacedByReplayOnAllServers,
  23   waitUntilLiveWaitingOnAllServers
  24 } from '@shared/server-commands'
  25
  26 const expect = chai.expect
  27
  28 describe('Save replay setting', function () {
  29   let servers: PeerTubeServer[] = []
  30   let liveVideoUUID: string
  31   let ffmpegCommand: FfmpegCommand
  32
  33   async function createLiveWrapper (options: { permanent: boolean, replay: boolean }) {
  34     if (liveVideoUUID) {
  35       try {
  36         await servers[0].videos.remove({ id: liveVideoUUID })
  37         await waitJobs(servers)
  38       } catch {}
  39     }
  40
  41     const attributes: LiveVideoCreate = {
  42       channelId: servers[0].store.channel.id,
  43       privacy: VideoPrivacy.PUBLIC,
  44       name: 'my super live',
  45       saveReplay: options.replay,
  46       permanentLive: options.permanent
  47     }
  48
  49     const { uuid } = await servers[0].live.create({ fields: attributes })
  50     return uuid
  51   }
  52
  53   async function publishLive (options: { permanent: boolean, replay: boolean }) {
  54     liveVideoUUID = await createLiveWrapper(options)
  55
  56     const ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID })
  57     await waitUntilLivePublishedOnAllServers(servers, liveVideoUUID)
  58
  59     const liveDetails = await servers[0].videos.get({ id: liveVideoUUID })
  60
  61     await waitJobs(servers)
  62     await checkVideosExist(liveVideoUUID, true, HttpStatusCode.OK_200)
  63
  64     return { ffmpegCommand, liveDetails }
  65   }
  66
  67   async function publishLiveAndDelete (options: { permanent: boolean, replay: boolean }) {
  68     const { ffmpegCommand, liveDetails } = await publishLive(options)
  69
  70     await Promise.all([
  71       servers[0].videos.remove({ id: liveVideoUUID }),
  72       testFfmpegStreamError(ffmpegCommand, true)
  73     ])
  74
  75     await waitJobs(servers)
  76     await wait(5000)
  77     await waitJobs(servers)
  78
  79     return { liveDetails }
  80   }
  81
  82   async function publishLiveAndBlacklist (options: { permanent: boolean, replay: boolean }) {
  83     const { ffmpegCommand, liveDetails } = await publishLive(options)
  84
  85     await Promise.all([
  86       servers[0].blacklist.add({ videoId: liveVideoUUID, reason: 'bad live', unfederate: true }),
  87       testFfmpegStreamError(ffmpegCommand, true)
  88     ])
  89
  90     await waitJobs(servers)
  91     await wait(5000)
  92     await waitJobs(servers)
  93
  94     return { liveDetails }
  95   }
  96
  97   async function checkVideosExist (videoId: string, existsInList: boolean, expectedStatus?: number) {
  98     for (const server of servers) {
  99       const length = existsInList ? 1 : 0
 100
 101       const { data, total } = await server.videos.list()
 102       expect(data).to.have.lengthOf(length)
 103       expect(total).to.equal(length)
 104
 105       if (expectedStatus) {
 106         await server.videos.get({ id: videoId, expectedStatus })
 107       }
 108     }
 109   }
 110
 111   async function checkVideoState (videoId: string, state: VideoState) {
 112     for (const server of servers) {
 113       const video = await server.videos.get({ id: videoId })
 114       expect(video.state.id).to.equal(state)
 115     }
 116   }
 117
 118   before(async function () {
 119     this.timeout(120000)
 120
 121     servers = await createMultipleServers(2)
 122
 123     // Get the access tokens
 124     await setAccessTokensToServers(servers)
 125     await setDefaultVideoChannel(servers)
 126
 127     // Server 1 and server 2 follow each other
 128     await doubleFollow(servers[0], servers[1])
 129
 130     await servers[0].config.updateCustomSubConfig({
 131       newConfig: {
 132         live: {
 133           enabled: true,
 134           allowReplay: true,
 135           maxDuration: -1,
 136           transcoding: {
 137             enabled: false,
 138             resolutions: ConfigCommand.getCustomConfigResolutions(true)
 139           }
 140         }
 141       }
 142     })
 143   })
 144
 145   describe('With save replay disabled', function () {
 146     let sessionStartDateMin: Date
 147     let sessionStartDateMax: Date
 148     let sessionEndDateMin: Date
 149
 150     it('Should correctly create and federate the "waiting for stream" live', async function () {
 151       this.timeout(20000)
 152
 153       liveVideoUUID = await createLiveWrapper({ permanent: false, replay: false })
 154
 155       await waitJobs(servers)
 156
 157       await checkVideosExist(liveVideoUUID, false, HttpStatusCode.OK_200)
 158       await checkVideoState(liveVideoUUID, VideoState.WAITING_FOR_LIVE)
 159     })
 160
 161     it('Should correctly have updated the live and federated it when streaming in the live', async function () {
 162       this.timeout(30000)
 163
 164       ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID })
 165
 166       sessionStartDateMin = new Date()
 167       await waitUntilLivePublishedOnAllServers(servers, liveVideoUUID)
 168       sessionStartDateMax = new Date()
 169
 170       await waitJobs(servers)
 171
 172       await checkVideosExist(liveVideoUUID, true, HttpStatusCode.OK_200)
 173       await checkVideoState(liveVideoUUID, VideoState.PUBLISHED)
 174     })
 175
 176     it('Should correctly delete the video files after the stream ended', async function () {
 177       this.timeout(40000)
 178
 179       sessionEndDateMin = new Date()
 180       await stopFfmpeg(ffmpegCommand)
 181
 182       for (const server of servers) {
 183         await server.live.waitUntilEnded({ videoId: liveVideoUUID })
 184       }
 185       await waitJobs(servers)
 186
 187       // Live still exist, but cannot be played anymore
 188       await checkVideosExist(liveVideoUUID, false, HttpStatusCode.OK_200)
 189       await checkVideoState(liveVideoUUID, VideoState.LIVE_ENDED)
 190
 191       // No resolutions saved since we did not save replay
 192       await checkLiveCleanup(servers[0], liveVideoUUID, [])
 193     })
 194
 195     it('Should have appropriate ended session', async function () {
 196       const { data, total } = await servers[0].live.listSessions({ videoId: liveVideoUUID })
 197       expect(total).to.equal(1)
 198       expect(data).to.have.lengthOf(1)
 199
 200       const session = data[0]
 201
 202       const startDate = new Date(session.startDate)
 203       expect(startDate).to.be.above(sessionStartDateMin)
 204       expect(startDate).to.be.below(sessionStartDateMax)
 205
 206       expect(session.endDate).to.exist
 207       expect(new Date(session.endDate)).to.be.above(sessionEndDateMin)
 208
 209       expect(session.saveReplay).to.be.false
 210       expect(session.error).to.not.exist
 211       expect(session.replayVideo).to.not.exist
 212     })
 213
 214     it('Should correctly terminate the stream on blacklist and delete the live', async function () {
 215       this.timeout(40000)
 216
 217       await publishLiveAndBlacklist({ permanent: false, replay: false })
 218
 219       await checkVideosExist(liveVideoUUID, false)
 220
 221       await servers[0].videos.get({ id: liveVideoUUID, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 })
 222       await servers[1].videos.get({ id: liveVideoUUID, expectedStatus: HttpStatusCode.NOT_FOUND_404 })
 223
 224       await wait(5000)
 225       await waitJobs(servers)
 226       await checkLiveCleanup(servers[0], liveVideoUUID, [])
 227     })
 228
 229     it('Should have blacklisted session error', async function () {
 230       const session = await servers[0].live.findLatestSession({ videoId: liveVideoUUID })
 231       expect(session.startDate).to.exist
 232       expect(session.endDate).to.exist
 233
 234       expect(session.error).to.equal(LiveVideoError.BLACKLISTED)
 235       expect(session.replayVideo).to.not.exist
 236     })
 237
 238     it('Should correctly terminate the stream on delete and delete the video', async function () {
 239       this.timeout(40000)
 240
 241       await publishLiveAndDelete({ permanent: false, replay: false })
 242
 243       await checkVideosExist(liveVideoUUID, false, HttpStatusCode.NOT_FOUND_404)
 244       await checkLiveCleanup(servers[0], liveVideoUUID, [])
 245     })
 246   })
 247
 248   describe('With save replay enabled on non permanent live', function () {
 249
 250     it('Should correctly create and federate the "waiting for stream" live', async function () {
 251       this.timeout(20000)
 252
 253       liveVideoUUID = await createLiveWrapper({ permanent: false, replay: true })
 254
 255       await waitJobs(servers)
 256
 257       await checkVideosExist(liveVideoUUID, false, HttpStatusCode.OK_200)
 258       await checkVideoState(liveVideoUUID, VideoState.WAITING_FOR_LIVE)
 259     })
 260
 261     it('Should correctly have updated the live and federated it when streaming in the live', async function () {
 262       this.timeout(20000)
 263
 264       ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID })
 265       await waitUntilLivePublishedOnAllServers(servers, liveVideoUUID)
 266
 267       await waitJobs(servers)
 268
 269       await checkVideosExist(liveVideoUUID, true, HttpStatusCode.OK_200)
 270       await checkVideoState(liveVideoUUID, VideoState.PUBLISHED)
 271     })
 272
 273     it('Should correctly have saved the live and federated it after the streaming', async function () {
 274       this.timeout(30000)
 275
 276       const session = await servers[0].live.findLatestSession({ videoId: liveVideoUUID })
 277       expect(session.endDate).to.not.exist
 278       expect(session.endingProcessed).to.be.false
 279       expect(session.saveReplay).to.be.true
 280
 281       await stopFfmpeg(ffmpegCommand)
 282
 283       await waitUntilLiveReplacedByReplayOnAllServers(servers, liveVideoUUID)
 284       await waitJobs(servers)
 285
 286       // Live has been transcoded
 287       await checkVideosExist(liveVideoUUID, true, HttpStatusCode.OK_200)
 288       await checkVideoState(liveVideoUUID, VideoState.PUBLISHED)
 289     })
 290
 291     it('Should find the replay live session', async function () {
 292       const session = await servers[0].live.getReplaySession({ videoId: liveVideoUUID })
 293
 294       expect(session).to.exist
 295
 296       expect(session.startDate).to.exist
 297       expect(session.endDate).to.exist
 298
 299       expect(session.error).to.not.exist
 300       expect(session.saveReplay).to.be.true
 301       expect(session.endingProcessed).to.be.true
 302
 303       expect(session.replayVideo).to.exist
 304       expect(session.replayVideo.id).to.exist
 305       expect(session.replayVideo.shortUUID).to.exist
 306       expect(session.replayVideo.uuid).to.equal(liveVideoUUID)
 307     })
 308
 309     it('Should update the saved live and correctly federate the updated attributes', async function () {
 310       this.timeout(30000)
 311
 312       await servers[0].videos.update({ id: liveVideoUUID, attributes: { name: 'video updated' } })
 313       await waitJobs(servers)
 314
 315       for (const server of servers) {
 316         const video = await server.videos.get({ id: liveVideoUUID })
 317         expect(video.name).to.equal('video updated')
 318         expect(video.isLive).to.be.false
 319       }
 320     })
 321
 322     it('Should have cleaned up the live files', async function () {
 323       await checkLiveCleanup(servers[0], liveVideoUUID, [ 720 ])
 324     })
 325
 326     it('Should correctly terminate the stream on blacklist and blacklist the saved replay video', async function () {
 327       this.timeout(120000)
 328
 329       await publishLiveAndBlacklist({ permanent: false, replay: true })
 330
 331       await checkVideosExist(liveVideoUUID, false)
 332
 333       await servers[0].videos.get({ id: liveVideoUUID, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 })
 334       await servers[1].videos.get({ id: liveVideoUUID, expectedStatus: HttpStatusCode.NOT_FOUND_404 })
 335
 336       await wait(5000)
 337       await waitJobs(servers)
 338       await checkLiveCleanup(servers[0], liveVideoUUID, [ 720 ])
 339     })
 340
 341     it('Should correctly terminate the stream on delete and delete the video', async function () {
 342       this.timeout(40000)
 343
 344       await publishLiveAndDelete({ permanent: false, replay: true })
 345
 346       await checkVideosExist(liveVideoUUID, false, HttpStatusCode.NOT_FOUND_404)
 347       await checkLiveCleanup(servers[0], liveVideoUUID, [])
 348     })
 349   })
 350
 351   describe('With save replay enabled on permanent live', function () {
 352     let lastReplayUUID: string
 353
 354     it('Should correctly create and federate the "waiting for stream" live', async function () {
 355       this.timeout(20000)
 356
 357       liveVideoUUID = await createLiveWrapper({ permanent: true, replay: true })
 358
 359       await waitJobs(servers)
 360
 361       await checkVideosExist(liveVideoUUID, false, HttpStatusCode.OK_200)
 362       await checkVideoState(liveVideoUUID, VideoState.WAITING_FOR_LIVE)
 363     })
 364
 365     it('Should correctly have updated the live and federated it when streaming in the live', async function () {
 366       this.timeout(20000)
 367
 368       ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID })
 369       await waitUntilLivePublishedOnAllServers(servers, liveVideoUUID)
 370
 371       await waitJobs(servers)
 372
 373       await checkVideosExist(liveVideoUUID, true, HttpStatusCode.OK_200)
 374       await checkVideoState(liveVideoUUID, VideoState.PUBLISHED)
 375     })
 376
 377     it('Should correctly have saved the live and federated it after the streaming', async function () {
 378       this.timeout(30000)
 379
 380       const liveDetails = await servers[0].videos.get({ id: liveVideoUUID })
 381
 382       await stopFfmpeg(ffmpegCommand)
 383
 384       await waitUntilLiveWaitingOnAllServers(servers, liveVideoUUID)
 385       await waitJobs(servers)
 386
 387       const video = await findExternalSavedVideo(servers[0], liveDetails)
 388       expect(video).to.exist
 389
 390       for (const server of servers) {
 391         await server.videos.get({ id: video.uuid })
 392       }
 393
 394       lastReplayUUID = video.uuid
 395     })
 396
 397     it('Should have appropriate ended session and replay live session', async function () {
 398       const { data, total } = await servers[0].live.listSessions({ videoId: liveVideoUUID })
 399       expect(total).to.equal(1)
 400       expect(data).to.have.lengthOf(1)
 401
 402       const sessionFromLive = data[0]
 403       const sessionFromReplay = await servers[0].live.getReplaySession({ videoId: lastReplayUUID })
 404
 405       for (const session of [ sessionFromLive, sessionFromReplay ]) {
 406         expect(session.startDate).to.exist
 407         expect(session.endDate).to.exist
 408
 409         expect(session.error).to.not.exist
 410
 411         expect(session.replayVideo).to.exist
 412         expect(session.replayVideo.id).to.exist
 413         expect(session.replayVideo.shortUUID).to.exist
 414         expect(session.replayVideo.uuid).to.equal(lastReplayUUID)
 415       }
 416     })
 417
 418     it('Should have cleaned up the live files', async function () {
 419       await checkLiveCleanup(servers[0], liveVideoUUID, [])
 420     })
 421
 422     it('Should correctly terminate the stream on blacklist and blacklist the saved replay video', async function () {
 423       this.timeout(120000)
 424
 425       await servers[0].videos.remove({ id: lastReplayUUID })
 426       const { liveDetails } = await publishLiveAndBlacklist({ permanent: true, replay: true })
 427
 428       const replay = await findExternalSavedVideo(servers[0], liveDetails)
 429       expect(replay).to.exist
 430
 431       for (const videoId of [ liveVideoUUID, replay.uuid ]) {
 432         await checkVideosExist(videoId, false)
 433
 434         await servers[0].videos.get({ id: videoId, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 })
 435         await servers[1].videos.get({ id: videoId, expectedStatus: HttpStatusCode.NOT_FOUND_404 })
 436       }
 437
 438       await checkLiveCleanup(servers[0], liveVideoUUID, [])
 439     })
 440
 441     it('Should correctly terminate the stream on delete and not save the video', async function () {
 442       this.timeout(40000)
 443
 444       const { liveDetails } = await publishLiveAndDelete({ permanent: true, replay: true })
 445
 446       const replay = await findExternalSavedVideo(servers[0], liveDetails)
 447       expect(replay).to.not.exist
 448
 449       await checkVideosExist(liveVideoUUID, false, HttpStatusCode.NOT_FOUND_404)
 450       await checkLiveCleanup(servers[0], liveVideoUUID, [])
 451     })
 452   })
 453
 454   after(async function () {
 455     await cleanupTests(servers)
 456   })
 457 })