server/tests/api/live/live-save-replay.ts

   1 /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
   2
   3 import 'mocha'
   4 import * as chai from 'chai'
   5 import { FfmpegCommand } from 'fluent-ffmpeg'
   6 import { checkLiveCleanup } from '@server/tests/shared'
   7 import { wait } from '@shared/core-utils'
   8 import { HttpStatusCode, LiveVideoCreate, LiveVideoError, VideoPrivacy, VideoState } from '@shared/models'
   9 import {
  10   cleanupTests,
  11   ConfigCommand,
  12   createMultipleServers,
  13   doubleFollow,
  14   findExternalSavedVideo,
  15   PeerTubeServer,
  16   setAccessTokensToServers,
  17   setDefaultVideoChannel,
  18   stopFfmpeg,
  19   testFfmpegStreamError,
  20   waitJobs,
  21   waitUntilLivePublishedOnAllServers,
  22   waitUntilLiveReplacedByReplayOnAllServers,
  23   waitUntilLiveWaitingOnAllServers
  24 } from '@shared/server-commands'
  25
  26 const expect = chai.expect
  27
  28 describe('Save replay setting', function () {
  29   let servers: PeerTubeServer[] = []
  30   let liveVideoUUID: string
  31   let ffmpegCommand: FfmpegCommand
  32
  33   async function createLiveWrapper (options: { permanent: boolean, replay: boolean }) {
  34     if (liveVideoUUID) {
  35       try {
  36         await servers[0].videos.remove({ id: liveVideoUUID })
  37         await waitJobs(servers)
  38       } catch {}
  39     }
  40
  41     const attributes: LiveVideoCreate = {
  42       channelId: servers[0].store.channel.id,
  43       privacy: VideoPrivacy.PUBLIC,
  44       name: 'my super live',
  45       saveReplay: options.replay,
  46       permanentLive: options.permanent
  47     }
  48
  49     const { uuid } = await servers[0].live.create({ fields: attributes })
  50     return uuid
  51   }
  52
  53   async function publishLive (options: { permanent: boolean, replay: boolean }) {
  54     liveVideoUUID = await createLiveWrapper(options)
  55
  56     const ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID })
  57     await waitUntilLivePublishedOnAllServers(servers, liveVideoUUID)
  58
  59     const liveDetails = await servers[0].videos.get({ id: liveVideoUUID })
  60
  61     await waitJobs(servers)
  62     await checkVideosExist(liveVideoUUID, true, HttpStatusCode.OK_200)
  63
  64     return { ffmpegCommand, liveDetails }
  65   }
  66
  67   async function publishLiveAndDelete (options: { permanent: boolean, replay: boolean }) {
  68     const { ffmpegCommand, liveDetails } = await publishLive(options)
  69
  70     await Promise.all([
  71       servers[0].videos.remove({ id: liveVideoUUID }),
  72       testFfmpegStreamError(ffmpegCommand, true)
  73     ])
  74
  75     await waitJobs(servers)
  76     await wait(5000)
  77     await waitJobs(servers)
  78
  79     return { liveDetails }
  80   }
  81
  82   async function publishLiveAndBlacklist (options: { permanent: boolean, replay: boolean }) {
  83     const { ffmpegCommand, liveDetails } = await publishLive(options)
  84
  85     await Promise.all([
  86       servers[0].blacklist.add({ videoId: liveVideoUUID, reason: 'bad live', unfederate: true }),
  87       testFfmpegStreamError(ffmpegCommand, true)
  88     ])
  89
  90     await waitJobs(servers)
  91     await wait(5000)
  92     await waitJobs(servers)
  93
  94     return { liveDetails }
  95   }
  96
  97   async function checkVideosExist (videoId: string, existsInList: boolean, expectedStatus?: number) {
  98     for (const server of servers) {
  99       const length = existsInList ? 1 : 0
 100
 101       const { data, total } = await server.videos.list()
 102       expect(data).to.have.lengthOf(length)
 103       expect(total).to.equal(length)
 104
 105       if (expectedStatus) {
 106         await server.videos.get({ id: videoId, expectedStatus })
 107       }
 108     }
 109   }
 110
 111   async function checkVideoState (videoId: string, state: VideoState) {
 112     for (const server of servers) {
 113       const video = await server.videos.get({ id: videoId })
 114       expect(video.state.id).to.equal(state)
 115     }
 116   }
 117
 118   before(async function () {
 119     this.timeout(120000)
 120
 121     servers = await createMultipleServers(2)
 122
 123     // Get the access tokens
 124     await setAccessTokensToServers(servers)
 125     await setDefaultVideoChannel(servers)
 126
 127     // Server 1 and server 2 follow each other
 128     await doubleFollow(servers[0], servers[1])
 129
 130     await servers[0].config.updateCustomSubConfig({
 131       newConfig: {
 132         live: {
 133           enabled: true,
 134           allowReplay: true,
 135           maxDuration: -1,
 136           transcoding: {
 137             enabled: false,
 138             resolutions: ConfigCommand.getCustomConfigResolutions(true)
 139           }
 140         }
 141       }
 142     })
 143   })
 144
 145   describe('With save replay disabled', function () {
 146     let sessionStartDateMin: Date
 147     let sessionStartDateMax: Date
 148     let sessionEndDateMin: Date
 149
 150     it('Should correctly create and federate the "waiting for stream" live', async function () {
 151       this.timeout(20000)
 152
 153       liveVideoUUID = await createLiveWrapper({ permanent: false, replay: false })
 154
 155       await waitJobs(servers)
 156
 157       await checkVideosExist(liveVideoUUID, false, HttpStatusCode.OK_200)
 158       await checkVideoState(liveVideoUUID, VideoState.WAITING_FOR_LIVE)
 159     })
 160
 161     it('Should correctly have updated the live and federated it when streaming in the live', async function () {
 162       this.timeout(30000)
 163
 164       ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID })
 165
 166       sessionStartDateMin = new Date()
 167       await waitUntilLivePublishedOnAllServers(servers, liveVideoUUID)
 168       sessionStartDateMax = new Date()
 169
 170       await waitJobs(servers)
 171
 172       await checkVideosExist(liveVideoUUID, true, HttpStatusCode.OK_200)
 173       await checkVideoState(liveVideoUUID, VideoState.PUBLISHED)
 174     })
 175
 176     it('Should correctly delete the video files after the stream ended', async function () {
 177       this.timeout(40000)
 178
 179       sessionEndDateMin = new Date()
 180       await stopFfmpeg(ffmpegCommand)
 181
 182       for (const server of servers) {
 183         await server.live.waitUntilEnded({ videoId: liveVideoUUID })
 184       }
 185       await waitJobs(servers)
 186
 187       // Live still exist, but cannot be played anymore
 188       await checkVideosExist(liveVideoUUID, false, HttpStatusCode.OK_200)
 189       await checkVideoState(liveVideoUUID, VideoState.LIVE_ENDED)
 190
 191       // No resolutions saved since we did not save replay
 192       await checkLiveCleanup(servers[0], liveVideoUUID, [])
 193     })
 194
 195     it('Should have appropriate ended session', async function () {
 196       const { data, total } = await servers[0].live.listSessions({ videoId: liveVideoUUID })
 197       expect(total).to.equal(1)
 198       expect(data).to.have.lengthOf(1)
 199
 200       const session = data[0]
 201
 202       const startDate = new Date(session.startDate)
 203       expect(startDate).to.be.above(sessionStartDateMin)
 204       expect(startDate).to.be.below(sessionStartDateMax)
 205
 206       expect(session.endDate).to.exist
 207       expect(new Date(session.endDate)).to.be.above(sessionEndDateMin)
 208
 209       expect(session.error).to.not.exist
 210       expect(session.replayVideo).to.not.exist
 211     })
 212
 213     it('Should correctly terminate the stream on blacklist and delete the live', async function () {
 214       this.timeout(40000)
 215
 216       await publishLiveAndBlacklist({ permanent: false, replay: false })
 217
 218       await checkVideosExist(liveVideoUUID, false)
 219
 220       await servers[0].videos.get({ id: liveVideoUUID, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 })
 221       await servers[1].videos.get({ id: liveVideoUUID, expectedStatus: HttpStatusCode.NOT_FOUND_404 })
 222
 223       await wait(5000)
 224       await waitJobs(servers)
 225       await checkLiveCleanup(servers[0], liveVideoUUID, [])
 226     })
 227
 228     it('Should have blacklisted session error', async function () {
 229       const session = await servers[0].live.findLatestSession({ videoId: liveVideoUUID })
 230       expect(session.startDate).to.exist
 231       expect(session.endDate).to.exist
 232
 233       expect(session.error).to.equal(LiveVideoError.BLACKLISTED)
 234       expect(session.replayVideo).to.not.exist
 235     })
 236
 237     it('Should correctly terminate the stream on delete and delete the video', async function () {
 238       this.timeout(40000)
 239
 240       await publishLiveAndDelete({ permanent: false, replay: false })
 241
 242       await checkVideosExist(liveVideoUUID, false, HttpStatusCode.NOT_FOUND_404)
 243       await checkLiveCleanup(servers[0], liveVideoUUID, [])
 244     })
 245   })
 246
 247   describe('With save replay enabled on non permanent live', function () {
 248
 249     it('Should correctly create and federate the "waiting for stream" live', async function () {
 250       this.timeout(20000)
 251
 252       liveVideoUUID = await createLiveWrapper({ permanent: false, replay: true })
 253
 254       await waitJobs(servers)
 255
 256       await checkVideosExist(liveVideoUUID, false, HttpStatusCode.OK_200)
 257       await checkVideoState(liveVideoUUID, VideoState.WAITING_FOR_LIVE)
 258     })
 259
 260     it('Should correctly have updated the live and federated it when streaming in the live', async function () {
 261       this.timeout(20000)
 262
 263       ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID })
 264       await waitUntilLivePublishedOnAllServers(servers, liveVideoUUID)
 265
 266       await waitJobs(servers)
 267
 268       await checkVideosExist(liveVideoUUID, true, HttpStatusCode.OK_200)
 269       await checkVideoState(liveVideoUUID, VideoState.PUBLISHED)
 270     })
 271
 272     it('Should correctly have saved the live and federated it after the streaming', async function () {
 273       this.timeout(30000)
 274
 275       await stopFfmpeg(ffmpegCommand)
 276
 277       await waitUntilLiveReplacedByReplayOnAllServers(servers, liveVideoUUID)
 278       await waitJobs(servers)
 279
 280       // Live has been transcoded
 281       await checkVideosExist(liveVideoUUID, true, HttpStatusCode.OK_200)
 282       await checkVideoState(liveVideoUUID, VideoState.PUBLISHED)
 283     })
 284
 285     it('Should find the replay live session', async function () {
 286       const session = await servers[0].live.getReplaySession({ videoId: liveVideoUUID })
 287
 288       expect(session).to.exist
 289
 290       expect(session.startDate).to.exist
 291       expect(session.endDate).to.exist
 292
 293       expect(session.error).to.not.exist
 294
 295       expect(session.replayVideo).to.exist
 296       expect(session.replayVideo.id).to.exist
 297       expect(session.replayVideo.shortUUID).to.exist
 298       expect(session.replayVideo.uuid).to.equal(liveVideoUUID)
 299     })
 300
 301     it('Should update the saved live and correctly federate the updated attributes', async function () {
 302       this.timeout(30000)
 303
 304       await servers[0].videos.update({ id: liveVideoUUID, attributes: { name: 'video updated' } })
 305       await waitJobs(servers)
 306
 307       for (const server of servers) {
 308         const video = await server.videos.get({ id: liveVideoUUID })
 309         expect(video.name).to.equal('video updated')
 310         expect(video.isLive).to.be.false
 311       }
 312     })
 313
 314     it('Should have cleaned up the live files', async function () {
 315       await checkLiveCleanup(servers[0], liveVideoUUID, [ 720 ])
 316     })
 317
 318     it('Should correctly terminate the stream on blacklist and blacklist the saved replay video', async function () {
 319       this.timeout(40000)
 320
 321       await publishLiveAndBlacklist({ permanent: false, replay: true })
 322
 323       await checkVideosExist(liveVideoUUID, false)
 324
 325       await servers[0].videos.get({ id: liveVideoUUID, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 })
 326       await servers[1].videos.get({ id: liveVideoUUID, expectedStatus: HttpStatusCode.NOT_FOUND_404 })
 327
 328       await wait(5000)
 329       await waitJobs(servers)
 330       await checkLiveCleanup(servers[0], liveVideoUUID, [ 720 ])
 331     })
 332
 333     it('Should correctly terminate the stream on delete and delete the video', async function () {
 334       this.timeout(40000)
 335
 336       await publishLiveAndDelete({ permanent: false, replay: true })
 337
 338       await checkVideosExist(liveVideoUUID, false, HttpStatusCode.NOT_FOUND_404)
 339       await checkLiveCleanup(servers[0], liveVideoUUID, [])
 340     })
 341   })
 342
 343   describe('With save replay enabled on permanent live', function () {
 344     let lastReplayUUID: string
 345
 346     it('Should correctly create and federate the "waiting for stream" live', async function () {
 347       this.timeout(20000)
 348
 349       liveVideoUUID = await createLiveWrapper({ permanent: true, replay: true })
 350
 351       await waitJobs(servers)
 352
 353       await checkVideosExist(liveVideoUUID, false, HttpStatusCode.OK_200)
 354       await checkVideoState(liveVideoUUID, VideoState.WAITING_FOR_LIVE)
 355     })
 356
 357     it('Should correctly have updated the live and federated it when streaming in the live', async function () {
 358       this.timeout(20000)
 359
 360       ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID })
 361       await waitUntilLivePublishedOnAllServers(servers, liveVideoUUID)
 362
 363       await waitJobs(servers)
 364
 365       await checkVideosExist(liveVideoUUID, true, HttpStatusCode.OK_200)
 366       await checkVideoState(liveVideoUUID, VideoState.PUBLISHED)
 367     })
 368
 369     it('Should correctly have saved the live and federated it after the streaming', async function () {
 370       this.timeout(30000)
 371
 372       const liveDetails = await servers[0].videos.get({ id: liveVideoUUID })
 373
 374       await stopFfmpeg(ffmpegCommand)
 375
 376       await waitUntilLiveWaitingOnAllServers(servers, liveVideoUUID)
 377       await waitJobs(servers)
 378
 379       const video = await findExternalSavedVideo(servers[0], liveDetails)
 380       expect(video).to.exist
 381
 382       for (const server of servers) {
 383         await server.videos.get({ id: video.uuid })
 384       }
 385
 386       lastReplayUUID = video.uuid
 387     })
 388
 389     it('Should have appropriate ended session and replay live session', async function () {
 390       const { data, total } = await servers[0].live.listSessions({ videoId: liveVideoUUID })
 391       expect(total).to.equal(1)
 392       expect(data).to.have.lengthOf(1)
 393
 394       const sessionFromLive = data[0]
 395       const sessionFromReplay = await servers[0].live.getReplaySession({ videoId: lastReplayUUID })
 396
 397       for (const session of [ sessionFromLive, sessionFromReplay ]) {
 398         expect(session.startDate).to.exist
 399         expect(session.endDate).to.exist
 400
 401         expect(session.error).to.not.exist
 402
 403         expect(session.replayVideo).to.exist
 404         expect(session.replayVideo.id).to.exist
 405         expect(session.replayVideo.shortUUID).to.exist
 406         expect(session.replayVideo.uuid).to.equal(lastReplayUUID)
 407       }
 408     })
 409
 410     it('Should have cleaned up the live files', async function () {
 411       await checkLiveCleanup(servers[0], liveVideoUUID, [])
 412     })
 413
 414     it('Should correctly terminate the stream on blacklist and blacklist the saved replay video', async function () {
 415       this.timeout(60000)
 416
 417       await servers[0].videos.remove({ id: lastReplayUUID })
 418       const { liveDetails } = await publishLiveAndBlacklist({ permanent: true, replay: true })
 419
 420       const replay = await findExternalSavedVideo(servers[0], liveDetails)
 421       expect(replay).to.exist
 422
 423       for (const videoId of [ liveVideoUUID, replay.uuid ]) {
 424         await checkVideosExist(videoId, false)
 425
 426         await servers[0].videos.get({ id: videoId, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 })
 427         await servers[1].videos.get({ id: videoId, expectedStatus: HttpStatusCode.NOT_FOUND_404 })
 428       }
 429
 430       await checkLiveCleanup(servers[0], liveVideoUUID, [])
 431     })
 432
 433     it('Should correctly terminate the stream on delete and not save the video', async function () {
 434       this.timeout(40000)
 435
 436       const { liveDetails } = await publishLiveAndDelete({ permanent: true, replay: true })
 437
 438       const replay = await findExternalSavedVideo(servers[0], liveDetails)
 439       expect(replay).to.not.exist
 440
 441       await checkVideosExist(liveVideoUUID, false, HttpStatusCode.NOT_FOUND_404)
 442       await checkLiveCleanup(servers[0], liveVideoUUID, [])
 443     })
 444   })
 445
 446   after(async function () {
 447     await cleanupTests(servers)
 448   })
 449 })