server/tests/api/live/live-save-replay.ts

   1 /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
   2
   3 import { expect } from 'chai'
   4 import { FfmpegCommand } from 'fluent-ffmpeg'
   5 import { checkLiveCleanup } from '@server/tests/shared'
   6 import { wait } from '@shared/core-utils'
   7 import { HttpStatusCode, LiveVideoCreate, LiveVideoError, VideoPrivacy, VideoState } from '@shared/models'
   8 import {
   9   cleanupTests,
  10   ConfigCommand,
  11   createMultipleServers,
  12   doubleFollow,
  13   findExternalSavedVideo,
  14   PeerTubeServer,
  15   setAccessTokensToServers,
  16   setDefaultVideoChannel,
  17   stopFfmpeg,
  18   testFfmpegStreamError,
  19   waitJobs,
  20   waitUntilLivePublishedOnAllServers,
  21   waitUntilLiveReplacedByReplayOnAllServers,
  22   waitUntilLiveWaitingOnAllServers
  23 } from '@shared/server-commands'
  24
  25 describe('Save replay setting', function () {
  26   let servers: PeerTubeServer[] = []
  27   let liveVideoUUID: string
  28   let ffmpegCommand: FfmpegCommand
  29
  30   async function createLiveWrapper (options: { permanent: boolean, replay: boolean }) {
  31     if (liveVideoUUID) {
  32       try {
  33         await servers[0].videos.remove({ id: liveVideoUUID })
  34         await waitJobs(servers)
  35       } catch {}
  36     }
  37
  38     const attributes: LiveVideoCreate = {
  39       channelId: servers[0].store.channel.id,
  40       privacy: VideoPrivacy.PUBLIC,
  41       name: 'my super live',
  42       saveReplay: options.replay,
  43       permanentLive: options.permanent
  44     }
  45
  46     const { uuid } = await servers[0].live.create({ fields: attributes })
  47     return uuid
  48   }
  49
  50   async function publishLive (options: { permanent: boolean, replay: boolean }) {
  51     liveVideoUUID = await createLiveWrapper(options)
  52
  53     const ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID })
  54     await waitUntilLivePublishedOnAllServers(servers, liveVideoUUID)
  55
  56     const liveDetails = await servers[0].videos.get({ id: liveVideoUUID })
  57
  58     await waitJobs(servers)
  59     await checkVideosExist(liveVideoUUID, true, HttpStatusCode.OK_200)
  60
  61     return { ffmpegCommand, liveDetails }
  62   }
  63
  64   async function publishLiveAndDelete (options: { permanent: boolean, replay: boolean }) {
  65     const { ffmpegCommand, liveDetails } = await publishLive(options)
  66
  67     await Promise.all([
  68       servers[0].videos.remove({ id: liveVideoUUID }),
  69       testFfmpegStreamError(ffmpegCommand, true)
  70     ])
  71
  72     await waitJobs(servers)
  73     await wait(5000)
  74     await waitJobs(servers)
  75
  76     return { liveDetails }
  77   }
  78
  79   async function publishLiveAndBlacklist (options: { permanent: boolean, replay: boolean }) {
  80     const { ffmpegCommand, liveDetails } = await publishLive(options)
  81
  82     await Promise.all([
  83       servers[0].blacklist.add({ videoId: liveVideoUUID, reason: 'bad live', unfederate: true }),
  84       testFfmpegStreamError(ffmpegCommand, true)
  85     ])
  86
  87     await waitJobs(servers)
  88     await wait(5000)
  89     await waitJobs(servers)
  90
  91     return { liveDetails }
  92   }
  93
  94   async function checkVideosExist (videoId: string, existsInList: boolean, expectedStatus?: number) {
  95     for (const server of servers) {
  96       const length = existsInList ? 1 : 0
  97
  98       const { data, total } = await server.videos.list()
  99       expect(data).to.have.lengthOf(length)
 100       expect(total).to.equal(length)
 101
 102       if (expectedStatus) {
 103         await server.videos.get({ id: videoId, expectedStatus })
 104       }
 105     }
 106   }
 107
 108   async function checkVideoState (videoId: string, state: VideoState) {
 109     for (const server of servers) {
 110       const video = await server.videos.get({ id: videoId })
 111       expect(video.state.id).to.equal(state)
 112     }
 113   }
 114
 115   before(async function () {
 116     this.timeout(120000)
 117
 118     servers = await createMultipleServers(2)
 119
 120     // Get the access tokens
 121     await setAccessTokensToServers(servers)
 122     await setDefaultVideoChannel(servers)
 123
 124     // Server 1 and server 2 follow each other
 125     await doubleFollow(servers[0], servers[1])
 126
 127     await servers[0].config.updateCustomSubConfig({
 128       newConfig: {
 129         live: {
 130           enabled: true,
 131           allowReplay: true,
 132           maxDuration: -1,
 133           transcoding: {
 134             enabled: false,
 135             resolutions: ConfigCommand.getCustomConfigResolutions(true)
 136           }
 137         }
 138       }
 139     })
 140   })
 141
 142   describe('With save replay disabled', function () {
 143     let sessionStartDateMin: Date
 144     let sessionStartDateMax: Date
 145     let sessionEndDateMin: Date
 146
 147     it('Should correctly create and federate the "waiting for stream" live', async function () {
 148       this.timeout(20000)
 149
 150       liveVideoUUID = await createLiveWrapper({ permanent: false, replay: false })
 151
 152       await waitJobs(servers)
 153
 154       await checkVideosExist(liveVideoUUID, false, HttpStatusCode.OK_200)
 155       await checkVideoState(liveVideoUUID, VideoState.WAITING_FOR_LIVE)
 156     })
 157
 158     it('Should correctly have updated the live and federated it when streaming in the live', async function () {
 159       this.timeout(30000)
 160
 161       ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID })
 162
 163       sessionStartDateMin = new Date()
 164       await waitUntilLivePublishedOnAllServers(servers, liveVideoUUID)
 165       sessionStartDateMax = new Date()
 166
 167       await waitJobs(servers)
 168
 169       await checkVideosExist(liveVideoUUID, true, HttpStatusCode.OK_200)
 170       await checkVideoState(liveVideoUUID, VideoState.PUBLISHED)
 171     })
 172
 173     it('Should correctly delete the video files after the stream ended', async function () {
 174       this.timeout(40000)
 175
 176       sessionEndDateMin = new Date()
 177       await stopFfmpeg(ffmpegCommand)
 178
 179       for (const server of servers) {
 180         await server.live.waitUntilEnded({ videoId: liveVideoUUID })
 181       }
 182       await waitJobs(servers)
 183
 184       // Live still exist, but cannot be played anymore
 185       await checkVideosExist(liveVideoUUID, false, HttpStatusCode.OK_200)
 186       await checkVideoState(liveVideoUUID, VideoState.LIVE_ENDED)
 187
 188       // No resolutions saved since we did not save replay
 189       await checkLiveCleanup({ server: servers[0], videoUUID: liveVideoUUID, permanent: false })
 190     })
 191
 192     it('Should have appropriate ended session', async function () {
 193       const { data, total } = await servers[0].live.listSessions({ videoId: liveVideoUUID })
 194       expect(total).to.equal(1)
 195       expect(data).to.have.lengthOf(1)
 196
 197       const session = data[0]
 198
 199       const startDate = new Date(session.startDate)
 200       expect(startDate).to.be.above(sessionStartDateMin)
 201       expect(startDate).to.be.below(sessionStartDateMax)
 202
 203       expect(session.endDate).to.exist
 204       expect(new Date(session.endDate)).to.be.above(sessionEndDateMin)
 205
 206       expect(session.saveReplay).to.be.false
 207       expect(session.error).to.not.exist
 208       expect(session.replayVideo).to.not.exist
 209     })
 210
 211     it('Should correctly terminate the stream on blacklist and delete the live', async function () {
 212       this.timeout(40000)
 213
 214       await publishLiveAndBlacklist({ permanent: false, replay: false })
 215
 216       await checkVideosExist(liveVideoUUID, false)
 217
 218       await servers[0].videos.get({ id: liveVideoUUID, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 })
 219       await servers[1].videos.get({ id: liveVideoUUID, expectedStatus: HttpStatusCode.NOT_FOUND_404 })
 220
 221       await wait(5000)
 222       await waitJobs(servers)
 223       await checkLiveCleanup({ server: servers[0], videoUUID: liveVideoUUID, permanent: false })
 224     })
 225
 226     it('Should have blacklisted session error', async function () {
 227       const session = await servers[0].live.findLatestSession({ videoId: liveVideoUUID })
 228       expect(session.startDate).to.exist
 229       expect(session.endDate).to.exist
 230
 231       expect(session.error).to.equal(LiveVideoError.BLACKLISTED)
 232       expect(session.replayVideo).to.not.exist
 233     })
 234
 235     it('Should correctly terminate the stream on delete and delete the video', async function () {
 236       this.timeout(40000)
 237
 238       await publishLiveAndDelete({ permanent: false, replay: false })
 239
 240       await checkVideosExist(liveVideoUUID, false, HttpStatusCode.NOT_FOUND_404)
 241       await checkLiveCleanup({ server: servers[0], videoUUID: liveVideoUUID, permanent: false })
 242     })
 243   })
 244
 245   describe('With save replay enabled on non permanent live', function () {
 246
 247     it('Should correctly create and federate the "waiting for stream" live', async function () {
 248       this.timeout(20000)
 249
 250       liveVideoUUID = await createLiveWrapper({ permanent: false, replay: true })
 251
 252       await waitJobs(servers)
 253
 254       await checkVideosExist(liveVideoUUID, false, HttpStatusCode.OK_200)
 255       await checkVideoState(liveVideoUUID, VideoState.WAITING_FOR_LIVE)
 256     })
 257
 258     it('Should correctly have updated the live and federated it when streaming in the live', async function () {
 259       this.timeout(20000)
 260
 261       ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID })
 262       await waitUntilLivePublishedOnAllServers(servers, liveVideoUUID)
 263
 264       await waitJobs(servers)
 265
 266       await checkVideosExist(liveVideoUUID, true, HttpStatusCode.OK_200)
 267       await checkVideoState(liveVideoUUID, VideoState.PUBLISHED)
 268     })
 269
 270     it('Should correctly have saved the live and federated it after the streaming', async function () {
 271       this.timeout(30000)
 272
 273       const session = await servers[0].live.findLatestSession({ videoId: liveVideoUUID })
 274       expect(session.endDate).to.not.exist
 275       expect(session.endingProcessed).to.be.false
 276       expect(session.saveReplay).to.be.true
 277
 278       await stopFfmpeg(ffmpegCommand)
 279
 280       await waitUntilLiveReplacedByReplayOnAllServers(servers, liveVideoUUID)
 281       await waitJobs(servers)
 282
 283       // Live has been transcoded
 284       await checkVideosExist(liveVideoUUID, true, HttpStatusCode.OK_200)
 285       await checkVideoState(liveVideoUUID, VideoState.PUBLISHED)
 286     })
 287
 288     it('Should find the replay live session', async function () {
 289       const session = await servers[0].live.getReplaySession({ videoId: liveVideoUUID })
 290
 291       expect(session).to.exist
 292
 293       expect(session.startDate).to.exist
 294       expect(session.endDate).to.exist
 295
 296       expect(session.error).to.not.exist
 297       expect(session.saveReplay).to.be.true
 298       expect(session.endingProcessed).to.be.true
 299
 300       expect(session.replayVideo).to.exist
 301       expect(session.replayVideo.id).to.exist
 302       expect(session.replayVideo.shortUUID).to.exist
 303       expect(session.replayVideo.uuid).to.equal(liveVideoUUID)
 304     })
 305
 306     it('Should update the saved live and correctly federate the updated attributes', async function () {
 307       this.timeout(30000)
 308
 309       await servers[0].videos.update({ id: liveVideoUUID, attributes: { name: 'video updated' } })
 310       await waitJobs(servers)
 311
 312       for (const server of servers) {
 313         const video = await server.videos.get({ id: liveVideoUUID })
 314         expect(video.name).to.equal('video updated')
 315         expect(video.isLive).to.be.false
 316       }
 317     })
 318
 319     it('Should have cleaned up the live files', async function () {
 320       await checkLiveCleanup({ server: servers[0], videoUUID: liveVideoUUID, permanent: false, savedResolutions: [ 720 ] })
 321     })
 322
 323     it('Should correctly terminate the stream on blacklist and blacklist the saved replay video', async function () {
 324       this.timeout(120000)
 325
 326       await publishLiveAndBlacklist({ permanent: false, replay: true })
 327
 328       await checkVideosExist(liveVideoUUID, false)
 329
 330       await servers[0].videos.get({ id: liveVideoUUID, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 })
 331       await servers[1].videos.get({ id: liveVideoUUID, expectedStatus: HttpStatusCode.NOT_FOUND_404 })
 332
 333       await wait(5000)
 334       await waitJobs(servers)
 335       await checkLiveCleanup({ server: servers[0], videoUUID: liveVideoUUID, permanent: false, savedResolutions: [ 720 ] })
 336     })
 337
 338     it('Should correctly terminate the stream on delete and delete the video', async function () {
 339       this.timeout(40000)
 340
 341       await publishLiveAndDelete({ permanent: false, replay: true })
 342
 343       await checkVideosExist(liveVideoUUID, false, HttpStatusCode.NOT_FOUND_404)
 344       await checkLiveCleanup({ server: servers[0], videoUUID: liveVideoUUID, permanent: false })
 345     })
 346   })
 347
 348   describe('With save replay enabled on permanent live', function () {
 349     let lastReplayUUID: string
 350
 351     it('Should correctly create and federate the "waiting for stream" live', async function () {
 352       this.timeout(20000)
 353
 354       liveVideoUUID = await createLiveWrapper({ permanent: true, replay: true })
 355
 356       await waitJobs(servers)
 357
 358       await checkVideosExist(liveVideoUUID, false, HttpStatusCode.OK_200)
 359       await checkVideoState(liveVideoUUID, VideoState.WAITING_FOR_LIVE)
 360     })
 361
 362     it('Should correctly have updated the live and federated it when streaming in the live', async function () {
 363       this.timeout(20000)
 364
 365       ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID })
 366       await waitUntilLivePublishedOnAllServers(servers, liveVideoUUID)
 367
 368       await waitJobs(servers)
 369
 370       await checkVideosExist(liveVideoUUID, true, HttpStatusCode.OK_200)
 371       await checkVideoState(liveVideoUUID, VideoState.PUBLISHED)
 372     })
 373
 374     it('Should correctly have saved the live and federated it after the streaming', async function () {
 375       this.timeout(30000)
 376
 377       const liveDetails = await servers[0].videos.get({ id: liveVideoUUID })
 378
 379       await stopFfmpeg(ffmpegCommand)
 380
 381       await waitUntilLiveWaitingOnAllServers(servers, liveVideoUUID)
 382       await waitJobs(servers)
 383
 384       const video = await findExternalSavedVideo(servers[0], liveDetails)
 385       expect(video).to.exist
 386
 387       for (const server of servers) {
 388         await server.videos.get({ id: video.uuid })
 389       }
 390
 391       lastReplayUUID = video.uuid
 392     })
 393
 394     it('Should have appropriate ended session and replay live session', async function () {
 395       const { data, total } = await servers[0].live.listSessions({ videoId: liveVideoUUID })
 396       expect(total).to.equal(1)
 397       expect(data).to.have.lengthOf(1)
 398
 399       const sessionFromLive = data[0]
 400       const sessionFromReplay = await servers[0].live.getReplaySession({ videoId: lastReplayUUID })
 401
 402       for (const session of [ sessionFromLive, sessionFromReplay ]) {
 403         expect(session.startDate).to.exist
 404         expect(session.endDate).to.exist
 405
 406         expect(session.error).to.not.exist
 407
 408         expect(session.replayVideo).to.exist
 409         expect(session.replayVideo.id).to.exist
 410         expect(session.replayVideo.shortUUID).to.exist
 411         expect(session.replayVideo.uuid).to.equal(lastReplayUUID)
 412       }
 413     })
 414
 415     it('Should have cleaned up the live files', async function () {
 416       await checkLiveCleanup({ server: servers[0], videoUUID: liveVideoUUID, permanent: false })
 417     })
 418
 419     it('Should correctly terminate the stream on blacklist and blacklist the saved replay video', async function () {
 420       this.timeout(120000)
 421
 422       await servers[0].videos.remove({ id: lastReplayUUID })
 423       const { liveDetails } = await publishLiveAndBlacklist({ permanent: true, replay: true })
 424
 425       const replay = await findExternalSavedVideo(servers[0], liveDetails)
 426       expect(replay).to.exist
 427
 428       for (const videoId of [ liveVideoUUID, replay.uuid ]) {
 429         await checkVideosExist(videoId, false)
 430
 431         await servers[0].videos.get({ id: videoId, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 })
 432         await servers[1].videos.get({ id: videoId, expectedStatus: HttpStatusCode.NOT_FOUND_404 })
 433       }
 434
 435       await checkLiveCleanup({ server: servers[0], videoUUID: liveVideoUUID, permanent: false })
 436     })
 437
 438     it('Should correctly terminate the stream on delete and not save the video', async function () {
 439       this.timeout(40000)
 440
 441       const { liveDetails } = await publishLiveAndDelete({ permanent: true, replay: true })
 442
 443       const replay = await findExternalSavedVideo(servers[0], liveDetails)
 444       expect(replay).to.not.exist
 445
 446       await checkVideosExist(liveVideoUUID, false, HttpStatusCode.NOT_FOUND_404)
 447       await checkLiveCleanup({ server: servers[0], videoUUID: liveVideoUUID, permanent: false })
 448     })
 449   })
 450
 451   after(async function () {
 452     await cleanupTests(servers)
 453   })
 454 })