server/tests/api/live/live-save-replay.ts

   1 /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
   2
   3 import * as chai from 'chai'
   4 import { FfmpegCommand } from 'fluent-ffmpeg'
   5 import { checkLiveCleanup } from '@server/tests/shared'
   6 import { wait } from '@shared/core-utils'
   7 import { HttpStatusCode, LiveVideoCreate, LiveVideoError, VideoPrivacy, VideoState } from '@shared/models'
   8 import {
   9   cleanupTests,
  10   ConfigCommand,
  11   createMultipleServers,
  12   doubleFollow,
  13   findExternalSavedVideo,
  14   PeerTubeServer,
  15   setAccessTokensToServers,
  16   setDefaultVideoChannel,
  17   stopFfmpeg,
  18   testFfmpegStreamError,
  19   waitJobs,
  20   waitUntilLivePublishedOnAllServers,
  21   waitUntilLiveReplacedByReplayOnAllServers,
  22   waitUntilLiveWaitingOnAllServers
  23 } from '@shared/server-commands'
  24
  25 const expect = chai.expect
  26
  27 describe('Save replay setting', function () {
  28   let servers: PeerTubeServer[] = []
  29   let liveVideoUUID: string
  30   let ffmpegCommand: FfmpegCommand
  31
  32   async function createLiveWrapper (options: { permanent: boolean, replay: boolean }) {
  33     if (liveVideoUUID) {
  34       try {
  35         await servers[0].videos.remove({ id: liveVideoUUID })
  36         await waitJobs(servers)
  37       } catch {}
  38     }
  39
  40     const attributes: LiveVideoCreate = {
  41       channelId: servers[0].store.channel.id,
  42       privacy: VideoPrivacy.PUBLIC,
  43       name: 'my super live',
  44       saveReplay: options.replay,
  45       permanentLive: options.permanent
  46     }
  47
  48     const { uuid } = await servers[0].live.create({ fields: attributes })
  49     return uuid
  50   }
  51
  52   async function publishLive (options: { permanent: boolean, replay: boolean }) {
  53     liveVideoUUID = await createLiveWrapper(options)
  54
  55     const ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID })
  56     await waitUntilLivePublishedOnAllServers(servers, liveVideoUUID)
  57
  58     const liveDetails = await servers[0].videos.get({ id: liveVideoUUID })
  59
  60     await waitJobs(servers)
  61     await checkVideosExist(liveVideoUUID, true, HttpStatusCode.OK_200)
  62
  63     return { ffmpegCommand, liveDetails }
  64   }
  65
  66   async function publishLiveAndDelete (options: { permanent: boolean, replay: boolean }) {
  67     const { ffmpegCommand, liveDetails } = await publishLive(options)
  68
  69     await Promise.all([
  70       servers[0].videos.remove({ id: liveVideoUUID }),
  71       testFfmpegStreamError(ffmpegCommand, true)
  72     ])
  73
  74     await waitJobs(servers)
  75     await wait(5000)
  76     await waitJobs(servers)
  77
  78     return { liveDetails }
  79   }
  80
  81   async function publishLiveAndBlacklist (options: { permanent: boolean, replay: boolean }) {
  82     const { ffmpegCommand, liveDetails } = await publishLive(options)
  83
  84     await Promise.all([
  85       servers[0].blacklist.add({ videoId: liveVideoUUID, reason: 'bad live', unfederate: true }),
  86       testFfmpegStreamError(ffmpegCommand, true)
  87     ])
  88
  89     await waitJobs(servers)
  90     await wait(5000)
  91     await waitJobs(servers)
  92
  93     return { liveDetails }
  94   }
  95
  96   async function checkVideosExist (videoId: string, existsInList: boolean, expectedStatus?: number) {
  97     for (const server of servers) {
  98       const length = existsInList ? 1 : 0
  99
 100       const { data, total } = await server.videos.list()
 101       expect(data).to.have.lengthOf(length)
 102       expect(total).to.equal(length)
 103
 104       if (expectedStatus) {
 105         await server.videos.get({ id: videoId, expectedStatus })
 106       }
 107     }
 108   }
 109
 110   async function checkVideoState (videoId: string, state: VideoState) {
 111     for (const server of servers) {
 112       const video = await server.videos.get({ id: videoId })
 113       expect(video.state.id).to.equal(state)
 114     }
 115   }
 116
 117   before(async function () {
 118     this.timeout(120000)
 119
 120     servers = await createMultipleServers(2)
 121
 122     // Get the access tokens
 123     await setAccessTokensToServers(servers)
 124     await setDefaultVideoChannel(servers)
 125
 126     // Server 1 and server 2 follow each other
 127     await doubleFollow(servers[0], servers[1])
 128
 129     await servers[0].config.updateCustomSubConfig({
 130       newConfig: {
 131         live: {
 132           enabled: true,
 133           allowReplay: true,
 134           maxDuration: -1,
 135           transcoding: {
 136             enabled: false,
 137             resolutions: ConfigCommand.getCustomConfigResolutions(true)
 138           }
 139         }
 140       }
 141     })
 142   })
 143
 144   describe('With save replay disabled', function () {
 145     let sessionStartDateMin: Date
 146     let sessionStartDateMax: Date
 147     let sessionEndDateMin: Date
 148
 149     it('Should correctly create and federate the "waiting for stream" live', async function () {
 150       this.timeout(20000)
 151
 152       liveVideoUUID = await createLiveWrapper({ permanent: false, replay: false })
 153
 154       await waitJobs(servers)
 155
 156       await checkVideosExist(liveVideoUUID, false, HttpStatusCode.OK_200)
 157       await checkVideoState(liveVideoUUID, VideoState.WAITING_FOR_LIVE)
 158     })
 159
 160     it('Should correctly have updated the live and federated it when streaming in the live', async function () {
 161       this.timeout(30000)
 162
 163       ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID })
 164
 165       sessionStartDateMin = new Date()
 166       await waitUntilLivePublishedOnAllServers(servers, liveVideoUUID)
 167       sessionStartDateMax = new Date()
 168
 169       await waitJobs(servers)
 170
 171       await checkVideosExist(liveVideoUUID, true, HttpStatusCode.OK_200)
 172       await checkVideoState(liveVideoUUID, VideoState.PUBLISHED)
 173     })
 174
 175     it('Should correctly delete the video files after the stream ended', async function () {
 176       this.timeout(40000)
 177
 178       sessionEndDateMin = new Date()
 179       await stopFfmpeg(ffmpegCommand)
 180
 181       for (const server of servers) {
 182         await server.live.waitUntilEnded({ videoId: liveVideoUUID })
 183       }
 184       await waitJobs(servers)
 185
 186       // Live still exist, but cannot be played anymore
 187       await checkVideosExist(liveVideoUUID, false, HttpStatusCode.OK_200)
 188       await checkVideoState(liveVideoUUID, VideoState.LIVE_ENDED)
 189
 190       // No resolutions saved since we did not save replay
 191       await checkLiveCleanup(servers[0], liveVideoUUID, [])
 192     })
 193
 194     it('Should have appropriate ended session', async function () {
 195       const { data, total } = await servers[0].live.listSessions({ videoId: liveVideoUUID })
 196       expect(total).to.equal(1)
 197       expect(data).to.have.lengthOf(1)
 198
 199       const session = data[0]
 200
 201       const startDate = new Date(session.startDate)
 202       expect(startDate).to.be.above(sessionStartDateMin)
 203       expect(startDate).to.be.below(sessionStartDateMax)
 204
 205       expect(session.endDate).to.exist
 206       expect(new Date(session.endDate)).to.be.above(sessionEndDateMin)
 207
 208       expect(session.saveReplay).to.be.false
 209       expect(session.error).to.not.exist
 210       expect(session.replayVideo).to.not.exist
 211     })
 212
 213     it('Should correctly terminate the stream on blacklist and delete the live', async function () {
 214       this.timeout(40000)
 215
 216       await publishLiveAndBlacklist({ permanent: false, replay: false })
 217
 218       await checkVideosExist(liveVideoUUID, false)
 219
 220       await servers[0].videos.get({ id: liveVideoUUID, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 })
 221       await servers[1].videos.get({ id: liveVideoUUID, expectedStatus: HttpStatusCode.NOT_FOUND_404 })
 222
 223       await wait(5000)
 224       await waitJobs(servers)
 225       await checkLiveCleanup(servers[0], liveVideoUUID, [])
 226     })
 227
 228     it('Should have blacklisted session error', async function () {
 229       const session = await servers[0].live.findLatestSession({ videoId: liveVideoUUID })
 230       expect(session.startDate).to.exist
 231       expect(session.endDate).to.exist
 232
 233       expect(session.error).to.equal(LiveVideoError.BLACKLISTED)
 234       expect(session.replayVideo).to.not.exist
 235     })
 236
 237     it('Should correctly terminate the stream on delete and delete the video', async function () {
 238       this.timeout(40000)
 239
 240       await publishLiveAndDelete({ permanent: false, replay: false })
 241
 242       await checkVideosExist(liveVideoUUID, false, HttpStatusCode.NOT_FOUND_404)
 243       await checkLiveCleanup(servers[0], liveVideoUUID, [])
 244     })
 245   })
 246
 247   describe('With save replay enabled on non permanent live', function () {
 248
 249     it('Should correctly create and federate the "waiting for stream" live', async function () {
 250       this.timeout(20000)
 251
 252       liveVideoUUID = await createLiveWrapper({ permanent: false, replay: true })
 253
 254       await waitJobs(servers)
 255
 256       await checkVideosExist(liveVideoUUID, false, HttpStatusCode.OK_200)
 257       await checkVideoState(liveVideoUUID, VideoState.WAITING_FOR_LIVE)
 258     })
 259
 260     it('Should correctly have updated the live and federated it when streaming in the live', async function () {
 261       this.timeout(20000)
 262
 263       ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID })
 264       await waitUntilLivePublishedOnAllServers(servers, liveVideoUUID)
 265
 266       await waitJobs(servers)
 267
 268       await checkVideosExist(liveVideoUUID, true, HttpStatusCode.OK_200)
 269       await checkVideoState(liveVideoUUID, VideoState.PUBLISHED)
 270     })
 271
 272     it('Should correctly have saved the live and federated it after the streaming', async function () {
 273       this.timeout(30000)
 274
 275       const session = await servers[0].live.findLatestSession({ videoId: liveVideoUUID })
 276       expect(session.endDate).to.not.exist
 277       expect(session.endingProcessed).to.be.false
 278       expect(session.saveReplay).to.be.true
 279
 280       await stopFfmpeg(ffmpegCommand)
 281
 282       await waitUntilLiveReplacedByReplayOnAllServers(servers, liveVideoUUID)
 283       await waitJobs(servers)
 284
 285       // Live has been transcoded
 286       await checkVideosExist(liveVideoUUID, true, HttpStatusCode.OK_200)
 287       await checkVideoState(liveVideoUUID, VideoState.PUBLISHED)
 288     })
 289
 290     it('Should find the replay live session', async function () {
 291       const session = await servers[0].live.getReplaySession({ videoId: liveVideoUUID })
 292
 293       expect(session).to.exist
 294
 295       expect(session.startDate).to.exist
 296       expect(session.endDate).to.exist
 297
 298       expect(session.error).to.not.exist
 299       expect(session.saveReplay).to.be.true
 300       expect(session.endingProcessed).to.be.true
 301
 302       expect(session.replayVideo).to.exist
 303       expect(session.replayVideo.id).to.exist
 304       expect(session.replayVideo.shortUUID).to.exist
 305       expect(session.replayVideo.uuid).to.equal(liveVideoUUID)
 306     })
 307
 308     it('Should update the saved live and correctly federate the updated attributes', async function () {
 309       this.timeout(30000)
 310
 311       await servers[0].videos.update({ id: liveVideoUUID, attributes: { name: 'video updated' } })
 312       await waitJobs(servers)
 313
 314       for (const server of servers) {
 315         const video = await server.videos.get({ id: liveVideoUUID })
 316         expect(video.name).to.equal('video updated')
 317         expect(video.isLive).to.be.false
 318       }
 319     })
 320
 321     it('Should have cleaned up the live files', async function () {
 322       await checkLiveCleanup(servers[0], liveVideoUUID, [ 720 ])
 323     })
 324
 325     it('Should correctly terminate the stream on blacklist and blacklist the saved replay video', async function () {
 326       this.timeout(120000)
 327
 328       await publishLiveAndBlacklist({ permanent: false, replay: true })
 329
 330       await checkVideosExist(liveVideoUUID, false)
 331
 332       await servers[0].videos.get({ id: liveVideoUUID, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 })
 333       await servers[1].videos.get({ id: liveVideoUUID, expectedStatus: HttpStatusCode.NOT_FOUND_404 })
 334
 335       await wait(5000)
 336       await waitJobs(servers)
 337       await checkLiveCleanup(servers[0], liveVideoUUID, [ 720 ])
 338     })
 339
 340     it('Should correctly terminate the stream on delete and delete the video', async function () {
 341       this.timeout(40000)
 342
 343       await publishLiveAndDelete({ permanent: false, replay: true })
 344
 345       await checkVideosExist(liveVideoUUID, false, HttpStatusCode.NOT_FOUND_404)
 346       await checkLiveCleanup(servers[0], liveVideoUUID, [])
 347     })
 348   })
 349
 350   describe('With save replay enabled on permanent live', function () {
 351     let lastReplayUUID: string
 352
 353     it('Should correctly create and federate the "waiting for stream" live', async function () {
 354       this.timeout(20000)
 355
 356       liveVideoUUID = await createLiveWrapper({ permanent: true, replay: true })
 357
 358       await waitJobs(servers)
 359
 360       await checkVideosExist(liveVideoUUID, false, HttpStatusCode.OK_200)
 361       await checkVideoState(liveVideoUUID, VideoState.WAITING_FOR_LIVE)
 362     })
 363
 364     it('Should correctly have updated the live and federated it when streaming in the live', async function () {
 365       this.timeout(20000)
 366
 367       ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID })
 368       await waitUntilLivePublishedOnAllServers(servers, liveVideoUUID)
 369
 370       await waitJobs(servers)
 371
 372       await checkVideosExist(liveVideoUUID, true, HttpStatusCode.OK_200)
 373       await checkVideoState(liveVideoUUID, VideoState.PUBLISHED)
 374     })
 375
 376     it('Should correctly have saved the live and federated it after the streaming', async function () {
 377       this.timeout(30000)
 378
 379       const liveDetails = await servers[0].videos.get({ id: liveVideoUUID })
 380
 381       await stopFfmpeg(ffmpegCommand)
 382
 383       await waitUntilLiveWaitingOnAllServers(servers, liveVideoUUID)
 384       await waitJobs(servers)
 385
 386       const video = await findExternalSavedVideo(servers[0], liveDetails)
 387       expect(video).to.exist
 388
 389       for (const server of servers) {
 390         await server.videos.get({ id: video.uuid })
 391       }
 392
 393       lastReplayUUID = video.uuid
 394     })
 395
 396     it('Should have appropriate ended session and replay live session', async function () {
 397       const { data, total } = await servers[0].live.listSessions({ videoId: liveVideoUUID })
 398       expect(total).to.equal(1)
 399       expect(data).to.have.lengthOf(1)
 400
 401       const sessionFromLive = data[0]
 402       const sessionFromReplay = await servers[0].live.getReplaySession({ videoId: lastReplayUUID })
 403
 404       for (const session of [ sessionFromLive, sessionFromReplay ]) {
 405         expect(session.startDate).to.exist
 406         expect(session.endDate).to.exist
 407
 408         expect(session.error).to.not.exist
 409
 410         expect(session.replayVideo).to.exist
 411         expect(session.replayVideo.id).to.exist
 412         expect(session.replayVideo.shortUUID).to.exist
 413         expect(session.replayVideo.uuid).to.equal(lastReplayUUID)
 414       }
 415     })
 416
 417     it('Should have cleaned up the live files', async function () {
 418       await checkLiveCleanup(servers[0], liveVideoUUID, [])
 419     })
 420
 421     it('Should correctly terminate the stream on blacklist and blacklist the saved replay video', async function () {
 422       this.timeout(120000)
 423
 424       await servers[0].videos.remove({ id: lastReplayUUID })
 425       const { liveDetails } = await publishLiveAndBlacklist({ permanent: true, replay: true })
 426
 427       const replay = await findExternalSavedVideo(servers[0], liveDetails)
 428       expect(replay).to.exist
 429
 430       for (const videoId of [ liveVideoUUID, replay.uuid ]) {
 431         await checkVideosExist(videoId, false)
 432
 433         await servers[0].videos.get({ id: videoId, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 })
 434         await servers[1].videos.get({ id: videoId, expectedStatus: HttpStatusCode.NOT_FOUND_404 })
 435       }
 436
 437       await checkLiveCleanup(servers[0], liveVideoUUID, [])
 438     })
 439
 440     it('Should correctly terminate the stream on delete and not save the video', async function () {
 441       this.timeout(40000)
 442
 443       const { liveDetails } = await publishLiveAndDelete({ permanent: true, replay: true })
 444
 445       const replay = await findExternalSavedVideo(servers[0], liveDetails)
 446       expect(replay).to.not.exist
 447
 448       await checkVideosExist(liveVideoUUID, false, HttpStatusCode.NOT_FOUND_404)
 449       await checkLiveCleanup(servers[0], liveVideoUUID, [])
 450     })
 451   })
 452
 453   after(async function () {
 454     await cleanupTests(servers)
 455   })
 456 })