server/tests/api/live/live-save-replay.ts

   1 /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
   2
   3 import 'mocha'
   4 import * as chai from 'chai'
   5 import { FfmpegCommand } from 'fluent-ffmpeg'
   6 import { checkLiveCleanup } from '@server/tests/shared'
   7 import { wait } from '@shared/core-utils'
   8 import { HttpStatusCode, LiveVideoCreate, VideoPrivacy, VideoState } from '@shared/models'
   9 import {
  10   cleanupTests,
  11   ConfigCommand,
  12   createMultipleServers,
  13   doubleFollow,
  14   findExternalSavedVideo,
  15   PeerTubeServer,
  16   setAccessTokensToServers,
  17   setDefaultVideoChannel,
  18   stopFfmpeg,
  19   testFfmpegStreamError,
  20   waitJobs,
  21   waitUntilLivePublishedOnAllServers,
  22   waitUntilLiveReplacedByReplayOnAllServers,
  23   waitUntilLiveWaitingOnAllServers
  24 } from '@shared/server-commands'
  25
  26 const expect = chai.expect
  27
  28 describe('Save replay setting', function () {
  29   let servers: PeerTubeServer[] = []
  30   let liveVideoUUID: string
  31   let ffmpegCommand: FfmpegCommand
  32
  33   async function createLiveWrapper (options: { permanent: boolean, replay: boolean }) {
  34     if (liveVideoUUID) {
  35       try {
  36         await servers[0].videos.remove({ id: liveVideoUUID })
  37         await waitJobs(servers)
  38       } catch {}
  39     }
  40
  41     const attributes: LiveVideoCreate = {
  42       channelId: servers[0].store.channel.id,
  43       privacy: VideoPrivacy.PUBLIC,
  44       name: 'my super live',
  45       saveReplay: options.replay,
  46       permanentLive: options.permanent
  47     }
  48
  49     const { uuid } = await servers[0].live.create({ fields: attributes })
  50     return uuid
  51   }
  52
  53   async function checkVideosExist (videoId: string, existsInList: boolean, expectedStatus?: number) {
  54     for (const server of servers) {
  55       const length = existsInList ? 1 : 0
  56
  57       const { data, total } = await server.videos.list()
  58       expect(data).to.have.lengthOf(length)
  59       expect(total).to.equal(length)
  60
  61       if (expectedStatus) {
  62         await server.videos.get({ id: videoId, expectedStatus })
  63       }
  64     }
  65   }
  66
  67   async function checkVideoState (videoId: string, state: VideoState) {
  68     for (const server of servers) {
  69       const video = await server.videos.get({ id: videoId })
  70       expect(video.state.id).to.equal(state)
  71     }
  72   }
  73
  74   before(async function () {
  75     this.timeout(120000)
  76
  77     servers = await createMultipleServers(2)
  78
  79     // Get the access tokens
  80     await setAccessTokensToServers(servers)
  81     await setDefaultVideoChannel(servers)
  82
  83     // Server 1 and server 2 follow each other
  84     await doubleFollow(servers[0], servers[1])
  85
  86     await servers[0].config.updateCustomSubConfig({
  87       newConfig: {
  88         live: {
  89           enabled: true,
  90           allowReplay: true,
  91           maxDuration: -1,
  92           transcoding: {
  93             enabled: false,
  94             resolutions: ConfigCommand.getCustomConfigResolutions(true)
  95           }
  96         }
  97       }
  98     })
  99   })
 100
 101   describe('With save replay disabled', function () {
 102
 103     before(async function () {
 104       this.timeout(10000)
 105     })
 106
 107     it('Should correctly create and federate the "waiting for stream" live', async function () {
 108       this.timeout(20000)
 109
 110       liveVideoUUID = await createLiveWrapper({ permanent: false, replay: false })
 111
 112       await waitJobs(servers)
 113
 114       await checkVideosExist(liveVideoUUID, false, HttpStatusCode.OK_200)
 115       await checkVideoState(liveVideoUUID, VideoState.WAITING_FOR_LIVE)
 116     })
 117
 118     it('Should correctly have updated the live and federated it when streaming in the live', async function () {
 119       this.timeout(30000)
 120
 121       ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID })
 122
 123       await waitUntilLivePublishedOnAllServers(servers, liveVideoUUID)
 124
 125       await waitJobs(servers)
 126
 127       await checkVideosExist(liveVideoUUID, true, HttpStatusCode.OK_200)
 128       await checkVideoState(liveVideoUUID, VideoState.PUBLISHED)
 129     })
 130
 131     it('Should correctly delete the video files after the stream ended', async function () {
 132       this.timeout(40000)
 133
 134       await stopFfmpeg(ffmpegCommand)
 135
 136       for (const server of servers) {
 137         await server.live.waitUntilEnded({ videoId: liveVideoUUID })
 138       }
 139       await waitJobs(servers)
 140
 141       // Live still exist, but cannot be played anymore
 142       await checkVideosExist(liveVideoUUID, false, HttpStatusCode.OK_200)
 143       await checkVideoState(liveVideoUUID, VideoState.LIVE_ENDED)
 144
 145       // No resolutions saved since we did not save replay
 146       await checkLiveCleanup(servers[0], liveVideoUUID, [])
 147     })
 148
 149     it('Should correctly terminate the stream on blacklist and delete the live', async function () {
 150       this.timeout(40000)
 151
 152       liveVideoUUID = await createLiveWrapper({ permanent: false, replay: false })
 153
 154       ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID })
 155
 156       await waitUntilLivePublishedOnAllServers(servers, liveVideoUUID)
 157
 158       await waitJobs(servers)
 159       await checkVideosExist(liveVideoUUID, true, HttpStatusCode.OK_200)
 160
 161       await Promise.all([
 162         servers[0].blacklist.add({ videoId: liveVideoUUID, reason: 'bad live', unfederate: true }),
 163         testFfmpegStreamError(ffmpegCommand, true)
 164       ])
 165
 166       await waitJobs(servers)
 167
 168       await checkVideosExist(liveVideoUUID, false)
 169
 170       await servers[0].videos.get({ id: liveVideoUUID, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 })
 171       await servers[1].videos.get({ id: liveVideoUUID, expectedStatus: HttpStatusCode.NOT_FOUND_404 })
 172
 173       await wait(5000)
 174       await waitJobs(servers)
 175       await checkLiveCleanup(servers[0], liveVideoUUID, [])
 176     })
 177
 178     it('Should correctly terminate the stream on delete and delete the video', async function () {
 179       this.timeout(40000)
 180
 181       liveVideoUUID = await createLiveWrapper({ permanent: false, replay: false })
 182
 183       ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID })
 184
 185       await waitUntilLivePublishedOnAllServers(servers, liveVideoUUID)
 186
 187       await waitJobs(servers)
 188       await checkVideosExist(liveVideoUUID, true, HttpStatusCode.OK_200)
 189
 190       await Promise.all([
 191         testFfmpegStreamError(ffmpegCommand, true),
 192         servers[0].videos.remove({ id: liveVideoUUID })
 193       ])
 194
 195       await wait(5000)
 196       await waitJobs(servers)
 197
 198       await checkVideosExist(liveVideoUUID, false, HttpStatusCode.NOT_FOUND_404)
 199       await checkLiveCleanup(servers[0], liveVideoUUID, [])
 200     })
 201   })
 202
 203   describe('With save replay enabled on non permanent live', function () {
 204
 205     it('Should correctly create and federate the "waiting for stream" live', async function () {
 206       this.timeout(20000)
 207
 208       liveVideoUUID = await createLiveWrapper({ permanent: false, replay: true })
 209
 210       await waitJobs(servers)
 211
 212       await checkVideosExist(liveVideoUUID, false, HttpStatusCode.OK_200)
 213       await checkVideoState(liveVideoUUID, VideoState.WAITING_FOR_LIVE)
 214     })
 215
 216     it('Should correctly have updated the live and federated it when streaming in the live', async function () {
 217       this.timeout(20000)
 218
 219       ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID })
 220       await waitUntilLivePublishedOnAllServers(servers, liveVideoUUID)
 221
 222       await waitJobs(servers)
 223
 224       await checkVideosExist(liveVideoUUID, true, HttpStatusCode.OK_200)
 225       await checkVideoState(liveVideoUUID, VideoState.PUBLISHED)
 226     })
 227
 228     it('Should correctly have saved the live and federated it after the streaming', async function () {
 229       this.timeout(30000)
 230
 231       await stopFfmpeg(ffmpegCommand)
 232
 233       await waitUntilLiveReplacedByReplayOnAllServers(servers, liveVideoUUID)
 234       await waitJobs(servers)
 235
 236       // Live has been transcoded
 237       await checkVideosExist(liveVideoUUID, true, HttpStatusCode.OK_200)
 238       await checkVideoState(liveVideoUUID, VideoState.PUBLISHED)
 239     })
 240
 241     it('Should update the saved live and correctly federate the updated attributes', async function () {
 242       this.timeout(30000)
 243
 244       await servers[0].videos.update({ id: liveVideoUUID, attributes: { name: 'video updated' } })
 245       await waitJobs(servers)
 246
 247       for (const server of servers) {
 248         const video = await server.videos.get({ id: liveVideoUUID })
 249         expect(video.name).to.equal('video updated')
 250         expect(video.isLive).to.be.false
 251       }
 252     })
 253
 254     it('Should have cleaned up the live files', async function () {
 255       await checkLiveCleanup(servers[0], liveVideoUUID, [ 720 ])
 256     })
 257
 258     it('Should correctly terminate the stream on blacklist and blacklist the saved replay video', async function () {
 259       this.timeout(40000)
 260
 261       liveVideoUUID = await createLiveWrapper({ permanent: false, replay: true })
 262
 263       ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID })
 264       await waitUntilLivePublishedOnAllServers(servers, liveVideoUUID)
 265
 266       await waitJobs(servers)
 267       await checkVideosExist(liveVideoUUID, true, HttpStatusCode.OK_200)
 268
 269       await Promise.all([
 270         servers[0].blacklist.add({ videoId: liveVideoUUID, reason: 'bad live', unfederate: true }),
 271         testFfmpegStreamError(ffmpegCommand, true)
 272       ])
 273
 274       await waitJobs(servers)
 275
 276       await checkVideosExist(liveVideoUUID, false)
 277
 278       await servers[0].videos.get({ id: liveVideoUUID, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 })
 279       await servers[1].videos.get({ id: liveVideoUUID, expectedStatus: HttpStatusCode.NOT_FOUND_404 })
 280
 281       await wait(5000)
 282       await waitJobs(servers)
 283       await checkLiveCleanup(servers[0], liveVideoUUID, [ 720 ])
 284     })
 285
 286     it('Should correctly terminate the stream on delete and delete the video', async function () {
 287       this.timeout(40000)
 288
 289       liveVideoUUID = await createLiveWrapper({ permanent: false, replay: true })
 290
 291       ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID })
 292       await waitUntilLivePublishedOnAllServers(servers, liveVideoUUID)
 293
 294       await waitJobs(servers)
 295       await checkVideosExist(liveVideoUUID, true, HttpStatusCode.OK_200)
 296
 297       await Promise.all([
 298         servers[0].videos.remove({ id: liveVideoUUID }),
 299         testFfmpegStreamError(ffmpegCommand, true)
 300       ])
 301
 302       await wait(5000)
 303       await waitJobs(servers)
 304
 305       await checkVideosExist(liveVideoUUID, false, HttpStatusCode.NOT_FOUND_404)
 306       await checkLiveCleanup(servers[0], liveVideoUUID, [])
 307     })
 308   })
 309
 310   describe('With save replay enabled on permanent live', function () {
 311     let lastReplayUUID: string
 312
 313     it('Should correctly create and federate the "waiting for stream" live', async function () {
 314       this.timeout(20000)
 315
 316       liveVideoUUID = await createLiveWrapper({ permanent: true, replay: true })
 317
 318       await waitJobs(servers)
 319
 320       await checkVideosExist(liveVideoUUID, false, HttpStatusCode.OK_200)
 321       await checkVideoState(liveVideoUUID, VideoState.WAITING_FOR_LIVE)
 322     })
 323
 324     it('Should correctly have updated the live and federated it when streaming in the live', async function () {
 325       this.timeout(20000)
 326
 327       ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID })
 328       await waitUntilLivePublishedOnAllServers(servers, liveVideoUUID)
 329
 330       await waitJobs(servers)
 331
 332       await checkVideosExist(liveVideoUUID, true, HttpStatusCode.OK_200)
 333       await checkVideoState(liveVideoUUID, VideoState.PUBLISHED)
 334     })
 335
 336     it('Should correctly have saved the live and federated it after the streaming', async function () {
 337       this.timeout(30000)
 338
 339       const liveDetails = await servers[0].videos.get({ id: liveVideoUUID })
 340
 341       await stopFfmpeg(ffmpegCommand)
 342
 343       await waitUntilLiveWaitingOnAllServers(servers, liveVideoUUID)
 344       await waitJobs(servers)
 345
 346       const video = await findExternalSavedVideo(servers[0], liveDetails)
 347       expect(video).to.exist
 348
 349       for (const server of servers) {
 350         await server.videos.get({ id: video.uuid })
 351       }
 352
 353       lastReplayUUID = video.uuid
 354     })
 355
 356     it('Should have cleaned up the live files', async function () {
 357       await checkLiveCleanup(servers[0], liveVideoUUID, [])
 358     })
 359
 360     it('Should correctly terminate the stream on blacklist and blacklist the saved replay video', async function () {
 361       this.timeout(60000)
 362
 363       await servers[0].videos.remove({ id: lastReplayUUID })
 364
 365       liveVideoUUID = await createLiveWrapper({ permanent: true, replay: true })
 366
 367       ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID })
 368       await waitUntilLivePublishedOnAllServers(servers, liveVideoUUID)
 369
 370       const liveDetails = await servers[0].videos.get({ id: liveVideoUUID })
 371
 372       await waitJobs(servers)
 373       await checkVideosExist(liveVideoUUID, true, HttpStatusCode.OK_200)
 374
 375       await Promise.all([
 376         servers[0].blacklist.add({ videoId: liveVideoUUID, reason: 'bad live', unfederate: true }),
 377         testFfmpegStreamError(ffmpegCommand, true)
 378       ])
 379
 380       await waitJobs(servers)
 381       await wait(5000)
 382       await waitJobs(servers)
 383
 384       const replay = await findExternalSavedVideo(servers[0], liveDetails)
 385       expect(replay).to.exist
 386
 387       for (const videoId of [ liveVideoUUID, replay.uuid ]) {
 388         await checkVideosExist(videoId, false)
 389
 390         await servers[0].videos.get({ id: videoId, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 })
 391         await servers[1].videos.get({ id: videoId, expectedStatus: HttpStatusCode.NOT_FOUND_404 })
 392       }
 393
 394       await checkLiveCleanup(servers[0], liveVideoUUID, [])
 395     })
 396
 397     it('Should correctly terminate the stream on delete and not save the video', async function () {
 398       this.timeout(40000)
 399
 400       liveVideoUUID = await createLiveWrapper({ permanent: true, replay: true })
 401
 402       ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID })
 403       await waitUntilLivePublishedOnAllServers(servers, liveVideoUUID)
 404
 405       const liveDetails = await servers[0].videos.get({ id: liveVideoUUID })
 406
 407       await waitJobs(servers)
 408       await checkVideosExist(liveVideoUUID, true, HttpStatusCode.OK_200)
 409
 410       await Promise.all([
 411         servers[0].videos.remove({ id: liveVideoUUID }),
 412         testFfmpegStreamError(ffmpegCommand, true)
 413       ])
 414
 415       await wait(5000)
 416       await waitJobs(servers)
 417
 418       const replay = await findExternalSavedVideo(servers[0], liveDetails)
 419       expect(replay).to.not.exist
 420
 421       await checkVideosExist(liveVideoUUID, false, HttpStatusCode.NOT_FOUND_404)
 422       await checkLiveCleanup(servers[0], liveVideoUUID, [])
 423     })
 424   })
 425
 426   after(async function () {
 427     await cleanupTests(servers)
 428   })
 429 })