server/tests/api/live/live-save-replay.ts

   1 /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
   2
   3 import 'mocha'
   4 import * as chai from 'chai'
   5 import { FfmpegCommand } from 'fluent-ffmpeg'
   6 import { checkLiveCleanup } from '@server/tests/shared'
   7 import { wait } from '@shared/core-utils'
   8 import { HttpStatusCode, LiveVideoCreate, VideoPrivacy, VideoState } from '@shared/models'
   9 import {
  10   cleanupTests,
  11   ConfigCommand,
  12   createMultipleServers,
  13   doubleFollow,
  14   findExternalSavedVideo,
  15   PeerTubeServer,
  16   setAccessTokensToServers,
  17   setDefaultVideoChannel,
  18   stopFfmpeg,
  19   testFfmpegStreamError,
  20   waitJobs,
  21   waitUntilLivePublishedOnAllServers,
  22   waitUntilLiveReplacedByReplayOnAllServers,
  23   waitUntilLiveWaitingOnAllServers
  24 } from '@shared/server-commands'
  25
  26 const expect = chai.expect
  27
  28 describe('Save replay setting', function () {
  29   let servers: PeerTubeServer[] = []
  30   let liveVideoUUID: string
  31   let ffmpegCommand: FfmpegCommand
  32
  33   async function createLiveWrapper (options: { permanent: boolean, replay: boolean }) {
  34     if (liveVideoUUID) {
  35       try {
  36         await servers[0].videos.remove({ id: liveVideoUUID })
  37         await waitJobs(servers)
  38       } catch {}
  39     }
  40
  41     const attributes: LiveVideoCreate = {
  42       channelId: servers[0].store.channel.id,
  43       privacy: VideoPrivacy.PUBLIC,
  44       name: 'my super live',
  45       saveReplay: options.replay,
  46       permanentLive: options.permanent
  47     }
  48
  49     const { uuid } = await servers[0].live.create({ fields: attributes })
  50     return uuid
  51   }
  52
  53   async function publishLive (options: { permanent: boolean, replay: boolean }) {
  54     liveVideoUUID = await createLiveWrapper(options)
  55
  56     const ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID })
  57     await waitUntilLivePublishedOnAllServers(servers, liveVideoUUID)
  58
  59     const liveDetails = await servers[0].videos.get({ id: liveVideoUUID })
  60
  61     await waitJobs(servers)
  62     await checkVideosExist(liveVideoUUID, true, HttpStatusCode.OK_200)
  63
  64     return { ffmpegCommand, liveDetails }
  65   }
  66
  67   async function publishLiveAndDelete (options: { permanent: boolean, replay: boolean }) {
  68     const { ffmpegCommand, liveDetails } = await publishLive(options)
  69
  70     await Promise.all([
  71       servers[0].videos.remove({ id: liveVideoUUID }),
  72       testFfmpegStreamError(ffmpegCommand, true)
  73     ])
  74
  75     await waitJobs(servers)
  76     await wait(5000)
  77     await waitJobs(servers)
  78
  79     return { liveDetails }
  80   }
  81
  82   async function publishLiveAndBlacklist (options: { permanent: boolean, replay: boolean }) {
  83     const { ffmpegCommand, liveDetails } = await publishLive(options)
  84
  85     await Promise.all([
  86       servers[0].blacklist.add({ videoId: liveVideoUUID, reason: 'bad live', unfederate: true }),
  87       testFfmpegStreamError(ffmpegCommand, true)
  88     ])
  89
  90     await waitJobs(servers)
  91     await wait(5000)
  92     await waitJobs(servers)
  93
  94     return { liveDetails }
  95   }
  96
  97   async function checkVideosExist (videoId: string, existsInList: boolean, expectedStatus?: number) {
  98     for (const server of servers) {
  99       const length = existsInList ? 1 : 0
 100
 101       const { data, total } = await server.videos.list()
 102       expect(data).to.have.lengthOf(length)
 103       expect(total).to.equal(length)
 104
 105       if (expectedStatus) {
 106         await server.videos.get({ id: videoId, expectedStatus })
 107       }
 108     }
 109   }
 110
 111   async function checkVideoState (videoId: string, state: VideoState) {
 112     for (const server of servers) {
 113       const video = await server.videos.get({ id: videoId })
 114       expect(video.state.id).to.equal(state)
 115     }
 116   }
 117
 118   before(async function () {
 119     this.timeout(120000)
 120
 121     servers = await createMultipleServers(2)
 122
 123     // Get the access tokens
 124     await setAccessTokensToServers(servers)
 125     await setDefaultVideoChannel(servers)
 126
 127     // Server 1 and server 2 follow each other
 128     await doubleFollow(servers[0], servers[1])
 129
 130     await servers[0].config.updateCustomSubConfig({
 131       newConfig: {
 132         live: {
 133           enabled: true,
 134           allowReplay: true,
 135           maxDuration: -1,
 136           transcoding: {
 137             enabled: false,
 138             resolutions: ConfigCommand.getCustomConfigResolutions(true)
 139           }
 140         }
 141       }
 142     })
 143   })
 144
 145   describe('With save replay disabled', function () {
 146
 147     it('Should correctly create and federate the "waiting for stream" live', async function () {
 148       this.timeout(20000)
 149
 150       liveVideoUUID = await createLiveWrapper({ permanent: false, replay: false })
 151
 152       await waitJobs(servers)
 153
 154       await checkVideosExist(liveVideoUUID, false, HttpStatusCode.OK_200)
 155       await checkVideoState(liveVideoUUID, VideoState.WAITING_FOR_LIVE)
 156     })
 157
 158     it('Should correctly have updated the live and federated it when streaming in the live', async function () {
 159       this.timeout(30000)
 160
 161       ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID })
 162
 163       await waitUntilLivePublishedOnAllServers(servers, liveVideoUUID)
 164
 165       await waitJobs(servers)
 166
 167       await checkVideosExist(liveVideoUUID, true, HttpStatusCode.OK_200)
 168       await checkVideoState(liveVideoUUID, VideoState.PUBLISHED)
 169     })
 170
 171     it('Should correctly delete the video files after the stream ended', async function () {
 172       this.timeout(40000)
 173
 174       await stopFfmpeg(ffmpegCommand)
 175
 176       for (const server of servers) {
 177         await server.live.waitUntilEnded({ videoId: liveVideoUUID })
 178       }
 179       await waitJobs(servers)
 180
 181       // Live still exist, but cannot be played anymore
 182       await checkVideosExist(liveVideoUUID, false, HttpStatusCode.OK_200)
 183       await checkVideoState(liveVideoUUID, VideoState.LIVE_ENDED)
 184
 185       // No resolutions saved since we did not save replay
 186       await checkLiveCleanup(servers[0], liveVideoUUID, [])
 187     })
 188
 189     it('Should correctly terminate the stream on blacklist and delete the live', async function () {
 190       this.timeout(40000)
 191
 192       await publishLiveAndBlacklist({ permanent: false, replay: false })
 193
 194       await checkVideosExist(liveVideoUUID, false)
 195
 196       await servers[0].videos.get({ id: liveVideoUUID, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 })
 197       await servers[1].videos.get({ id: liveVideoUUID, expectedStatus: HttpStatusCode.NOT_FOUND_404 })
 198
 199       await wait(5000)
 200       await waitJobs(servers)
 201       await checkLiveCleanup(servers[0], liveVideoUUID, [])
 202     })
 203
 204     it('Should correctly terminate the stream on delete and delete the video', async function () {
 205       this.timeout(40000)
 206
 207       await publishLiveAndDelete({ permanent: false, replay: false })
 208
 209       await checkVideosExist(liveVideoUUID, false, HttpStatusCode.NOT_FOUND_404)
 210       await checkLiveCleanup(servers[0], liveVideoUUID, [])
 211     })
 212   })
 213
 214   describe('With save replay enabled on non permanent live', function () {
 215
 216     it('Should correctly create and federate the "waiting for stream" live', async function () {
 217       this.timeout(20000)
 218
 219       liveVideoUUID = await createLiveWrapper({ permanent: false, replay: true })
 220
 221       await waitJobs(servers)
 222
 223       await checkVideosExist(liveVideoUUID, false, HttpStatusCode.OK_200)
 224       await checkVideoState(liveVideoUUID, VideoState.WAITING_FOR_LIVE)
 225     })
 226
 227     it('Should correctly have updated the live and federated it when streaming in the live', async function () {
 228       this.timeout(20000)
 229
 230       ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID })
 231       await waitUntilLivePublishedOnAllServers(servers, liveVideoUUID)
 232
 233       await waitJobs(servers)
 234
 235       await checkVideosExist(liveVideoUUID, true, HttpStatusCode.OK_200)
 236       await checkVideoState(liveVideoUUID, VideoState.PUBLISHED)
 237     })
 238
 239     it('Should correctly have saved the live and federated it after the streaming', async function () {
 240       this.timeout(30000)
 241
 242       await stopFfmpeg(ffmpegCommand)
 243
 244       await waitUntilLiveReplacedByReplayOnAllServers(servers, liveVideoUUID)
 245       await waitJobs(servers)
 246
 247       // Live has been transcoded
 248       await checkVideosExist(liveVideoUUID, true, HttpStatusCode.OK_200)
 249       await checkVideoState(liveVideoUUID, VideoState.PUBLISHED)
 250     })
 251
 252     it('Should update the saved live and correctly federate the updated attributes', async function () {
 253       this.timeout(30000)
 254
 255       await servers[0].videos.update({ id: liveVideoUUID, attributes: { name: 'video updated' } })
 256       await waitJobs(servers)
 257
 258       for (const server of servers) {
 259         const video = await server.videos.get({ id: liveVideoUUID })
 260         expect(video.name).to.equal('video updated')
 261         expect(video.isLive).to.be.false
 262       }
 263     })
 264
 265     it('Should have cleaned up the live files', async function () {
 266       await checkLiveCleanup(servers[0], liveVideoUUID, [ 720 ])
 267     })
 268
 269     it('Should correctly terminate the stream on blacklist and blacklist the saved replay video', async function () {
 270       this.timeout(40000)
 271
 272       await publishLiveAndBlacklist({ permanent: false, replay: true })
 273
 274       await checkVideosExist(liveVideoUUID, false)
 275
 276       await servers[0].videos.get({ id: liveVideoUUID, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 })
 277       await servers[1].videos.get({ id: liveVideoUUID, expectedStatus: HttpStatusCode.NOT_FOUND_404 })
 278
 279       await wait(5000)
 280       await waitJobs(servers)
 281       await checkLiveCleanup(servers[0], liveVideoUUID, [ 720 ])
 282     })
 283
 284     it('Should correctly terminate the stream on delete and delete the video', async function () {
 285       this.timeout(40000)
 286
 287       await publishLiveAndDelete({ permanent: false, replay: true })
 288
 289       await checkVideosExist(liveVideoUUID, false, HttpStatusCode.NOT_FOUND_404)
 290       await checkLiveCleanup(servers[0], liveVideoUUID, [])
 291     })
 292   })
 293
 294   describe('With save replay enabled on permanent live', function () {
 295     let lastReplayUUID: string
 296
 297     it('Should correctly create and federate the "waiting for stream" live', async function () {
 298       this.timeout(20000)
 299
 300       liveVideoUUID = await createLiveWrapper({ permanent: true, replay: true })
 301
 302       await waitJobs(servers)
 303
 304       await checkVideosExist(liveVideoUUID, false, HttpStatusCode.OK_200)
 305       await checkVideoState(liveVideoUUID, VideoState.WAITING_FOR_LIVE)
 306     })
 307
 308     it('Should correctly have updated the live and federated it when streaming in the live', async function () {
 309       this.timeout(20000)
 310
 311       ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID })
 312       await waitUntilLivePublishedOnAllServers(servers, liveVideoUUID)
 313
 314       await waitJobs(servers)
 315
 316       await checkVideosExist(liveVideoUUID, true, HttpStatusCode.OK_200)
 317       await checkVideoState(liveVideoUUID, VideoState.PUBLISHED)
 318     })
 319
 320     it('Should correctly have saved the live and federated it after the streaming', async function () {
 321       this.timeout(30000)
 322
 323       const liveDetails = await servers[0].videos.get({ id: liveVideoUUID })
 324
 325       await stopFfmpeg(ffmpegCommand)
 326
 327       await waitUntilLiveWaitingOnAllServers(servers, liveVideoUUID)
 328       await waitJobs(servers)
 329
 330       const video = await findExternalSavedVideo(servers[0], liveDetails)
 331       expect(video).to.exist
 332
 333       for (const server of servers) {
 334         await server.videos.get({ id: video.uuid })
 335       }
 336
 337       lastReplayUUID = video.uuid
 338     })
 339
 340     it('Should have cleaned up the live files', async function () {
 341       await checkLiveCleanup(servers[0], liveVideoUUID, [])
 342     })
 343
 344     it('Should correctly terminate the stream on blacklist and blacklist the saved replay video', async function () {
 345       this.timeout(60000)
 346
 347       await servers[0].videos.remove({ id: lastReplayUUID })
 348       const { liveDetails } = await publishLiveAndBlacklist({ permanent: true, replay: true })
 349
 350       const replay = await findExternalSavedVideo(servers[0], liveDetails)
 351       expect(replay).to.exist
 352
 353       for (const videoId of [ liveVideoUUID, replay.uuid ]) {
 354         await checkVideosExist(videoId, false)
 355
 356         await servers[0].videos.get({ id: videoId, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 })
 357         await servers[1].videos.get({ id: videoId, expectedStatus: HttpStatusCode.NOT_FOUND_404 })
 358       }
 359
 360       await checkLiveCleanup(servers[0], liveVideoUUID, [])
 361     })
 362
 363     it('Should correctly terminate the stream on delete and not save the video', async function () {
 364       this.timeout(40000)
 365
 366       const { liveDetails } = await publishLiveAndDelete({ permanent: true, replay: true })
 367
 368       const replay = await findExternalSavedVideo(servers[0], liveDetails)
 369       expect(replay).to.not.exist
 370
 371       await checkVideosExist(liveVideoUUID, false, HttpStatusCode.NOT_FOUND_404)
 372       await checkLiveCleanup(servers[0], liveVideoUUID, [])
 373     })
 374   })
 375
 376   after(async function () {
 377     await cleanupTests(servers)
 378   })
 379 })