server/tests/api/object-storage/live.ts

   1 /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
   2
   3 import { expect } from 'chai'
   4 import { expectStartWith, MockObjectStorageProxy, SQLCommand, testLiveVideoResolutions } from '@server/tests/shared'
   5 import { areMockObjectStorageTestsDisabled } from '@shared/core-utils'
   6 import { HttpStatusCode, LiveVideoCreate, VideoPrivacy } from '@shared/models'
   7 import {
   8   cleanupTests,
   9   createMultipleServers,
  10   doubleFollow,
  11   findExternalSavedVideo,
  12   makeRawRequest,
  13   ObjectStorageCommand,
  14   PeerTubeServer,
  15   setAccessTokensToServers,
  16   setDefaultVideoChannel,
  17   stopFfmpeg,
  18   waitJobs,
  19   waitUntilLivePublishedOnAllServers,
  20   waitUntilLiveReplacedByReplayOnAllServers,
  21   waitUntilLiveWaitingOnAllServers
  22 } from '@shared/server-commands'
  23
  24 async function createLive (server: PeerTubeServer, permanent: boolean) {
  25   const attributes: LiveVideoCreate = {
  26     channelId: server.store.channel.id,
  27     privacy: VideoPrivacy.PUBLIC,
  28     name: 'my super live',
  29     saveReplay: true,
  30     replaySettings: { privacy: VideoPrivacy.PUBLIC },
  31     permanentLive: permanent
  32   }
  33
  34   const { uuid } = await server.live.create({ fields: attributes })
  35
  36   return uuid
  37 }
  38
  39 async function checkFilesExist (options: {
  40   servers: PeerTubeServer[]
  41   videoUUID: string
  42   numberOfFiles: number
  43   objectStorage: ObjectStorageCommand
  44 }) {
  45   const { servers, videoUUID, numberOfFiles, objectStorage } = options
  46
  47   for (const server of servers) {
  48     const video = await server.videos.get({ id: videoUUID })
  49
  50     expect(video.files).to.have.lengthOf(0)
  51     expect(video.streamingPlaylists).to.have.lengthOf(1)
  52
  53     const files = video.streamingPlaylists[0].files
  54     expect(files).to.have.lengthOf(numberOfFiles)
  55
  56     for (const file of files) {
  57       expectStartWith(file.fileUrl, objectStorage.getMockPlaylistBaseUrl())
  58
  59       await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 })
  60     }
  61   }
  62 }
  63
  64 async function checkFilesCleanup (options: {
  65   server: PeerTubeServer
  66   videoUUID: string
  67   resolutions: number[]
  68   objectStorage: ObjectStorageCommand
  69 }) {
  70   const { server, videoUUID, resolutions, objectStorage } = options
  71
  72   const resolutionFiles = resolutions.map((_value, i) => `${i}.m3u8`)
  73
  74   for (const playlistName of [ 'master.m3u8' ].concat(resolutionFiles)) {
  75     await server.live.getPlaylistFile({
  76       videoUUID,
  77       playlistName,
  78       expectedStatus: HttpStatusCode.NOT_FOUND_404,
  79       objectStorage
  80     })
  81   }
  82
  83   await server.live.getSegmentFile({
  84     videoUUID,
  85     playlistNumber: 0,
  86     segment: 0,
  87     objectStorage,
  88     expectedStatus: HttpStatusCode.NOT_FOUND_404
  89   })
  90 }
  91
  92 describe('Object storage for lives', function () {
  93   if (areMockObjectStorageTestsDisabled()) return
  94
  95   let servers: PeerTubeServer[]
  96   let sqlCommandServer1: SQLCommand
  97   const objectStorage = new ObjectStorageCommand()
  98
  99   before(async function () {
 100     this.timeout(120000)
 101
 102     await objectStorage.prepareDefaultMockBuckets()
 103     servers = await createMultipleServers(2, objectStorage.getDefaultMockConfig())
 104
 105     await setAccessTokensToServers(servers)
 106     await setDefaultVideoChannel(servers)
 107     await doubleFollow(servers[0], servers[1])
 108
 109     await servers[0].config.enableTranscoding()
 110
 111     sqlCommandServer1 = new SQLCommand(servers[0])
 112   })
 113
 114   describe('Without live transcoding', function () {
 115     let videoUUID: string
 116
 117     before(async function () {
 118       await servers[0].config.enableLive({ transcoding: false })
 119
 120       videoUUID = await createLive(servers[0], false)
 121     })
 122
 123     it('Should create a live and publish it on object storage', async function () {
 124       this.timeout(220000)
 125
 126       const ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: videoUUID })
 127       await waitUntilLivePublishedOnAllServers(servers, videoUUID)
 128
 129       await testLiveVideoResolutions({
 130         originServer: servers[0],
 131         sqlCommand: sqlCommandServer1,
 132         servers,
 133         liveVideoId: videoUUID,
 134         resolutions: [ 720 ],
 135         transcoded: false,
 136         objectStorage
 137       })
 138
 139       await stopFfmpeg(ffmpegCommand)
 140     })
 141
 142     it('Should have saved the replay on object storage', async function () {
 143       this.timeout(220000)
 144
 145       await waitUntilLiveReplacedByReplayOnAllServers(servers, videoUUID)
 146       await waitJobs(servers)
 147
 148       await checkFilesExist({ servers, videoUUID, numberOfFiles: 1, objectStorage })
 149     })
 150
 151     it('Should have cleaned up live files from object storage', async function () {
 152       await checkFilesCleanup({ server: servers[0], videoUUID, resolutions: [ 720 ], objectStorage })
 153     })
 154   })
 155
 156   describe('With live transcoding', function () {
 157     const resolutions = [ 720, 480, 360, 240, 144 ]
 158
 159     before(async function () {
 160       await servers[0].config.enableLive({ transcoding: true })
 161     })
 162
 163     describe('Normal replay', function () {
 164       let videoUUIDNonPermanent: string
 165
 166       before(async function () {
 167         videoUUIDNonPermanent = await createLive(servers[0], false)
 168       })
 169
 170       it('Should create a live and publish it on object storage', async function () {
 171         this.timeout(240000)
 172
 173         const ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: videoUUIDNonPermanent })
 174         await waitUntilLivePublishedOnAllServers(servers, videoUUIDNonPermanent)
 175
 176         await testLiveVideoResolutions({
 177           originServer: servers[0],
 178           sqlCommand: sqlCommandServer1,
 179           servers,
 180           liveVideoId: videoUUIDNonPermanent,
 181           resolutions,
 182           transcoded: true,
 183           objectStorage
 184         })
 185
 186         await stopFfmpeg(ffmpegCommand)
 187       })
 188
 189       it('Should have saved the replay on object storage', async function () {
 190         this.timeout(220000)
 191
 192         await waitUntilLiveReplacedByReplayOnAllServers(servers, videoUUIDNonPermanent)
 193         await waitJobs(servers)
 194
 195         await checkFilesExist({ servers, videoUUID: videoUUIDNonPermanent, numberOfFiles: 5, objectStorage })
 196       })
 197
 198       it('Should have cleaned up live files from object storage', async function () {
 199         await checkFilesCleanup({ server: servers[0], videoUUID: videoUUIDNonPermanent, resolutions, objectStorage })
 200       })
 201     })
 202
 203     describe('Permanent replay', function () {
 204       let videoUUIDPermanent: string
 205
 206       before(async function () {
 207         videoUUIDPermanent = await createLive(servers[0], true)
 208       })
 209
 210       it('Should create a live and publish it on object storage', async function () {
 211         this.timeout(240000)
 212
 213         const ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: videoUUIDPermanent })
 214         await waitUntilLivePublishedOnAllServers(servers, videoUUIDPermanent)
 215
 216         await testLiveVideoResolutions({
 217           originServer: servers[0],
 218           sqlCommand: sqlCommandServer1,
 219           servers,
 220           liveVideoId: videoUUIDPermanent,
 221           resolutions,
 222           transcoded: true,
 223           objectStorage
 224         })
 225
 226         await stopFfmpeg(ffmpegCommand)
 227       })
 228
 229       it('Should have saved the replay on object storage', async function () {
 230         this.timeout(220000)
 231
 232         await waitUntilLiveWaitingOnAllServers(servers, videoUUIDPermanent)
 233         await waitJobs(servers)
 234
 235         const videoLiveDetails = await servers[0].videos.get({ id: videoUUIDPermanent })
 236         const replay = await findExternalSavedVideo(servers[0], videoLiveDetails)
 237
 238         await checkFilesExist({ servers, videoUUID: replay.uuid, numberOfFiles: 5, objectStorage })
 239       })
 240
 241       it('Should have cleaned up live files from object storage', async function () {
 242         await checkFilesCleanup({ server: servers[0], videoUUID: videoUUIDPermanent, resolutions, objectStorage })
 243       })
 244     })
 245   })
 246
 247   describe('With object storage base url', function () {
 248     const mockObjectStorageProxy = new MockObjectStorageProxy()
 249     let baseMockUrl: string
 250
 251     before(async function () {
 252       this.timeout(120000)
 253
 254       const port = await mockObjectStorageProxy.initialize()
 255       const bucketName = objectStorage.getMockStreamingPlaylistsBucketName()
 256       baseMockUrl = `http://127.0.0.1:${port}/${bucketName}`
 257
 258       await objectStorage.prepareDefaultMockBuckets()
 259
 260       const config = {
 261         object_storage: {
 262           enabled: true,
 263           endpoint: 'http://' + ObjectStorageCommand.getMockEndpointHost(),
 264           region: ObjectStorageCommand.getMockRegion(),
 265
 266           credentials: ObjectStorageCommand.getMockCredentialsConfig(),
 267
 268           streaming_playlists: {
 269             bucket_name: bucketName,
 270             prefix: '',
 271             base_url: baseMockUrl
 272           }
 273         }
 274       }
 275
 276       await servers[0].kill()
 277       await servers[0].run(config)
 278
 279       await servers[0].config.enableLive({ transcoding: true, resolutions: 'min' })
 280     })
 281
 282     it('Should publish a live and replace the base url', async function () {
 283       this.timeout(240000)
 284
 285       const videoUUIDPermanent = await createLive(servers[0], true)
 286
 287       const ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: videoUUIDPermanent })
 288       await waitUntilLivePublishedOnAllServers(servers, videoUUIDPermanent)
 289
 290       await testLiveVideoResolutions({
 291         originServer: servers[0],
 292         sqlCommand: sqlCommandServer1,
 293         servers,
 294         liveVideoId: videoUUIDPermanent,
 295         resolutions: [ 720 ],
 296         transcoded: true,
 297         objectStorage,
 298         objectStorageBaseUrl: baseMockUrl
 299       })
 300
 301       await stopFfmpeg(ffmpegCommand)
 302     })
 303   })
 304
 305   after(async function () {
 306     await sqlCommandServer1.cleanup()
 307     await objectStorage.cleanupMock()
 308
 309     await cleanupTests(servers)
 310   })
 311 })