server/tests/api/object-storage/videos.ts

   1 /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
   2
   3 import bytes from 'bytes'
   4 import { expect } from 'chai'
   5 import { stat } from 'fs-extra'
   6 import { merge } from 'lodash'
   7 import {
   8   checkTmpIsEmpty,
   9   checkWebTorrentWorks,
  10   expectLogDoesNotContain,
  11   expectStartWith,
  12   generateHighBitrateVideo,
  13   MockObjectStorageProxy,
  14   SQLCommand
  15 } from '@server/tests/shared'
  16 import { areMockObjectStorageTestsDisabled } from '@shared/core-utils'
  17 import { sha1 } from '@shared/extra-utils'
  18 import { HttpStatusCode, VideoDetails } from '@shared/models'
  19 import {
  20   cleanupTests,
  21   createMultipleServers,
  22   createSingleServer,
  23   doubleFollow,
  24   killallServers,
  25   makeRawRequest,
  26   ObjectStorageCommand,
  27   PeerTubeServer,
  28   setAccessTokensToServers,
  29   waitJobs
  30 } from '@shared/server-commands'
  31
  32 async function checkFiles (options: {
  33   server: PeerTubeServer
  34   originServer: PeerTubeServer
  35   originSQLCommand: SQLCommand
  36
  37   video: VideoDetails
  38
  39   baseMockUrl?: string
  40
  41   playlistBucket: string
  42   playlistPrefix?: string
  43
  44   webtorrentBucket: string
  45   webtorrentPrefix?: string
  46 }) {
  47   const {
  48     server,
  49     originServer,
  50     originSQLCommand,
  51     video,
  52     playlistBucket,
  53     webtorrentBucket,
  54     baseMockUrl,
  55     playlistPrefix,
  56     webtorrentPrefix
  57   } = options
  58
  59   let allFiles = video.files
  60
  61   for (const file of video.files) {
  62     const baseUrl = baseMockUrl
  63       ? `${baseMockUrl}/${webtorrentBucket}/`
  64       : `http://${webtorrentBucket}.${ObjectStorageCommand.getMockEndpointHost()}/`
  65
  66     const prefix = webtorrentPrefix || ''
  67     const start = baseUrl + prefix
  68
  69     expectStartWith(file.fileUrl, start)
  70
  71     const res = await makeRawRequest({ url: file.fileDownloadUrl, expectedStatus: HttpStatusCode.FOUND_302 })
  72     const location = res.headers['location']
  73     expectStartWith(location, start)
  74
  75     await makeRawRequest({ url: location, expectedStatus: HttpStatusCode.OK_200 })
  76   }
  77
  78   const hls = video.streamingPlaylists[0]
  79
  80   if (hls) {
  81     allFiles = allFiles.concat(hls.files)
  82
  83     const baseUrl = baseMockUrl
  84       ? `${baseMockUrl}/${playlistBucket}/`
  85       : `http://${playlistBucket}.${ObjectStorageCommand.getMockEndpointHost()}/`
  86
  87     const prefix = playlistPrefix || ''
  88     const start = baseUrl + prefix
  89
  90     expectStartWith(hls.playlistUrl, start)
  91     expectStartWith(hls.segmentsSha256Url, start)
  92
  93     await makeRawRequest({ url: hls.playlistUrl, expectedStatus: HttpStatusCode.OK_200 })
  94
  95     const resSha = await makeRawRequest({ url: hls.segmentsSha256Url, expectedStatus: HttpStatusCode.OK_200 })
  96     expect(JSON.stringify(resSha.body)).to.not.throw
  97
  98     let i = 0
  99     for (const file of hls.files) {
 100       expectStartWith(file.fileUrl, start)
 101
 102       const res = await makeRawRequest({ url: file.fileDownloadUrl, expectedStatus: HttpStatusCode.FOUND_302 })
 103       const location = res.headers['location']
 104       expectStartWith(location, start)
 105
 106       await makeRawRequest({ url: location, expectedStatus: HttpStatusCode.OK_200 })
 107
 108       if (originServer.internalServerNumber === server.internalServerNumber) {
 109         const infohash = sha1(`${2 + hls.playlistUrl}+V${i}`)
 110         const dbInfohashes = await originSQLCommand.getPlaylistInfohash(hls.id)
 111
 112         expect(dbInfohashes).to.include(infohash)
 113       }
 114
 115       i++
 116     }
 117   }
 118
 119   for (const file of allFiles) {
 120     await checkWebTorrentWorks(file.magnetUri)
 121
 122     const res = await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 })
 123     expect(res.body).to.have.length.above(100)
 124   }
 125
 126   return allFiles.map(f => f.fileUrl)
 127 }
 128
 129 function runTestSuite (options: {
 130   fixture?: string
 131
 132   maxUploadPart?: string
 133
 134   playlistBucket: string
 135   playlistPrefix?: string
 136
 137   webtorrentBucket: string
 138   webtorrentPrefix?: string
 139
 140   useMockBaseUrl?: boolean
 141 }) {
 142   const mockObjectStorageProxy = new MockObjectStorageProxy()
 143   const { fixture } = options
 144   let baseMockUrl: string
 145
 146   let servers: PeerTubeServer[]
 147   let sqlCommands: SQLCommand[] = []
 148
 149   let keptUrls: string[] = []
 150
 151   const uuidsToDelete: string[] = []
 152   let deletedUrls: string[] = []
 153
 154   before(async function () {
 155     this.timeout(240000)
 156
 157     const port = await mockObjectStorageProxy.initialize()
 158     baseMockUrl = options.useMockBaseUrl
 159       ? `http://127.0.0.1:${port}`
 160       : undefined
 161
 162     await ObjectStorageCommand.createMockBucket(options.playlistBucket)
 163     await ObjectStorageCommand.createMockBucket(options.webtorrentBucket)
 164
 165     const config = {
 166       object_storage: {
 167         enabled: true,
 168         endpoint: 'http://' + ObjectStorageCommand.getMockEndpointHost(),
 169         region: ObjectStorageCommand.getMockRegion(),
 170
 171         credentials: ObjectStorageCommand.getMockCredentialsConfig(),
 172
 173         max_upload_part: options.maxUploadPart || '5MB',
 174
 175         streaming_playlists: {
 176           bucket_name: options.playlistBucket,
 177           prefix: options.playlistPrefix,
 178           base_url: baseMockUrl
 179             ? `${baseMockUrl}/${options.playlistBucket}`
 180             : undefined
 181         },
 182
 183         videos: {
 184           bucket_name: options.webtorrentBucket,
 185           prefix: options.webtorrentPrefix,
 186           base_url: baseMockUrl
 187             ? `${baseMockUrl}/${options.webtorrentBucket}`
 188             : undefined
 189         }
 190       }
 191     }
 192
 193     servers = await createMultipleServers(2, config)
 194
 195     await setAccessTokensToServers(servers)
 196     await doubleFollow(servers[0], servers[1])
 197
 198     for (const server of servers) {
 199       const { uuid } = await server.videos.quickUpload({ name: 'video to keep' })
 200       await waitJobs(servers)
 201
 202       const files = await server.videos.listFiles({ id: uuid })
 203       keptUrls = keptUrls.concat(files.map(f => f.fileUrl))
 204     }
 205
 206     sqlCommands = servers.map(s => new SQLCommand(s))
 207   })
 208
 209   it('Should upload a video and move it to the object storage without transcoding', async function () {
 210     this.timeout(40000)
 211
 212     const { uuid } = await servers[0].videos.quickUpload({ name: 'video 1', fixture })
 213     uuidsToDelete.push(uuid)
 214
 215     await waitJobs(servers)
 216
 217     for (const server of servers) {
 218       const video = await server.videos.get({ id: uuid })
 219       const files = await checkFiles({ ...options, server, originServer: servers[0], originSQLCommand: sqlCommands[0], video, baseMockUrl })
 220
 221       deletedUrls = deletedUrls.concat(files)
 222     }
 223   })
 224
 225   it('Should upload a video and move it to the object storage with transcoding', async function () {
 226     this.timeout(120000)
 227
 228     const { uuid } = await servers[1].videos.quickUpload({ name: 'video 2', fixture })
 229     uuidsToDelete.push(uuid)
 230
 231     await waitJobs(servers)
 232
 233     for (const server of servers) {
 234       const video = await server.videos.get({ id: uuid })
 235       const files = await checkFiles({ ...options, server, originServer: servers[0], originSQLCommand: sqlCommands[0], video, baseMockUrl })
 236
 237       deletedUrls = deletedUrls.concat(files)
 238     }
 239   })
 240
 241   it('Should fetch correctly all the files', async function () {
 242     for (const url of deletedUrls.concat(keptUrls)) {
 243       await makeRawRequest({ url, expectedStatus: HttpStatusCode.OK_200 })
 244     }
 245   })
 246
 247   it('Should correctly delete the files', async function () {
 248     await servers[0].videos.remove({ id: uuidsToDelete[0] })
 249     await servers[1].videos.remove({ id: uuidsToDelete[1] })
 250
 251     await waitJobs(servers)
 252
 253     for (const url of deletedUrls) {
 254       await makeRawRequest({ url, expectedStatus: HttpStatusCode.NOT_FOUND_404 })
 255     }
 256   })
 257
 258   it('Should have kept other files', async function () {
 259     for (const url of keptUrls) {
 260       await makeRawRequest({ url, expectedStatus: HttpStatusCode.OK_200 })
 261     }
 262   })
 263
 264   it('Should have an empty tmp directory', async function () {
 265     for (const server of servers) {
 266       await checkTmpIsEmpty(server)
 267     }
 268   })
 269
 270   it('Should not have downloaded files from object storage', async function () {
 271     for (const server of servers) {
 272       await expectLogDoesNotContain(server, 'from object storage')
 273     }
 274   })
 275
 276   after(async function () {
 277     await mockObjectStorageProxy.terminate()
 278
 279     for (const sqlCommand of sqlCommands) {
 280       await sqlCommand.cleanup()
 281     }
 282
 283     await cleanupTests(servers)
 284   })
 285 }
 286
 287 describe('Object storage for videos', function () {
 288   if (areMockObjectStorageTestsDisabled()) return
 289
 290   describe('Test config', function () {
 291     let server: PeerTubeServer
 292
 293     const baseConfig = {
 294       object_storage: {
 295         enabled: true,
 296         endpoint: 'http://' + ObjectStorageCommand.getMockEndpointHost(),
 297         region: ObjectStorageCommand.getMockRegion(),
 298
 299         credentials: ObjectStorageCommand.getMockCredentialsConfig(),
 300
 301         streaming_playlists: {
 302           bucket_name: ObjectStorageCommand.DEFAULT_PLAYLIST_MOCK_BUCKET
 303         },
 304
 305         videos: {
 306           bucket_name: ObjectStorageCommand.DEFAULT_WEBTORRENT_MOCK_BUCKET
 307         }
 308       }
 309     }
 310
 311     const badCredentials = {
 312       access_key_id: 'AKIAIOSFODNN7EXAMPLE',
 313       secret_access_key: 'aJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY'
 314     }
 315
 316     it('Should fail with same bucket names without prefix', function (done) {
 317       const config = merge({}, baseConfig, {
 318         object_storage: {
 319           streaming_playlists: {
 320             bucket_name: 'aaa'
 321           },
 322
 323           videos: {
 324             bucket_name: 'aaa'
 325           }
 326         }
 327       })
 328
 329       createSingleServer(1, config)
 330         .then(() => done(new Error('Did not throw')))
 331         .catch(() => done())
 332     })
 333
 334     it('Should fail with bad credentials', async function () {
 335       this.timeout(60000)
 336
 337       await ObjectStorageCommand.prepareDefaultMockBuckets()
 338
 339       const config = merge({}, baseConfig, {
 340         object_storage: {
 341           credentials: badCredentials
 342         }
 343       })
 344
 345       server = await createSingleServer(1, config)
 346       await setAccessTokensToServers([ server ])
 347
 348       const { uuid } = await server.videos.quickUpload({ name: 'video' })
 349
 350       await waitJobs([ server ], { skipDelayed: true })
 351       const video = await server.videos.get({ id: uuid })
 352
 353       expectStartWith(video.files[0].fileUrl, server.url)
 354
 355       await killallServers([ server ])
 356     })
 357
 358     it('Should succeed with credentials from env', async function () {
 359       this.timeout(60000)
 360
 361       await ObjectStorageCommand.prepareDefaultMockBuckets()
 362
 363       const config = merge({}, baseConfig, {
 364         object_storage: {
 365           credentials: {
 366             access_key_id: '',
 367             secret_access_key: ''
 368           }
 369         }
 370       })
 371
 372       const goodCredentials = ObjectStorageCommand.getMockCredentialsConfig()
 373
 374       server = await createSingleServer(1, config, {
 375         env: {
 376           AWS_ACCESS_KEY_ID: goodCredentials.access_key_id,
 377           AWS_SECRET_ACCESS_KEY: goodCredentials.secret_access_key
 378         }
 379       })
 380
 381       await setAccessTokensToServers([ server ])
 382
 383       const { uuid } = await server.videos.quickUpload({ name: 'video' })
 384
 385       await waitJobs([ server ], { skipDelayed: true })
 386       const video = await server.videos.get({ id: uuid })
 387
 388       expectStartWith(video.files[0].fileUrl, ObjectStorageCommand.getMockWebTorrentBaseUrl())
 389     })
 390
 391     after(async function () {
 392       await cleanupTests([ server ])
 393     })
 394   })
 395
 396   describe('Test simple object storage', function () {
 397     runTestSuite({
 398       playlistBucket: 'streaming-playlists',
 399       webtorrentBucket: 'videos'
 400     })
 401   })
 402
 403   describe('Test object storage with prefix', function () {
 404     runTestSuite({
 405       playlistBucket: 'mybucket',
 406       webtorrentBucket: 'mybucket',
 407
 408       playlistPrefix: 'streaming-playlists_',
 409       webtorrentPrefix: 'webtorrent_'
 410     })
 411   })
 412
 413   describe('Test object storage with prefix and base URL', function () {
 414     runTestSuite({
 415       playlistBucket: 'mybucket',
 416       webtorrentBucket: 'mybucket',
 417
 418       playlistPrefix: 'streaming-playlists/',
 419       webtorrentPrefix: 'webtorrent/',
 420
 421       useMockBaseUrl: true
 422     })
 423   })
 424
 425   describe('Test object storage with file bigger than upload part', function () {
 426     let fixture: string
 427     const maxUploadPart = '5MB'
 428
 429     before(async function () {
 430       this.timeout(120000)
 431
 432       fixture = await generateHighBitrateVideo()
 433
 434       const { size } = await stat(fixture)
 435
 436       if (bytes.parse(maxUploadPart) > size) {
 437         throw Error(`Fixture file is too small (${size}) to make sense for this test.`)
 438       }
 439     })
 440
 441     runTestSuite({
 442       maxUploadPart,
 443       playlistBucket: 'streaming-playlists',
 444       webtorrentBucket: 'videos',
 445       fixture
 446     })
 447   })
 448 })