server/tests/api/object-storage/videos.ts

   1 /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
   2
   3 import bytes from 'bytes'
   4 import { expect } from 'chai'
   5 import { stat } from 'fs-extra'
   6 import { merge } from 'lodash'
   7 import {
   8   checkTmpIsEmpty,
   9   expectLogDoesNotContain,
  10   expectStartWith,
  11   generateHighBitrateVideo,
  12   MockObjectStorageProxy
  13 } from '@server/tests/shared'
  14 import { areMockObjectStorageTestsDisabled } from '@shared/core-utils'
  15 import { HttpStatusCode, VideoDetails } from '@shared/models'
  16 import {
  17   cleanupTests,
  18   createMultipleServers,
  19   createSingleServer,
  20   doubleFollow,
  21   killallServers,
  22   makeRawRequest,
  23   ObjectStorageCommand,
  24   PeerTubeServer,
  25   setAccessTokensToServers,
  26   waitJobs,
  27   webtorrentAdd
  28 } from '@shared/server-commands'
  29 import { sha1 } from '@shared/extra-utils'
  30
  31 async function checkFiles (options: {
  32   server: PeerTubeServer
  33   originServer: PeerTubeServer
  34
  35   video: VideoDetails
  36
  37   baseMockUrl?: string
  38
  39   playlistBucket: string
  40   playlistPrefix?: string
  41
  42   webtorrentBucket: string
  43   webtorrentPrefix?: string
  44 }) {
  45   const {
  46     server,
  47     originServer,
  48     video,
  49     playlistBucket,
  50     webtorrentBucket,
  51     baseMockUrl,
  52     playlistPrefix,
  53     webtorrentPrefix
  54   } = options
  55
  56   let allFiles = video.files
  57
  58   for (const file of video.files) {
  59     const baseUrl = baseMockUrl
  60       ? `${baseMockUrl}/${webtorrentBucket}/`
  61       : `http://${webtorrentBucket}.${ObjectStorageCommand.getMockEndpointHost()}/`
  62
  63     const prefix = webtorrentPrefix || ''
  64     const start = baseUrl + prefix
  65
  66     expectStartWith(file.fileUrl, start)
  67
  68     const res = await makeRawRequest({ url: file.fileDownloadUrl, expectedStatus: HttpStatusCode.FOUND_302 })
  69     const location = res.headers['location']
  70     expectStartWith(location, start)
  71
  72     await makeRawRequest({ url: location, expectedStatus: HttpStatusCode.OK_200 })
  73   }
  74
  75   const hls = video.streamingPlaylists[0]
  76
  77   if (hls) {
  78     allFiles = allFiles.concat(hls.files)
  79
  80     const baseUrl = baseMockUrl
  81       ? `${baseMockUrl}/${playlistBucket}/`
  82       : `http://${playlistBucket}.${ObjectStorageCommand.getMockEndpointHost()}/`
  83
  84     const prefix = playlistPrefix || ''
  85     const start = baseUrl + prefix
  86
  87     expectStartWith(hls.playlistUrl, start)
  88     expectStartWith(hls.segmentsSha256Url, start)
  89
  90     await makeRawRequest({ url: hls.playlistUrl, expectedStatus: HttpStatusCode.OK_200 })
  91
  92     const resSha = await makeRawRequest({ url: hls.segmentsSha256Url, expectedStatus: HttpStatusCode.OK_200 })
  93     expect(JSON.stringify(resSha.body)).to.not.throw
  94
  95     let i = 0
  96     for (const file of hls.files) {
  97       expectStartWith(file.fileUrl, start)
  98
  99       const res = await makeRawRequest({ url: file.fileDownloadUrl, expectedStatus: HttpStatusCode.FOUND_302 })
 100       const location = res.headers['location']
 101       expectStartWith(location, start)
 102
 103       await makeRawRequest({ url: location, expectedStatus: HttpStatusCode.OK_200 })
 104
 105       if (originServer.internalServerNumber === server.internalServerNumber) {
 106         const infohash = sha1(`${2 + hls.playlistUrl}+V${i}`)
 107         const dbInfohashes = await originServer.sql.getPlaylistInfohash(hls.id)
 108
 109         expect(dbInfohashes).to.include(infohash)
 110       }
 111
 112       i++
 113     }
 114   }
 115
 116   for (const file of allFiles) {
 117     const torrent = await webtorrentAdd(file.magnetUri, true)
 118
 119     expect(torrent.files).to.be.an('array')
 120     expect(torrent.files.length).to.equal(1)
 121     expect(torrent.files[0].path).to.exist.and.to.not.equal('')
 122
 123     const res = await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 })
 124     expect(res.body).to.have.length.above(100)
 125   }
 126
 127   return allFiles.map(f => f.fileUrl)
 128 }
 129
 130 function runTestSuite (options: {
 131   fixture?: string
 132
 133   maxUploadPart?: string
 134
 135   playlistBucket: string
 136   playlistPrefix?: string
 137
 138   webtorrentBucket: string
 139   webtorrentPrefix?: string
 140
 141   useMockBaseUrl?: boolean
 142 }) {
 143   const mockObjectStorageProxy = new MockObjectStorageProxy()
 144   const { fixture } = options
 145   let baseMockUrl: string
 146
 147   let servers: PeerTubeServer[]
 148
 149   let keptUrls: string[] = []
 150
 151   const uuidsToDelete: string[] = []
 152   let deletedUrls: string[] = []
 153
 154   before(async function () {
 155     this.timeout(120000)
 156
 157     const port = await mockObjectStorageProxy.initialize()
 158     baseMockUrl = options.useMockBaseUrl
 159       ? `http://127.0.0.1:${port}`
 160       : undefined
 161
 162     await ObjectStorageCommand.createMockBucket(options.playlistBucket)
 163     await ObjectStorageCommand.createMockBucket(options.webtorrentBucket)
 164
 165     const config = {
 166       object_storage: {
 167         enabled: true,
 168         endpoint: 'http://' + ObjectStorageCommand.getMockEndpointHost(),
 169         region: ObjectStorageCommand.getMockRegion(),
 170
 171         credentials: ObjectStorageCommand.getMockCredentialsConfig(),
 172
 173         max_upload_part: options.maxUploadPart || '5MB',
 174
 175         streaming_playlists: {
 176           bucket_name: options.playlistBucket,
 177           prefix: options.playlistPrefix,
 178           base_url: baseMockUrl
 179             ? `${baseMockUrl}/${options.playlistBucket}`
 180             : undefined
 181         },
 182
 183         videos: {
 184           bucket_name: options.webtorrentBucket,
 185           prefix: options.webtorrentPrefix,
 186           base_url: baseMockUrl
 187             ? `${baseMockUrl}/${options.webtorrentBucket}`
 188             : undefined
 189         }
 190       }
 191     }
 192
 193     servers = await createMultipleServers(2, config)
 194
 195     await setAccessTokensToServers(servers)
 196     await doubleFollow(servers[0], servers[1])
 197
 198     for (const server of servers) {
 199       const { uuid } = await server.videos.quickUpload({ name: 'video to keep' })
 200       await waitJobs(servers)
 201
 202       const files = await server.videos.listFiles({ id: uuid })
 203       keptUrls = keptUrls.concat(files.map(f => f.fileUrl))
 204     }
 205   })
 206
 207   it('Should upload a video and move it to the object storage without transcoding', async function () {
 208     this.timeout(40000)
 209
 210     const { uuid } = await servers[0].videos.quickUpload({ name: 'video 1', fixture })
 211     uuidsToDelete.push(uuid)
 212
 213     await waitJobs(servers)
 214
 215     for (const server of servers) {
 216       const video = await server.videos.get({ id: uuid })
 217       const files = await checkFiles({ ...options, server, originServer: servers[0], video, baseMockUrl })
 218
 219       deletedUrls = deletedUrls.concat(files)
 220     }
 221   })
 222
 223   it('Should upload a video and move it to the object storage with transcoding', async function () {
 224     this.timeout(120000)
 225
 226     const { uuid } = await servers[1].videos.quickUpload({ name: 'video 2', fixture })
 227     uuidsToDelete.push(uuid)
 228
 229     await waitJobs(servers)
 230
 231     for (const server of servers) {
 232       const video = await server.videos.get({ id: uuid })
 233       const files = await checkFiles({ ...options, server, originServer: servers[0], video, baseMockUrl })
 234
 235       deletedUrls = deletedUrls.concat(files)
 236     }
 237   })
 238
 239   it('Should fetch correctly all the files', async function () {
 240     for (const url of deletedUrls.concat(keptUrls)) {
 241       await makeRawRequest({ url, expectedStatus: HttpStatusCode.OK_200 })
 242     }
 243   })
 244
 245   it('Should correctly delete the files', async function () {
 246     await servers[0].videos.remove({ id: uuidsToDelete[0] })
 247     await servers[1].videos.remove({ id: uuidsToDelete[1] })
 248
 249     await waitJobs(servers)
 250
 251     for (const url of deletedUrls) {
 252       await makeRawRequest({ url, expectedStatus: HttpStatusCode.NOT_FOUND_404 })
 253     }
 254   })
 255
 256   it('Should have kept other files', async function () {
 257     for (const url of keptUrls) {
 258       await makeRawRequest({ url, expectedStatus: HttpStatusCode.OK_200 })
 259     }
 260   })
 261
 262   it('Should have an empty tmp directory', async function () {
 263     for (const server of servers) {
 264       await checkTmpIsEmpty(server)
 265     }
 266   })
 267
 268   it('Should not have downloaded files from object storage', async function () {
 269     for (const server of servers) {
 270       await expectLogDoesNotContain(server, 'from object storage')
 271     }
 272   })
 273
 274   after(async function () {
 275     await mockObjectStorageProxy.terminate()
 276
 277     await cleanupTests(servers)
 278   })
 279 }
 280
 281 describe('Object storage for videos', function () {
 282   if (areMockObjectStorageTestsDisabled()) return
 283
 284   describe('Test config', function () {
 285     let server: PeerTubeServer
 286
 287     const baseConfig = {
 288       object_storage: {
 289         enabled: true,
 290         endpoint: 'http://' + ObjectStorageCommand.getMockEndpointHost(),
 291         region: ObjectStorageCommand.getMockRegion(),
 292
 293         credentials: ObjectStorageCommand.getMockCredentialsConfig(),
 294
 295         streaming_playlists: {
 296           bucket_name: ObjectStorageCommand.DEFAULT_PLAYLIST_MOCK_BUCKET
 297         },
 298
 299         videos: {
 300           bucket_name: ObjectStorageCommand.DEFAULT_WEBTORRENT_MOCK_BUCKET
 301         }
 302       }
 303     }
 304
 305     const badCredentials = {
 306       access_key_id: 'AKIAIOSFODNN7EXAMPLE',
 307       secret_access_key: 'aJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY'
 308     }
 309
 310     it('Should fail with same bucket names without prefix', function (done) {
 311       const config = merge({}, baseConfig, {
 312         object_storage: {
 313           streaming_playlists: {
 314             bucket_name: 'aaa'
 315           },
 316
 317           videos: {
 318             bucket_name: 'aaa'
 319           }
 320         }
 321       })
 322
 323       createSingleServer(1, config)
 324         .then(() => done(new Error('Did not throw')))
 325         .catch(() => done())
 326     })
 327
 328     it('Should fail with bad credentials', async function () {
 329       this.timeout(60000)
 330
 331       await ObjectStorageCommand.prepareDefaultMockBuckets()
 332
 333       const config = merge({}, baseConfig, {
 334         object_storage: {
 335           credentials: badCredentials
 336         }
 337       })
 338
 339       server = await createSingleServer(1, config)
 340       await setAccessTokensToServers([ server ])
 341
 342       const { uuid } = await server.videos.quickUpload({ name: 'video' })
 343
 344       await waitJobs([ server ], { skipDelayed: true })
 345       const video = await server.videos.get({ id: uuid })
 346
 347       expectStartWith(video.files[0].fileUrl, server.url)
 348
 349       await killallServers([ server ])
 350     })
 351
 352     it('Should succeed with credentials from env', async function () {
 353       this.timeout(60000)
 354
 355       await ObjectStorageCommand.prepareDefaultMockBuckets()
 356
 357       const config = merge({}, baseConfig, {
 358         object_storage: {
 359           credentials: {
 360             access_key_id: '',
 361             secret_access_key: ''
 362           }
 363         }
 364       })
 365
 366       const goodCredentials = ObjectStorageCommand.getMockCredentialsConfig()
 367
 368       server = await createSingleServer(1, config, {
 369         env: {
 370           AWS_ACCESS_KEY_ID: goodCredentials.access_key_id,
 371           AWS_SECRET_ACCESS_KEY: goodCredentials.secret_access_key
 372         }
 373       })
 374
 375       await setAccessTokensToServers([ server ])
 376
 377       const { uuid } = await server.videos.quickUpload({ name: 'video' })
 378
 379       await waitJobs([ server ], { skipDelayed: true })
 380       const video = await server.videos.get({ id: uuid })
 381
 382       expectStartWith(video.files[0].fileUrl, ObjectStorageCommand.getMockWebTorrentBaseUrl())
 383     })
 384
 385     after(async function () {
 386       await killallServers([ server ])
 387     })
 388   })
 389
 390   describe('Test simple object storage', function () {
 391     runTestSuite({
 392       playlistBucket: 'streaming-playlists',
 393       webtorrentBucket: 'videos'
 394     })
 395   })
 396
 397   describe('Test object storage with prefix', function () {
 398     runTestSuite({
 399       playlistBucket: 'mybucket',
 400       webtorrentBucket: 'mybucket',
 401
 402       playlistPrefix: 'streaming-playlists_',
 403       webtorrentPrefix: 'webtorrent_'
 404     })
 405   })
 406
 407   describe('Test object storage with prefix and base URL', function () {
 408     runTestSuite({
 409       playlistBucket: 'mybucket',
 410       webtorrentBucket: 'mybucket',
 411
 412       playlistPrefix: 'streaming-playlists/',
 413       webtorrentPrefix: 'webtorrent/',
 414
 415       useMockBaseUrl: true
 416     })
 417   })
 418
 419   describe('Test object storage with file bigger than upload part', function () {
 420     let fixture: string
 421     const maxUploadPart = '5MB'
 422
 423     before(async function () {
 424       this.timeout(120000)
 425
 426       fixture = await generateHighBitrateVideo()
 427
 428       const { size } = await stat(fixture)
 429
 430       if (bytes.parse(maxUploadPart) > size) {
 431         throw Error(`Fixture file is too small (${size}) to make sense for this test.`)
 432       }
 433     })
 434
 435     runTestSuite({
 436       maxUploadPart,
 437       playlistBucket: 'streaming-playlists',
 438       webtorrentBucket: 'videos',
 439       fixture
 440     })
 441   })
 442 })