server/tests/api/object-storage/videos.ts

   1 /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
   2
   3 import bytes from 'bytes'
   4 import * as chai from 'chai'
   5 import { stat } from 'fs-extra'
   6 import { merge } from 'lodash'
   7 import {
   8   checkTmpIsEmpty,
   9   expectLogDoesNotContain,
  10   expectStartWith,
  11   generateHighBitrateVideo,
  12   MockObjectStorage
  13 } from '@server/tests/shared'
  14 import { areObjectStorageTestsDisabled } from '@shared/core-utils'
  15 import { HttpStatusCode, VideoDetails } from '@shared/models'
  16 import {
  17   cleanupTests,
  18   createMultipleServers,
  19   createSingleServer,
  20   doubleFollow,
  21   killallServers,
  22   makeRawRequest,
  23   ObjectStorageCommand,
  24   PeerTubeServer,
  25   setAccessTokensToServers,
  26   waitJobs,
  27   webtorrentAdd
  28 } from '@shared/server-commands'
  29
  30 const expect = chai.expect
  31
  32 async function checkFiles (options: {
  33   video: VideoDetails
  34
  35   baseMockUrl?: string
  36
  37   playlistBucket: string
  38   playlistPrefix?: string
  39
  40   webtorrentBucket: string
  41   webtorrentPrefix?: string
  42 }) {
  43   const {
  44     video,
  45     playlistBucket,
  46     webtorrentBucket,
  47     baseMockUrl,
  48     playlistPrefix,
  49     webtorrentPrefix
  50   } = options
  51
  52   let allFiles = video.files
  53
  54   for (const file of video.files) {
  55     const baseUrl = baseMockUrl
  56       ? `${baseMockUrl}/${webtorrentBucket}/`
  57       : `http://${webtorrentBucket}.${ObjectStorageCommand.getEndpointHost()}/`
  58
  59     const prefix = webtorrentPrefix || ''
  60     const start = baseUrl + prefix
  61
  62     expectStartWith(file.fileUrl, start)
  63
  64     const res = await makeRawRequest(file.fileDownloadUrl, HttpStatusCode.FOUND_302)
  65     const location = res.headers['location']
  66     expectStartWith(location, start)
  67
  68     await makeRawRequest(location, HttpStatusCode.OK_200)
  69   }
  70
  71   const hls = video.streamingPlaylists[0]
  72
  73   if (hls) {
  74     allFiles = allFiles.concat(hls.files)
  75
  76     const baseUrl = baseMockUrl
  77       ? `${baseMockUrl}/${playlistBucket}/`
  78       : `http://${playlistBucket}.${ObjectStorageCommand.getEndpointHost()}/`
  79
  80     const prefix = playlistPrefix || ''
  81     const start = baseUrl + prefix
  82
  83     expectStartWith(hls.playlistUrl, start)
  84     expectStartWith(hls.segmentsSha256Url, start)
  85
  86     await makeRawRequest(hls.playlistUrl, HttpStatusCode.OK_200)
  87
  88     const resSha = await makeRawRequest(hls.segmentsSha256Url, HttpStatusCode.OK_200)
  89     expect(JSON.stringify(resSha.body)).to.not.throw
  90
  91     for (const file of hls.files) {
  92       expectStartWith(file.fileUrl, start)
  93
  94       const res = await makeRawRequest(file.fileDownloadUrl, HttpStatusCode.FOUND_302)
  95       const location = res.headers['location']
  96       expectStartWith(location, start)
  97
  98       await makeRawRequest(location, HttpStatusCode.OK_200)
  99     }
 100   }
 101
 102   for (const file of allFiles) {
 103     const torrent = await webtorrentAdd(file.magnetUri, true)
 104
 105     expect(torrent.files).to.be.an('array')
 106     expect(torrent.files.length).to.equal(1)
 107     expect(torrent.files[0].path).to.exist.and.to.not.equal('')
 108
 109     const res = await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200)
 110     expect(res.body).to.have.length.above(100)
 111   }
 112
 113   return allFiles.map(f => f.fileUrl)
 114 }
 115
 116 function runTestSuite (options: {
 117   fixture?: string
 118
 119   maxUploadPart?: string
 120
 121   playlistBucket: string
 122   playlistPrefix?: string
 123
 124   webtorrentBucket: string
 125   webtorrentPrefix?: string
 126
 127   useMockBaseUrl?: boolean
 128 }) {
 129   const mockObjectStorage = new MockObjectStorage()
 130   const { fixture } = options
 131   let baseMockUrl: string
 132
 133   let servers: PeerTubeServer[]
 134
 135   let keptUrls: string[] = []
 136
 137   const uuidsToDelete: string[] = []
 138   let deletedUrls: string[] = []
 139
 140   before(async function () {
 141     this.timeout(120000)
 142
 143     const port = await mockObjectStorage.initialize()
 144     baseMockUrl = options.useMockBaseUrl ? `http://localhost:${port}` : undefined
 145
 146     await ObjectStorageCommand.createBucket(options.playlistBucket)
 147     await ObjectStorageCommand.createBucket(options.webtorrentBucket)
 148
 149     const config = {
 150       object_storage: {
 151         enabled: true,
 152         endpoint: 'http://' + ObjectStorageCommand.getEndpointHost(),
 153         region: ObjectStorageCommand.getRegion(),
 154
 155         credentials: ObjectStorageCommand.getCredentialsConfig(),
 156
 157         max_upload_part: options.maxUploadPart || '5MB',
 158
 159         streaming_playlists: {
 160           bucket_name: options.playlistBucket,
 161           prefix: options.playlistPrefix,
 162           base_url: baseMockUrl
 163             ? `${baseMockUrl}/${options.playlistBucket}`
 164             : undefined
 165         },
 166
 167         videos: {
 168           bucket_name: options.webtorrentBucket,
 169           prefix: options.webtorrentPrefix,
 170           base_url: baseMockUrl
 171             ? `${baseMockUrl}/${options.webtorrentBucket}`
 172             : undefined
 173         }
 174       }
 175     }
 176
 177     servers = await createMultipleServers(2, config)
 178
 179     await setAccessTokensToServers(servers)
 180     await doubleFollow(servers[0], servers[1])
 181
 182     for (const server of servers) {
 183       const { uuid } = await server.videos.quickUpload({ name: 'video to keep' })
 184       await waitJobs(servers)
 185
 186       const files = await server.videos.listFiles({ id: uuid })
 187       keptUrls = keptUrls.concat(files.map(f => f.fileUrl))
 188     }
 189   })
 190
 191   it('Should upload a video and move it to the object storage without transcoding', async function () {
 192     this.timeout(40000)
 193
 194     const { uuid } = await servers[0].videos.quickUpload({ name: 'video 1', fixture })
 195     uuidsToDelete.push(uuid)
 196
 197     await waitJobs(servers)
 198
 199     for (const server of servers) {
 200       const video = await server.videos.get({ id: uuid })
 201       const files = await checkFiles({ ...options, video, baseMockUrl })
 202
 203       deletedUrls = deletedUrls.concat(files)
 204     }
 205   })
 206
 207   it('Should upload a video and move it to the object storage with transcoding', async function () {
 208     this.timeout(120000)
 209
 210     const { uuid } = await servers[1].videos.quickUpload({ name: 'video 2', fixture })
 211     uuidsToDelete.push(uuid)
 212
 213     await waitJobs(servers)
 214
 215     for (const server of servers) {
 216       const video = await server.videos.get({ id: uuid })
 217       const files = await checkFiles({ ...options, video, baseMockUrl })
 218
 219       deletedUrls = deletedUrls.concat(files)
 220     }
 221   })
 222
 223   it('Should fetch correctly all the files', async function () {
 224     for (const url of deletedUrls.concat(keptUrls)) {
 225       await makeRawRequest(url, HttpStatusCode.OK_200)
 226     }
 227   })
 228
 229   it('Should correctly delete the files', async function () {
 230     await servers[0].videos.remove({ id: uuidsToDelete[0] })
 231     await servers[1].videos.remove({ id: uuidsToDelete[1] })
 232
 233     await waitJobs(servers)
 234
 235     for (const url of deletedUrls) {
 236       await makeRawRequest(url, HttpStatusCode.NOT_FOUND_404)
 237     }
 238   })
 239
 240   it('Should have kept other files', async function () {
 241     for (const url of keptUrls) {
 242       await makeRawRequest(url, HttpStatusCode.OK_200)
 243     }
 244   })
 245
 246   it('Should have an empty tmp directory', async function () {
 247     for (const server of servers) {
 248       await checkTmpIsEmpty(server)
 249     }
 250   })
 251
 252   it('Should not have downloaded files from object storage', async function () {
 253     for (const server of servers) {
 254       await expectLogDoesNotContain(server, 'from object storage')
 255     }
 256   })
 257
 258   after(async function () {
 259     await mockObjectStorage.terminate()
 260
 261     await cleanupTests(servers)
 262   })
 263 }
 264
 265 describe('Object storage for videos', function () {
 266   if (areObjectStorageTestsDisabled()) return
 267
 268   describe('Test config', function () {
 269     let server: PeerTubeServer
 270
 271     const baseConfig = {
 272       object_storage: {
 273         enabled: true,
 274         endpoint: 'http://' + ObjectStorageCommand.getEndpointHost(),
 275         region: ObjectStorageCommand.getRegion(),
 276
 277         credentials: ObjectStorageCommand.getCredentialsConfig(),
 278
 279         streaming_playlists: {
 280           bucket_name: ObjectStorageCommand.DEFAULT_PLAYLIST_BUCKET
 281         },
 282
 283         videos: {
 284           bucket_name: ObjectStorageCommand.DEFAULT_WEBTORRENT_BUCKET
 285         }
 286       }
 287     }
 288
 289     const badCredentials = {
 290       access_key_id: 'AKIAIOSFODNN7EXAMPLE',
 291       secret_access_key: 'aJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY'
 292     }
 293
 294     it('Should fail with same bucket names without prefix', function (done) {
 295       const config = merge({}, baseConfig, {
 296         object_storage: {
 297           streaming_playlists: {
 298             bucket_name: 'aaa'
 299           },
 300
 301           videos: {
 302             bucket_name: 'aaa'
 303           }
 304         }
 305       })
 306
 307       createSingleServer(1, config)
 308         .then(() => done(new Error('Did not throw')))
 309         .catch(() => done())
 310     })
 311
 312     it('Should fail with bad credentials', async function () {
 313       this.timeout(60000)
 314
 315       await ObjectStorageCommand.prepareDefaultBuckets()
 316
 317       const config = merge({}, baseConfig, {
 318         object_storage: {
 319           credentials: badCredentials
 320         }
 321       })
 322
 323       server = await createSingleServer(1, config)
 324       await setAccessTokensToServers([ server ])
 325
 326       const { uuid } = await server.videos.quickUpload({ name: 'video' })
 327
 328       await waitJobs([ server ], true)
 329       const video = await server.videos.get({ id: uuid })
 330
 331       expectStartWith(video.files[0].fileUrl, server.url)
 332
 333       await killallServers([ server ])
 334     })
 335
 336     it('Should succeed with credentials from env', async function () {
 337       this.timeout(60000)
 338
 339       await ObjectStorageCommand.prepareDefaultBuckets()
 340
 341       const config = merge({}, baseConfig, {
 342         object_storage: {
 343           credentials: {
 344             access_key_id: '',
 345             secret_access_key: ''
 346           }
 347         }
 348       })
 349
 350       const goodCredentials = ObjectStorageCommand.getCredentialsConfig()
 351
 352       server = await createSingleServer(1, config, {
 353         env: {
 354           AWS_ACCESS_KEY_ID: goodCredentials.access_key_id,
 355           AWS_SECRET_ACCESS_KEY: goodCredentials.secret_access_key
 356         }
 357       })
 358
 359       await setAccessTokensToServers([ server ])
 360
 361       const { uuid } = await server.videos.quickUpload({ name: 'video' })
 362
 363       await waitJobs([ server ], true)
 364       const video = await server.videos.get({ id: uuid })
 365
 366       expectStartWith(video.files[0].fileUrl, ObjectStorageCommand.getWebTorrentBaseUrl())
 367     })
 368
 369     after(async function () {
 370       await killallServers([ server ])
 371     })
 372   })
 373
 374   describe('Test simple object storage', function () {
 375     runTestSuite({
 376       playlistBucket: 'streaming-playlists',
 377       webtorrentBucket: 'videos'
 378     })
 379   })
 380
 381   describe('Test object storage with prefix', function () {
 382     runTestSuite({
 383       playlistBucket: 'mybucket',
 384       webtorrentBucket: 'mybucket',
 385
 386       playlistPrefix: 'streaming-playlists_',
 387       webtorrentPrefix: 'webtorrent_'
 388     })
 389   })
 390
 391   describe('Test object storage with prefix and base URL', function () {
 392     runTestSuite({
 393       playlistBucket: 'mybucket',
 394       webtorrentBucket: 'mybucket',
 395
 396       playlistPrefix: 'streaming-playlists/',
 397       webtorrentPrefix: 'webtorrent/',
 398
 399       useMockBaseUrl: true
 400     })
 401   })
 402
 403   describe('Test object storage with file bigger than upload part', function () {
 404     let fixture: string
 405     const maxUploadPart = '5MB'
 406
 407     before(async function () {
 408       this.timeout(120000)
 409
 410       fixture = await generateHighBitrateVideo()
 411
 412       const { size } = await stat(fixture)
 413
 414       if (bytes.parse(maxUploadPart) > size) {
 415         throw Error(`Fixture file is too small (${size}) to make sense for this test.`)
 416       }
 417     })
 418
 419     runTestSuite({
 420       maxUploadPart,
 421       playlistBucket: 'streaming-playlists',
 422       webtorrentBucket: 'videos',
 423       fixture
 424     })
 425   })
 426 })