server/tests/api/object-storage/videos.ts

   1 /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
   2
   3 import bytes from 'bytes'
   4 import { expect } from 'chai'
   5 import { stat } from 'fs-extra'
   6 import { merge } from 'lodash'
   7 import {
   8   checkTmpIsEmpty,
   9   expectLogDoesNotContain,
  10   expectStartWith,
  11   generateHighBitrateVideo,
  12   MockObjectStorageProxy
  13 } from '@server/tests/shared'
  14 import { areMockObjectStorageTestsDisabled } from '@shared/core-utils'
  15 import { HttpStatusCode, VideoDetails } from '@shared/models'
  16 import {
  17   cleanupTests,
  18   createMultipleServers,
  19   createSingleServer,
  20   doubleFollow,
  21   killallServers,
  22   makeRawRequest,
  23   ObjectStorageCommand,
  24   PeerTubeServer,
  25   setAccessTokensToServers,
  26   waitJobs,
  27   webtorrentAdd
  28 } from '@shared/server-commands'
  29
  30 async function checkFiles (options: {
  31   video: VideoDetails
  32
  33   baseMockUrl?: string
  34
  35   playlistBucket: string
  36   playlistPrefix?: string
  37
  38   webtorrentBucket: string
  39   webtorrentPrefix?: string
  40 }) {
  41   const {
  42     video,
  43     playlistBucket,
  44     webtorrentBucket,
  45     baseMockUrl,
  46     playlistPrefix,
  47     webtorrentPrefix
  48   } = options
  49
  50   let allFiles = video.files
  51
  52   for (const file of video.files) {
  53     const baseUrl = baseMockUrl
  54       ? `${baseMockUrl}/${webtorrentBucket}/`
  55       : `http://${webtorrentBucket}.${ObjectStorageCommand.getMockEndpointHost()}/`
  56
  57     const prefix = webtorrentPrefix || ''
  58     const start = baseUrl + prefix
  59
  60     expectStartWith(file.fileUrl, start)
  61
  62     const res = await makeRawRequest({ url: file.fileDownloadUrl, expectedStatus: HttpStatusCode.FOUND_302 })
  63     const location = res.headers['location']
  64     expectStartWith(location, start)
  65
  66     await makeRawRequest({ url: location, expectedStatus: HttpStatusCode.OK_200 })
  67   }
  68
  69   const hls = video.streamingPlaylists[0]
  70
  71   if (hls) {
  72     allFiles = allFiles.concat(hls.files)
  73
  74     const baseUrl = baseMockUrl
  75       ? `${baseMockUrl}/${playlistBucket}/`
  76       : `http://${playlistBucket}.${ObjectStorageCommand.getMockEndpointHost()}/`
  77
  78     const prefix = playlistPrefix || ''
  79     const start = baseUrl + prefix
  80
  81     expectStartWith(hls.playlistUrl, start)
  82     expectStartWith(hls.segmentsSha256Url, start)
  83
  84     await makeRawRequest({ url: hls.playlistUrl, expectedStatus: HttpStatusCode.OK_200 })
  85
  86     const resSha = await makeRawRequest({ url: hls.segmentsSha256Url, expectedStatus: HttpStatusCode.OK_200 })
  87     expect(JSON.stringify(resSha.body)).to.not.throw
  88
  89     for (const file of hls.files) {
  90       expectStartWith(file.fileUrl, start)
  91
  92       const res = await makeRawRequest({ url: file.fileDownloadUrl, expectedStatus: HttpStatusCode.FOUND_302 })
  93       const location = res.headers['location']
  94       expectStartWith(location, start)
  95
  96       await makeRawRequest({ url: location, expectedStatus: HttpStatusCode.OK_200 })
  97     }
  98   }
  99
 100   for (const file of allFiles) {
 101     const torrent = await webtorrentAdd(file.magnetUri, true)
 102
 103     expect(torrent.files).to.be.an('array')
 104     expect(torrent.files.length).to.equal(1)
 105     expect(torrent.files[0].path).to.exist.and.to.not.equal('')
 106
 107     const res = await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 })
 108     expect(res.body).to.have.length.above(100)
 109   }
 110
 111   return allFiles.map(f => f.fileUrl)
 112 }
 113
 114 function runTestSuite (options: {
 115   fixture?: string
 116
 117   maxUploadPart?: string
 118
 119   playlistBucket: string
 120   playlistPrefix?: string
 121
 122   webtorrentBucket: string
 123   webtorrentPrefix?: string
 124
 125   useMockBaseUrl?: boolean
 126 }) {
 127   const mockObjectStorageProxy = new MockObjectStorageProxy()
 128   const { fixture } = options
 129   let baseMockUrl: string
 130
 131   let servers: PeerTubeServer[]
 132
 133   let keptUrls: string[] = []
 134
 135   const uuidsToDelete: string[] = []
 136   let deletedUrls: string[] = []
 137
 138   before(async function () {
 139     this.timeout(120000)
 140
 141     const port = await mockObjectStorageProxy.initialize()
 142     baseMockUrl = options.useMockBaseUrl
 143       ? `http://127.0.0.1:${port}`
 144       : undefined
 145
 146     await ObjectStorageCommand.createMockBucket(options.playlistBucket)
 147     await ObjectStorageCommand.createMockBucket(options.webtorrentBucket)
 148
 149     const config = {
 150       object_storage: {
 151         enabled: true,
 152         endpoint: 'http://' + ObjectStorageCommand.getMockEndpointHost(),
 153         region: ObjectStorageCommand.getMockRegion(),
 154
 155         credentials: ObjectStorageCommand.getMockCredentialsConfig(),
 156
 157         max_upload_part: options.maxUploadPart || '5MB',
 158
 159         streaming_playlists: {
 160           bucket_name: options.playlistBucket,
 161           prefix: options.playlistPrefix,
 162           base_url: baseMockUrl
 163             ? `${baseMockUrl}/${options.playlistBucket}`
 164             : undefined
 165         },
 166
 167         videos: {
 168           bucket_name: options.webtorrentBucket,
 169           prefix: options.webtorrentPrefix,
 170           base_url: baseMockUrl
 171             ? `${baseMockUrl}/${options.webtorrentBucket}`
 172             : undefined
 173         }
 174       }
 175     }
 176
 177     servers = await createMultipleServers(2, config)
 178
 179     await setAccessTokensToServers(servers)
 180     await doubleFollow(servers[0], servers[1])
 181
 182     for (const server of servers) {
 183       const { uuid } = await server.videos.quickUpload({ name: 'video to keep' })
 184       await waitJobs(servers)
 185
 186       const files = await server.videos.listFiles({ id: uuid })
 187       keptUrls = keptUrls.concat(files.map(f => f.fileUrl))
 188     }
 189   })
 190
 191   it('Should upload a video and move it to the object storage without transcoding', async function () {
 192     this.timeout(40000)
 193
 194     const { uuid } = await servers[0].videos.quickUpload({ name: 'video 1', fixture })
 195     uuidsToDelete.push(uuid)
 196
 197     await waitJobs(servers)
 198
 199     for (const server of servers) {
 200       const video = await server.videos.get({ id: uuid })
 201       const files = await checkFiles({ ...options, video, baseMockUrl })
 202
 203       deletedUrls = deletedUrls.concat(files)
 204     }
 205   })
 206
 207   it('Should upload a video and move it to the object storage with transcoding', async function () {
 208     this.timeout(120000)
 209
 210     const { uuid } = await servers[1].videos.quickUpload({ name: 'video 2', fixture })
 211     uuidsToDelete.push(uuid)
 212
 213     await waitJobs(servers)
 214
 215     for (const server of servers) {
 216       const video = await server.videos.get({ id: uuid })
 217       const files = await checkFiles({ ...options, video, baseMockUrl })
 218
 219       deletedUrls = deletedUrls.concat(files)
 220     }
 221   })
 222
 223   it('Should fetch correctly all the files', async function () {
 224     for (const url of deletedUrls.concat(keptUrls)) {
 225       await makeRawRequest({ url, expectedStatus: HttpStatusCode.OK_200 })
 226     }
 227   })
 228
 229   it('Should correctly delete the files', async function () {
 230     await servers[0].videos.remove({ id: uuidsToDelete[0] })
 231     await servers[1].videos.remove({ id: uuidsToDelete[1] })
 232
 233     await waitJobs(servers)
 234
 235     for (const url of deletedUrls) {
 236       await makeRawRequest({ url, expectedStatus: HttpStatusCode.NOT_FOUND_404 })
 237     }
 238   })
 239
 240   it('Should have kept other files', async function () {
 241     for (const url of keptUrls) {
 242       await makeRawRequest({ url, expectedStatus: HttpStatusCode.OK_200 })
 243     }
 244   })
 245
 246   it('Should have an empty tmp directory', async function () {
 247     for (const server of servers) {
 248       await checkTmpIsEmpty(server)
 249     }
 250   })
 251
 252   it('Should not have downloaded files from object storage', async function () {
 253     for (const server of servers) {
 254       await expectLogDoesNotContain(server, 'from object storage')
 255     }
 256   })
 257
 258   after(async function () {
 259     await mockObjectStorageProxy.terminate()
 260
 261     await cleanupTests(servers)
 262   })
 263 }
 264
 265 describe('Object storage for videos', function () {
 266   if (areMockObjectStorageTestsDisabled()) return
 267
 268   describe('Test config', function () {
 269     let server: PeerTubeServer
 270
 271     const baseConfig = {
 272       object_storage: {
 273         enabled: true,
 274         endpoint: 'http://' + ObjectStorageCommand.getMockEndpointHost(),
 275         region: ObjectStorageCommand.getMockRegion(),
 276
 277         credentials: ObjectStorageCommand.getMockCredentialsConfig(),
 278
 279         streaming_playlists: {
 280           bucket_name: ObjectStorageCommand.DEFAULT_PLAYLIST_MOCK_BUCKET
 281         },
 282
 283         videos: {
 284           bucket_name: ObjectStorageCommand.DEFAULT_WEBTORRENT_MOCK_BUCKET
 285         }
 286       }
 287     }
 288
 289     const badCredentials = {
 290       access_key_id: 'AKIAIOSFODNN7EXAMPLE',
 291       secret_access_key: 'aJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY'
 292     }
 293
 294     it('Should fail with same bucket names without prefix', function (done) {
 295       const config = merge({}, baseConfig, {
 296         object_storage: {
 297           streaming_playlists: {
 298             bucket_name: 'aaa'
 299           },
 300
 301           videos: {
 302             bucket_name: 'aaa'
 303           }
 304         }
 305       })
 306
 307       createSingleServer(1, config)
 308         .then(() => done(new Error('Did not throw')))
 309         .catch(() => done())
 310     })
 311
 312     it('Should fail with bad credentials', async function () {
 313       this.timeout(60000)
 314
 315       await ObjectStorageCommand.prepareDefaultMockBuckets()
 316
 317       const config = merge({}, baseConfig, {
 318         object_storage: {
 319           credentials: badCredentials
 320         }
 321       })
 322
 323       server = await createSingleServer(1, config)
 324       await setAccessTokensToServers([ server ])
 325
 326       const { uuid } = await server.videos.quickUpload({ name: 'video' })
 327
 328       await waitJobs([ server ], { skipDelayed: true })
 329       const video = await server.videos.get({ id: uuid })
 330
 331       expectStartWith(video.files[0].fileUrl, server.url)
 332
 333       await killallServers([ server ])
 334     })
 335
 336     it('Should succeed with credentials from env', async function () {
 337       this.timeout(60000)
 338
 339       await ObjectStorageCommand.prepareDefaultMockBuckets()
 340
 341       const config = merge({}, baseConfig, {
 342         object_storage: {
 343           credentials: {
 344             access_key_id: '',
 345             secret_access_key: ''
 346           }
 347         }
 348       })
 349
 350       const goodCredentials = ObjectStorageCommand.getMockCredentialsConfig()
 351
 352       server = await createSingleServer(1, config, {
 353         env: {
 354           AWS_ACCESS_KEY_ID: goodCredentials.access_key_id,
 355           AWS_SECRET_ACCESS_KEY: goodCredentials.secret_access_key
 356         }
 357       })
 358
 359       await setAccessTokensToServers([ server ])
 360
 361       const { uuid } = await server.videos.quickUpload({ name: 'video' })
 362
 363       await waitJobs([ server ], { skipDelayed: true })
 364       const video = await server.videos.get({ id: uuid })
 365
 366       expectStartWith(video.files[0].fileUrl, ObjectStorageCommand.getMockWebTorrentBaseUrl())
 367     })
 368
 369     after(async function () {
 370       await killallServers([ server ])
 371     })
 372   })
 373
 374   describe('Test simple object storage', function () {
 375     runTestSuite({
 376       playlistBucket: 'streaming-playlists',
 377       webtorrentBucket: 'videos'
 378     })
 379   })
 380
 381   describe('Test object storage with prefix', function () {
 382     runTestSuite({
 383       playlistBucket: 'mybucket',
 384       webtorrentBucket: 'mybucket',
 385
 386       playlistPrefix: 'streaming-playlists_',
 387       webtorrentPrefix: 'webtorrent_'
 388     })
 389   })
 390
 391   describe('Test object storage with prefix and base URL', function () {
 392     runTestSuite({
 393       playlistBucket: 'mybucket',
 394       webtorrentBucket: 'mybucket',
 395
 396       playlistPrefix: 'streaming-playlists/',
 397       webtorrentPrefix: 'webtorrent/',
 398
 399       useMockBaseUrl: true
 400     })
 401   })
 402
 403   describe('Test object storage with file bigger than upload part', function () {
 404     let fixture: string
 405     const maxUploadPart = '5MB'
 406
 407     before(async function () {
 408       this.timeout(120000)
 409
 410       fixture = await generateHighBitrateVideo()
 411
 412       const { size } = await stat(fixture)
 413
 414       if (bytes.parse(maxUploadPart) > size) {
 415         throw Error(`Fixture file is too small (${size}) to make sense for this test.`)
 416       }
 417     })
 418
 419     runTestSuite({
 420       maxUploadPart,
 421       playlistBucket: 'streaming-playlists',
 422       webtorrentBucket: 'videos',
 423       fixture
 424     })
 425   })
 426 })