server/tests/api/object-storage/videos.ts

   1 /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
   2
   3 import 'mocha'
   4 import * as chai from 'chai'
   5 import { merge } from 'lodash'
   6 import { checkTmpIsEmpty, expectLogDoesNotContain, expectStartWith, MockObjectStorage } from '@server/tests/shared'
   7 import { areObjectStorageTestsDisabled } from '@shared/core-utils'
   8 import { HttpStatusCode, VideoDetails } from '@shared/models'
   9 import {
  10   cleanupTests,
  11   createMultipleServers,
  12   createSingleServer,
  13   doubleFollow,
  14   killallServers,
  15   makeRawRequest,
  16   ObjectStorageCommand,
  17   PeerTubeServer,
  18   setAccessTokensToServers,
  19   waitJobs,
  20   webtorrentAdd
  21 } from '@shared/server-commands'
  22
  23 const expect = chai.expect
  24
  25 async function checkFiles (options: {
  26   video: VideoDetails
  27
  28   baseMockUrl?: string
  29
  30   playlistBucket: string
  31   playlistPrefix?: string
  32
  33   webtorrentBucket: string
  34   webtorrentPrefix?: string
  35 }) {
  36   const {
  37     video,
  38     playlistBucket,
  39     webtorrentBucket,
  40     baseMockUrl,
  41     playlistPrefix,
  42     webtorrentPrefix
  43   } = options
  44
  45   let allFiles = video.files
  46
  47   for (const file of video.files) {
  48     const baseUrl = baseMockUrl
  49       ? `${baseMockUrl}/${webtorrentBucket}/`
  50       : `http://${webtorrentBucket}.${ObjectStorageCommand.getEndpointHost()}/`
  51
  52     const prefix = webtorrentPrefix || ''
  53     const start = baseUrl + prefix
  54
  55     expectStartWith(file.fileUrl, start)
  56
  57     const res = await makeRawRequest(file.fileDownloadUrl, HttpStatusCode.FOUND_302)
  58     const location = res.headers['location']
  59     expectStartWith(location, start)
  60
  61     await makeRawRequest(location, HttpStatusCode.OK_200)
  62   }
  63
  64   const hls = video.streamingPlaylists[0]
  65
  66   if (hls) {
  67     allFiles = allFiles.concat(hls.files)
  68
  69     const baseUrl = baseMockUrl
  70       ? `${baseMockUrl}/${playlistBucket}/`
  71       : `http://${playlistBucket}.${ObjectStorageCommand.getEndpointHost()}/`
  72
  73     const prefix = playlistPrefix || ''
  74     const start = baseUrl + prefix
  75
  76     expectStartWith(hls.playlistUrl, start)
  77     expectStartWith(hls.segmentsSha256Url, start)
  78
  79     await makeRawRequest(hls.playlistUrl, HttpStatusCode.OK_200)
  80
  81     const resSha = await makeRawRequest(hls.segmentsSha256Url, HttpStatusCode.OK_200)
  82     expect(JSON.stringify(resSha.body)).to.not.throw
  83
  84     for (const file of hls.files) {
  85       expectStartWith(file.fileUrl, start)
  86
  87       const res = await makeRawRequest(file.fileDownloadUrl, HttpStatusCode.FOUND_302)
  88       const location = res.headers['location']
  89       expectStartWith(location, start)
  90
  91       await makeRawRequest(location, HttpStatusCode.OK_200)
  92     }
  93   }
  94
  95   for (const file of allFiles) {
  96     const torrent = await webtorrentAdd(file.magnetUri, true)
  97
  98     expect(torrent.files).to.be.an('array')
  99     expect(torrent.files.length).to.equal(1)
 100     expect(torrent.files[0].path).to.exist.and.to.not.equal('')
 101
 102     const res = await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200)
 103     expect(res.body).to.have.length.above(100)
 104   }
 105
 106   return allFiles.map(f => f.fileUrl)
 107 }
 108
 109 function runTestSuite (options: {
 110   playlistBucket: string
 111   playlistPrefix?: string
 112
 113   webtorrentBucket: string
 114   webtorrentPrefix?: string
 115
 116   useMockBaseUrl?: boolean
 117
 118   maxUploadPart?: string
 119 }) {
 120   const mockObjectStorage = new MockObjectStorage()
 121   let baseMockUrl: string
 122
 123   let servers: PeerTubeServer[]
 124
 125   let keptUrls: string[] = []
 126
 127   const uuidsToDelete: string[] = []
 128   let deletedUrls: string[] = []
 129
 130   before(async function () {
 131     this.timeout(120000)
 132
 133     const port = await mockObjectStorage.initialize()
 134     baseMockUrl = options.useMockBaseUrl ? `http://localhost:${port}` : undefined
 135
 136     await ObjectStorageCommand.createBucket(options.playlistBucket)
 137     await ObjectStorageCommand.createBucket(options.webtorrentBucket)
 138
 139     const config = {
 140       object_storage: {
 141         enabled: true,
 142         endpoint: 'http://' + ObjectStorageCommand.getEndpointHost(),
 143         region: ObjectStorageCommand.getRegion(),
 144
 145         credentials: ObjectStorageCommand.getCredentialsConfig(),
 146
 147         max_upload_part: options.maxUploadPart || '2MB',
 148
 149         streaming_playlists: {
 150           bucket_name: options.playlistBucket,
 151           prefix: options.playlistPrefix,
 152           base_url: baseMockUrl
 153             ? `${baseMockUrl}/${options.playlistBucket}`
 154             : undefined
 155         },
 156
 157         videos: {
 158           bucket_name: options.webtorrentBucket,
 159           prefix: options.webtorrentPrefix,
 160           base_url: baseMockUrl
 161             ? `${baseMockUrl}/${options.webtorrentBucket}`
 162             : undefined
 163         }
 164       }
 165     }
 166
 167     servers = await createMultipleServers(2, config)
 168
 169     await setAccessTokensToServers(servers)
 170     await doubleFollow(servers[0], servers[1])
 171
 172     for (const server of servers) {
 173       const { uuid } = await server.videos.quickUpload({ name: 'video to keep' })
 174       await waitJobs(servers)
 175
 176       const files = await server.videos.listFiles({ id: uuid })
 177       keptUrls = keptUrls.concat(files.map(f => f.fileUrl))
 178     }
 179   })
 180
 181   it('Should upload a video and move it to the object storage without transcoding', async function () {
 182     this.timeout(40000)
 183
 184     const { uuid } = await servers[0].videos.quickUpload({ name: 'video 1' })
 185     uuidsToDelete.push(uuid)
 186
 187     await waitJobs(servers)
 188
 189     for (const server of servers) {
 190       const video = await server.videos.get({ id: uuid })
 191       const files = await checkFiles({ ...options, video, baseMockUrl })
 192
 193       deletedUrls = deletedUrls.concat(files)
 194     }
 195   })
 196
 197   it('Should upload a video and move it to the object storage with transcoding', async function () {
 198     this.timeout(120000)
 199
 200     const { uuid } = await servers[1].videos.quickUpload({ name: 'video 2' })
 201     uuidsToDelete.push(uuid)
 202
 203     await waitJobs(servers)
 204
 205     for (const server of servers) {
 206       const video = await server.videos.get({ id: uuid })
 207       const files = await checkFiles({ ...options, video, baseMockUrl })
 208
 209       deletedUrls = deletedUrls.concat(files)
 210     }
 211   })
 212
 213   it('Should fetch correctly all the files', async function () {
 214     for (const url of deletedUrls.concat(keptUrls)) {
 215       await makeRawRequest(url, HttpStatusCode.OK_200)
 216     }
 217   })
 218
 219   it('Should correctly delete the files', async function () {
 220     await servers[0].videos.remove({ id: uuidsToDelete[0] })
 221     await servers[1].videos.remove({ id: uuidsToDelete[1] })
 222
 223     await waitJobs(servers)
 224
 225     for (const url of deletedUrls) {
 226       await makeRawRequest(url, HttpStatusCode.NOT_FOUND_404)
 227     }
 228   })
 229
 230   it('Should have kept other files', async function () {
 231     for (const url of keptUrls) {
 232       await makeRawRequest(url, HttpStatusCode.OK_200)
 233     }
 234   })
 235
 236   it('Should have an empty tmp directory', async function () {
 237     for (const server of servers) {
 238       await checkTmpIsEmpty(server)
 239     }
 240   })
 241
 242   it('Should not have downloaded files from object storage', async function () {
 243     for (const server of servers) {
 244       await expectLogDoesNotContain(server, 'from object storage')
 245     }
 246   })
 247
 248   after(async function () {
 249     await mockObjectStorage.terminate()
 250
 251     await cleanupTests(servers)
 252   })
 253 }
 254
 255 describe('Object storage for videos', function () {
 256   if (areObjectStorageTestsDisabled()) return
 257
 258   describe('Test config', function () {
 259     let server: PeerTubeServer
 260
 261     const baseConfig = {
 262       object_storage: {
 263         enabled: true,
 264         endpoint: 'http://' + ObjectStorageCommand.getEndpointHost(),
 265         region: ObjectStorageCommand.getRegion(),
 266
 267         credentials: ObjectStorageCommand.getCredentialsConfig(),
 268
 269         streaming_playlists: {
 270           bucket_name: ObjectStorageCommand.DEFAULT_PLAYLIST_BUCKET
 271         },
 272
 273         videos: {
 274           bucket_name: ObjectStorageCommand.DEFAULT_WEBTORRENT_BUCKET
 275         }
 276       }
 277     }
 278
 279     const badCredentials = {
 280       access_key_id: 'AKIAIOSFODNN7EXAMPLE',
 281       secret_access_key: 'aJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY'
 282     }
 283
 284     it('Should fail with same bucket names without prefix', function (done) {
 285       const config = merge({}, baseConfig, {
 286         object_storage: {
 287           streaming_playlists: {
 288             bucket_name: 'aaa'
 289           },
 290
 291           videos: {
 292             bucket_name: 'aaa'
 293           }
 294         }
 295       })
 296
 297       createSingleServer(1, config)
 298         .then(() => done(new Error('Did not throw')))
 299         .catch(() => done())
 300     })
 301
 302     it('Should fail with bad credentials', async function () {
 303       this.timeout(60000)
 304
 305       await ObjectStorageCommand.prepareDefaultBuckets()
 306
 307       const config = merge({}, baseConfig, {
 308         object_storage: {
 309           credentials: badCredentials
 310         }
 311       })
 312
 313       server = await createSingleServer(1, config)
 314       await setAccessTokensToServers([ server ])
 315
 316       const { uuid } = await server.videos.quickUpload({ name: 'video' })
 317
 318       await waitJobs([ server ], true)
 319       const video = await server.videos.get({ id: uuid })
 320
 321       expectStartWith(video.files[0].fileUrl, server.url)
 322
 323       await killallServers([ server ])
 324     })
 325
 326     it('Should succeed with credentials from env', async function () {
 327       this.timeout(60000)
 328
 329       await ObjectStorageCommand.prepareDefaultBuckets()
 330
 331       const config = merge({}, baseConfig, {
 332         object_storage: {
 333           credentials: {
 334             access_key_id: '',
 335             secret_access_key: ''
 336           }
 337         }
 338       })
 339
 340       const goodCredentials = ObjectStorageCommand.getCredentialsConfig()
 341
 342       server = await createSingleServer(1, config, {
 343         env: {
 344           AWS_ACCESS_KEY_ID: goodCredentials.access_key_id,
 345           AWS_SECRET_ACCESS_KEY: goodCredentials.secret_access_key
 346         }
 347       })
 348
 349       await setAccessTokensToServers([ server ])
 350
 351       const { uuid } = await server.videos.quickUpload({ name: 'video' })
 352
 353       await waitJobs([ server ], true)
 354       const video = await server.videos.get({ id: uuid })
 355
 356       expectStartWith(video.files[0].fileUrl, ObjectStorageCommand.getWebTorrentBaseUrl())
 357     })
 358
 359     after(async function () {
 360       await killallServers([ server ])
 361     })
 362   })
 363
 364   describe('Test simple object storage', function () {
 365     runTestSuite({
 366       playlistBucket: 'streaming-playlists',
 367       webtorrentBucket: 'videos'
 368     })
 369   })
 370
 371   describe('Test object storage with prefix', function () {
 372     runTestSuite({
 373       playlistBucket: 'mybucket',
 374       webtorrentBucket: 'mybucket',
 375
 376       playlistPrefix: 'streaming-playlists_',
 377       webtorrentPrefix: 'webtorrent_'
 378     })
 379   })
 380
 381   describe('Test object storage with prefix and base URL', function () {
 382     runTestSuite({
 383       playlistBucket: 'mybucket',
 384       webtorrentBucket: 'mybucket',
 385
 386       playlistPrefix: 'streaming-playlists/',
 387       webtorrentPrefix: 'webtorrent/',
 388
 389       useMockBaseUrl: true
 390     })
 391   })
 392
 393   describe('Test object storage with small upload part', function () {
 394     runTestSuite({
 395       playlistBucket: 'streaming-playlists',
 396       webtorrentBucket: 'videos',
 397
 398       maxUploadPart: '5KB'
 399     })
 400   })
 401 })