1 /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
3 import bytes from 'bytes'
4 import { expect } from 'chai'
5 import { stat } from 'fs-extra'
6 import { merge } from 'lodash'
9 expectLogDoesNotContain,
11 generateHighBitrateVideo,
13 } from '@server/tests/shared'
14 import { areObjectStorageTestsDisabled } from '@shared/core-utils'
15 import { HttpStatusCode, VideoDetails } from '@shared/models'
18 createMultipleServers,
25 setAccessTokensToServers,
28 } from '@shared/server-commands'
30 async function checkFiles (options: {
35 playlistBucket: string
36 playlistPrefix?: string
38 webtorrentBucket: string
39 webtorrentPrefix?: string
50 let allFiles = video.files
52 for (const file of video.files) {
53 const baseUrl = baseMockUrl
54 ? `${baseMockUrl}/${webtorrentBucket}/`
55 : `http://${webtorrentBucket}.${ObjectStorageCommand.getEndpointHost()}/`
57 const prefix = webtorrentPrefix || ''
58 const start = baseUrl + prefix
60 expectStartWith(file.fileUrl, start)
62 const res = await makeRawRequest(file.fileDownloadUrl, HttpStatusCode.FOUND_302)
63 const location = res.headers['location']
64 expectStartWith(location, start)
66 await makeRawRequest(location, HttpStatusCode.OK_200)
69 const hls = video.streamingPlaylists[0]
72 allFiles = allFiles.concat(hls.files)
74 const baseUrl = baseMockUrl
75 ? `${baseMockUrl}/${playlistBucket}/`
76 : `http://${playlistBucket}.${ObjectStorageCommand.getEndpointHost()}/`
78 const prefix = playlistPrefix || ''
79 const start = baseUrl + prefix
81 expectStartWith(hls.playlistUrl, start)
82 expectStartWith(hls.segmentsSha256Url, start)
84 await makeRawRequest(hls.playlistUrl, HttpStatusCode.OK_200)
86 const resSha = await makeRawRequest(hls.segmentsSha256Url, HttpStatusCode.OK_200)
87 expect(JSON.stringify(resSha.body)).to.not.throw
89 for (const file of hls.files) {
90 expectStartWith(file.fileUrl, start)
92 const res = await makeRawRequest(file.fileDownloadUrl, HttpStatusCode.FOUND_302)
93 const location = res.headers['location']
94 expectStartWith(location, start)
96 await makeRawRequest(location, HttpStatusCode.OK_200)
100 for (const file of allFiles) {
101 const torrent = await webtorrentAdd(file.magnetUri, true)
103 expect(torrent.files).to.be.an('array')
104 expect(torrent.files.length).to.equal(1)
105 expect(torrent.files[0].path).to.exist.and.to.not.equal('')
107 const res = await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200)
108 expect(res.body).to.have.length.above(100)
111 return allFiles.map(f => f.fileUrl)
114 function runTestSuite (options: {
117 maxUploadPart?: string
119 playlistBucket: string
120 playlistPrefix?: string
122 webtorrentBucket: string
123 webtorrentPrefix?: string
125 useMockBaseUrl?: boolean
127 const mockObjectStorage = new MockObjectStorage()
128 const { fixture } = options
129 let baseMockUrl: string
131 let servers: PeerTubeServer[]
133 let keptUrls: string[] = []
135 const uuidsToDelete: string[] = []
136 let deletedUrls: string[] = []
138 before(async function () {
141 const port = await mockObjectStorage.initialize()
142 baseMockUrl = options.useMockBaseUrl ? `http://localhost:${port}` : undefined
144 await ObjectStorageCommand.createBucket(options.playlistBucket)
145 await ObjectStorageCommand.createBucket(options.webtorrentBucket)
150 endpoint: 'http://' + ObjectStorageCommand.getEndpointHost(),
151 region: ObjectStorageCommand.getRegion(),
153 credentials: ObjectStorageCommand.getCredentialsConfig(),
155 max_upload_part: options.maxUploadPart || '5MB',
157 streaming_playlists: {
158 bucket_name: options.playlistBucket,
159 prefix: options.playlistPrefix,
160 base_url: baseMockUrl
161 ? `${baseMockUrl}/${options.playlistBucket}`
166 bucket_name: options.webtorrentBucket,
167 prefix: options.webtorrentPrefix,
168 base_url: baseMockUrl
169 ? `${baseMockUrl}/${options.webtorrentBucket}`
175 servers = await createMultipleServers(2, config)
177 await setAccessTokensToServers(servers)
178 await doubleFollow(servers[0], servers[1])
180 for (const server of servers) {
181 const { uuid } = await server.videos.quickUpload({ name: 'video to keep' })
182 await waitJobs(servers)
184 const files = await server.videos.listFiles({ id: uuid })
185 keptUrls = keptUrls.concat(files.map(f => f.fileUrl))
189 it('Should upload a video and move it to the object storage without transcoding', async function () {
192 const { uuid } = await servers[0].videos.quickUpload({ name: 'video 1', fixture })
193 uuidsToDelete.push(uuid)
195 await waitJobs(servers)
197 for (const server of servers) {
198 const video = await server.videos.get({ id: uuid })
199 const files = await checkFiles({ ...options, video, baseMockUrl })
201 deletedUrls = deletedUrls.concat(files)
205 it('Should upload a video and move it to the object storage with transcoding', async function () {
208 const { uuid } = await servers[1].videos.quickUpload({ name: 'video 2', fixture })
209 uuidsToDelete.push(uuid)
211 await waitJobs(servers)
213 for (const server of servers) {
214 const video = await server.videos.get({ id: uuid })
215 const files = await checkFiles({ ...options, video, baseMockUrl })
217 deletedUrls = deletedUrls.concat(files)
221 it('Should fetch correctly all the files', async function () {
222 for (const url of deletedUrls.concat(keptUrls)) {
223 await makeRawRequest(url, HttpStatusCode.OK_200)
227 it('Should correctly delete the files', async function () {
228 await servers[0].videos.remove({ id: uuidsToDelete[0] })
229 await servers[1].videos.remove({ id: uuidsToDelete[1] })
231 await waitJobs(servers)
233 for (const url of deletedUrls) {
234 await makeRawRequest(url, HttpStatusCode.NOT_FOUND_404)
238 it('Should have kept other files', async function () {
239 for (const url of keptUrls) {
240 await makeRawRequest(url, HttpStatusCode.OK_200)
244 it('Should have an empty tmp directory', async function () {
245 for (const server of servers) {
246 await checkTmpIsEmpty(server)
250 it('Should not have downloaded files from object storage', async function () {
251 for (const server of servers) {
252 await expectLogDoesNotContain(server, 'from object storage')
256 after(async function () {
257 await mockObjectStorage.terminate()
259 await cleanupTests(servers)
263 describe('Object storage for videos', function () {
264 if (areObjectStorageTestsDisabled()) return
266 describe('Test config', function () {
267 let server: PeerTubeServer
272 endpoint: 'http://' + ObjectStorageCommand.getEndpointHost(),
273 region: ObjectStorageCommand.getRegion(),
275 credentials: ObjectStorageCommand.getCredentialsConfig(),
277 streaming_playlists: {
278 bucket_name: ObjectStorageCommand.DEFAULT_PLAYLIST_BUCKET
282 bucket_name: ObjectStorageCommand.DEFAULT_WEBTORRENT_BUCKET
287 const badCredentials = {
288 access_key_id: 'AKIAIOSFODNN7EXAMPLE',
289 secret_access_key: 'aJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY'
292 it('Should fail with same bucket names without prefix', function (done) {
293 const config = merge({}, baseConfig, {
295 streaming_playlists: {
305 createSingleServer(1, config)
306 .then(() => done(new Error('Did not throw')))
310 it('Should fail with bad credentials', async function () {
313 await ObjectStorageCommand.prepareDefaultBuckets()
315 const config = merge({}, baseConfig, {
317 credentials: badCredentials
321 server = await createSingleServer(1, config)
322 await setAccessTokensToServers([ server ])
324 const { uuid } = await server.videos.quickUpload({ name: 'video' })
326 await waitJobs([ server ], true)
327 const video = await server.videos.get({ id: uuid })
329 expectStartWith(video.files[0].fileUrl, server.url)
331 await killallServers([ server ])
334 it('Should succeed with credentials from env', async function () {
337 await ObjectStorageCommand.prepareDefaultBuckets()
339 const config = merge({}, baseConfig, {
343 secret_access_key: ''
348 const goodCredentials = ObjectStorageCommand.getCredentialsConfig()
350 server = await createSingleServer(1, config, {
352 AWS_ACCESS_KEY_ID: goodCredentials.access_key_id,
353 AWS_SECRET_ACCESS_KEY: goodCredentials.secret_access_key
357 await setAccessTokensToServers([ server ])
359 const { uuid } = await server.videos.quickUpload({ name: 'video' })
361 await waitJobs([ server ], true)
362 const video = await server.videos.get({ id: uuid })
364 expectStartWith(video.files[0].fileUrl, ObjectStorageCommand.getWebTorrentBaseUrl())
367 after(async function () {
368 await killallServers([ server ])
372 describe('Test simple object storage', function () {
374 playlistBucket: 'streaming-playlists',
375 webtorrentBucket: 'videos'
379 describe('Test object storage with prefix', function () {
381 playlistBucket: 'mybucket',
382 webtorrentBucket: 'mybucket',
384 playlistPrefix: 'streaming-playlists_',
385 webtorrentPrefix: 'webtorrent_'
389 describe('Test object storage with prefix and base URL', function () {
391 playlistBucket: 'mybucket',
392 webtorrentBucket: 'mybucket',
394 playlistPrefix: 'streaming-playlists/',
395 webtorrentPrefix: 'webtorrent/',
401 describe('Test object storage with file bigger than upload part', function () {
403 const maxUploadPart = '5MB'
405 before(async function () {
408 fixture = await generateHighBitrateVideo()
410 const { size } = await stat(fixture)
412 if (bytes.parse(maxUploadPart) > size) {
413 throw Error(`Fixture file is too small (${size}) to make sense for this test.`)
419 playlistBucket: 'streaming-playlists',
420 webtorrentBucket: 'videos',