1 /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
4 import bytes from 'bytes'
5 import * as chai from 'chai'
6 import { stat } from 'fs-extra'
7 import { merge } from 'lodash'
10 expectLogDoesNotContain,
12 generateHighBitrateVideo,
14 } from '@server/tests/shared'
15 import { areObjectStorageTestsDisabled } from '@shared/core-utils'
16 import { HttpStatusCode, VideoDetails } from '@shared/models'
19 createMultipleServers,
26 setAccessTokensToServers,
29 } from '@shared/server-commands'
31 const expect = chai.expect
33 async function checkFiles (options: {
38 playlistBucket: string
39 playlistPrefix?: string
41 webtorrentBucket: string
42 webtorrentPrefix?: string
53 let allFiles = video.files
55 for (const file of video.files) {
56 const baseUrl = baseMockUrl
57 ? `${baseMockUrl}/${webtorrentBucket}/`
58 : `http://${webtorrentBucket}.${ObjectStorageCommand.getEndpointHost()}/`
60 const prefix = webtorrentPrefix || ''
61 const start = baseUrl + prefix
63 expectStartWith(file.fileUrl, start)
65 const res = await makeRawRequest(file.fileDownloadUrl, HttpStatusCode.FOUND_302)
66 const location = res.headers['location']
67 expectStartWith(location, start)
69 await makeRawRequest(location, HttpStatusCode.OK_200)
72 const hls = video.streamingPlaylists[0]
75 allFiles = allFiles.concat(hls.files)
77 const baseUrl = baseMockUrl
78 ? `${baseMockUrl}/${playlistBucket}/`
79 : `http://${playlistBucket}.${ObjectStorageCommand.getEndpointHost()}/`
81 const prefix = playlistPrefix || ''
82 const start = baseUrl + prefix
84 expectStartWith(hls.playlistUrl, start)
85 expectStartWith(hls.segmentsSha256Url, start)
87 await makeRawRequest(hls.playlistUrl, HttpStatusCode.OK_200)
89 const resSha = await makeRawRequest(hls.segmentsSha256Url, HttpStatusCode.OK_200)
90 expect(JSON.stringify(resSha.body)).to.not.throw
92 for (const file of hls.files) {
93 expectStartWith(file.fileUrl, start)
95 const res = await makeRawRequest(file.fileDownloadUrl, HttpStatusCode.FOUND_302)
96 const location = res.headers['location']
97 expectStartWith(location, start)
99 await makeRawRequest(location, HttpStatusCode.OK_200)
103 for (const file of allFiles) {
104 const torrent = await webtorrentAdd(file.magnetUri, true)
106 expect(torrent.files).to.be.an('array')
107 expect(torrent.files.length).to.equal(1)
108 expect(torrent.files[0].path).to.exist.and.to.not.equal('')
110 const res = await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200)
111 expect(res.body).to.have.length.above(100)
114 return allFiles.map(f => f.fileUrl)
117 function runTestSuite (options: {
120 maxUploadPart?: string
122 playlistBucket: string
123 playlistPrefix?: string
125 webtorrentBucket: string
126 webtorrentPrefix?: string
128 useMockBaseUrl?: boolean
130 const mockObjectStorage = new MockObjectStorage()
131 const { fixture } = options
132 let baseMockUrl: string
134 let servers: PeerTubeServer[]
136 let keptUrls: string[] = []
138 const uuidsToDelete: string[] = []
139 let deletedUrls: string[] = []
141 before(async function () {
144 const port = await mockObjectStorage.initialize()
145 baseMockUrl = options.useMockBaseUrl ? `http://localhost:${port}` : undefined
147 await ObjectStorageCommand.createBucket(options.playlistBucket)
148 await ObjectStorageCommand.createBucket(options.webtorrentBucket)
153 endpoint: 'http://' + ObjectStorageCommand.getEndpointHost(),
154 region: ObjectStorageCommand.getRegion(),
156 credentials: ObjectStorageCommand.getCredentialsConfig(),
158 max_upload_part: options.maxUploadPart || '5MB',
160 streaming_playlists: {
161 bucket_name: options.playlistBucket,
162 prefix: options.playlistPrefix,
163 base_url: baseMockUrl
164 ? `${baseMockUrl}/${options.playlistBucket}`
169 bucket_name: options.webtorrentBucket,
170 prefix: options.webtorrentPrefix,
171 base_url: baseMockUrl
172 ? `${baseMockUrl}/${options.webtorrentBucket}`
178 servers = await createMultipleServers(2, config)
180 await setAccessTokensToServers(servers)
181 await doubleFollow(servers[0], servers[1])
183 for (const server of servers) {
184 const { uuid } = await server.videos.quickUpload({ name: 'video to keep' })
185 await waitJobs(servers)
187 const files = await server.videos.listFiles({ id: uuid })
188 keptUrls = keptUrls.concat(files.map(f => f.fileUrl))
192 it('Should upload a video and move it to the object storage without transcoding', async function () {
195 const { uuid } = await servers[0].videos.quickUpload({ name: 'video 1', fixture })
196 uuidsToDelete.push(uuid)
198 await waitJobs(servers)
200 for (const server of servers) {
201 const video = await server.videos.get({ id: uuid })
202 const files = await checkFiles({ ...options, video, baseMockUrl })
204 deletedUrls = deletedUrls.concat(files)
208 it('Should upload a video and move it to the object storage with transcoding', async function () {
211 const { uuid } = await servers[1].videos.quickUpload({ name: 'video 2', fixture })
212 uuidsToDelete.push(uuid)
214 await waitJobs(servers)
216 for (const server of servers) {
217 const video = await server.videos.get({ id: uuid })
218 const files = await checkFiles({ ...options, video, baseMockUrl })
220 deletedUrls = deletedUrls.concat(files)
224 it('Should fetch correctly all the files', async function () {
225 for (const url of deletedUrls.concat(keptUrls)) {
226 await makeRawRequest(url, HttpStatusCode.OK_200)
230 it('Should correctly delete the files', async function () {
231 await servers[0].videos.remove({ id: uuidsToDelete[0] })
232 await servers[1].videos.remove({ id: uuidsToDelete[1] })
234 await waitJobs(servers)
236 for (const url of deletedUrls) {
237 await makeRawRequest(url, HttpStatusCode.NOT_FOUND_404)
241 it('Should have kept other files', async function () {
242 for (const url of keptUrls) {
243 await makeRawRequest(url, HttpStatusCode.OK_200)
247 it('Should have an empty tmp directory', async function () {
248 for (const server of servers) {
249 await checkTmpIsEmpty(server)
253 it('Should not have downloaded files from object storage', async function () {
254 for (const server of servers) {
255 await expectLogDoesNotContain(server, 'from object storage')
259 after(async function () {
260 await mockObjectStorage.terminate()
262 await cleanupTests(servers)
266 describe('Object storage for videos', function () {
267 if (areObjectStorageTestsDisabled()) return
269 describe('Test config', function () {
270 let server: PeerTubeServer
275 endpoint: 'http://' + ObjectStorageCommand.getEndpointHost(),
276 region: ObjectStorageCommand.getRegion(),
278 credentials: ObjectStorageCommand.getCredentialsConfig(),
280 streaming_playlists: {
281 bucket_name: ObjectStorageCommand.DEFAULT_PLAYLIST_BUCKET
285 bucket_name: ObjectStorageCommand.DEFAULT_WEBTORRENT_BUCKET
290 const badCredentials = {
291 access_key_id: 'AKIAIOSFODNN7EXAMPLE',
292 secret_access_key: 'aJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY'
295 it('Should fail with same bucket names without prefix', function (done) {
296 const config = merge({}, baseConfig, {
298 streaming_playlists: {
308 createSingleServer(1, config)
309 .then(() => done(new Error('Did not throw')))
313 it('Should fail with bad credentials', async function () {
316 await ObjectStorageCommand.prepareDefaultBuckets()
318 const config = merge({}, baseConfig, {
320 credentials: badCredentials
324 server = await createSingleServer(1, config)
325 await setAccessTokensToServers([ server ])
327 const { uuid } = await server.videos.quickUpload({ name: 'video' })
329 await waitJobs([ server ], true)
330 const video = await server.videos.get({ id: uuid })
332 expectStartWith(video.files[0].fileUrl, server.url)
334 await killallServers([ server ])
337 it('Should succeed with credentials from env', async function () {
340 await ObjectStorageCommand.prepareDefaultBuckets()
342 const config = merge({}, baseConfig, {
346 secret_access_key: ''
351 const goodCredentials = ObjectStorageCommand.getCredentialsConfig()
353 server = await createSingleServer(1, config, {
355 AWS_ACCESS_KEY_ID: goodCredentials.access_key_id,
356 AWS_SECRET_ACCESS_KEY: goodCredentials.secret_access_key
360 await setAccessTokensToServers([ server ])
362 const { uuid } = await server.videos.quickUpload({ name: 'video' })
364 await waitJobs([ server ], true)
365 const video = await server.videos.get({ id: uuid })
367 expectStartWith(video.files[0].fileUrl, ObjectStorageCommand.getWebTorrentBaseUrl())
370 after(async function () {
371 await killallServers([ server ])
375 describe('Test simple object storage', function () {
377 playlistBucket: 'streaming-playlists',
378 webtorrentBucket: 'videos'
382 describe('Test object storage with prefix', function () {
384 playlistBucket: 'mybucket',
385 webtorrentBucket: 'mybucket',
387 playlistPrefix: 'streaming-playlists_',
388 webtorrentPrefix: 'webtorrent_'
392 describe('Test object storage with prefix and base URL', function () {
394 playlistBucket: 'mybucket',
395 webtorrentBucket: 'mybucket',
397 playlistPrefix: 'streaming-playlists/',
398 webtorrentPrefix: 'webtorrent/',
404 describe('Test object storage with file bigger than upload part', function () {
406 const maxUploadPart = '5MB'
408 before(async function () {
409 fixture = await generateHighBitrateVideo()
411 const { size } = await stat(fixture)
413 if (bytes.parse(maxUploadPart) > size) {
414 throw Error(`Fixture file is too small (${size}) to make sense for this test.`)
420 playlistBucket: 'streaming-playlists',
421 webtorrentBucket: 'videos',