1 /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
3 import bytes from 'bytes'
4 import { expect } from 'chai'
5 import { stat } from 'fs-extra'
6 import { merge } from 'lodash'
10 expectLogDoesNotContain,
12 generateHighBitrateVideo,
13 MockObjectStorageProxy,
15 } from '@server/tests/shared'
16 import { areMockObjectStorageTestsDisabled } from '@shared/core-utils'
17 import { sha1 } from '@shared/extra-utils'
18 import { HttpStatusCode, VideoDetails } from '@shared/models'
21 createMultipleServers,
28 setAccessTokensToServers,
30 } from '@shared/server-commands'
32 async function checkFiles (options: {
33 server: PeerTubeServer
34 originServer: PeerTubeServer
35 originSQLCommand: SQLCommand
41 playlistBucket: string
42 playlistPrefix?: string
44 webtorrentBucket: string
45 webtorrentPrefix?: string
59 let allFiles = video.files
61 for (const file of video.files) {
62 const baseUrl = baseMockUrl
63 ? `${baseMockUrl}/${webtorrentBucket}/`
64 : `http://${webtorrentBucket}.${ObjectStorageCommand.getMockEndpointHost()}/`
66 const prefix = webtorrentPrefix || ''
67 const start = baseUrl + prefix
69 expectStartWith(file.fileUrl, start)
71 const res = await makeRawRequest({ url: file.fileDownloadUrl, expectedStatus: HttpStatusCode.FOUND_302 })
72 const location = res.headers['location']
73 expectStartWith(location, start)
75 await makeRawRequest({ url: location, expectedStatus: HttpStatusCode.OK_200 })
78 const hls = video.streamingPlaylists[0]
81 allFiles = allFiles.concat(hls.files)
83 const baseUrl = baseMockUrl
84 ? `${baseMockUrl}/${playlistBucket}/`
85 : `http://${playlistBucket}.${ObjectStorageCommand.getMockEndpointHost()}/`
87 const prefix = playlistPrefix || ''
88 const start = baseUrl + prefix
90 expectStartWith(hls.playlistUrl, start)
91 expectStartWith(hls.segmentsSha256Url, start)
93 await makeRawRequest({ url: hls.playlistUrl, expectedStatus: HttpStatusCode.OK_200 })
95 const resSha = await makeRawRequest({ url: hls.segmentsSha256Url, expectedStatus: HttpStatusCode.OK_200 })
96 expect(JSON.stringify(resSha.body)).to.not.throw
99 for (const file of hls.files) {
100 expectStartWith(file.fileUrl, start)
102 const res = await makeRawRequest({ url: file.fileDownloadUrl, expectedStatus: HttpStatusCode.FOUND_302 })
103 const location = res.headers['location']
104 expectStartWith(location, start)
106 await makeRawRequest({ url: location, expectedStatus: HttpStatusCode.OK_200 })
108 if (originServer.internalServerNumber === server.internalServerNumber) {
109 const infohash = sha1(`${2 + hls.playlistUrl}+V${i}`)
110 const dbInfohashes = await originSQLCommand.getPlaylistInfohash(hls.id)
112 expect(dbInfohashes).to.include(infohash)
119 for (const file of allFiles) {
120 await checkWebTorrentWorks(file.magnetUri)
122 const res = await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 })
123 expect(res.body).to.have.length.above(100)
126 return allFiles.map(f => f.fileUrl)
129 function runTestSuite (options: {
132 maxUploadPart?: string
134 playlistBucket: string
135 playlistPrefix?: string
137 webtorrentBucket: string
138 webtorrentPrefix?: string
140 useMockBaseUrl?: boolean
142 const mockObjectStorageProxy = new MockObjectStorageProxy()
143 const { fixture } = options
144 let baseMockUrl: string
146 let servers: PeerTubeServer[]
147 let sqlCommands: SQLCommand[]
149 let keptUrls: string[] = []
151 const uuidsToDelete: string[] = []
152 let deletedUrls: string[] = []
154 before(async function () {
157 const port = await mockObjectStorageProxy.initialize()
158 baseMockUrl = options.useMockBaseUrl
159 ? `http://127.0.0.1:${port}`
162 await ObjectStorageCommand.createMockBucket(options.playlistBucket)
163 await ObjectStorageCommand.createMockBucket(options.webtorrentBucket)
168 endpoint: 'http://' + ObjectStorageCommand.getMockEndpointHost(),
169 region: ObjectStorageCommand.getMockRegion(),
171 credentials: ObjectStorageCommand.getMockCredentialsConfig(),
173 max_upload_part: options.maxUploadPart || '5MB',
175 streaming_playlists: {
176 bucket_name: options.playlistBucket,
177 prefix: options.playlistPrefix,
178 base_url: baseMockUrl
179 ? `${baseMockUrl}/${options.playlistBucket}`
184 bucket_name: options.webtorrentBucket,
185 prefix: options.webtorrentPrefix,
186 base_url: baseMockUrl
187 ? `${baseMockUrl}/${options.webtorrentBucket}`
193 servers = await createMultipleServers(2, config)
195 await setAccessTokensToServers(servers)
196 await doubleFollow(servers[0], servers[1])
198 for (const server of servers) {
199 const { uuid } = await server.videos.quickUpload({ name: 'video to keep' })
200 await waitJobs(servers)
202 const files = await server.videos.listFiles({ id: uuid })
203 keptUrls = keptUrls.concat(files.map(f => f.fileUrl))
206 sqlCommands = servers.map(s => new SQLCommand(s))
209 it('Should upload a video and move it to the object storage without transcoding', async function () {
212 const { uuid } = await servers[0].videos.quickUpload({ name: 'video 1', fixture })
213 uuidsToDelete.push(uuid)
215 await waitJobs(servers)
217 for (const server of servers) {
218 const video = await server.videos.get({ id: uuid })
219 const files = await checkFiles({ ...options, server, originServer: servers[0], originSQLCommand: sqlCommands[0], video, baseMockUrl })
221 deletedUrls = deletedUrls.concat(files)
225 it('Should upload a video and move it to the object storage with transcoding', async function () {
228 const { uuid } = await servers[1].videos.quickUpload({ name: 'video 2', fixture })
229 uuidsToDelete.push(uuid)
231 await waitJobs(servers)
233 for (const server of servers) {
234 const video = await server.videos.get({ id: uuid })
235 const files = await checkFiles({ ...options, server, originServer: servers[0], originSQLCommand: sqlCommands[0], video, baseMockUrl })
237 deletedUrls = deletedUrls.concat(files)
241 it('Should fetch correctly all the files', async function () {
242 for (const url of deletedUrls.concat(keptUrls)) {
243 await makeRawRequest({ url, expectedStatus: HttpStatusCode.OK_200 })
247 it('Should correctly delete the files', async function () {
248 await servers[0].videos.remove({ id: uuidsToDelete[0] })
249 await servers[1].videos.remove({ id: uuidsToDelete[1] })
251 await waitJobs(servers)
253 for (const url of deletedUrls) {
254 await makeRawRequest({ url, expectedStatus: HttpStatusCode.NOT_FOUND_404 })
258 it('Should have kept other files', async function () {
259 for (const url of keptUrls) {
260 await makeRawRequest({ url, expectedStatus: HttpStatusCode.OK_200 })
264 it('Should have an empty tmp directory', async function () {
265 for (const server of servers) {
266 await checkTmpIsEmpty(server)
270 it('Should not have downloaded files from object storage', async function () {
271 for (const server of servers) {
272 await expectLogDoesNotContain(server, 'from object storage')
276 after(async function () {
277 await mockObjectStorageProxy.terminate()
279 for (const sqlCommand of sqlCommands) {
280 await sqlCommand.cleanup()
283 await cleanupTests(servers)
287 describe('Object storage for videos', function () {
288 if (areMockObjectStorageTestsDisabled()) return
290 describe('Test config', function () {
291 let server: PeerTubeServer
296 endpoint: 'http://' + ObjectStorageCommand.getMockEndpointHost(),
297 region: ObjectStorageCommand.getMockRegion(),
299 credentials: ObjectStorageCommand.getMockCredentialsConfig(),
301 streaming_playlists: {
302 bucket_name: ObjectStorageCommand.DEFAULT_PLAYLIST_MOCK_BUCKET
306 bucket_name: ObjectStorageCommand.DEFAULT_WEBTORRENT_MOCK_BUCKET
311 const badCredentials = {
312 access_key_id: 'AKIAIOSFODNN7EXAMPLE',
313 secret_access_key: 'aJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY'
316 it('Should fail with same bucket names without prefix', function (done) {
317 const config = merge({}, baseConfig, {
319 streaming_playlists: {
329 createSingleServer(1, config)
330 .then(() => done(new Error('Did not throw')))
334 it('Should fail with bad credentials', async function () {
337 await ObjectStorageCommand.prepareDefaultMockBuckets()
339 const config = merge({}, baseConfig, {
341 credentials: badCredentials
345 server = await createSingleServer(1, config)
346 await setAccessTokensToServers([ server ])
348 const { uuid } = await server.videos.quickUpload({ name: 'video' })
350 await waitJobs([ server ], { skipDelayed: true })
351 const video = await server.videos.get({ id: uuid })
353 expectStartWith(video.files[0].fileUrl, server.url)
355 await killallServers([ server ])
358 it('Should succeed with credentials from env', async function () {
361 await ObjectStorageCommand.prepareDefaultMockBuckets()
363 const config = merge({}, baseConfig, {
367 secret_access_key: ''
372 const goodCredentials = ObjectStorageCommand.getMockCredentialsConfig()
374 server = await createSingleServer(1, config, {
376 AWS_ACCESS_KEY_ID: goodCredentials.access_key_id,
377 AWS_SECRET_ACCESS_KEY: goodCredentials.secret_access_key
381 await setAccessTokensToServers([ server ])
383 const { uuid } = await server.videos.quickUpload({ name: 'video' })
385 await waitJobs([ server ], { skipDelayed: true })
386 const video = await server.videos.get({ id: uuid })
388 expectStartWith(video.files[0].fileUrl, ObjectStorageCommand.getMockWebTorrentBaseUrl())
391 after(async function () {
392 await killallServers([ server ])
396 describe('Test simple object storage', function () {
398 playlistBucket: 'streaming-playlists',
399 webtorrentBucket: 'videos'
403 describe('Test object storage with prefix', function () {
405 playlistBucket: 'mybucket',
406 webtorrentBucket: 'mybucket',
408 playlistPrefix: 'streaming-playlists_',
409 webtorrentPrefix: 'webtorrent_'
413 describe('Test object storage with prefix and base URL', function () {
415 playlistBucket: 'mybucket',
416 webtorrentBucket: 'mybucket',
418 playlistPrefix: 'streaming-playlists/',
419 webtorrentPrefix: 'webtorrent/',
425 describe('Test object storage with file bigger than upload part', function () {
427 const maxUploadPart = '5MB'
429 before(async function () {
432 fixture = await generateHighBitrateVideo()
434 const { size } = await stat(fixture)
436 if (bytes.parse(maxUploadPart) > size) {
437 throw Error(`Fixture file is too small (${size}) to make sense for this test.`)
443 playlistBucket: 'streaming-playlists',
444 webtorrentBucket: 'videos',