1 /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
3 import bytes from 'bytes'
4 import { expect } from 'chai'
5 import { stat } from 'fs-extra'
6 import { merge } from 'lodash'
9 expectLogDoesNotContain,
11 generateHighBitrateVideo,
12 MockObjectStorageProxy
13 } from '@server/tests/shared'
14 import { areMockObjectStorageTestsDisabled } from '@shared/core-utils'
15 import { HttpStatusCode, VideoDetails } from '@shared/models'
18 createMultipleServers,
25 setAccessTokensToServers,
28 } from '@shared/server-commands'
29 import { sha1 } from '@shared/extra-utils'
31 async function checkFiles (options: {
32 server: PeerTubeServer
33 originServer: PeerTubeServer
39 playlistBucket: string
40 playlistPrefix?: string
42 webtorrentBucket: string
43 webtorrentPrefix?: string
56 let allFiles = video.files
58 for (const file of video.files) {
59 const baseUrl = baseMockUrl
60 ? `${baseMockUrl}/${webtorrentBucket}/`
61 : `http://${webtorrentBucket}.${ObjectStorageCommand.getMockEndpointHost()}/`
63 const prefix = webtorrentPrefix || ''
64 const start = baseUrl + prefix
66 expectStartWith(file.fileUrl, start)
68 const res = await makeRawRequest({ url: file.fileDownloadUrl, expectedStatus: HttpStatusCode.FOUND_302 })
69 const location = res.headers['location']
70 expectStartWith(location, start)
72 await makeRawRequest({ url: location, expectedStatus: HttpStatusCode.OK_200 })
75 const hls = video.streamingPlaylists[0]
78 allFiles = allFiles.concat(hls.files)
80 const baseUrl = baseMockUrl
81 ? `${baseMockUrl}/${playlistBucket}/`
82 : `http://${playlistBucket}.${ObjectStorageCommand.getMockEndpointHost()}/`
84 const prefix = playlistPrefix || ''
85 const start = baseUrl + prefix
87 expectStartWith(hls.playlistUrl, start)
88 expectStartWith(hls.segmentsSha256Url, start)
90 await makeRawRequest({ url: hls.playlistUrl, expectedStatus: HttpStatusCode.OK_200 })
92 const resSha = await makeRawRequest({ url: hls.segmentsSha256Url, expectedStatus: HttpStatusCode.OK_200 })
93 expect(JSON.stringify(resSha.body)).to.not.throw
96 for (const file of hls.files) {
97 expectStartWith(file.fileUrl, start)
99 const res = await makeRawRequest({ url: file.fileDownloadUrl, expectedStatus: HttpStatusCode.FOUND_302 })
100 const location = res.headers['location']
101 expectStartWith(location, start)
103 await makeRawRequest({ url: location, expectedStatus: HttpStatusCode.OK_200 })
105 if (originServer.internalServerNumber === server.internalServerNumber) {
106 const infohash = sha1(`${2 + hls.playlistUrl}+V${i}`)
107 const dbInfohashes = await originServer.sql.getPlaylistInfohash(hls.id)
109 expect(dbInfohashes).to.include(infohash)
116 for (const file of allFiles) {
117 const torrent = await webtorrentAdd(file.magnetUri, true)
119 expect(torrent.files).to.be.an('array')
120 expect(torrent.files.length).to.equal(1)
121 expect(torrent.files[0].path).to.exist.and.to.not.equal('')
123 const res = await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 })
124 expect(res.body).to.have.length.above(100)
127 return allFiles.map(f => f.fileUrl)
130 function runTestSuite (options: {
133 maxUploadPart?: string
135 playlistBucket: string
136 playlistPrefix?: string
138 webtorrentBucket: string
139 webtorrentPrefix?: string
141 useMockBaseUrl?: boolean
143 const mockObjectStorageProxy = new MockObjectStorageProxy()
144 const { fixture } = options
145 let baseMockUrl: string
147 let servers: PeerTubeServer[]
149 let keptUrls: string[] = []
151 const uuidsToDelete: string[] = []
152 let deletedUrls: string[] = []
154 before(async function () {
157 const port = await mockObjectStorageProxy.initialize()
158 baseMockUrl = options.useMockBaseUrl
159 ? `http://127.0.0.1:${port}`
162 await ObjectStorageCommand.createMockBucket(options.playlistBucket)
163 await ObjectStorageCommand.createMockBucket(options.webtorrentBucket)
168 endpoint: 'http://' + ObjectStorageCommand.getMockEndpointHost(),
169 region: ObjectStorageCommand.getMockRegion(),
171 credentials: ObjectStorageCommand.getMockCredentialsConfig(),
173 max_upload_part: options.maxUploadPart || '5MB',
175 streaming_playlists: {
176 bucket_name: options.playlistBucket,
177 prefix: options.playlistPrefix,
178 base_url: baseMockUrl
179 ? `${baseMockUrl}/${options.playlistBucket}`
184 bucket_name: options.webtorrentBucket,
185 prefix: options.webtorrentPrefix,
186 base_url: baseMockUrl
187 ? `${baseMockUrl}/${options.webtorrentBucket}`
193 servers = await createMultipleServers(2, config)
195 await setAccessTokensToServers(servers)
196 await doubleFollow(servers[0], servers[1])
198 for (const server of servers) {
199 const { uuid } = await server.videos.quickUpload({ name: 'video to keep' })
200 await waitJobs(servers)
202 const files = await server.videos.listFiles({ id: uuid })
203 keptUrls = keptUrls.concat(files.map(f => f.fileUrl))
207 it('Should upload a video and move it to the object storage without transcoding', async function () {
210 const { uuid } = await servers[0].videos.quickUpload({ name: 'video 1', fixture })
211 uuidsToDelete.push(uuid)
213 await waitJobs(servers)
215 for (const server of servers) {
216 const video = await server.videos.get({ id: uuid })
217 const files = await checkFiles({ ...options, server, originServer: servers[0], video, baseMockUrl })
219 deletedUrls = deletedUrls.concat(files)
223 it('Should upload a video and move it to the object storage with transcoding', async function () {
226 const { uuid } = await servers[1].videos.quickUpload({ name: 'video 2', fixture })
227 uuidsToDelete.push(uuid)
229 await waitJobs(servers)
231 for (const server of servers) {
232 const video = await server.videos.get({ id: uuid })
233 const files = await checkFiles({ ...options, server, originServer: servers[0], video, baseMockUrl })
235 deletedUrls = deletedUrls.concat(files)
239 it('Should fetch correctly all the files', async function () {
240 for (const url of deletedUrls.concat(keptUrls)) {
241 await makeRawRequest({ url, expectedStatus: HttpStatusCode.OK_200 })
245 it('Should correctly delete the files', async function () {
246 await servers[0].videos.remove({ id: uuidsToDelete[0] })
247 await servers[1].videos.remove({ id: uuidsToDelete[1] })
249 await waitJobs(servers)
251 for (const url of deletedUrls) {
252 await makeRawRequest({ url, expectedStatus: HttpStatusCode.NOT_FOUND_404 })
256 it('Should have kept other files', async function () {
257 for (const url of keptUrls) {
258 await makeRawRequest({ url, expectedStatus: HttpStatusCode.OK_200 })
262 it('Should have an empty tmp directory', async function () {
263 for (const server of servers) {
264 await checkTmpIsEmpty(server)
268 it('Should not have downloaded files from object storage', async function () {
269 for (const server of servers) {
270 await expectLogDoesNotContain(server, 'from object storage')
274 after(async function () {
275 await mockObjectStorageProxy.terminate()
277 await cleanupTests(servers)
281 describe('Object storage for videos', function () {
282 if (areMockObjectStorageTestsDisabled()) return
284 describe('Test config', function () {
285 let server: PeerTubeServer
290 endpoint: 'http://' + ObjectStorageCommand.getMockEndpointHost(),
291 region: ObjectStorageCommand.getMockRegion(),
293 credentials: ObjectStorageCommand.getMockCredentialsConfig(),
295 streaming_playlists: {
296 bucket_name: ObjectStorageCommand.DEFAULT_PLAYLIST_MOCK_BUCKET
300 bucket_name: ObjectStorageCommand.DEFAULT_WEBTORRENT_MOCK_BUCKET
305 const badCredentials = {
306 access_key_id: 'AKIAIOSFODNN7EXAMPLE',
307 secret_access_key: 'aJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY'
310 it('Should fail with same bucket names without prefix', function (done) {
311 const config = merge({}, baseConfig, {
313 streaming_playlists: {
323 createSingleServer(1, config)
324 .then(() => done(new Error('Did not throw')))
328 it('Should fail with bad credentials', async function () {
331 await ObjectStorageCommand.prepareDefaultMockBuckets()
333 const config = merge({}, baseConfig, {
335 credentials: badCredentials
339 server = await createSingleServer(1, config)
340 await setAccessTokensToServers([ server ])
342 const { uuid } = await server.videos.quickUpload({ name: 'video' })
344 await waitJobs([ server ], { skipDelayed: true })
345 const video = await server.videos.get({ id: uuid })
347 expectStartWith(video.files[0].fileUrl, server.url)
349 await killallServers([ server ])
352 it('Should succeed with credentials from env', async function () {
355 await ObjectStorageCommand.prepareDefaultMockBuckets()
357 const config = merge({}, baseConfig, {
361 secret_access_key: ''
366 const goodCredentials = ObjectStorageCommand.getMockCredentialsConfig()
368 server = await createSingleServer(1, config, {
370 AWS_ACCESS_KEY_ID: goodCredentials.access_key_id,
371 AWS_SECRET_ACCESS_KEY: goodCredentials.secret_access_key
375 await setAccessTokensToServers([ server ])
377 const { uuid } = await server.videos.quickUpload({ name: 'video' })
379 await waitJobs([ server ], { skipDelayed: true })
380 const video = await server.videos.get({ id: uuid })
382 expectStartWith(video.files[0].fileUrl, ObjectStorageCommand.getMockWebTorrentBaseUrl())
385 after(async function () {
386 await killallServers([ server ])
390 describe('Test simple object storage', function () {
392 playlistBucket: 'streaming-playlists',
393 webtorrentBucket: 'videos'
397 describe('Test object storage with prefix', function () {
399 playlistBucket: 'mybucket',
400 webtorrentBucket: 'mybucket',
402 playlistPrefix: 'streaming-playlists_',
403 webtorrentPrefix: 'webtorrent_'
407 describe('Test object storage with prefix and base URL', function () {
409 playlistBucket: 'mybucket',
410 webtorrentBucket: 'mybucket',
412 playlistPrefix: 'streaming-playlists/',
413 webtorrentPrefix: 'webtorrent/',
419 describe('Test object storage with file bigger than upload part', function () {
421 const maxUploadPart = '5MB'
423 before(async function () {
426 fixture = await generateHighBitrateVideo()
428 const { size } = await stat(fixture)
430 if (bytes.parse(maxUploadPart) > size) {
431 throw Error(`Fixture file is too small (${size}) to make sense for this test.`)
437 playlistBucket: 'streaming-playlists',
438 webtorrentBucket: 'videos',