1 /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
4 import * as chai from 'chai'
5 import { merge } from 'lodash'
6 import { checkTmpIsEmpty, expectLogDoesNotContain, expectStartWith, MockObjectStorage } from '@server/tests/shared'
7 import { areObjectStorageTestsDisabled } from '@shared/core-utils'
8 import { HttpStatusCode, VideoDetails } from '@shared/models'
11 createMultipleServers,
18 setAccessTokensToServers,
21 } from '@shared/server-commands'
23 const expect = chai.expect
25 async function checkFiles (options: {
30 playlistBucket: string
31 playlistPrefix?: string
33 webtorrentBucket: string
34 webtorrentPrefix?: string
45 let allFiles = video.files
47 for (const file of video.files) {
48 const baseUrl = baseMockUrl
49 ? `${baseMockUrl}/${webtorrentBucket}/`
50 : `http://${webtorrentBucket}.${ObjectStorageCommand.getEndpointHost()}/`
52 const prefix = webtorrentPrefix || ''
53 const start = baseUrl + prefix
55 expectStartWith(file.fileUrl, start)
57 const res = await makeRawRequest(file.fileDownloadUrl, HttpStatusCode.FOUND_302)
58 const location = res.headers['location']
59 expectStartWith(location, start)
61 await makeRawRequest(location, HttpStatusCode.OK_200)
64 const hls = video.streamingPlaylists[0]
67 allFiles = allFiles.concat(hls.files)
69 const baseUrl = baseMockUrl
70 ? `${baseMockUrl}/${playlistBucket}/`
71 : `http://${playlistBucket}.${ObjectStorageCommand.getEndpointHost()}/`
73 const prefix = playlistPrefix || ''
74 const start = baseUrl + prefix
76 expectStartWith(hls.playlistUrl, start)
77 expectStartWith(hls.segmentsSha256Url, start)
79 await makeRawRequest(hls.playlistUrl, HttpStatusCode.OK_200)
81 const resSha = await makeRawRequest(hls.segmentsSha256Url, HttpStatusCode.OK_200)
82 expect(JSON.stringify(resSha.body)).to.not.throw
84 for (const file of hls.files) {
85 expectStartWith(file.fileUrl, start)
87 const res = await makeRawRequest(file.fileDownloadUrl, HttpStatusCode.FOUND_302)
88 const location = res.headers['location']
89 expectStartWith(location, start)
91 await makeRawRequest(location, HttpStatusCode.OK_200)
95 for (const file of allFiles) {
96 const torrent = await webtorrentAdd(file.magnetUri, true)
98 expect(torrent.files).to.be.an('array')
99 expect(torrent.files.length).to.equal(1)
100 expect(torrent.files[0].path).to.exist.and.to.not.equal('')
102 const res = await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200)
103 expect(res.body).to.have.length.above(100)
106 return allFiles.map(f => f.fileUrl)
109 function runTestSuite (options: {
110 playlistBucket: string
111 playlistPrefix?: string
113 webtorrentBucket: string
114 webtorrentPrefix?: string
116 useMockBaseUrl?: boolean
118 maxUploadPart?: string
120 const mockObjectStorage = new MockObjectStorage()
121 let baseMockUrl: string
123 let servers: PeerTubeServer[]
125 let keptUrls: string[] = []
127 const uuidsToDelete: string[] = []
128 let deletedUrls: string[] = []
130 before(async function () {
133 const port = await mockObjectStorage.initialize()
134 baseMockUrl = options.useMockBaseUrl ? `http://localhost:${port}` : undefined
136 await ObjectStorageCommand.createBucket(options.playlistBucket)
137 await ObjectStorageCommand.createBucket(options.webtorrentBucket)
142 endpoint: 'http://' + ObjectStorageCommand.getEndpointHost(),
143 region: ObjectStorageCommand.getRegion(),
145 credentials: ObjectStorageCommand.getCredentialsConfig(),
147 max_upload_part: options.maxUploadPart || '2MB',
149 streaming_playlists: {
150 bucket_name: options.playlistBucket,
151 prefix: options.playlistPrefix,
152 base_url: baseMockUrl
153 ? `${baseMockUrl}/${options.playlistBucket}`
158 bucket_name: options.webtorrentBucket,
159 prefix: options.webtorrentPrefix,
160 base_url: baseMockUrl
161 ? `${baseMockUrl}/${options.webtorrentBucket}`
167 servers = await createMultipleServers(2, config)
169 await setAccessTokensToServers(servers)
170 await doubleFollow(servers[0], servers[1])
172 for (const server of servers) {
173 const { uuid } = await server.videos.quickUpload({ name: 'video to keep' })
174 await waitJobs(servers)
176 const files = await server.videos.listFiles({ id: uuid })
177 keptUrls = keptUrls.concat(files.map(f => f.fileUrl))
181 it('Should upload a video and move it to the object storage without transcoding', async function () {
184 const { uuid } = await servers[0].videos.quickUpload({ name: 'video 1' })
185 uuidsToDelete.push(uuid)
187 await waitJobs(servers)
189 for (const server of servers) {
190 const video = await server.videos.get({ id: uuid })
191 const files = await checkFiles({ ...options, video, baseMockUrl })
193 deletedUrls = deletedUrls.concat(files)
197 it('Should upload a video and move it to the object storage with transcoding', async function () {
200 const { uuid } = await servers[1].videos.quickUpload({ name: 'video 2' })
201 uuidsToDelete.push(uuid)
203 await waitJobs(servers)
205 for (const server of servers) {
206 const video = await server.videos.get({ id: uuid })
207 const files = await checkFiles({ ...options, video, baseMockUrl })
209 deletedUrls = deletedUrls.concat(files)
213 it('Should fetch correctly all the files', async function () {
214 for (const url of deletedUrls.concat(keptUrls)) {
215 await makeRawRequest(url, HttpStatusCode.OK_200)
219 it('Should correctly delete the files', async function () {
220 await servers[0].videos.remove({ id: uuidsToDelete[0] })
221 await servers[1].videos.remove({ id: uuidsToDelete[1] })
223 await waitJobs(servers)
225 for (const url of deletedUrls) {
226 await makeRawRequest(url, HttpStatusCode.NOT_FOUND_404)
230 it('Should have kept other files', async function () {
231 for (const url of keptUrls) {
232 await makeRawRequest(url, HttpStatusCode.OK_200)
236 it('Should have an empty tmp directory', async function () {
237 for (const server of servers) {
238 await checkTmpIsEmpty(server)
242 it('Should not have downloaded files from object storage', async function () {
243 for (const server of servers) {
244 await expectLogDoesNotContain(server, 'from object storage')
248 after(async function () {
249 await mockObjectStorage.terminate()
251 await cleanupTests(servers)
255 describe('Object storage for videos', function () {
256 if (areObjectStorageTestsDisabled()) return
258 describe('Test config', function () {
259 let server: PeerTubeServer
264 endpoint: 'http://' + ObjectStorageCommand.getEndpointHost(),
265 region: ObjectStorageCommand.getRegion(),
267 credentials: ObjectStorageCommand.getCredentialsConfig(),
269 streaming_playlists: {
270 bucket_name: ObjectStorageCommand.DEFAULT_PLAYLIST_BUCKET
274 bucket_name: ObjectStorageCommand.DEFAULT_WEBTORRENT_BUCKET
279 const badCredentials = {
280 access_key_id: 'AKIAIOSFODNN7EXAMPLE',
281 secret_access_key: 'aJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY'
284 it('Should fail with same bucket names without prefix', function (done) {
285 const config = merge({}, baseConfig, {
287 streaming_playlists: {
297 createSingleServer(1, config)
298 .then(() => done(new Error('Did not throw')))
302 it('Should fail with bad credentials', async function () {
305 await ObjectStorageCommand.prepareDefaultBuckets()
307 const config = merge({}, baseConfig, {
309 credentials: badCredentials
313 server = await createSingleServer(1, config)
314 await setAccessTokensToServers([ server ])
316 const { uuid } = await server.videos.quickUpload({ name: 'video' })
318 await waitJobs([ server ], true)
319 const video = await server.videos.get({ id: uuid })
321 expectStartWith(video.files[0].fileUrl, server.url)
323 await killallServers([ server ])
326 it('Should succeed with credentials from env', async function () {
329 await ObjectStorageCommand.prepareDefaultBuckets()
331 const config = merge({}, baseConfig, {
335 secret_access_key: ''
340 const goodCredentials = ObjectStorageCommand.getCredentialsConfig()
342 server = await createSingleServer(1, config, {
344 AWS_ACCESS_KEY_ID: goodCredentials.access_key_id,
345 AWS_SECRET_ACCESS_KEY: goodCredentials.secret_access_key
349 await setAccessTokensToServers([ server ])
351 const { uuid } = await server.videos.quickUpload({ name: 'video' })
353 await waitJobs([ server ], true)
354 const video = await server.videos.get({ id: uuid })
356 expectStartWith(video.files[0].fileUrl, ObjectStorageCommand.getWebTorrentBaseUrl())
359 after(async function () {
360 await killallServers([ server ])
364 describe('Test simple object storage', function () {
366 playlistBucket: 'streaming-playlists',
367 webtorrentBucket: 'videos'
371 describe('Test object storage with prefix', function () {
373 playlistBucket: 'mybucket',
374 webtorrentBucket: 'mybucket',
376 playlistPrefix: 'streaming-playlists_',
377 webtorrentPrefix: 'webtorrent_'
381 describe('Test object storage with prefix and base URL', function () {
383 playlistBucket: 'mybucket',
384 webtorrentBucket: 'mybucket',
386 playlistPrefix: 'streaming-playlists/',
387 webtorrentPrefix: 'webtorrent/',
393 describe('Test object storage with small upload part', function () {
395 playlistBucket: 'streaming-playlists',
396 webtorrentBucket: 'videos',