1 /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
4 import * as chai from 'chai'
5 import { merge } from 'lodash'
7 areObjectStorageTestsDisabled,
10 createMultipleServers,
13 expectLogDoesNotContain,
20 setAccessTokensToServers,
23 } from '@shared/extra-utils'
24 import { HttpStatusCode, VideoDetails } from '@shared/models'
26 const expect = chai.expect
28 async function checkFiles (options: {
33 playlistBucket: string
34 playlistPrefix?: string
36 webtorrentBucket: string
37 webtorrentPrefix?: string
48 let allFiles = video.files
50 for (const file of video.files) {
51 const baseUrl = baseMockUrl
52 ? `${baseMockUrl}/${webtorrentBucket}/`
53 : `http://${webtorrentBucket}.${ObjectStorageCommand.getEndpointHost()}/`
55 const prefix = webtorrentPrefix || ''
56 const start = baseUrl + prefix
58 expectStartWith(file.fileUrl, start)
60 const res = await makeRawRequest(file.fileDownloadUrl, HttpStatusCode.FOUND_302)
61 const location = res.headers['location']
62 expectStartWith(location, start)
64 await makeRawRequest(location, HttpStatusCode.OK_200)
67 const hls = video.streamingPlaylists[0]
70 allFiles = allFiles.concat(hls.files)
72 const baseUrl = baseMockUrl
73 ? `${baseMockUrl}/${playlistBucket}/`
74 : `http://${playlistBucket}.${ObjectStorageCommand.getEndpointHost()}/`
76 const prefix = playlistPrefix || ''
77 const start = baseUrl + prefix
79 expectStartWith(hls.playlistUrl, start)
80 expectStartWith(hls.segmentsSha256Url, start)
82 await makeRawRequest(hls.playlistUrl, HttpStatusCode.OK_200)
84 const resSha = await makeRawRequest(hls.segmentsSha256Url, HttpStatusCode.OK_200)
85 expect(JSON.stringify(resSha.body)).to.not.throw
87 for (const file of hls.files) {
88 expectStartWith(file.fileUrl, start)
90 const res = await makeRawRequest(file.fileDownloadUrl, HttpStatusCode.FOUND_302)
91 const location = res.headers['location']
92 expectStartWith(location, start)
94 await makeRawRequest(location, HttpStatusCode.OK_200)
98 for (const file of allFiles) {
99 const torrent = await webtorrentAdd(file.magnetUri, true)
101 expect(torrent.files).to.be.an('array')
102 expect(torrent.files.length).to.equal(1)
103 expect(torrent.files[0].path).to.exist.and.to.not.equal('')
105 const res = await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200)
106 expect(res.body).to.have.length.above(100)
109 return allFiles.map(f => f.fileUrl)
112 function runTestSuite (options: {
113 playlistBucket: string
114 playlistPrefix?: string
116 webtorrentBucket: string
117 webtorrentPrefix?: string
119 useMockBaseUrl?: boolean
121 maxUploadPart?: string
123 const mockObjectStorage = new MockObjectStorage()
124 let baseMockUrl: string
126 let servers: PeerTubeServer[]
128 let keptUrls: string[] = []
130 const uuidsToDelete: string[] = []
131 let deletedUrls: string[] = []
133 before(async function () {
136 const port = await mockObjectStorage.initialize()
137 baseMockUrl = options.useMockBaseUrl ? `http://localhost:${port}` : undefined
139 await ObjectStorageCommand.createBucket(options.playlistBucket)
140 await ObjectStorageCommand.createBucket(options.webtorrentBucket)
145 endpoint: 'http://' + ObjectStorageCommand.getEndpointHost(),
146 region: ObjectStorageCommand.getRegion(),
148 credentials: ObjectStorageCommand.getCredentialsConfig(),
150 max_upload_part: options.maxUploadPart || '2MB',
152 streaming_playlists: {
153 bucket_name: options.playlistBucket,
154 prefix: options.playlistPrefix,
155 base_url: baseMockUrl
156 ? `${baseMockUrl}/${options.playlistBucket}`
161 bucket_name: options.webtorrentBucket,
162 prefix: options.webtorrentPrefix,
163 base_url: baseMockUrl
164 ? `${baseMockUrl}/${options.webtorrentBucket}`
170 servers = await createMultipleServers(2, config)
172 await setAccessTokensToServers(servers)
173 await doubleFollow(servers[0], servers[1])
175 for (const server of servers) {
176 const { uuid } = await server.videos.quickUpload({ name: 'video to keep' })
177 await waitJobs(servers)
179 const files = await server.videos.listFiles({ id: uuid })
180 keptUrls = keptUrls.concat(files.map(f => f.fileUrl))
184 it('Should upload a video and move it to the object storage without transcoding', async function () {
187 const { uuid } = await servers[0].videos.quickUpload({ name: 'video 1' })
188 uuidsToDelete.push(uuid)
190 await waitJobs(servers)
192 for (const server of servers) {
193 const video = await server.videos.get({ id: uuid })
194 const files = await checkFiles({ ...options, video, baseMockUrl })
196 deletedUrls = deletedUrls.concat(files)
200 it('Should upload a video and move it to the object storage with transcoding', async function () {
203 const { uuid } = await servers[1].videos.quickUpload({ name: 'video 2' })
204 uuidsToDelete.push(uuid)
206 await waitJobs(servers)
208 for (const server of servers) {
209 const video = await server.videos.get({ id: uuid })
210 const files = await checkFiles({ ...options, video, baseMockUrl })
212 deletedUrls = deletedUrls.concat(files)
216 it('Should correctly delete the files', async function () {
217 await servers[0].videos.remove({ id: uuidsToDelete[0] })
218 await servers[1].videos.remove({ id: uuidsToDelete[1] })
220 await waitJobs(servers)
222 for (const url of deletedUrls) {
223 await makeRawRequest(url, HttpStatusCode.NOT_FOUND_404)
227 it('Should have kept other files', async function () {
228 for (const url of keptUrls) {
229 await makeRawRequest(url, HttpStatusCode.OK_200)
233 it('Should have an empty tmp directory', async function () {
234 for (const server of servers) {
235 await checkTmpIsEmpty(server)
239 it('Should not have downloaded files from object storage', async function () {
240 for (const server of servers) {
241 await expectLogDoesNotContain(server, 'from object storage')
245 after(async function () {
246 mockObjectStorage.terminate()
248 await cleanupTests(servers)
252 describe('Object storage for videos', function () {
253 if (areObjectStorageTestsDisabled()) return
255 describe('Test config', function () {
256 let server: PeerTubeServer
261 endpoint: 'http://' + ObjectStorageCommand.getEndpointHost(),
262 region: ObjectStorageCommand.getRegion(),
264 credentials: ObjectStorageCommand.getCredentialsConfig(),
266 streaming_playlists: {
267 bucket_name: ObjectStorageCommand.DEFAULT_PLAYLIST_BUCKET
271 bucket_name: ObjectStorageCommand.DEFAULT_WEBTORRENT_BUCKET
276 const badCredentials = {
277 access_key_id: 'AKIAIOSFODNN7EXAMPLE',
278 secret_access_key: 'aJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY'
281 it('Should fail with same bucket names without prefix', function (done) {
282 const config = merge({}, baseConfig, {
284 streaming_playlists: {
294 createSingleServer(1, config)
295 .then(() => done(new Error('Did not throw')))
299 it('Should fail with bad credentials', async function () {
302 await ObjectStorageCommand.prepareDefaultBuckets()
304 const config = merge({}, baseConfig, {
306 credentials: badCredentials
310 server = await createSingleServer(1, config)
311 await setAccessTokensToServers([ server ])
313 const { uuid } = await server.videos.quickUpload({ name: 'video' })
315 await waitJobs([ server ], true)
316 const video = await server.videos.get({ id: uuid })
318 expectStartWith(video.files[0].fileUrl, server.url)
320 await killallServers([ server ])
323 it('Should succeed with credentials from env', async function () {
326 await ObjectStorageCommand.prepareDefaultBuckets()
328 const config = merge({}, baseConfig, {
332 secret_access_key: ''
337 const goodCredentials = ObjectStorageCommand.getCredentialsConfig()
339 server = await createSingleServer(1, config, {
341 AWS_ACCESS_KEY_ID: goodCredentials.access_key_id,
342 AWS_SECRET_ACCESS_KEY: goodCredentials.secret_access_key
346 await setAccessTokensToServers([ server ])
348 const { uuid } = await server.videos.quickUpload({ name: 'video' })
350 await waitJobs([ server ], true)
351 const video = await server.videos.get({ id: uuid })
353 expectStartWith(video.files[0].fileUrl, ObjectStorageCommand.getWebTorrentBaseUrl())
356 after(async function () {
357 await killallServers([ server ])
361 describe('Test simple object storage', function () {
363 playlistBucket: 'streaming-playlists',
364 webtorrentBucket: 'videos'
368 describe('Test object storage with prefix', function () {
370 playlistBucket: 'mybucket',
371 webtorrentBucket: 'mybucket',
373 playlistPrefix: 'streaming-playlists_',
374 webtorrentPrefix: 'webtorrent_'
378 describe('Test object storage with prefix and base URL', function () {
380 playlistBucket: 'mybucket',
381 webtorrentBucket: 'mybucket',
383 playlistPrefix: 'streaming-playlists_',
384 webtorrentPrefix: 'webtorrent_',
390 describe('Test object storage with small upload part', function () {
392 playlistBucket: 'streaming-playlists',
393 webtorrentBucket: 'videos',