1 /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
4 import * as chai from 'chai'
5 import { merge } from 'lodash'
7 areObjectStorageTestsDisabled,
10 createMultipleServers,
19 setAccessTokensToServers,
22 } from '@shared/extra-utils'
23 import { HttpStatusCode, VideoDetails } from '@shared/models'
25 const expect = chai.expect
27 async function checkFiles (options: {
32 playlistBucket: string
33 playlistPrefix?: string
35 webtorrentBucket: string
36 webtorrentPrefix?: string
47 let allFiles = video.files
49 for (const file of video.files) {
50 const baseUrl = baseMockUrl
51 ? `${baseMockUrl}/${webtorrentBucket}/`
52 : `http://${webtorrentBucket}.${ObjectStorageCommand.getEndpointHost()}/`
54 const prefix = webtorrentPrefix || ''
55 const start = baseUrl + prefix
57 expectStartWith(file.fileUrl, start)
59 const res = await makeRawRequest(file.fileDownloadUrl, HttpStatusCode.FOUND_302)
60 const location = res.headers['location']
61 expectStartWith(location, start)
63 await makeRawRequest(location, HttpStatusCode.OK_200)
66 const hls = video.streamingPlaylists[0]
69 allFiles = allFiles.concat(hls.files)
71 const baseUrl = baseMockUrl
72 ? `${baseMockUrl}/${playlistBucket}/`
73 : `http://${playlistBucket}.${ObjectStorageCommand.getEndpointHost()}/`
75 const prefix = playlistPrefix || ''
76 const start = baseUrl + prefix
78 expectStartWith(hls.playlistUrl, start)
79 expectStartWith(hls.segmentsSha256Url, start)
81 await makeRawRequest(hls.playlistUrl, HttpStatusCode.OK_200)
83 const resSha = await makeRawRequest(hls.segmentsSha256Url, HttpStatusCode.OK_200)
84 expect(JSON.stringify(resSha.body)).to.not.throw
86 for (const file of hls.files) {
87 expectStartWith(file.fileUrl, start)
89 const res = await makeRawRequest(file.fileDownloadUrl, HttpStatusCode.FOUND_302)
90 const location = res.headers['location']
91 expectStartWith(location, start)
93 await makeRawRequest(location, HttpStatusCode.OK_200)
97 for (const file of allFiles) {
98 const torrent = await webtorrentAdd(file.magnetUri, true)
100 expect(torrent.files).to.be.an('array')
101 expect(torrent.files.length).to.equal(1)
102 expect(torrent.files[0].path).to.exist.and.to.not.equal('')
104 const res = await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200)
105 expect(res.body).to.have.length.above(100)
108 return allFiles.map(f => f.fileUrl)
111 function runTestSuite (options: {
112 playlistBucket: string
113 playlistPrefix?: string
115 webtorrentBucket: string
116 webtorrentPrefix?: string
118 useMockBaseUrl?: boolean
120 maxUploadPart?: string
122 const mockObjectStorage = new MockObjectStorage()
123 let baseMockUrl: string
125 let servers: PeerTubeServer[]
127 let keptUrls: string[] = []
129 const uuidsToDelete: string[] = []
130 let deletedUrls: string[] = []
132 before(async function () {
135 const port = await mockObjectStorage.initialize()
136 baseMockUrl = options.useMockBaseUrl ? `http://localhost:${port}` : undefined
138 await ObjectStorageCommand.createBucket(options.playlistBucket)
139 await ObjectStorageCommand.createBucket(options.webtorrentBucket)
144 endpoint: 'http://' + ObjectStorageCommand.getEndpointHost(),
145 region: ObjectStorageCommand.getRegion(),
147 credentials: ObjectStorageCommand.getCredentialsConfig(),
149 max_upload_part: options.maxUploadPart || '2MB',
151 streaming_playlists: {
152 bucket_name: options.playlistBucket,
153 prefix: options.playlistPrefix,
154 base_url: baseMockUrl
155 ? `${baseMockUrl}/${options.playlistBucket}`
160 bucket_name: options.webtorrentBucket,
161 prefix: options.webtorrentPrefix,
162 base_url: baseMockUrl
163 ? `${baseMockUrl}/${options.webtorrentBucket}`
169 servers = await createMultipleServers(2, config)
171 await setAccessTokensToServers(servers)
172 await doubleFollow(servers[0], servers[1])
174 for (const server of servers) {
175 const { uuid } = await server.videos.quickUpload({ name: 'video to keep' })
176 await waitJobs(servers)
178 const files = await server.videos.listFiles({ id: uuid })
179 keptUrls = keptUrls.concat(files.map(f => f.fileUrl))
183 it('Should upload a video and move it to the object storage without transcoding', async function () {
186 const { uuid } = await servers[0].videos.quickUpload({ name: 'video 1' })
187 uuidsToDelete.push(uuid)
189 await waitJobs(servers)
191 for (const server of servers) {
192 const video = await server.videos.get({ id: uuid })
193 const files = await checkFiles({ ...options, video, baseMockUrl })
195 deletedUrls = deletedUrls.concat(files)
199 it('Should upload a video and move it to the object storage with transcoding', async function () {
202 const { uuid } = await servers[1].videos.quickUpload({ name: 'video 2' })
203 uuidsToDelete.push(uuid)
205 await waitJobs(servers)
207 for (const server of servers) {
208 const video = await server.videos.get({ id: uuid })
209 const files = await checkFiles({ ...options, video, baseMockUrl })
211 deletedUrls = deletedUrls.concat(files)
215 it('Should correctly delete the files', async function () {
216 await servers[0].videos.remove({ id: uuidsToDelete[0] })
217 await servers[1].videos.remove({ id: uuidsToDelete[1] })
219 await waitJobs(servers)
221 for (const url of deletedUrls) {
222 await makeRawRequest(url, HttpStatusCode.NOT_FOUND_404)
226 it('Should have kept other files', async function () {
227 for (const url of keptUrls) {
228 await makeRawRequest(url, HttpStatusCode.OK_200)
232 it('Should have an empty tmp directory', async function () {
233 for (const server of servers) {
234 await checkTmpIsEmpty(server)
238 after(async function () {
239 mockObjectStorage.terminate()
241 await cleanupTests(servers)
245 describe('Object storage for videos', function () {
246 if (areObjectStorageTestsDisabled()) return
248 describe('Test config', function () {
249 let server: PeerTubeServer
254 endpoint: 'http://' + ObjectStorageCommand.getEndpointHost(),
255 region: ObjectStorageCommand.getRegion(),
257 credentials: ObjectStorageCommand.getCredentialsConfig(),
259 streaming_playlists: {
260 bucket_name: ObjectStorageCommand.DEFAULT_PLAYLIST_BUCKET
264 bucket_name: ObjectStorageCommand.DEFAULT_WEBTORRENT_BUCKET
269 const badCredentials = {
270 access_key_id: 'AKIAIOSFODNN7EXAMPLE',
271 secret_access_key: 'aJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY'
274 it('Should fail with same bucket names without prefix', function (done) {
275 const config = merge({}, baseConfig, {
277 streaming_playlists: {
287 createSingleServer(1, config)
288 .then(() => done(new Error('Did not throw')))
292 it('Should fail with bad credentials', async function () {
295 await ObjectStorageCommand.prepareDefaultBuckets()
297 const config = merge({}, baseConfig, {
299 credentials: badCredentials
303 server = await createSingleServer(1, config)
304 await setAccessTokensToServers([ server ])
306 const { uuid } = await server.videos.quickUpload({ name: 'video' })
308 await waitJobs([ server ], true)
309 const video = await server.videos.get({ id: uuid })
311 expectStartWith(video.files[0].fileUrl, server.url)
313 await killallServers([ server ])
316 it('Should succeed with credentials from env', async function () {
319 await ObjectStorageCommand.prepareDefaultBuckets()
321 const config = merge({}, baseConfig, {
325 secret_access_key: ''
330 const goodCredentials = ObjectStorageCommand.getCredentialsConfig()
332 server = await createSingleServer(1, config, {
334 AWS_ACCESS_KEY_ID: goodCredentials.access_key_id,
335 AWS_SECRET_ACCESS_KEY: goodCredentials.secret_access_key
339 await setAccessTokensToServers([ server ])
341 const { uuid } = await server.videos.quickUpload({ name: 'video' })
343 await waitJobs([ server ], true)
344 const video = await server.videos.get({ id: uuid })
346 expectStartWith(video.files[0].fileUrl, ObjectStorageCommand.getWebTorrentBaseUrl())
349 after(async function () {
350 await killallServers([ server ])
354 describe('Test simple object storage', function () {
356 playlistBucket: 'streaming-playlists',
357 webtorrentBucket: 'videos'
361 describe('Test object storage with prefix', function () {
363 playlistBucket: 'mybucket',
364 webtorrentBucket: 'mybucket',
366 playlistPrefix: 'streaming-playlists_',
367 webtorrentPrefix: 'webtorrent_'
371 describe('Test object storage with prefix and base URL', function () {
373 playlistBucket: 'mybucket',
374 webtorrentBucket: 'mybucket',
376 playlistPrefix: 'streaming-playlists_',
377 webtorrentPrefix: 'webtorrent_',
383 describe('Test object storage with small upload part', function () {
385 playlistBucket: 'streaming-playlists',
386 webtorrentBucket: 'videos',