]>
Commit | Line | Data |
---|---|---|
0305db28 JB |
1 | /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ |
2 | ||
156cdbac | 3 | import bytes from 'bytes' |
86347717 | 4 | import { expect } from 'chai' |
156cdbac | 5 | import { stat } from 'fs-extra' |
0305db28 | 6 | import { merge } from 'lodash' |
156cdbac | 7 | import { |
8 | checkTmpIsEmpty, | |
9 | expectLogDoesNotContain, | |
10 | expectStartWith, | |
11 | generateHighBitrateVideo, | |
8059e050 | 12 | MockObjectStorageProxy |
156cdbac | 13 | } from '@server/tests/shared' |
9ab330b9 | 14 | import { areMockObjectStorageTestsDisabled } from '@shared/core-utils' |
c55e3d72 | 15 | import { HttpStatusCode, VideoDetails } from '@shared/models' |
0305db28 | 16 | import { |
0305db28 JB |
17 | cleanupTests, |
18 | createMultipleServers, | |
19 | createSingleServer, | |
20 | doubleFollow, | |
0305db28 JB |
21 | killallServers, |
22 | makeRawRequest, | |
0305db28 JB |
23 | ObjectStorageCommand, |
24 | PeerTubeServer, | |
25 | setAccessTokensToServers, | |
26 | waitJobs, | |
27 | webtorrentAdd | |
bf54587a | 28 | } from '@shared/server-commands' |
1c41b5c1 | 29 | import { sha1 } from '@shared/extra-utils' |
0305db28 | 30 | |
0305db28 | 31 | async function checkFiles (options: { |
1c41b5c1 C |
32 | server: PeerTubeServer |
33 | originServer: PeerTubeServer | |
34 | ||
0305db28 JB |
35 | video: VideoDetails |
36 | ||
37 | baseMockUrl?: string | |
38 | ||
39 | playlistBucket: string | |
40 | playlistPrefix?: string | |
41 | ||
42 | webtorrentBucket: string | |
43 | webtorrentPrefix?: string | |
44 | }) { | |
45 | const { | |
1c41b5c1 C |
46 | server, |
47 | originServer, | |
0305db28 JB |
48 | video, |
49 | playlistBucket, | |
50 | webtorrentBucket, | |
51 | baseMockUrl, | |
52 | playlistPrefix, | |
53 | webtorrentPrefix | |
54 | } = options | |
55 | ||
56 | let allFiles = video.files | |
57 | ||
58 | for (const file of video.files) { | |
59 | const baseUrl = baseMockUrl | |
60 | ? `${baseMockUrl}/${webtorrentBucket}/` | |
9ab330b9 | 61 | : `http://${webtorrentBucket}.${ObjectStorageCommand.getMockEndpointHost()}/` |
0305db28 JB |
62 | |
63 | const prefix = webtorrentPrefix || '' | |
64 | const start = baseUrl + prefix | |
65 | ||
66 | expectStartWith(file.fileUrl, start) | |
67 | ||
3545e72c | 68 | const res = await makeRawRequest({ url: file.fileDownloadUrl, expectedStatus: HttpStatusCode.FOUND_302 }) |
0305db28 JB |
69 | const location = res.headers['location'] |
70 | expectStartWith(location, start) | |
71 | ||
3545e72c | 72 | await makeRawRequest({ url: location, expectedStatus: HttpStatusCode.OK_200 }) |
0305db28 JB |
73 | } |
74 | ||
75 | const hls = video.streamingPlaylists[0] | |
76 | ||
77 | if (hls) { | |
78 | allFiles = allFiles.concat(hls.files) | |
79 | ||
80 | const baseUrl = baseMockUrl | |
81 | ? `${baseMockUrl}/${playlistBucket}/` | |
9ab330b9 | 82 | : `http://${playlistBucket}.${ObjectStorageCommand.getMockEndpointHost()}/` |
0305db28 JB |
83 | |
84 | const prefix = playlistPrefix || '' | |
85 | const start = baseUrl + prefix | |
86 | ||
87 | expectStartWith(hls.playlistUrl, start) | |
88 | expectStartWith(hls.segmentsSha256Url, start) | |
89 | ||
3545e72c | 90 | await makeRawRequest({ url: hls.playlistUrl, expectedStatus: HttpStatusCode.OK_200 }) |
0305db28 | 91 | |
3545e72c | 92 | const resSha = await makeRawRequest({ url: hls.segmentsSha256Url, expectedStatus: HttpStatusCode.OK_200 }) |
0305db28 JB |
93 | expect(JSON.stringify(resSha.body)).to.not.throw |
94 | ||
1c41b5c1 | 95 | let i = 0 |
0305db28 JB |
96 | for (const file of hls.files) { |
97 | expectStartWith(file.fileUrl, start) | |
98 | ||
3545e72c | 99 | const res = await makeRawRequest({ url: file.fileDownloadUrl, expectedStatus: HttpStatusCode.FOUND_302 }) |
0305db28 JB |
100 | const location = res.headers['location'] |
101 | expectStartWith(location, start) | |
102 | ||
3545e72c | 103 | await makeRawRequest({ url: location, expectedStatus: HttpStatusCode.OK_200 }) |
1c41b5c1 C |
104 | |
105 | if (originServer.internalServerNumber === server.internalServerNumber) { | |
106 | const infohash = sha1(`${2 + hls.playlistUrl}+V${i}`) | |
107 | const dbInfohashes = await originServer.sql.getPlaylistInfohash(hls.id) | |
108 | ||
109 | expect(dbInfohashes).to.include(infohash) | |
110 | } | |
111 | ||
112 | i++ | |
0305db28 JB |
113 | } |
114 | } | |
115 | ||
116 | for (const file of allFiles) { | |
117 | const torrent = await webtorrentAdd(file.magnetUri, true) | |
118 | ||
119 | expect(torrent.files).to.be.an('array') | |
120 | expect(torrent.files.length).to.equal(1) | |
121 | expect(torrent.files[0].path).to.exist.and.to.not.equal('') | |
122 | ||
3545e72c | 123 | const res = await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 }) |
0305db28 JB |
124 | expect(res.body).to.have.length.above(100) |
125 | } | |
126 | ||
127 | return allFiles.map(f => f.fileUrl) | |
128 | } | |
129 | ||
130 | function runTestSuite (options: { | |
156cdbac | 131 | fixture?: string |
132 | ||
133 | maxUploadPart?: string | |
134 | ||
0305db28 JB |
135 | playlistBucket: string |
136 | playlistPrefix?: string | |
137 | ||
138 | webtorrentBucket: string | |
139 | webtorrentPrefix?: string | |
140 | ||
141 | useMockBaseUrl?: boolean | |
0305db28 | 142 | }) { |
8059e050 | 143 | const mockObjectStorageProxy = new MockObjectStorageProxy() |
156cdbac | 144 | const { fixture } = options |
0305db28 JB |
145 | let baseMockUrl: string |
146 | ||
147 | let servers: PeerTubeServer[] | |
148 | ||
149 | let keptUrls: string[] = [] | |
150 | ||
151 | const uuidsToDelete: string[] = [] | |
152 | let deletedUrls: string[] = [] | |
153 | ||
154 | before(async function () { | |
155 | this.timeout(120000) | |
156 | ||
8059e050 C |
157 | const port = await mockObjectStorageProxy.initialize() |
158 | baseMockUrl = options.useMockBaseUrl | |
159 | ? `http://127.0.0.1:${port}` | |
160 | : undefined | |
0305db28 | 161 | |
9ab330b9 C |
162 | await ObjectStorageCommand.createMockBucket(options.playlistBucket) |
163 | await ObjectStorageCommand.createMockBucket(options.webtorrentBucket) | |
0305db28 JB |
164 | |
165 | const config = { | |
166 | object_storage: { | |
167 | enabled: true, | |
9ab330b9 C |
168 | endpoint: 'http://' + ObjectStorageCommand.getMockEndpointHost(), |
169 | region: ObjectStorageCommand.getMockRegion(), | |
0305db28 | 170 | |
9ab330b9 | 171 | credentials: ObjectStorageCommand.getMockCredentialsConfig(), |
0305db28 | 172 | |
156cdbac | 173 | max_upload_part: options.maxUploadPart || '5MB', |
0305db28 JB |
174 | |
175 | streaming_playlists: { | |
176 | bucket_name: options.playlistBucket, | |
177 | prefix: options.playlistPrefix, | |
178 | base_url: baseMockUrl | |
179 | ? `${baseMockUrl}/${options.playlistBucket}` | |
180 | : undefined | |
181 | }, | |
182 | ||
183 | videos: { | |
184 | bucket_name: options.webtorrentBucket, | |
185 | prefix: options.webtorrentPrefix, | |
186 | base_url: baseMockUrl | |
187 | ? `${baseMockUrl}/${options.webtorrentBucket}` | |
188 | : undefined | |
189 | } | |
190 | } | |
191 | } | |
192 | ||
193 | servers = await createMultipleServers(2, config) | |
194 | ||
195 | await setAccessTokensToServers(servers) | |
196 | await doubleFollow(servers[0], servers[1]) | |
197 | ||
198 | for (const server of servers) { | |
199 | const { uuid } = await server.videos.quickUpload({ name: 'video to keep' }) | |
200 | await waitJobs(servers) | |
201 | ||
202 | const files = await server.videos.listFiles({ id: uuid }) | |
203 | keptUrls = keptUrls.concat(files.map(f => f.fileUrl)) | |
204 | } | |
205 | }) | |
206 | ||
207 | it('Should upload a video and move it to the object storage without transcoding', async function () { | |
d1bfbdeb | 208 | this.timeout(40000) |
0305db28 | 209 | |
156cdbac | 210 | const { uuid } = await servers[0].videos.quickUpload({ name: 'video 1', fixture }) |
0305db28 JB |
211 | uuidsToDelete.push(uuid) |
212 | ||
213 | await waitJobs(servers) | |
214 | ||
215 | for (const server of servers) { | |
216 | const video = await server.videos.get({ id: uuid }) | |
1c41b5c1 | 217 | const files = await checkFiles({ ...options, server, originServer: servers[0], video, baseMockUrl }) |
0305db28 JB |
218 | |
219 | deletedUrls = deletedUrls.concat(files) | |
220 | } | |
221 | }) | |
222 | ||
223 | it('Should upload a video and move it to the object storage with transcoding', async function () { | |
1ffb5fb6 | 224 | this.timeout(120000) |
0305db28 | 225 | |
156cdbac | 226 | const { uuid } = await servers[1].videos.quickUpload({ name: 'video 2', fixture }) |
0305db28 JB |
227 | uuidsToDelete.push(uuid) |
228 | ||
229 | await waitJobs(servers) | |
230 | ||
231 | for (const server of servers) { | |
232 | const video = await server.videos.get({ id: uuid }) | |
1c41b5c1 | 233 | const files = await checkFiles({ ...options, server, originServer: servers[0], video, baseMockUrl }) |
0305db28 JB |
234 | |
235 | deletedUrls = deletedUrls.concat(files) | |
236 | } | |
237 | }) | |
238 | ||
87699a09 C |
239 | it('Should fetch correctly all the files', async function () { |
240 | for (const url of deletedUrls.concat(keptUrls)) { | |
3545e72c | 241 | await makeRawRequest({ url, expectedStatus: HttpStatusCode.OK_200 }) |
87699a09 C |
242 | } |
243 | }) | |
244 | ||
0305db28 JB |
245 | it('Should correctly delete the files', async function () { |
246 | await servers[0].videos.remove({ id: uuidsToDelete[0] }) | |
247 | await servers[1].videos.remove({ id: uuidsToDelete[1] }) | |
248 | ||
249 | await waitJobs(servers) | |
250 | ||
251 | for (const url of deletedUrls) { | |
3545e72c | 252 | await makeRawRequest({ url, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) |
0305db28 JB |
253 | } |
254 | }) | |
255 | ||
256 | it('Should have kept other files', async function () { | |
257 | for (const url of keptUrls) { | |
3545e72c | 258 | await makeRawRequest({ url, expectedStatus: HttpStatusCode.OK_200 }) |
0305db28 JB |
259 | } |
260 | }) | |
261 | ||
262 | it('Should have an empty tmp directory', async function () { | |
263 | for (const server of servers) { | |
264 | await checkTmpIsEmpty(server) | |
265 | } | |
266 | }) | |
267 | ||
1f6125be C |
268 | it('Should not have downloaded files from object storage', async function () { |
269 | for (const server of servers) { | |
270 | await expectLogDoesNotContain(server, 'from object storage') | |
271 | } | |
272 | }) | |
273 | ||
0305db28 | 274 | after(async function () { |
8059e050 | 275 | await mockObjectStorageProxy.terminate() |
0305db28 JB |
276 | |
277 | await cleanupTests(servers) | |
278 | }) | |
279 | } | |
280 | ||
281 | describe('Object storage for videos', function () { | |
9ab330b9 | 282 | if (areMockObjectStorageTestsDisabled()) return |
0305db28 JB |
283 | |
284 | describe('Test config', function () { | |
285 | let server: PeerTubeServer | |
286 | ||
287 | const baseConfig = { | |
288 | object_storage: { | |
289 | enabled: true, | |
9ab330b9 C |
290 | endpoint: 'http://' + ObjectStorageCommand.getMockEndpointHost(), |
291 | region: ObjectStorageCommand.getMockRegion(), | |
0305db28 | 292 | |
9ab330b9 | 293 | credentials: ObjectStorageCommand.getMockCredentialsConfig(), |
0305db28 JB |
294 | |
295 | streaming_playlists: { | |
9ab330b9 | 296 | bucket_name: ObjectStorageCommand.DEFAULT_PLAYLIST_MOCK_BUCKET |
0305db28 JB |
297 | }, |
298 | ||
299 | videos: { | |
9ab330b9 | 300 | bucket_name: ObjectStorageCommand.DEFAULT_WEBTORRENT_MOCK_BUCKET |
0305db28 JB |
301 | } |
302 | } | |
303 | } | |
304 | ||
305 | const badCredentials = { | |
306 | access_key_id: 'AKIAIOSFODNN7EXAMPLE', | |
307 | secret_access_key: 'aJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY' | |
308 | } | |
309 | ||
310 | it('Should fail with same bucket names without prefix', function (done) { | |
311 | const config = merge({}, baseConfig, { | |
312 | object_storage: { | |
313 | streaming_playlists: { | |
314 | bucket_name: 'aaa' | |
315 | }, | |
316 | ||
317 | videos: { | |
318 | bucket_name: 'aaa' | |
319 | } | |
320 | } | |
321 | }) | |
322 | ||
323 | createSingleServer(1, config) | |
324 | .then(() => done(new Error('Did not throw'))) | |
325 | .catch(() => done()) | |
326 | }) | |
327 | ||
328 | it('Should fail with bad credentials', async function () { | |
329 | this.timeout(60000) | |
330 | ||
9ab330b9 | 331 | await ObjectStorageCommand.prepareDefaultMockBuckets() |
0305db28 JB |
332 | |
333 | const config = merge({}, baseConfig, { | |
334 | object_storage: { | |
335 | credentials: badCredentials | |
336 | } | |
337 | }) | |
338 | ||
339 | server = await createSingleServer(1, config) | |
340 | await setAccessTokensToServers([ server ]) | |
341 | ||
342 | const { uuid } = await server.videos.quickUpload({ name: 'video' }) | |
343 | ||
e82cb087 | 344 | await waitJobs([ server ], { skipDelayed: true }) |
0305db28 JB |
345 | const video = await server.videos.get({ id: uuid }) |
346 | ||
347 | expectStartWith(video.files[0].fileUrl, server.url) | |
348 | ||
349 | await killallServers([ server ]) | |
350 | }) | |
351 | ||
352 | it('Should succeed with credentials from env', async function () { | |
353 | this.timeout(60000) | |
354 | ||
9ab330b9 | 355 | await ObjectStorageCommand.prepareDefaultMockBuckets() |
0305db28 JB |
356 | |
357 | const config = merge({}, baseConfig, { | |
358 | object_storage: { | |
359 | credentials: { | |
360 | access_key_id: '', | |
361 | secret_access_key: '' | |
362 | } | |
363 | } | |
364 | }) | |
365 | ||
9ab330b9 | 366 | const goodCredentials = ObjectStorageCommand.getMockCredentialsConfig() |
0305db28 JB |
367 | |
368 | server = await createSingleServer(1, config, { | |
369 | env: { | |
370 | AWS_ACCESS_KEY_ID: goodCredentials.access_key_id, | |
371 | AWS_SECRET_ACCESS_KEY: goodCredentials.secret_access_key | |
372 | } | |
373 | }) | |
374 | ||
375 | await setAccessTokensToServers([ server ]) | |
376 | ||
377 | const { uuid } = await server.videos.quickUpload({ name: 'video' }) | |
378 | ||
e82cb087 | 379 | await waitJobs([ server ], { skipDelayed: true }) |
0305db28 JB |
380 | const video = await server.videos.get({ id: uuid }) |
381 | ||
9ab330b9 | 382 | expectStartWith(video.files[0].fileUrl, ObjectStorageCommand.getMockWebTorrentBaseUrl()) |
0305db28 JB |
383 | }) |
384 | ||
385 | after(async function () { | |
386 | await killallServers([ server ]) | |
387 | }) | |
388 | }) | |
389 | ||
390 | describe('Test simple object storage', function () { | |
391 | runTestSuite({ | |
392 | playlistBucket: 'streaming-playlists', | |
393 | webtorrentBucket: 'videos' | |
394 | }) | |
395 | }) | |
396 | ||
397 | describe('Test object storage with prefix', function () { | |
398 | runTestSuite({ | |
399 | playlistBucket: 'mybucket', | |
400 | webtorrentBucket: 'mybucket', | |
401 | ||
402 | playlistPrefix: 'streaming-playlists_', | |
403 | webtorrentPrefix: 'webtorrent_' | |
404 | }) | |
405 | }) | |
406 | ||
407 | describe('Test object storage with prefix and base URL', function () { | |
408 | runTestSuite({ | |
409 | playlistBucket: 'mybucket', | |
410 | webtorrentBucket: 'mybucket', | |
411 | ||
70430c27 C |
412 | playlistPrefix: 'streaming-playlists/', |
413 | webtorrentPrefix: 'webtorrent/', | |
0305db28 JB |
414 | |
415 | useMockBaseUrl: true | |
416 | }) | |
417 | }) | |
418 | ||
156cdbac | 419 | describe('Test object storage with file bigger than upload part', function () { |
420 | let fixture: string | |
421 | const maxUploadPart = '5MB' | |
422 | ||
423 | before(async function () { | |
cf0dd18a C |
424 | this.timeout(120000) |
425 | ||
156cdbac | 426 | fixture = await generateHighBitrateVideo() |
427 | ||
428 | const { size } = await stat(fixture) | |
429 | ||
430 | if (bytes.parse(maxUploadPart) > size) { | |
431 | throw Error(`Fixture file is too small (${size}) to make sense for this test.`) | |
432 | } | |
433 | }) | |
434 | ||
0305db28 | 435 | runTestSuite({ |
156cdbac | 436 | maxUploadPart, |
0305db28 JB |
437 | playlistBucket: 'streaming-playlists', |
438 | webtorrentBucket: 'videos', | |
156cdbac | 439 | fixture |
0305db28 JB |
440 | }) |
441 | }) | |
442 | }) |