]>
Commit | Line | Data |
---|---|---|
0305db28 JB |
1 | /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ |
2 | ||
156cdbac | 3 | import bytes from 'bytes' |
86347717 | 4 | import { expect } from 'chai' |
156cdbac | 5 | import { stat } from 'fs-extra' |
0305db28 | 6 | import { merge } from 'lodash' |
156cdbac | 7 | import { |
8 | checkTmpIsEmpty, | |
d102de1b | 9 | checkWebTorrentWorks, |
156cdbac | 10 | expectLogDoesNotContain, |
11 | expectStartWith, | |
12 | generateHighBitrateVideo, | |
d102de1b C |
13 | MockObjectStorageProxy, |
14 | SQLCommand | |
156cdbac | 15 | } from '@server/tests/shared' |
9ab330b9 | 16 | import { areMockObjectStorageTestsDisabled } from '@shared/core-utils' |
d102de1b | 17 | import { sha1 } from '@shared/extra-utils' |
c55e3d72 | 18 | import { HttpStatusCode, VideoDetails } from '@shared/models' |
0305db28 | 19 | import { |
0305db28 JB |
20 | cleanupTests, |
21 | createMultipleServers, | |
22 | createSingleServer, | |
23 | doubleFollow, | |
0305db28 JB |
24 | killallServers, |
25 | makeRawRequest, | |
0305db28 JB |
26 | ObjectStorageCommand, |
27 | PeerTubeServer, | |
28 | setAccessTokensToServers, | |
d102de1b | 29 | waitJobs |
bf54587a | 30 | } from '@shared/server-commands' |
0305db28 | 31 | |
0305db28 | 32 | async function checkFiles (options: { |
1c41b5c1 C |
33 | server: PeerTubeServer |
34 | originServer: PeerTubeServer | |
d102de1b | 35 | originSQLCommand: SQLCommand |
1c41b5c1 | 36 | |
0305db28 JB |
37 | video: VideoDetails |
38 | ||
39 | baseMockUrl?: string | |
40 | ||
41 | playlistBucket: string | |
42 | playlistPrefix?: string | |
43 | ||
44 | webtorrentBucket: string | |
45 | webtorrentPrefix?: string | |
46 | }) { | |
47 | const { | |
1c41b5c1 C |
48 | server, |
49 | originServer, | |
d102de1b | 50 | originSQLCommand, |
0305db28 JB |
51 | video, |
52 | playlistBucket, | |
53 | webtorrentBucket, | |
54 | baseMockUrl, | |
55 | playlistPrefix, | |
56 | webtorrentPrefix | |
57 | } = options | |
58 | ||
59 | let allFiles = video.files | |
60 | ||
61 | for (const file of video.files) { | |
62 | const baseUrl = baseMockUrl | |
63 | ? `${baseMockUrl}/${webtorrentBucket}/` | |
9ab330b9 | 64 | : `http://${webtorrentBucket}.${ObjectStorageCommand.getMockEndpointHost()}/` |
0305db28 JB |
65 | |
66 | const prefix = webtorrentPrefix || '' | |
67 | const start = baseUrl + prefix | |
68 | ||
69 | expectStartWith(file.fileUrl, start) | |
70 | ||
3545e72c | 71 | const res = await makeRawRequest({ url: file.fileDownloadUrl, expectedStatus: HttpStatusCode.FOUND_302 }) |
0305db28 JB |
72 | const location = res.headers['location'] |
73 | expectStartWith(location, start) | |
74 | ||
3545e72c | 75 | await makeRawRequest({ url: location, expectedStatus: HttpStatusCode.OK_200 }) |
0305db28 JB |
76 | } |
77 | ||
78 | const hls = video.streamingPlaylists[0] | |
79 | ||
80 | if (hls) { | |
81 | allFiles = allFiles.concat(hls.files) | |
82 | ||
83 | const baseUrl = baseMockUrl | |
84 | ? `${baseMockUrl}/${playlistBucket}/` | |
9ab330b9 | 85 | : `http://${playlistBucket}.${ObjectStorageCommand.getMockEndpointHost()}/` |
0305db28 JB |
86 | |
87 | const prefix = playlistPrefix || '' | |
88 | const start = baseUrl + prefix | |
89 | ||
90 | expectStartWith(hls.playlistUrl, start) | |
91 | expectStartWith(hls.segmentsSha256Url, start) | |
92 | ||
3545e72c | 93 | await makeRawRequest({ url: hls.playlistUrl, expectedStatus: HttpStatusCode.OK_200 }) |
0305db28 | 94 | |
3545e72c | 95 | const resSha = await makeRawRequest({ url: hls.segmentsSha256Url, expectedStatus: HttpStatusCode.OK_200 }) |
0305db28 JB |
96 | expect(JSON.stringify(resSha.body)).to.not.throw |
97 | ||
1c41b5c1 | 98 | let i = 0 |
0305db28 JB |
99 | for (const file of hls.files) { |
100 | expectStartWith(file.fileUrl, start) | |
101 | ||
3545e72c | 102 | const res = await makeRawRequest({ url: file.fileDownloadUrl, expectedStatus: HttpStatusCode.FOUND_302 }) |
0305db28 JB |
103 | const location = res.headers['location'] |
104 | expectStartWith(location, start) | |
105 | ||
3545e72c | 106 | await makeRawRequest({ url: location, expectedStatus: HttpStatusCode.OK_200 }) |
1c41b5c1 C |
107 | |
108 | if (originServer.internalServerNumber === server.internalServerNumber) { | |
109 | const infohash = sha1(`${2 + hls.playlistUrl}+V${i}`) | |
d102de1b | 110 | const dbInfohashes = await originSQLCommand.getPlaylistInfohash(hls.id) |
1c41b5c1 C |
111 | |
112 | expect(dbInfohashes).to.include(infohash) | |
113 | } | |
114 | ||
115 | i++ | |
0305db28 JB |
116 | } |
117 | } | |
118 | ||
119 | for (const file of allFiles) { | |
d102de1b | 120 | await checkWebTorrentWorks(file.magnetUri) |
0305db28 | 121 | |
3545e72c | 122 | const res = await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 }) |
0305db28 JB |
123 | expect(res.body).to.have.length.above(100) |
124 | } | |
125 | ||
126 | return allFiles.map(f => f.fileUrl) | |
127 | } | |
128 | ||
129 | function runTestSuite (options: { | |
156cdbac | 130 | fixture?: string |
131 | ||
132 | maxUploadPart?: string | |
133 | ||
0305db28 JB |
134 | playlistBucket: string |
135 | playlistPrefix?: string | |
136 | ||
137 | webtorrentBucket: string | |
138 | webtorrentPrefix?: string | |
139 | ||
140 | useMockBaseUrl?: boolean | |
0305db28 | 141 | }) { |
8059e050 | 142 | const mockObjectStorageProxy = new MockObjectStorageProxy() |
156cdbac | 143 | const { fixture } = options |
0305db28 JB |
144 | let baseMockUrl: string |
145 | ||
146 | let servers: PeerTubeServer[] | |
c106db14 | 147 | let sqlCommands: SQLCommand[] = [] |
f8918990 | 148 | const objectStorage = new ObjectStorageCommand() |
0305db28 JB |
149 | |
150 | let keptUrls: string[] = [] | |
151 | ||
152 | const uuidsToDelete: string[] = [] | |
153 | let deletedUrls: string[] = [] | |
154 | ||
155 | before(async function () { | |
5a05c145 | 156 | this.timeout(240000) |
0305db28 | 157 | |
8059e050 C |
158 | const port = await mockObjectStorageProxy.initialize() |
159 | baseMockUrl = options.useMockBaseUrl | |
160 | ? `http://127.0.0.1:${port}` | |
161 | : undefined | |
0305db28 | 162 | |
f8918990 C |
163 | await objectStorage.createMockBucket(options.playlistBucket) |
164 | await objectStorage.createMockBucket(options.webtorrentBucket) | |
0305db28 JB |
165 | |
166 | const config = { | |
167 | object_storage: { | |
168 | enabled: true, | |
9ab330b9 C |
169 | endpoint: 'http://' + ObjectStorageCommand.getMockEndpointHost(), |
170 | region: ObjectStorageCommand.getMockRegion(), | |
0305db28 | 171 | |
9ab330b9 | 172 | credentials: ObjectStorageCommand.getMockCredentialsConfig(), |
0305db28 | 173 | |
156cdbac | 174 | max_upload_part: options.maxUploadPart || '5MB', |
0305db28 JB |
175 | |
176 | streaming_playlists: { | |
177 | bucket_name: options.playlistBucket, | |
178 | prefix: options.playlistPrefix, | |
179 | base_url: baseMockUrl | |
180 | ? `${baseMockUrl}/${options.playlistBucket}` | |
181 | : undefined | |
182 | }, | |
183 | ||
184 | videos: { | |
185 | bucket_name: options.webtorrentBucket, | |
186 | prefix: options.webtorrentPrefix, | |
187 | base_url: baseMockUrl | |
188 | ? `${baseMockUrl}/${options.webtorrentBucket}` | |
189 | : undefined | |
190 | } | |
191 | } | |
192 | } | |
193 | ||
194 | servers = await createMultipleServers(2, config) | |
195 | ||
196 | await setAccessTokensToServers(servers) | |
197 | await doubleFollow(servers[0], servers[1]) | |
198 | ||
199 | for (const server of servers) { | |
200 | const { uuid } = await server.videos.quickUpload({ name: 'video to keep' }) | |
201 | await waitJobs(servers) | |
202 | ||
203 | const files = await server.videos.listFiles({ id: uuid }) | |
204 | keptUrls = keptUrls.concat(files.map(f => f.fileUrl)) | |
205 | } | |
d102de1b C |
206 | |
207 | sqlCommands = servers.map(s => new SQLCommand(s)) | |
0305db28 JB |
208 | }) |
209 | ||
210 | it('Should upload a video and move it to the object storage without transcoding', async function () { | |
d1bfbdeb | 211 | this.timeout(40000) |
0305db28 | 212 | |
156cdbac | 213 | const { uuid } = await servers[0].videos.quickUpload({ name: 'video 1', fixture }) |
0305db28 JB |
214 | uuidsToDelete.push(uuid) |
215 | ||
216 | await waitJobs(servers) | |
217 | ||
218 | for (const server of servers) { | |
219 | const video = await server.videos.get({ id: uuid }) | |
d102de1b | 220 | const files = await checkFiles({ ...options, server, originServer: servers[0], originSQLCommand: sqlCommands[0], video, baseMockUrl }) |
0305db28 JB |
221 | |
222 | deletedUrls = deletedUrls.concat(files) | |
223 | } | |
224 | }) | |
225 | ||
226 | it('Should upload a video and move it to the object storage with transcoding', async function () { | |
1ffb5fb6 | 227 | this.timeout(120000) |
0305db28 | 228 | |
156cdbac | 229 | const { uuid } = await servers[1].videos.quickUpload({ name: 'video 2', fixture }) |
0305db28 JB |
230 | uuidsToDelete.push(uuid) |
231 | ||
232 | await waitJobs(servers) | |
233 | ||
234 | for (const server of servers) { | |
235 | const video = await server.videos.get({ id: uuid }) | |
d102de1b | 236 | const files = await checkFiles({ ...options, server, originServer: servers[0], originSQLCommand: sqlCommands[0], video, baseMockUrl }) |
0305db28 JB |
237 | |
238 | deletedUrls = deletedUrls.concat(files) | |
239 | } | |
240 | }) | |
241 | ||
87699a09 C |
242 | it('Should fetch correctly all the files', async function () { |
243 | for (const url of deletedUrls.concat(keptUrls)) { | |
3545e72c | 244 | await makeRawRequest({ url, expectedStatus: HttpStatusCode.OK_200 }) |
87699a09 C |
245 | } |
246 | }) | |
247 | ||
0305db28 JB |
248 | it('Should correctly delete the files', async function () { |
249 | await servers[0].videos.remove({ id: uuidsToDelete[0] }) | |
250 | await servers[1].videos.remove({ id: uuidsToDelete[1] }) | |
251 | ||
252 | await waitJobs(servers) | |
253 | ||
254 | for (const url of deletedUrls) { | |
3545e72c | 255 | await makeRawRequest({ url, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) |
0305db28 JB |
256 | } |
257 | }) | |
258 | ||
259 | it('Should have kept other files', async function () { | |
260 | for (const url of keptUrls) { | |
3545e72c | 261 | await makeRawRequest({ url, expectedStatus: HttpStatusCode.OK_200 }) |
0305db28 JB |
262 | } |
263 | }) | |
264 | ||
265 | it('Should have an empty tmp directory', async function () { | |
266 | for (const server of servers) { | |
267 | await checkTmpIsEmpty(server) | |
268 | } | |
269 | }) | |
270 | ||
1f6125be C |
271 | it('Should not have downloaded files from object storage', async function () { |
272 | for (const server of servers) { | |
273 | await expectLogDoesNotContain(server, 'from object storage') | |
274 | } | |
275 | }) | |
276 | ||
0305db28 | 277 | after(async function () { |
8059e050 | 278 | await mockObjectStorageProxy.terminate() |
f8918990 | 279 | await objectStorage.cleanupMock() |
0305db28 | 280 | |
d102de1b C |
281 | for (const sqlCommand of sqlCommands) { |
282 | await sqlCommand.cleanup() | |
283 | } | |
284 | ||
0305db28 JB |
285 | await cleanupTests(servers) |
286 | }) | |
287 | } | |
288 | ||
289 | describe('Object storage for videos', function () { | |
9ab330b9 | 290 | if (areMockObjectStorageTestsDisabled()) return |
0305db28 | 291 | |
f8918990 C |
292 | const objectStorage = new ObjectStorageCommand() |
293 | ||
0305db28 JB |
294 | describe('Test config', function () { |
295 | let server: PeerTubeServer | |
296 | ||
f8918990 | 297 | const baseConfig = objectStorage.getDefaultMockConfig() |
0305db28 JB |
298 | |
299 | const badCredentials = { | |
300 | access_key_id: 'AKIAIOSFODNN7EXAMPLE', | |
301 | secret_access_key: 'aJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY' | |
302 | } | |
303 | ||
304 | it('Should fail with same bucket names without prefix', function (done) { | |
305 | const config = merge({}, baseConfig, { | |
306 | object_storage: { | |
307 | streaming_playlists: { | |
308 | bucket_name: 'aaa' | |
309 | }, | |
310 | ||
311 | videos: { | |
312 | bucket_name: 'aaa' | |
313 | } | |
314 | } | |
315 | }) | |
316 | ||
317 | createSingleServer(1, config) | |
318 | .then(() => done(new Error('Did not throw'))) | |
319 | .catch(() => done()) | |
320 | }) | |
321 | ||
322 | it('Should fail with bad credentials', async function () { | |
323 | this.timeout(60000) | |
324 | ||
f8918990 | 325 | await objectStorage.prepareDefaultMockBuckets() |
0305db28 JB |
326 | |
327 | const config = merge({}, baseConfig, { | |
328 | object_storage: { | |
329 | credentials: badCredentials | |
330 | } | |
331 | }) | |
332 | ||
333 | server = await createSingleServer(1, config) | |
334 | await setAccessTokensToServers([ server ]) | |
335 | ||
336 | const { uuid } = await server.videos.quickUpload({ name: 'video' }) | |
337 | ||
e82cb087 | 338 | await waitJobs([ server ], { skipDelayed: true }) |
0305db28 JB |
339 | const video = await server.videos.get({ id: uuid }) |
340 | ||
341 | expectStartWith(video.files[0].fileUrl, server.url) | |
342 | ||
343 | await killallServers([ server ]) | |
344 | }) | |
345 | ||
346 | it('Should succeed with credentials from env', async function () { | |
347 | this.timeout(60000) | |
348 | ||
f8918990 | 349 | await objectStorage.prepareDefaultMockBuckets() |
0305db28 JB |
350 | |
351 | const config = merge({}, baseConfig, { | |
352 | object_storage: { | |
353 | credentials: { | |
354 | access_key_id: '', | |
355 | secret_access_key: '' | |
356 | } | |
357 | } | |
358 | }) | |
359 | ||
9ab330b9 | 360 | const goodCredentials = ObjectStorageCommand.getMockCredentialsConfig() |
0305db28 JB |
361 | |
362 | server = await createSingleServer(1, config, { | |
363 | env: { | |
364 | AWS_ACCESS_KEY_ID: goodCredentials.access_key_id, | |
365 | AWS_SECRET_ACCESS_KEY: goodCredentials.secret_access_key | |
366 | } | |
367 | }) | |
368 | ||
369 | await setAccessTokensToServers([ server ]) | |
370 | ||
371 | const { uuid } = await server.videos.quickUpload({ name: 'video' }) | |
372 | ||
e82cb087 | 373 | await waitJobs([ server ], { skipDelayed: true }) |
0305db28 JB |
374 | const video = await server.videos.get({ id: uuid }) |
375 | ||
f8918990 | 376 | expectStartWith(video.files[0].fileUrl, objectStorage.getMockWebVideosBaseUrl()) |
0305db28 JB |
377 | }) |
378 | ||
379 | after(async function () { | |
f8918990 C |
380 | await objectStorage.cleanupMock() |
381 | ||
644391be | 382 | await cleanupTests([ server ]) |
0305db28 JB |
383 | }) |
384 | }) | |
385 | ||
386 | describe('Test simple object storage', function () { | |
387 | runTestSuite({ | |
f8918990 C |
388 | playlistBucket: objectStorage.getMockBucketName('streaming-playlists'), |
389 | webtorrentBucket: objectStorage.getMockBucketName('videos') | |
0305db28 JB |
390 | }) |
391 | }) | |
392 | ||
393 | describe('Test object storage with prefix', function () { | |
394 | runTestSuite({ | |
f8918990 C |
395 | playlistBucket: objectStorage.getMockBucketName('mybucket'), |
396 | webtorrentBucket: objectStorage.getMockBucketName('mybucket'), | |
0305db28 JB |
397 | |
398 | playlistPrefix: 'streaming-playlists_', | |
399 | webtorrentPrefix: 'webtorrent_' | |
400 | }) | |
401 | }) | |
402 | ||
403 | describe('Test object storage with prefix and base URL', function () { | |
404 | runTestSuite({ | |
f8918990 C |
405 | playlistBucket: objectStorage.getMockBucketName('mybucket'), |
406 | webtorrentBucket: objectStorage.getMockBucketName('mybucket'), | |
0305db28 | 407 | |
70430c27 C |
408 | playlistPrefix: 'streaming-playlists/', |
409 | webtorrentPrefix: 'webtorrent/', | |
0305db28 JB |
410 | |
411 | useMockBaseUrl: true | |
412 | }) | |
413 | }) | |
414 | ||
156cdbac | 415 | describe('Test object storage with file bigger than upload part', function () { |
416 | let fixture: string | |
417 | const maxUploadPart = '5MB' | |
418 | ||
419 | before(async function () { | |
cf0dd18a C |
420 | this.timeout(120000) |
421 | ||
156cdbac | 422 | fixture = await generateHighBitrateVideo() |
423 | ||
424 | const { size } = await stat(fixture) | |
425 | ||
426 | if (bytes.parse(maxUploadPart) > size) { | |
427 | throw Error(`Fixture file is too small (${size}) to make sense for this test.`) | |
428 | } | |
429 | }) | |
430 | ||
0305db28 | 431 | runTestSuite({ |
156cdbac | 432 | maxUploadPart, |
f8918990 C |
433 | playlistBucket: objectStorage.getMockBucketName('streaming-playlists'), |
434 | webtorrentBucket: objectStorage.getMockBucketName('videos'), | |
156cdbac | 435 | fixture |
0305db28 JB |
436 | }) |
437 | }) | |
438 | }) |