]>
Commit | Line | Data |
---|---|---|
0305db28 JB |
1 | /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ |
2 | ||
3 | import 'mocha' | |
156cdbac | 4 | import bytes from 'bytes' |
0305db28 | 5 | import * as chai from 'chai' |
156cdbac | 6 | import { stat } from 'fs-extra' |
0305db28 | 7 | import { merge } from 'lodash' |
156cdbac | 8 | import { |
9 | checkTmpIsEmpty, | |
10 | expectLogDoesNotContain, | |
11 | expectStartWith, | |
12 | generateHighBitrateVideo, | |
13 | MockObjectStorage | |
14 | } from '@server/tests/shared' | |
c55e3d72 C |
15 | import { areObjectStorageTestsDisabled } from '@shared/core-utils' |
16 | import { HttpStatusCode, VideoDetails } from '@shared/models' | |
0305db28 | 17 | import { |
0305db28 JB |
18 | cleanupTests, |
19 | createMultipleServers, | |
20 | createSingleServer, | |
21 | doubleFollow, | |
0305db28 JB |
22 | killallServers, |
23 | makeRawRequest, | |
0305db28 JB |
24 | ObjectStorageCommand, |
25 | PeerTubeServer, | |
26 | setAccessTokensToServers, | |
27 | waitJobs, | |
28 | webtorrentAdd | |
bf54587a | 29 | } from '@shared/server-commands' |
0305db28 JB |
30 | |
31 | const expect = chai.expect | |
32 | ||
33 | async function checkFiles (options: { | |
34 | video: VideoDetails | |
35 | ||
36 | baseMockUrl?: string | |
37 | ||
38 | playlistBucket: string | |
39 | playlistPrefix?: string | |
40 | ||
41 | webtorrentBucket: string | |
42 | webtorrentPrefix?: string | |
43 | }) { | |
44 | const { | |
45 | video, | |
46 | playlistBucket, | |
47 | webtorrentBucket, | |
48 | baseMockUrl, | |
49 | playlistPrefix, | |
50 | webtorrentPrefix | |
51 | } = options | |
52 | ||
53 | let allFiles = video.files | |
54 | ||
55 | for (const file of video.files) { | |
56 | const baseUrl = baseMockUrl | |
57 | ? `${baseMockUrl}/${webtorrentBucket}/` | |
58 | : `http://${webtorrentBucket}.${ObjectStorageCommand.getEndpointHost()}/` | |
59 | ||
60 | const prefix = webtorrentPrefix || '' | |
61 | const start = baseUrl + prefix | |
62 | ||
63 | expectStartWith(file.fileUrl, start) | |
64 | ||
65 | const res = await makeRawRequest(file.fileDownloadUrl, HttpStatusCode.FOUND_302) | |
66 | const location = res.headers['location'] | |
67 | expectStartWith(location, start) | |
68 | ||
69 | await makeRawRequest(location, HttpStatusCode.OK_200) | |
70 | } | |
71 | ||
72 | const hls = video.streamingPlaylists[0] | |
73 | ||
74 | if (hls) { | |
75 | allFiles = allFiles.concat(hls.files) | |
76 | ||
77 | const baseUrl = baseMockUrl | |
78 | ? `${baseMockUrl}/${playlistBucket}/` | |
79 | : `http://${playlistBucket}.${ObjectStorageCommand.getEndpointHost()}/` | |
80 | ||
81 | const prefix = playlistPrefix || '' | |
82 | const start = baseUrl + prefix | |
83 | ||
84 | expectStartWith(hls.playlistUrl, start) | |
85 | expectStartWith(hls.segmentsSha256Url, start) | |
86 | ||
87 | await makeRawRequest(hls.playlistUrl, HttpStatusCode.OK_200) | |
88 | ||
89 | const resSha = await makeRawRequest(hls.segmentsSha256Url, HttpStatusCode.OK_200) | |
90 | expect(JSON.stringify(resSha.body)).to.not.throw | |
91 | ||
92 | for (const file of hls.files) { | |
93 | expectStartWith(file.fileUrl, start) | |
94 | ||
95 | const res = await makeRawRequest(file.fileDownloadUrl, HttpStatusCode.FOUND_302) | |
96 | const location = res.headers['location'] | |
97 | expectStartWith(location, start) | |
98 | ||
99 | await makeRawRequest(location, HttpStatusCode.OK_200) | |
100 | } | |
101 | } | |
102 | ||
103 | for (const file of allFiles) { | |
104 | const torrent = await webtorrentAdd(file.magnetUri, true) | |
105 | ||
106 | expect(torrent.files).to.be.an('array') | |
107 | expect(torrent.files.length).to.equal(1) | |
108 | expect(torrent.files[0].path).to.exist.and.to.not.equal('') | |
109 | ||
110 | const res = await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200) | |
111 | expect(res.body).to.have.length.above(100) | |
112 | } | |
113 | ||
114 | return allFiles.map(f => f.fileUrl) | |
115 | } | |
116 | ||
117 | function runTestSuite (options: { | |
156cdbac | 118 | fixture?: string |
119 | ||
120 | maxUploadPart?: string | |
121 | ||
0305db28 JB |
122 | playlistBucket: string |
123 | playlistPrefix?: string | |
124 | ||
125 | webtorrentBucket: string | |
126 | webtorrentPrefix?: string | |
127 | ||
128 | useMockBaseUrl?: boolean | |
0305db28 JB |
129 | }) { |
130 | const mockObjectStorage = new MockObjectStorage() | |
156cdbac | 131 | const { fixture } = options |
0305db28 JB |
132 | let baseMockUrl: string |
133 | ||
134 | let servers: PeerTubeServer[] | |
135 | ||
136 | let keptUrls: string[] = [] | |
137 | ||
138 | const uuidsToDelete: string[] = [] | |
139 | let deletedUrls: string[] = [] | |
140 | ||
141 | before(async function () { | |
142 | this.timeout(120000) | |
143 | ||
144 | const port = await mockObjectStorage.initialize() | |
145 | baseMockUrl = options.useMockBaseUrl ? `http://localhost:${port}` : undefined | |
146 | ||
147 | await ObjectStorageCommand.createBucket(options.playlistBucket) | |
148 | await ObjectStorageCommand.createBucket(options.webtorrentBucket) | |
149 | ||
150 | const config = { | |
151 | object_storage: { | |
152 | enabled: true, | |
153 | endpoint: 'http://' + ObjectStorageCommand.getEndpointHost(), | |
154 | region: ObjectStorageCommand.getRegion(), | |
155 | ||
156 | credentials: ObjectStorageCommand.getCredentialsConfig(), | |
157 | ||
156cdbac | 158 | max_upload_part: options.maxUploadPart || '5MB', |
0305db28 JB |
159 | |
160 | streaming_playlists: { | |
161 | bucket_name: options.playlistBucket, | |
162 | prefix: options.playlistPrefix, | |
163 | base_url: baseMockUrl | |
164 | ? `${baseMockUrl}/${options.playlistBucket}` | |
165 | : undefined | |
166 | }, | |
167 | ||
168 | videos: { | |
169 | bucket_name: options.webtorrentBucket, | |
170 | prefix: options.webtorrentPrefix, | |
171 | base_url: baseMockUrl | |
172 | ? `${baseMockUrl}/${options.webtorrentBucket}` | |
173 | : undefined | |
174 | } | |
175 | } | |
176 | } | |
177 | ||
178 | servers = await createMultipleServers(2, config) | |
179 | ||
180 | await setAccessTokensToServers(servers) | |
181 | await doubleFollow(servers[0], servers[1]) | |
182 | ||
183 | for (const server of servers) { | |
184 | const { uuid } = await server.videos.quickUpload({ name: 'video to keep' }) | |
185 | await waitJobs(servers) | |
186 | ||
187 | const files = await server.videos.listFiles({ id: uuid }) | |
188 | keptUrls = keptUrls.concat(files.map(f => f.fileUrl)) | |
189 | } | |
190 | }) | |
191 | ||
192 | it('Should upload a video and move it to the object storage without transcoding', async function () { | |
d1bfbdeb | 193 | this.timeout(40000) |
0305db28 | 194 | |
156cdbac | 195 | const { uuid } = await servers[0].videos.quickUpload({ name: 'video 1', fixture }) |
0305db28 JB |
196 | uuidsToDelete.push(uuid) |
197 | ||
198 | await waitJobs(servers) | |
199 | ||
200 | for (const server of servers) { | |
201 | const video = await server.videos.get({ id: uuid }) | |
202 | const files = await checkFiles({ ...options, video, baseMockUrl }) | |
203 | ||
204 | deletedUrls = deletedUrls.concat(files) | |
205 | } | |
206 | }) | |
207 | ||
208 | it('Should upload a video and move it to the object storage with transcoding', async function () { | |
1ffb5fb6 | 209 | this.timeout(120000) |
0305db28 | 210 | |
156cdbac | 211 | const { uuid } = await servers[1].videos.quickUpload({ name: 'video 2', fixture }) |
0305db28 JB |
212 | uuidsToDelete.push(uuid) |
213 | ||
214 | await waitJobs(servers) | |
215 | ||
216 | for (const server of servers) { | |
217 | const video = await server.videos.get({ id: uuid }) | |
218 | const files = await checkFiles({ ...options, video, baseMockUrl }) | |
219 | ||
220 | deletedUrls = deletedUrls.concat(files) | |
221 | } | |
222 | }) | |
223 | ||
87699a09 C |
224 | it('Should fetch correctly all the files', async function () { |
225 | for (const url of deletedUrls.concat(keptUrls)) { | |
226 | await makeRawRequest(url, HttpStatusCode.OK_200) | |
227 | } | |
228 | }) | |
229 | ||
0305db28 JB |
230 | it('Should correctly delete the files', async function () { |
231 | await servers[0].videos.remove({ id: uuidsToDelete[0] }) | |
232 | await servers[1].videos.remove({ id: uuidsToDelete[1] }) | |
233 | ||
234 | await waitJobs(servers) | |
235 | ||
236 | for (const url of deletedUrls) { | |
237 | await makeRawRequest(url, HttpStatusCode.NOT_FOUND_404) | |
238 | } | |
239 | }) | |
240 | ||
241 | it('Should have kept other files', async function () { | |
242 | for (const url of keptUrls) { | |
243 | await makeRawRequest(url, HttpStatusCode.OK_200) | |
244 | } | |
245 | }) | |
246 | ||
247 | it('Should have an empty tmp directory', async function () { | |
248 | for (const server of servers) { | |
249 | await checkTmpIsEmpty(server) | |
250 | } | |
251 | }) | |
252 | ||
1f6125be C |
253 | it('Should not have downloaded files from object storage', async function () { |
254 | for (const server of servers) { | |
255 | await expectLogDoesNotContain(server, 'from object storage') | |
256 | } | |
257 | }) | |
258 | ||
0305db28 | 259 | after(async function () { |
70430c27 | 260 | await mockObjectStorage.terminate() |
0305db28 JB |
261 | |
262 | await cleanupTests(servers) | |
263 | }) | |
264 | } | |
265 | ||
266 | describe('Object storage for videos', function () { | |
267 | if (areObjectStorageTestsDisabled()) return | |
268 | ||
269 | describe('Test config', function () { | |
270 | let server: PeerTubeServer | |
271 | ||
272 | const baseConfig = { | |
273 | object_storage: { | |
274 | enabled: true, | |
275 | endpoint: 'http://' + ObjectStorageCommand.getEndpointHost(), | |
276 | region: ObjectStorageCommand.getRegion(), | |
277 | ||
278 | credentials: ObjectStorageCommand.getCredentialsConfig(), | |
279 | ||
280 | streaming_playlists: { | |
281 | bucket_name: ObjectStorageCommand.DEFAULT_PLAYLIST_BUCKET | |
282 | }, | |
283 | ||
284 | videos: { | |
285 | bucket_name: ObjectStorageCommand.DEFAULT_WEBTORRENT_BUCKET | |
286 | } | |
287 | } | |
288 | } | |
289 | ||
290 | const badCredentials = { | |
291 | access_key_id: 'AKIAIOSFODNN7EXAMPLE', | |
292 | secret_access_key: 'aJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY' | |
293 | } | |
294 | ||
295 | it('Should fail with same bucket names without prefix', function (done) { | |
296 | const config = merge({}, baseConfig, { | |
297 | object_storage: { | |
298 | streaming_playlists: { | |
299 | bucket_name: 'aaa' | |
300 | }, | |
301 | ||
302 | videos: { | |
303 | bucket_name: 'aaa' | |
304 | } | |
305 | } | |
306 | }) | |
307 | ||
308 | createSingleServer(1, config) | |
309 | .then(() => done(new Error('Did not throw'))) | |
310 | .catch(() => done()) | |
311 | }) | |
312 | ||
313 | it('Should fail with bad credentials', async function () { | |
314 | this.timeout(60000) | |
315 | ||
316 | await ObjectStorageCommand.prepareDefaultBuckets() | |
317 | ||
318 | const config = merge({}, baseConfig, { | |
319 | object_storage: { | |
320 | credentials: badCredentials | |
321 | } | |
322 | }) | |
323 | ||
324 | server = await createSingleServer(1, config) | |
325 | await setAccessTokensToServers([ server ]) | |
326 | ||
327 | const { uuid } = await server.videos.quickUpload({ name: 'video' }) | |
328 | ||
329 | await waitJobs([ server ], true) | |
330 | const video = await server.videos.get({ id: uuid }) | |
331 | ||
332 | expectStartWith(video.files[0].fileUrl, server.url) | |
333 | ||
334 | await killallServers([ server ]) | |
335 | }) | |
336 | ||
337 | it('Should succeed with credentials from env', async function () { | |
338 | this.timeout(60000) | |
339 | ||
340 | await ObjectStorageCommand.prepareDefaultBuckets() | |
341 | ||
342 | const config = merge({}, baseConfig, { | |
343 | object_storage: { | |
344 | credentials: { | |
345 | access_key_id: '', | |
346 | secret_access_key: '' | |
347 | } | |
348 | } | |
349 | }) | |
350 | ||
351 | const goodCredentials = ObjectStorageCommand.getCredentialsConfig() | |
352 | ||
353 | server = await createSingleServer(1, config, { | |
354 | env: { | |
355 | AWS_ACCESS_KEY_ID: goodCredentials.access_key_id, | |
356 | AWS_SECRET_ACCESS_KEY: goodCredentials.secret_access_key | |
357 | } | |
358 | }) | |
359 | ||
360 | await setAccessTokensToServers([ server ]) | |
361 | ||
362 | const { uuid } = await server.videos.quickUpload({ name: 'video' }) | |
363 | ||
364 | await waitJobs([ server ], true) | |
365 | const video = await server.videos.get({ id: uuid }) | |
366 | ||
367 | expectStartWith(video.files[0].fileUrl, ObjectStorageCommand.getWebTorrentBaseUrl()) | |
368 | }) | |
369 | ||
370 | after(async function () { | |
371 | await killallServers([ server ]) | |
372 | }) | |
373 | }) | |
374 | ||
375 | describe('Test simple object storage', function () { | |
376 | runTestSuite({ | |
377 | playlistBucket: 'streaming-playlists', | |
378 | webtorrentBucket: 'videos' | |
379 | }) | |
380 | }) | |
381 | ||
382 | describe('Test object storage with prefix', function () { | |
383 | runTestSuite({ | |
384 | playlistBucket: 'mybucket', | |
385 | webtorrentBucket: 'mybucket', | |
386 | ||
387 | playlistPrefix: 'streaming-playlists_', | |
388 | webtorrentPrefix: 'webtorrent_' | |
389 | }) | |
390 | }) | |
391 | ||
392 | describe('Test object storage with prefix and base URL', function () { | |
393 | runTestSuite({ | |
394 | playlistBucket: 'mybucket', | |
395 | webtorrentBucket: 'mybucket', | |
396 | ||
70430c27 C |
397 | playlistPrefix: 'streaming-playlists/', |
398 | webtorrentPrefix: 'webtorrent/', | |
0305db28 JB |
399 | |
400 | useMockBaseUrl: true | |
401 | }) | |
402 | }) | |
403 | ||
156cdbac | 404 | describe('Test object storage with file bigger than upload part', function () { |
405 | let fixture: string | |
406 | const maxUploadPart = '5MB' | |
407 | ||
408 | before(async function () { | |
cf0dd18a C |
409 | this.timeout(120000) |
410 | ||
156cdbac | 411 | fixture = await generateHighBitrateVideo() |
412 | ||
413 | const { size } = await stat(fixture) | |
414 | ||
415 | if (bytes.parse(maxUploadPart) > size) { | |
416 | throw Error(`Fixture file is too small (${size}) to make sense for this test.`) | |
417 | } | |
418 | }) | |
419 | ||
0305db28 | 420 | runTestSuite({ |
156cdbac | 421 | maxUploadPart, |
0305db28 JB |
422 | playlistBucket: 'streaming-playlists', |
423 | webtorrentBucket: 'videos', | |
156cdbac | 424 | fixture |
0305db28 JB |
425 | }) |
426 | }) | |
427 | }) |