]>
Commit | Line | Data |
---|---|---|
1 | /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ | |
2 | ||
3 | import bytes from 'bytes' | |
4 | import { expect } from 'chai' | |
5 | import { stat } from 'fs-extra' | |
6 | import { merge } from 'lodash' | |
7 | import { | |
8 | checkTmpIsEmpty, | |
9 | expectLogDoesNotContain, | |
10 | expectStartWith, | |
11 | generateHighBitrateVideo, | |
12 | MockObjectStorage | |
13 | } from '@server/tests/shared' | |
14 | import { areMockObjectStorageTestsDisabled } from '@shared/core-utils' | |
15 | import { HttpStatusCode, VideoDetails } from '@shared/models' | |
16 | import { | |
17 | cleanupTests, | |
18 | createMultipleServers, | |
19 | createSingleServer, | |
20 | doubleFollow, | |
21 | killallServers, | |
22 | makeRawRequest, | |
23 | ObjectStorageCommand, | |
24 | PeerTubeServer, | |
25 | setAccessTokensToServers, | |
26 | waitJobs, | |
27 | webtorrentAdd | |
28 | } from '@shared/server-commands' | |
29 | ||
30 | async function checkFiles (options: { | |
31 | video: VideoDetails | |
32 | ||
33 | baseMockUrl?: string | |
34 | ||
35 | playlistBucket: string | |
36 | playlistPrefix?: string | |
37 | ||
38 | webtorrentBucket: string | |
39 | webtorrentPrefix?: string | |
40 | }) { | |
41 | const { | |
42 | video, | |
43 | playlistBucket, | |
44 | webtorrentBucket, | |
45 | baseMockUrl, | |
46 | playlistPrefix, | |
47 | webtorrentPrefix | |
48 | } = options | |
49 | ||
50 | let allFiles = video.files | |
51 | ||
52 | for (const file of video.files) { | |
53 | const baseUrl = baseMockUrl | |
54 | ? `${baseMockUrl}/${webtorrentBucket}/` | |
55 | : `http://${webtorrentBucket}.${ObjectStorageCommand.getMockEndpointHost()}/` | |
56 | ||
57 | const prefix = webtorrentPrefix || '' | |
58 | const start = baseUrl + prefix | |
59 | ||
60 | expectStartWith(file.fileUrl, start) | |
61 | ||
62 | const res = await makeRawRequest({ url: file.fileDownloadUrl, expectedStatus: HttpStatusCode.FOUND_302 }) | |
63 | const location = res.headers['location'] | |
64 | expectStartWith(location, start) | |
65 | ||
66 | await makeRawRequest({ url: location, expectedStatus: HttpStatusCode.OK_200 }) | |
67 | } | |
68 | ||
69 | const hls = video.streamingPlaylists[0] | |
70 | ||
71 | if (hls) { | |
72 | allFiles = allFiles.concat(hls.files) | |
73 | ||
74 | const baseUrl = baseMockUrl | |
75 | ? `${baseMockUrl}/${playlistBucket}/` | |
76 | : `http://${playlistBucket}.${ObjectStorageCommand.getMockEndpointHost()}/` | |
77 | ||
78 | const prefix = playlistPrefix || '' | |
79 | const start = baseUrl + prefix | |
80 | ||
81 | expectStartWith(hls.playlistUrl, start) | |
82 | expectStartWith(hls.segmentsSha256Url, start) | |
83 | ||
84 | await makeRawRequest({ url: hls.playlistUrl, expectedStatus: HttpStatusCode.OK_200 }) | |
85 | ||
86 | const resSha = await makeRawRequest({ url: hls.segmentsSha256Url, expectedStatus: HttpStatusCode.OK_200 }) | |
87 | expect(JSON.stringify(resSha.body)).to.not.throw | |
88 | ||
89 | for (const file of hls.files) { | |
90 | expectStartWith(file.fileUrl, start) | |
91 | ||
92 | const res = await makeRawRequest({ url: file.fileDownloadUrl, expectedStatus: HttpStatusCode.FOUND_302 }) | |
93 | const location = res.headers['location'] | |
94 | expectStartWith(location, start) | |
95 | ||
96 | await makeRawRequest({ url: location, expectedStatus: HttpStatusCode.OK_200 }) | |
97 | } | |
98 | } | |
99 | ||
100 | for (const file of allFiles) { | |
101 | const torrent = await webtorrentAdd(file.magnetUri, true) | |
102 | ||
103 | expect(torrent.files).to.be.an('array') | |
104 | expect(torrent.files.length).to.equal(1) | |
105 | expect(torrent.files[0].path).to.exist.and.to.not.equal('') | |
106 | ||
107 | const res = await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 }) | |
108 | expect(res.body).to.have.length.above(100) | |
109 | } | |
110 | ||
111 | return allFiles.map(f => f.fileUrl) | |
112 | } | |
113 | ||
114 | function runTestSuite (options: { | |
115 | fixture?: string | |
116 | ||
117 | maxUploadPart?: string | |
118 | ||
119 | playlistBucket: string | |
120 | playlistPrefix?: string | |
121 | ||
122 | webtorrentBucket: string | |
123 | webtorrentPrefix?: string | |
124 | ||
125 | useMockBaseUrl?: boolean | |
126 | }) { | |
127 | const mockObjectStorage = new MockObjectStorage() | |
128 | const { fixture } = options | |
129 | let baseMockUrl: string | |
130 | ||
131 | let servers: PeerTubeServer[] | |
132 | ||
133 | let keptUrls: string[] = [] | |
134 | ||
135 | const uuidsToDelete: string[] = [] | |
136 | let deletedUrls: string[] = [] | |
137 | ||
138 | before(async function () { | |
139 | this.timeout(120000) | |
140 | ||
141 | const port = await mockObjectStorage.initialize() | |
142 | baseMockUrl = options.useMockBaseUrl ? `http://localhost:${port}` : undefined | |
143 | ||
144 | await ObjectStorageCommand.createMockBucket(options.playlistBucket) | |
145 | await ObjectStorageCommand.createMockBucket(options.webtorrentBucket) | |
146 | ||
147 | const config = { | |
148 | object_storage: { | |
149 | enabled: true, | |
150 | endpoint: 'http://' + ObjectStorageCommand.getMockEndpointHost(), | |
151 | region: ObjectStorageCommand.getMockRegion(), | |
152 | ||
153 | credentials: ObjectStorageCommand.getMockCredentialsConfig(), | |
154 | ||
155 | max_upload_part: options.maxUploadPart || '5MB', | |
156 | ||
157 | streaming_playlists: { | |
158 | bucket_name: options.playlistBucket, | |
159 | prefix: options.playlistPrefix, | |
160 | base_url: baseMockUrl | |
161 | ? `${baseMockUrl}/${options.playlistBucket}` | |
162 | : undefined | |
163 | }, | |
164 | ||
165 | videos: { | |
166 | bucket_name: options.webtorrentBucket, | |
167 | prefix: options.webtorrentPrefix, | |
168 | base_url: baseMockUrl | |
169 | ? `${baseMockUrl}/${options.webtorrentBucket}` | |
170 | : undefined | |
171 | } | |
172 | } | |
173 | } | |
174 | ||
175 | servers = await createMultipleServers(2, config) | |
176 | ||
177 | await setAccessTokensToServers(servers) | |
178 | await doubleFollow(servers[0], servers[1]) | |
179 | ||
180 | for (const server of servers) { | |
181 | const { uuid } = await server.videos.quickUpload({ name: 'video to keep' }) | |
182 | await waitJobs(servers) | |
183 | ||
184 | const files = await server.videos.listFiles({ id: uuid }) | |
185 | keptUrls = keptUrls.concat(files.map(f => f.fileUrl)) | |
186 | } | |
187 | }) | |
188 | ||
189 | it('Should upload a video and move it to the object storage without transcoding', async function () { | |
190 | this.timeout(40000) | |
191 | ||
192 | const { uuid } = await servers[0].videos.quickUpload({ name: 'video 1', fixture }) | |
193 | uuidsToDelete.push(uuid) | |
194 | ||
195 | await waitJobs(servers) | |
196 | ||
197 | for (const server of servers) { | |
198 | const video = await server.videos.get({ id: uuid }) | |
199 | const files = await checkFiles({ ...options, video, baseMockUrl }) | |
200 | ||
201 | deletedUrls = deletedUrls.concat(files) | |
202 | } | |
203 | }) | |
204 | ||
205 | it('Should upload a video and move it to the object storage with transcoding', async function () { | |
206 | this.timeout(120000) | |
207 | ||
208 | const { uuid } = await servers[1].videos.quickUpload({ name: 'video 2', fixture }) | |
209 | uuidsToDelete.push(uuid) | |
210 | ||
211 | await waitJobs(servers) | |
212 | ||
213 | for (const server of servers) { | |
214 | const video = await server.videos.get({ id: uuid }) | |
215 | const files = await checkFiles({ ...options, video, baseMockUrl }) | |
216 | ||
217 | deletedUrls = deletedUrls.concat(files) | |
218 | } | |
219 | }) | |
220 | ||
221 | it('Should fetch correctly all the files', async function () { | |
222 | for (const url of deletedUrls.concat(keptUrls)) { | |
223 | await makeRawRequest({ url, expectedStatus: HttpStatusCode.OK_200 }) | |
224 | } | |
225 | }) | |
226 | ||
227 | it('Should correctly delete the files', async function () { | |
228 | await servers[0].videos.remove({ id: uuidsToDelete[0] }) | |
229 | await servers[1].videos.remove({ id: uuidsToDelete[1] }) | |
230 | ||
231 | await waitJobs(servers) | |
232 | ||
233 | for (const url of deletedUrls) { | |
234 | await makeRawRequest({ url, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) | |
235 | } | |
236 | }) | |
237 | ||
238 | it('Should have kept other files', async function () { | |
239 | for (const url of keptUrls) { | |
240 | await makeRawRequest({ url, expectedStatus: HttpStatusCode.OK_200 }) | |
241 | } | |
242 | }) | |
243 | ||
244 | it('Should have an empty tmp directory', async function () { | |
245 | for (const server of servers) { | |
246 | await checkTmpIsEmpty(server) | |
247 | } | |
248 | }) | |
249 | ||
250 | it('Should not have downloaded files from object storage', async function () { | |
251 | for (const server of servers) { | |
252 | await expectLogDoesNotContain(server, 'from object storage') | |
253 | } | |
254 | }) | |
255 | ||
256 | after(async function () { | |
257 | await mockObjectStorage.terminate() | |
258 | ||
259 | await cleanupTests(servers) | |
260 | }) | |
261 | } | |
262 | ||
263 | describe('Object storage for videos', function () { | |
264 | if (areMockObjectStorageTestsDisabled()) return | |
265 | ||
266 | describe('Test config', function () { | |
267 | let server: PeerTubeServer | |
268 | ||
269 | const baseConfig = { | |
270 | object_storage: { | |
271 | enabled: true, | |
272 | endpoint: 'http://' + ObjectStorageCommand.getMockEndpointHost(), | |
273 | region: ObjectStorageCommand.getMockRegion(), | |
274 | ||
275 | credentials: ObjectStorageCommand.getMockCredentialsConfig(), | |
276 | ||
277 | streaming_playlists: { | |
278 | bucket_name: ObjectStorageCommand.DEFAULT_PLAYLIST_MOCK_BUCKET | |
279 | }, | |
280 | ||
281 | videos: { | |
282 | bucket_name: ObjectStorageCommand.DEFAULT_WEBTORRENT_MOCK_BUCKET | |
283 | } | |
284 | } | |
285 | } | |
286 | ||
287 | const badCredentials = { | |
288 | access_key_id: 'AKIAIOSFODNN7EXAMPLE', | |
289 | secret_access_key: 'aJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY' | |
290 | } | |
291 | ||
292 | it('Should fail with same bucket names without prefix', function (done) { | |
293 | const config = merge({}, baseConfig, { | |
294 | object_storage: { | |
295 | streaming_playlists: { | |
296 | bucket_name: 'aaa' | |
297 | }, | |
298 | ||
299 | videos: { | |
300 | bucket_name: 'aaa' | |
301 | } | |
302 | } | |
303 | }) | |
304 | ||
305 | createSingleServer(1, config) | |
306 | .then(() => done(new Error('Did not throw'))) | |
307 | .catch(() => done()) | |
308 | }) | |
309 | ||
310 | it('Should fail with bad credentials', async function () { | |
311 | this.timeout(60000) | |
312 | ||
313 | await ObjectStorageCommand.prepareDefaultMockBuckets() | |
314 | ||
315 | const config = merge({}, baseConfig, { | |
316 | object_storage: { | |
317 | credentials: badCredentials | |
318 | } | |
319 | }) | |
320 | ||
321 | server = await createSingleServer(1, config) | |
322 | await setAccessTokensToServers([ server ]) | |
323 | ||
324 | const { uuid } = await server.videos.quickUpload({ name: 'video' }) | |
325 | ||
326 | await waitJobs([ server ], true) | |
327 | const video = await server.videos.get({ id: uuid }) | |
328 | ||
329 | expectStartWith(video.files[0].fileUrl, server.url) | |
330 | ||
331 | await killallServers([ server ]) | |
332 | }) | |
333 | ||
334 | it('Should succeed with credentials from env', async function () { | |
335 | this.timeout(60000) | |
336 | ||
337 | await ObjectStorageCommand.prepareDefaultMockBuckets() | |
338 | ||
339 | const config = merge({}, baseConfig, { | |
340 | object_storage: { | |
341 | credentials: { | |
342 | access_key_id: '', | |
343 | secret_access_key: '' | |
344 | } | |
345 | } | |
346 | }) | |
347 | ||
348 | const goodCredentials = ObjectStorageCommand.getMockCredentialsConfig() | |
349 | ||
350 | server = await createSingleServer(1, config, { | |
351 | env: { | |
352 | AWS_ACCESS_KEY_ID: goodCredentials.access_key_id, | |
353 | AWS_SECRET_ACCESS_KEY: goodCredentials.secret_access_key | |
354 | } | |
355 | }) | |
356 | ||
357 | await setAccessTokensToServers([ server ]) | |
358 | ||
359 | const { uuid } = await server.videos.quickUpload({ name: 'video' }) | |
360 | ||
361 | await waitJobs([ server ], true) | |
362 | const video = await server.videos.get({ id: uuid }) | |
363 | ||
364 | expectStartWith(video.files[0].fileUrl, ObjectStorageCommand.getMockWebTorrentBaseUrl()) | |
365 | }) | |
366 | ||
367 | after(async function () { | |
368 | await killallServers([ server ]) | |
369 | }) | |
370 | }) | |
371 | ||
372 | describe('Test simple object storage', function () { | |
373 | runTestSuite({ | |
374 | playlistBucket: 'streaming-playlists', | |
375 | webtorrentBucket: 'videos' | |
376 | }) | |
377 | }) | |
378 | ||
379 | describe('Test object storage with prefix', function () { | |
380 | runTestSuite({ | |
381 | playlistBucket: 'mybucket', | |
382 | webtorrentBucket: 'mybucket', | |
383 | ||
384 | playlistPrefix: 'streaming-playlists_', | |
385 | webtorrentPrefix: 'webtorrent_' | |
386 | }) | |
387 | }) | |
388 | ||
389 | describe('Test object storage with prefix and base URL', function () { | |
390 | runTestSuite({ | |
391 | playlistBucket: 'mybucket', | |
392 | webtorrentBucket: 'mybucket', | |
393 | ||
394 | playlistPrefix: 'streaming-playlists/', | |
395 | webtorrentPrefix: 'webtorrent/', | |
396 | ||
397 | useMockBaseUrl: true | |
398 | }) | |
399 | }) | |
400 | ||
401 | describe('Test object storage with file bigger than upload part', function () { | |
402 | let fixture: string | |
403 | const maxUploadPart = '5MB' | |
404 | ||
405 | before(async function () { | |
406 | this.timeout(120000) | |
407 | ||
408 | fixture = await generateHighBitrateVideo() | |
409 | ||
410 | const { size } = await stat(fixture) | |
411 | ||
412 | if (bytes.parse(maxUploadPart) > size) { | |
413 | throw Error(`Fixture file is too small (${size}) to make sense for this test.`) | |
414 | } | |
415 | }) | |
416 | ||
417 | runTestSuite({ | |
418 | maxUploadPart, | |
419 | playlistBucket: 'streaming-playlists', | |
420 | webtorrentBucket: 'videos', | |
421 | fixture | |
422 | }) | |
423 | }) | |
424 | }) |