]>
Commit | Line | Data |
---|---|---|
0305db28 JB |
1 | /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ |
2 | ||
3 | import 'mocha' | |
4 | import * as chai from 'chai' | |
5 | import { merge } from 'lodash' | |
c55e3d72 C |
6 | import { checkTmpIsEmpty, expectLogDoesNotContain, expectStartWith, MockObjectStorage } from '@server/tests/shared' |
7 | import { areObjectStorageTestsDisabled } from '@shared/core-utils' | |
8 | import { HttpStatusCode, VideoDetails } from '@shared/models' | |
0305db28 | 9 | import { |
0305db28 JB |
10 | cleanupTests, |
11 | createMultipleServers, | |
12 | createSingleServer, | |
13 | doubleFollow, | |
0305db28 JB |
14 | killallServers, |
15 | makeRawRequest, | |
0305db28 JB |
16 | ObjectStorageCommand, |
17 | PeerTubeServer, | |
18 | setAccessTokensToServers, | |
19 | waitJobs, | |
20 | webtorrentAdd | |
bf54587a | 21 | } from '@shared/server-commands' |
0305db28 JB |
22 | |
23 | const expect = chai.expect | |
24 | ||
25 | async function checkFiles (options: { | |
26 | video: VideoDetails | |
27 | ||
28 | baseMockUrl?: string | |
29 | ||
30 | playlistBucket: string | |
31 | playlistPrefix?: string | |
32 | ||
33 | webtorrentBucket: string | |
34 | webtorrentPrefix?: string | |
35 | }) { | |
36 | const { | |
37 | video, | |
38 | playlistBucket, | |
39 | webtorrentBucket, | |
40 | baseMockUrl, | |
41 | playlistPrefix, | |
42 | webtorrentPrefix | |
43 | } = options | |
44 | ||
45 | let allFiles = video.files | |
46 | ||
47 | for (const file of video.files) { | |
48 | const baseUrl = baseMockUrl | |
49 | ? `${baseMockUrl}/${webtorrentBucket}/` | |
50 | : `http://${webtorrentBucket}.${ObjectStorageCommand.getEndpointHost()}/` | |
51 | ||
52 | const prefix = webtorrentPrefix || '' | |
53 | const start = baseUrl + prefix | |
54 | ||
55 | expectStartWith(file.fileUrl, start) | |
56 | ||
57 | const res = await makeRawRequest(file.fileDownloadUrl, HttpStatusCode.FOUND_302) | |
58 | const location = res.headers['location'] | |
59 | expectStartWith(location, start) | |
60 | ||
61 | await makeRawRequest(location, HttpStatusCode.OK_200) | |
62 | } | |
63 | ||
64 | const hls = video.streamingPlaylists[0] | |
65 | ||
66 | if (hls) { | |
67 | allFiles = allFiles.concat(hls.files) | |
68 | ||
69 | const baseUrl = baseMockUrl | |
70 | ? `${baseMockUrl}/${playlistBucket}/` | |
71 | : `http://${playlistBucket}.${ObjectStorageCommand.getEndpointHost()}/` | |
72 | ||
73 | const prefix = playlistPrefix || '' | |
74 | const start = baseUrl + prefix | |
75 | ||
76 | expectStartWith(hls.playlistUrl, start) | |
77 | expectStartWith(hls.segmentsSha256Url, start) | |
78 | ||
79 | await makeRawRequest(hls.playlistUrl, HttpStatusCode.OK_200) | |
80 | ||
81 | const resSha = await makeRawRequest(hls.segmentsSha256Url, HttpStatusCode.OK_200) | |
82 | expect(JSON.stringify(resSha.body)).to.not.throw | |
83 | ||
84 | for (const file of hls.files) { | |
85 | expectStartWith(file.fileUrl, start) | |
86 | ||
87 | const res = await makeRawRequest(file.fileDownloadUrl, HttpStatusCode.FOUND_302) | |
88 | const location = res.headers['location'] | |
89 | expectStartWith(location, start) | |
90 | ||
91 | await makeRawRequest(location, HttpStatusCode.OK_200) | |
92 | } | |
93 | } | |
94 | ||
95 | for (const file of allFiles) { | |
96 | const torrent = await webtorrentAdd(file.magnetUri, true) | |
97 | ||
98 | expect(torrent.files).to.be.an('array') | |
99 | expect(torrent.files.length).to.equal(1) | |
100 | expect(torrent.files[0].path).to.exist.and.to.not.equal('') | |
101 | ||
102 | const res = await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200) | |
103 | expect(res.body).to.have.length.above(100) | |
104 | } | |
105 | ||
106 | return allFiles.map(f => f.fileUrl) | |
107 | } | |
108 | ||
109 | function runTestSuite (options: { | |
110 | playlistBucket: string | |
111 | playlistPrefix?: string | |
112 | ||
113 | webtorrentBucket: string | |
114 | webtorrentPrefix?: string | |
115 | ||
116 | useMockBaseUrl?: boolean | |
117 | ||
118 | maxUploadPart?: string | |
119 | }) { | |
120 | const mockObjectStorage = new MockObjectStorage() | |
121 | let baseMockUrl: string | |
122 | ||
123 | let servers: PeerTubeServer[] | |
124 | ||
125 | let keptUrls: string[] = [] | |
126 | ||
127 | const uuidsToDelete: string[] = [] | |
128 | let deletedUrls: string[] = [] | |
129 | ||
130 | before(async function () { | |
131 | this.timeout(120000) | |
132 | ||
133 | const port = await mockObjectStorage.initialize() | |
134 | baseMockUrl = options.useMockBaseUrl ? `http://localhost:${port}` : undefined | |
135 | ||
136 | await ObjectStorageCommand.createBucket(options.playlistBucket) | |
137 | await ObjectStorageCommand.createBucket(options.webtorrentBucket) | |
138 | ||
139 | const config = { | |
140 | object_storage: { | |
141 | enabled: true, | |
142 | endpoint: 'http://' + ObjectStorageCommand.getEndpointHost(), | |
143 | region: ObjectStorageCommand.getRegion(), | |
144 | ||
145 | credentials: ObjectStorageCommand.getCredentialsConfig(), | |
146 | ||
147 | max_upload_part: options.maxUploadPart || '2MB', | |
148 | ||
149 | streaming_playlists: { | |
150 | bucket_name: options.playlistBucket, | |
151 | prefix: options.playlistPrefix, | |
152 | base_url: baseMockUrl | |
153 | ? `${baseMockUrl}/${options.playlistBucket}` | |
154 | : undefined | |
155 | }, | |
156 | ||
157 | videos: { | |
158 | bucket_name: options.webtorrentBucket, | |
159 | prefix: options.webtorrentPrefix, | |
160 | base_url: baseMockUrl | |
161 | ? `${baseMockUrl}/${options.webtorrentBucket}` | |
162 | : undefined | |
163 | } | |
164 | } | |
165 | } | |
166 | ||
167 | servers = await createMultipleServers(2, config) | |
168 | ||
169 | await setAccessTokensToServers(servers) | |
170 | await doubleFollow(servers[0], servers[1]) | |
171 | ||
172 | for (const server of servers) { | |
173 | const { uuid } = await server.videos.quickUpload({ name: 'video to keep' }) | |
174 | await waitJobs(servers) | |
175 | ||
176 | const files = await server.videos.listFiles({ id: uuid }) | |
177 | keptUrls = keptUrls.concat(files.map(f => f.fileUrl)) | |
178 | } | |
179 | }) | |
180 | ||
181 | it('Should upload a video and move it to the object storage without transcoding', async function () { | |
d1bfbdeb | 182 | this.timeout(40000) |
0305db28 JB |
183 | |
184 | const { uuid } = await servers[0].videos.quickUpload({ name: 'video 1' }) | |
185 | uuidsToDelete.push(uuid) | |
186 | ||
187 | await waitJobs(servers) | |
188 | ||
189 | for (const server of servers) { | |
190 | const video = await server.videos.get({ id: uuid }) | |
191 | const files = await checkFiles({ ...options, video, baseMockUrl }) | |
192 | ||
193 | deletedUrls = deletedUrls.concat(files) | |
194 | } | |
195 | }) | |
196 | ||
197 | it('Should upload a video and move it to the object storage with transcoding', async function () { | |
1ffb5fb6 | 198 | this.timeout(120000) |
0305db28 JB |
199 | |
200 | const { uuid } = await servers[1].videos.quickUpload({ name: 'video 2' }) | |
201 | uuidsToDelete.push(uuid) | |
202 | ||
203 | await waitJobs(servers) | |
204 | ||
205 | for (const server of servers) { | |
206 | const video = await server.videos.get({ id: uuid }) | |
207 | const files = await checkFiles({ ...options, video, baseMockUrl }) | |
208 | ||
209 | deletedUrls = deletedUrls.concat(files) | |
210 | } | |
211 | }) | |
212 | ||
87699a09 C |
213 | it('Should fetch correctly all the files', async function () { |
214 | for (const url of deletedUrls.concat(keptUrls)) { | |
215 | await makeRawRequest(url, HttpStatusCode.OK_200) | |
216 | } | |
217 | }) | |
218 | ||
0305db28 JB |
219 | it('Should correctly delete the files', async function () { |
220 | await servers[0].videos.remove({ id: uuidsToDelete[0] }) | |
221 | await servers[1].videos.remove({ id: uuidsToDelete[1] }) | |
222 | ||
223 | await waitJobs(servers) | |
224 | ||
225 | for (const url of deletedUrls) { | |
226 | await makeRawRequest(url, HttpStatusCode.NOT_FOUND_404) | |
227 | } | |
228 | }) | |
229 | ||
230 | it('Should have kept other files', async function () { | |
231 | for (const url of keptUrls) { | |
232 | await makeRawRequest(url, HttpStatusCode.OK_200) | |
233 | } | |
234 | }) | |
235 | ||
236 | it('Should have an empty tmp directory', async function () { | |
237 | for (const server of servers) { | |
238 | await checkTmpIsEmpty(server) | |
239 | } | |
240 | }) | |
241 | ||
1f6125be C |
242 | it('Should not have downloaded files from object storage', async function () { |
243 | for (const server of servers) { | |
244 | await expectLogDoesNotContain(server, 'from object storage') | |
245 | } | |
246 | }) | |
247 | ||
0305db28 | 248 | after(async function () { |
70430c27 | 249 | await mockObjectStorage.terminate() |
0305db28 JB |
250 | |
251 | await cleanupTests(servers) | |
252 | }) | |
253 | } | |
254 | ||
255 | describe('Object storage for videos', function () { | |
256 | if (areObjectStorageTestsDisabled()) return | |
257 | ||
258 | describe('Test config', function () { | |
259 | let server: PeerTubeServer | |
260 | ||
261 | const baseConfig = { | |
262 | object_storage: { | |
263 | enabled: true, | |
264 | endpoint: 'http://' + ObjectStorageCommand.getEndpointHost(), | |
265 | region: ObjectStorageCommand.getRegion(), | |
266 | ||
267 | credentials: ObjectStorageCommand.getCredentialsConfig(), | |
268 | ||
269 | streaming_playlists: { | |
270 | bucket_name: ObjectStorageCommand.DEFAULT_PLAYLIST_BUCKET | |
271 | }, | |
272 | ||
273 | videos: { | |
274 | bucket_name: ObjectStorageCommand.DEFAULT_WEBTORRENT_BUCKET | |
275 | } | |
276 | } | |
277 | } | |
278 | ||
279 | const badCredentials = { | |
280 | access_key_id: 'AKIAIOSFODNN7EXAMPLE', | |
281 | secret_access_key: 'aJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY' | |
282 | } | |
283 | ||
284 | it('Should fail with same bucket names without prefix', function (done) { | |
285 | const config = merge({}, baseConfig, { | |
286 | object_storage: { | |
287 | streaming_playlists: { | |
288 | bucket_name: 'aaa' | |
289 | }, | |
290 | ||
291 | videos: { | |
292 | bucket_name: 'aaa' | |
293 | } | |
294 | } | |
295 | }) | |
296 | ||
297 | createSingleServer(1, config) | |
298 | .then(() => done(new Error('Did not throw'))) | |
299 | .catch(() => done()) | |
300 | }) | |
301 | ||
302 | it('Should fail with bad credentials', async function () { | |
303 | this.timeout(60000) | |
304 | ||
305 | await ObjectStorageCommand.prepareDefaultBuckets() | |
306 | ||
307 | const config = merge({}, baseConfig, { | |
308 | object_storage: { | |
309 | credentials: badCredentials | |
310 | } | |
311 | }) | |
312 | ||
313 | server = await createSingleServer(1, config) | |
314 | await setAccessTokensToServers([ server ]) | |
315 | ||
316 | const { uuid } = await server.videos.quickUpload({ name: 'video' }) | |
317 | ||
318 | await waitJobs([ server ], true) | |
319 | const video = await server.videos.get({ id: uuid }) | |
320 | ||
321 | expectStartWith(video.files[0].fileUrl, server.url) | |
322 | ||
323 | await killallServers([ server ]) | |
324 | }) | |
325 | ||
326 | it('Should succeed with credentials from env', async function () { | |
327 | this.timeout(60000) | |
328 | ||
329 | await ObjectStorageCommand.prepareDefaultBuckets() | |
330 | ||
331 | const config = merge({}, baseConfig, { | |
332 | object_storage: { | |
333 | credentials: { | |
334 | access_key_id: '', | |
335 | secret_access_key: '' | |
336 | } | |
337 | } | |
338 | }) | |
339 | ||
340 | const goodCredentials = ObjectStorageCommand.getCredentialsConfig() | |
341 | ||
342 | server = await createSingleServer(1, config, { | |
343 | env: { | |
344 | AWS_ACCESS_KEY_ID: goodCredentials.access_key_id, | |
345 | AWS_SECRET_ACCESS_KEY: goodCredentials.secret_access_key | |
346 | } | |
347 | }) | |
348 | ||
349 | await setAccessTokensToServers([ server ]) | |
350 | ||
351 | const { uuid } = await server.videos.quickUpload({ name: 'video' }) | |
352 | ||
353 | await waitJobs([ server ], true) | |
354 | const video = await server.videos.get({ id: uuid }) | |
355 | ||
356 | expectStartWith(video.files[0].fileUrl, ObjectStorageCommand.getWebTorrentBaseUrl()) | |
357 | }) | |
358 | ||
359 | after(async function () { | |
360 | await killallServers([ server ]) | |
361 | }) | |
362 | }) | |
363 | ||
364 | describe('Test simple object storage', function () { | |
365 | runTestSuite({ | |
366 | playlistBucket: 'streaming-playlists', | |
367 | webtorrentBucket: 'videos' | |
368 | }) | |
369 | }) | |
370 | ||
371 | describe('Test object storage with prefix', function () { | |
372 | runTestSuite({ | |
373 | playlistBucket: 'mybucket', | |
374 | webtorrentBucket: 'mybucket', | |
375 | ||
376 | playlistPrefix: 'streaming-playlists_', | |
377 | webtorrentPrefix: 'webtorrent_' | |
378 | }) | |
379 | }) | |
380 | ||
381 | describe('Test object storage with prefix and base URL', function () { | |
382 | runTestSuite({ | |
383 | playlistBucket: 'mybucket', | |
384 | webtorrentBucket: 'mybucket', | |
385 | ||
70430c27 C |
386 | playlistPrefix: 'streaming-playlists/', |
387 | webtorrentPrefix: 'webtorrent/', | |
0305db28 JB |
388 | |
389 | useMockBaseUrl: true | |
390 | }) | |
391 | }) | |
392 | ||
393 | describe('Test object storage with small upload part', function () { | |
394 | runTestSuite({ | |
395 | playlistBucket: 'streaming-playlists', | |
396 | webtorrentBucket: 'videos', | |
397 | ||
398 | maxUploadPart: '5KB' | |
399 | }) | |
400 | }) | |
401 | }) |