]>
Commit | Line | Data |
---|---|---|
0305db28 JB |
1 | /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ |
2 | ||
3 | import 'mocha' | |
4 | import * as chai from 'chai' | |
5 | import { merge } from 'lodash' | |
6 | import { | |
7 | areObjectStorageTestsDisabled, | |
8 | checkTmpIsEmpty, | |
9 | cleanupTests, | |
10 | createMultipleServers, | |
11 | createSingleServer, | |
12 | doubleFollow, | |
13 | expectStartWith, | |
14 | killallServers, | |
15 | makeRawRequest, | |
16 | MockObjectStorage, | |
17 | ObjectStorageCommand, | |
18 | PeerTubeServer, | |
19 | setAccessTokensToServers, | |
20 | waitJobs, | |
21 | webtorrentAdd | |
22 | } from '@shared/extra-utils' | |
23 | import { HttpStatusCode, VideoDetails } from '@shared/models' | |
24 | ||
25 | const expect = chai.expect | |
26 | ||
27 | async function checkFiles (options: { | |
28 | video: VideoDetails | |
29 | ||
30 | baseMockUrl?: string | |
31 | ||
32 | playlistBucket: string | |
33 | playlistPrefix?: string | |
34 | ||
35 | webtorrentBucket: string | |
36 | webtorrentPrefix?: string | |
37 | }) { | |
38 | const { | |
39 | video, | |
40 | playlistBucket, | |
41 | webtorrentBucket, | |
42 | baseMockUrl, | |
43 | playlistPrefix, | |
44 | webtorrentPrefix | |
45 | } = options | |
46 | ||
47 | let allFiles = video.files | |
48 | ||
49 | for (const file of video.files) { | |
50 | const baseUrl = baseMockUrl | |
51 | ? `${baseMockUrl}/${webtorrentBucket}/` | |
52 | : `http://${webtorrentBucket}.${ObjectStorageCommand.getEndpointHost()}/` | |
53 | ||
54 | const prefix = webtorrentPrefix || '' | |
55 | const start = baseUrl + prefix | |
56 | ||
57 | expectStartWith(file.fileUrl, start) | |
58 | ||
59 | const res = await makeRawRequest(file.fileDownloadUrl, HttpStatusCode.FOUND_302) | |
60 | const location = res.headers['location'] | |
61 | expectStartWith(location, start) | |
62 | ||
63 | await makeRawRequest(location, HttpStatusCode.OK_200) | |
64 | } | |
65 | ||
66 | const hls = video.streamingPlaylists[0] | |
67 | ||
68 | if (hls) { | |
69 | allFiles = allFiles.concat(hls.files) | |
70 | ||
71 | const baseUrl = baseMockUrl | |
72 | ? `${baseMockUrl}/${playlistBucket}/` | |
73 | : `http://${playlistBucket}.${ObjectStorageCommand.getEndpointHost()}/` | |
74 | ||
75 | const prefix = playlistPrefix || '' | |
76 | const start = baseUrl + prefix | |
77 | ||
78 | expectStartWith(hls.playlistUrl, start) | |
79 | expectStartWith(hls.segmentsSha256Url, start) | |
80 | ||
81 | await makeRawRequest(hls.playlistUrl, HttpStatusCode.OK_200) | |
82 | ||
83 | const resSha = await makeRawRequest(hls.segmentsSha256Url, HttpStatusCode.OK_200) | |
84 | expect(JSON.stringify(resSha.body)).to.not.throw | |
85 | ||
86 | for (const file of hls.files) { | |
87 | expectStartWith(file.fileUrl, start) | |
88 | ||
89 | const res = await makeRawRequest(file.fileDownloadUrl, HttpStatusCode.FOUND_302) | |
90 | const location = res.headers['location'] | |
91 | expectStartWith(location, start) | |
92 | ||
93 | await makeRawRequest(location, HttpStatusCode.OK_200) | |
94 | } | |
95 | } | |
96 | ||
97 | for (const file of allFiles) { | |
98 | const torrent = await webtorrentAdd(file.magnetUri, true) | |
99 | ||
100 | expect(torrent.files).to.be.an('array') | |
101 | expect(torrent.files.length).to.equal(1) | |
102 | expect(torrent.files[0].path).to.exist.and.to.not.equal('') | |
103 | ||
104 | const res = await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200) | |
105 | expect(res.body).to.have.length.above(100) | |
106 | } | |
107 | ||
108 | return allFiles.map(f => f.fileUrl) | |
109 | } | |
110 | ||
111 | function runTestSuite (options: { | |
112 | playlistBucket: string | |
113 | playlistPrefix?: string | |
114 | ||
115 | webtorrentBucket: string | |
116 | webtorrentPrefix?: string | |
117 | ||
118 | useMockBaseUrl?: boolean | |
119 | ||
120 | maxUploadPart?: string | |
121 | }) { | |
122 | const mockObjectStorage = new MockObjectStorage() | |
123 | let baseMockUrl: string | |
124 | ||
125 | let servers: PeerTubeServer[] | |
126 | ||
127 | let keptUrls: string[] = [] | |
128 | ||
129 | const uuidsToDelete: string[] = [] | |
130 | let deletedUrls: string[] = [] | |
131 | ||
132 | before(async function () { | |
133 | this.timeout(120000) | |
134 | ||
135 | const port = await mockObjectStorage.initialize() | |
136 | baseMockUrl = options.useMockBaseUrl ? `http://localhost:${port}` : undefined | |
137 | ||
138 | await ObjectStorageCommand.createBucket(options.playlistBucket) | |
139 | await ObjectStorageCommand.createBucket(options.webtorrentBucket) | |
140 | ||
141 | const config = { | |
142 | object_storage: { | |
143 | enabled: true, | |
144 | endpoint: 'http://' + ObjectStorageCommand.getEndpointHost(), | |
145 | region: ObjectStorageCommand.getRegion(), | |
146 | ||
147 | credentials: ObjectStorageCommand.getCredentialsConfig(), | |
148 | ||
149 | max_upload_part: options.maxUploadPart || '2MB', | |
150 | ||
151 | streaming_playlists: { | |
152 | bucket_name: options.playlistBucket, | |
153 | prefix: options.playlistPrefix, | |
154 | base_url: baseMockUrl | |
155 | ? `${baseMockUrl}/${options.playlistBucket}` | |
156 | : undefined | |
157 | }, | |
158 | ||
159 | videos: { | |
160 | bucket_name: options.webtorrentBucket, | |
161 | prefix: options.webtorrentPrefix, | |
162 | base_url: baseMockUrl | |
163 | ? `${baseMockUrl}/${options.webtorrentBucket}` | |
164 | : undefined | |
165 | } | |
166 | } | |
167 | } | |
168 | ||
169 | servers = await createMultipleServers(2, config) | |
170 | ||
171 | await setAccessTokensToServers(servers) | |
172 | await doubleFollow(servers[0], servers[1]) | |
173 | ||
174 | for (const server of servers) { | |
175 | const { uuid } = await server.videos.quickUpload({ name: 'video to keep' }) | |
176 | await waitJobs(servers) | |
177 | ||
178 | const files = await server.videos.listFiles({ id: uuid }) | |
179 | keptUrls = keptUrls.concat(files.map(f => f.fileUrl)) | |
180 | } | |
181 | }) | |
182 | ||
183 | it('Should upload a video and move it to the object storage without transcoding', async function () { | |
184 | this.timeout(20000) | |
185 | ||
186 | const { uuid } = await servers[0].videos.quickUpload({ name: 'video 1' }) | |
187 | uuidsToDelete.push(uuid) | |
188 | ||
189 | await waitJobs(servers) | |
190 | ||
191 | for (const server of servers) { | |
192 | const video = await server.videos.get({ id: uuid }) | |
193 | const files = await checkFiles({ ...options, video, baseMockUrl }) | |
194 | ||
195 | deletedUrls = deletedUrls.concat(files) | |
196 | } | |
197 | }) | |
198 | ||
199 | it('Should upload a video and move it to the object storage with transcoding', async function () { | |
200 | this.timeout(40000) | |
201 | ||
202 | const { uuid } = await servers[1].videos.quickUpload({ name: 'video 2' }) | |
203 | uuidsToDelete.push(uuid) | |
204 | ||
205 | await waitJobs(servers) | |
206 | ||
207 | for (const server of servers) { | |
208 | const video = await server.videos.get({ id: uuid }) | |
209 | const files = await checkFiles({ ...options, video, baseMockUrl }) | |
210 | ||
211 | deletedUrls = deletedUrls.concat(files) | |
212 | } | |
213 | }) | |
214 | ||
215 | it('Should correctly delete the files', async function () { | |
216 | await servers[0].videos.remove({ id: uuidsToDelete[0] }) | |
217 | await servers[1].videos.remove({ id: uuidsToDelete[1] }) | |
218 | ||
219 | await waitJobs(servers) | |
220 | ||
221 | for (const url of deletedUrls) { | |
222 | await makeRawRequest(url, HttpStatusCode.NOT_FOUND_404) | |
223 | } | |
224 | }) | |
225 | ||
226 | it('Should have kept other files', async function () { | |
227 | for (const url of keptUrls) { | |
228 | await makeRawRequest(url, HttpStatusCode.OK_200) | |
229 | } | |
230 | }) | |
231 | ||
232 | it('Should have an empty tmp directory', async function () { | |
233 | for (const server of servers) { | |
234 | await checkTmpIsEmpty(server) | |
235 | } | |
236 | }) | |
237 | ||
238 | after(async function () { | |
239 | mockObjectStorage.terminate() | |
240 | ||
241 | await cleanupTests(servers) | |
242 | }) | |
243 | } | |
244 | ||
245 | describe('Object storage for videos', function () { | |
246 | if (areObjectStorageTestsDisabled()) return | |
247 | ||
248 | describe('Test config', function () { | |
249 | let server: PeerTubeServer | |
250 | ||
251 | const baseConfig = { | |
252 | object_storage: { | |
253 | enabled: true, | |
254 | endpoint: 'http://' + ObjectStorageCommand.getEndpointHost(), | |
255 | region: ObjectStorageCommand.getRegion(), | |
256 | ||
257 | credentials: ObjectStorageCommand.getCredentialsConfig(), | |
258 | ||
259 | streaming_playlists: { | |
260 | bucket_name: ObjectStorageCommand.DEFAULT_PLAYLIST_BUCKET | |
261 | }, | |
262 | ||
263 | videos: { | |
264 | bucket_name: ObjectStorageCommand.DEFAULT_WEBTORRENT_BUCKET | |
265 | } | |
266 | } | |
267 | } | |
268 | ||
269 | const badCredentials = { | |
270 | access_key_id: 'AKIAIOSFODNN7EXAMPLE', | |
271 | secret_access_key: 'aJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY' | |
272 | } | |
273 | ||
274 | it('Should fail with same bucket names without prefix', function (done) { | |
275 | const config = merge({}, baseConfig, { | |
276 | object_storage: { | |
277 | streaming_playlists: { | |
278 | bucket_name: 'aaa' | |
279 | }, | |
280 | ||
281 | videos: { | |
282 | bucket_name: 'aaa' | |
283 | } | |
284 | } | |
285 | }) | |
286 | ||
287 | createSingleServer(1, config) | |
288 | .then(() => done(new Error('Did not throw'))) | |
289 | .catch(() => done()) | |
290 | }) | |
291 | ||
292 | it('Should fail with bad credentials', async function () { | |
293 | this.timeout(60000) | |
294 | ||
295 | await ObjectStorageCommand.prepareDefaultBuckets() | |
296 | ||
297 | const config = merge({}, baseConfig, { | |
298 | object_storage: { | |
299 | credentials: badCredentials | |
300 | } | |
301 | }) | |
302 | ||
303 | server = await createSingleServer(1, config) | |
304 | await setAccessTokensToServers([ server ]) | |
305 | ||
306 | const { uuid } = await server.videos.quickUpload({ name: 'video' }) | |
307 | ||
308 | await waitJobs([ server ], true) | |
309 | const video = await server.videos.get({ id: uuid }) | |
310 | ||
311 | expectStartWith(video.files[0].fileUrl, server.url) | |
312 | ||
313 | await killallServers([ server ]) | |
314 | }) | |
315 | ||
316 | it('Should succeed with credentials from env', async function () { | |
317 | this.timeout(60000) | |
318 | ||
319 | await ObjectStorageCommand.prepareDefaultBuckets() | |
320 | ||
321 | const config = merge({}, baseConfig, { | |
322 | object_storage: { | |
323 | credentials: { | |
324 | access_key_id: '', | |
325 | secret_access_key: '' | |
326 | } | |
327 | } | |
328 | }) | |
329 | ||
330 | const goodCredentials = ObjectStorageCommand.getCredentialsConfig() | |
331 | ||
332 | server = await createSingleServer(1, config, { | |
333 | env: { | |
334 | AWS_ACCESS_KEY_ID: goodCredentials.access_key_id, | |
335 | AWS_SECRET_ACCESS_KEY: goodCredentials.secret_access_key | |
336 | } | |
337 | }) | |
338 | ||
339 | await setAccessTokensToServers([ server ]) | |
340 | ||
341 | const { uuid } = await server.videos.quickUpload({ name: 'video' }) | |
342 | ||
343 | await waitJobs([ server ], true) | |
344 | const video = await server.videos.get({ id: uuid }) | |
345 | ||
346 | expectStartWith(video.files[0].fileUrl, ObjectStorageCommand.getWebTorrentBaseUrl()) | |
347 | }) | |
348 | ||
349 | after(async function () { | |
350 | await killallServers([ server ]) | |
351 | }) | |
352 | }) | |
353 | ||
354 | describe('Test simple object storage', function () { | |
355 | runTestSuite({ | |
356 | playlistBucket: 'streaming-playlists', | |
357 | webtorrentBucket: 'videos' | |
358 | }) | |
359 | }) | |
360 | ||
361 | describe('Test object storage with prefix', function () { | |
362 | runTestSuite({ | |
363 | playlistBucket: 'mybucket', | |
364 | webtorrentBucket: 'mybucket', | |
365 | ||
366 | playlistPrefix: 'streaming-playlists_', | |
367 | webtorrentPrefix: 'webtorrent_' | |
368 | }) | |
369 | }) | |
370 | ||
371 | describe('Test object storage with prefix and base URL', function () { | |
372 | runTestSuite({ | |
373 | playlistBucket: 'mybucket', | |
374 | webtorrentBucket: 'mybucket', | |
375 | ||
376 | playlistPrefix: 'streaming-playlists_', | |
377 | webtorrentPrefix: 'webtorrent_', | |
378 | ||
379 | useMockBaseUrl: true | |
380 | }) | |
381 | }) | |
382 | ||
383 | describe('Test object storage with small upload part', function () { | |
384 | runTestSuite({ | |
385 | playlistBucket: 'streaming-playlists', | |
386 | webtorrentBucket: 'videos', | |
387 | ||
388 | maxUploadPart: '5KB' | |
389 | }) | |
390 | }) | |
391 | }) |