aboutsummaryrefslogtreecommitdiffhomepage
path: root/server/tests/api/object-storage/videos.ts
diff options
context:
space:
mode:
Diffstat (limited to 'server/tests/api/object-storage/videos.ts')
-rw-r--r--server/tests/api/object-storage/videos.ts438
1 files changed, 0 insertions, 438 deletions
diff --git a/server/tests/api/object-storage/videos.ts b/server/tests/api/object-storage/videos.ts
deleted file mode 100644
index dcc52ef06..000000000
--- a/server/tests/api/object-storage/videos.ts
+++ /dev/null
@@ -1,438 +0,0 @@
1/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
2
3import bytes from 'bytes'
4import { expect } from 'chai'
5import { stat } from 'fs-extra'
6import { merge } from 'lodash'
7import {
8 checkTmpIsEmpty,
9 checkWebTorrentWorks,
10 expectLogDoesNotContain,
11 expectStartWith,
12 generateHighBitrateVideo,
13 MockObjectStorageProxy,
14 SQLCommand
15} from '@server/tests/shared'
16import { areMockObjectStorageTestsDisabled } from '@shared/core-utils'
17import { sha1 } from '@shared/extra-utils'
18import { HttpStatusCode, VideoDetails } from '@shared/models'
19import {
20 cleanupTests,
21 createMultipleServers,
22 createSingleServer,
23 doubleFollow,
24 killallServers,
25 makeRawRequest,
26 ObjectStorageCommand,
27 PeerTubeServer,
28 setAccessTokensToServers,
29 waitJobs
30} from '@shared/server-commands'
31
32async function checkFiles (options: {
33 server: PeerTubeServer
34 originServer: PeerTubeServer
35 originSQLCommand: SQLCommand
36
37 video: VideoDetails
38
39 baseMockUrl?: string
40
41 playlistBucket: string
42 playlistPrefix?: string
43
44 webVideoBucket: string
45 webVideoPrefix?: string
46}) {
47 const {
48 server,
49 originServer,
50 originSQLCommand,
51 video,
52 playlistBucket,
53 webVideoBucket,
54 baseMockUrl,
55 playlistPrefix,
56 webVideoPrefix
57 } = options
58
59 let allFiles = video.files
60
61 for (const file of video.files) {
62 const baseUrl = baseMockUrl
63 ? `${baseMockUrl}/${webVideoBucket}/`
64 : `http://${webVideoBucket}.${ObjectStorageCommand.getMockEndpointHost()}/`
65
66 const prefix = webVideoPrefix || ''
67 const start = baseUrl + prefix
68
69 expectStartWith(file.fileUrl, start)
70
71 const res = await makeRawRequest({ url: file.fileDownloadUrl, expectedStatus: HttpStatusCode.FOUND_302 })
72 const location = res.headers['location']
73 expectStartWith(location, start)
74
75 await makeRawRequest({ url: location, expectedStatus: HttpStatusCode.OK_200 })
76 }
77
78 const hls = video.streamingPlaylists[0]
79
80 if (hls) {
81 allFiles = allFiles.concat(hls.files)
82
83 const baseUrl = baseMockUrl
84 ? `${baseMockUrl}/${playlistBucket}/`
85 : `http://${playlistBucket}.${ObjectStorageCommand.getMockEndpointHost()}/`
86
87 const prefix = playlistPrefix || ''
88 const start = baseUrl + prefix
89
90 expectStartWith(hls.playlistUrl, start)
91 expectStartWith(hls.segmentsSha256Url, start)
92
93 await makeRawRequest({ url: hls.playlistUrl, expectedStatus: HttpStatusCode.OK_200 })
94
95 const resSha = await makeRawRequest({ url: hls.segmentsSha256Url, expectedStatus: HttpStatusCode.OK_200 })
96 expect(JSON.stringify(resSha.body)).to.not.throw
97
98 let i = 0
99 for (const file of hls.files) {
100 expectStartWith(file.fileUrl, start)
101
102 const res = await makeRawRequest({ url: file.fileDownloadUrl, expectedStatus: HttpStatusCode.FOUND_302 })
103 const location = res.headers['location']
104 expectStartWith(location, start)
105
106 await makeRawRequest({ url: location, expectedStatus: HttpStatusCode.OK_200 })
107
108 if (originServer.internalServerNumber === server.internalServerNumber) {
109 const infohash = sha1(`${2 + hls.playlistUrl}+V${i}`)
110 const dbInfohashes = await originSQLCommand.getPlaylistInfohash(hls.id)
111
112 expect(dbInfohashes).to.include(infohash)
113 }
114
115 i++
116 }
117 }
118
119 for (const file of allFiles) {
120 await checkWebTorrentWorks(file.magnetUri)
121
122 const res = await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 })
123 expect(res.body).to.have.length.above(100)
124 }
125
126 return allFiles.map(f => f.fileUrl)
127}
128
129function runTestSuite (options: {
130 fixture?: string
131
132 maxUploadPart?: string
133
134 playlistBucket: string
135 playlistPrefix?: string
136
137 webVideoBucket: string
138 webVideoPrefix?: string
139
140 useMockBaseUrl?: boolean
141}) {
142 const mockObjectStorageProxy = new MockObjectStorageProxy()
143 const { fixture } = options
144 let baseMockUrl: string
145
146 let servers: PeerTubeServer[]
147 let sqlCommands: SQLCommand[] = []
148 const objectStorage = new ObjectStorageCommand()
149
150 let keptUrls: string[] = []
151
152 const uuidsToDelete: string[] = []
153 let deletedUrls: string[] = []
154
155 before(async function () {
156 this.timeout(240000)
157
158 const port = await mockObjectStorageProxy.initialize()
159 baseMockUrl = options.useMockBaseUrl
160 ? `http://127.0.0.1:${port}`
161 : undefined
162
163 await objectStorage.createMockBucket(options.playlistBucket)
164 await objectStorage.createMockBucket(options.webVideoBucket)
165
166 const config = {
167 object_storage: {
168 enabled: true,
169 endpoint: 'http://' + ObjectStorageCommand.getMockEndpointHost(),
170 region: ObjectStorageCommand.getMockRegion(),
171
172 credentials: ObjectStorageCommand.getMockCredentialsConfig(),
173
174 max_upload_part: options.maxUploadPart || '5MB',
175
176 streaming_playlists: {
177 bucket_name: options.playlistBucket,
178 prefix: options.playlistPrefix,
179 base_url: baseMockUrl
180 ? `${baseMockUrl}/${options.playlistBucket}`
181 : undefined
182 },
183
184 web_videos: {
185 bucket_name: options.webVideoBucket,
186 prefix: options.webVideoPrefix,
187 base_url: baseMockUrl
188 ? `${baseMockUrl}/${options.webVideoBucket}`
189 : undefined
190 }
191 }
192 }
193
194 servers = await createMultipleServers(2, config)
195
196 await setAccessTokensToServers(servers)
197 await doubleFollow(servers[0], servers[1])
198
199 for (const server of servers) {
200 const { uuid } = await server.videos.quickUpload({ name: 'video to keep' })
201 await waitJobs(servers)
202
203 const files = await server.videos.listFiles({ id: uuid })
204 keptUrls = keptUrls.concat(files.map(f => f.fileUrl))
205 }
206
207 sqlCommands = servers.map(s => new SQLCommand(s))
208 })
209
210 it('Should upload a video and move it to the object storage without transcoding', async function () {
211 this.timeout(40000)
212
213 const { uuid } = await servers[0].videos.quickUpload({ name: 'video 1', fixture })
214 uuidsToDelete.push(uuid)
215
216 await waitJobs(servers)
217
218 for (const server of servers) {
219 const video = await server.videos.get({ id: uuid })
220 const files = await checkFiles({ ...options, server, originServer: servers[0], originSQLCommand: sqlCommands[0], video, baseMockUrl })
221
222 deletedUrls = deletedUrls.concat(files)
223 }
224 })
225
226 it('Should upload a video and move it to the object storage with transcoding', async function () {
227 this.timeout(120000)
228
229 const { uuid } = await servers[1].videos.quickUpload({ name: 'video 2', fixture })
230 uuidsToDelete.push(uuid)
231
232 await waitJobs(servers)
233
234 for (const server of servers) {
235 const video = await server.videos.get({ id: uuid })
236 const files = await checkFiles({ ...options, server, originServer: servers[0], originSQLCommand: sqlCommands[0], video, baseMockUrl })
237
238 deletedUrls = deletedUrls.concat(files)
239 }
240 })
241
242 it('Should fetch correctly all the files', async function () {
243 for (const url of deletedUrls.concat(keptUrls)) {
244 await makeRawRequest({ url, expectedStatus: HttpStatusCode.OK_200 })
245 }
246 })
247
248 it('Should correctly delete the files', async function () {
249 await servers[0].videos.remove({ id: uuidsToDelete[0] })
250 await servers[1].videos.remove({ id: uuidsToDelete[1] })
251
252 await waitJobs(servers)
253
254 for (const url of deletedUrls) {
255 await makeRawRequest({ url, expectedStatus: HttpStatusCode.NOT_FOUND_404 })
256 }
257 })
258
259 it('Should have kept other files', async function () {
260 for (const url of keptUrls) {
261 await makeRawRequest({ url, expectedStatus: HttpStatusCode.OK_200 })
262 }
263 })
264
265 it('Should have an empty tmp directory', async function () {
266 for (const server of servers) {
267 await checkTmpIsEmpty(server)
268 }
269 })
270
271 it('Should not have downloaded files from object storage', async function () {
272 for (const server of servers) {
273 await expectLogDoesNotContain(server, 'from object storage')
274 }
275 })
276
277 after(async function () {
278 await mockObjectStorageProxy.terminate()
279 await objectStorage.cleanupMock()
280
281 for (const sqlCommand of sqlCommands) {
282 await sqlCommand.cleanup()
283 }
284
285 await cleanupTests(servers)
286 })
287}
288
289describe('Object storage for videos', function () {
290 if (areMockObjectStorageTestsDisabled()) return
291
292 const objectStorage = new ObjectStorageCommand()
293
294 describe('Test config', function () {
295 let server: PeerTubeServer
296
297 const baseConfig = objectStorage.getDefaultMockConfig()
298
299 const badCredentials = {
300 access_key_id: 'AKIAIOSFODNN7EXAMPLE',
301 secret_access_key: 'aJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY'
302 }
303
304 it('Should fail with same bucket names without prefix', function (done) {
305 const config = merge({}, baseConfig, {
306 object_storage: {
307 streaming_playlists: {
308 bucket_name: 'aaa'
309 },
310
311 web_videos: {
312 bucket_name: 'aaa'
313 }
314 }
315 })
316
317 createSingleServer(1, config)
318 .then(() => done(new Error('Did not throw')))
319 .catch(() => done())
320 })
321
322 it('Should fail with bad credentials', async function () {
323 this.timeout(60000)
324
325 await objectStorage.prepareDefaultMockBuckets()
326
327 const config = merge({}, baseConfig, {
328 object_storage: {
329 credentials: badCredentials
330 }
331 })
332
333 server = await createSingleServer(1, config)
334 await setAccessTokensToServers([ server ])
335
336 const { uuid } = await server.videos.quickUpload({ name: 'video' })
337
338 await waitJobs([ server ], { skipDelayed: true })
339 const video = await server.videos.get({ id: uuid })
340
341 expectStartWith(video.files[0].fileUrl, server.url)
342
343 await killallServers([ server ])
344 })
345
346 it('Should succeed with credentials from env', async function () {
347 this.timeout(60000)
348
349 await objectStorage.prepareDefaultMockBuckets()
350
351 const config = merge({}, baseConfig, {
352 object_storage: {
353 credentials: {
354 access_key_id: '',
355 secret_access_key: ''
356 }
357 }
358 })
359
360 const goodCredentials = ObjectStorageCommand.getMockCredentialsConfig()
361
362 server = await createSingleServer(1, config, {
363 env: {
364 AWS_ACCESS_KEY_ID: goodCredentials.access_key_id,
365 AWS_SECRET_ACCESS_KEY: goodCredentials.secret_access_key
366 }
367 })
368
369 await setAccessTokensToServers([ server ])
370
371 const { uuid } = await server.videos.quickUpload({ name: 'video' })
372
373 await waitJobs([ server ], { skipDelayed: true })
374 const video = await server.videos.get({ id: uuid })
375
376 expectStartWith(video.files[0].fileUrl, objectStorage.getMockWebVideosBaseUrl())
377 })
378
379 after(async function () {
380 await objectStorage.cleanupMock()
381
382 await cleanupTests([ server ])
383 })
384 })
385
386 describe('Test simple object storage', function () {
387 runTestSuite({
388 playlistBucket: objectStorage.getMockBucketName('streaming-playlists'),
389 webVideoBucket: objectStorage.getMockBucketName('web-videos')
390 })
391 })
392
393 describe('Test object storage with prefix', function () {
394 runTestSuite({
395 playlistBucket: objectStorage.getMockBucketName('mybucket'),
396 webVideoBucket: objectStorage.getMockBucketName('mybucket'),
397
398 playlistPrefix: 'streaming-playlists_',
399 webVideoPrefix: 'webvideo_'
400 })
401 })
402
403 describe('Test object storage with prefix and base URL', function () {
404 runTestSuite({
405 playlistBucket: objectStorage.getMockBucketName('mybucket'),
406 webVideoBucket: objectStorage.getMockBucketName('mybucket'),
407
408 playlistPrefix: 'streaming-playlists/',
409 webVideoPrefix: 'webvideo/',
410
411 useMockBaseUrl: true
412 })
413 })
414
415 describe('Test object storage with file bigger than upload part', function () {
416 let fixture: string
417 const maxUploadPart = '5MB'
418
419 before(async function () {
420 this.timeout(120000)
421
422 fixture = await generateHighBitrateVideo()
423
424 const { size } = await stat(fixture)
425
426 if (bytes.parse(maxUploadPart) > size) {
427 throw Error(`Fixture file is too small (${size}) to make sense for this test.`)
428 }
429 })
430
431 runTestSuite({
432 maxUploadPart,
433 playlistBucket: objectStorage.getMockBucketName('streaming-playlists'),
434 webVideoBucket: objectStorage.getMockBucketName('web-videos'),
435 fixture
436 })
437 })
438})
generated by cgit v1.2.3 (git 2.25.1) at 2025-04-16 03:18:58 +0000