diff options
author | Chocobozzz <me@florianbigard.com> | 2023-07-31 14:34:36 +0200 |
---|---|---|
committer | Chocobozzz <me@florianbigard.com> | 2023-08-11 15:02:33 +0200 |
commit | 3a4992633ee62d5edfbb484d9c6bcb3cf158489d (patch) | |
tree | e4510b39bdac9c318fdb4b47018d08f15368b8f0 /server/tests/api/object-storage | |
parent | 04d1da5621d25d59bd5fa1543b725c497bf5d9a8 (diff) | |
download | PeerTube-3a4992633ee62d5edfbb484d9c6bcb3cf158489d.tar.gz PeerTube-3a4992633ee62d5edfbb484d9c6bcb3cf158489d.tar.zst PeerTube-3a4992633ee62d5edfbb484d9c6bcb3cf158489d.zip |
Migrate server to ESM
Sorry for the very big commit that may lead to git log issues and merge
conflicts, but it's a major step forward:
* Server can be faster at startup because imports() are async and we can
easily lazy import big modules
* Angular doesn't seem to support ES import (with .js extension), so we
had to correctly organize peertube into a monorepo:
* Use yarn workspace feature
* Use typescript reference projects for dependencies
* Shared projects have been moved into "packages", each one is now a
node module (with a dedicated package.json/tsconfig.json)
* server/tools have been moved into apps/ and is now a dedicated app
bundled and published on NPM so users don't have to build peertube
cli tools manually
* server/tests have been moved into packages/ so we don't compile
them every time we want to run the server
* Use isolatedModule option:
* Had to move from const enum to const
(https://www.typescriptlang.org/docs/handbook/enums.html#objects-vs-enums)
* Had to explictely specify "type" imports when used in decorators
* Prefer tsx (that uses esbuild under the hood) instead of ts-node to
load typescript files (tests with mocha or scripts):
* To reduce test complexity as esbuild doesn't support decorator
metadata, we only test server files that do not import server
models
* We still build tests files into js files for a faster CI
* Remove unmaintained peertube CLI import script
* Removed some barrels to speed up execution (less imports)
Diffstat (limited to 'server/tests/api/object-storage')
-rw-r--r-- | server/tests/api/object-storage/index.ts | 4 | ||||
-rw-r--r-- | server/tests/api/object-storage/live.ts | 311 | ||||
-rw-r--r-- | server/tests/api/object-storage/video-imports.ts | 111 | ||||
-rw-r--r-- | server/tests/api/object-storage/video-static-file-privacy.ts | 570 | ||||
-rw-r--r-- | server/tests/api/object-storage/videos.ts | 438 |
5 files changed, 0 insertions, 1434 deletions
diff --git a/server/tests/api/object-storage/index.ts b/server/tests/api/object-storage/index.ts deleted file mode 100644 index 1f4489fa3..000000000 --- a/server/tests/api/object-storage/index.ts +++ /dev/null | |||
@@ -1,4 +0,0 @@ | |||
1 | export * from './live' | ||
2 | export * from './video-imports' | ||
3 | export * from './video-static-file-privacy' | ||
4 | export * from './videos' | ||
diff --git a/server/tests/api/object-storage/live.ts b/server/tests/api/object-storage/live.ts deleted file mode 100644 index 07ff4763b..000000000 --- a/server/tests/api/object-storage/live.ts +++ /dev/null | |||
@@ -1,311 +0,0 @@ | |||
1 | /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ | ||
2 | |||
3 | import { expect } from 'chai' | ||
4 | import { expectStartWith, MockObjectStorageProxy, SQLCommand, testLiveVideoResolutions } from '@server/tests/shared' | ||
5 | import { areMockObjectStorageTestsDisabled } from '@shared/core-utils' | ||
6 | import { HttpStatusCode, LiveVideoCreate, VideoPrivacy } from '@shared/models' | ||
7 | import { | ||
8 | cleanupTests, | ||
9 | createMultipleServers, | ||
10 | doubleFollow, | ||
11 | findExternalSavedVideo, | ||
12 | makeRawRequest, | ||
13 | ObjectStorageCommand, | ||
14 | PeerTubeServer, | ||
15 | setAccessTokensToServers, | ||
16 | setDefaultVideoChannel, | ||
17 | stopFfmpeg, | ||
18 | waitJobs, | ||
19 | waitUntilLivePublishedOnAllServers, | ||
20 | waitUntilLiveReplacedByReplayOnAllServers, | ||
21 | waitUntilLiveWaitingOnAllServers | ||
22 | } from '@shared/server-commands' | ||
23 | |||
24 | async function createLive (server: PeerTubeServer, permanent: boolean) { | ||
25 | const attributes: LiveVideoCreate = { | ||
26 | channelId: server.store.channel.id, | ||
27 | privacy: VideoPrivacy.PUBLIC, | ||
28 | name: 'my super live', | ||
29 | saveReplay: true, | ||
30 | replaySettings: { privacy: VideoPrivacy.PUBLIC }, | ||
31 | permanentLive: permanent | ||
32 | } | ||
33 | |||
34 | const { uuid } = await server.live.create({ fields: attributes }) | ||
35 | |||
36 | return uuid | ||
37 | } | ||
38 | |||
39 | async function checkFilesExist (options: { | ||
40 | servers: PeerTubeServer[] | ||
41 | videoUUID: string | ||
42 | numberOfFiles: number | ||
43 | objectStorage: ObjectStorageCommand | ||
44 | }) { | ||
45 | const { servers, videoUUID, numberOfFiles, objectStorage } = options | ||
46 | |||
47 | for (const server of servers) { | ||
48 | const video = await server.videos.get({ id: videoUUID }) | ||
49 | |||
50 | expect(video.files).to.have.lengthOf(0) | ||
51 | expect(video.streamingPlaylists).to.have.lengthOf(1) | ||
52 | |||
53 | const files = video.streamingPlaylists[0].files | ||
54 | expect(files).to.have.lengthOf(numberOfFiles) | ||
55 | |||
56 | for (const file of files) { | ||
57 | expectStartWith(file.fileUrl, objectStorage.getMockPlaylistBaseUrl()) | ||
58 | |||
59 | await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 }) | ||
60 | } | ||
61 | } | ||
62 | } | ||
63 | |||
64 | async function checkFilesCleanup (options: { | ||
65 | server: PeerTubeServer | ||
66 | videoUUID: string | ||
67 | resolutions: number[] | ||
68 | objectStorage: ObjectStorageCommand | ||
69 | }) { | ||
70 | const { server, videoUUID, resolutions, objectStorage } = options | ||
71 | |||
72 | const resolutionFiles = resolutions.map((_value, i) => `${i}.m3u8`) | ||
73 | |||
74 | for (const playlistName of [ 'master.m3u8' ].concat(resolutionFiles)) { | ||
75 | await server.live.getPlaylistFile({ | ||
76 | videoUUID, | ||
77 | playlistName, | ||
78 | expectedStatus: HttpStatusCode.NOT_FOUND_404, | ||
79 | objectStorage | ||
80 | }) | ||
81 | } | ||
82 | |||
83 | await server.live.getSegmentFile({ | ||
84 | videoUUID, | ||
85 | playlistNumber: 0, | ||
86 | segment: 0, | ||
87 | objectStorage, | ||
88 | expectedStatus: HttpStatusCode.NOT_FOUND_404 | ||
89 | }) | ||
90 | } | ||
91 | |||
92 | describe('Object storage for lives', function () { | ||
93 | if (areMockObjectStorageTestsDisabled()) return | ||
94 | |||
95 | let servers: PeerTubeServer[] | ||
96 | let sqlCommandServer1: SQLCommand | ||
97 | const objectStorage = new ObjectStorageCommand() | ||
98 | |||
99 | before(async function () { | ||
100 | this.timeout(120000) | ||
101 | |||
102 | await objectStorage.prepareDefaultMockBuckets() | ||
103 | servers = await createMultipleServers(2, objectStorage.getDefaultMockConfig()) | ||
104 | |||
105 | await setAccessTokensToServers(servers) | ||
106 | await setDefaultVideoChannel(servers) | ||
107 | await doubleFollow(servers[0], servers[1]) | ||
108 | |||
109 | await servers[0].config.enableTranscoding() | ||
110 | |||
111 | sqlCommandServer1 = new SQLCommand(servers[0]) | ||
112 | }) | ||
113 | |||
114 | describe('Without live transcoding', function () { | ||
115 | let videoUUID: string | ||
116 | |||
117 | before(async function () { | ||
118 | await servers[0].config.enableLive({ transcoding: false }) | ||
119 | |||
120 | videoUUID = await createLive(servers[0], false) | ||
121 | }) | ||
122 | |||
123 | it('Should create a live and publish it on object storage', async function () { | ||
124 | this.timeout(220000) | ||
125 | |||
126 | const ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: videoUUID }) | ||
127 | await waitUntilLivePublishedOnAllServers(servers, videoUUID) | ||
128 | |||
129 | await testLiveVideoResolutions({ | ||
130 | originServer: servers[0], | ||
131 | sqlCommand: sqlCommandServer1, | ||
132 | servers, | ||
133 | liveVideoId: videoUUID, | ||
134 | resolutions: [ 720 ], | ||
135 | transcoded: false, | ||
136 | objectStorage | ||
137 | }) | ||
138 | |||
139 | await stopFfmpeg(ffmpegCommand) | ||
140 | }) | ||
141 | |||
142 | it('Should have saved the replay on object storage', async function () { | ||
143 | this.timeout(220000) | ||
144 | |||
145 | await waitUntilLiveReplacedByReplayOnAllServers(servers, videoUUID) | ||
146 | await waitJobs(servers) | ||
147 | |||
148 | await checkFilesExist({ servers, videoUUID, numberOfFiles: 1, objectStorage }) | ||
149 | }) | ||
150 | |||
151 | it('Should have cleaned up live files from object storage', async function () { | ||
152 | await checkFilesCleanup({ server: servers[0], videoUUID, resolutions: [ 720 ], objectStorage }) | ||
153 | }) | ||
154 | }) | ||
155 | |||
156 | describe('With live transcoding', function () { | ||
157 | const resolutions = [ 720, 480, 360, 240, 144 ] | ||
158 | |||
159 | before(async function () { | ||
160 | await servers[0].config.enableLive({ transcoding: true }) | ||
161 | }) | ||
162 | |||
163 | describe('Normal replay', function () { | ||
164 | let videoUUIDNonPermanent: string | ||
165 | |||
166 | before(async function () { | ||
167 | videoUUIDNonPermanent = await createLive(servers[0], false) | ||
168 | }) | ||
169 | |||
170 | it('Should create a live and publish it on object storage', async function () { | ||
171 | this.timeout(240000) | ||
172 | |||
173 | const ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: videoUUIDNonPermanent }) | ||
174 | await waitUntilLivePublishedOnAllServers(servers, videoUUIDNonPermanent) | ||
175 | |||
176 | await testLiveVideoResolutions({ | ||
177 | originServer: servers[0], | ||
178 | sqlCommand: sqlCommandServer1, | ||
179 | servers, | ||
180 | liveVideoId: videoUUIDNonPermanent, | ||
181 | resolutions, | ||
182 | transcoded: true, | ||
183 | objectStorage | ||
184 | }) | ||
185 | |||
186 | await stopFfmpeg(ffmpegCommand) | ||
187 | }) | ||
188 | |||
189 | it('Should have saved the replay on object storage', async function () { | ||
190 | this.timeout(220000) | ||
191 | |||
192 | await waitUntilLiveReplacedByReplayOnAllServers(servers, videoUUIDNonPermanent) | ||
193 | await waitJobs(servers) | ||
194 | |||
195 | await checkFilesExist({ servers, videoUUID: videoUUIDNonPermanent, numberOfFiles: 5, objectStorage }) | ||
196 | }) | ||
197 | |||
198 | it('Should have cleaned up live files from object storage', async function () { | ||
199 | await checkFilesCleanup({ server: servers[0], videoUUID: videoUUIDNonPermanent, resolutions, objectStorage }) | ||
200 | }) | ||
201 | }) | ||
202 | |||
203 | describe('Permanent replay', function () { | ||
204 | let videoUUIDPermanent: string | ||
205 | |||
206 | before(async function () { | ||
207 | videoUUIDPermanent = await createLive(servers[0], true) | ||
208 | }) | ||
209 | |||
210 | it('Should create a live and publish it on object storage', async function () { | ||
211 | this.timeout(240000) | ||
212 | |||
213 | const ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: videoUUIDPermanent }) | ||
214 | await waitUntilLivePublishedOnAllServers(servers, videoUUIDPermanent) | ||
215 | |||
216 | await testLiveVideoResolutions({ | ||
217 | originServer: servers[0], | ||
218 | sqlCommand: sqlCommandServer1, | ||
219 | servers, | ||
220 | liveVideoId: videoUUIDPermanent, | ||
221 | resolutions, | ||
222 | transcoded: true, | ||
223 | objectStorage | ||
224 | }) | ||
225 | |||
226 | await stopFfmpeg(ffmpegCommand) | ||
227 | }) | ||
228 | |||
229 | it('Should have saved the replay on object storage', async function () { | ||
230 | this.timeout(220000) | ||
231 | |||
232 | await waitUntilLiveWaitingOnAllServers(servers, videoUUIDPermanent) | ||
233 | await waitJobs(servers) | ||
234 | |||
235 | const videoLiveDetails = await servers[0].videos.get({ id: videoUUIDPermanent }) | ||
236 | const replay = await findExternalSavedVideo(servers[0], videoLiveDetails) | ||
237 | |||
238 | await checkFilesExist({ servers, videoUUID: replay.uuid, numberOfFiles: 5, objectStorage }) | ||
239 | }) | ||
240 | |||
241 | it('Should have cleaned up live files from object storage', async function () { | ||
242 | await checkFilesCleanup({ server: servers[0], videoUUID: videoUUIDPermanent, resolutions, objectStorage }) | ||
243 | }) | ||
244 | }) | ||
245 | }) | ||
246 | |||
247 | describe('With object storage base url', function () { | ||
248 | const mockObjectStorageProxy = new MockObjectStorageProxy() | ||
249 | let baseMockUrl: string | ||
250 | |||
251 | before(async function () { | ||
252 | this.timeout(120000) | ||
253 | |||
254 | const port = await mockObjectStorageProxy.initialize() | ||
255 | const bucketName = objectStorage.getMockStreamingPlaylistsBucketName() | ||
256 | baseMockUrl = `http://127.0.0.1:${port}/${bucketName}` | ||
257 | |||
258 | await objectStorage.prepareDefaultMockBuckets() | ||
259 | |||
260 | const config = { | ||
261 | object_storage: { | ||
262 | enabled: true, | ||
263 | endpoint: 'http://' + ObjectStorageCommand.getMockEndpointHost(), | ||
264 | region: ObjectStorageCommand.getMockRegion(), | ||
265 | |||
266 | credentials: ObjectStorageCommand.getMockCredentialsConfig(), | ||
267 | |||
268 | streaming_playlists: { | ||
269 | bucket_name: bucketName, | ||
270 | prefix: '', | ||
271 | base_url: baseMockUrl | ||
272 | } | ||
273 | } | ||
274 | } | ||
275 | |||
276 | await servers[0].kill() | ||
277 | await servers[0].run(config) | ||
278 | |||
279 | await servers[0].config.enableLive({ transcoding: true, resolutions: 'min' }) | ||
280 | }) | ||
281 | |||
282 | it('Should publish a live and replace the base url', async function () { | ||
283 | this.timeout(240000) | ||
284 | |||
285 | const videoUUIDPermanent = await createLive(servers[0], true) | ||
286 | |||
287 | const ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: videoUUIDPermanent }) | ||
288 | await waitUntilLivePublishedOnAllServers(servers, videoUUIDPermanent) | ||
289 | |||
290 | await testLiveVideoResolutions({ | ||
291 | originServer: servers[0], | ||
292 | sqlCommand: sqlCommandServer1, | ||
293 | servers, | ||
294 | liveVideoId: videoUUIDPermanent, | ||
295 | resolutions: [ 720 ], | ||
296 | transcoded: true, | ||
297 | objectStorage, | ||
298 | objectStorageBaseUrl: baseMockUrl | ||
299 | }) | ||
300 | |||
301 | await stopFfmpeg(ffmpegCommand) | ||
302 | }) | ||
303 | }) | ||
304 | |||
305 | after(async function () { | ||
306 | await sqlCommandServer1.cleanup() | ||
307 | await objectStorage.cleanupMock() | ||
308 | |||
309 | await cleanupTests(servers) | ||
310 | }) | ||
311 | }) | ||
diff --git a/server/tests/api/object-storage/video-imports.ts b/server/tests/api/object-storage/video-imports.ts deleted file mode 100644 index 57150e5a6..000000000 --- a/server/tests/api/object-storage/video-imports.ts +++ /dev/null | |||
@@ -1,111 +0,0 @@ | |||
1 | /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ | ||
2 | |||
3 | import { expect } from 'chai' | ||
4 | import { expectStartWith, FIXTURE_URLS } from '@server/tests/shared' | ||
5 | import { areMockObjectStorageTestsDisabled } from '@shared/core-utils' | ||
6 | import { HttpStatusCode, VideoPrivacy } from '@shared/models' | ||
7 | import { | ||
8 | cleanupTests, | ||
9 | createSingleServer, | ||
10 | makeRawRequest, | ||
11 | ObjectStorageCommand, | ||
12 | PeerTubeServer, | ||
13 | setAccessTokensToServers, | ||
14 | setDefaultVideoChannel, | ||
15 | waitJobs | ||
16 | } from '@shared/server-commands' | ||
17 | |||
18 | async function importVideo (server: PeerTubeServer) { | ||
19 | const attributes = { | ||
20 | name: 'import 2', | ||
21 | privacy: VideoPrivacy.PUBLIC, | ||
22 | channelId: server.store.channel.id, | ||
23 | targetUrl: FIXTURE_URLS.goodVideo720 | ||
24 | } | ||
25 | |||
26 | const { video: { uuid } } = await server.imports.importVideo({ attributes }) | ||
27 | |||
28 | return uuid | ||
29 | } | ||
30 | |||
31 | describe('Object storage for video import', function () { | ||
32 | if (areMockObjectStorageTestsDisabled()) return | ||
33 | |||
34 | let server: PeerTubeServer | ||
35 | const objectStorage = new ObjectStorageCommand() | ||
36 | |||
37 | before(async function () { | ||
38 | this.timeout(120000) | ||
39 | |||
40 | await objectStorage.prepareDefaultMockBuckets() | ||
41 | |||
42 | server = await createSingleServer(1, objectStorage.getDefaultMockConfig()) | ||
43 | |||
44 | await setAccessTokensToServers([ server ]) | ||
45 | await setDefaultVideoChannel([ server ]) | ||
46 | |||
47 | await server.config.enableImports() | ||
48 | }) | ||
49 | |||
50 | describe('Without transcoding', async function () { | ||
51 | |||
52 | before(async function () { | ||
53 | await server.config.disableTranscoding() | ||
54 | }) | ||
55 | |||
56 | it('Should import a video and have sent it to object storage', async function () { | ||
57 | this.timeout(120000) | ||
58 | |||
59 | const uuid = await importVideo(server) | ||
60 | await waitJobs(server) | ||
61 | |||
62 | const video = await server.videos.get({ id: uuid }) | ||
63 | |||
64 | expect(video.files).to.have.lengthOf(1) | ||
65 | expect(video.streamingPlaylists).to.have.lengthOf(0) | ||
66 | |||
67 | const fileUrl = video.files[0].fileUrl | ||
68 | expectStartWith(fileUrl, objectStorage.getMockWebVideosBaseUrl()) | ||
69 | |||
70 | await makeRawRequest({ url: fileUrl, expectedStatus: HttpStatusCode.OK_200 }) | ||
71 | }) | ||
72 | }) | ||
73 | |||
74 | describe('With transcoding', async function () { | ||
75 | |||
76 | before(async function () { | ||
77 | await server.config.enableTranscoding() | ||
78 | }) | ||
79 | |||
80 | it('Should import a video and have sent it to object storage', async function () { | ||
81 | this.timeout(120000) | ||
82 | |||
83 | const uuid = await importVideo(server) | ||
84 | await waitJobs(server) | ||
85 | |||
86 | const video = await server.videos.get({ id: uuid }) | ||
87 | |||
88 | expect(video.files).to.have.lengthOf(5) | ||
89 | expect(video.streamingPlaylists).to.have.lengthOf(1) | ||
90 | expect(video.streamingPlaylists[0].files).to.have.lengthOf(5) | ||
91 | |||
92 | for (const file of video.files) { | ||
93 | expectStartWith(file.fileUrl, objectStorage.getMockWebVideosBaseUrl()) | ||
94 | |||
95 | await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 }) | ||
96 | } | ||
97 | |||
98 | for (const file of video.streamingPlaylists[0].files) { | ||
99 | expectStartWith(file.fileUrl, objectStorage.getMockPlaylistBaseUrl()) | ||
100 | |||
101 | await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 }) | ||
102 | } | ||
103 | }) | ||
104 | }) | ||
105 | |||
106 | after(async function () { | ||
107 | await objectStorage.cleanupMock() | ||
108 | |||
109 | await cleanupTests([ server ]) | ||
110 | }) | ||
111 | }) | ||
diff --git a/server/tests/api/object-storage/video-static-file-privacy.ts b/server/tests/api/object-storage/video-static-file-privacy.ts deleted file mode 100644 index 64ab542a5..000000000 --- a/server/tests/api/object-storage/video-static-file-privacy.ts +++ /dev/null | |||
@@ -1,570 +0,0 @@ | |||
1 | /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ | ||
2 | |||
3 | import { expect } from 'chai' | ||
4 | import { basename } from 'path' | ||
5 | import { checkVideoFileTokenReinjection, expectStartWith, SQLCommand } from '@server/tests/shared' | ||
6 | import { areScalewayObjectStorageTestsDisabled, getAllFiles, getHLS } from '@shared/core-utils' | ||
7 | import { HttpStatusCode, LiveVideo, VideoDetails, VideoPrivacy } from '@shared/models' | ||
8 | import { | ||
9 | cleanupTests, | ||
10 | createSingleServer, | ||
11 | findExternalSavedVideo, | ||
12 | makeRawRequest, | ||
13 | ObjectStorageCommand, | ||
14 | PeerTubeServer, | ||
15 | sendRTMPStream, | ||
16 | setAccessTokensToServers, | ||
17 | setDefaultVideoChannel, | ||
18 | stopFfmpeg, | ||
19 | waitJobs | ||
20 | } from '@shared/server-commands' | ||
21 | |||
22 | function extractFilenameFromUrl (url: string) { | ||
23 | const parts = basename(url).split(':') | ||
24 | |||
25 | return parts[parts.length - 1] | ||
26 | } | ||
27 | |||
28 | describe('Object storage for video static file privacy', function () { | ||
29 | // We need real world object storage to check ACL | ||
30 | if (areScalewayObjectStorageTestsDisabled()) return | ||
31 | |||
32 | let server: PeerTubeServer | ||
33 | let sqlCommand: SQLCommand | ||
34 | let userToken: string | ||
35 | |||
36 | // --------------------------------------------------------------------------- | ||
37 | |||
38 | async function checkPrivateVODFiles (uuid: string) { | ||
39 | const video = await server.videos.getWithToken({ id: uuid }) | ||
40 | |||
41 | for (const file of video.files) { | ||
42 | expectStartWith(file.fileUrl, server.url + '/object-storage-proxy/web-videos/private/') | ||
43 | |||
44 | await makeRawRequest({ url: file.fileUrl, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 }) | ||
45 | } | ||
46 | |||
47 | for (const file of getAllFiles(video)) { | ||
48 | const internalFileUrl = await sqlCommand.getInternalFileUrl(file.id) | ||
49 | expectStartWith(internalFileUrl, ObjectStorageCommand.getScalewayBaseUrl()) | ||
50 | await makeRawRequest({ url: internalFileUrl, token: server.accessToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) | ||
51 | } | ||
52 | |||
53 | const hls = getHLS(video) | ||
54 | |||
55 | if (hls) { | ||
56 | for (const url of [ hls.playlistUrl, hls.segmentsSha256Url ]) { | ||
57 | expectStartWith(url, server.url + '/object-storage-proxy/streaming-playlists/hls/private/') | ||
58 | } | ||
59 | |||
60 | await makeRawRequest({ url: hls.playlistUrl, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 }) | ||
61 | await makeRawRequest({ url: hls.segmentsSha256Url, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 }) | ||
62 | |||
63 | for (const file of hls.files) { | ||
64 | expectStartWith(file.fileUrl, server.url + '/object-storage-proxy/streaming-playlists/hls/private/') | ||
65 | |||
66 | await makeRawRequest({ url: file.fileUrl, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 }) | ||
67 | } | ||
68 | } | ||
69 | } | ||
70 | |||
71 | async function checkPublicVODFiles (uuid: string) { | ||
72 | const video = await server.videos.getWithToken({ id: uuid }) | ||
73 | |||
74 | for (const file of getAllFiles(video)) { | ||
75 | expectStartWith(file.fileUrl, ObjectStorageCommand.getScalewayBaseUrl()) | ||
76 | |||
77 | await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 }) | ||
78 | } | ||
79 | |||
80 | const hls = getHLS(video) | ||
81 | |||
82 | if (hls) { | ||
83 | expectStartWith(hls.playlistUrl, ObjectStorageCommand.getScalewayBaseUrl()) | ||
84 | expectStartWith(hls.segmentsSha256Url, ObjectStorageCommand.getScalewayBaseUrl()) | ||
85 | |||
86 | await makeRawRequest({ url: hls.playlistUrl, expectedStatus: HttpStatusCode.OK_200 }) | ||
87 | await makeRawRequest({ url: hls.segmentsSha256Url, expectedStatus: HttpStatusCode.OK_200 }) | ||
88 | } | ||
89 | } | ||
90 | |||
91 | // --------------------------------------------------------------------------- | ||
92 | |||
93 | before(async function () { | ||
94 | this.timeout(120000) | ||
95 | |||
96 | server = await createSingleServer(1, ObjectStorageCommand.getDefaultScalewayConfig({ serverNumber: 1 })) | ||
97 | await setAccessTokensToServers([ server ]) | ||
98 | await setDefaultVideoChannel([ server ]) | ||
99 | |||
100 | await server.config.enableMinimumTranscoding() | ||
101 | |||
102 | userToken = await server.users.generateUserAndToken('user1') | ||
103 | |||
104 | sqlCommand = new SQLCommand(server) | ||
105 | }) | ||
106 | |||
107 | describe('VOD', function () { | ||
108 | let privateVideoUUID: string | ||
109 | let publicVideoUUID: string | ||
110 | let passwordProtectedVideoUUID: string | ||
111 | let userPrivateVideoUUID: string | ||
112 | |||
113 | const correctPassword = 'my super password' | ||
114 | const correctPasswordHeader = { 'x-peertube-video-password': correctPassword } | ||
115 | const incorrectPasswordHeader = { 'x-peertube-video-password': correctPassword + 'toto' } | ||
116 | |||
117 | // --------------------------------------------------------------------------- | ||
118 | |||
119 | async function getSampleFileUrls (videoId: string) { | ||
120 | const video = await server.videos.getWithToken({ id: videoId }) | ||
121 | |||
122 | return { | ||
123 | webVideoFile: video.files[0].fileUrl, | ||
124 | hlsFile: getHLS(video).files[0].fileUrl | ||
125 | } | ||
126 | } | ||
127 | |||
128 | // --------------------------------------------------------------------------- | ||
129 | |||
130 | it('Should upload a private video and have appropriate object storage ACL', async function () { | ||
131 | this.timeout(120000) | ||
132 | |||
133 | { | ||
134 | const { uuid } = await server.videos.quickUpload({ name: 'video', privacy: VideoPrivacy.PRIVATE }) | ||
135 | privateVideoUUID = uuid | ||
136 | } | ||
137 | |||
138 | { | ||
139 | const { uuid } = await server.videos.quickUpload({ name: 'user video', token: userToken, privacy: VideoPrivacy.PRIVATE }) | ||
140 | userPrivateVideoUUID = uuid | ||
141 | } | ||
142 | |||
143 | await waitJobs([ server ]) | ||
144 | |||
145 | await checkPrivateVODFiles(privateVideoUUID) | ||
146 | }) | ||
147 | |||
148 | it('Should upload a password protected video and have appropriate object storage ACL', async function () { | ||
149 | this.timeout(120000) | ||
150 | |||
151 | { | ||
152 | const { uuid } = await server.videos.quickUpload({ | ||
153 | name: 'video', | ||
154 | privacy: VideoPrivacy.PASSWORD_PROTECTED, | ||
155 | videoPasswords: [ correctPassword ] | ||
156 | }) | ||
157 | passwordProtectedVideoUUID = uuid | ||
158 | } | ||
159 | await waitJobs([ server ]) | ||
160 | |||
161 | await checkPrivateVODFiles(passwordProtectedVideoUUID) | ||
162 | }) | ||
163 | |||
164 | it('Should upload a public video and have appropriate object storage ACL', async function () { | ||
165 | this.timeout(120000) | ||
166 | |||
167 | const { uuid } = await server.videos.quickUpload({ name: 'video', privacy: VideoPrivacy.UNLISTED }) | ||
168 | await waitJobs([ server ]) | ||
169 | |||
170 | publicVideoUUID = uuid | ||
171 | |||
172 | await checkPublicVODFiles(publicVideoUUID) | ||
173 | }) | ||
174 | |||
175 | it('Should not get files without appropriate OAuth token', async function () { | ||
176 | this.timeout(60000) | ||
177 | |||
178 | const { webVideoFile, hlsFile } = await getSampleFileUrls(privateVideoUUID) | ||
179 | |||
180 | await makeRawRequest({ url: webVideoFile, token: userToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) | ||
181 | await makeRawRequest({ url: webVideoFile, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 }) | ||
182 | |||
183 | await makeRawRequest({ url: hlsFile, token: userToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) | ||
184 | await makeRawRequest({ url: hlsFile, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 }) | ||
185 | }) | ||
186 | |||
187 | it('Should not get files without appropriate password or appropriate OAuth token', async function () { | ||
188 | this.timeout(60000) | ||
189 | |||
190 | const { webVideoFile, hlsFile } = await getSampleFileUrls(passwordProtectedVideoUUID) | ||
191 | |||
192 | await makeRawRequest({ url: webVideoFile, token: userToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) | ||
193 | await makeRawRequest({ | ||
194 | url: webVideoFile, | ||
195 | token: null, | ||
196 | headers: incorrectPasswordHeader, | ||
197 | expectedStatus: HttpStatusCode.FORBIDDEN_403 | ||
198 | }) | ||
199 | await makeRawRequest({ url: webVideoFile, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 }) | ||
200 | await makeRawRequest({ | ||
201 | url: webVideoFile, | ||
202 | token: null, | ||
203 | headers: correctPasswordHeader, | ||
204 | expectedStatus: HttpStatusCode.OK_200 | ||
205 | }) | ||
206 | |||
207 | await makeRawRequest({ url: hlsFile, token: userToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) | ||
208 | await makeRawRequest({ | ||
209 | url: hlsFile, | ||
210 | token: null, | ||
211 | headers: incorrectPasswordHeader, | ||
212 | expectedStatus: HttpStatusCode.FORBIDDEN_403 | ||
213 | }) | ||
214 | await makeRawRequest({ url: hlsFile, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 }) | ||
215 | await makeRawRequest({ | ||
216 | url: hlsFile, | ||
217 | token: null, | ||
218 | headers: correctPasswordHeader, | ||
219 | expectedStatus: HttpStatusCode.OK_200 | ||
220 | }) | ||
221 | }) | ||
222 | |||
223 | it('Should not get HLS file of another video', async function () { | ||
224 | this.timeout(60000) | ||
225 | |||
226 | const privateVideo = await server.videos.getWithToken({ id: privateVideoUUID }) | ||
227 | const hlsFilename = basename(getHLS(privateVideo).files[0].fileUrl) | ||
228 | |||
229 | const badUrl = server.url + '/object-storage-proxy/streaming-playlists/hls/private/' + userPrivateVideoUUID + '/' + hlsFilename | ||
230 | const goodUrl = server.url + '/object-storage-proxy/streaming-playlists/hls/private/' + privateVideoUUID + '/' + hlsFilename | ||
231 | |||
232 | await makeRawRequest({ url: badUrl, token: server.accessToken, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) | ||
233 | await makeRawRequest({ url: goodUrl, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 }) | ||
234 | }) | ||
235 | |||
236 | it('Should correctly check OAuth, video file token of private video', async function () { | ||
237 | this.timeout(60000) | ||
238 | |||
239 | const badVideoFileToken = await server.videoToken.getVideoFileToken({ token: userToken, videoId: userPrivateVideoUUID }) | ||
240 | const goodVideoFileToken = await server.videoToken.getVideoFileToken({ videoId: privateVideoUUID }) | ||
241 | |||
242 | const { webVideoFile, hlsFile } = await getSampleFileUrls(privateVideoUUID) | ||
243 | |||
244 | for (const url of [ webVideoFile, hlsFile ]) { | ||
245 | await makeRawRequest({ url, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) | ||
246 | await makeRawRequest({ url, token: userToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) | ||
247 | await makeRawRequest({ url, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 }) | ||
248 | |||
249 | await makeRawRequest({ url, query: { videoFileToken: badVideoFileToken }, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) | ||
250 | await makeRawRequest({ url, query: { videoFileToken: goodVideoFileToken }, expectedStatus: HttpStatusCode.OK_200 }) | ||
251 | |||
252 | } | ||
253 | }) | ||
254 | |||
255 | it('Should correctly check OAuth, video file token or video password of password protected video', async function () { | ||
256 | this.timeout(60000) | ||
257 | |||
258 | const badVideoFileToken = await server.videoToken.getVideoFileToken({ token: userToken, videoId: userPrivateVideoUUID }) | ||
259 | const goodVideoFileToken = await server.videoToken.getVideoFileToken({ | ||
260 | videoId: passwordProtectedVideoUUID, | ||
261 | videoPassword: correctPassword | ||
262 | }) | ||
263 | |||
264 | const { webVideoFile, hlsFile } = await getSampleFileUrls(passwordProtectedVideoUUID) | ||
265 | |||
266 | for (const url of [ hlsFile, webVideoFile ]) { | ||
267 | await makeRawRequest({ url, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) | ||
268 | await makeRawRequest({ url, token: userToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) | ||
269 | await makeRawRequest({ url, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 }) | ||
270 | |||
271 | await makeRawRequest({ url, query: { videoFileToken: badVideoFileToken }, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) | ||
272 | await makeRawRequest({ url, query: { videoFileToken: goodVideoFileToken }, expectedStatus: HttpStatusCode.OK_200 }) | ||
273 | |||
274 | await makeRawRequest({ | ||
275 | url, | ||
276 | headers: incorrectPasswordHeader, | ||
277 | expectedStatus: HttpStatusCode.FORBIDDEN_403 | ||
278 | }) | ||
279 | await makeRawRequest({ url, headers: correctPasswordHeader, expectedStatus: HttpStatusCode.OK_200 }) | ||
280 | } | ||
281 | }) | ||
282 | |||
283 | it('Should reinject video file token', async function () { | ||
284 | this.timeout(120000) | ||
285 | |||
286 | const videoFileToken = await server.videoToken.getVideoFileToken({ videoId: privateVideoUUID }) | ||
287 | |||
288 | await checkVideoFileTokenReinjection({ | ||
289 | server, | ||
290 | videoUUID: privateVideoUUID, | ||
291 | videoFileToken, | ||
292 | resolutions: [ 240, 720 ], | ||
293 | isLive: false | ||
294 | }) | ||
295 | }) | ||
296 | |||
297 | it('Should update public video to private', async function () { | ||
298 | this.timeout(60000) | ||
299 | |||
300 | await server.videos.update({ id: publicVideoUUID, attributes: { privacy: VideoPrivacy.INTERNAL } }) | ||
301 | |||
302 | await checkPrivateVODFiles(publicVideoUUID) | ||
303 | }) | ||
304 | |||
305 | it('Should update private video to public', async function () { | ||
306 | this.timeout(60000) | ||
307 | |||
308 | await server.videos.update({ id: publicVideoUUID, attributes: { privacy: VideoPrivacy.PUBLIC } }) | ||
309 | |||
310 | await checkPublicVODFiles(publicVideoUUID) | ||
311 | }) | ||
312 | }) | ||
313 | |||
314 | describe('Live', function () { | ||
315 | let normalLiveId: string | ||
316 | let normalLive: LiveVideo | ||
317 | |||
318 | let permanentLiveId: string | ||
319 | let permanentLive: LiveVideo | ||
320 | |||
321 | let passwordProtectedLiveId: string | ||
322 | let passwordProtectedLive: LiveVideo | ||
323 | |||
324 | const correctPassword = 'my super password' | ||
325 | |||
326 | let unrelatedFileToken: string | ||
327 | |||
328 | // --------------------------------------------------------------------------- | ||
329 | |||
330 | async function checkLiveFiles (live: LiveVideo, liveId: string, videoPassword?: string) { | ||
331 | const ffmpegCommand = sendRTMPStream({ rtmpBaseUrl: live.rtmpUrl, streamKey: live.streamKey }) | ||
332 | await server.live.waitUntilPublished({ videoId: liveId }) | ||
333 | |||
334 | const video = videoPassword | ||
335 | ? await server.videos.getWithPassword({ id: liveId, password: videoPassword }) | ||
336 | : await server.videos.getWithToken({ id: liveId }) | ||
337 | |||
338 | const fileToken = videoPassword | ||
339 | ? await server.videoToken.getVideoFileToken({ token: null, videoId: video.uuid, videoPassword }) | ||
340 | : await server.videoToken.getVideoFileToken({ videoId: video.uuid }) | ||
341 | |||
342 | const hls = video.streamingPlaylists[0] | ||
343 | |||
344 | for (const url of [ hls.playlistUrl, hls.segmentsSha256Url ]) { | ||
345 | expectStartWith(url, server.url + '/object-storage-proxy/streaming-playlists/hls/private/') | ||
346 | |||
347 | await makeRawRequest({ url: hls.playlistUrl, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 }) | ||
348 | await makeRawRequest({ url: hls.segmentsSha256Url, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 }) | ||
349 | |||
350 | await makeRawRequest({ url, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 }) | ||
351 | await makeRawRequest({ url, query: { videoFileToken: fileToken }, expectedStatus: HttpStatusCode.OK_200 }) | ||
352 | if (videoPassword) { | ||
353 | await makeRawRequest({ url, headers: { 'x-peertube-video-password': videoPassword }, expectedStatus: HttpStatusCode.OK_200 }) | ||
354 | } | ||
355 | await makeRawRequest({ url, token: userToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) | ||
356 | await makeRawRequest({ url, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) | ||
357 | await makeRawRequest({ url, query: { videoFileToken: unrelatedFileToken }, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) | ||
358 | if (videoPassword) { | ||
359 | await makeRawRequest({ | ||
360 | url, | ||
361 | headers: { 'x-peertube-video-password': 'incorrectPassword' }, | ||
362 | expectedStatus: HttpStatusCode.FORBIDDEN_403 | ||
363 | }) | ||
364 | } | ||
365 | } | ||
366 | |||
367 | await stopFfmpeg(ffmpegCommand) | ||
368 | } | ||
369 | |||
370 | async function checkReplay (replay: VideoDetails) { | ||
371 | const fileToken = await server.videoToken.getVideoFileToken({ videoId: replay.uuid }) | ||
372 | |||
373 | const hls = replay.streamingPlaylists[0] | ||
374 | expect(hls.files).to.not.have.lengthOf(0) | ||
375 | |||
376 | for (const file of hls.files) { | ||
377 | await makeRawRequest({ url: file.fileUrl, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 }) | ||
378 | await makeRawRequest({ url: file.fileUrl, query: { videoFileToken: fileToken }, expectedStatus: HttpStatusCode.OK_200 }) | ||
379 | |||
380 | await makeRawRequest({ url: file.fileUrl, token: userToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) | ||
381 | await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) | ||
382 | await makeRawRequest({ | ||
383 | url: file.fileUrl, | ||
384 | query: { videoFileToken: unrelatedFileToken }, | ||
385 | expectedStatus: HttpStatusCode.FORBIDDEN_403 | ||
386 | }) | ||
387 | } | ||
388 | |||
389 | for (const url of [ hls.playlistUrl, hls.segmentsSha256Url ]) { | ||
390 | expectStartWith(url, server.url + '/object-storage-proxy/streaming-playlists/hls/private/') | ||
391 | |||
392 | await makeRawRequest({ url, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 }) | ||
393 | await makeRawRequest({ url, query: { videoFileToken: fileToken }, expectedStatus: HttpStatusCode.OK_200 }) | ||
394 | |||
395 | await makeRawRequest({ url, token: userToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) | ||
396 | await makeRawRequest({ url, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) | ||
397 | await makeRawRequest({ url, query: { videoFileToken: unrelatedFileToken }, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) | ||
398 | } | ||
399 | } | ||
400 | |||
401 | // --------------------------------------------------------------------------- | ||
402 | |||
403 | before(async function () { | ||
404 | await server.config.enableMinimumTranscoding() | ||
405 | |||
406 | const { uuid } = await server.videos.quickUpload({ name: 'another video' }) | ||
407 | unrelatedFileToken = await server.videoToken.getVideoFileToken({ videoId: uuid }) | ||
408 | |||
409 | await server.config.enableLive({ | ||
410 | allowReplay: true, | ||
411 | transcoding: true, | ||
412 | resolutions: 'min' | ||
413 | }) | ||
414 | |||
415 | { | ||
416 | const { video, live } = await server.live.quickCreate({ | ||
417 | saveReplay: true, | ||
418 | permanentLive: false, | ||
419 | privacy: VideoPrivacy.PRIVATE | ||
420 | }) | ||
421 | normalLiveId = video.uuid | ||
422 | normalLive = live | ||
423 | } | ||
424 | |||
425 | { | ||
426 | const { video, live } = await server.live.quickCreate({ | ||
427 | saveReplay: true, | ||
428 | permanentLive: true, | ||
429 | privacy: VideoPrivacy.PRIVATE | ||
430 | }) | ||
431 | permanentLiveId = video.uuid | ||
432 | permanentLive = live | ||
433 | } | ||
434 | |||
435 | { | ||
436 | const { video, live } = await server.live.quickCreate({ | ||
437 | saveReplay: false, | ||
438 | permanentLive: false, | ||
439 | privacy: VideoPrivacy.PASSWORD_PROTECTED, | ||
440 | videoPasswords: [ correctPassword ] | ||
441 | }) | ||
442 | passwordProtectedLiveId = video.uuid | ||
443 | passwordProtectedLive = live | ||
444 | } | ||
445 | }) | ||
446 | |||
447 | it('Should create a private normal live and have a private static path', async function () { | ||
448 | this.timeout(240000) | ||
449 | |||
450 | await checkLiveFiles(normalLive, normalLiveId) | ||
451 | }) | ||
452 | |||
453 | it('Should create a private permanent live and have a private static path', async function () { | ||
454 | this.timeout(240000) | ||
455 | |||
456 | await checkLiveFiles(permanentLive, permanentLiveId) | ||
457 | }) | ||
458 | |||
459 | it('Should create a password protected live and have a private static path', async function () { | ||
460 | this.timeout(240000) | ||
461 | |||
462 | await checkLiveFiles(passwordProtectedLive, passwordProtectedLiveId, correctPassword) | ||
463 | }) | ||
464 | |||
465 | it('Should reinject video file token in permanent live', async function () { | ||
466 | this.timeout(240000) | ||
467 | |||
468 | const ffmpegCommand = sendRTMPStream({ rtmpBaseUrl: permanentLive.rtmpUrl, streamKey: permanentLive.streamKey }) | ||
469 | await server.live.waitUntilPublished({ videoId: permanentLiveId }) | ||
470 | |||
471 | const video = await server.videos.getWithToken({ id: permanentLiveId }) | ||
472 | const videoFileToken = await server.videoToken.getVideoFileToken({ videoId: video.uuid }) | ||
473 | |||
474 | await checkVideoFileTokenReinjection({ | ||
475 | server, | ||
476 | videoUUID: permanentLiveId, | ||
477 | videoFileToken, | ||
478 | resolutions: [ 720 ], | ||
479 | isLive: true | ||
480 | }) | ||
481 | |||
482 | await stopFfmpeg(ffmpegCommand) | ||
483 | }) | ||
484 | |||
485 | it('Should have created a replay of the normal live with a private static path', async function () { | ||
486 | this.timeout(240000) | ||
487 | |||
488 | await server.live.waitUntilReplacedByReplay({ videoId: normalLiveId }) | ||
489 | |||
490 | const replay = await server.videos.getWithToken({ id: normalLiveId }) | ||
491 | await checkReplay(replay) | ||
492 | }) | ||
493 | |||
494 | it('Should have created a replay of the permanent live with a private static path', async function () { | ||
495 | this.timeout(240000) | ||
496 | |||
497 | await server.live.waitUntilWaiting({ videoId: permanentLiveId }) | ||
498 | await waitJobs([ server ]) | ||
499 | |||
500 | const live = await server.videos.getWithToken({ id: permanentLiveId }) | ||
501 | const replayFromList = await findExternalSavedVideo(server, live) | ||
502 | const replay = await server.videos.getWithToken({ id: replayFromList.id }) | ||
503 | |||
504 | await checkReplay(replay) | ||
505 | }) | ||
506 | }) | ||
507 | |||
508 | describe('With private files proxy disabled and public ACL for private files', function () { | ||
509 | let videoUUID: string | ||
510 | |||
511 | before(async function () { | ||
512 | this.timeout(240000) | ||
513 | |||
514 | await server.kill() | ||
515 | |||
516 | const config = ObjectStorageCommand.getDefaultScalewayConfig({ | ||
517 | serverNumber: 1, | ||
518 | enablePrivateProxy: false, | ||
519 | privateACL: 'public-read' | ||
520 | }) | ||
521 | await server.run(config) | ||
522 | |||
523 | const { uuid } = await server.videos.quickUpload({ name: 'video', privacy: VideoPrivacy.PRIVATE }) | ||
524 | videoUUID = uuid | ||
525 | |||
526 | await waitJobs([ server ]) | ||
527 | }) | ||
528 | |||
529 | it('Should display object storage path for a private video and be able to access them', async function () { | ||
530 | this.timeout(60000) | ||
531 | |||
532 | await checkPublicVODFiles(videoUUID) | ||
533 | }) | ||
534 | |||
535 | it('Should not be able to access object storage proxy', async function () { | ||
536 | const privateVideo = await server.videos.getWithToken({ id: videoUUID }) | ||
537 | const webVideoFilename = extractFilenameFromUrl(privateVideo.files[0].fileUrl) | ||
538 | const hlsFilename = extractFilenameFromUrl(getHLS(privateVideo).files[0].fileUrl) | ||
539 | |||
540 | await makeRawRequest({ | ||
541 | url: server.url + '/object-storage-proxy/web-videos/private/' + webVideoFilename, | ||
542 | token: server.accessToken, | ||
543 | expectedStatus: HttpStatusCode.BAD_REQUEST_400 | ||
544 | }) | ||
545 | |||
546 | await makeRawRequest({ | ||
547 | url: server.url + '/object-storage-proxy/streaming-playlists/hls/private/' + videoUUID + '/' + hlsFilename, | ||
548 | token: server.accessToken, | ||
549 | expectedStatus: HttpStatusCode.BAD_REQUEST_400 | ||
550 | }) | ||
551 | }) | ||
552 | }) | ||
553 | |||
554 | after(async function () { | ||
555 | this.timeout(240000) | ||
556 | |||
557 | const { data } = await server.videos.listAllForAdmin() | ||
558 | |||
559 | for (const v of data) { | ||
560 | await server.videos.remove({ id: v.uuid }) | ||
561 | } | ||
562 | |||
563 | for (const v of data) { | ||
564 | await server.servers.waitUntilLog('Removed files of video ' + v.url) | ||
565 | } | ||
566 | |||
567 | await sqlCommand.cleanup() | ||
568 | await cleanupTests([ server ]) | ||
569 | }) | ||
570 | }) | ||
diff --git a/server/tests/api/object-storage/videos.ts b/server/tests/api/object-storage/videos.ts deleted file mode 100644 index dcc52ef06..000000000 --- a/server/tests/api/object-storage/videos.ts +++ /dev/null | |||
@@ -1,438 +0,0 @@ | |||
1 | /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ | ||
2 | |||
3 | import bytes from 'bytes' | ||
4 | import { expect } from 'chai' | ||
5 | import { stat } from 'fs-extra' | ||
6 | import { merge } from 'lodash' | ||
7 | import { | ||
8 | checkTmpIsEmpty, | ||
9 | checkWebTorrentWorks, | ||
10 | expectLogDoesNotContain, | ||
11 | expectStartWith, | ||
12 | generateHighBitrateVideo, | ||
13 | MockObjectStorageProxy, | ||
14 | SQLCommand | ||
15 | } from '@server/tests/shared' | ||
16 | import { areMockObjectStorageTestsDisabled } from '@shared/core-utils' | ||
17 | import { sha1 } from '@shared/extra-utils' | ||
18 | import { HttpStatusCode, VideoDetails } from '@shared/models' | ||
19 | import { | ||
20 | cleanupTests, | ||
21 | createMultipleServers, | ||
22 | createSingleServer, | ||
23 | doubleFollow, | ||
24 | killallServers, | ||
25 | makeRawRequest, | ||
26 | ObjectStorageCommand, | ||
27 | PeerTubeServer, | ||
28 | setAccessTokensToServers, | ||
29 | waitJobs | ||
30 | } from '@shared/server-commands' | ||
31 | |||
32 | async function checkFiles (options: { | ||
33 | server: PeerTubeServer | ||
34 | originServer: PeerTubeServer | ||
35 | originSQLCommand: SQLCommand | ||
36 | |||
37 | video: VideoDetails | ||
38 | |||
39 | baseMockUrl?: string | ||
40 | |||
41 | playlistBucket: string | ||
42 | playlistPrefix?: string | ||
43 | |||
44 | webVideoBucket: string | ||
45 | webVideoPrefix?: string | ||
46 | }) { | ||
47 | const { | ||
48 | server, | ||
49 | originServer, | ||
50 | originSQLCommand, | ||
51 | video, | ||
52 | playlistBucket, | ||
53 | webVideoBucket, | ||
54 | baseMockUrl, | ||
55 | playlistPrefix, | ||
56 | webVideoPrefix | ||
57 | } = options | ||
58 | |||
59 | let allFiles = video.files | ||
60 | |||
61 | for (const file of video.files) { | ||
62 | const baseUrl = baseMockUrl | ||
63 | ? `${baseMockUrl}/${webVideoBucket}/` | ||
64 | : `http://${webVideoBucket}.${ObjectStorageCommand.getMockEndpointHost()}/` | ||
65 | |||
66 | const prefix = webVideoPrefix || '' | ||
67 | const start = baseUrl + prefix | ||
68 | |||
69 | expectStartWith(file.fileUrl, start) | ||
70 | |||
71 | const res = await makeRawRequest({ url: file.fileDownloadUrl, expectedStatus: HttpStatusCode.FOUND_302 }) | ||
72 | const location = res.headers['location'] | ||
73 | expectStartWith(location, start) | ||
74 | |||
75 | await makeRawRequest({ url: location, expectedStatus: HttpStatusCode.OK_200 }) | ||
76 | } | ||
77 | |||
78 | const hls = video.streamingPlaylists[0] | ||
79 | |||
80 | if (hls) { | ||
81 | allFiles = allFiles.concat(hls.files) | ||
82 | |||
83 | const baseUrl = baseMockUrl | ||
84 | ? `${baseMockUrl}/${playlistBucket}/` | ||
85 | : `http://${playlistBucket}.${ObjectStorageCommand.getMockEndpointHost()}/` | ||
86 | |||
87 | const prefix = playlistPrefix || '' | ||
88 | const start = baseUrl + prefix | ||
89 | |||
90 | expectStartWith(hls.playlistUrl, start) | ||
91 | expectStartWith(hls.segmentsSha256Url, start) | ||
92 | |||
93 | await makeRawRequest({ url: hls.playlistUrl, expectedStatus: HttpStatusCode.OK_200 }) | ||
94 | |||
95 | const resSha = await makeRawRequest({ url: hls.segmentsSha256Url, expectedStatus: HttpStatusCode.OK_200 }) | ||
96 | expect(JSON.stringify(resSha.body)).to.not.throw | ||
97 | |||
98 | let i = 0 | ||
99 | for (const file of hls.files) { | ||
100 | expectStartWith(file.fileUrl, start) | ||
101 | |||
102 | const res = await makeRawRequest({ url: file.fileDownloadUrl, expectedStatus: HttpStatusCode.FOUND_302 }) | ||
103 | const location = res.headers['location'] | ||
104 | expectStartWith(location, start) | ||
105 | |||
106 | await makeRawRequest({ url: location, expectedStatus: HttpStatusCode.OK_200 }) | ||
107 | |||
108 | if (originServer.internalServerNumber === server.internalServerNumber) { | ||
109 | const infohash = sha1(`${2 + hls.playlistUrl}+V${i}`) | ||
110 | const dbInfohashes = await originSQLCommand.getPlaylistInfohash(hls.id) | ||
111 | |||
112 | expect(dbInfohashes).to.include(infohash) | ||
113 | } | ||
114 | |||
115 | i++ | ||
116 | } | ||
117 | } | ||
118 | |||
119 | for (const file of allFiles) { | ||
120 | await checkWebTorrentWorks(file.magnetUri) | ||
121 | |||
122 | const res = await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 }) | ||
123 | expect(res.body).to.have.length.above(100) | ||
124 | } | ||
125 | |||
126 | return allFiles.map(f => f.fileUrl) | ||
127 | } | ||
128 | |||
129 | function runTestSuite (options: { | ||
130 | fixture?: string | ||
131 | |||
132 | maxUploadPart?: string | ||
133 | |||
134 | playlistBucket: string | ||
135 | playlistPrefix?: string | ||
136 | |||
137 | webVideoBucket: string | ||
138 | webVideoPrefix?: string | ||
139 | |||
140 | useMockBaseUrl?: boolean | ||
141 | }) { | ||
142 | const mockObjectStorageProxy = new MockObjectStorageProxy() | ||
143 | const { fixture } = options | ||
144 | let baseMockUrl: string | ||
145 | |||
146 | let servers: PeerTubeServer[] | ||
147 | let sqlCommands: SQLCommand[] = [] | ||
148 | const objectStorage = new ObjectStorageCommand() | ||
149 | |||
150 | let keptUrls: string[] = [] | ||
151 | |||
152 | const uuidsToDelete: string[] = [] | ||
153 | let deletedUrls: string[] = [] | ||
154 | |||
155 | before(async function () { | ||
156 | this.timeout(240000) | ||
157 | |||
158 | const port = await mockObjectStorageProxy.initialize() | ||
159 | baseMockUrl = options.useMockBaseUrl | ||
160 | ? `http://127.0.0.1:${port}` | ||
161 | : undefined | ||
162 | |||
163 | await objectStorage.createMockBucket(options.playlistBucket) | ||
164 | await objectStorage.createMockBucket(options.webVideoBucket) | ||
165 | |||
166 | const config = { | ||
167 | object_storage: { | ||
168 | enabled: true, | ||
169 | endpoint: 'http://' + ObjectStorageCommand.getMockEndpointHost(), | ||
170 | region: ObjectStorageCommand.getMockRegion(), | ||
171 | |||
172 | credentials: ObjectStorageCommand.getMockCredentialsConfig(), | ||
173 | |||
174 | max_upload_part: options.maxUploadPart || '5MB', | ||
175 | |||
176 | streaming_playlists: { | ||
177 | bucket_name: options.playlistBucket, | ||
178 | prefix: options.playlistPrefix, | ||
179 | base_url: baseMockUrl | ||
180 | ? `${baseMockUrl}/${options.playlistBucket}` | ||
181 | : undefined | ||
182 | }, | ||
183 | |||
184 | web_videos: { | ||
185 | bucket_name: options.webVideoBucket, | ||
186 | prefix: options.webVideoPrefix, | ||
187 | base_url: baseMockUrl | ||
188 | ? `${baseMockUrl}/${options.webVideoBucket}` | ||
189 | : undefined | ||
190 | } | ||
191 | } | ||
192 | } | ||
193 | |||
194 | servers = await createMultipleServers(2, config) | ||
195 | |||
196 | await setAccessTokensToServers(servers) | ||
197 | await doubleFollow(servers[0], servers[1]) | ||
198 | |||
199 | for (const server of servers) { | ||
200 | const { uuid } = await server.videos.quickUpload({ name: 'video to keep' }) | ||
201 | await waitJobs(servers) | ||
202 | |||
203 | const files = await server.videos.listFiles({ id: uuid }) | ||
204 | keptUrls = keptUrls.concat(files.map(f => f.fileUrl)) | ||
205 | } | ||
206 | |||
207 | sqlCommands = servers.map(s => new SQLCommand(s)) | ||
208 | }) | ||
209 | |||
210 | it('Should upload a video and move it to the object storage without transcoding', async function () { | ||
211 | this.timeout(40000) | ||
212 | |||
213 | const { uuid } = await servers[0].videos.quickUpload({ name: 'video 1', fixture }) | ||
214 | uuidsToDelete.push(uuid) | ||
215 | |||
216 | await waitJobs(servers) | ||
217 | |||
218 | for (const server of servers) { | ||
219 | const video = await server.videos.get({ id: uuid }) | ||
220 | const files = await checkFiles({ ...options, server, originServer: servers[0], originSQLCommand: sqlCommands[0], video, baseMockUrl }) | ||
221 | |||
222 | deletedUrls = deletedUrls.concat(files) | ||
223 | } | ||
224 | }) | ||
225 | |||
226 | it('Should upload a video and move it to the object storage with transcoding', async function () { | ||
227 | this.timeout(120000) | ||
228 | |||
229 | const { uuid } = await servers[1].videos.quickUpload({ name: 'video 2', fixture }) | ||
230 | uuidsToDelete.push(uuid) | ||
231 | |||
232 | await waitJobs(servers) | ||
233 | |||
234 | for (const server of servers) { | ||
235 | const video = await server.videos.get({ id: uuid }) | ||
236 | const files = await checkFiles({ ...options, server, originServer: servers[0], originSQLCommand: sqlCommands[0], video, baseMockUrl }) | ||
237 | |||
238 | deletedUrls = deletedUrls.concat(files) | ||
239 | } | ||
240 | }) | ||
241 | |||
242 | it('Should fetch correctly all the files', async function () { | ||
243 | for (const url of deletedUrls.concat(keptUrls)) { | ||
244 | await makeRawRequest({ url, expectedStatus: HttpStatusCode.OK_200 }) | ||
245 | } | ||
246 | }) | ||
247 | |||
248 | it('Should correctly delete the files', async function () { | ||
249 | await servers[0].videos.remove({ id: uuidsToDelete[0] }) | ||
250 | await servers[1].videos.remove({ id: uuidsToDelete[1] }) | ||
251 | |||
252 | await waitJobs(servers) | ||
253 | |||
254 | for (const url of deletedUrls) { | ||
255 | await makeRawRequest({ url, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) | ||
256 | } | ||
257 | }) | ||
258 | |||
259 | it('Should have kept other files', async function () { | ||
260 | for (const url of keptUrls) { | ||
261 | await makeRawRequest({ url, expectedStatus: HttpStatusCode.OK_200 }) | ||
262 | } | ||
263 | }) | ||
264 | |||
265 | it('Should have an empty tmp directory', async function () { | ||
266 | for (const server of servers) { | ||
267 | await checkTmpIsEmpty(server) | ||
268 | } | ||
269 | }) | ||
270 | |||
271 | it('Should not have downloaded files from object storage', async function () { | ||
272 | for (const server of servers) { | ||
273 | await expectLogDoesNotContain(server, 'from object storage') | ||
274 | } | ||
275 | }) | ||
276 | |||
277 | after(async function () { | ||
278 | await mockObjectStorageProxy.terminate() | ||
279 | await objectStorage.cleanupMock() | ||
280 | |||
281 | for (const sqlCommand of sqlCommands) { | ||
282 | await sqlCommand.cleanup() | ||
283 | } | ||
284 | |||
285 | await cleanupTests(servers) | ||
286 | }) | ||
287 | } | ||
288 | |||
289 | describe('Object storage for videos', function () { | ||
290 | if (areMockObjectStorageTestsDisabled()) return | ||
291 | |||
292 | const objectStorage = new ObjectStorageCommand() | ||
293 | |||
294 | describe('Test config', function () { | ||
295 | let server: PeerTubeServer | ||
296 | |||
297 | const baseConfig = objectStorage.getDefaultMockConfig() | ||
298 | |||
299 | const badCredentials = { | ||
300 | access_key_id: 'AKIAIOSFODNN7EXAMPLE', | ||
301 | secret_access_key: 'aJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY' | ||
302 | } | ||
303 | |||
304 | it('Should fail with same bucket names without prefix', function (done) { | ||
305 | const config = merge({}, baseConfig, { | ||
306 | object_storage: { | ||
307 | streaming_playlists: { | ||
308 | bucket_name: 'aaa' | ||
309 | }, | ||
310 | |||
311 | web_videos: { | ||
312 | bucket_name: 'aaa' | ||
313 | } | ||
314 | } | ||
315 | }) | ||
316 | |||
317 | createSingleServer(1, config) | ||
318 | .then(() => done(new Error('Did not throw'))) | ||
319 | .catch(() => done()) | ||
320 | }) | ||
321 | |||
322 | it('Should fail with bad credentials', async function () { | ||
323 | this.timeout(60000) | ||
324 | |||
325 | await objectStorage.prepareDefaultMockBuckets() | ||
326 | |||
327 | const config = merge({}, baseConfig, { | ||
328 | object_storage: { | ||
329 | credentials: badCredentials | ||
330 | } | ||
331 | }) | ||
332 | |||
333 | server = await createSingleServer(1, config) | ||
334 | await setAccessTokensToServers([ server ]) | ||
335 | |||
336 | const { uuid } = await server.videos.quickUpload({ name: 'video' }) | ||
337 | |||
338 | await waitJobs([ server ], { skipDelayed: true }) | ||
339 | const video = await server.videos.get({ id: uuid }) | ||
340 | |||
341 | expectStartWith(video.files[0].fileUrl, server.url) | ||
342 | |||
343 | await killallServers([ server ]) | ||
344 | }) | ||
345 | |||
346 | it('Should succeed with credentials from env', async function () { | ||
347 | this.timeout(60000) | ||
348 | |||
349 | await objectStorage.prepareDefaultMockBuckets() | ||
350 | |||
351 | const config = merge({}, baseConfig, { | ||
352 | object_storage: { | ||
353 | credentials: { | ||
354 | access_key_id: '', | ||
355 | secret_access_key: '' | ||
356 | } | ||
357 | } | ||
358 | }) | ||
359 | |||
360 | const goodCredentials = ObjectStorageCommand.getMockCredentialsConfig() | ||
361 | |||
362 | server = await createSingleServer(1, config, { | ||
363 | env: { | ||
364 | AWS_ACCESS_KEY_ID: goodCredentials.access_key_id, | ||
365 | AWS_SECRET_ACCESS_KEY: goodCredentials.secret_access_key | ||
366 | } | ||
367 | }) | ||
368 | |||
369 | await setAccessTokensToServers([ server ]) | ||
370 | |||
371 | const { uuid } = await server.videos.quickUpload({ name: 'video' }) | ||
372 | |||
373 | await waitJobs([ server ], { skipDelayed: true }) | ||
374 | const video = await server.videos.get({ id: uuid }) | ||
375 | |||
376 | expectStartWith(video.files[0].fileUrl, objectStorage.getMockWebVideosBaseUrl()) | ||
377 | }) | ||
378 | |||
379 | after(async function () { | ||
380 | await objectStorage.cleanupMock() | ||
381 | |||
382 | await cleanupTests([ server ]) | ||
383 | }) | ||
384 | }) | ||
385 | |||
386 | describe('Test simple object storage', function () { | ||
387 | runTestSuite({ | ||
388 | playlistBucket: objectStorage.getMockBucketName('streaming-playlists'), | ||
389 | webVideoBucket: objectStorage.getMockBucketName('web-videos') | ||
390 | }) | ||
391 | }) | ||
392 | |||
393 | describe('Test object storage with prefix', function () { | ||
394 | runTestSuite({ | ||
395 | playlistBucket: objectStorage.getMockBucketName('mybucket'), | ||
396 | webVideoBucket: objectStorage.getMockBucketName('mybucket'), | ||
397 | |||
398 | playlistPrefix: 'streaming-playlists_', | ||
399 | webVideoPrefix: 'webvideo_' | ||
400 | }) | ||
401 | }) | ||
402 | |||
403 | describe('Test object storage with prefix and base URL', function () { | ||
404 | runTestSuite({ | ||
405 | playlistBucket: objectStorage.getMockBucketName('mybucket'), | ||
406 | webVideoBucket: objectStorage.getMockBucketName('mybucket'), | ||
407 | |||
408 | playlistPrefix: 'streaming-playlists/', | ||
409 | webVideoPrefix: 'webvideo/', | ||
410 | |||
411 | useMockBaseUrl: true | ||
412 | }) | ||
413 | }) | ||
414 | |||
415 | describe('Test object storage with file bigger than upload part', function () { | ||
416 | let fixture: string | ||
417 | const maxUploadPart = '5MB' | ||
418 | |||
419 | before(async function () { | ||
420 | this.timeout(120000) | ||
421 | |||
422 | fixture = await generateHighBitrateVideo() | ||
423 | |||
424 | const { size } = await stat(fixture) | ||
425 | |||
426 | if (bytes.parse(maxUploadPart) > size) { | ||
427 | throw Error(`Fixture file is too small (${size}) to make sense for this test.`) | ||
428 | } | ||
429 | }) | ||
430 | |||
431 | runTestSuite({ | ||
432 | maxUploadPart, | ||
433 | playlistBucket: objectStorage.getMockBucketName('streaming-playlists'), | ||
434 | webVideoBucket: objectStorage.getMockBucketName('web-videos'), | ||
435 | fixture | ||
436 | }) | ||
437 | }) | ||
438 | }) | ||