aboutsummaryrefslogtreecommitdiffhomepage
path: root/server/tests/api
diff options
context:
space:
mode:
Diffstat (limited to 'server/tests/api')
-rw-r--r--server/tests/api/index.ts1
-rw-r--r--server/tests/api/live/live-save-replay.ts30
-rw-r--r--server/tests/api/object-storage/index.ts3
-rw-r--r--server/tests/api/object-storage/live.ts136
-rw-r--r--server/tests/api/object-storage/video-imports.ts112
-rw-r--r--server/tests/api/object-storage/videos.ts391
-rw-r--r--server/tests/api/redundancy/redundancy.ts6
-rw-r--r--server/tests/api/videos/video-hls.ts73
8 files changed, 715 insertions, 37 deletions
diff --git a/server/tests/api/index.ts b/server/tests/api/index.ts
index b62e2f5f7..19301c0b9 100644
--- a/server/tests/api/index.ts
+++ b/server/tests/api/index.ts
@@ -2,6 +2,7 @@
2import './activitypub' 2import './activitypub'
3import './check-params' 3import './check-params'
4import './moderation' 4import './moderation'
5import './object-storage'
5import './notifications' 6import './notifications'
6import './redundancy' 7import './redundancy'
7import './search' 8import './search'
diff --git a/server/tests/api/live/live-save-replay.ts b/server/tests/api/live/live-save-replay.ts
index 8f1fb78a5..6c4ea90ca 100644
--- a/server/tests/api/live/live-save-replay.ts
+++ b/server/tests/api/live/live-save-replay.ts
@@ -15,7 +15,9 @@ import {
15 stopFfmpeg, 15 stopFfmpeg,
16 testFfmpegStreamError, 16 testFfmpegStreamError,
17 wait, 17 wait,
18 waitJobs 18 waitJobs,
19 waitUntilLivePublishedOnAllServers,
20 waitUntilLiveSavedOnAllServers
19} from '@shared/extra-utils' 21} from '@shared/extra-utils'
20import { HttpStatusCode, LiveVideoCreate, VideoPrivacy, VideoState } from '@shared/models' 22import { HttpStatusCode, LiveVideoCreate, VideoPrivacy, VideoState } from '@shared/models'
21 23
@@ -66,18 +68,6 @@ describe('Save replay setting', function () {
66 } 68 }
67 } 69 }
68 70
69 async function waitUntilLivePublishedOnAllServers (videoId: string) {
70 for (const server of servers) {
71 await server.live.waitUntilPublished({ videoId })
72 }
73 }
74
75 async function waitUntilLiveSavedOnAllServers (videoId: string) {
76 for (const server of servers) {
77 await server.live.waitUntilSaved({ videoId })
78 }
79 }
80
81 before(async function () { 71 before(async function () {
82 this.timeout(120000) 72 this.timeout(120000)
83 73
@@ -127,7 +117,7 @@ describe('Save replay setting', function () {
127 117
128 ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID }) 118 ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID })
129 119
130 await waitUntilLivePublishedOnAllServers(liveVideoUUID) 120 await waitUntilLivePublishedOnAllServers(servers, liveVideoUUID)
131 121
132 await waitJobs(servers) 122 await waitJobs(servers)
133 123
@@ -160,7 +150,7 @@ describe('Save replay setting', function () {
160 150
161 ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID }) 151 ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID })
162 152
163 await waitUntilLivePublishedOnAllServers(liveVideoUUID) 153 await waitUntilLivePublishedOnAllServers(servers, liveVideoUUID)
164 154
165 await waitJobs(servers) 155 await waitJobs(servers)
166 await checkVideosExist(liveVideoUUID, true, HttpStatusCode.OK_200) 156 await checkVideosExist(liveVideoUUID, true, HttpStatusCode.OK_200)
@@ -189,7 +179,7 @@ describe('Save replay setting', function () {
189 179
190 ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID }) 180 ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID })
191 181
192 await waitUntilLivePublishedOnAllServers(liveVideoUUID) 182 await waitUntilLivePublishedOnAllServers(servers, liveVideoUUID)
193 183
194 await waitJobs(servers) 184 await waitJobs(servers)
195 await checkVideosExist(liveVideoUUID, true, HttpStatusCode.OK_200) 185 await checkVideosExist(liveVideoUUID, true, HttpStatusCode.OK_200)
@@ -224,7 +214,7 @@ describe('Save replay setting', function () {
224 this.timeout(20000) 214 this.timeout(20000)
225 215
226 ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID }) 216 ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID })
227 await waitUntilLivePublishedOnAllServers(liveVideoUUID) 217 await waitUntilLivePublishedOnAllServers(servers, liveVideoUUID)
228 218
229 await waitJobs(servers) 219 await waitJobs(servers)
230 220
@@ -237,7 +227,7 @@ describe('Save replay setting', function () {
237 227
238 await stopFfmpeg(ffmpegCommand) 228 await stopFfmpeg(ffmpegCommand)
239 229
240 await waitUntilLiveSavedOnAllServers(liveVideoUUID) 230 await waitUntilLiveSavedOnAllServers(servers, liveVideoUUID)
241 await waitJobs(servers) 231 await waitJobs(servers)
242 232
243 // Live has been transcoded 233 // Live has been transcoded
@@ -268,7 +258,7 @@ describe('Save replay setting', function () {
268 liveVideoUUID = await createLiveWrapper(true) 258 liveVideoUUID = await createLiveWrapper(true)
269 259
270 ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID }) 260 ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID })
271 await waitUntilLivePublishedOnAllServers(liveVideoUUID) 261 await waitUntilLivePublishedOnAllServers(servers, liveVideoUUID)
272 262
273 await waitJobs(servers) 263 await waitJobs(servers)
274 await checkVideosExist(liveVideoUUID, true, HttpStatusCode.OK_200) 264 await checkVideosExist(liveVideoUUID, true, HttpStatusCode.OK_200)
@@ -296,7 +286,7 @@ describe('Save replay setting', function () {
296 liveVideoUUID = await createLiveWrapper(true) 286 liveVideoUUID = await createLiveWrapper(true)
297 287
298 ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID }) 288 ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: liveVideoUUID })
299 await waitUntilLivePublishedOnAllServers(liveVideoUUID) 289 await waitUntilLivePublishedOnAllServers(servers, liveVideoUUID)
300 290
301 await waitJobs(servers) 291 await waitJobs(servers)
302 await checkVideosExist(liveVideoUUID, true, HttpStatusCode.OK_200) 292 await checkVideosExist(liveVideoUUID, true, HttpStatusCode.OK_200)
diff --git a/server/tests/api/object-storage/index.ts b/server/tests/api/object-storage/index.ts
new file mode 100644
index 000000000..f319d6ef5
--- /dev/null
+++ b/server/tests/api/object-storage/index.ts
@@ -0,0 +1,3 @@
1export * from './live'
2export * from './video-imports'
3export * from './videos'
diff --git a/server/tests/api/object-storage/live.ts b/server/tests/api/object-storage/live.ts
new file mode 100644
index 000000000..d3e6777f2
--- /dev/null
+++ b/server/tests/api/object-storage/live.ts
@@ -0,0 +1,136 @@
1/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
2
3import 'mocha'
4import * as chai from 'chai'
5import { FfmpegCommand } from 'fluent-ffmpeg'
6import {
7 areObjectStorageTestsDisabled,
8 createMultipleServers,
9 doubleFollow,
10 expectStartWith,
11 killallServers,
12 makeRawRequest,
13 ObjectStorageCommand,
14 PeerTubeServer,
15 setAccessTokensToServers,
16 setDefaultVideoChannel,
17 stopFfmpeg,
18 waitJobs,
19 waitUntilLivePublishedOnAllServers,
20 waitUntilLiveSavedOnAllServers
21} from '@shared/extra-utils'
22import { HttpStatusCode, LiveVideoCreate, VideoFile, VideoPrivacy } from '@shared/models'
23
24const expect = chai.expect
25
26async function createLive (server: PeerTubeServer) {
27 const attributes: LiveVideoCreate = {
28 channelId: server.store.channel.id,
29 privacy: VideoPrivacy.PUBLIC,
30 name: 'my super live',
31 saveReplay: true
32 }
33
34 const { uuid } = await server.live.create({ fields: attributes })
35
36 return uuid
37}
38
39async function checkFiles (files: VideoFile[]) {
40 for (const file of files) {
41 expectStartWith(file.fileUrl, ObjectStorageCommand.getPlaylistBaseUrl())
42
43 await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200)
44 }
45}
46
47describe('Object storage for lives', function () {
48 if (areObjectStorageTestsDisabled()) return
49
50 let ffmpegCommand: FfmpegCommand
51 let servers: PeerTubeServer[]
52 let videoUUID: string
53
54 before(async function () {
55 this.timeout(120000)
56
57 await ObjectStorageCommand.prepareDefaultBuckets()
58
59 servers = await createMultipleServers(2, ObjectStorageCommand.getDefaultConfig())
60
61 await setAccessTokensToServers(servers)
62 await setDefaultVideoChannel(servers)
63 await doubleFollow(servers[0], servers[1])
64
65 await servers[0].config.enableTranscoding()
66 })
67
68 describe('Without live transcoding', async function () {
69
70 before(async function () {
71 await servers[0].config.enableLive({ transcoding: false })
72
73 videoUUID = await createLive(servers[0])
74 })
75
76 it('Should create a live and save the replay on object storage', async function () {
77 this.timeout(220000)
78
79 ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: videoUUID })
80 await waitUntilLivePublishedOnAllServers(servers, videoUUID)
81
82 await stopFfmpeg(ffmpegCommand)
83
84 await waitUntilLiveSavedOnAllServers(servers, videoUUID)
85 await waitJobs(servers)
86
87 for (const server of servers) {
88 const video = await server.videos.get({ id: videoUUID })
89
90 expect(video.files).to.have.lengthOf(0)
91 expect(video.streamingPlaylists).to.have.lengthOf(1)
92
93 const files = video.streamingPlaylists[0].files
94
95 await checkFiles(files)
96 }
97 })
98 })
99
100 describe('With live transcoding', async function () {
101
102 before(async function () {
103 await servers[0].config.enableLive({ transcoding: true })
104
105 videoUUID = await createLive(servers[0])
106 })
107
108 it('Should import a video and have sent it to object storage', async function () {
109 this.timeout(240000)
110
111 ffmpegCommand = await servers[0].live.sendRTMPStreamInVideo({ videoId: videoUUID })
112 await waitUntilLivePublishedOnAllServers(servers, videoUUID)
113
114 await stopFfmpeg(ffmpegCommand)
115
116 await waitUntilLiveSavedOnAllServers(servers, videoUUID)
117 await waitJobs(servers)
118
119 for (const server of servers) {
120 const video = await server.videos.get({ id: videoUUID })
121
122 expect(video.files).to.have.lengthOf(0)
123 expect(video.streamingPlaylists).to.have.lengthOf(1)
124
125 const files = video.streamingPlaylists[0].files
126 expect(files).to.have.lengthOf(4)
127
128 await checkFiles(files)
129 }
130 })
131 })
132
133 after(async function () {
134 await killallServers(servers)
135 })
136})
diff --git a/server/tests/api/object-storage/video-imports.ts b/server/tests/api/object-storage/video-imports.ts
new file mode 100644
index 000000000..efc01f550
--- /dev/null
+++ b/server/tests/api/object-storage/video-imports.ts
@@ -0,0 +1,112 @@
1/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
2
3import 'mocha'
4import * as chai from 'chai'
5import {
6 areObjectStorageTestsDisabled,
7 createSingleServer,
8 expectStartWith,
9 FIXTURE_URLS,
10 killallServers,
11 makeRawRequest,
12 ObjectStorageCommand,
13 PeerTubeServer,
14 setAccessTokensToServers,
15 setDefaultVideoChannel,
16 waitJobs
17} from '@shared/extra-utils'
18import { HttpStatusCode, VideoPrivacy } from '@shared/models'
19
20const expect = chai.expect
21
22async function importVideo (server: PeerTubeServer) {
23 const attributes = {
24 name: 'import 2',
25 privacy: VideoPrivacy.PUBLIC,
26 channelId: server.store.channel.id,
27 targetUrl: FIXTURE_URLS.goodVideo720
28 }
29
30 const { video: { uuid } } = await server.imports.importVideo({ attributes })
31
32 return uuid
33}
34
35describe('Object storage for video import', function () {
36 if (areObjectStorageTestsDisabled()) return
37
38 let server: PeerTubeServer
39
40 before(async function () {
41 this.timeout(120000)
42
43 await ObjectStorageCommand.prepareDefaultBuckets()
44
45 server = await createSingleServer(1, ObjectStorageCommand.getDefaultConfig())
46
47 await setAccessTokensToServers([ server ])
48 await setDefaultVideoChannel([ server ])
49
50 await server.config.enableImports()
51 })
52
53 describe('Without transcoding', async function () {
54
55 before(async function () {
56 await server.config.disableTranscoding()
57 })
58
59 it('Should import a video and have sent it to object storage', async function () {
60 this.timeout(120000)
61
62 const uuid = await importVideo(server)
63 await waitJobs(server)
64
65 const video = await server.videos.get({ id: uuid })
66
67 expect(video.files).to.have.lengthOf(1)
68 expect(video.streamingPlaylists).to.have.lengthOf(0)
69
70 const fileUrl = video.files[0].fileUrl
71 expectStartWith(fileUrl, ObjectStorageCommand.getWebTorrentBaseUrl())
72
73 await makeRawRequest(fileUrl, HttpStatusCode.OK_200)
74 })
75 })
76
77 describe('With transcoding', async function () {
78
79 before(async function () {
80 await server.config.enableTranscoding()
81 })
82
83 it('Should import a video and have sent it to object storage', async function () {
84 this.timeout(120000)
85
86 const uuid = await importVideo(server)
87 await waitJobs(server)
88
89 const video = await server.videos.get({ id: uuid })
90
91 expect(video.files).to.have.lengthOf(4)
92 expect(video.streamingPlaylists).to.have.lengthOf(1)
93 expect(video.streamingPlaylists[0].files).to.have.lengthOf(4)
94
95 for (const file of video.files) {
96 expectStartWith(file.fileUrl, ObjectStorageCommand.getWebTorrentBaseUrl())
97
98 await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200)
99 }
100
101 for (const file of video.streamingPlaylists[0].files) {
102 expectStartWith(file.fileUrl, ObjectStorageCommand.getPlaylistBaseUrl())
103
104 await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200)
105 }
106 })
107 })
108
109 after(async function () {
110 await killallServers([ server ])
111 })
112})
diff --git a/server/tests/api/object-storage/videos.ts b/server/tests/api/object-storage/videos.ts
new file mode 100644
index 000000000..3958bd3d7
--- /dev/null
+++ b/server/tests/api/object-storage/videos.ts
@@ -0,0 +1,391 @@
1/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
2
3import 'mocha'
4import * as chai from 'chai'
5import { merge } from 'lodash'
6import {
7 areObjectStorageTestsDisabled,
8 checkTmpIsEmpty,
9 cleanupTests,
10 createMultipleServers,
11 createSingleServer,
12 doubleFollow,
13 expectStartWith,
14 killallServers,
15 makeRawRequest,
16 MockObjectStorage,
17 ObjectStorageCommand,
18 PeerTubeServer,
19 setAccessTokensToServers,
20 waitJobs,
21 webtorrentAdd
22} from '@shared/extra-utils'
23import { HttpStatusCode, VideoDetails } from '@shared/models'
24
25const expect = chai.expect
26
27async function checkFiles (options: {
28 video: VideoDetails
29
30 baseMockUrl?: string
31
32 playlistBucket: string
33 playlistPrefix?: string
34
35 webtorrentBucket: string
36 webtorrentPrefix?: string
37}) {
38 const {
39 video,
40 playlistBucket,
41 webtorrentBucket,
42 baseMockUrl,
43 playlistPrefix,
44 webtorrentPrefix
45 } = options
46
47 let allFiles = video.files
48
49 for (const file of video.files) {
50 const baseUrl = baseMockUrl
51 ? `${baseMockUrl}/${webtorrentBucket}/`
52 : `http://${webtorrentBucket}.${ObjectStorageCommand.getEndpointHost()}/`
53
54 const prefix = webtorrentPrefix || ''
55 const start = baseUrl + prefix
56
57 expectStartWith(file.fileUrl, start)
58
59 const res = await makeRawRequest(file.fileDownloadUrl, HttpStatusCode.FOUND_302)
60 const location = res.headers['location']
61 expectStartWith(location, start)
62
63 await makeRawRequest(location, HttpStatusCode.OK_200)
64 }
65
66 const hls = video.streamingPlaylists[0]
67
68 if (hls) {
69 allFiles = allFiles.concat(hls.files)
70
71 const baseUrl = baseMockUrl
72 ? `${baseMockUrl}/${playlistBucket}/`
73 : `http://${playlistBucket}.${ObjectStorageCommand.getEndpointHost()}/`
74
75 const prefix = playlistPrefix || ''
76 const start = baseUrl + prefix
77
78 expectStartWith(hls.playlistUrl, start)
79 expectStartWith(hls.segmentsSha256Url, start)
80
81 await makeRawRequest(hls.playlistUrl, HttpStatusCode.OK_200)
82
83 const resSha = await makeRawRequest(hls.segmentsSha256Url, HttpStatusCode.OK_200)
84 expect(JSON.stringify(resSha.body)).to.not.throw
85
86 for (const file of hls.files) {
87 expectStartWith(file.fileUrl, start)
88
89 const res = await makeRawRequest(file.fileDownloadUrl, HttpStatusCode.FOUND_302)
90 const location = res.headers['location']
91 expectStartWith(location, start)
92
93 await makeRawRequest(location, HttpStatusCode.OK_200)
94 }
95 }
96
97 for (const file of allFiles) {
98 const torrent = await webtorrentAdd(file.magnetUri, true)
99
100 expect(torrent.files).to.be.an('array')
101 expect(torrent.files.length).to.equal(1)
102 expect(torrent.files[0].path).to.exist.and.to.not.equal('')
103
104 const res = await makeRawRequest(file.fileUrl, HttpStatusCode.OK_200)
105 expect(res.body).to.have.length.above(100)
106 }
107
108 return allFiles.map(f => f.fileUrl)
109}
110
111function runTestSuite (options: {
112 playlistBucket: string
113 playlistPrefix?: string
114
115 webtorrentBucket: string
116 webtorrentPrefix?: string
117
118 useMockBaseUrl?: boolean
119
120 maxUploadPart?: string
121}) {
122 const mockObjectStorage = new MockObjectStorage()
123 let baseMockUrl: string
124
125 let servers: PeerTubeServer[]
126
127 let keptUrls: string[] = []
128
129 const uuidsToDelete: string[] = []
130 let deletedUrls: string[] = []
131
132 before(async function () {
133 this.timeout(120000)
134
135 const port = await mockObjectStorage.initialize()
136 baseMockUrl = options.useMockBaseUrl ? `http://localhost:${port}` : undefined
137
138 await ObjectStorageCommand.createBucket(options.playlistBucket)
139 await ObjectStorageCommand.createBucket(options.webtorrentBucket)
140
141 const config = {
142 object_storage: {
143 enabled: true,
144 endpoint: 'http://' + ObjectStorageCommand.getEndpointHost(),
145 region: ObjectStorageCommand.getRegion(),
146
147 credentials: ObjectStorageCommand.getCredentialsConfig(),
148
149 max_upload_part: options.maxUploadPart || '2MB',
150
151 streaming_playlists: {
152 bucket_name: options.playlistBucket,
153 prefix: options.playlistPrefix,
154 base_url: baseMockUrl
155 ? `${baseMockUrl}/${options.playlistBucket}`
156 : undefined
157 },
158
159 videos: {
160 bucket_name: options.webtorrentBucket,
161 prefix: options.webtorrentPrefix,
162 base_url: baseMockUrl
163 ? `${baseMockUrl}/${options.webtorrentBucket}`
164 : undefined
165 }
166 }
167 }
168
169 servers = await createMultipleServers(2, config)
170
171 await setAccessTokensToServers(servers)
172 await doubleFollow(servers[0], servers[1])
173
174 for (const server of servers) {
175 const { uuid } = await server.videos.quickUpload({ name: 'video to keep' })
176 await waitJobs(servers)
177
178 const files = await server.videos.listFiles({ id: uuid })
179 keptUrls = keptUrls.concat(files.map(f => f.fileUrl))
180 }
181 })
182
183 it('Should upload a video and move it to the object storage without transcoding', async function () {
184 this.timeout(20000)
185
186 const { uuid } = await servers[0].videos.quickUpload({ name: 'video 1' })
187 uuidsToDelete.push(uuid)
188
189 await waitJobs(servers)
190
191 for (const server of servers) {
192 const video = await server.videos.get({ id: uuid })
193 const files = await checkFiles({ ...options, video, baseMockUrl })
194
195 deletedUrls = deletedUrls.concat(files)
196 }
197 })
198
199 it('Should upload a video and move it to the object storage with transcoding', async function () {
200 this.timeout(40000)
201
202 const { uuid } = await servers[1].videos.quickUpload({ name: 'video 2' })
203 uuidsToDelete.push(uuid)
204
205 await waitJobs(servers)
206
207 for (const server of servers) {
208 const video = await server.videos.get({ id: uuid })
209 const files = await checkFiles({ ...options, video, baseMockUrl })
210
211 deletedUrls = deletedUrls.concat(files)
212 }
213 })
214
215 it('Should correctly delete the files', async function () {
216 await servers[0].videos.remove({ id: uuidsToDelete[0] })
217 await servers[1].videos.remove({ id: uuidsToDelete[1] })
218
219 await waitJobs(servers)
220
221 for (const url of deletedUrls) {
222 await makeRawRequest(url, HttpStatusCode.NOT_FOUND_404)
223 }
224 })
225
226 it('Should have kept other files', async function () {
227 for (const url of keptUrls) {
228 await makeRawRequest(url, HttpStatusCode.OK_200)
229 }
230 })
231
232 it('Should have an empty tmp directory', async function () {
233 for (const server of servers) {
234 await checkTmpIsEmpty(server)
235 }
236 })
237
238 after(async function () {
239 mockObjectStorage.terminate()
240
241 await cleanupTests(servers)
242 })
243}
244
245describe('Object storage for videos', function () {
246 if (areObjectStorageTestsDisabled()) return
247
248 describe('Test config', function () {
249 let server: PeerTubeServer
250
251 const baseConfig = {
252 object_storage: {
253 enabled: true,
254 endpoint: 'http://' + ObjectStorageCommand.getEndpointHost(),
255 region: ObjectStorageCommand.getRegion(),
256
257 credentials: ObjectStorageCommand.getCredentialsConfig(),
258
259 streaming_playlists: {
260 bucket_name: ObjectStorageCommand.DEFAULT_PLAYLIST_BUCKET
261 },
262
263 videos: {
264 bucket_name: ObjectStorageCommand.DEFAULT_WEBTORRENT_BUCKET
265 }
266 }
267 }
268
269 const badCredentials = {
270 access_key_id: 'AKIAIOSFODNN7EXAMPLE',
271 secret_access_key: 'aJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY'
272 }
273
274 it('Should fail with same bucket names without prefix', function (done) {
275 const config = merge({}, baseConfig, {
276 object_storage: {
277 streaming_playlists: {
278 bucket_name: 'aaa'
279 },
280
281 videos: {
282 bucket_name: 'aaa'
283 }
284 }
285 })
286
287 createSingleServer(1, config)
288 .then(() => done(new Error('Did not throw')))
289 .catch(() => done())
290 })
291
292 it('Should fail with bad credentials', async function () {
293 this.timeout(60000)
294
295 await ObjectStorageCommand.prepareDefaultBuckets()
296
297 const config = merge({}, baseConfig, {
298 object_storage: {
299 credentials: badCredentials
300 }
301 })
302
303 server = await createSingleServer(1, config)
304 await setAccessTokensToServers([ server ])
305
306 const { uuid } = await server.videos.quickUpload({ name: 'video' })
307
308 await waitJobs([ server ], true)
309 const video = await server.videos.get({ id: uuid })
310
311 expectStartWith(video.files[0].fileUrl, server.url)
312
313 await killallServers([ server ])
314 })
315
316 it('Should succeed with credentials from env', async function () {
317 this.timeout(60000)
318
319 await ObjectStorageCommand.prepareDefaultBuckets()
320
321 const config = merge({}, baseConfig, {
322 object_storage: {
323 credentials: {
324 access_key_id: '',
325 secret_access_key: ''
326 }
327 }
328 })
329
330 const goodCredentials = ObjectStorageCommand.getCredentialsConfig()
331
332 server = await createSingleServer(1, config, {
333 env: {
334 AWS_ACCESS_KEY_ID: goodCredentials.access_key_id,
335 AWS_SECRET_ACCESS_KEY: goodCredentials.secret_access_key
336 }
337 })
338
339 await setAccessTokensToServers([ server ])
340
341 const { uuid } = await server.videos.quickUpload({ name: 'video' })
342
343 await waitJobs([ server ], true)
344 const video = await server.videos.get({ id: uuid })
345
346 expectStartWith(video.files[0].fileUrl, ObjectStorageCommand.getWebTorrentBaseUrl())
347 })
348
349 after(async function () {
350 await killallServers([ server ])
351 })
352 })
353
354 describe('Test simple object storage', function () {
355 runTestSuite({
356 playlistBucket: 'streaming-playlists',
357 webtorrentBucket: 'videos'
358 })
359 })
360
361 describe('Test object storage with prefix', function () {
362 runTestSuite({
363 playlistBucket: 'mybucket',
364 webtorrentBucket: 'mybucket',
365
366 playlistPrefix: 'streaming-playlists_',
367 webtorrentPrefix: 'webtorrent_'
368 })
369 })
370
371 describe('Test object storage with prefix and base URL', function () {
372 runTestSuite({
373 playlistBucket: 'mybucket',
374 webtorrentBucket: 'mybucket',
375
376 playlistPrefix: 'streaming-playlists_',
377 webtorrentPrefix: 'webtorrent_',
378
379 useMockBaseUrl: true
380 })
381 })
382
383 describe('Test object storage with small upload part', function () {
384 runTestSuite({
385 playlistBucket: 'streaming-playlists',
386 webtorrentBucket: 'videos',
387
388 maxUploadPart: '5KB'
389 })
390 })
391})
diff --git a/server/tests/api/redundancy/redundancy.ts b/server/tests/api/redundancy/redundancy.ts
index e1a12f5f8..3400b1d9a 100644
--- a/server/tests/api/redundancy/redundancy.ts
+++ b/server/tests/api/redundancy/redundancy.ts
@@ -207,14 +207,14 @@ async function check1PlaylistRedundancies (videoUUID?: string) {
207 expect(redundancy.baseUrl).to.equal(servers[0].url + '/static/redundancy/hls/' + videoUUID) 207 expect(redundancy.baseUrl).to.equal(servers[0].url + '/static/redundancy/hls/' + videoUUID)
208 } 208 }
209 209
210 const baseUrlPlaylist = servers[1].url + '/static/streaming-playlists/hls' 210 const baseUrlPlaylist = servers[1].url + '/static/streaming-playlists/hls/' + videoUUID
211 const baseUrlSegment = servers[0].url + '/static/redundancy/hls' 211 const baseUrlSegment = servers[0].url + '/static/redundancy/hls/' + videoUUID
212 212
213 const video = await servers[0].videos.get({ id: videoUUID }) 213 const video = await servers[0].videos.get({ id: videoUUID })
214 const hlsPlaylist = video.streamingPlaylists[0] 214 const hlsPlaylist = video.streamingPlaylists[0]
215 215
216 for (const resolution of [ 240, 360, 480, 720 ]) { 216 for (const resolution of [ 240, 360, 480, 720 ]) {
217 await checkSegmentHash({ server: servers[1], baseUrlPlaylist, baseUrlSegment, videoUUID, resolution, hlsPlaylist }) 217 await checkSegmentHash({ server: servers[1], baseUrlPlaylist, baseUrlSegment, resolution, hlsPlaylist })
218 } 218 }
219 219
220 const { hlsFilenames } = await ensureSameFilenames(videoUUID) 220 const { hlsFilenames } = await ensureSameFilenames(videoUUID)
diff --git a/server/tests/api/videos/video-hls.ts b/server/tests/api/videos/video-hls.ts
index 961f0e617..2c829f532 100644
--- a/server/tests/api/videos/video-hls.ts
+++ b/server/tests/api/videos/video-hls.ts
@@ -5,6 +5,7 @@ import * as chai from 'chai'
5import { basename, join } from 'path' 5import { basename, join } from 'path'
6import { removeFragmentedMP4Ext, uuidRegex } from '@shared/core-utils' 6import { removeFragmentedMP4Ext, uuidRegex } from '@shared/core-utils'
7import { 7import {
8 areObjectStorageTestsDisabled,
8 checkDirectoryIsEmpty, 9 checkDirectoryIsEmpty,
9 checkResolutionsInMasterPlaylist, 10 checkResolutionsInMasterPlaylist,
10 checkSegmentHash, 11 checkSegmentHash,
@@ -12,7 +13,9 @@ import {
12 cleanupTests, 13 cleanupTests,
13 createMultipleServers, 14 createMultipleServers,
14 doubleFollow, 15 doubleFollow,
16 expectStartWith,
15 makeRawRequest, 17 makeRawRequest,
18 ObjectStorageCommand,
16 PeerTubeServer, 19 PeerTubeServer,
17 setAccessTokensToServers, 20 setAccessTokensToServers,
18 waitJobs, 21 waitJobs,
@@ -23,8 +26,19 @@ import { DEFAULT_AUDIO_RESOLUTION } from '../../../initializers/constants'
23 26
24const expect = chai.expect 27const expect = chai.expect
25 28
26async function checkHlsPlaylist (servers: PeerTubeServer[], videoUUID: string, hlsOnly: boolean, resolutions = [ 240, 360, 480, 720 ]) { 29async function checkHlsPlaylist (options: {
27 for (const server of servers) { 30 servers: PeerTubeServer[]
31 videoUUID: string
32 hlsOnly: boolean
33
34 resolutions?: number[]
35 objectStorageBaseUrl: string
36}) {
37 const { videoUUID, hlsOnly, objectStorageBaseUrl } = options
38
39 const resolutions = options.resolutions ?? [ 240, 360, 480, 720 ]
40
41 for (const server of options.servers) {
28 const videoDetails = await server.videos.get({ id: videoUUID }) 42 const videoDetails = await server.videos.get({ id: videoUUID })
29 const baseUrl = `http://${videoDetails.account.host}` 43 const baseUrl = `http://${videoDetails.account.host}`
30 44
@@ -48,9 +62,15 @@ async function checkHlsPlaylist (servers: PeerTubeServer[], videoUUID: string, h
48 expect(file.torrentUrl).to.match( 62 expect(file.torrentUrl).to.match(
49 new RegExp(`http://${server.host}/lazy-static/torrents/${uuidRegex}-${file.resolution.id}-hls.torrent`) 63 new RegExp(`http://${server.host}/lazy-static/torrents/${uuidRegex}-${file.resolution.id}-hls.torrent`)
50 ) 64 )
51 expect(file.fileUrl).to.match( 65
52 new RegExp(`${baseUrl}/static/streaming-playlists/hls/${videoDetails.uuid}/${uuidRegex}-${file.resolution.id}-fragmented.mp4`) 66 if (objectStorageBaseUrl) {
53 ) 67 expectStartWith(file.fileUrl, objectStorageBaseUrl)
68 } else {
69 expect(file.fileUrl).to.match(
70 new RegExp(`${baseUrl}/static/streaming-playlists/hls/${videoDetails.uuid}/${uuidRegex}-${file.resolution.id}-fragmented.mp4`)
71 )
72 }
73
54 expect(file.resolution.label).to.equal(resolution + 'p') 74 expect(file.resolution.label).to.equal(resolution + 'p')
55 75
56 await makeRawRequest(file.torrentUrl, HttpStatusCode.OK_200) 76 await makeRawRequest(file.torrentUrl, HttpStatusCode.OK_200)
@@ -80,9 +100,11 @@ async function checkHlsPlaylist (servers: PeerTubeServer[], videoUUID: string, h
80 const file = hlsFiles.find(f => f.resolution.id === resolution) 100 const file = hlsFiles.find(f => f.resolution.id === resolution)
81 const playlistName = removeFragmentedMP4Ext(basename(file.fileUrl)) + '.m3u8' 101 const playlistName = removeFragmentedMP4Ext(basename(file.fileUrl)) + '.m3u8'
82 102
83 const subPlaylist = await server.streamingPlaylists.get({ 103 const url = objectStorageBaseUrl
84 url: `${baseUrl}/static/streaming-playlists/hls/${videoUUID}/${playlistName}` 104 ? `${objectStorageBaseUrl}hls_${videoUUID}/${playlistName}`
85 }) 105 : `${baseUrl}/static/streaming-playlists/hls/${videoUUID}/${playlistName}`
106
107 const subPlaylist = await server.streamingPlaylists.get({ url })
86 108
87 expect(subPlaylist).to.match(new RegExp(`${uuidRegex}-${resolution}-fragmented.mp4`)) 109 expect(subPlaylist).to.match(new RegExp(`${uuidRegex}-${resolution}-fragmented.mp4`))
88 expect(subPlaylist).to.contain(basename(file.fileUrl)) 110 expect(subPlaylist).to.contain(basename(file.fileUrl))
@@ -90,14 +112,15 @@ async function checkHlsPlaylist (servers: PeerTubeServer[], videoUUID: string, h
90 } 112 }
91 113
92 { 114 {
93 const baseUrlAndPath = baseUrl + '/static/streaming-playlists/hls' 115 const baseUrlAndPath = objectStorageBaseUrl
116 ? objectStorageBaseUrl + 'hls_' + videoUUID
117 : baseUrl + '/static/streaming-playlists/hls/' + videoUUID
94 118
95 for (const resolution of resolutions) { 119 for (const resolution of resolutions) {
96 await checkSegmentHash({ 120 await checkSegmentHash({
97 server, 121 server,
98 baseUrlPlaylist: baseUrlAndPath, 122 baseUrlPlaylist: baseUrlAndPath,
99 baseUrlSegment: baseUrlAndPath, 123 baseUrlSegment: baseUrlAndPath,
100 videoUUID,
101 resolution, 124 resolution,
102 hlsPlaylist 125 hlsPlaylist
103 }) 126 })
@@ -111,7 +134,7 @@ describe('Test HLS videos', function () {
111 let videoUUID = '' 134 let videoUUID = ''
112 let videoAudioUUID = '' 135 let videoAudioUUID = ''
113 136
114 function runTestSuite (hlsOnly: boolean) { 137 function runTestSuite (hlsOnly: boolean, objectStorageBaseUrl?: string) {
115 138
116 it('Should upload a video and transcode it to HLS', async function () { 139 it('Should upload a video and transcode it to HLS', async function () {
117 this.timeout(120000) 140 this.timeout(120000)
@@ -121,7 +144,7 @@ describe('Test HLS videos', function () {
121 144
122 await waitJobs(servers) 145 await waitJobs(servers)
123 146
124 await checkHlsPlaylist(servers, videoUUID, hlsOnly) 147 await checkHlsPlaylist({ servers, videoUUID, hlsOnly, objectStorageBaseUrl })
125 }) 148 })
126 149
127 it('Should upload an audio file and transcode it to HLS', async function () { 150 it('Should upload an audio file and transcode it to HLS', async function () {
@@ -132,7 +155,13 @@ describe('Test HLS videos', function () {
132 155
133 await waitJobs(servers) 156 await waitJobs(servers)
134 157
135 await checkHlsPlaylist(servers, videoAudioUUID, hlsOnly, [ DEFAULT_AUDIO_RESOLUTION, 360, 240 ]) 158 await checkHlsPlaylist({
159 servers,
160 videoUUID: videoAudioUUID,
161 hlsOnly,
162 resolutions: [ DEFAULT_AUDIO_RESOLUTION, 360, 240 ],
163 objectStorageBaseUrl
164 })
136 }) 165 })
137 166
138 it('Should update the video', async function () { 167 it('Should update the video', async function () {
@@ -142,7 +171,7 @@ describe('Test HLS videos', function () {
142 171
143 await waitJobs(servers) 172 await waitJobs(servers)
144 173
145 await checkHlsPlaylist(servers, videoUUID, hlsOnly) 174 await checkHlsPlaylist({ servers, videoUUID, hlsOnly, objectStorageBaseUrl })
146 }) 175 })
147 176
148 it('Should delete videos', async function () { 177 it('Should delete videos', async function () {
@@ -229,6 +258,22 @@ describe('Test HLS videos', function () {
229 runTestSuite(true) 258 runTestSuite(true)
230 }) 259 })
231 260
261 describe('With object storage enabled', function () {
262 if (areObjectStorageTestsDisabled()) return
263
264 before(async function () {
265 this.timeout(120000)
266
267 const configOverride = ObjectStorageCommand.getDefaultConfig()
268 await ObjectStorageCommand.prepareDefaultBuckets()
269
270 await servers[0].kill()
271 await servers[0].run(configOverride)
272 })
273
274 runTestSuite(true, ObjectStorageCommand.getPlaylistBaseUrl())
275 })
276
232 after(async function () { 277 after(async function () {
233 await cleanupTests(servers) 278 await cleanupTests(servers)
234 }) 279 })