]>
Commit | Line | Data |
---|---|---|
c48e82b5 C |
1 | /* tslint:disable:no-unused-expression */ |
2 | ||
3 | import * as chai from 'chai' | |
4 | import 'mocha' | |
5 | import { VideoDetails } from '../../../../shared/models/videos' | |
6 | import { | |
7 | doubleFollow, | |
8 | flushAndRunMultipleServers, | |
c48e82b5 C |
9 | getFollowingListPaginationAndSort, |
10 | getVideo, | |
993cef4b | 11 | immutableAssign, |
ebdb6124 | 12 | killallServers, makeGetRequest, |
993cef4b | 13 | root, |
c48e82b5 | 14 | ServerInfo, |
161b061d | 15 | setAccessTokensToServers, unfollow, |
c48e82b5 | 16 | uploadVideo, |
993cef4b C |
17 | viewVideo, |
18 | wait | |
c48e82b5 C |
19 | } from '../../utils' |
20 | import { waitJobs } from '../../utils/server/jobs' | |
21 | import * as magnetUtil from 'magnet-uri' | |
22 | import { updateRedundancy } from '../../utils/server/redundancy' | |
23 | import { ActorFollow } from '../../../../shared/models/actors' | |
24 | import { readdir } from 'fs-extra' | |
25 | import { join } from 'path' | |
b36f41ca | 26 | import { VideoRedundancyStrategy } from '../../../../shared/models/redundancy' |
4b5384f6 C |
27 | import { getStats } from '../../utils/server/stats' |
28 | import { ServerStats } from '../../../../shared/models/server/server-stats.model' | |
c48e82b5 C |
29 | |
30 | const expect = chai.expect | |
31 | ||
b36f41ca C |
32 | let servers: ServerInfo[] = [] |
33 | let video1Server2UUID: string | |
b36f41ca | 34 | |
e5565833 | 35 | function checkMagnetWebseeds (file: { magnetUri: string, resolution: { id: number } }, baseWebseeds: string[], server: ServerInfo) { |
c48e82b5 C |
36 | const parsed = magnetUtil.decode(file.magnetUri) |
37 | ||
38 | for (const ws of baseWebseeds) { | |
39 | const found = parsed.urlList.find(url => url === `${ws}-${file.resolution.id}.mp4`) | |
e5565833 | 40 | expect(found, `Webseed ${ws} not found in ${file.magnetUri} on server ${server.url}`).to.not.be.undefined |
c48e82b5 | 41 | } |
161b061d C |
42 | |
43 | expect(parsed.urlList).to.have.lengthOf(baseWebseeds.length) | |
c48e82b5 C |
44 | } |
45 | ||
3f6b6a56 | 46 | async function runServers (strategy: VideoRedundancyStrategy, additionalParams: any = {}) { |
b36f41ca C |
47 | const config = { |
48 | redundancy: { | |
993cef4b C |
49 | videos: { |
50 | check_interval: '5 seconds', | |
51 | strategies: [ | |
52 | immutableAssign({ | |
e5565833 | 53 | min_lifetime: '1 hour', |
993cef4b C |
54 | strategy: strategy, |
55 | size: '100KB' | |
56 | }, additionalParams) | |
57 | ] | |
58 | } | |
b36f41ca C |
59 | } |
60 | } | |
61 | servers = await flushAndRunMultipleServers(3, config) | |
c48e82b5 | 62 | |
b36f41ca C |
63 | // Get the access tokens |
64 | await setAccessTokensToServers(servers) | |
c48e82b5 | 65 | |
b36f41ca C |
66 | { |
67 | const res = await uploadVideo(servers[ 1 ].url, servers[ 1 ].accessToken, { name: 'video 1 server 2' }) | |
68 | video1Server2UUID = res.body.video.uuid | |
c48e82b5 | 69 | |
b36f41ca C |
70 | await viewVideo(servers[ 1 ].url, video1Server2UUID) |
71 | } | |
c48e82b5 | 72 | |
b36f41ca | 73 | await waitJobs(servers) |
c48e82b5 | 74 | |
b36f41ca C |
75 | // Server 1 and server 2 follow each other |
76 | await doubleFollow(servers[ 0 ], servers[ 1 ]) | |
77 | // Server 1 and server 3 follow each other | |
78 | await doubleFollow(servers[ 0 ], servers[ 2 ]) | |
79 | // Server 2 and server 3 follow each other | |
80 | await doubleFollow(servers[ 1 ], servers[ 2 ]) | |
81 | ||
82 | await waitJobs(servers) | |
83 | } | |
c48e82b5 | 84 | |
e5565833 C |
85 | async function check1WebSeed (strategy: VideoRedundancyStrategy, videoUUID?: string) { |
86 | if (!videoUUID) videoUUID = video1Server2UUID | |
87 | ||
b36f41ca | 88 | const webseeds = [ |
e5565833 | 89 | 'http://localhost:9002/static/webseed/' + videoUUID |
b36f41ca | 90 | ] |
c48e82b5 | 91 | |
b36f41ca | 92 | for (const server of servers) { |
4b5384f6 | 93 | { |
e5565833 | 94 | const res = await getVideo(server.url, videoUUID) |
c48e82b5 | 95 | |
4b5384f6 | 96 | const video: VideoDetails = res.body |
e5565833 C |
97 | for (const f of video.files) { |
98 | checkMagnetWebseeds(f, webseeds, server) | |
99 | } | |
4b5384f6 | 100 | } |
e5565833 C |
101 | } |
102 | } | |
4b5384f6 | 103 | |
e5565833 C |
104 | async function checkStatsWith2Webseed (strategy: VideoRedundancyStrategy) { |
105 | const res = await getStats(servers[0].url) | |
106 | const data: ServerStats = res.body | |
4b5384f6 | 107 | |
e5565833 C |
108 | expect(data.videosRedundancy).to.have.lengthOf(1) |
109 | const stat = data.videosRedundancy[0] | |
4b5384f6 | 110 | |
e5565833 C |
111 | expect(stat.strategy).to.equal(strategy) |
112 | expect(stat.totalSize).to.equal(102400) | |
113 | expect(stat.totalUsed).to.be.at.least(1).and.below(102401) | |
114 | expect(stat.totalVideoFiles).to.equal(4) | |
115 | expect(stat.totalVideos).to.equal(1) | |
b36f41ca C |
116 | } |
117 | ||
e5565833 C |
118 | async function checkStatsWith1Webseed (strategy: VideoRedundancyStrategy) { |
119 | const res = await getStats(servers[0].url) | |
120 | const data: ServerStats = res.body | |
b36f41ca | 121 | |
e5565833 | 122 | expect(data.videosRedundancy).to.have.lengthOf(1) |
b36f41ca | 123 | |
e5565833 C |
124 | const stat = data.videosRedundancy[0] |
125 | expect(stat.strategy).to.equal(strategy) | |
126 | expect(stat.totalSize).to.equal(102400) | |
127 | expect(stat.totalUsed).to.equal(0) | |
128 | expect(stat.totalVideoFiles).to.equal(0) | |
129 | expect(stat.totalVideos).to.equal(0) | |
b36f41ca | 130 | } |
c48e82b5 | 131 | |
e5565833 C |
132 | async function check2Webseeds (strategy: VideoRedundancyStrategy, videoUUID?: string) { |
133 | if (!videoUUID) videoUUID = video1Server2UUID | |
c48e82b5 | 134 | |
b36f41ca | 135 | const webseeds = [ |
e5565833 C |
136 | 'http://localhost:9001/static/webseed/' + videoUUID, |
137 | 'http://localhost:9002/static/webseed/' + videoUUID | |
b36f41ca | 138 | ] |
c48e82b5 | 139 | |
b36f41ca | 140 | for (const server of servers) { |
161b061d | 141 | const res = await getVideo(server.url, videoUUID) |
b36f41ca | 142 | |
161b061d C |
143 | const video: VideoDetails = res.body |
144 | ||
145 | for (const file of video.files) { | |
146 | checkMagnetWebseeds(file, webseeds, server) | |
b36f41ca | 147 | |
161b061d C |
148 | // Only servers 1 and 2 have the video |
149 | if (server.serverNumber !== 3) { | |
150 | await makeGetRequest({ | |
151 | url: server.url, | |
152 | statusCodeExpected: 200, | |
153 | path: '/static/webseed/' + `${videoUUID}-${file.resolution.id}.mp4`, | |
154 | contentType: null | |
155 | }) | |
4b5384f6 | 156 | } |
c48e82b5 | 157 | } |
b36f41ca | 158 | } |
c48e82b5 | 159 | |
ebdb6124 C |
160 | for (const directory of [ 'test1', 'test2' ]) { |
161 | const files = await readdir(join(root(), directory, 'videos')) | |
162 | expect(files).to.have.length.at.least(4) | |
c48e82b5 | 163 | |
ebdb6124 C |
164 | for (const resolution of [ 240, 360, 480, 720 ]) { |
165 | expect(files.find(f => f === `${videoUUID}-${resolution}.mp4`)).to.not.be.undefined | |
166 | } | |
b36f41ca | 167 | } |
e5565833 | 168 | } |
4b5384f6 | 169 | |
e5565833 C |
170 | async function enableRedundancyOnServer1 () { |
171 | await updateRedundancy(servers[ 0 ].url, servers[ 0 ].accessToken, servers[ 1 ].host, true) | |
4b5384f6 | 172 | |
e5565833 C |
173 | const res = await getFollowingListPaginationAndSort(servers[ 0 ].url, 0, 5, '-createdAt') |
174 | const follows: ActorFollow[] = res.body.data | |
175 | const server2 = follows.find(f => f.following.host === 'localhost:9002') | |
176 | const server3 = follows.find(f => f.following.host === 'localhost:9003') | |
4b5384f6 | 177 | |
e5565833 C |
178 | expect(server3).to.not.be.undefined |
179 | expect(server3.following.hostRedundancyAllowed).to.be.false | |
180 | ||
181 | expect(server2).to.not.be.undefined | |
182 | expect(server2.following.hostRedundancyAllowed).to.be.true | |
b36f41ca | 183 | } |
c48e82b5 | 184 | |
161b061d C |
185 | async function disableRedundancyOnServer1 () { |
186 | await updateRedundancy(servers[ 0 ].url, servers[ 0 ].accessToken, servers[ 1 ].host, false) | |
187 | ||
188 | const res = await getFollowingListPaginationAndSort(servers[ 0 ].url, 0, 5, '-createdAt') | |
189 | const follows: ActorFollow[] = res.body.data | |
190 | const server2 = follows.find(f => f.following.host === 'localhost:9002') | |
191 | const server3 = follows.find(f => f.following.host === 'localhost:9003') | |
192 | ||
193 | expect(server3).to.not.be.undefined | |
194 | expect(server3.following.hostRedundancyAllowed).to.be.false | |
195 | ||
196 | expect(server2).to.not.be.undefined | |
197 | expect(server2.following.hostRedundancyAllowed).to.be.false | |
198 | } | |
199 | ||
b36f41ca C |
200 | async function cleanServers () { |
201 | killallServers(servers) | |
202 | } | |
c48e82b5 | 203 | |
b36f41ca | 204 | describe('Test videos redundancy', function () { |
c48e82b5 | 205 | |
b36f41ca | 206 | describe('With most-views strategy', function () { |
4b5384f6 | 207 | const strategy = 'most-views' |
c48e82b5 | 208 | |
b36f41ca C |
209 | before(function () { |
210 | this.timeout(120000) | |
c48e82b5 | 211 | |
4b5384f6 | 212 | return runServers(strategy) |
b36f41ca | 213 | }) |
c48e82b5 | 214 | |
e5565833 C |
215 | it('Should have 1 webseed on the first video', async function () { |
216 | await check1WebSeed(strategy) | |
217 | await checkStatsWith1Webseed(strategy) | |
b36f41ca | 218 | }) |
c48e82b5 | 219 | |
3f6b6a56 | 220 | it('Should enable redundancy on server 1', function () { |
e5565833 | 221 | return enableRedundancyOnServer1() |
b36f41ca | 222 | }) |
c48e82b5 | 223 | |
e5565833 | 224 | it('Should have 2 webseed on the first video', async function () { |
b36f41ca | 225 | this.timeout(40000) |
c48e82b5 | 226 | |
e5565833 C |
227 | await waitJobs(servers) |
228 | await wait(15000) | |
229 | await waitJobs(servers) | |
230 | ||
231 | await check2Webseeds(strategy) | |
232 | await checkStatsWith2Webseed(strategy) | |
b36f41ca | 233 | }) |
c48e82b5 | 234 | |
161b061d C |
235 | it('Should undo redundancy on server 1 and remove duplicated videos', async function () { |
236 | this.timeout(40000) | |
237 | ||
238 | await disableRedundancyOnServer1() | |
239 | ||
240 | await waitJobs(servers) | |
241 | await wait(5000) | |
242 | ||
243 | await check1WebSeed(strategy) | |
244 | }) | |
245 | ||
b36f41ca C |
246 | after(function () { |
247 | return cleanServers() | |
248 | }) | |
c48e82b5 C |
249 | }) |
250 | ||
b36f41ca | 251 | describe('With trending strategy', function () { |
4b5384f6 | 252 | const strategy = 'trending' |
c48e82b5 | 253 | |
b36f41ca C |
254 | before(function () { |
255 | this.timeout(120000) | |
256 | ||
4b5384f6 | 257 | return runServers(strategy) |
b36f41ca C |
258 | }) |
259 | ||
e5565833 C |
260 | it('Should have 1 webseed on the first video', async function () { |
261 | await check1WebSeed(strategy) | |
262 | await checkStatsWith1Webseed(strategy) | |
b36f41ca C |
263 | }) |
264 | ||
3f6b6a56 | 265 | it('Should enable redundancy on server 1', function () { |
e5565833 | 266 | return enableRedundancyOnServer1() |
b36f41ca C |
267 | }) |
268 | ||
e5565833 | 269 | it('Should have 2 webseed on the first video', async function () { |
3f6b6a56 C |
270 | this.timeout(40000) |
271 | ||
e5565833 C |
272 | await waitJobs(servers) |
273 | await wait(15000) | |
274 | await waitJobs(servers) | |
275 | ||
276 | await check2Webseeds(strategy) | |
277 | await checkStatsWith2Webseed(strategy) | |
3f6b6a56 C |
278 | }) |
279 | ||
161b061d C |
280 | it('Should unfollow on server 1 and remove duplicated videos', async function () { |
281 | this.timeout(40000) | |
282 | ||
283 | await unfollow(servers[0].url, servers[0].accessToken, servers[1]) | |
284 | ||
285 | await waitJobs(servers) | |
286 | await wait(5000) | |
287 | ||
288 | await check1WebSeed(strategy) | |
289 | }) | |
290 | ||
3f6b6a56 C |
291 | after(function () { |
292 | return cleanServers() | |
293 | }) | |
294 | }) | |
295 | ||
296 | describe('With recently added strategy', function () { | |
4b5384f6 | 297 | const strategy = 'recently-added' |
3f6b6a56 C |
298 | |
299 | before(function () { | |
300 | this.timeout(120000) | |
301 | ||
e5565833 | 302 | return runServers(strategy, { min_views: 3 }) |
3f6b6a56 C |
303 | }) |
304 | ||
e5565833 C |
305 | it('Should have 1 webseed on the first video', async function () { |
306 | await check1WebSeed(strategy) | |
307 | await checkStatsWith1Webseed(strategy) | |
3f6b6a56 C |
308 | }) |
309 | ||
310 | it('Should enable redundancy on server 1', function () { | |
e5565833 | 311 | return enableRedundancyOnServer1() |
3f6b6a56 C |
312 | }) |
313 | ||
314 | it('Should still have 1 webseed on the first video', async function () { | |
315 | this.timeout(40000) | |
316 | ||
317 | await waitJobs(servers) | |
318 | await wait(15000) | |
319 | await waitJobs(servers) | |
320 | ||
e5565833 C |
321 | await check1WebSeed(strategy) |
322 | await checkStatsWith1Webseed(strategy) | |
3f6b6a56 C |
323 | }) |
324 | ||
e5565833 | 325 | it('Should view 2 times the first video to have > min_views config', async function () { |
3f6b6a56 C |
326 | this.timeout(40000) |
327 | ||
328 | await viewVideo(servers[ 0 ].url, video1Server2UUID) | |
329 | await viewVideo(servers[ 2 ].url, video1Server2UUID) | |
330 | ||
331 | await wait(10000) | |
332 | await waitJobs(servers) | |
333 | }) | |
334 | ||
e5565833 | 335 | it('Should have 2 webseed on the first video', async function () { |
b36f41ca C |
336 | this.timeout(40000) |
337 | ||
e5565833 C |
338 | await waitJobs(servers) |
339 | await wait(15000) | |
340 | await waitJobs(servers) | |
341 | ||
342 | await check2Webseeds(strategy) | |
343 | await checkStatsWith2Webseed(strategy) | |
344 | }) | |
345 | ||
346 | after(function () { | |
347 | return cleanServers() | |
348 | }) | |
349 | }) | |
350 | ||
351 | describe('Test expiration', function () { | |
352 | const strategy = 'recently-added' | |
353 | ||
354 | async function checkContains (servers: ServerInfo[], str: string) { | |
355 | for (const server of servers) { | |
356 | const res = await getVideo(server.url, video1Server2UUID) | |
357 | const video: VideoDetails = res.body | |
358 | ||
359 | for (const f of video.files) { | |
360 | expect(f.magnetUri).to.contain(str) | |
361 | } | |
362 | } | |
363 | } | |
364 | ||
365 | async function checkNotContains (servers: ServerInfo[], str: string) { | |
366 | for (const server of servers) { | |
367 | const res = await getVideo(server.url, video1Server2UUID) | |
368 | const video: VideoDetails = res.body | |
369 | ||
370 | for (const f of video.files) { | |
371 | expect(f.magnetUri).to.not.contain(str) | |
372 | } | |
373 | } | |
374 | } | |
375 | ||
376 | before(async function () { | |
377 | this.timeout(120000) | |
378 | ||
379 | await runServers(strategy, { min_lifetime: '7 seconds', min_views: 0 }) | |
380 | ||
381 | await enableRedundancyOnServer1() | |
382 | }) | |
383 | ||
384 | it('Should still have 2 webseeds after 10 seconds', async function () { | |
385 | this.timeout(40000) | |
386 | ||
387 | await wait(10000) | |
388 | ||
389 | try { | |
390 | await checkContains(servers, 'http%3A%2F%2Flocalhost%3A9001') | |
391 | } catch { | |
392 | // Maybe a server deleted a redundancy in the scheduler | |
393 | await wait(2000) | |
394 | ||
395 | await checkContains(servers, 'http%3A%2F%2Flocalhost%3A9001') | |
396 | } | |
397 | }) | |
398 | ||
399 | it('Should stop server 1 and expire video redundancy', async function () { | |
400 | this.timeout(40000) | |
401 | ||
402 | killallServers([ servers[0] ]) | |
403 | ||
404 | await wait(10000) | |
405 | ||
406 | await checkNotContains([ servers[1], servers[2] ], 'http%3A%2F%2Flocalhost%3A9001') | |
407 | }) | |
408 | ||
409 | after(function () { | |
410 | return killallServers([ servers[1], servers[2] ]) | |
411 | }) | |
412 | }) | |
413 | ||
414 | describe('Test file replacement', function () { | |
415 | let video2Server2UUID: string | |
416 | const strategy = 'recently-added' | |
417 | ||
418 | before(async function () { | |
419 | this.timeout(120000) | |
420 | ||
421 | await runServers(strategy, { min_lifetime: '7 seconds', min_views: 0 }) | |
422 | ||
423 | await enableRedundancyOnServer1() | |
424 | ||
425 | await waitJobs(servers) | |
426 | await wait(5000) | |
427 | await waitJobs(servers) | |
428 | ||
429 | await check2Webseeds(strategy) | |
430 | await checkStatsWith2Webseed(strategy) | |
431 | ||
432 | const res = await uploadVideo(servers[ 1 ].url, servers[ 1 ].accessToken, { name: 'video 2 server 2' }) | |
433 | video2Server2UUID = res.body.video.uuid | |
434 | }) | |
435 | ||
436 | it('Should cache video 2 webseed on the first video', async function () { | |
437 | this.timeout(40000) | |
438 | this.retries(3) | |
439 | ||
440 | await waitJobs(servers) | |
441 | ||
442 | await wait(7000) | |
443 | ||
444 | await check1WebSeed(strategy, video1Server2UUID) | |
445 | await check2Webseeds(strategy, video2Server2UUID) | |
b36f41ca C |
446 | }) |
447 | ||
448 | after(function () { | |
449 | return cleanServers() | |
450 | }) | |
c48e82b5 C |
451 | }) |
452 | }) |