]>
Commit | Line | Data |
---|---|---|
c48e82b5 C |
1 | /* tslint:disable:no-unused-expression */ |
2 | ||
3 | import * as chai from 'chai' | |
4 | import 'mocha' | |
5 | import { VideoDetails } from '../../../../shared/models/videos' | |
6 | import { | |
7 | doubleFollow, | |
8 | flushAndRunMultipleServers, | |
c48e82b5 C |
9 | getFollowingListPaginationAndSort, |
10 | getVideo, | |
993cef4b | 11 | immutableAssign, |
ebdb6124 | 12 | killallServers, makeGetRequest, |
993cef4b | 13 | root, |
c48e82b5 | 14 | ServerInfo, |
161b061d | 15 | setAccessTokensToServers, unfollow, |
c48e82b5 | 16 | uploadVideo, |
993cef4b | 17 | viewVideo, |
792e5b8e C |
18 | wait, |
19 | waitUntilLog | |
c48e82b5 C |
20 | } from '../../utils' |
21 | import { waitJobs } from '../../utils/server/jobs' | |
22 | import * as magnetUtil from 'magnet-uri' | |
23 | import { updateRedundancy } from '../../utils/server/redundancy' | |
24 | import { ActorFollow } from '../../../../shared/models/actors' | |
25 | import { readdir } from 'fs-extra' | |
26 | import { join } from 'path' | |
b36f41ca | 27 | import { VideoRedundancyStrategy } from '../../../../shared/models/redundancy' |
4b5384f6 C |
28 | import { getStats } from '../../utils/server/stats' |
29 | import { ServerStats } from '../../../../shared/models/server/server-stats.model' | |
c48e82b5 C |
30 | |
31 | const expect = chai.expect | |
32 | ||
b36f41ca C |
33 | let servers: ServerInfo[] = [] |
34 | let video1Server2UUID: string | |
b36f41ca | 35 | |
e5565833 | 36 | function checkMagnetWebseeds (file: { magnetUri: string, resolution: { id: number } }, baseWebseeds: string[], server: ServerInfo) { |
c48e82b5 C |
37 | const parsed = magnetUtil.decode(file.magnetUri) |
38 | ||
39 | for (const ws of baseWebseeds) { | |
40 | const found = parsed.urlList.find(url => url === `${ws}-${file.resolution.id}.mp4`) | |
e5565833 | 41 | expect(found, `Webseed ${ws} not found in ${file.magnetUri} on server ${server.url}`).to.not.be.undefined |
c48e82b5 | 42 | } |
161b061d C |
43 | |
44 | expect(parsed.urlList).to.have.lengthOf(baseWebseeds.length) | |
c48e82b5 C |
45 | } |
46 | ||
3f6b6a56 | 47 | async function runServers (strategy: VideoRedundancyStrategy, additionalParams: any = {}) { |
b36f41ca C |
48 | const config = { |
49 | redundancy: { | |
993cef4b C |
50 | videos: { |
51 | check_interval: '5 seconds', | |
52 | strategies: [ | |
53 | immutableAssign({ | |
e5565833 | 54 | min_lifetime: '1 hour', |
993cef4b C |
55 | strategy: strategy, |
56 | size: '100KB' | |
57 | }, additionalParams) | |
58 | ] | |
59 | } | |
b36f41ca C |
60 | } |
61 | } | |
62 | servers = await flushAndRunMultipleServers(3, config) | |
c48e82b5 | 63 | |
b36f41ca C |
64 | // Get the access tokens |
65 | await setAccessTokensToServers(servers) | |
c48e82b5 | 66 | |
b36f41ca C |
67 | { |
68 | const res = await uploadVideo(servers[ 1 ].url, servers[ 1 ].accessToken, { name: 'video 1 server 2' }) | |
69 | video1Server2UUID = res.body.video.uuid | |
c48e82b5 | 70 | |
b36f41ca C |
71 | await viewVideo(servers[ 1 ].url, video1Server2UUID) |
72 | } | |
c48e82b5 | 73 | |
b36f41ca | 74 | await waitJobs(servers) |
c48e82b5 | 75 | |
b36f41ca C |
76 | // Server 1 and server 2 follow each other |
77 | await doubleFollow(servers[ 0 ], servers[ 1 ]) | |
78 | // Server 1 and server 3 follow each other | |
79 | await doubleFollow(servers[ 0 ], servers[ 2 ]) | |
80 | // Server 2 and server 3 follow each other | |
81 | await doubleFollow(servers[ 1 ], servers[ 2 ]) | |
82 | ||
83 | await waitJobs(servers) | |
84 | } | |
c48e82b5 | 85 | |
e5565833 C |
86 | async function check1WebSeed (strategy: VideoRedundancyStrategy, videoUUID?: string) { |
87 | if (!videoUUID) videoUUID = video1Server2UUID | |
88 | ||
b36f41ca | 89 | const webseeds = [ |
e5565833 | 90 | 'http://localhost:9002/static/webseed/' + videoUUID |
b36f41ca | 91 | ] |
c48e82b5 | 92 | |
b36f41ca | 93 | for (const server of servers) { |
4b5384f6 | 94 | { |
e5565833 | 95 | const res = await getVideo(server.url, videoUUID) |
c48e82b5 | 96 | |
4b5384f6 | 97 | const video: VideoDetails = res.body |
e5565833 C |
98 | for (const f of video.files) { |
99 | checkMagnetWebseeds(f, webseeds, server) | |
100 | } | |
4b5384f6 | 101 | } |
e5565833 C |
102 | } |
103 | } | |
4b5384f6 | 104 | |
e5565833 C |
105 | async function checkStatsWith2Webseed (strategy: VideoRedundancyStrategy) { |
106 | const res = await getStats(servers[0].url) | |
107 | const data: ServerStats = res.body | |
4b5384f6 | 108 | |
e5565833 C |
109 | expect(data.videosRedundancy).to.have.lengthOf(1) |
110 | const stat = data.videosRedundancy[0] | |
4b5384f6 | 111 | |
e5565833 C |
112 | expect(stat.strategy).to.equal(strategy) |
113 | expect(stat.totalSize).to.equal(102400) | |
114 | expect(stat.totalUsed).to.be.at.least(1).and.below(102401) | |
115 | expect(stat.totalVideoFiles).to.equal(4) | |
116 | expect(stat.totalVideos).to.equal(1) | |
b36f41ca C |
117 | } |
118 | ||
e5565833 C |
119 | async function checkStatsWith1Webseed (strategy: VideoRedundancyStrategy) { |
120 | const res = await getStats(servers[0].url) | |
121 | const data: ServerStats = res.body | |
b36f41ca | 122 | |
e5565833 | 123 | expect(data.videosRedundancy).to.have.lengthOf(1) |
b36f41ca | 124 | |
e5565833 C |
125 | const stat = data.videosRedundancy[0] |
126 | expect(stat.strategy).to.equal(strategy) | |
127 | expect(stat.totalSize).to.equal(102400) | |
128 | expect(stat.totalUsed).to.equal(0) | |
129 | expect(stat.totalVideoFiles).to.equal(0) | |
130 | expect(stat.totalVideos).to.equal(0) | |
b36f41ca | 131 | } |
c48e82b5 | 132 | |
e5565833 C |
133 | async function check2Webseeds (strategy: VideoRedundancyStrategy, videoUUID?: string) { |
134 | if (!videoUUID) videoUUID = video1Server2UUID | |
c48e82b5 | 135 | |
b36f41ca | 136 | const webseeds = [ |
e5565833 C |
137 | 'http://localhost:9001/static/webseed/' + videoUUID, |
138 | 'http://localhost:9002/static/webseed/' + videoUUID | |
b36f41ca | 139 | ] |
c48e82b5 | 140 | |
b36f41ca | 141 | for (const server of servers) { |
161b061d | 142 | const res = await getVideo(server.url, videoUUID) |
b36f41ca | 143 | |
161b061d C |
144 | const video: VideoDetails = res.body |
145 | ||
146 | for (const file of video.files) { | |
147 | checkMagnetWebseeds(file, webseeds, server) | |
b36f41ca | 148 | |
161b061d C |
149 | // Only servers 1 and 2 have the video |
150 | if (server.serverNumber !== 3) { | |
151 | await makeGetRequest({ | |
152 | url: server.url, | |
153 | statusCodeExpected: 200, | |
154 | path: '/static/webseed/' + `${videoUUID}-${file.resolution.id}.mp4`, | |
155 | contentType: null | |
156 | }) | |
4b5384f6 | 157 | } |
c48e82b5 | 158 | } |
b36f41ca | 159 | } |
c48e82b5 | 160 | |
ebdb6124 C |
161 | for (const directory of [ 'test1', 'test2' ]) { |
162 | const files = await readdir(join(root(), directory, 'videos')) | |
163 | expect(files).to.have.length.at.least(4) | |
c48e82b5 | 164 | |
ebdb6124 C |
165 | for (const resolution of [ 240, 360, 480, 720 ]) { |
166 | expect(files.find(f => f === `${videoUUID}-${resolution}.mp4`)).to.not.be.undefined | |
167 | } | |
b36f41ca | 168 | } |
e5565833 | 169 | } |
4b5384f6 | 170 | |
e5565833 C |
171 | async function enableRedundancyOnServer1 () { |
172 | await updateRedundancy(servers[ 0 ].url, servers[ 0 ].accessToken, servers[ 1 ].host, true) | |
4b5384f6 | 173 | |
e5565833 C |
174 | const res = await getFollowingListPaginationAndSort(servers[ 0 ].url, 0, 5, '-createdAt') |
175 | const follows: ActorFollow[] = res.body.data | |
176 | const server2 = follows.find(f => f.following.host === 'localhost:9002') | |
177 | const server3 = follows.find(f => f.following.host === 'localhost:9003') | |
4b5384f6 | 178 | |
e5565833 C |
179 | expect(server3).to.not.be.undefined |
180 | expect(server3.following.hostRedundancyAllowed).to.be.false | |
181 | ||
182 | expect(server2).to.not.be.undefined | |
183 | expect(server2.following.hostRedundancyAllowed).to.be.true | |
b36f41ca | 184 | } |
c48e82b5 | 185 | |
161b061d C |
186 | async function disableRedundancyOnServer1 () { |
187 | await updateRedundancy(servers[ 0 ].url, servers[ 0 ].accessToken, servers[ 1 ].host, false) | |
188 | ||
189 | const res = await getFollowingListPaginationAndSort(servers[ 0 ].url, 0, 5, '-createdAt') | |
190 | const follows: ActorFollow[] = res.body.data | |
191 | const server2 = follows.find(f => f.following.host === 'localhost:9002') | |
192 | const server3 = follows.find(f => f.following.host === 'localhost:9003') | |
193 | ||
194 | expect(server3).to.not.be.undefined | |
195 | expect(server3.following.hostRedundancyAllowed).to.be.false | |
196 | ||
197 | expect(server2).to.not.be.undefined | |
198 | expect(server2.following.hostRedundancyAllowed).to.be.false | |
199 | } | |
200 | ||
b36f41ca C |
201 | async function cleanServers () { |
202 | killallServers(servers) | |
203 | } | |
c48e82b5 | 204 | |
b36f41ca | 205 | describe('Test videos redundancy', function () { |
c48e82b5 | 206 | |
b36f41ca | 207 | describe('With most-views strategy', function () { |
4b5384f6 | 208 | const strategy = 'most-views' |
c48e82b5 | 209 | |
b36f41ca C |
210 | before(function () { |
211 | this.timeout(120000) | |
c48e82b5 | 212 | |
4b5384f6 | 213 | return runServers(strategy) |
b36f41ca | 214 | }) |
c48e82b5 | 215 | |
e5565833 C |
216 | it('Should have 1 webseed on the first video', async function () { |
217 | await check1WebSeed(strategy) | |
218 | await checkStatsWith1Webseed(strategy) | |
b36f41ca | 219 | }) |
c48e82b5 | 220 | |
3f6b6a56 | 221 | it('Should enable redundancy on server 1', function () { |
e5565833 | 222 | return enableRedundancyOnServer1() |
b36f41ca | 223 | }) |
c48e82b5 | 224 | |
e5565833 | 225 | it('Should have 2 webseed on the first video', async function () { |
b36f41ca | 226 | this.timeout(40000) |
c48e82b5 | 227 | |
e5565833 | 228 | await waitJobs(servers) |
792e5b8e | 229 | await waitUntilLog(servers[0], 'Duplicated ', 4) |
e5565833 C |
230 | await waitJobs(servers) |
231 | ||
232 | await check2Webseeds(strategy) | |
233 | await checkStatsWith2Webseed(strategy) | |
b36f41ca | 234 | }) |
c48e82b5 | 235 | |
161b061d C |
236 | it('Should undo redundancy on server 1 and remove duplicated videos', async function () { |
237 | this.timeout(40000) | |
238 | ||
239 | await disableRedundancyOnServer1() | |
240 | ||
241 | await waitJobs(servers) | |
242 | await wait(5000) | |
243 | ||
244 | await check1WebSeed(strategy) | |
245 | }) | |
246 | ||
b36f41ca C |
247 | after(function () { |
248 | return cleanServers() | |
249 | }) | |
c48e82b5 C |
250 | }) |
251 | ||
b36f41ca | 252 | describe('With trending strategy', function () { |
4b5384f6 | 253 | const strategy = 'trending' |
c48e82b5 | 254 | |
b36f41ca C |
255 | before(function () { |
256 | this.timeout(120000) | |
257 | ||
4b5384f6 | 258 | return runServers(strategy) |
b36f41ca C |
259 | }) |
260 | ||
e5565833 C |
261 | it('Should have 1 webseed on the first video', async function () { |
262 | await check1WebSeed(strategy) | |
263 | await checkStatsWith1Webseed(strategy) | |
b36f41ca C |
264 | }) |
265 | ||
3f6b6a56 | 266 | it('Should enable redundancy on server 1', function () { |
e5565833 | 267 | return enableRedundancyOnServer1() |
b36f41ca C |
268 | }) |
269 | ||
e5565833 | 270 | it('Should have 2 webseed on the first video', async function () { |
3f6b6a56 C |
271 | this.timeout(40000) |
272 | ||
e5565833 | 273 | await waitJobs(servers) |
792e5b8e | 274 | await waitUntilLog(servers[0], 'Duplicated ', 4) |
e5565833 C |
275 | await waitJobs(servers) |
276 | ||
277 | await check2Webseeds(strategy) | |
278 | await checkStatsWith2Webseed(strategy) | |
3f6b6a56 C |
279 | }) |
280 | ||
161b061d C |
281 | it('Should unfollow on server 1 and remove duplicated videos', async function () { |
282 | this.timeout(40000) | |
283 | ||
284 | await unfollow(servers[0].url, servers[0].accessToken, servers[1]) | |
285 | ||
286 | await waitJobs(servers) | |
287 | await wait(5000) | |
288 | ||
289 | await check1WebSeed(strategy) | |
290 | }) | |
291 | ||
3f6b6a56 C |
292 | after(function () { |
293 | return cleanServers() | |
294 | }) | |
295 | }) | |
296 | ||
297 | describe('With recently added strategy', function () { | |
4b5384f6 | 298 | const strategy = 'recently-added' |
3f6b6a56 C |
299 | |
300 | before(function () { | |
301 | this.timeout(120000) | |
302 | ||
e5565833 | 303 | return runServers(strategy, { min_views: 3 }) |
3f6b6a56 C |
304 | }) |
305 | ||
e5565833 C |
306 | it('Should have 1 webseed on the first video', async function () { |
307 | await check1WebSeed(strategy) | |
308 | await checkStatsWith1Webseed(strategy) | |
3f6b6a56 C |
309 | }) |
310 | ||
311 | it('Should enable redundancy on server 1', function () { | |
e5565833 | 312 | return enableRedundancyOnServer1() |
3f6b6a56 C |
313 | }) |
314 | ||
315 | it('Should still have 1 webseed on the first video', async function () { | |
316 | this.timeout(40000) | |
317 | ||
318 | await waitJobs(servers) | |
319 | await wait(15000) | |
320 | await waitJobs(servers) | |
321 | ||
e5565833 C |
322 | await check1WebSeed(strategy) |
323 | await checkStatsWith1Webseed(strategy) | |
3f6b6a56 C |
324 | }) |
325 | ||
e5565833 | 326 | it('Should view 2 times the first video to have > min_views config', async function () { |
3f6b6a56 C |
327 | this.timeout(40000) |
328 | ||
329 | await viewVideo(servers[ 0 ].url, video1Server2UUID) | |
330 | await viewVideo(servers[ 2 ].url, video1Server2UUID) | |
331 | ||
332 | await wait(10000) | |
333 | await waitJobs(servers) | |
334 | }) | |
335 | ||
e5565833 | 336 | it('Should have 2 webseed on the first video', async function () { |
b36f41ca C |
337 | this.timeout(40000) |
338 | ||
e5565833 | 339 | await waitJobs(servers) |
792e5b8e | 340 | await waitUntilLog(servers[0], 'Duplicated ', 4) |
e5565833 C |
341 | await waitJobs(servers) |
342 | ||
343 | await check2Webseeds(strategy) | |
344 | await checkStatsWith2Webseed(strategy) | |
345 | }) | |
346 | ||
347 | after(function () { | |
348 | return cleanServers() | |
349 | }) | |
350 | }) | |
351 | ||
352 | describe('Test expiration', function () { | |
353 | const strategy = 'recently-added' | |
354 | ||
355 | async function checkContains (servers: ServerInfo[], str: string) { | |
356 | for (const server of servers) { | |
357 | const res = await getVideo(server.url, video1Server2UUID) | |
358 | const video: VideoDetails = res.body | |
359 | ||
360 | for (const f of video.files) { | |
361 | expect(f.magnetUri).to.contain(str) | |
362 | } | |
363 | } | |
364 | } | |
365 | ||
366 | async function checkNotContains (servers: ServerInfo[], str: string) { | |
367 | for (const server of servers) { | |
368 | const res = await getVideo(server.url, video1Server2UUID) | |
369 | const video: VideoDetails = res.body | |
370 | ||
371 | for (const f of video.files) { | |
372 | expect(f.magnetUri).to.not.contain(str) | |
373 | } | |
374 | } | |
375 | } | |
376 | ||
377 | before(async function () { | |
378 | this.timeout(120000) | |
379 | ||
380 | await runServers(strategy, { min_lifetime: '7 seconds', min_views: 0 }) | |
381 | ||
382 | await enableRedundancyOnServer1() | |
383 | }) | |
384 | ||
385 | it('Should still have 2 webseeds after 10 seconds', async function () { | |
386 | this.timeout(40000) | |
387 | ||
388 | await wait(10000) | |
389 | ||
390 | try { | |
391 | await checkContains(servers, 'http%3A%2F%2Flocalhost%3A9001') | |
392 | } catch { | |
393 | // Maybe a server deleted a redundancy in the scheduler | |
394 | await wait(2000) | |
395 | ||
396 | await checkContains(servers, 'http%3A%2F%2Flocalhost%3A9001') | |
397 | } | |
398 | }) | |
399 | ||
400 | it('Should stop server 1 and expire video redundancy', async function () { | |
401 | this.timeout(40000) | |
402 | ||
403 | killallServers([ servers[0] ]) | |
404 | ||
405 | await wait(10000) | |
406 | ||
407 | await checkNotContains([ servers[1], servers[2] ], 'http%3A%2F%2Flocalhost%3A9001') | |
408 | }) | |
409 | ||
410 | after(function () { | |
411 | return killallServers([ servers[1], servers[2] ]) | |
412 | }) | |
413 | }) | |
414 | ||
415 | describe('Test file replacement', function () { | |
416 | let video2Server2UUID: string | |
417 | const strategy = 'recently-added' | |
418 | ||
419 | before(async function () { | |
420 | this.timeout(120000) | |
421 | ||
422 | await runServers(strategy, { min_lifetime: '7 seconds', min_views: 0 }) | |
423 | ||
424 | await enableRedundancyOnServer1() | |
425 | ||
426 | await waitJobs(servers) | |
792e5b8e | 427 | await waitUntilLog(servers[0], 'Duplicated ', 4) |
e5565833 C |
428 | await waitJobs(servers) |
429 | ||
430 | await check2Webseeds(strategy) | |
431 | await checkStatsWith2Webseed(strategy) | |
432 | ||
433 | const res = await uploadVideo(servers[ 1 ].url, servers[ 1 ].accessToken, { name: 'video 2 server 2' }) | |
434 | video2Server2UUID = res.body.video.uuid | |
435 | }) | |
436 | ||
437 | it('Should cache video 2 webseed on the first video', async function () { | |
792e5b8e | 438 | this.timeout(50000) |
e5565833 C |
439 | |
440 | await waitJobs(servers) | |
441 | ||
442 | await wait(7000) | |
443 | ||
792e5b8e C |
444 | try { |
445 | await check1WebSeed(strategy, video1Server2UUID) | |
446 | await check2Webseeds(strategy, video2Server2UUID) | |
447 | } catch { | |
448 | await wait(7000) | |
449 | ||
450 | await check1WebSeed(strategy, video1Server2UUID) | |
451 | await check2Webseeds(strategy, video2Server2UUID) | |
452 | } | |
b36f41ca C |
453 | }) |
454 | ||
455 | after(function () { | |
456 | return cleanServers() | |
457 | }) | |
c48e82b5 C |
458 | }) |
459 | }) |