diff options
Diffstat (limited to 'server/tests/api/redundancy/redundancy.ts')
-rw-r--r-- | server/tests/api/redundancy/redundancy.ts | 182 |
1 files changed, 137 insertions, 45 deletions
diff --git a/server/tests/api/redundancy/redundancy.ts b/server/tests/api/redundancy/redundancy.ts index 1cdf93aa1..c5037a541 100644 --- a/server/tests/api/redundancy/redundancy.ts +++ b/server/tests/api/redundancy/redundancy.ts | |||
@@ -1,11 +1,12 @@ | |||
1 | /* tslint:disable:no-unused-expression */ | 1 | /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ |
2 | 2 | ||
3 | import * as chai from 'chai' | 3 | import * as chai from 'chai' |
4 | import 'mocha' | 4 | import 'mocha' |
5 | import { VideoDetails } from '../../../../shared/models/videos' | 5 | import { VideoDetails } from '../../../../shared/models/videos' |
6 | import { | 6 | import { |
7 | checkSegmentHash, | 7 | checkSegmentHash, |
8 | checkVideoFilesWereRemoved, cleanupTests, | 8 | checkVideoFilesWereRemoved, |
9 | cleanupTests, | ||
9 | doubleFollow, | 10 | doubleFollow, |
10 | flushAndRunMultipleServers, | 11 | flushAndRunMultipleServers, |
11 | getFollowingListPaginationAndSort, | 12 | getFollowingListPaginationAndSort, |
@@ -28,11 +29,16 @@ import { | |||
28 | import { waitJobs } from '../../../../shared/extra-utils/server/jobs' | 29 | import { waitJobs } from '../../../../shared/extra-utils/server/jobs' |
29 | 30 | ||
30 | import * as magnetUtil from 'magnet-uri' | 31 | import * as magnetUtil from 'magnet-uri' |
31 | import { updateRedundancy } from '../../../../shared/extra-utils/server/redundancy' | 32 | import { |
33 | addVideoRedundancy, | ||
34 | listVideoRedundancies, | ||
35 | removeVideoRedundancy, | ||
36 | updateRedundancy | ||
37 | } from '../../../../shared/extra-utils/server/redundancy' | ||
32 | import { ActorFollow } from '../../../../shared/models/actors' | 38 | import { ActorFollow } from '../../../../shared/models/actors' |
33 | import { readdir } from 'fs-extra' | 39 | import { readdir } from 'fs-extra' |
34 | import { join } from 'path' | 40 | import { join } from 'path' |
35 | import { VideoRedundancyStrategy } from '../../../../shared/models/redundancy' | 41 | import { VideoRedundancy, VideoRedundancyStrategy, VideoRedundancyStrategyWithManual } from '../../../../shared/models/redundancy' |
36 | import { getStats } from '../../../../shared/extra-utils/server/stats' | 42 | import { getStats } from '../../../../shared/extra-utils/server/stats' |
37 | import { ServerStats } from '../../../../shared/models/server/server-stats.model' | 43 | import { ServerStats } from '../../../../shared/models/server/server-stats.model' |
38 | 44 | ||
@@ -40,6 +46,7 @@ const expect = chai.expect | |||
40 | 46 | ||
41 | let servers: ServerInfo[] = [] | 47 | let servers: ServerInfo[] = [] |
42 | let video1Server2UUID: string | 48 | let video1Server2UUID: string |
49 | let video1Server2Id: number | ||
43 | 50 | ||
44 | function checkMagnetWebseeds (file: { magnetUri: string, resolution: { id: number } }, baseWebseeds: string[], server: ServerInfo) { | 51 | function checkMagnetWebseeds (file: { magnetUri: string, resolution: { id: number } }, baseWebseeds: string[], server: ServerInfo) { |
45 | const parsed = magnetUtil.decode(file.magnetUri) | 52 | const parsed = magnetUtil.decode(file.magnetUri) |
@@ -52,7 +59,19 @@ function checkMagnetWebseeds (file: { magnetUri: string, resolution: { id: numbe | |||
52 | expect(parsed.urlList).to.have.lengthOf(baseWebseeds.length) | 59 | expect(parsed.urlList).to.have.lengthOf(baseWebseeds.length) |
53 | } | 60 | } |
54 | 61 | ||
55 | async function flushAndRunServers (strategy: VideoRedundancyStrategy, additionalParams: any = {}) { | 62 | async function flushAndRunServers (strategy: VideoRedundancyStrategy | null, additionalParams: any = {}) { |
63 | const strategies: any[] = [] | ||
64 | |||
65 | if (strategy !== null) { | ||
66 | strategies.push( | ||
67 | immutableAssign({ | ||
68 | min_lifetime: '1 hour', | ||
69 | strategy: strategy, | ||
70 | size: '400KB' | ||
71 | }, additionalParams) | ||
72 | ) | ||
73 | } | ||
74 | |||
56 | const config = { | 75 | const config = { |
57 | transcoding: { | 76 | transcoding: { |
58 | hls: { | 77 | hls: { |
@@ -62,36 +81,32 @@ async function flushAndRunServers (strategy: VideoRedundancyStrategy, additional | |||
62 | redundancy: { | 81 | redundancy: { |
63 | videos: { | 82 | videos: { |
64 | check_interval: '5 seconds', | 83 | check_interval: '5 seconds', |
65 | strategies: [ | 84 | strategies |
66 | immutableAssign({ | ||
67 | min_lifetime: '1 hour', | ||
68 | strategy: strategy, | ||
69 | size: '400KB' | ||
70 | }, additionalParams) | ||
71 | ] | ||
72 | } | 85 | } |
73 | } | 86 | } |
74 | } | 87 | } |
88 | |||
75 | servers = await flushAndRunMultipleServers(3, config) | 89 | servers = await flushAndRunMultipleServers(3, config) |
76 | 90 | ||
77 | // Get the access tokens | 91 | // Get the access tokens |
78 | await setAccessTokensToServers(servers) | 92 | await setAccessTokensToServers(servers) |
79 | 93 | ||
80 | { | 94 | { |
81 | const res = await uploadVideo(servers[ 1 ].url, servers[ 1 ].accessToken, { name: 'video 1 server 2' }) | 95 | const res = await uploadVideo(servers[1].url, servers[1].accessToken, { name: 'video 1 server 2' }) |
82 | video1Server2UUID = res.body.video.uuid | 96 | video1Server2UUID = res.body.video.uuid |
97 | video1Server2Id = res.body.video.id | ||
83 | 98 | ||
84 | await viewVideo(servers[ 1 ].url, video1Server2UUID) | 99 | await viewVideo(servers[1].url, video1Server2UUID) |
85 | } | 100 | } |
86 | 101 | ||
87 | await waitJobs(servers) | 102 | await waitJobs(servers) |
88 | 103 | ||
89 | // Server 1 and server 2 follow each other | 104 | // Server 1 and server 2 follow each other |
90 | await doubleFollow(servers[ 0 ], servers[ 1 ]) | 105 | await doubleFollow(servers[0], servers[1]) |
91 | // Server 1 and server 3 follow each other | 106 | // Server 1 and server 3 follow each other |
92 | await doubleFollow(servers[ 0 ], servers[ 2 ]) | 107 | await doubleFollow(servers[0], servers[2]) |
93 | // Server 2 and server 3 follow each other | 108 | // Server 2 and server 3 follow each other |
94 | await doubleFollow(servers[ 1 ], servers[ 2 ]) | 109 | await doubleFollow(servers[1], servers[2]) |
95 | 110 | ||
96 | await waitJobs(servers) | 111 | await waitJobs(servers) |
97 | } | 112 | } |
@@ -100,7 +115,7 @@ async function check1WebSeed (videoUUID?: string) { | |||
100 | if (!videoUUID) videoUUID = video1Server2UUID | 115 | if (!videoUUID) videoUUID = video1Server2UUID |
101 | 116 | ||
102 | const webseeds = [ | 117 | const webseeds = [ |
103 | `http://localhost:${servers[ 1 ].port}/static/webseed/${videoUUID}` | 118 | `http://localhost:${servers[1].port}/static/webseed/${videoUUID}` |
104 | ] | 119 | ] |
105 | 120 | ||
106 | for (const server of servers) { | 121 | for (const server of servers) { |
@@ -118,8 +133,8 @@ async function check2Webseeds (videoUUID?: string) { | |||
118 | if (!videoUUID) videoUUID = video1Server2UUID | 133 | if (!videoUUID) videoUUID = video1Server2UUID |
119 | 134 | ||
120 | const webseeds = [ | 135 | const webseeds = [ |
121 | `http://localhost:${servers[ 0 ].port}/static/redundancy/${videoUUID}`, | 136 | `http://localhost:${servers[0].port}/static/redundancy/${videoUUID}`, |
122 | `http://localhost:${servers[ 1 ].port}/static/webseed/${videoUUID}` | 137 | `http://localhost:${servers[1].port}/static/webseed/${videoUUID}` |
123 | ] | 138 | ] |
124 | 139 | ||
125 | for (const server of servers) { | 140 | for (const server of servers) { |
@@ -216,41 +231,50 @@ async function check1PlaylistRedundancies (videoUUID?: string) { | |||
216 | } | 231 | } |
217 | } | 232 | } |
218 | 233 | ||
219 | async function checkStatsWith2Webseed (strategy: VideoRedundancyStrategy) { | 234 | async function checkStatsGlobal (strategy: VideoRedundancyStrategyWithManual) { |
235 | let totalSize: number = null | ||
236 | let statsLength = 1 | ||
237 | |||
238 | if (strategy !== 'manual') { | ||
239 | totalSize = 409600 | ||
240 | statsLength = 2 | ||
241 | } | ||
242 | |||
220 | const res = await getStats(servers[0].url) | 243 | const res = await getStats(servers[0].url) |
221 | const data: ServerStats = res.body | 244 | const data: ServerStats = res.body |
222 | 245 | ||
223 | expect(data.videosRedundancy).to.have.lengthOf(1) | 246 | expect(data.videosRedundancy).to.have.lengthOf(statsLength) |
224 | const stat = data.videosRedundancy[0] | ||
225 | 247 | ||
248 | const stat = data.videosRedundancy[0] | ||
226 | expect(stat.strategy).to.equal(strategy) | 249 | expect(stat.strategy).to.equal(strategy) |
227 | expect(stat.totalSize).to.equal(409600) | 250 | expect(stat.totalSize).to.equal(totalSize) |
251 | |||
252 | return stat | ||
253 | } | ||
254 | |||
255 | async function checkStatsWith2Webseed (strategy: VideoRedundancyStrategyWithManual) { | ||
256 | const stat = await checkStatsGlobal(strategy) | ||
257 | |||
228 | expect(stat.totalUsed).to.be.at.least(1).and.below(409601) | 258 | expect(stat.totalUsed).to.be.at.least(1).and.below(409601) |
229 | expect(stat.totalVideoFiles).to.equal(4) | 259 | expect(stat.totalVideoFiles).to.equal(4) |
230 | expect(stat.totalVideos).to.equal(1) | 260 | expect(stat.totalVideos).to.equal(1) |
231 | } | 261 | } |
232 | 262 | ||
233 | async function checkStatsWith1Webseed (strategy: VideoRedundancyStrategy) { | 263 | async function checkStatsWith1Webseed (strategy: VideoRedundancyStrategyWithManual) { |
234 | const res = await getStats(servers[0].url) | 264 | const stat = await checkStatsGlobal(strategy) |
235 | const data: ServerStats = res.body | ||
236 | 265 | ||
237 | expect(data.videosRedundancy).to.have.lengthOf(1) | ||
238 | |||
239 | const stat = data.videosRedundancy[0] | ||
240 | expect(stat.strategy).to.equal(strategy) | ||
241 | expect(stat.totalSize).to.equal(409600) | ||
242 | expect(stat.totalUsed).to.equal(0) | 266 | expect(stat.totalUsed).to.equal(0) |
243 | expect(stat.totalVideoFiles).to.equal(0) | 267 | expect(stat.totalVideoFiles).to.equal(0) |
244 | expect(stat.totalVideos).to.equal(0) | 268 | expect(stat.totalVideos).to.equal(0) |
245 | } | 269 | } |
246 | 270 | ||
247 | async function enableRedundancyOnServer1 () { | 271 | async function enableRedundancyOnServer1 () { |
248 | await updateRedundancy(servers[ 0 ].url, servers[ 0 ].accessToken, servers[ 1 ].host, true) | 272 | await updateRedundancy(servers[0].url, servers[0].accessToken, servers[1].host, true) |
249 | 273 | ||
250 | const res = await getFollowingListPaginationAndSort({ url: servers[ 0 ].url, start: 0, count: 5, sort: '-createdAt' }) | 274 | const res = await getFollowingListPaginationAndSort({ url: servers[0].url, start: 0, count: 5, sort: '-createdAt' }) |
251 | const follows: ActorFollow[] = res.body.data | 275 | const follows: ActorFollow[] = res.body.data |
252 | const server2 = follows.find(f => f.following.host === `localhost:${servers[ 1 ].port}`) | 276 | const server2 = follows.find(f => f.following.host === `localhost:${servers[1].port}`) |
253 | const server3 = follows.find(f => f.following.host === `localhost:${servers[ 2 ].port}`) | 277 | const server3 = follows.find(f => f.following.host === `localhost:${servers[2].port}`) |
254 | 278 | ||
255 | expect(server3).to.not.be.undefined | 279 | expect(server3).to.not.be.undefined |
256 | expect(server3.following.hostRedundancyAllowed).to.be.false | 280 | expect(server3.following.hostRedundancyAllowed).to.be.false |
@@ -260,12 +284,12 @@ async function enableRedundancyOnServer1 () { | |||
260 | } | 284 | } |
261 | 285 | ||
262 | async function disableRedundancyOnServer1 () { | 286 | async function disableRedundancyOnServer1 () { |
263 | await updateRedundancy(servers[ 0 ].url, servers[ 0 ].accessToken, servers[ 1 ].host, false) | 287 | await updateRedundancy(servers[0].url, servers[0].accessToken, servers[1].host, false) |
264 | 288 | ||
265 | const res = await getFollowingListPaginationAndSort({ url: servers[ 0 ].url, start: 0, count: 5, sort: '-createdAt' }) | 289 | const res = await getFollowingListPaginationAndSort({ url: servers[0].url, start: 0, count: 5, sort: '-createdAt' }) |
266 | const follows: ActorFollow[] = res.body.data | 290 | const follows: ActorFollow[] = res.body.data |
267 | const server2 = follows.find(f => f.following.host === `localhost:${servers[ 1 ].port}`) | 291 | const server2 = follows.find(f => f.following.host === `localhost:${servers[1].port}`) |
268 | const server3 = follows.find(f => f.following.host === `localhost:${servers[ 2 ].port}`) | 292 | const server3 = follows.find(f => f.following.host === `localhost:${servers[2].port}`) |
269 | 293 | ||
270 | expect(server3).to.not.be.undefined | 294 | expect(server3).to.not.be.undefined |
271 | expect(server3.following.hostRedundancyAllowed).to.be.false | 295 | expect(server3.following.hostRedundancyAllowed).to.be.false |
@@ -410,8 +434,8 @@ describe('Test videos redundancy', function () { | |||
410 | it('Should view 2 times the first video to have > min_views config', async function () { | 434 | it('Should view 2 times the first video to have > min_views config', async function () { |
411 | this.timeout(80000) | 435 | this.timeout(80000) |
412 | 436 | ||
413 | await viewVideo(servers[ 0 ].url, video1Server2UUID) | 437 | await viewVideo(servers[0].url, video1Server2UUID) |
414 | await viewVideo(servers[ 2 ].url, video1Server2UUID) | 438 | await viewVideo(servers[2].url, video1Server2UUID) |
415 | 439 | ||
416 | await wait(10000) | 440 | await wait(10000) |
417 | await waitJobs(servers) | 441 | await waitJobs(servers) |
@@ -446,6 +470,74 @@ describe('Test videos redundancy', function () { | |||
446 | }) | 470 | }) |
447 | }) | 471 | }) |
448 | 472 | ||
473 | describe('With manual strategy', function () { | ||
474 | before(function () { | ||
475 | this.timeout(120000) | ||
476 | |||
477 | return flushAndRunServers(null) | ||
478 | }) | ||
479 | |||
480 | it('Should have 1 webseed on the first video', async function () { | ||
481 | await check1WebSeed() | ||
482 | await check0PlaylistRedundancies() | ||
483 | await checkStatsWith1Webseed('manual') | ||
484 | }) | ||
485 | |||
486 | it('Should create a redundancy on first video', async function () { | ||
487 | await addVideoRedundancy({ | ||
488 | url: servers[0].url, | ||
489 | accessToken: servers[0].accessToken, | ||
490 | videoId: video1Server2Id | ||
491 | }) | ||
492 | }) | ||
493 | |||
494 | it('Should have 2 webseeds on the first video', async function () { | ||
495 | this.timeout(80000) | ||
496 | |||
497 | await waitJobs(servers) | ||
498 | await waitUntilLog(servers[0], 'Duplicated ', 5) | ||
499 | await waitJobs(servers) | ||
500 | |||
501 | await check2Webseeds() | ||
502 | await check1PlaylistRedundancies() | ||
503 | await checkStatsWith2Webseed('manual') | ||
504 | }) | ||
505 | |||
506 | it('Should manually remove redundancies on server 1 and remove duplicated videos', async function () { | ||
507 | this.timeout(80000) | ||
508 | |||
509 | const res = await listVideoRedundancies({ | ||
510 | url: servers[0].url, | ||
511 | accessToken: servers[0].accessToken, | ||
512 | target: 'remote-videos' | ||
513 | }) | ||
514 | |||
515 | const videos = res.body.data as VideoRedundancy[] | ||
516 | expect(videos).to.have.lengthOf(1) | ||
517 | |||
518 | const video = videos[0] | ||
519 | for (const r of video.redundancies.files.concat(video.redundancies.streamingPlaylists)) { | ||
520 | await removeVideoRedundancy({ | ||
521 | url: servers[0].url, | ||
522 | accessToken: servers[0].accessToken, | ||
523 | redundancyId: r.id | ||
524 | }) | ||
525 | } | ||
526 | |||
527 | await waitJobs(servers) | ||
528 | await wait(5000) | ||
529 | |||
530 | await check1WebSeed() | ||
531 | await check0PlaylistRedundancies() | ||
532 | |||
533 | await checkVideoFilesWereRemoved(video1Server2UUID, servers[0].serverNumber, [ 'videos' ]) | ||
534 | }) | ||
535 | |||
536 | after(async function () { | ||
537 | await cleanupTests(servers) | ||
538 | }) | ||
539 | }) | ||
540 | |||
449 | describe('Test expiration', function () { | 541 | describe('Test expiration', function () { |
450 | const strategy = 'recently-added' | 542 | const strategy = 'recently-added' |
451 | 543 | ||
@@ -528,7 +620,7 @@ describe('Test videos redundancy', function () { | |||
528 | await check1PlaylistRedundancies() | 620 | await check1PlaylistRedundancies() |
529 | await checkStatsWith2Webseed(strategy) | 621 | await checkStatsWith2Webseed(strategy) |
530 | 622 | ||
531 | const res = await uploadVideo(servers[ 1 ].url, servers[ 1 ].accessToken, { name: 'video 2 server 2' }) | 623 | const res = await uploadVideo(servers[1].url, servers[1].accessToken, { name: 'video 2 server 2' }) |
532 | video2Server2UUID = res.body.video.uuid | 624 | video2Server2UUID = res.body.video.uuid |
533 | }) | 625 | }) |
534 | 626 | ||
@@ -560,8 +652,8 @@ describe('Test videos redundancy', function () { | |||
560 | 652 | ||
561 | await waitJobs(servers) | 653 | await waitJobs(servers) |
562 | 654 | ||
563 | killallServers([ servers[ 0 ] ]) | 655 | killallServers([ servers[0] ]) |
564 | await reRunServer(servers[ 0 ], { | 656 | await reRunServer(servers[0], { |
565 | redundancy: { | 657 | redundancy: { |
566 | videos: { | 658 | videos: { |
567 | check_interval: '1 second', | 659 | check_interval: '1 second', |