From e5565833f62b97f62ea75eba5b479963ae78b873 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Mon, 24 Sep 2018 13:07:33 +0200 Subject: Improve redundancy: add 'min_lifetime' configuration --- server/tests/api/server/redundancy.ts | 264 ++++++++++++++++++++++++---------- 1 file changed, 191 insertions(+), 73 deletions(-) (limited to 'server/tests/api') diff --git a/server/tests/api/server/redundancy.ts b/server/tests/api/server/redundancy.ts index 6ce4b9dd1..a773e3de4 100644 --- a/server/tests/api/server/redundancy.ts +++ b/server/tests/api/server/redundancy.ts @@ -31,14 +31,13 @@ const expect = chai.expect let servers: ServerInfo[] = [] let video1Server2UUID: string -let video2Server2UUID: string -function checkMagnetWebseeds (file: { magnetUri: string, resolution: { id: number } }, baseWebseeds: string[]) { +function checkMagnetWebseeds (file: { magnetUri: string, resolution: { id: number } }, baseWebseeds: string[], server: ServerInfo) { const parsed = magnetUtil.decode(file.magnetUri) for (const ws of baseWebseeds) { const found = parsed.urlList.find(url => url === `${ws}-${file.resolution.id}.mp4`) - expect(found, `Webseed ${ws} not found in ${file.magnetUri}`).to.not.be.undefined + expect(found, `Webseed ${ws} not found in ${file.magnetUri} on server ${server.url}`).to.not.be.undefined } } @@ -49,6 +48,7 @@ async function runServers (strategy: VideoRedundancyStrategy, additionalParams: check_interval: '5 seconds', strategies: [ immutableAssign({ + min_lifetime: '1 hour', strategy: strategy, size: '100KB' }, additionalParams) @@ -68,11 +68,6 @@ async function runServers (strategy: VideoRedundancyStrategy, additionalParams: await viewVideo(servers[ 1 ].url, video1Server2UUID) } - { - const res = await uploadVideo(servers[ 1 ].url, servers[ 1 ].accessToken, { name: 'video 2 server 2' }) - video2Server2UUID = res.body.video.uuid - } - await waitJobs(servers) // Server 1 and server 2 follow each other @@ -85,68 +80,69 @@ async function runServers (strategy: VideoRedundancyStrategy, additionalParams: await waitJobs(servers) } -async function check1WebSeed (strategy: VideoRedundancyStrategy) { +async function check1WebSeed (strategy: VideoRedundancyStrategy, videoUUID?: string) { + if (!videoUUID) videoUUID = video1Server2UUID + const webseeds = [ - 'http://localhost:9002/static/webseed/' + video1Server2UUID + 'http://localhost:9002/static/webseed/' + videoUUID ] for (const server of servers) { { - const res = await getVideo(server.url, video1Server2UUID) + const res = await getVideo(server.url, videoUUID) const video: VideoDetails = res.body - video.files.forEach(f => checkMagnetWebseeds(f, webseeds)) + for (const f of video.files) { + checkMagnetWebseeds(f, webseeds, server) + } } + } +} - { - const res = await getStats(server.url) - const data: ServerStats = res.body +async function checkStatsWith2Webseed (strategy: VideoRedundancyStrategy) { + const res = await getStats(servers[0].url) + const data: ServerStats = res.body - expect(data.videosRedundancy).to.have.lengthOf(1) + expect(data.videosRedundancy).to.have.lengthOf(1) + const stat = data.videosRedundancy[0] - const stat = data.videosRedundancy[0] - expect(stat.strategy).to.equal(strategy) - expect(stat.totalSize).to.equal(102400) - expect(stat.totalUsed).to.equal(0) - expect(stat.totalVideoFiles).to.equal(0) - expect(stat.totalVideos).to.equal(0) - } - } + expect(stat.strategy).to.equal(strategy) + expect(stat.totalSize).to.equal(102400) + expect(stat.totalUsed).to.be.at.least(1).and.below(102401) + expect(stat.totalVideoFiles).to.equal(4) + expect(stat.totalVideos).to.equal(1) } -async function enableRedundancy () { - await updateRedundancy(servers[ 0 ].url, servers[ 0 ].accessToken, servers[ 1 ].host, true) +async function checkStatsWith1Webseed (strategy: VideoRedundancyStrategy) { + const res = await getStats(servers[0].url) + const data: ServerStats = res.body - const res = await getFollowingListPaginationAndSort(servers[ 0 ].url, 0, 5, '-createdAt') - const follows: ActorFollow[] = res.body.data - const server2 = follows.find(f => f.following.host === 'localhost:9002') - const server3 = follows.find(f => f.following.host === 'localhost:9003') - - expect(server3).to.not.be.undefined - expect(server3.following.hostRedundancyAllowed).to.be.false + expect(data.videosRedundancy).to.have.lengthOf(1) - expect(server2).to.not.be.undefined - expect(server2.following.hostRedundancyAllowed).to.be.true + const stat = data.videosRedundancy[0] + expect(stat.strategy).to.equal(strategy) + expect(stat.totalSize).to.equal(102400) + expect(stat.totalUsed).to.equal(0) + expect(stat.totalVideoFiles).to.equal(0) + expect(stat.totalVideos).to.equal(0) } -async function check2Webseeds (strategy: VideoRedundancyStrategy) { - await waitJobs(servers) - await wait(15000) - await waitJobs(servers) +async function check2Webseeds (strategy: VideoRedundancyStrategy, videoUUID?: string) { + if (!videoUUID) videoUUID = video1Server2UUID const webseeds = [ - 'http://localhost:9001/static/webseed/' + video1Server2UUID, - 'http://localhost:9002/static/webseed/' + video1Server2UUID + 'http://localhost:9001/static/webseed/' + videoUUID, + 'http://localhost:9002/static/webseed/' + videoUUID ] for (const server of servers) { { - const res = await getVideo(server.url, video1Server2UUID) + const res = await getVideo(server.url, videoUUID) const video: VideoDetails = res.body for (const file of video.files) { - checkMagnetWebseeds(file, webseeds) + checkMagnetWebseeds(file, webseeds, server) } } } @@ -155,22 +151,23 @@ async function check2Webseeds (strategy: VideoRedundancyStrategy) { expect(files).to.have.lengthOf(4) for (const resolution of [ 240, 360, 480, 720 ]) { - expect(files.find(f => f === `${video1Server2UUID}-${resolution}.mp4`)).to.not.be.undefined + expect(files.find(f => f === `${videoUUID}-${resolution}.mp4`)).to.not.be.undefined } +} - { - const res = await getStats(servers[0].url) - const data: ServerStats = res.body +async function enableRedundancyOnServer1 () { + await updateRedundancy(servers[ 0 ].url, servers[ 0 ].accessToken, servers[ 1 ].host, true) - expect(data.videosRedundancy).to.have.lengthOf(1) - const stat = data.videosRedundancy[0] + const res = await getFollowingListPaginationAndSort(servers[ 0 ].url, 0, 5, '-createdAt') + const follows: ActorFollow[] = res.body.data + const server2 = follows.find(f => f.following.host === 'localhost:9002') + const server3 = follows.find(f => f.following.host === 'localhost:9003') - expect(stat.strategy).to.equal(strategy) - expect(stat.totalSize).to.equal(102400) - expect(stat.totalUsed).to.be.at.least(1).and.below(102401) - expect(stat.totalVideoFiles).to.equal(4) - expect(stat.totalVideos).to.equal(1) - } + expect(server3).to.not.be.undefined + expect(server3.following.hostRedundancyAllowed).to.be.false + + expect(server2).to.not.be.undefined + expect(server2.following.hostRedundancyAllowed).to.be.true } async function cleanServers () { @@ -188,18 +185,24 @@ describe('Test videos redundancy', function () { return runServers(strategy) }) - it('Should have 1 webseed on the first video', function () { - return check1WebSeed(strategy) + it('Should have 1 webseed on the first video', async function () { + await check1WebSeed(strategy) + await checkStatsWith1Webseed(strategy) }) it('Should enable redundancy on server 1', function () { - return enableRedundancy() + return enableRedundancyOnServer1() }) - it('Should have 2 webseed on the first video', function () { + it('Should have 2 webseed on the first video', async function () { this.timeout(40000) - return check2Webseeds(strategy) + await waitJobs(servers) + await wait(15000) + await waitJobs(servers) + + await check2Webseeds(strategy) + await checkStatsWith2Webseed(strategy) }) after(function () { @@ -216,18 +219,24 @@ describe('Test videos redundancy', function () { return runServers(strategy) }) - it('Should have 1 webseed on the first video', function () { - return check1WebSeed(strategy) + it('Should have 1 webseed on the first video', async function () { + await check1WebSeed(strategy) + await checkStatsWith1Webseed(strategy) }) it('Should enable redundancy on server 1', function () { - return enableRedundancy() + return enableRedundancyOnServer1() }) - it('Should have 2 webseed on the first video', function () { + it('Should have 2 webseed on the first video', async function () { this.timeout(40000) - return check2Webseeds(strategy) + await waitJobs(servers) + await wait(15000) + await waitJobs(servers) + + await check2Webseeds(strategy) + await checkStatsWith2Webseed(strategy) }) after(function () { @@ -241,15 +250,16 @@ describe('Test videos redundancy', function () { before(function () { this.timeout(120000) - return runServers(strategy, { minViews: 3 }) + return runServers(strategy, { min_views: 3 }) }) - it('Should have 1 webseed on the first video', function () { - return check1WebSeed(strategy) + it('Should have 1 webseed on the first video', async function () { + await check1WebSeed(strategy) + await checkStatsWith1Webseed(strategy) }) it('Should enable redundancy on server 1', function () { - return enableRedundancy() + return enableRedundancyOnServer1() }) it('Should still have 1 webseed on the first video', async function () { @@ -259,10 +269,11 @@ describe('Test videos redundancy', function () { await wait(15000) await waitJobs(servers) - return check1WebSeed(strategy) + await check1WebSeed(strategy) + await checkStatsWith1Webseed(strategy) }) - it('Should view 2 times the first video', async function () { + it('Should view 2 times the first video to have > min_views config', async function () { this.timeout(40000) await viewVideo(servers[ 0 ].url, video1Server2UUID) @@ -272,10 +283,117 @@ describe('Test videos redundancy', function () { await waitJobs(servers) }) - it('Should have 2 webseed on the first video', function () { + it('Should have 2 webseed on the first video', async function () { this.timeout(40000) - return check2Webseeds(strategy) + await waitJobs(servers) + await wait(15000) + await waitJobs(servers) + + await check2Webseeds(strategy) + await checkStatsWith2Webseed(strategy) + }) + + after(function () { + return cleanServers() + }) + }) + + describe('Test expiration', function () { + const strategy = 'recently-added' + + async function checkContains (servers: ServerInfo[], str: string) { + for (const server of servers) { + const res = await getVideo(server.url, video1Server2UUID) + const video: VideoDetails = res.body + + for (const f of video.files) { + expect(f.magnetUri).to.contain(str) + } + } + } + + async function checkNotContains (servers: ServerInfo[], str: string) { + for (const server of servers) { + const res = await getVideo(server.url, video1Server2UUID) + const video: VideoDetails = res.body + + for (const f of video.files) { + expect(f.magnetUri).to.not.contain(str) + } + } + } + + before(async function () { + this.timeout(120000) + + await runServers(strategy, { min_lifetime: '7 seconds', min_views: 0 }) + + await enableRedundancyOnServer1() + }) + + it('Should still have 2 webseeds after 10 seconds', async function () { + this.timeout(40000) + + await wait(10000) + + try { + await checkContains(servers, 'http%3A%2F%2Flocalhost%3A9001') + } catch { + // Maybe a server deleted a redundancy in the scheduler + await wait(2000) + + await checkContains(servers, 'http%3A%2F%2Flocalhost%3A9001') + } + }) + + it('Should stop server 1 and expire video redundancy', async function () { + this.timeout(40000) + + killallServers([ servers[0] ]) + + await wait(10000) + + await checkNotContains([ servers[1], servers[2] ], 'http%3A%2F%2Flocalhost%3A9001') + }) + + after(function () { + return killallServers([ servers[1], servers[2] ]) + }) + }) + + describe('Test file replacement', function () { + let video2Server2UUID: string + const strategy = 'recently-added' + + before(async function () { + this.timeout(120000) + + await runServers(strategy, { min_lifetime: '7 seconds', min_views: 0 }) + + await enableRedundancyOnServer1() + + await waitJobs(servers) + await wait(5000) + await waitJobs(servers) + + await check2Webseeds(strategy) + await checkStatsWith2Webseed(strategy) + + const res = await uploadVideo(servers[ 1 ].url, servers[ 1 ].accessToken, { name: 'video 2 server 2' }) + video2Server2UUID = res.body.video.uuid + }) + + it('Should cache video 2 webseed on the first video', async function () { + this.timeout(40000) + this.retries(3) + + await waitJobs(servers) + + await wait(7000) + + await check1WebSeed(strategy, video1Server2UUID) + await check2Webseeds(strategy, video2Server2UUID) }) after(function () { -- cgit v1.2.3