From 26370ce469913f48b1c10b91d68ac8cb67586ce1 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Tue, 16 Oct 2018 10:55:29 +0200 Subject: Move redundancy in its own travis container --- server/tests/api/redundancy/index.ts | 1 + server/tests/api/redundancy/redundancy.ts | 483 ++++++++++++++++++++++++++++++ 2 files changed, 484 insertions(+) create mode 100644 server/tests/api/redundancy/index.ts create mode 100644 server/tests/api/redundancy/redundancy.ts (limited to 'server/tests/api/redundancy') diff --git a/server/tests/api/redundancy/index.ts b/server/tests/api/redundancy/index.ts new file mode 100644 index 000000000..8e69b95a6 --- /dev/null +++ b/server/tests/api/redundancy/index.ts @@ -0,0 +1 @@ +import './redundancy' diff --git a/server/tests/api/redundancy/redundancy.ts b/server/tests/api/redundancy/redundancy.ts new file mode 100644 index 000000000..1960854b6 --- /dev/null +++ b/server/tests/api/redundancy/redundancy.ts @@ -0,0 +1,483 @@ +/* tslint:disable:no-unused-expression */ + +import * as chai from 'chai' +import 'mocha' +import { VideoDetails } from '../../../../shared/models/videos' +import { + doubleFollow, + flushAndRunMultipleServers, + getFollowingListPaginationAndSort, + getVideo, + immutableAssign, + killallServers, makeGetRequest, + root, + ServerInfo, + setAccessTokensToServers, unfollow, + uploadVideo, + viewVideo, + wait, + waitUntilLog, + checkVideoFilesWereRemoved, removeVideo +} from '../../utils' +import { waitJobs } from '../../utils/server/jobs' +import * as magnetUtil from 'magnet-uri' +import { updateRedundancy } from '../../utils/server/redundancy' +import { ActorFollow } from '../../../../shared/models/actors' +import { readdir } from 'fs-extra' +import { join } from 'path' +import { VideoRedundancyStrategy } from '../../../../shared/models/redundancy' +import { getStats } from '../../utils/server/stats' +import { ServerStats } from '../../../../shared/models/server/server-stats.model' + +const expect = chai.expect + +let servers: ServerInfo[] = [] +let video1Server2UUID: string + +function checkMagnetWebseeds (file: { magnetUri: string, resolution: { id: number } }, baseWebseeds: string[], server: ServerInfo) { + const parsed = magnetUtil.decode(file.magnetUri) + + for (const ws of baseWebseeds) { + const found = parsed.urlList.find(url => url === `${ws}-${file.resolution.id}.mp4`) + expect(found, `Webseed ${ws} not found in ${file.magnetUri} on server ${server.url}`).to.not.be.undefined + } + + expect(parsed.urlList).to.have.lengthOf(baseWebseeds.length) +} + +async function runServers (strategy: VideoRedundancyStrategy, additionalParams: any = {}) { + const config = { + redundancy: { + videos: { + check_interval: '5 seconds', + strategies: [ + immutableAssign({ + min_lifetime: '1 hour', + strategy: strategy, + size: '100KB' + }, additionalParams) + ] + } + } + } + servers = await flushAndRunMultipleServers(3, config) + + // Get the access tokens + await setAccessTokensToServers(servers) + + { + const res = await uploadVideo(servers[ 1 ].url, servers[ 1 ].accessToken, { name: 'video 1 server 2' }) + video1Server2UUID = res.body.video.uuid + + await viewVideo(servers[ 1 ].url, video1Server2UUID) + } + + await waitJobs(servers) + + // Server 1 and server 2 follow each other + await doubleFollow(servers[ 0 ], servers[ 1 ]) + // Server 1 and server 3 follow each other + await doubleFollow(servers[ 0 ], servers[ 2 ]) + // Server 2 and server 3 follow each other + await doubleFollow(servers[ 1 ], servers[ 2 ]) + + await waitJobs(servers) +} + +async function check1WebSeed (strategy: VideoRedundancyStrategy, videoUUID?: string) { + if (!videoUUID) videoUUID = video1Server2UUID + + const webseeds = [ + 'http://localhost:9002/static/webseed/' + videoUUID + ] + + for (const server of servers) { + { + const res = await getVideo(server.url, videoUUID) + + const video: VideoDetails = res.body + for (const f of video.files) { + checkMagnetWebseeds(f, webseeds, server) + } + } + } +} + +async function checkStatsWith2Webseed (strategy: VideoRedundancyStrategy) { + const res = await getStats(servers[0].url) + const data: ServerStats = res.body + + expect(data.videosRedundancy).to.have.lengthOf(1) + const stat = data.videosRedundancy[0] + + expect(stat.strategy).to.equal(strategy) + expect(stat.totalSize).to.equal(102400) + expect(stat.totalUsed).to.be.at.least(1).and.below(102401) + expect(stat.totalVideoFiles).to.equal(4) + expect(stat.totalVideos).to.equal(1) +} + +async function checkStatsWith1Webseed (strategy: VideoRedundancyStrategy) { + const res = await getStats(servers[0].url) + const data: ServerStats = res.body + + expect(data.videosRedundancy).to.have.lengthOf(1) + + const stat = data.videosRedundancy[0] + expect(stat.strategy).to.equal(strategy) + expect(stat.totalSize).to.equal(102400) + expect(stat.totalUsed).to.equal(0) + expect(stat.totalVideoFiles).to.equal(0) + expect(stat.totalVideos).to.equal(0) +} + +async function check2Webseeds (strategy: VideoRedundancyStrategy, videoUUID?: string) { + if (!videoUUID) videoUUID = video1Server2UUID + + const webseeds = [ + 'http://localhost:9001/static/webseed/' + videoUUID, + 'http://localhost:9002/static/webseed/' + videoUUID + ] + + for (const server of servers) { + const res = await getVideo(server.url, videoUUID) + + const video: VideoDetails = res.body + + for (const file of video.files) { + checkMagnetWebseeds(file, webseeds, server) + + // Only servers 1 and 2 have the video + if (server.serverNumber !== 3) { + await makeGetRequest({ + url: server.url, + statusCodeExpected: 200, + path: '/static/webseed/' + `${videoUUID}-${file.resolution.id}.mp4`, + contentType: null + }) + } + } + } + + for (const directory of [ 'test1', 'test2' ]) { + const files = await readdir(join(root(), directory, 'videos')) + expect(files).to.have.length.at.least(4) + + for (const resolution of [ 240, 360, 480, 720 ]) { + expect(files.find(f => f === `${videoUUID}-${resolution}.mp4`)).to.not.be.undefined + } + } +} + +async function enableRedundancyOnServer1 () { + await updateRedundancy(servers[ 0 ].url, servers[ 0 ].accessToken, servers[ 1 ].host, true) + + const res = await getFollowingListPaginationAndSort(servers[ 0 ].url, 0, 5, '-createdAt') + const follows: ActorFollow[] = res.body.data + const server2 = follows.find(f => f.following.host === 'localhost:9002') + const server3 = follows.find(f => f.following.host === 'localhost:9003') + + expect(server3).to.not.be.undefined + expect(server3.following.hostRedundancyAllowed).to.be.false + + expect(server2).to.not.be.undefined + expect(server2.following.hostRedundancyAllowed).to.be.true +} + +async function disableRedundancyOnServer1 () { + await updateRedundancy(servers[ 0 ].url, servers[ 0 ].accessToken, servers[ 1 ].host, false) + + const res = await getFollowingListPaginationAndSort(servers[ 0 ].url, 0, 5, '-createdAt') + const follows: ActorFollow[] = res.body.data + const server2 = follows.find(f => f.following.host === 'localhost:9002') + const server3 = follows.find(f => f.following.host === 'localhost:9003') + + expect(server3).to.not.be.undefined + expect(server3.following.hostRedundancyAllowed).to.be.false + + expect(server2).to.not.be.undefined + expect(server2.following.hostRedundancyAllowed).to.be.false +} + +async function cleanServers () { + killallServers(servers) +} + +describe('Test videos redundancy', function () { + + describe('With most-views strategy', function () { + const strategy = 'most-views' + + before(function () { + this.timeout(120000) + + return runServers(strategy) + }) + + it('Should have 1 webseed on the first video', async function () { + await check1WebSeed(strategy) + await checkStatsWith1Webseed(strategy) + }) + + it('Should enable redundancy on server 1', function () { + return enableRedundancyOnServer1() + }) + + it('Should have 2 webseed on the first video', async function () { + this.timeout(40000) + + await waitJobs(servers) + await waitUntilLog(servers[0], 'Duplicated ', 4) + await waitJobs(servers) + + await check2Webseeds(strategy) + await checkStatsWith2Webseed(strategy) + }) + + it('Should undo redundancy on server 1 and remove duplicated videos', async function () { + this.timeout(40000) + + await disableRedundancyOnServer1() + + await waitJobs(servers) + await wait(5000) + + await check1WebSeed(strategy) + + await checkVideoFilesWereRemoved(video1Server2UUID, servers[0].serverNumber, [ 'videos' ]) + }) + + after(function () { + return cleanServers() + }) + }) + + describe('With trending strategy', function () { + const strategy = 'trending' + + before(function () { + this.timeout(120000) + + return runServers(strategy) + }) + + it('Should have 1 webseed on the first video', async function () { + await check1WebSeed(strategy) + await checkStatsWith1Webseed(strategy) + }) + + it('Should enable redundancy on server 1', function () { + return enableRedundancyOnServer1() + }) + + it('Should have 2 webseed on the first video', async function () { + this.timeout(40000) + + await waitJobs(servers) + await waitUntilLog(servers[0], 'Duplicated ', 4) + await waitJobs(servers) + + await check2Webseeds(strategy) + await checkStatsWith2Webseed(strategy) + }) + + it('Should unfollow on server 1 and remove duplicated videos', async function () { + this.timeout(40000) + + await unfollow(servers[0].url, servers[0].accessToken, servers[1]) + + await waitJobs(servers) + await wait(5000) + + await check1WebSeed(strategy) + + await checkVideoFilesWereRemoved(video1Server2UUID, servers[0].serverNumber, [ 'videos' ]) + }) + + after(function () { + return cleanServers() + }) + }) + + describe('With recently added strategy', function () { + const strategy = 'recently-added' + + before(function () { + this.timeout(120000) + + return runServers(strategy, { min_views: 3 }) + }) + + it('Should have 1 webseed on the first video', async function () { + await check1WebSeed(strategy) + await checkStatsWith1Webseed(strategy) + }) + + it('Should enable redundancy on server 1', function () { + return enableRedundancyOnServer1() + }) + + it('Should still have 1 webseed on the first video', async function () { + this.timeout(40000) + + await waitJobs(servers) + await wait(15000) + await waitJobs(servers) + + await check1WebSeed(strategy) + await checkStatsWith1Webseed(strategy) + }) + + it('Should view 2 times the first video to have > min_views config', async function () { + this.timeout(40000) + + await viewVideo(servers[ 0 ].url, video1Server2UUID) + await viewVideo(servers[ 2 ].url, video1Server2UUID) + + await wait(10000) + await waitJobs(servers) + }) + + it('Should have 2 webseed on the first video', async function () { + this.timeout(40000) + + await waitJobs(servers) + await waitUntilLog(servers[0], 'Duplicated ', 4) + await waitJobs(servers) + + await check2Webseeds(strategy) + await checkStatsWith2Webseed(strategy) + }) + + it('Should remove the video and the redundancy files', async function () { + this.timeout(20000) + + await removeVideo(servers[1].url, servers[1].accessToken, video1Server2UUID) + + await waitJobs(servers) + + for (const server of servers) { + await checkVideoFilesWereRemoved(video1Server2UUID, server.serverNumber) + } + }) + + after(function () { + return cleanServers() + }) + }) + + describe('Test expiration', function () { + const strategy = 'recently-added' + + async function checkContains (servers: ServerInfo[], str: string) { + for (const server of servers) { + const res = await getVideo(server.url, video1Server2UUID) + const video: VideoDetails = res.body + + for (const f of video.files) { + expect(f.magnetUri).to.contain(str) + } + } + } + + async function checkNotContains (servers: ServerInfo[], str: string) { + for (const server of servers) { + const res = await getVideo(server.url, video1Server2UUID) + const video: VideoDetails = res.body + + for (const f of video.files) { + expect(f.magnetUri).to.not.contain(str) + } + } + } + + before(async function () { + this.timeout(120000) + + await runServers(strategy, { min_lifetime: '7 seconds', min_views: 0 }) + + await enableRedundancyOnServer1() + }) + + it('Should still have 2 webseeds after 10 seconds', async function () { + this.timeout(40000) + + await wait(10000) + + try { + await checkContains(servers, 'http%3A%2F%2Flocalhost%3A9001') + } catch { + // Maybe a server deleted a redundancy in the scheduler + await wait(2000) + + await checkContains(servers, 'http%3A%2F%2Flocalhost%3A9001') + } + }) + + it('Should stop server 1 and expire video redundancy', async function () { + this.timeout(40000) + + killallServers([ servers[0] ]) + + await wait(10000) + + await checkNotContains([ servers[1], servers[2] ], 'http%3A%2F%2Flocalhost%3A9001') + }) + + after(function () { + return killallServers([ servers[1], servers[2] ]) + }) + }) + + describe('Test file replacement', function () { + let video2Server2UUID: string + const strategy = 'recently-added' + + before(async function () { + this.timeout(120000) + + await runServers(strategy, { min_lifetime: '7 seconds', min_views: 0 }) + + await enableRedundancyOnServer1() + + await waitJobs(servers) + await waitUntilLog(servers[0], 'Duplicated ', 4) + await waitJobs(servers) + + await check2Webseeds(strategy) + await checkStatsWith2Webseed(strategy) + + const res = await uploadVideo(servers[ 1 ].url, servers[ 1 ].accessToken, { name: 'video 2 server 2' }) + video2Server2UUID = res.body.video.uuid + }) + + it('Should cache video 2 webseed on the first video', async function () { + this.timeout(50000) + + await waitJobs(servers) + + await wait(7000) + + try { + await check1WebSeed(strategy, video1Server2UUID) + await check2Webseeds(strategy, video2Server2UUID) + } catch { + await wait(3000) + + try { + await check1WebSeed(strategy, video1Server2UUID) + await check2Webseeds(strategy, video2Server2UUID) + } catch { + await wait(5000) + + await check1WebSeed(strategy, video1Server2UUID) + await check2Webseeds(strategy, video2Server2UUID) + } + } + }) + + after(function () { + return cleanServers() + }) + }) +}) -- cgit v1.2.3 From 9639bd175726b73f8fe664b5ced12a72407b1f0b Mon Sep 17 00:00:00 2001 From: buoyantair Date: Mon, 29 Oct 2018 22:18:31 +0530 Subject: Move utils to /shared Move utils used by /server/tools/* & /server/tests/**/* into /shared folder. Issue: #1336 --- server/tests/api/redundancy/redundancy.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'server/tests/api/redundancy') diff --git a/server/tests/api/redundancy/redundancy.ts b/server/tests/api/redundancy/redundancy.ts index 1960854b6..0af52023c 100644 --- a/server/tests/api/redundancy/redundancy.ts +++ b/server/tests/api/redundancy/redundancy.ts @@ -18,15 +18,15 @@ import { wait, waitUntilLog, checkVideoFilesWereRemoved, removeVideo -} from '../../utils' -import { waitJobs } from '../../utils/server/jobs' +} from '../../../../shared/utils' +import { waitJobs } from '../../../../shared/utils/server/jobs' import * as magnetUtil from 'magnet-uri' -import { updateRedundancy } from '../../utils/server/redundancy' +import { updateRedundancy } from '../../../../shared/utils/server/redundancy' import { ActorFollow } from '../../../../shared/models/actors' import { readdir } from 'fs-extra' import { join } from 'path' import { VideoRedundancyStrategy } from '../../../../shared/models/redundancy' -import { getStats } from '../../utils/server/stats' +import { getStats } from '../../../../shared/utils/server/stats' import { ServerStats } from '../../../../shared/models/server/server-stats.model' const expect = chai.expect -- cgit v1.2.3 From 6cb3482ceba2e0564a05b525699f29a1f5ff20a2 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Thu, 15 Nov 2018 11:20:23 +0100 Subject: Remove wrong redundancy test --- server/tests/api/redundancy/redundancy.ts | 36 ++++++++++++++----------------- 1 file changed, 16 insertions(+), 20 deletions(-) (limited to 'server/tests/api/redundancy') diff --git a/server/tests/api/redundancy/redundancy.ts b/server/tests/api/redundancy/redundancy.ts index 1960854b6..47f4e59fc 100644 --- a/server/tests/api/redundancy/redundancy.ts +++ b/server/tests/api/redundancy/redundancy.ts @@ -54,7 +54,7 @@ async function runServers (strategy: VideoRedundancyStrategy, additionalParams: immutableAssign({ min_lifetime: '1 hour', strategy: strategy, - size: '100KB' + size: '200KB' }, additionalParams) ] } @@ -111,8 +111,8 @@ async function checkStatsWith2Webseed (strategy: VideoRedundancyStrategy) { const stat = data.videosRedundancy[0] expect(stat.strategy).to.equal(strategy) - expect(stat.totalSize).to.equal(102400) - expect(stat.totalUsed).to.be.at.least(1).and.below(102401) + expect(stat.totalSize).to.equal(204800) + expect(stat.totalUsed).to.be.at.least(1).and.below(204801) expect(stat.totalVideoFiles).to.equal(4) expect(stat.totalVideos).to.equal(1) } @@ -125,7 +125,7 @@ async function checkStatsWith1Webseed (strategy: VideoRedundancyStrategy) { const stat = data.videosRedundancy[0] expect(stat.strategy).to.equal(strategy) - expect(stat.totalSize).to.equal(102400) + expect(stat.totalSize).to.equal(204800) expect(stat.totalUsed).to.equal(0) expect(stat.totalVideoFiles).to.equal(0) expect(stat.totalVideos).to.equal(0) @@ -223,7 +223,7 @@ describe('Test videos redundancy', function () { return enableRedundancyOnServer1() }) - it('Should have 2 webseed on the first video', async function () { + it('Should have 2 webseeds on the first video', async function () { this.timeout(40000) await waitJobs(servers) @@ -270,7 +270,7 @@ describe('Test videos redundancy', function () { return enableRedundancyOnServer1() }) - it('Should have 2 webseed on the first video', async function () { + it('Should have 2 webseeds on the first video', async function () { this.timeout(40000) await waitJobs(servers) @@ -338,7 +338,7 @@ describe('Test videos redundancy', function () { await waitJobs(servers) }) - it('Should have 2 webseed on the first video', async function () { + it('Should have 2 webseeds on the first video', async function () { this.timeout(40000) await waitJobs(servers) @@ -419,7 +419,7 @@ describe('Test videos redundancy', function () { killallServers([ servers[0] ]) - await wait(10000) + await wait(15000) await checkNotContains([ servers[1], servers[2] ], 'http%3A%2F%2Flocalhost%3A9001') }) @@ -451,27 +451,23 @@ describe('Test videos redundancy', function () { video2Server2UUID = res.body.video.uuid }) - it('Should cache video 2 webseed on the first video', async function () { - this.timeout(50000) + it('Should cache video 2 webseeds on the first video', async function () { + this.timeout(120000) await waitJobs(servers) - await wait(7000) + let checked = false - try { - await check1WebSeed(strategy, video1Server2UUID) - await check2Webseeds(strategy, video2Server2UUID) - } catch { - await wait(3000) + while (checked === false) { + await wait(1000) try { await check1WebSeed(strategy, video1Server2UUID) await check2Webseeds(strategy, video2Server2UUID) - } catch { - await wait(5000) - await check1WebSeed(strategy, video1Server2UUID) - await check2Webseeds(strategy, video2Server2UUID) + checked = true + } catch { + checked = false } } }) -- cgit v1.2.3 From 58d515e32fe1d0133435b3a5e550c6ff24906fff Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Fri, 16 Nov 2018 16:48:17 +0100 Subject: Fix images size when downloading them --- server/tests/api/redundancy/redundancy.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'server/tests/api/redundancy') diff --git a/server/tests/api/redundancy/redundancy.ts b/server/tests/api/redundancy/redundancy.ts index 47f4e59fc..a8a2f305f 100644 --- a/server/tests/api/redundancy/redundancy.ts +++ b/server/tests/api/redundancy/redundancy.ts @@ -17,7 +17,7 @@ import { viewVideo, wait, waitUntilLog, - checkVideoFilesWereRemoved, removeVideo + checkVideoFilesWereRemoved, removeVideo, getVideoWithToken } from '../../utils' import { waitJobs } from '../../utils/server/jobs' import * as magnetUtil from 'magnet-uri' @@ -93,7 +93,8 @@ async function check1WebSeed (strategy: VideoRedundancyStrategy, videoUUID?: str for (const server of servers) { { - const res = await getVideo(server.url, videoUUID) + // With token to avoid issues with video follow constraints + const res = await getVideoWithToken(server.url, server.accessToken, videoUUID) const video: VideoDetails = res.body for (const f of video.files) { -- cgit v1.2.3 From b9fffa297f49a84df8ffd0d7b842599bc88a8e3e Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Tue, 4 Dec 2018 17:08:55 +0100 Subject: Create redundancy endpoint --- server/tests/api/redundancy/redundancy.ts | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) (limited to 'server/tests/api/redundancy') diff --git a/server/tests/api/redundancy/redundancy.ts b/server/tests/api/redundancy/redundancy.ts index a8a2f305f..5b29a503a 100644 --- a/server/tests/api/redundancy/redundancy.ts +++ b/server/tests/api/redundancy/redundancy.ts @@ -136,7 +136,7 @@ async function check2Webseeds (strategy: VideoRedundancyStrategy, videoUUID?: st if (!videoUUID) videoUUID = video1Server2UUID const webseeds = [ - 'http://localhost:9001/static/webseed/' + videoUUID, + 'http://localhost:9001/static/redundancy/' + videoUUID, 'http://localhost:9002/static/webseed/' + videoUUID ] @@ -148,20 +148,23 @@ async function check2Webseeds (strategy: VideoRedundancyStrategy, videoUUID?: st for (const file of video.files) { checkMagnetWebseeds(file, webseeds, server) - // Only servers 1 and 2 have the video - if (server.serverNumber !== 3) { - await makeGetRequest({ - url: server.url, - statusCodeExpected: 200, - path: '/static/webseed/' + `${videoUUID}-${file.resolution.id}.mp4`, - contentType: null - }) - } + await makeGetRequest({ + url: servers[0].url, + statusCodeExpected: 200, + path: '/static/redundancy/' + `${videoUUID}-${file.resolution.id}.mp4`, + contentType: null + }) + await makeGetRequest({ + url: servers[1].url, + statusCodeExpected: 200, + path: '/static/webseed/' + `${videoUUID}-${file.resolution.id}.mp4`, + contentType: null + }) } } - for (const directory of [ 'test1', 'test2' ]) { - const files = await readdir(join(root(), directory, 'videos')) + for (const directory of [ 'test1/redundancy', 'test2/videos' ]) { + const files = await readdir(join(root(), directory)) expect(files).to.have.length.at.least(4) for (const resolution of [ 240, 360, 480, 720 ]) { -- cgit v1.2.3 From 092092969633bbcf6d4891a083ea497a7d5c3154 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Tue, 29 Jan 2019 08:37:25 +0100 Subject: Add hls support on server --- server/tests/api/redundancy/redundancy.ts | 212 +++++++++++++++++++++--------- 1 file changed, 147 insertions(+), 65 deletions(-) (limited to 'server/tests/api/redundancy') diff --git a/server/tests/api/redundancy/redundancy.ts b/server/tests/api/redundancy/redundancy.ts index 9d3ce8153..5b99309fb 100644 --- a/server/tests/api/redundancy/redundancy.ts +++ b/server/tests/api/redundancy/redundancy.ts @@ -17,7 +17,7 @@ import { viewVideo, wait, waitUntilLog, - checkVideoFilesWereRemoved, removeVideo, getVideoWithToken + checkVideoFilesWereRemoved, removeVideo, getVideoWithToken, reRunServer } from '../../../../shared/utils' import { waitJobs } from '../../../../shared/utils/server/jobs' @@ -48,6 +48,11 @@ function checkMagnetWebseeds (file: { magnetUri: string, resolution: { id: numbe async function runServers (strategy: VideoRedundancyStrategy, additionalParams: any = {}) { const config = { + transcoding: { + hls: { + enabled: true + } + }, redundancy: { videos: { check_interval: '5 seconds', @@ -85,7 +90,7 @@ async function runServers (strategy: VideoRedundancyStrategy, additionalParams: await waitJobs(servers) } -async function check1WebSeed (strategy: VideoRedundancyStrategy, videoUUID?: string) { +async function check1WebSeed (videoUUID?: string) { if (!videoUUID) videoUUID = video1Server2UUID const webseeds = [ @@ -93,47 +98,17 @@ async function check1WebSeed (strategy: VideoRedundancyStrategy, videoUUID?: str ] for (const server of servers) { - { - // With token to avoid issues with video follow constraints - const res = await getVideoWithToken(server.url, server.accessToken, videoUUID) + // With token to avoid issues with video follow constraints + const res = await getVideoWithToken(server.url, server.accessToken, videoUUID) - const video: VideoDetails = res.body - for (const f of video.files) { - checkMagnetWebseeds(f, webseeds, server) - } + const video: VideoDetails = res.body + for (const f of video.files) { + checkMagnetWebseeds(f, webseeds, server) } } } -async function checkStatsWith2Webseed (strategy: VideoRedundancyStrategy) { - const res = await getStats(servers[0].url) - const data: ServerStats = res.body - - expect(data.videosRedundancy).to.have.lengthOf(1) - const stat = data.videosRedundancy[0] - - expect(stat.strategy).to.equal(strategy) - expect(stat.totalSize).to.equal(204800) - expect(stat.totalUsed).to.be.at.least(1).and.below(204801) - expect(stat.totalVideoFiles).to.equal(4) - expect(stat.totalVideos).to.equal(1) -} - -async function checkStatsWith1Webseed (strategy: VideoRedundancyStrategy) { - const res = await getStats(servers[0].url) - const data: ServerStats = res.body - - expect(data.videosRedundancy).to.have.lengthOf(1) - - const stat = data.videosRedundancy[0] - expect(stat.strategy).to.equal(strategy) - expect(stat.totalSize).to.equal(204800) - expect(stat.totalUsed).to.equal(0) - expect(stat.totalVideoFiles).to.equal(0) - expect(stat.totalVideos).to.equal(0) -} - -async function check2Webseeds (strategy: VideoRedundancyStrategy, videoUUID?: string) { +async function check2Webseeds (videoUUID?: string) { if (!videoUUID) videoUUID = video1Server2UUID const webseeds = [ @@ -158,7 +133,7 @@ async function check2Webseeds (strategy: VideoRedundancyStrategy, videoUUID?: st await makeGetRequest({ url: servers[1].url, statusCodeExpected: 200, - path: '/static/webseed/' + `${videoUUID}-${file.resolution.id}.mp4`, + path: `/static/webseed/${videoUUID}-${file.resolution.id}.mp4`, contentType: null }) } @@ -174,6 +149,81 @@ async function check2Webseeds (strategy: VideoRedundancyStrategy, videoUUID?: st } } +async function check0PlaylistRedundancies (videoUUID?: string) { + if (!videoUUID) videoUUID = video1Server2UUID + + for (const server of servers) { + // With token to avoid issues with video follow constraints + const res = await getVideoWithToken(server.url, server.accessToken, videoUUID) + const video: VideoDetails = res.body + + expect(video.streamingPlaylists).to.be.an('array') + expect(video.streamingPlaylists).to.have.lengthOf(1) + expect(video.streamingPlaylists[0].redundancies).to.have.lengthOf(0) + } +} + +async function check1PlaylistRedundancies (videoUUID?: string) { + if (!videoUUID) videoUUID = video1Server2UUID + + for (const server of servers) { + const res = await getVideo(server.url, videoUUID) + const video: VideoDetails = res.body + + expect(video.streamingPlaylists).to.have.lengthOf(1) + expect(video.streamingPlaylists[0].redundancies).to.have.lengthOf(1) + + const redundancy = video.streamingPlaylists[0].redundancies[0] + + expect(redundancy.baseUrl).to.equal(servers[0].url + '/static/redundancy/hls/' + videoUUID) + } + + await makeGetRequest({ + url: servers[0].url, + statusCodeExpected: 200, + path: `/static/redundancy/hls/${videoUUID}/360_000.ts`, + contentType: null + }) + + for (const directory of [ 'test1/redundancy/hls', 'test2/playlists/hls' ]) { + const files = await readdir(join(root(), directory, videoUUID)) + expect(files).to.have.length.at.least(4) + + for (const resolution of [ 240, 360, 480, 720 ]) { + expect(files.find(f => f === `${resolution}_000.ts`)).to.not.be.undefined + expect(files.find(f => f === `${resolution}_001.ts`)).to.not.be.undefined + } + } +} + +async function checkStatsWith2Webseed (strategy: VideoRedundancyStrategy) { + const res = await getStats(servers[0].url) + const data: ServerStats = res.body + + expect(data.videosRedundancy).to.have.lengthOf(1) + const stat = data.videosRedundancy[0] + + expect(stat.strategy).to.equal(strategy) + expect(stat.totalSize).to.equal(204800) + expect(stat.totalUsed).to.be.at.least(1).and.below(204801) + expect(stat.totalVideoFiles).to.equal(4) + expect(stat.totalVideos).to.equal(1) +} + +async function checkStatsWith1Webseed (strategy: VideoRedundancyStrategy) { + const res = await getStats(servers[0].url) + const data: ServerStats = res.body + + expect(data.videosRedundancy).to.have.lengthOf(1) + + const stat = data.videosRedundancy[0] + expect(stat.strategy).to.equal(strategy) + expect(stat.totalSize).to.equal(204800) + expect(stat.totalUsed).to.equal(0) + expect(stat.totalVideoFiles).to.equal(0) + expect(stat.totalVideos).to.equal(0) +} + async function enableRedundancyOnServer1 () { await updateRedundancy(servers[ 0 ].url, servers[ 0 ].accessToken, servers[ 1 ].host, true) @@ -220,7 +270,8 @@ describe('Test videos redundancy', function () { }) it('Should have 1 webseed on the first video', async function () { - await check1WebSeed(strategy) + await check1WebSeed() + await check0PlaylistRedundancies() await checkStatsWith1Webseed(strategy) }) @@ -229,27 +280,29 @@ describe('Test videos redundancy', function () { }) it('Should have 2 webseeds on the first video', async function () { - this.timeout(40000) + this.timeout(80000) await waitJobs(servers) - await waitUntilLog(servers[0], 'Duplicated ', 4) + await waitUntilLog(servers[0], 'Duplicated ', 5) await waitJobs(servers) - await check2Webseeds(strategy) + await check2Webseeds() + await check1PlaylistRedundancies() await checkStatsWith2Webseed(strategy) }) it('Should undo redundancy on server 1 and remove duplicated videos', async function () { - this.timeout(40000) + this.timeout(80000) await disableRedundancyOnServer1() await waitJobs(servers) await wait(5000) - await check1WebSeed(strategy) + await check1WebSeed() + await check0PlaylistRedundancies() - await checkVideoFilesWereRemoved(video1Server2UUID, servers[0].serverNumber, [ 'videos' ]) + await checkVideoFilesWereRemoved(video1Server2UUID, servers[0].serverNumber, [ 'videos', join('playlists', 'hls') ]) }) after(function () { @@ -267,7 +320,8 @@ describe('Test videos redundancy', function () { }) it('Should have 1 webseed on the first video', async function () { - await check1WebSeed(strategy) + await check1WebSeed() + await check0PlaylistRedundancies() await checkStatsWith1Webseed(strategy) }) @@ -276,25 +330,27 @@ describe('Test videos redundancy', function () { }) it('Should have 2 webseeds on the first video', async function () { - this.timeout(40000) + this.timeout(80000) await waitJobs(servers) - await waitUntilLog(servers[0], 'Duplicated ', 4) + await waitUntilLog(servers[0], 'Duplicated ', 5) await waitJobs(servers) - await check2Webseeds(strategy) + await check2Webseeds() + await check1PlaylistRedundancies() await checkStatsWith2Webseed(strategy) }) it('Should unfollow on server 1 and remove duplicated videos', async function () { - this.timeout(40000) + this.timeout(80000) await unfollow(servers[0].url, servers[0].accessToken, servers[1]) await waitJobs(servers) await wait(5000) - await check1WebSeed(strategy) + await check1WebSeed() + await check0PlaylistRedundancies() await checkVideoFilesWereRemoved(video1Server2UUID, servers[0].serverNumber, [ 'videos' ]) }) @@ -314,7 +370,8 @@ describe('Test videos redundancy', function () { }) it('Should have 1 webseed on the first video', async function () { - await check1WebSeed(strategy) + await check1WebSeed() + await check0PlaylistRedundancies() await checkStatsWith1Webseed(strategy) }) @@ -323,18 +380,19 @@ describe('Test videos redundancy', function () { }) it('Should still have 1 webseed on the first video', async function () { - this.timeout(40000) + this.timeout(80000) await waitJobs(servers) await wait(15000) await waitJobs(servers) - await check1WebSeed(strategy) + await check1WebSeed() + await check0PlaylistRedundancies() await checkStatsWith1Webseed(strategy) }) it('Should view 2 times the first video to have > min_views config', async function () { - this.timeout(40000) + this.timeout(80000) await viewVideo(servers[ 0 ].url, video1Server2UUID) await viewVideo(servers[ 2 ].url, video1Server2UUID) @@ -344,13 +402,14 @@ describe('Test videos redundancy', function () { }) it('Should have 2 webseeds on the first video', async function () { - this.timeout(40000) + this.timeout(80000) await waitJobs(servers) - await waitUntilLog(servers[0], 'Duplicated ', 4) + await waitUntilLog(servers[0], 'Duplicated ', 5) await waitJobs(servers) - await check2Webseeds(strategy) + await check2Webseeds() + await check1PlaylistRedundancies() await checkStatsWith2Webseed(strategy) }) @@ -405,7 +464,7 @@ describe('Test videos redundancy', function () { }) it('Should still have 2 webseeds after 10 seconds', async function () { - this.timeout(40000) + this.timeout(80000) await wait(10000) @@ -420,7 +479,7 @@ describe('Test videos redundancy', function () { }) it('Should stop server 1 and expire video redundancy', async function () { - this.timeout(40000) + this.timeout(80000) killallServers([ servers[0] ]) @@ -446,10 +505,11 @@ describe('Test videos redundancy', function () { await enableRedundancyOnServer1() await waitJobs(servers) - await waitUntilLog(servers[0], 'Duplicated ', 4) + await waitUntilLog(servers[0], 'Duplicated ', 5) await waitJobs(servers) - await check2Webseeds(strategy) + await check2Webseeds() + await check1PlaylistRedundancies() await checkStatsWith2Webseed(strategy) const res = await uploadVideo(servers[ 1 ].url, servers[ 1 ].accessToken, { name: 'video 2 server 2' }) @@ -467,8 +527,10 @@ describe('Test videos redundancy', function () { await wait(1000) try { - await check1WebSeed(strategy, video1Server2UUID) - await check2Webseeds(strategy, video2Server2UUID) + await check1WebSeed(video1Server2UUID) + await check0PlaylistRedundancies(video1Server2UUID) + await check2Webseeds(video2Server2UUID) + await check1PlaylistRedundancies(video2Server2UUID) checked = true } catch { @@ -477,6 +539,26 @@ describe('Test videos redundancy', function () { } }) + it('Should disable strategy and remove redundancies', async function () { + this.timeout(80000) + + await waitJobs(servers) + + killallServers([ servers[ 0 ] ]) + await reRunServer(servers[ 0 ], { + redundancy: { + videos: { + check_interval: '1 second', + strategies: [] + } + } + }) + + await waitJobs(servers) + + await checkVideoFilesWereRemoved(video1Server2UUID, servers[0].serverNumber, [ join('redundancy', 'hls') ]) + }) + after(function () { return cleanServers() }) -- cgit v1.2.3 From 4c280004ce62bf11ddb091854c28f1e1d54a54d6 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Thu, 7 Feb 2019 15:08:19 +0100 Subject: Use a single file instead of segments for HLS --- server/tests/api/redundancy/redundancy.ts | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) (limited to 'server/tests/api/redundancy') diff --git a/server/tests/api/redundancy/redundancy.ts b/server/tests/api/redundancy/redundancy.ts index 5b99309fb..778611fff 100644 --- a/server/tests/api/redundancy/redundancy.ts +++ b/server/tests/api/redundancy/redundancy.ts @@ -17,7 +17,7 @@ import { viewVideo, wait, waitUntilLog, - checkVideoFilesWereRemoved, removeVideo, getVideoWithToken, reRunServer + checkVideoFilesWereRemoved, removeVideo, getVideoWithToken, reRunServer, checkSegmentHash } from '../../../../shared/utils' import { waitJobs } from '../../../../shared/utils/server/jobs' @@ -178,20 +178,24 @@ async function check1PlaylistRedundancies (videoUUID?: string) { expect(redundancy.baseUrl).to.equal(servers[0].url + '/static/redundancy/hls/' + videoUUID) } - await makeGetRequest({ - url: servers[0].url, - statusCodeExpected: 200, - path: `/static/redundancy/hls/${videoUUID}/360_000.ts`, - contentType: null - }) + const baseUrlPlaylist = servers[1].url + '/static/playlists/hls' + const baseUrlSegment = servers[0].url + '/static/redundancy/hls' + + const res = await getVideo(servers[0].url, videoUUID) + const hlsPlaylist = (res.body as VideoDetails).streamingPlaylists[0] + + for (const resolution of [ 240, 360, 480, 720 ]) { + await checkSegmentHash(baseUrlPlaylist, baseUrlSegment, videoUUID, resolution, hlsPlaylist) + } for (const directory of [ 'test1/redundancy/hls', 'test2/playlists/hls' ]) { const files = await readdir(join(root(), directory, videoUUID)) expect(files).to.have.length.at.least(4) for (const resolution of [ 240, 360, 480, 720 ]) { - expect(files.find(f => f === `${resolution}_000.ts`)).to.not.be.undefined - expect(files.find(f => f === `${resolution}_001.ts`)).to.not.be.undefined + const filename = `${videoUUID}-${resolution}-fragmented.mp4` + + expect(files.find(f => f === filename)).to.not.be.undefined } } } -- cgit v1.2.3