From c48e82b5e0478434de30626d14594a97f2402e7c Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Tue, 11 Sep 2018 16:27:07 +0200 Subject: Basic video redundancy implementation --- server/tests/api/check-params/follows.ts | 9 -- server/tests/api/check-params/index.ts | 8 +- server/tests/api/check-params/redundancy.ts | 103 ++++++++++++++++++++ server/tests/api/server/index.ts | 1 + server/tests/api/server/redundancy.ts | 140 ++++++++++++++++++++++++++++ server/tests/utils/server/follows.ts | 1 - server/tests/utils/server/redundancy.ts | 17 ++++ 7 files changed, 266 insertions(+), 13 deletions(-) create mode 100644 server/tests/api/check-params/redundancy.ts create mode 100644 server/tests/api/server/redundancy.ts create mode 100644 server/tests/utils/server/redundancy.ts (limited to 'server/tests') diff --git a/server/tests/api/check-params/follows.ts b/server/tests/api/check-params/follows.ts index 2bc3b27d9..cdc95c81a 100644 --- a/server/tests/api/check-params/follows.ts +++ b/server/tests/api/check-params/follows.ts @@ -169,15 +169,6 @@ describe('Test server follows API validators', function () { statusCodeExpected: 404 }) }) - - it('Should succeed with the correct parameters', async function () { - await makeDeleteRequest({ - url: server.url, - path: path + '/localhost:9002', - token: server.accessToken, - statusCodeExpected: 404 - }) - }) }) }) diff --git a/server/tests/api/check-params/index.ts b/server/tests/api/check-params/index.ts index 777acbb0f..44460a167 100644 --- a/server/tests/api/check-params/index.ts +++ b/server/tests/api/check-params/index.ts @@ -1,15 +1,17 @@ // Order of the tests we want to execute import './accounts' +import './config' import './follows' import './jobs' +import './redundancy' +import './search' import './services' +import './user-subscriptions' import './users' import './video-abuses' import './video-blacklist' import './video-captions' import './video-channels' import './video-comments' -import './videos' import './video-imports' -import './search' -import './user-subscriptions' +import './videos' diff --git a/server/tests/api/check-params/redundancy.ts b/server/tests/api/check-params/redundancy.ts new file mode 100644 index 000000000..aa588e3dd --- /dev/null +++ b/server/tests/api/check-params/redundancy.ts @@ -0,0 +1,103 @@ +/* tslint:disable:no-unused-expression */ + +import 'mocha' + +import { + createUser, + doubleFollow, + flushAndRunMultipleServers, + flushTests, + killallServers, + makePutBodyRequest, + ServerInfo, + setAccessTokensToServers, + userLogin +} from '../../utils' + +describe('Test server redundancy API validators', function () { + let servers: ServerInfo[] + let userAccessToken = null + + // --------------------------------------------------------------- + + before(async function () { + this.timeout(30000) + + await flushTests() + servers = await flushAndRunMultipleServers(2) + + await setAccessTokensToServers(servers) + await doubleFollow(servers[0], servers[1]) + + const user = { + username: 'user1', + password: 'password' + } + + await createUser(servers[0].url, servers[0].accessToken, user.username, user.password) + userAccessToken = await userLogin(servers[0], user) + }) + + describe('When updating redundancy', function () { + const path = '/api/v1/server/redundancy' + + it('Should fail with an invalid token', async function () { + await makePutBodyRequest({ + url: servers[0].url, + path: path + '/localhost:9002', + fields: { redundancyAllowed: true }, + token: 'fake_token', + statusCodeExpected: 401 + }) + }) + + it('Should fail if the user is not an administrator', async function () { + await makePutBodyRequest({ + url: servers[0].url, + path: path + '/localhost:9002', + fields: { redundancyAllowed: true }, + token: userAccessToken, + statusCodeExpected: 403 + }) + }) + + it('Should fail if we do not follow this server', async function () { + await makePutBodyRequest({ + url: servers[0].url, + path: path + '/example.com', + fields: { redundancyAllowed: true }, + token: servers[0].accessToken, + statusCodeExpected: 404 + }) + }) + + it('Should fail without de redundancyAllowed param', async function () { + await makePutBodyRequest({ + url: servers[0].url, + path: path + '/localhost:9002', + fields: { blabla: true }, + token: servers[0].accessToken, + statusCodeExpected: 400 + }) + }) + + it('Should succeed with the correct parameters', async function () { + await makePutBodyRequest({ + url: servers[0].url, + path: path + '/localhost:9002', + fields: { redundancyAllowed: true }, + token: servers[0].accessToken, + statusCodeExpected: 204 + }) + }) + }) + + after(async function () { + killallServers(servers) + + // Keep the logs if the test failed + if (this['ok']) { + await flushTests() + } + }) +}) diff --git a/server/tests/api/server/index.ts b/server/tests/api/server/index.ts index eeb8b7a28..c74c68a33 100644 --- a/server/tests/api/server/index.ts +++ b/server/tests/api/server/index.ts @@ -3,6 +3,7 @@ import './email' import './follows' import './handle-down' import './jobs' +import './redundancy' import './reverse-proxy' import './stats' import './tracker' diff --git a/server/tests/api/server/redundancy.ts b/server/tests/api/server/redundancy.ts new file mode 100644 index 000000000..c0ec75a45 --- /dev/null +++ b/server/tests/api/server/redundancy.ts @@ -0,0 +1,140 @@ +/* tslint:disable:no-unused-expression */ + +import * as chai from 'chai' +import 'mocha' +import { VideoDetails } from '../../../../shared/models/videos' +import { + doubleFollow, + flushAndRunMultipleServers, + flushTests, + getFollowingListPaginationAndSort, + getVideo, + killallServers, + ServerInfo, + setAccessTokensToServers, + uploadVideo, + wait, + root, viewVideo +} from '../../utils' +import { waitJobs } from '../../utils/server/jobs' +import * as magnetUtil from 'magnet-uri' +import { updateRedundancy } from '../../utils/server/redundancy' +import { ActorFollow } from '../../../../shared/models/actors' +import { readdir } from 'fs-extra' +import { join } from 'path' + +const expect = chai.expect + +function checkMagnetWebseeds (file: { magnetUri: string, resolution: { id: number } }, baseWebseeds: string[]) { + const parsed = magnetUtil.decode(file.magnetUri) + + for (const ws of baseWebseeds) { + const found = parsed.urlList.find(url => url === `${ws}-${file.resolution.id}.mp4`) + expect(found, `Webseed ${ws} not found in ${file.magnetUri}`).to.not.be.undefined + } +} + +describe('Test videos redundancy', function () { + let servers: ServerInfo[] = [] + let video1Server2UUID: string + let video2Server2UUID: string + + before(async function () { + this.timeout(120000) + + servers = await flushAndRunMultipleServers(3) + + // Get the access tokens + await setAccessTokensToServers(servers) + + { + const res = await uploadVideo(servers[ 1 ].url, servers[ 1 ].accessToken, { name: 'video 1 server 2' }) + video1Server2UUID = res.body.video.uuid + + await viewVideo(servers[1].url, video1Server2UUID) + } + + { + const res = await uploadVideo(servers[ 1 ].url, servers[ 1 ].accessToken, { name: 'video 2 server 2' }) + video2Server2UUID = res.body.video.uuid + } + + await waitJobs(servers) + + // Server 1 and server 2 follow each other + await doubleFollow(servers[0], servers[1]) + // Server 1 and server 3 follow each other + await doubleFollow(servers[0], servers[2]) + // Server 2 and server 3 follow each other + await doubleFollow(servers[1], servers[2]) + + await waitJobs(servers) + }) + + it('Should have 1 webseed on the first video', async function () { + const webseeds = [ + 'http://localhost:9002/static/webseed/' + video1Server2UUID + ] + + for (const server of servers) { + const res = await getVideo(server.url, video1Server2UUID) + + const video: VideoDetails = res.body + video.files.forEach(f => checkMagnetWebseeds(f, webseeds)) + } + }) + + it('Should enable redundancy on server 1', async function () { + await updateRedundancy(servers[0].url, servers[0].accessToken, servers[1].host, true) + + const res = await getFollowingListPaginationAndSort(servers[0].url, 0, 5, '-createdAt') + const follows: ActorFollow[] = res.body.data + const server2 = follows.find(f => f.following.host === 'localhost:9002') + const server3 = follows.find(f => f.following.host === 'localhost:9003') + + expect(server3).to.not.be.undefined + expect(server3.following.hostRedundancyAllowed).to.be.false + + expect(server2).to.not.be.undefined + expect(server2.following.hostRedundancyAllowed).to.be.true + }) + + it('Should have 2 webseed on the first video', async function () { + this.timeout(40000) + + await waitJobs(servers) + await wait(15000) + await waitJobs(servers) + + const webseeds = [ + 'http://localhost:9001/static/webseed/' + video1Server2UUID, + 'http://localhost:9002/static/webseed/' + video1Server2UUID + ] + + for (const server of servers) { + const res = await getVideo(server.url, video1Server2UUID) + + const video: VideoDetails = res.body + + for (const file of video.files) { + checkMagnetWebseeds(file, webseeds) + } + } + + const files = await readdir(join(root(), 'test1', 'videos')) + expect(files).to.have.lengthOf(4) + + for (const resolution of [ 240, 360, 480, 720 ]) { + expect(files.find(f => f === `${video1Server2UUID}-${resolution}.mp4`)).to.not.be.undefined + } + }) + + after(async function () { + killallServers(servers) + + // Keep the logs if the test failed + if (this['ok']) { + await flushTests() + } + }) +}) diff --git a/server/tests/utils/server/follows.ts b/server/tests/utils/server/follows.ts index d21fb5e58..8a65a958b 100644 --- a/server/tests/utils/server/follows.ts +++ b/server/tests/utils/server/follows.ts @@ -1,5 +1,4 @@ import * as request from 'supertest' -import { wait } from '../miscs/miscs' import { ServerInfo } from './servers' import { waitJobs } from './jobs' diff --git a/server/tests/utils/server/redundancy.ts b/server/tests/utils/server/redundancy.ts new file mode 100644 index 000000000..c39ff2c8b --- /dev/null +++ b/server/tests/utils/server/redundancy.ts @@ -0,0 +1,17 @@ +import { makePutBodyRequest } from '../requests/requests' + +async function updateRedundancy (url: string, accessToken: string, host: string, redundancyAllowed: boolean, expectedStatus = 204) { + const path = '/api/v1/server/redundancy/' + host + + return makePutBodyRequest({ + url, + path, + token: accessToken, + fields: { redundancyAllowed }, + statusCodeExpected: expectedStatus + }) +} + +export { + updateRedundancy +} -- cgit v1.2.3