From dab047092b51b453f175069573d8865fb17acdfc Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Wed, 7 Jul 2021 10:56:45 +0200 Subject: Introduce redundancy command --- server/tests/api/redundancy/manage-redundancy.ts | 165 +++++++-------------- .../tests/api/redundancy/redundancy-constraints.ts | 28 ++-- server/tests/api/redundancy/redundancy.ts | 49 +++--- shared/extra-utils/server/index.ts | 1 + shared/extra-utils/server/redundancy-command.ts | 77 ++++++++++ shared/extra-utils/server/redundancy.ts | 88 ----------- shared/extra-utils/server/servers.ts | 3 + 7 files changed, 165 insertions(+), 246 deletions(-) create mode 100644 shared/extra-utils/server/redundancy-command.ts delete mode 100644 shared/extra-utils/server/redundancy.ts diff --git a/server/tests/api/redundancy/manage-redundancy.ts b/server/tests/api/redundancy/manage-redundancy.ts index 4253124c8..363e4cbfe 100644 --- a/server/tests/api/redundancy/manage-redundancy.ts +++ b/server/tests/api/redundancy/manage-redundancy.ts @@ -1,21 +1,21 @@ /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ -import * as chai from 'chai' import 'mocha' +import * as chai from 'chai' import { cleanupTests, doubleFollow, flushAndRunMultipleServers, getLocalIdByUUID, + RedundancyCommand, ServerInfo, setAccessTokensToServers, uploadVideo, uploadVideoAndGetId, + waitJobs, waitUntilLog -} from '../../../../shared/extra-utils' -import { waitJobs } from '../../../../shared/extra-utils/server/jobs' -import { addVideoRedundancy, listVideoRedundancies, removeVideoRedundancy, updateRedundancy } from '@shared/extra-utils/server/redundancy' -import { VideoPrivacy, VideoRedundanciesTarget, VideoRedundancy } from '@shared/models' +} from '@shared/extra-utils' +import { VideoPrivacy, VideoRedundanciesTarget } from '@shared/models' const expect = chai.expect @@ -27,6 +27,8 @@ describe('Test manage videos redundancy', function () { let video2Server2UUID: string let redundanciesToRemove: number[] = [] + let commands: RedundancyCommand[] + before(async function () { this.timeout(120000) @@ -55,6 +57,8 @@ describe('Test manage videos redundancy', function () { // Get the access tokens await setAccessTokensToServers(servers) + commands = servers.map(s => s.redundancyCommand) + { const res = await uploadVideo(servers[1].url, servers[1].accessToken, { name: 'video 1 server 2' }) video1Server2UUID = res.body.video.uuid @@ -69,21 +73,17 @@ describe('Test manage videos redundancy', function () { // Server 1 and server 2 follow each other await doubleFollow(servers[0], servers[1]) - await updateRedundancy(servers[0].url, servers[0].accessToken, servers[1].host, true) + await commands[0].updateRedundancy({ host: servers[1].host, redundancyAllowed: true }) await waitJobs(servers) }) it('Should not have redundancies on server 3', async function () { for (const target of targets) { - const res = await listVideoRedundancies({ - url: servers[2].url, - accessToken: servers[2].accessToken, - target - }) + const body = await commands[2].listVideos({ target }) - expect(res.body.total).to.equal(0) - expect(res.body.data).to.have.lengthOf(0) + expect(body.total).to.equal(0) + expect(body.data).to.have.lengthOf(0) } }) @@ -94,28 +94,19 @@ describe('Test manage videos redundancy', function () { await waitUntilLog(servers[0], 'Duplicated ', 10) await waitJobs(servers) - const res = await listVideoRedundancies({ - url: servers[1].url, - accessToken: servers[1].accessToken, - target: 'remote-videos' - }) + const body = await commands[1].listVideos({ target: 'remote-videos' }) - expect(res.body.total).to.equal(0) - expect(res.body.data).to.have.lengthOf(0) + expect(body.total).to.equal(0) + expect(body.data).to.have.lengthOf(0) }) it('Should have "my-videos" redundancies on server 2', async function () { this.timeout(120000) - const res = await listVideoRedundancies({ - url: servers[1].url, - accessToken: servers[1].accessToken, - target: 'my-videos' - }) - - expect(res.body.total).to.equal(2) + const body = await commands[1].listVideos({ target: 'my-videos' }) + expect(body.total).to.equal(2) - const videos = res.body.data as VideoRedundancy[] + const videos = body.data expect(videos).to.have.lengthOf(2) const videos1 = videos.find(v => v.uuid === video1Server2UUID) @@ -139,28 +130,19 @@ describe('Test manage videos redundancy', function () { }) it('Should not have "my-videos" redundancies on server 1', async function () { - const res = await listVideoRedundancies({ - url: servers[0].url, - accessToken: servers[0].accessToken, - target: 'my-videos' - }) + const body = await commands[0].listVideos({ target: 'my-videos' }) - expect(res.body.total).to.equal(0) - expect(res.body.data).to.have.lengthOf(0) + expect(body.total).to.equal(0) + expect(body.data).to.have.lengthOf(0) }) it('Should have "remote-videos" redundancies on server 1', async function () { this.timeout(120000) - const res = await listVideoRedundancies({ - url: servers[0].url, - accessToken: servers[0].accessToken, - target: 'remote-videos' - }) + const body = await commands[0].listVideos({ target: 'remote-videos' }) + expect(body.total).to.equal(2) - expect(res.body.total).to.equal(2) - - const videos = res.body.data as VideoRedundancy[] + const videos = body.data expect(videos).to.have.lengthOf(2) const videos1 = videos.find(v => v.uuid === video1Server2UUID) @@ -185,47 +167,40 @@ describe('Test manage videos redundancy', function () { it('Should correctly paginate and sort results', async function () { { - const res = await listVideoRedundancies({ - url: servers[0].url, - accessToken: servers[0].accessToken, + const body = await commands[0].listVideos({ target: 'remote-videos', sort: 'name', start: 0, count: 2 }) - const videos = res.body.data + const videos = body.data expect(videos[0].name).to.equal('video 1 server 2') expect(videos[1].name).to.equal('video 2 server 2') } { - const res = await listVideoRedundancies({ - url: servers[0].url, - accessToken: servers[0].accessToken, + const body = await commands[0].listVideos({ target: 'remote-videos', sort: '-name', start: 0, count: 2 }) - const videos = res.body.data + const videos = body.data expect(videos[0].name).to.equal('video 2 server 2') expect(videos[1].name).to.equal('video 1 server 2') } { - const res = await listVideoRedundancies({ - url: servers[0].url, - accessToken: servers[0].accessToken, + const body = await commands[0].listVideos({ target: 'remote-videos', sort: '-name', start: 1, count: 1 }) - const videos = res.body.data - expect(videos[0].name).to.equal('video 1 server 2') + expect(body.data[0].name).to.equal('video 1 server 2') } }) @@ -236,30 +211,23 @@ describe('Test manage videos redundancy', function () { await waitJobs(servers) const videoId = await getLocalIdByUUID(servers[0].url, uuid) - await addVideoRedundancy({ - url: servers[0].url, - accessToken: servers[0].accessToken, - videoId - }) + await commands[0].addVideo({ videoId }) await waitJobs(servers) await waitUntilLog(servers[0], 'Duplicated ', 15) await waitJobs(servers) { - const res = await listVideoRedundancies({ - url: servers[0].url, - accessToken: servers[0].accessToken, + const body = await commands[0].listVideos({ target: 'remote-videos', sort: '-name', start: 0, count: 5 }) - const videos = res.body.data - expect(videos[0].name).to.equal('video 3 server 2') + const video = body.data[0] - const video = videos[0] + expect(video.name).to.equal('video 3 server 2') expect(video.redundancies.files).to.have.lengthOf(4) expect(video.redundancies.streamingPlaylists).to.have.lengthOf(1) @@ -276,19 +244,15 @@ describe('Test manage videos redundancy', function () { } } - const res = await listVideoRedundancies({ - url: servers[1].url, - accessToken: servers[1].accessToken, + const body = await commands[1].listVideos({ target: 'my-videos', sort: '-name', start: 0, count: 5 }) - const videos = res.body.data - expect(videos[0].name).to.equal('video 3 server 2') - - const video = videos[0] + const video = body.data[0] + expect(video.name).to.equal('video 3 server 2') expect(video.redundancies.files).to.have.lengthOf(4) expect(video.redundancies.streamingPlaylists).to.have.lengthOf(1) @@ -307,64 +271,47 @@ describe('Test manage videos redundancy', function () { this.timeout(120000) for (const redundancyId of redundanciesToRemove) { - await removeVideoRedundancy({ - url: servers[0].url, - accessToken: servers[0].accessToken, - redundancyId - }) + await commands[0].removeVideo({ redundancyId }) } { - const res = await listVideoRedundancies({ - url: servers[0].url, - accessToken: servers[0].accessToken, + const body = await commands[0].listVideos({ target: 'remote-videos', sort: '-name', start: 0, count: 5 }) - const videos = res.body.data - expect(videos).to.have.lengthOf(2) + const videos = body.data - expect(videos[0].name).to.equal('video 2 server 2') + expect(videos).to.have.lengthOf(2) - redundanciesToRemove = [] const video = videos[0] + expect(video.name).to.equal('video 2 server 2') expect(video.redundancies.files).to.have.lengthOf(4) expect(video.redundancies.streamingPlaylists).to.have.lengthOf(1) const redundancies = video.redundancies.files.concat(video.redundancies.streamingPlaylists) - for (const r of redundancies) { - redundanciesToRemove.push(r.id) - } + redundanciesToRemove = redundancies.map(r => r.id) } }) it('Should remove another (auto) redundancy', async function () { - { - for (const redundancyId of redundanciesToRemove) { - await removeVideoRedundancy({ - url: servers[0].url, - accessToken: servers[0].accessToken, - redundancyId - }) - } + for (const redundancyId of redundanciesToRemove) { + await commands[0].removeVideo({ redundancyId }) + } - const res = await listVideoRedundancies({ - url: servers[0].url, - accessToken: servers[0].accessToken, - target: 'remote-videos', - sort: '-name', - start: 0, - count: 5 - }) + const body = await commands[0].listVideos({ + target: 'remote-videos', + sort: '-name', + start: 0, + count: 5 + }) - const videos = res.body.data - expect(videos[0].name).to.equal('video 1 server 2') - expect(videos).to.have.lengthOf(1) - } + const videos = body.data + expect(videos).to.have.lengthOf(1) + expect(videos[0].name).to.equal('video 1 server 2') }) after(async function () { diff --git a/server/tests/api/redundancy/redundancy-constraints.ts b/server/tests/api/redundancy/redundancy-constraints.ts index 602f4bc1b..a666976b3 100644 --- a/server/tests/api/redundancy/redundancy-constraints.ts +++ b/server/tests/api/redundancy/redundancy-constraints.ts @@ -1,9 +1,6 @@ /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ import 'mocha' -import * as chai from 'chai' -import { listVideoRedundancies, updateRedundancy } from '@shared/extra-utils/server/redundancy' -import { VideoPrivacy } from '@shared/models' import { cleanupTests, flushAndRunServer, @@ -13,9 +10,10 @@ import { setAccessTokensToServers, updateVideo, uploadVideo, + waitJobs, waitUntilLog -} from '../../../../shared/extra-utils' -import { waitJobs } from '../../../../shared/extra-utils/server/jobs' +} from '@shared/extra-utils' +import { VideoPrivacy } from '@shared/models' const expect = chai.expect @@ -50,23 +48,15 @@ describe('Test redundancy constraints', function () { } async function getTotalRedundanciesLocalServer () { - const res = await listVideoRedundancies({ - url: localServer.url, - accessToken: localServer.accessToken, - target: 'my-videos' - }) + const body = await localServer.redundancyCommand.listVideos({ target: 'my-videos' }) - return res.body.total + return body.total } async function getTotalRedundanciesRemoteServer () { - const res = await listVideoRedundancies({ - url: remoteServer.url, - accessToken: remoteServer.accessToken, - target: 'remote-videos' - }) + const body = await remoteServer.redundancyCommand.listVideos({ target: 'remote-videos' }) - return res.body.total + return body.total } before(async function () { @@ -99,7 +89,7 @@ describe('Test redundancy constraints', function () { // Server 1 and server 2 follow each other await remoteServer.followsCommand.follow({ targets: [ localServer.url ] }) await waitJobs(servers) - await updateRedundancy(remoteServer.url, remoteServer.accessToken, localServer.host, true) + await remoteServer.redundancyCommand.updateRedundancy({ host: localServer.host, redundancyAllowed: true }) await waitJobs(servers) }) @@ -161,7 +151,7 @@ describe('Test redundancy constraints', function () { } } } - await killallServers([ localServer ]) + killallServers([ localServer ]) await reRunServer(localServer, config) await uploadWrapper('video 3 server 2') diff --git a/server/tests/api/redundancy/redundancy.ts b/server/tests/api/redundancy/redundancy.ts index dfe8099ed..5b970473c 100644 --- a/server/tests/api/redundancy/redundancy.ts +++ b/server/tests/api/redundancy/redundancy.ts @@ -5,7 +5,8 @@ import * as chai from 'chai' import { readdir } from 'fs-extra' import * as magnetUtil from 'magnet-uri' import { join } from 'path' -import { HttpStatusCode } from '../../../../shared/core-utils/miscs/http-error-codes' +import { removeVideoRedundancy } from '@server/lib/redundancy' +import { HttpStatusCode } from '@shared/core-utils' import { checkSegmentHash, checkVideoFilesWereRemoved, @@ -26,19 +27,18 @@ import { uploadVideo, viewVideo, wait, + waitJobs, waitUntilLog -} from '../../../../shared/extra-utils' -import { waitJobs } from '../../../../shared/extra-utils/server/jobs' +} from '@shared/extra-utils' +import { getStats } from '@shared/extra-utils/server/stats' import { - addVideoRedundancy, - listVideoRedundancies, - removeVideoRedundancy, - updateRedundancy -} from '../../../../shared/extra-utils/server/redundancy' -import { getStats } from '../../../../shared/extra-utils/server/stats' -import { VideoRedundancy, VideoRedundancyStrategy, VideoRedundancyStrategyWithManual } from '../../../../shared/models/redundancy' -import { ServerStats } from '../../../../shared/models/server/server-stats.model' -import { VideoDetails, VideoPrivacy } from '../../../../shared/models/videos' + ServerStats, + VideoDetails, + VideoPrivacy, + VideoRedundancy, + VideoRedundancyStrategy, + VideoRedundancyStrategyWithManual +} from '@shared/models' const expect = chai.expect @@ -279,7 +279,7 @@ async function findServerFollows () { } async function enableRedundancyOnServer1 () { - await updateRedundancy(servers[0].url, servers[0].accessToken, servers[1].host, true) + await servers[0].redundancyCommand.updateRedundancy({ host: servers[1].host, redundancyAllowed: true }) const { server2, server3 } = await findServerFollows() @@ -291,7 +291,7 @@ async function enableRedundancyOnServer1 () { } async function disableRedundancyOnServer1 () { - await updateRedundancy(servers[0].url, servers[0].accessToken, servers[1].host, false) + await servers[0].redundancyCommand.updateRedundancy({ host: servers[1].host, redundancyAllowed: false }) const { server2, server3 } = await findServerFollows() @@ -551,11 +551,7 @@ describe('Test videos redundancy', function () { }) it('Should create a redundancy on first video', async function () { - await addVideoRedundancy({ - url: servers[0].url, - accessToken: servers[0].accessToken, - videoId: video1Server2Id - }) + await servers[0].redundancyCommand.addVideo({ videoId: video1Server2Id }) }) it('Should have 2 webseeds on the first video', async function () { @@ -573,22 +569,15 @@ describe('Test videos redundancy', function () { it('Should manually remove redundancies on server 1 and remove duplicated videos', async function () { this.timeout(80000) - const res = await listVideoRedundancies({ - url: servers[0].url, - accessToken: servers[0].accessToken, - target: 'remote-videos' - }) + const body = await servers[0].redundancyCommand.listVideos({ target: 'remote-videos' }) - const videos = res.body.data as VideoRedundancy[] + const videos = body.data expect(videos).to.have.lengthOf(1) const video = videos[0] + for (const r of video.redundancies.files.concat(video.redundancies.streamingPlaylists)) { - await removeVideoRedundancy({ - url: servers[0].url, - accessToken: servers[0].accessToken, - redundancyId: r.id - }) + await servers[0].redundancyCommand.removeVideo({ redundancyId: r.id }) } await waitJobs(servers) diff --git a/shared/extra-utils/server/index.ts b/shared/extra-utils/server/index.ts index e602fec7e..3ee70f0cf 100644 --- a/shared/extra-utils/server/index.ts +++ b/shared/extra-utils/server/index.ts @@ -6,3 +6,4 @@ export * from './jobs' export * from './jobs-command' export * from './plugins-command' export * from './plugins' +export * from './redundancy-command' diff --git a/shared/extra-utils/server/redundancy-command.ts b/shared/extra-utils/server/redundancy-command.ts new file mode 100644 index 000000000..d717d35f8 --- /dev/null +++ b/shared/extra-utils/server/redundancy-command.ts @@ -0,0 +1,77 @@ +import { ResultList, VideoRedundanciesTarget, VideoRedundancy } from '@shared/models' +import { HttpStatusCode } from '../../core-utils/miscs/http-error-codes' +import { AbstractCommand, OverrideCommandOptions } from '../shared' + +export class RedundancyCommand extends AbstractCommand { + + updateRedundancy (options: OverrideCommandOptions & { + host: string + redundancyAllowed: boolean + }) { + const { host, redundancyAllowed } = options + const path = '/api/v1/server/redundancy/' + host + + return this.putBodyRequest({ + ...options, + + path, + fields: { redundancyAllowed }, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } + + listVideos (options: OverrideCommandOptions & { + target: VideoRedundanciesTarget + start?: number + count?: number + sort?: string + }) { + const path = '/api/v1/server/redundancy/videos' + + const { target, start, count, sort } = options + + return this.getRequestBody>({ + ...options, + + path, + + query: { + start: start ?? 0, + count: count ?? 5, + sort: sort ?? 'name', + target + }, + + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } + + addVideo (options: OverrideCommandOptions & { + videoId: number + }) { + const path = '/api/v1/server/redundancy/videos' + const { videoId } = options + + return this.postBodyRequest({ + ...options, + + path, + fields: { videoId }, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } + + removeVideo (options: OverrideCommandOptions & { + redundancyId: number + }) { + const { redundancyId } = options + const path = '/api/v1/server/redundancy/videos/' + redundancyId + + return this.deleteRequest({ + ...options, + + path, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } +} diff --git a/shared/extra-utils/server/redundancy.ts b/shared/extra-utils/server/redundancy.ts deleted file mode 100644 index b83815a37..000000000 --- a/shared/extra-utils/server/redundancy.ts +++ /dev/null @@ -1,88 +0,0 @@ -import { makeDeleteRequest, makeGetRequest, makePostBodyRequest, makePutBodyRequest } from '../requests/requests' -import { VideoRedundanciesTarget } from '@shared/models' -import { HttpStatusCode } from '../../../shared/core-utils/miscs/http-error-codes' - -function updateRedundancy ( - url: string, - accessToken: string, - host: string, - redundancyAllowed: boolean, - expectedStatus = HttpStatusCode.NO_CONTENT_204 -) { - const path = '/api/v1/server/redundancy/' + host - - return makePutBodyRequest({ - url, - path, - token: accessToken, - fields: { redundancyAllowed }, - statusCodeExpected: expectedStatus - }) -} - -function listVideoRedundancies (options: { - url: string - accessToken: string - target: VideoRedundanciesTarget - start?: number - count?: number - sort?: string - statusCodeExpected?: HttpStatusCode -}) { - const path = '/api/v1/server/redundancy/videos' - - const { url, accessToken, target, statusCodeExpected, start, count, sort } = options - - return makeGetRequest({ - url, - token: accessToken, - path, - query: { - start: start ?? 0, - count: count ?? 5, - sort: sort ?? 'name', - target - }, - statusCodeExpected: statusCodeExpected || HttpStatusCode.OK_200 - }) -} - -function addVideoRedundancy (options: { - url: string - accessToken: string - videoId: number -}) { - const path = '/api/v1/server/redundancy/videos' - const { url, accessToken, videoId } = options - - return makePostBodyRequest({ - url, - token: accessToken, - path, - fields: { videoId }, - statusCodeExpected: HttpStatusCode.NO_CONTENT_204 - }) -} - -function removeVideoRedundancy (options: { - url: string - accessToken: string - redundancyId: number -}) { - const { url, accessToken, redundancyId } = options - const path = '/api/v1/server/redundancy/videos/' + redundancyId - - return makeDeleteRequest({ - url, - token: accessToken, - path, - statusCodeExpected: HttpStatusCode.NO_CONTENT_204 - }) -} - -export { - updateRedundancy, - listVideoRedundancies, - addVideoRedundancy, - removeVideoRedundancy -} diff --git a/shared/extra-utils/server/servers.ts b/shared/extra-utils/server/servers.ts index 79d6b7b1a..eaf39ecea 100644 --- a/shared/extra-utils/server/servers.ts +++ b/shared/extra-utils/server/servers.ts @@ -21,6 +21,7 @@ import { DebugCommand } from './debug-command' import { FollowsCommand } from './follows-command' import { JobsCommand } from './jobs-command' import { PluginsCommand } from './plugins-command' +import { RedundancyCommand } from './redundancy-command' interface ServerInfo { app: ChildProcess @@ -87,6 +88,7 @@ interface ServerInfo { followsCommand?: FollowsCommand jobsCommand?: JobsCommand pluginsCommand?: PluginsCommand + redundancyCommand?: RedundancyCommand } function parallelTests () { @@ -305,6 +307,7 @@ async function runServer (server: ServerInfo, configOverrideArg?: any, args = [] server.followsCommand = new FollowsCommand(server) server.jobsCommand = new JobsCommand(server) server.pluginsCommand = new PluginsCommand(server) + server.redundancyCommand = new RedundancyCommand(server) res(server) }) -- cgit v1.2.3