From 9639bd175726b73f8fe664b5ced12a72407b1f0b Mon Sep 17 00:00:00 2001 From: buoyantair Date: Mon, 29 Oct 2018 22:18:31 +0530 Subject: Move utils to /shared Move utils used by /server/tools/* & /server/tests/**/* into /shared folder. Issue: #1336 --- shared/utils/server/activitypub.ts | 15 +++ shared/utils/server/clients.ts | 19 ++++ shared/utils/server/config.ts | 135 +++++++++++++++++++++++++++ shared/utils/server/follows.ts | 79 ++++++++++++++++ shared/utils/server/jobs.ts | 77 +++++++++++++++ shared/utils/server/redundancy.ts | 17 ++++ shared/utils/server/servers.ts | 185 +++++++++++++++++++++++++++++++++++++ shared/utils/server/stats.ts | 22 +++++ 8 files changed, 549 insertions(+) create mode 100644 shared/utils/server/activitypub.ts create mode 100644 shared/utils/server/clients.ts create mode 100644 shared/utils/server/config.ts create mode 100644 shared/utils/server/follows.ts create mode 100644 shared/utils/server/jobs.ts create mode 100644 shared/utils/server/redundancy.ts create mode 100644 shared/utils/server/servers.ts create mode 100644 shared/utils/server/stats.ts (limited to 'shared/utils/server') diff --git a/shared/utils/server/activitypub.ts b/shared/utils/server/activitypub.ts new file mode 100644 index 000000000..cf3c1c3b3 --- /dev/null +++ b/shared/utils/server/activitypub.ts @@ -0,0 +1,15 @@ +import * as request from 'supertest' + +function makeActivityPubGetRequest (url: string, path: string) { + return request(url) + .get(path) + .set('Accept', 'application/activity+json,text/html;q=0.9,\\*/\\*;q=0.8') + .expect(200) + .expect('Content-Type', /json/) +} + +// --------------------------------------------------------------------------- + +export { + makeActivityPubGetRequest +} diff --git a/shared/utils/server/clients.ts b/shared/utils/server/clients.ts new file mode 100644 index 000000000..273aac747 --- /dev/null +++ b/shared/utils/server/clients.ts @@ -0,0 +1,19 @@ +import * as request from 'supertest' +import * as urlUtil from 'url' + +function getClient (url: string) { + const path = '/api/v1/oauth-clients/local' + + return request(url) + .get(path) + .set('Host', urlUtil.parse(url).host) + .set('Accept', 'application/json') + .expect(200) + .expect('Content-Type', /json/) +} + +// --------------------------------------------------------------------------- + +export { + getClient +} diff --git a/shared/utils/server/config.ts b/shared/utils/server/config.ts new file mode 100644 index 000000000..b85e02ab7 --- /dev/null +++ b/shared/utils/server/config.ts @@ -0,0 +1,135 @@ +import { makeDeleteRequest, makeGetRequest, makePutBodyRequest } from '../' +import { CustomConfig } from '../../../../shared/models/server/custom-config.model' + +function getConfig (url: string) { + const path = '/api/v1/config' + + return makeGetRequest({ + url, + path, + statusCodeExpected: 200 + }) +} + +function getAbout (url: string) { + const path = '/api/v1/config/about' + + return makeGetRequest({ + url, + path, + statusCodeExpected: 200 + }) +} + +function getCustomConfig (url: string, token: string, statusCodeExpected = 200) { + const path = '/api/v1/config/custom' + + return makeGetRequest({ + url, + token, + path, + statusCodeExpected + }) +} + +function updateCustomConfig (url: string, token: string, newCustomConfig: CustomConfig, statusCodeExpected = 200) { + const path = '/api/v1/config/custom' + + return makePutBodyRequest({ + url, + token, + path, + fields: newCustomConfig, + statusCodeExpected + }) +} + +function updateCustomSubConfig (url: string, token: string, newConfig: any) { + const updateParams: CustomConfig = { + instance: { + name: 'PeerTube updated', + shortDescription: 'my short description', + description: 'my super description', + terms: 'my super terms', + defaultClientRoute: '/videos/recently-added', + defaultNSFWPolicy: 'blur', + customizations: { + javascript: 'alert("coucou")', + css: 'body { background-color: red; }' + } + }, + services: { + twitter: { + username: '@MySuperUsername', + whitelisted: true + } + }, + cache: { + previews: { + size: 2 + }, + captions: { + size: 3 + } + }, + signup: { + enabled: false, + limit: 5, + requiresEmailVerification: false + }, + admin: { + email: 'superadmin1@example.com' + }, + user: { + videoQuota: 5242881, + videoQuotaDaily: 318742 + }, + transcoding: { + enabled: true, + threads: 1, + resolutions: { + '240p': false, + '360p': true, + '480p': true, + '720p': false, + '1080p': false + } + }, + import: { + videos: { + http: { + enabled: false + }, + torrent: { + enabled: false + } + } + } + } + + Object.assign(updateParams, newConfig) + + return updateCustomConfig(url, token, updateParams) +} + +function deleteCustomConfig (url: string, token: string, statusCodeExpected = 200) { + const path = '/api/v1/config/custom' + + return makeDeleteRequest({ + url, + token, + path, + statusCodeExpected + }) +} + +// --------------------------------------------------------------------------- + +export { + getConfig, + getCustomConfig, + updateCustomConfig, + getAbout, + deleteCustomConfig, + updateCustomSubConfig +} diff --git a/shared/utils/server/follows.ts b/shared/utils/server/follows.ts new file mode 100644 index 000000000..7741757a6 --- /dev/null +++ b/shared/utils/server/follows.ts @@ -0,0 +1,79 @@ +import * as request from 'supertest' +import { ServerInfo } from './servers' +import { waitJobs } from './jobs' + +function getFollowersListPaginationAndSort (url: string, start: number, count: number, sort: string, search?: string) { + const path = '/api/v1/server/followers' + + return request(url) + .get(path) + .query({ start }) + .query({ count }) + .query({ sort }) + .query({ search }) + .set('Accept', 'application/json') + .expect(200) + .expect('Content-Type', /json/) +} + +function getFollowingListPaginationAndSort (url: string, start: number, count: number, sort: string, search?: string) { + const path = '/api/v1/server/following' + + return request(url) + .get(path) + .query({ start }) + .query({ count }) + .query({ sort }) + .query({ search }) + .set('Accept', 'application/json') + .expect(200) + .expect('Content-Type', /json/) +} + +async function follow (follower: string, following: string[], accessToken: string, expectedStatus = 204) { + const path = '/api/v1/server/following' + + const followingHosts = following.map(f => f.replace(/^http:\/\//, '')) + const res = await request(follower) + .post(path) + .set('Accept', 'application/json') + .set('Authorization', 'Bearer ' + accessToken) + .send({ 'hosts': followingHosts }) + .expect(expectedStatus) + + return res +} + +async function unfollow (url: string, accessToken: string, target: ServerInfo, expectedStatus = 204) { + const path = '/api/v1/server/following/' + target.host + + const res = await request(url) + .delete(path) + .set('Accept', 'application/json') + .set('Authorization', 'Bearer ' + accessToken) + .expect(expectedStatus) + + return res +} + +async function doubleFollow (server1: ServerInfo, server2: ServerInfo) { + await Promise.all([ + follow(server1.url, [ server2.url ], server1.accessToken), + follow(server2.url, [ server1.url ], server2.accessToken) + ]) + + // Wait request propagation + await waitJobs([ server1, server2 ]) + + return true +} + +// --------------------------------------------------------------------------- + +export { + getFollowersListPaginationAndSort, + getFollowingListPaginationAndSort, + unfollow, + follow, + doubleFollow +} diff --git a/shared/utils/server/jobs.ts b/shared/utils/server/jobs.ts new file mode 100644 index 000000000..4c02cace5 --- /dev/null +++ b/shared/utils/server/jobs.ts @@ -0,0 +1,77 @@ +import * as request from 'supertest' +import { Job, JobState } from '../../../../shared/models' +import { ServerInfo, wait } from '../index' + +function getJobsList (url: string, accessToken: string, state: JobState) { + const path = '/api/v1/jobs/' + state + + return request(url) + .get(path) + .set('Accept', 'application/json') + .set('Authorization', 'Bearer ' + accessToken) + .expect(200) + .expect('Content-Type', /json/) +} + +function getJobsListPaginationAndSort (url: string, accessToken: string, state: JobState, start: number, count: number, sort: string) { + const path = '/api/v1/jobs/' + state + + return request(url) + .get(path) + .query({ start }) + .query({ count }) + .query({ sort }) + .set('Accept', 'application/json') + .set('Authorization', 'Bearer ' + accessToken) + .expect(200) + .expect('Content-Type', /json/) +} + +async function waitJobs (serversArg: ServerInfo[] | ServerInfo) { + let servers: ServerInfo[] + + if (Array.isArray(serversArg) === false) servers = [ serversArg as ServerInfo ] + else servers = serversArg as ServerInfo[] + + const states: JobState[] = [ 'waiting', 'active', 'delayed' ] + const tasks: Promise[] = [] + let pendingRequests: boolean + + do { + pendingRequests = false + + // Check if each server has pending request + for (const server of servers) { + for (const state of states) { + const p = getJobsListPaginationAndSort(server.url, server.accessToken, state, 0, 10, '-createdAt') + .then(res => res.body.data) + .then((jobs: Job[]) => jobs.filter(j => j.type !== 'videos-views')) + .then(jobs => { + if (jobs.length !== 0) pendingRequests = true + }) + tasks.push(p) + } + } + + await Promise.all(tasks) + + // Retry, in case of new jobs were created + if (pendingRequests === false) { + await wait(1000) + + await Promise.all(tasks) + } + + if (pendingRequests) { + await wait(1000) + } + } while (pendingRequests) +} + +// --------------------------------------------------------------------------- + +export { + getJobsList, + waitJobs, + getJobsListPaginationAndSort +} diff --git a/shared/utils/server/redundancy.ts b/shared/utils/server/redundancy.ts new file mode 100644 index 000000000..c39ff2c8b --- /dev/null +++ b/shared/utils/server/redundancy.ts @@ -0,0 +1,17 @@ +import { makePutBodyRequest } from '../requests/requests' + +async function updateRedundancy (url: string, accessToken: string, host: string, redundancyAllowed: boolean, expectedStatus = 204) { + const path = '/api/v1/server/redundancy/' + host + + return makePutBodyRequest({ + url, + path, + token: accessToken, + fields: { redundancyAllowed }, + statusCodeExpected: expectedStatus + }) +} + +export { + updateRedundancy +} diff --git a/shared/utils/server/servers.ts b/shared/utils/server/servers.ts new file mode 100644 index 000000000..3c946db27 --- /dev/null +++ b/shared/utils/server/servers.ts @@ -0,0 +1,185 @@ +import { ChildProcess, exec, fork } from 'child_process' +import { join } from 'path' +import { root, wait } from '../miscs/miscs' +import { readFile } from 'fs-extra' + +interface ServerInfo { + app: ChildProcess, + url: string + host: string + serverNumber: number + + client: { + id: string, + secret: string + } + + user: { + username: string, + password: string, + email?: string + } + + accessToken?: string + + video?: { + id: number + uuid: string + name: string + account: { + name: string + } + } + + remoteVideo?: { + id: number + uuid: string + } +} + +function flushAndRunMultipleServers (totalServers: number, configOverride?: Object) { + let apps = [] + let i = 0 + + return new Promise(res => { + function anotherServerDone (serverNumber, app) { + apps[serverNumber - 1] = app + i++ + if (i === totalServers) { + return res(apps) + } + } + + flushTests() + .then(() => { + for (let j = 1; j <= totalServers; j++) { + runServer(j, configOverride).then(app => anotherServerDone(j, app)) + } + }) + }) +} + +function flushTests () { + return new Promise((res, rej) => { + return exec('npm run clean:server:test', err => { + if (err) return rej(err) + + return res() + }) + }) +} + +function runServer (serverNumber: number, configOverride?: Object) { + const server: ServerInfo = { + app: null, + serverNumber: serverNumber, + url: `http://localhost:${9000 + serverNumber}`, + host: `localhost:${9000 + serverNumber}`, + client: { + id: null, + secret: null + }, + user: { + username: null, + password: null + } + } + + // These actions are async so we need to be sure that they have both been done + const serverRunString = { + 'Server listening': false + } + const key = 'Database peertube_test' + serverNumber + ' is ready' + serverRunString[key] = false + + const regexps = { + client_id: 'Client id: (.+)', + client_secret: 'Client secret: (.+)', + user_username: 'Username: (.+)', + user_password: 'User password: (.+)' + } + + // Share the environment + const env = Object.create(process.env) + env['NODE_ENV'] = 'test' + env['NODE_APP_INSTANCE'] = serverNumber.toString() + + if (configOverride !== undefined) { + env['NODE_CONFIG'] = JSON.stringify(configOverride) + } + + const options = { + silent: true, + env: env, + detached: true + } + + return new Promise(res => { + server.app = fork(join(__dirname, '..', '..', '..', '..', 'dist', 'server.js'), [], options) + server.app.stdout.on('data', function onStdout (data) { + let dontContinue = false + + // Capture things if we want to + for (const key of Object.keys(regexps)) { + const regexp = regexps[key] + const matches = data.toString().match(regexp) + if (matches !== null) { + if (key === 'client_id') server.client.id = matches[1] + else if (key === 'client_secret') server.client.secret = matches[1] + else if (key === 'user_username') server.user.username = matches[1] + else if (key === 'user_password') server.user.password = matches[1] + } + } + + // Check if all required sentences are here + for (const key of Object.keys(serverRunString)) { + if (data.toString().indexOf(key) !== -1) serverRunString[key] = true + if (serverRunString[key] === false) dontContinue = true + } + + // If no, there is maybe one thing not already initialized (client/user credentials generation...) + if (dontContinue === true) return + + server.app.stdout.removeListener('data', onStdout) + res(server) + }) + }) +} + +async function reRunServer (server: ServerInfo, configOverride?: any) { + const newServer = await runServer(server.serverNumber, configOverride) + server.app = newServer.app + + return server +} + +function killallServers (servers: ServerInfo[]) { + for (const server of servers) { + process.kill(-server.app.pid) + } +} + +async function waitUntilLog (server: ServerInfo, str: string, count = 1) { + const logfile = join(root(), 'test' + server.serverNumber, 'logs/peertube.log') + + while (true) { + const buf = await readFile(logfile) + + const matches = buf.toString().match(new RegExp(str, 'g')) + if (matches && matches.length === count) return + + await wait(1000) + } +} + +// --------------------------------------------------------------------------- + +export { + ServerInfo, + flushAndRunMultipleServers, + flushTests, + runServer, + killallServers, + reRunServer, + waitUntilLog +} diff --git a/shared/utils/server/stats.ts b/shared/utils/server/stats.ts new file mode 100644 index 000000000..01989d952 --- /dev/null +++ b/shared/utils/server/stats.ts @@ -0,0 +1,22 @@ +import { makeGetRequest } from '../' + +function getStats (url: string, useCache = false) { + const path = '/api/v1/server/stats' + + const query = { + t: useCache ? undefined : new Date().getTime() + } + + return makeGetRequest({ + url, + path, + query, + statusCodeExpected: 200 + }) +} + +// --------------------------------------------------------------------------- + +export { + getStats +} -- cgit v1.2.3