From 94565d52bb2883e09f16d1363170ac9c0dccb7a1 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Mon, 15 Apr 2019 15:26:15 +0200 Subject: Shared utils -> extra-utils Because they need dev dependencies --- shared/extra-utils/server/activitypub.ts | 14 ++ shared/extra-utils/server/clients.ts | 19 +++ shared/extra-utils/server/config.ts | 156 +++++++++++++++++++++ shared/extra-utils/server/contact-form.ts | 28 ++++ shared/extra-utils/server/follows.ts | 111 +++++++++++++++ shared/extra-utils/server/jobs.ts | 82 +++++++++++ shared/extra-utils/server/redundancy.ts | 17 +++ shared/extra-utils/server/servers.ts | 219 ++++++++++++++++++++++++++++++ shared/extra-utils/server/stats.ts | 22 +++ 9 files changed, 668 insertions(+) create mode 100644 shared/extra-utils/server/activitypub.ts create mode 100644 shared/extra-utils/server/clients.ts create mode 100644 shared/extra-utils/server/config.ts create mode 100644 shared/extra-utils/server/contact-form.ts create mode 100644 shared/extra-utils/server/follows.ts create mode 100644 shared/extra-utils/server/jobs.ts create mode 100644 shared/extra-utils/server/redundancy.ts create mode 100644 shared/extra-utils/server/servers.ts create mode 100644 shared/extra-utils/server/stats.ts (limited to 'shared/extra-utils/server') diff --git a/shared/extra-utils/server/activitypub.ts b/shared/extra-utils/server/activitypub.ts new file mode 100644 index 000000000..eccb198ca --- /dev/null +++ b/shared/extra-utils/server/activitypub.ts @@ -0,0 +1,14 @@ +import * as request from 'supertest' + +function makeActivityPubGetRequest (url: string, path: string, expectedStatus = 200) { + return request(url) + .get(path) + .set('Accept', 'application/activity+json,text/html;q=0.9,\\*/\\*;q=0.8') + .expect(expectedStatus) +} + +// --------------------------------------------------------------------------- + +export { + makeActivityPubGetRequest +} diff --git a/shared/extra-utils/server/clients.ts b/shared/extra-utils/server/clients.ts new file mode 100644 index 000000000..273aac747 --- /dev/null +++ b/shared/extra-utils/server/clients.ts @@ -0,0 +1,19 @@ +import * as request from 'supertest' +import * as urlUtil from 'url' + +function getClient (url: string) { + const path = '/api/v1/oauth-clients/local' + + return request(url) + .get(path) + .set('Host', urlUtil.parse(url).host) + .set('Accept', 'application/json') + .expect(200) + .expect('Content-Type', /json/) +} + +// --------------------------------------------------------------------------- + +export { + getClient +} diff --git a/shared/extra-utils/server/config.ts b/shared/extra-utils/server/config.ts new file mode 100644 index 000000000..deb77e9c0 --- /dev/null +++ b/shared/extra-utils/server/config.ts @@ -0,0 +1,156 @@ +import { makeDeleteRequest, makeGetRequest, makePutBodyRequest } from '../requests/requests' +import { CustomConfig } from '../../models/server/custom-config.model' + +function getConfig (url: string) { + const path = '/api/v1/config' + + return makeGetRequest({ + url, + path, + statusCodeExpected: 200 + }) +} + +function getAbout (url: string) { + const path = '/api/v1/config/about' + + return makeGetRequest({ + url, + path, + statusCodeExpected: 200 + }) +} + +function getCustomConfig (url: string, token: string, statusCodeExpected = 200) { + const path = '/api/v1/config/custom' + + return makeGetRequest({ + url, + token, + path, + statusCodeExpected + }) +} + +function updateCustomConfig (url: string, token: string, newCustomConfig: CustomConfig, statusCodeExpected = 200) { + const path = '/api/v1/config/custom' + + return makePutBodyRequest({ + url, + token, + path, + fields: newCustomConfig, + statusCodeExpected + }) +} + +function updateCustomSubConfig (url: string, token: string, newConfig: any) { + const updateParams: CustomConfig = { + instance: { + name: 'PeerTube updated', + shortDescription: 'my short description', + description: 'my super description', + terms: 'my super terms', + defaultClientRoute: '/videos/recently-added', + isNSFW: true, + defaultNSFWPolicy: 'blur', + customizations: { + javascript: 'alert("coucou")', + css: 'body { background-color: red; }' + } + }, + services: { + twitter: { + username: '@MySuperUsername', + whitelisted: true + } + }, + cache: { + previews: { + size: 2 + }, + captions: { + size: 3 + } + }, + signup: { + enabled: false, + limit: 5, + requiresEmailVerification: false + }, + admin: { + email: 'superadmin1@example.com' + }, + contactForm: { + enabled: true + }, + user: { + videoQuota: 5242881, + videoQuotaDaily: 318742 + }, + transcoding: { + enabled: true, + allowAdditionalExtensions: true, + threads: 1, + resolutions: { + '240p': false, + '360p': true, + '480p': true, + '720p': false, + '1080p': false + }, + hls: { + enabled: false + } + }, + import: { + videos: { + http: { + enabled: false + }, + torrent: { + enabled: false + } + } + }, + autoBlacklist: { + videos: { + ofUsers: { + enabled: false + } + } + }, + followers: { + instance: { + enabled: true, + manualApproval: false + } + } + } + + Object.assign(updateParams, newConfig) + + return updateCustomConfig(url, token, updateParams) +} + +function deleteCustomConfig (url: string, token: string, statusCodeExpected = 200) { + const path = '/api/v1/config/custom' + + return makeDeleteRequest({ + url, + token, + path, + statusCodeExpected + }) +} + +// --------------------------------------------------------------------------- + +export { + getConfig, + getCustomConfig, + updateCustomConfig, + getAbout, + deleteCustomConfig, + updateCustomSubConfig +} diff --git a/shared/extra-utils/server/contact-form.ts b/shared/extra-utils/server/contact-form.ts new file mode 100644 index 000000000..80394cf99 --- /dev/null +++ b/shared/extra-utils/server/contact-form.ts @@ -0,0 +1,28 @@ +import * as request from 'supertest' +import { ContactForm } from '../../models/server' + +function sendContactForm (options: { + url: string, + fromEmail: string, + fromName: string, + body: string, + expectedStatus?: number +}) { + const path = '/api/v1/server/contact' + + const body: ContactForm = { + fromEmail: options.fromEmail, + fromName: options.fromName, + body: options.body + } + return request(options.url) + .post(path) + .send(body) + .expect(options.expectedStatus || 204) +} + +// --------------------------------------------------------------------------- + +export { + sendContactForm +} diff --git a/shared/extra-utils/server/follows.ts b/shared/extra-utils/server/follows.ts new file mode 100644 index 000000000..1505804de --- /dev/null +++ b/shared/extra-utils/server/follows.ts @@ -0,0 +1,111 @@ +import * as request from 'supertest' +import { ServerInfo } from './servers' +import { waitJobs } from './jobs' +import { makeGetRequest, makePostBodyRequest } from '..' + +function getFollowersListPaginationAndSort (url: string, start: number, count: number, sort: string, search?: string) { + const path = '/api/v1/server/followers' + + return request(url) + .get(path) + .query({ start }) + .query({ count }) + .query({ sort }) + .query({ search }) + .set('Accept', 'application/json') + .expect(200) + .expect('Content-Type', /json/) +} + +function acceptFollower (url: string, token: string, follower: string, statusCodeExpected = 204) { + const path = '/api/v1/server/followers/' + follower + '/accept' + + return makePostBodyRequest({ + url, + token, + path, + statusCodeExpected + }) +} + +function rejectFollower (url: string, token: string, follower: string, statusCodeExpected = 204) { + const path = '/api/v1/server/followers/' + follower + '/reject' + + return makePostBodyRequest({ + url, + token, + path, + statusCodeExpected + }) +} + +function getFollowingListPaginationAndSort (url: string, start: number, count: number, sort: string, search?: string) { + const path = '/api/v1/server/following' + + return request(url) + .get(path) + .query({ start }) + .query({ count }) + .query({ sort }) + .query({ search }) + .set('Accept', 'application/json') + .expect(200) + .expect('Content-Type', /json/) +} + +function follow (follower: string, following: string[], accessToken: string, expectedStatus = 204) { + const path = '/api/v1/server/following' + + const followingHosts = following.map(f => f.replace(/^http:\/\//, '')) + return request(follower) + .post(path) + .set('Accept', 'application/json') + .set('Authorization', 'Bearer ' + accessToken) + .send({ 'hosts': followingHosts }) + .expect(expectedStatus) +} + +async function unfollow (url: string, accessToken: string, target: ServerInfo, expectedStatus = 204) { + const path = '/api/v1/server/following/' + target.host + + return request(url) + .delete(path) + .set('Accept', 'application/json') + .set('Authorization', 'Bearer ' + accessToken) + .expect(expectedStatus) +} + +function removeFollower (url: string, accessToken: string, follower: ServerInfo, expectedStatus = 204) { + const path = '/api/v1/server/followers/peertube@' + follower.host + + return request(url) + .delete(path) + .set('Accept', 'application/json') + .set('Authorization', 'Bearer ' + accessToken) + .expect(expectedStatus) +} + +async function doubleFollow (server1: ServerInfo, server2: ServerInfo) { + await Promise.all([ + follow(server1.url, [ server2.url ], server1.accessToken), + follow(server2.url, [ server1.url ], server2.accessToken) + ]) + + // Wait request propagation + await waitJobs([ server1, server2 ]) + + return true +} + +// --------------------------------------------------------------------------- + +export { + getFollowersListPaginationAndSort, + getFollowingListPaginationAndSort, + unfollow, + removeFollower, + follow, + doubleFollow, + acceptFollower, + rejectFollower +} diff --git a/shared/extra-utils/server/jobs.ts b/shared/extra-utils/server/jobs.ts new file mode 100644 index 000000000..692b5e24d --- /dev/null +++ b/shared/extra-utils/server/jobs.ts @@ -0,0 +1,82 @@ +import * as request from 'supertest' +import { Job, JobState } from '../../models' +import { wait } from '../miscs/miscs' +import { ServerInfo } from './servers' + +function getJobsList (url: string, accessToken: string, state: JobState) { + const path = '/api/v1/jobs/' + state + + return request(url) + .get(path) + .set('Accept', 'application/json') + .set('Authorization', 'Bearer ' + accessToken) + .expect(200) + .expect('Content-Type', /json/) +} + +function getJobsListPaginationAndSort (url: string, accessToken: string, state: JobState, start: number, count: number, sort: string) { + const path = '/api/v1/jobs/' + state + + return request(url) + .get(path) + .query({ start }) + .query({ count }) + .query({ sort }) + .set('Accept', 'application/json') + .set('Authorization', 'Bearer ' + accessToken) + .expect(200) + .expect('Content-Type', /json/) +} + +async function waitJobs (serversArg: ServerInfo[] | ServerInfo) { + const pendingJobWait = process.env.NODE_PENDING_JOB_WAIT ? parseInt(process.env.NODE_PENDING_JOB_WAIT, 10) : 2000 + let servers: ServerInfo[] + + if (Array.isArray(serversArg) === false) servers = [ serversArg as ServerInfo ] + else servers = serversArg as ServerInfo[] + + const states: JobState[] = [ 'waiting', 'active', 'delayed' ] + let pendingRequests = false + + function tasksBuilder () { + const tasks: Promise[] = [] + pendingRequests = false + + // Check if each server has pending request + for (const server of servers) { + for (const state of states) { + const p = getJobsListPaginationAndSort(server.url, server.accessToken, state, 0, 10, '-createdAt') + .then(res => res.body.data) + .then((jobs: Job[]) => jobs.filter(j => j.type !== 'videos-views')) + .then(jobs => { + if (jobs.length !== 0) pendingRequests = true + }) + tasks.push(p) + } + } + + return tasks + } + + do { + await Promise.all(tasksBuilder()) + + // Retry, in case of new jobs were created + if (pendingRequests === false) { + await wait(pendingJobWait) + await Promise.all(tasksBuilder()) + } + + if (pendingRequests) { + await wait(1000) + } + } while (pendingRequests) +} + +// --------------------------------------------------------------------------- + +export { + getJobsList, + waitJobs, + getJobsListPaginationAndSort +} diff --git a/shared/extra-utils/server/redundancy.ts b/shared/extra-utils/server/redundancy.ts new file mode 100644 index 000000000..c39ff2c8b --- /dev/null +++ b/shared/extra-utils/server/redundancy.ts @@ -0,0 +1,17 @@ +import { makePutBodyRequest } from '../requests/requests' + +async function updateRedundancy (url: string, accessToken: string, host: string, redundancyAllowed: boolean, expectedStatus = 204) { + const path = '/api/v1/server/redundancy/' + host + + return makePutBodyRequest({ + url, + path, + token: accessToken, + fields: { redundancyAllowed }, + statusCodeExpected: expectedStatus + }) +} + +export { + updateRedundancy +} diff --git a/shared/extra-utils/server/servers.ts b/shared/extra-utils/server/servers.ts new file mode 100644 index 000000000..5288d253a --- /dev/null +++ b/shared/extra-utils/server/servers.ts @@ -0,0 +1,219 @@ +/* tslint:disable:no-unused-expression */ + +import { ChildProcess, exec, fork } from 'child_process' +import { join } from 'path' +import { root, wait } from '../miscs/miscs' +import { readdir, readFile } from 'fs-extra' +import { existsSync } from 'fs' +import { expect } from 'chai' +import { VideoChannel } from '../../models/videos' + +interface ServerInfo { + app: ChildProcess, + url: string + host: string + serverNumber: number + + client: { + id: string, + secret: string + } + + user: { + username: string, + password: string, + email?: string + } + + accessToken?: string + videoChannel?: VideoChannel + + video?: { + id: number + uuid: string + name: string + account: { + name: string + } + } + + remoteVideo?: { + id: number + uuid: string + } + + videos?: { id: number, uuid: string }[] +} + +function flushAndRunMultipleServers (totalServers: number, configOverride?: Object) { + let apps = [] + let i = 0 + + return new Promise(res => { + function anotherServerDone (serverNumber, app) { + apps[serverNumber - 1] = app + i++ + if (i === totalServers) { + return res(apps) + } + } + + flushTests() + .then(() => { + for (let j = 1; j <= totalServers; j++) { + runServer(j, configOverride).then(app => anotherServerDone(j, app)) + } + }) + }) +} + +function flushTests () { + return new Promise((res, rej) => { + return exec('npm run clean:server:test', err => { + if (err) return rej(err) + + return res() + }) + }) +} + +function runServer (serverNumber: number, configOverride?: Object, args = []) { + const server: ServerInfo = { + app: null, + serverNumber: serverNumber, + url: `http://localhost:${9000 + serverNumber}`, + host: `localhost:${9000 + serverNumber}`, + client: { + id: null, + secret: null + }, + user: { + username: null, + password: null + } + } + + // These actions are async so we need to be sure that they have both been done + const serverRunString = { + 'Server listening': false + } + const key = 'Database peertube_test' + serverNumber + ' is ready' + serverRunString[key] = false + + const regexps = { + client_id: 'Client id: (.+)', + client_secret: 'Client secret: (.+)', + user_username: 'Username: (.+)', + user_password: 'User password: (.+)' + } + + // Share the environment + const env = Object.create(process.env) + env['NODE_ENV'] = 'test' + env['NODE_APP_INSTANCE'] = serverNumber.toString() + + if (configOverride !== undefined) { + env['NODE_CONFIG'] = JSON.stringify(configOverride) + } + + const options = { + silent: true, + env: env, + detached: true + } + + return new Promise(res => { + server.app = fork(join(root(), 'dist', 'server.js'), args, options) + server.app.stdout.on('data', function onStdout (data) { + let dontContinue = false + + // Capture things if we want to + for (const key of Object.keys(regexps)) { + const regexp = regexps[key] + const matches = data.toString().match(regexp) + if (matches !== null) { + if (key === 'client_id') server.client.id = matches[1] + else if (key === 'client_secret') server.client.secret = matches[1] + else if (key === 'user_username') server.user.username = matches[1] + else if (key === 'user_password') server.user.password = matches[1] + } + } + + // Check if all required sentences are here + for (const key of Object.keys(serverRunString)) { + if (data.toString().indexOf(key) !== -1) serverRunString[key] = true + if (serverRunString[key] === false) dontContinue = true + } + + // If no, there is maybe one thing not already initialized (client/user credentials generation...) + if (dontContinue === true) return + + server.app.stdout.removeListener('data', onStdout) + + process.on('exit', () => { + try { + process.kill(server.app.pid) + } catch { /* empty */ } + }) + + res(server) + }) + + }) +} + +async function reRunServer (server: ServerInfo, configOverride?: any) { + const newServer = await runServer(server.serverNumber, configOverride) + server.app = newServer.app + + return server +} + +async function checkTmpIsEmpty (server: ServerInfo) { + return checkDirectoryIsEmpty(server, 'tmp') +} + +async function checkDirectoryIsEmpty (server: ServerInfo, directory: string) { + const testDirectory = 'test' + server.serverNumber + + const directoryPath = join(root(), testDirectory, directory) + + const directoryExists = existsSync(directoryPath) + expect(directoryExists).to.be.true + + const files = await readdir(directoryPath) + expect(files).to.have.lengthOf(0) +} + +function killallServers (servers: ServerInfo[]) { + for (const server of servers) { + process.kill(-server.app.pid) + } +} + +async function waitUntilLog (server: ServerInfo, str: string, count = 1) { + const logfile = join(root(), 'test' + server.serverNumber, 'logs/peertube.log') + + while (true) { + const buf = await readFile(logfile) + + const matches = buf.toString().match(new RegExp(str, 'g')) + if (matches && matches.length === count) return + + await wait(1000) + } +} + +// --------------------------------------------------------------------------- + +export { + checkDirectoryIsEmpty, + checkTmpIsEmpty, + ServerInfo, + flushAndRunMultipleServers, + flushTests, + runServer, + killallServers, + reRunServer, + waitUntilLog +} diff --git a/shared/extra-utils/server/stats.ts b/shared/extra-utils/server/stats.ts new file mode 100644 index 000000000..6f079ad18 --- /dev/null +++ b/shared/extra-utils/server/stats.ts @@ -0,0 +1,22 @@ +import { makeGetRequest } from '../requests/requests' + +function getStats (url: string, useCache = false) { + const path = '/api/v1/server/stats' + + const query = { + t: useCache ? undefined : new Date().getTime() + } + + return makeGetRequest({ + url, + path, + query, + statusCodeExpected: 200 + }) +} + +// --------------------------------------------------------------------------- + +export { + getStats +} -- cgit v1.2.3