From 3a4992633ee62d5edfbb484d9c6bcb3cf158489d Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Mon, 31 Jul 2023 14:34:36 +0200 Subject: Migrate server to ESM Sorry for the very big commit that may lead to git log issues and merge conflicts, but it's a major step forward: * Server can be faster at startup because imports() are async and we can easily lazy import big modules * Angular doesn't seem to support ES import (with .js extension), so we had to correctly organize peertube into a monorepo: * Use yarn workspace feature * Use typescript reference projects for dependencies * Shared projects have been moved into "packages", each one is now a node module (with a dedicated package.json/tsconfig.json) * server/tools have been moved into apps/ and is now a dedicated app bundled and published on NPM so users don't have to build peertube cli tools manually * server/tests have been moved into packages/ so we don't compile them every time we want to run the server * Use isolatedModule option: * Had to move from const enum to const (https://www.typescriptlang.org/docs/handbook/enums.html#objects-vs-enums) * Had to explictely specify "type" imports when used in decorators * Prefer tsx (that uses esbuild under the hood) instead of ts-node to load typescript files (tests with mocha or scripts): * To reduce test complexity as esbuild doesn't support decorator metadata, we only test server files that do not import server models * We still build tests files into js files for a faster CI * Remove unmaintained peertube CLI import script * Removed some barrels to speed up execution (less imports) --- .../server-commands/src/server/config-command.ts | 576 +++++++++++++++++++++ .../src/server/contact-form-command.ts | 30 ++ .../server-commands/src/server/debug-command.ts | 33 ++ .../server-commands/src/server/follows-command.ts | 139 +++++ packages/server-commands/src/server/follows.ts | 20 + packages/server-commands/src/server/index.ts | 15 + .../server-commands/src/server/jobs-command.ts | 84 +++ packages/server-commands/src/server/jobs.ts | 117 +++++ .../server-commands/src/server/metrics-command.ts | 18 + .../src/server/object-storage-command.ts | 165 ++++++ .../server-commands/src/server/plugins-command.ts | 258 +++++++++ .../src/server/redundancy-command.ts | 80 +++ packages/server-commands/src/server/server.ts | 451 ++++++++++++++++ .../server-commands/src/server/servers-command.ts | 104 ++++ packages/server-commands/src/server/servers.ts | 68 +++ .../server-commands/src/server/stats-command.ts | 25 + 16 files changed, 2183 insertions(+) create mode 100644 packages/server-commands/src/server/config-command.ts create mode 100644 packages/server-commands/src/server/contact-form-command.ts create mode 100644 packages/server-commands/src/server/debug-command.ts create mode 100644 packages/server-commands/src/server/follows-command.ts create mode 100644 packages/server-commands/src/server/follows.ts create mode 100644 packages/server-commands/src/server/index.ts create mode 100644 packages/server-commands/src/server/jobs-command.ts create mode 100644 packages/server-commands/src/server/jobs.ts create mode 100644 packages/server-commands/src/server/metrics-command.ts create mode 100644 packages/server-commands/src/server/object-storage-command.ts create mode 100644 packages/server-commands/src/server/plugins-command.ts create mode 100644 packages/server-commands/src/server/redundancy-command.ts create mode 100644 packages/server-commands/src/server/server.ts create mode 100644 packages/server-commands/src/server/servers-command.ts create mode 100644 packages/server-commands/src/server/servers.ts create mode 100644 packages/server-commands/src/server/stats-command.ts (limited to 'packages/server-commands/src/server') diff --git a/packages/server-commands/src/server/config-command.ts b/packages/server-commands/src/server/config-command.ts new file mode 100644 index 000000000..8fcf0bd51 --- /dev/null +++ b/packages/server-commands/src/server/config-command.ts @@ -0,0 +1,576 @@ +import merge from 'lodash-es/merge.js' +import { About, CustomConfig, HttpStatusCode, ServerConfig } from '@peertube/peertube-models' +import { DeepPartial } from '@peertube/peertube-typescript-utils' +import { AbstractCommand, OverrideCommandOptions } from '../shared/abstract-command.js' + +export class ConfigCommand extends AbstractCommand { + + static getCustomConfigResolutions (enabled: boolean, with0p = false) { + return { + '0p': enabled && with0p, + '144p': enabled, + '240p': enabled, + '360p': enabled, + '480p': enabled, + '720p': enabled, + '1080p': enabled, + '1440p': enabled, + '2160p': enabled + } + } + + // --------------------------------------------------------------------------- + + static getEmailOverrideConfig (emailPort: number) { + return { + smtp: { + hostname: '127.0.0.1', + port: emailPort + } + } + } + + // --------------------------------------------------------------------------- + + enableSignup (requiresApproval: boolean, limit = -1) { + return this.updateExistingSubConfig({ + newConfig: { + signup: { + enabled: true, + requiresApproval, + limit + } + } + }) + } + + // --------------------------------------------------------------------------- + + disableImports () { + return this.setImportsEnabled(false) + } + + enableImports () { + return this.setImportsEnabled(true) + } + + private setImportsEnabled (enabled: boolean) { + return this.updateExistingSubConfig({ + newConfig: { + import: { + videos: { + http: { + enabled + }, + + torrent: { + enabled + } + } + } + } + }) + } + + // --------------------------------------------------------------------------- + + disableFileUpdate () { + return this.setFileUpdateEnabled(false) + } + + enableFileUpdate () { + return this.setFileUpdateEnabled(true) + } + + private setFileUpdateEnabled (enabled: boolean) { + return this.updateExistingSubConfig({ + newConfig: { + videoFile: { + update: { + enabled + } + } + } + }) + } + + // --------------------------------------------------------------------------- + + enableChannelSync () { + return this.setChannelSyncEnabled(true) + } + + disableChannelSync () { + return this.setChannelSyncEnabled(false) + } + + private setChannelSyncEnabled (enabled: boolean) { + return this.updateExistingSubConfig({ + newConfig: { + import: { + videoChannelSynchronization: { + enabled + } + } + } + }) + } + + // --------------------------------------------------------------------------- + + enableLive (options: { + allowReplay?: boolean + transcoding?: boolean + resolutions?: 'min' | 'max' // Default max + } = {}) { + const { allowReplay, transcoding, resolutions = 'max' } = options + + return this.updateExistingSubConfig({ + newConfig: { + live: { + enabled: true, + allowReplay: allowReplay ?? true, + transcoding: { + enabled: transcoding ?? true, + resolutions: ConfigCommand.getCustomConfigResolutions(resolutions === 'max') + } + } + } + }) + } + + disableTranscoding () { + return this.updateExistingSubConfig({ + newConfig: { + transcoding: { + enabled: false + }, + videoStudio: { + enabled: false + } + } + }) + } + + enableTranscoding (options: { + webVideo?: boolean // default true + hls?: boolean // default true + with0p?: boolean // default false + } = {}) { + const { webVideo = true, hls = true, with0p = false } = options + + return this.updateExistingSubConfig({ + newConfig: { + transcoding: { + enabled: true, + + allowAudioFiles: true, + allowAdditionalExtensions: true, + + resolutions: ConfigCommand.getCustomConfigResolutions(true, with0p), + + webVideos: { + enabled: webVideo + }, + hls: { + enabled: hls + } + } + } + }) + } + + enableMinimumTranscoding (options: { + webVideo?: boolean // default true + hls?: boolean // default true + } = {}) { + const { webVideo = true, hls = true } = options + + return this.updateExistingSubConfig({ + newConfig: { + transcoding: { + enabled: true, + + allowAudioFiles: true, + allowAdditionalExtensions: true, + + resolutions: { + ...ConfigCommand.getCustomConfigResolutions(false), + + '240p': true + }, + + webVideos: { + enabled: webVideo + }, + hls: { + enabled: hls + } + } + } + }) + } + + enableRemoteTranscoding () { + return this.updateExistingSubConfig({ + newConfig: { + transcoding: { + remoteRunners: { + enabled: true + } + }, + live: { + transcoding: { + remoteRunners: { + enabled: true + } + } + } + } + }) + } + + enableRemoteStudio () { + return this.updateExistingSubConfig({ + newConfig: { + videoStudio: { + remoteRunners: { + enabled: true + } + } + } + }) + } + + // --------------------------------------------------------------------------- + + enableStudio () { + return this.updateExistingSubConfig({ + newConfig: { + videoStudio: { + enabled: true + } + } + }) + } + + // --------------------------------------------------------------------------- + + getConfig (options: OverrideCommandOptions = {}) { + const path = '/api/v1/config' + + return this.getRequestBody({ + ...options, + + path, + implicitToken: false, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } + + async getIndexHTMLConfig (options: OverrideCommandOptions = {}) { + const text = await this.getRequestText({ + ...options, + + path: '/', + implicitToken: false, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + + const match = text.match('') + + // We parse the string twice, first to extract the string and then to extract the JSON + return JSON.parse(JSON.parse(match[1])) as ServerConfig + } + + getAbout (options: OverrideCommandOptions = {}) { + const path = '/api/v1/config/about' + + return this.getRequestBody({ + ...options, + + path, + implicitToken: false, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } + + getCustomConfig (options: OverrideCommandOptions = {}) { + const path = '/api/v1/config/custom' + + return this.getRequestBody({ + ...options, + + path, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } + + updateCustomConfig (options: OverrideCommandOptions & { + newCustomConfig: CustomConfig + }) { + const path = '/api/v1/config/custom' + + return this.putBodyRequest({ + ...options, + + path, + fields: options.newCustomConfig, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } + + deleteCustomConfig (options: OverrideCommandOptions = {}) { + const path = '/api/v1/config/custom' + + return this.deleteRequest({ + ...options, + + path, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } + + async updateExistingSubConfig (options: OverrideCommandOptions & { + newConfig: DeepPartial + }) { + const existing = await this.getCustomConfig({ ...options, expectedStatus: HttpStatusCode.OK_200 }) + + return this.updateCustomConfig({ ...options, newCustomConfig: merge({}, existing, options.newConfig) }) + } + + updateCustomSubConfig (options: OverrideCommandOptions & { + newConfig: DeepPartial + }) { + const newCustomConfig: CustomConfig = { + instance: { + name: 'PeerTube updated', + shortDescription: 'my short description', + description: 'my super description', + terms: 'my super terms', + codeOfConduct: 'my super coc', + + creationReason: 'my super creation reason', + moderationInformation: 'my super moderation information', + administrator: 'Kuja', + maintenanceLifetime: 'forever', + businessModel: 'my super business model', + hardwareInformation: '2vCore 3GB RAM', + + languages: [ 'en', 'es' ], + categories: [ 1, 2 ], + + isNSFW: true, + defaultNSFWPolicy: 'blur', + + defaultClientRoute: '/videos/recently-added', + + customizations: { + javascript: 'alert("coucou")', + css: 'body { background-color: red; }' + } + }, + theme: { + default: 'default' + }, + services: { + twitter: { + username: '@MySuperUsername', + whitelisted: true + } + }, + client: { + videos: { + miniature: { + preferAuthorDisplayName: false + } + }, + menu: { + login: { + redirectOnSingleExternalAuth: false + } + } + }, + cache: { + previews: { + size: 2 + }, + captions: { + size: 3 + }, + torrents: { + size: 4 + }, + storyboards: { + size: 5 + } + }, + signup: { + enabled: false, + limit: 5, + requiresApproval: true, + requiresEmailVerification: false, + minimumAge: 16 + }, + admin: { + email: 'superadmin1@example.com' + }, + contactForm: { + enabled: true + }, + user: { + history: { + videos: { + enabled: true + } + }, + videoQuota: 5242881, + videoQuotaDaily: 318742 + }, + videoChannels: { + maxPerUser: 20 + }, + transcoding: { + enabled: true, + remoteRunners: { + enabled: false + }, + allowAdditionalExtensions: true, + allowAudioFiles: true, + threads: 1, + concurrency: 3, + profile: 'default', + resolutions: { + '0p': false, + '144p': false, + '240p': false, + '360p': true, + '480p': true, + '720p': false, + '1080p': false, + '1440p': false, + '2160p': false + }, + alwaysTranscodeOriginalResolution: true, + webVideos: { + enabled: true + }, + hls: { + enabled: false + } + }, + live: { + enabled: true, + allowReplay: false, + latencySetting: { + enabled: false + }, + maxDuration: -1, + maxInstanceLives: -1, + maxUserLives: 50, + transcoding: { + enabled: true, + remoteRunners: { + enabled: false + }, + threads: 4, + profile: 'default', + resolutions: { + '144p': true, + '240p': true, + '360p': true, + '480p': true, + '720p': true, + '1080p': true, + '1440p': true, + '2160p': true + }, + alwaysTranscodeOriginalResolution: true + } + }, + videoStudio: { + enabled: false, + remoteRunners: { + enabled: false + } + }, + videoFile: { + update: { + enabled: false + } + }, + import: { + videos: { + concurrency: 3, + http: { + enabled: false + }, + torrent: { + enabled: false + } + }, + videoChannelSynchronization: { + enabled: false, + maxPerUser: 10 + } + }, + trending: { + videos: { + algorithms: { + enabled: [ 'hot', 'most-viewed', 'most-liked' ], + default: 'hot' + } + } + }, + autoBlacklist: { + videos: { + ofUsers: { + enabled: false + } + } + }, + followers: { + instance: { + enabled: true, + manualApproval: false + } + }, + followings: { + instance: { + autoFollowBack: { + enabled: false + }, + autoFollowIndex: { + indexUrl: 'https://instances.joinpeertube.org/api/v1/instances/hosts', + enabled: false + } + } + }, + broadcastMessage: { + enabled: true, + level: 'warning', + message: 'hello', + dismissable: true + }, + search: { + remoteUri: { + users: true, + anonymous: true + }, + searchIndex: { + enabled: true, + url: 'https://search.joinpeertube.org', + disableLocalSearch: true, + isDefaultSearch: true + } + } + } + + merge(newCustomConfig, options.newConfig) + + return this.updateCustomConfig({ ...options, newCustomConfig }) + } +} diff --git a/packages/server-commands/src/server/contact-form-command.ts b/packages/server-commands/src/server/contact-form-command.ts new file mode 100644 index 000000000..399e06d2f --- /dev/null +++ b/packages/server-commands/src/server/contact-form-command.ts @@ -0,0 +1,30 @@ +import { ContactForm, HttpStatusCode } from '@peertube/peertube-models' +import { AbstractCommand, OverrideCommandOptions } from '../shared/index.js' + +export class ContactFormCommand extends AbstractCommand { + + send (options: OverrideCommandOptions & { + fromEmail: string + fromName: string + subject: string + body: string + }) { + const path = '/api/v1/server/contact' + + const body: ContactForm = { + fromEmail: options.fromEmail, + fromName: options.fromName, + subject: options.subject, + body: options.body + } + + return this.postBodyRequest({ + ...options, + + path, + fields: body, + implicitToken: false, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } +} diff --git a/packages/server-commands/src/server/debug-command.ts b/packages/server-commands/src/server/debug-command.ts new file mode 100644 index 000000000..9bb7fda10 --- /dev/null +++ b/packages/server-commands/src/server/debug-command.ts @@ -0,0 +1,33 @@ +import { Debug, HttpStatusCode, SendDebugCommand } from '@peertube/peertube-models' +import { AbstractCommand, OverrideCommandOptions } from '../shared/index.js' + +export class DebugCommand extends AbstractCommand { + + getDebug (options: OverrideCommandOptions = {}) { + const path = '/api/v1/server/debug' + + return this.getRequestBody({ + ...options, + + path, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } + + sendCommand (options: OverrideCommandOptions & { + body: SendDebugCommand + }) { + const { body } = options + const path = '/api/v1/server/debug/run-command' + + return this.postBodyRequest({ + ...options, + + path, + fields: body, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } +} diff --git a/packages/server-commands/src/server/follows-command.ts b/packages/server-commands/src/server/follows-command.ts new file mode 100644 index 000000000..cdc263982 --- /dev/null +++ b/packages/server-commands/src/server/follows-command.ts @@ -0,0 +1,139 @@ +import { pick } from '@peertube/peertube-core-utils' +import { ActivityPubActorType, ActorFollow, FollowState, HttpStatusCode, ResultList, ServerFollowCreate } from '@peertube/peertube-models' +import { AbstractCommand, OverrideCommandOptions } from '../shared/index.js' +import { PeerTubeServer } from './server.js' + +export class FollowsCommand extends AbstractCommand { + + getFollowers (options: OverrideCommandOptions & { + start?: number + count?: number + sort?: string + search?: string + actorType?: ActivityPubActorType + state?: FollowState + } = {}) { + const path = '/api/v1/server/followers' + + const query = pick(options, [ 'start', 'count', 'sort', 'search', 'state', 'actorType' ]) + + return this.getRequestBody>({ + ...options, + + path, + query, + implicitToken: false, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } + + getFollowings (options: OverrideCommandOptions & { + start?: number + count?: number + sort?: string + search?: string + actorType?: ActivityPubActorType + state?: FollowState + } = {}) { + const path = '/api/v1/server/following' + + const query = pick(options, [ 'start', 'count', 'sort', 'search', 'state', 'actorType' ]) + + return this.getRequestBody>({ + ...options, + + path, + query, + implicitToken: false, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } + + follow (options: OverrideCommandOptions & { + hosts?: string[] + handles?: string[] + }) { + const path = '/api/v1/server/following' + + const fields: ServerFollowCreate = {} + + if (options.hosts) { + fields.hosts = options.hosts.map(f => f.replace(/^http:\/\//, '')) + } + + if (options.handles) { + fields.handles = options.handles + } + + return this.postBodyRequest({ + ...options, + + path, + fields, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } + + async unfollow (options: OverrideCommandOptions & { + target: PeerTubeServer | string + }) { + const { target } = options + + const handle = typeof target === 'string' + ? target + : target.host + + const path = '/api/v1/server/following/' + handle + + return this.deleteRequest({ + ...options, + + path, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } + + acceptFollower (options: OverrideCommandOptions & { + follower: string + }) { + const path = '/api/v1/server/followers/' + options.follower + '/accept' + + return this.postBodyRequest({ + ...options, + + path, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } + + rejectFollower (options: OverrideCommandOptions & { + follower: string + }) { + const path = '/api/v1/server/followers/' + options.follower + '/reject' + + return this.postBodyRequest({ + ...options, + + path, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } + + removeFollower (options: OverrideCommandOptions & { + follower: PeerTubeServer + }) { + const path = '/api/v1/server/followers/peertube@' + options.follower.host + + return this.deleteRequest({ + ...options, + + path, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } +} diff --git a/packages/server-commands/src/server/follows.ts b/packages/server-commands/src/server/follows.ts new file mode 100644 index 000000000..32304495a --- /dev/null +++ b/packages/server-commands/src/server/follows.ts @@ -0,0 +1,20 @@ +import { waitJobs } from './jobs.js' +import { PeerTubeServer } from './server.js' + +async function doubleFollow (server1: PeerTubeServer, server2: PeerTubeServer) { + await Promise.all([ + server1.follows.follow({ hosts: [ server2.url ] }), + server2.follows.follow({ hosts: [ server1.url ] }) + ]) + + // Wait request propagation + await waitJobs([ server1, server2 ]) + + return true +} + +// --------------------------------------------------------------------------- + +export { + doubleFollow +} diff --git a/packages/server-commands/src/server/index.ts b/packages/server-commands/src/server/index.ts new file mode 100644 index 000000000..c13972eca --- /dev/null +++ b/packages/server-commands/src/server/index.ts @@ -0,0 +1,15 @@ +export * from './config-command.js' +export * from './contact-form-command.js' +export * from './debug-command.js' +export * from './follows-command.js' +export * from './follows.js' +export * from './jobs.js' +export * from './jobs-command.js' +export * from './metrics-command.js' +export * from './object-storage-command.js' +export * from './plugins-command.js' +export * from './redundancy-command.js' +export * from './server.js' +export * from './servers-command.js' +export * from './servers.js' +export * from './stats-command.js' diff --git a/packages/server-commands/src/server/jobs-command.ts b/packages/server-commands/src/server/jobs-command.ts new file mode 100644 index 000000000..18aa0cd95 --- /dev/null +++ b/packages/server-commands/src/server/jobs-command.ts @@ -0,0 +1,84 @@ +import { pick } from '@peertube/peertube-core-utils' +import { HttpStatusCode, Job, JobState, JobType, ResultList } from '@peertube/peertube-models' +import { AbstractCommand, OverrideCommandOptions } from '../shared/index.js' + +export class JobsCommand extends AbstractCommand { + + async getLatest (options: OverrideCommandOptions & { + jobType: JobType + }) { + const { data } = await this.list({ ...options, start: 0, count: 1, sort: '-createdAt' }) + + if (data.length === 0) return undefined + + return data[0] + } + + pauseJobQueue (options: OverrideCommandOptions = {}) { + const path = '/api/v1/jobs/pause' + + return this.postBodyRequest({ + ...options, + + path, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } + + resumeJobQueue (options: OverrideCommandOptions = {}) { + const path = '/api/v1/jobs/resume' + + return this.postBodyRequest({ + ...options, + + path, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } + + list (options: OverrideCommandOptions & { + state?: JobState + jobType?: JobType + start?: number + count?: number + sort?: string + } = {}) { + const path = this.buildJobsUrl(options.state) + + const query = pick(options, [ 'start', 'count', 'sort', 'jobType' ]) + + return this.getRequestBody>({ + ...options, + + path, + query, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } + + listFailed (options: OverrideCommandOptions & { + jobType?: JobType + }) { + const path = this.buildJobsUrl('failed') + + return this.getRequestBody>({ + ...options, + + path, + query: { start: 0, count: 50 }, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } + + private buildJobsUrl (state?: JobState) { + let path = '/api/v1/jobs' + + if (state) path += '/' + state + + return path + } +} diff --git a/packages/server-commands/src/server/jobs.ts b/packages/server-commands/src/server/jobs.ts new file mode 100644 index 000000000..1f3b1f745 --- /dev/null +++ b/packages/server-commands/src/server/jobs.ts @@ -0,0 +1,117 @@ +import { expect } from 'chai' +import { wait } from '@peertube/peertube-core-utils' +import { JobState, JobType, RunnerJobState } from '@peertube/peertube-models' +import { PeerTubeServer } from './server.js' + +async function waitJobs ( + serversArg: PeerTubeServer[] | PeerTubeServer, + options: { + skipDelayed?: boolean // default false + runnerJobs?: boolean // default false + } = {} +) { + const { skipDelayed = false, runnerJobs = false } = options + + const pendingJobWait = process.env.NODE_PENDING_JOB_WAIT + ? parseInt(process.env.NODE_PENDING_JOB_WAIT, 10) + : 250 + + let servers: PeerTubeServer[] + + if (Array.isArray(serversArg) === false) servers = [ serversArg as PeerTubeServer ] + else servers = serversArg as PeerTubeServer[] + + const states: JobState[] = [ 'waiting', 'active' ] + if (!skipDelayed) states.push('delayed') + + const repeatableJobs: JobType[] = [ 'videos-views-stats', 'activitypub-cleaner' ] + let pendingRequests: boolean + + function tasksBuilder () { + const tasks: Promise[] = [] + + // Check if each server has pending request + for (const server of servers) { + if (process.env.DEBUG) console.log('Checking ' + server.url) + + for (const state of states) { + + const jobPromise = server.jobs.list({ + state, + start: 0, + count: 10, + sort: '-createdAt' + }).then(body => body.data) + .then(jobs => jobs.filter(j => !repeatableJobs.includes(j.type))) + .then(jobs => { + if (jobs.length !== 0) { + pendingRequests = true + + if (process.env.DEBUG) { + console.log(jobs) + } + } + }) + + tasks.push(jobPromise) + } + + const debugPromise = server.debug.getDebug() + .then(obj => { + if (obj.activityPubMessagesWaiting !== 0) { + pendingRequests = true + + if (process.env.DEBUG) { + console.log('AP messages waiting: ' + obj.activityPubMessagesWaiting) + } + } + }) + tasks.push(debugPromise) + + if (runnerJobs) { + const runnerJobsPromise = server.runnerJobs.list({ count: 100 }) + .then(({ data }) => { + for (const job of data) { + if (job.state.id !== RunnerJobState.COMPLETED) { + pendingRequests = true + + if (process.env.DEBUG) { + console.log(job) + } + } + } + }) + tasks.push(runnerJobsPromise) + } + } + + return tasks + } + + do { + pendingRequests = false + await Promise.all(tasksBuilder()) + + // Retry, in case of new jobs were created + if (pendingRequests === false) { + await wait(pendingJobWait) + await Promise.all(tasksBuilder()) + } + + if (pendingRequests) { + await wait(pendingJobWait) + } + } while (pendingRequests) +} + +async function expectNoFailedTranscodingJob (server: PeerTubeServer) { + const { data } = await server.jobs.listFailed({ jobType: 'video-transcoding' }) + expect(data).to.have.lengthOf(0) +} + +// --------------------------------------------------------------------------- + +export { + waitJobs, + expectNoFailedTranscodingJob +} diff --git a/packages/server-commands/src/server/metrics-command.ts b/packages/server-commands/src/server/metrics-command.ts new file mode 100644 index 000000000..1f969a024 --- /dev/null +++ b/packages/server-commands/src/server/metrics-command.ts @@ -0,0 +1,18 @@ +import { HttpStatusCode, PlaybackMetricCreate } from '@peertube/peertube-models' +import { AbstractCommand, OverrideCommandOptions } from '../shared/index.js' + +export class MetricsCommand extends AbstractCommand { + + addPlaybackMetric (options: OverrideCommandOptions & { metrics: PlaybackMetricCreate }) { + const path = '/api/v1/metrics/playback' + + return this.postBodyRequest({ + ...options, + + path, + fields: options.metrics, + implicitToken: false, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } +} diff --git a/packages/server-commands/src/server/object-storage-command.ts b/packages/server-commands/src/server/object-storage-command.ts new file mode 100644 index 000000000..ff8d5d75c --- /dev/null +++ b/packages/server-commands/src/server/object-storage-command.ts @@ -0,0 +1,165 @@ +import { randomInt } from 'crypto' +import { HttpStatusCode } from '@peertube/peertube-models' +import { makePostBodyRequest } from '../requests/index.js' + +export class ObjectStorageCommand { + static readonly DEFAULT_SCALEWAY_BUCKET = 'peertube-ci-test' + + private readonly bucketsCreated: string[] = [] + private readonly seed: number + + // --------------------------------------------------------------------------- + + constructor () { + this.seed = randomInt(0, 10000) + } + + static getMockCredentialsConfig () { + return { + access_key_id: 'AKIAIOSFODNN7EXAMPLE', + secret_access_key: 'wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY' + } + } + + static getMockEndpointHost () { + return 'localhost:9444' + } + + static getMockRegion () { + return 'us-east-1' + } + + getDefaultMockConfig () { + return { + object_storage: { + enabled: true, + endpoint: 'http://' + ObjectStorageCommand.getMockEndpointHost(), + region: ObjectStorageCommand.getMockRegion(), + + credentials: ObjectStorageCommand.getMockCredentialsConfig(), + + streaming_playlists: { + bucket_name: this.getMockStreamingPlaylistsBucketName() + }, + + web_videos: { + bucket_name: this.getMockWebVideosBucketName() + } + } + } + } + + getMockWebVideosBaseUrl () { + return `http://${this.getMockWebVideosBucketName()}.${ObjectStorageCommand.getMockEndpointHost()}/` + } + + getMockPlaylistBaseUrl () { + return `http://${this.getMockStreamingPlaylistsBucketName()}.${ObjectStorageCommand.getMockEndpointHost()}/` + } + + async prepareDefaultMockBuckets () { + await this.createMockBucket(this.getMockStreamingPlaylistsBucketName()) + await this.createMockBucket(this.getMockWebVideosBucketName()) + } + + async createMockBucket (name: string) { + this.bucketsCreated.push(name) + + await this.deleteMockBucket(name) + + await makePostBodyRequest({ + url: ObjectStorageCommand.getMockEndpointHost(), + path: '/ui/' + name + '?create', + expectedStatus: HttpStatusCode.TEMPORARY_REDIRECT_307 + }) + + await makePostBodyRequest({ + url: ObjectStorageCommand.getMockEndpointHost(), + path: '/ui/' + name + '?make-public', + expectedStatus: HttpStatusCode.TEMPORARY_REDIRECT_307 + }) + } + + async cleanupMock () { + for (const name of this.bucketsCreated) { + await this.deleteMockBucket(name) + } + } + + getMockStreamingPlaylistsBucketName (name = 'streaming-playlists') { + return this.getMockBucketName(name) + } + + getMockWebVideosBucketName (name = 'web-videos') { + return this.getMockBucketName(name) + } + + getMockBucketName (name: string) { + return `${this.seed}-${name}` + } + + private async deleteMockBucket (name: string) { + await makePostBodyRequest({ + url: ObjectStorageCommand.getMockEndpointHost(), + path: '/ui/' + name + '?delete', + expectedStatus: HttpStatusCode.TEMPORARY_REDIRECT_307 + }) + } + + // --------------------------------------------------------------------------- + + static getDefaultScalewayConfig (options: { + serverNumber: number + enablePrivateProxy?: boolean // default true + privateACL?: 'private' | 'public-read' // default 'private' + }) { + const { serverNumber, enablePrivateProxy = true, privateACL = 'private' } = options + + return { + object_storage: { + enabled: true, + endpoint: this.getScalewayEndpointHost(), + region: this.getScalewayRegion(), + + credentials: this.getScalewayCredentialsConfig(), + + upload_acl: { + private: privateACL + }, + + proxy: { + proxify_private_files: enablePrivateProxy + }, + + streaming_playlists: { + bucket_name: this.DEFAULT_SCALEWAY_BUCKET, + prefix: `test:server-${serverNumber}-streaming-playlists:` + }, + + web_videos: { + bucket_name: this.DEFAULT_SCALEWAY_BUCKET, + prefix: `test:server-${serverNumber}-web-videos:` + } + } + } + } + + static getScalewayCredentialsConfig () { + return { + access_key_id: process.env.OBJECT_STORAGE_SCALEWAY_KEY_ID, + secret_access_key: process.env.OBJECT_STORAGE_SCALEWAY_ACCESS_KEY + } + } + + static getScalewayEndpointHost () { + return 's3.fr-par.scw.cloud' + } + + static getScalewayRegion () { + return 'fr-par' + } + + static getScalewayBaseUrl () { + return `https://${this.DEFAULT_SCALEWAY_BUCKET}.${this.getScalewayEndpointHost()}/` + } +} diff --git a/packages/server-commands/src/server/plugins-command.ts b/packages/server-commands/src/server/plugins-command.ts new file mode 100644 index 000000000..f85ef0330 --- /dev/null +++ b/packages/server-commands/src/server/plugins-command.ts @@ -0,0 +1,258 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { readJSON, writeJSON } from 'fs-extra/esm' +import { join } from 'path' +import { + HttpStatusCode, + HttpStatusCodeType, + PeerTubePlugin, + PeerTubePluginIndex, + PeertubePluginIndexList, + PluginPackageJSON, + PluginTranslation, + PluginType_Type, + PublicServerSetting, + RegisteredServerSettings, + ResultList +} from '@peertube/peertube-models' +import { buildAbsoluteFixturePath } from '@peertube/peertube-node-utils' +import { AbstractCommand, OverrideCommandOptions } from '../shared/index.js' + +export class PluginsCommand extends AbstractCommand { + + static getPluginTestPath (suffix = '') { + return buildAbsoluteFixturePath('peertube-plugin-test' + suffix) + } + + list (options: OverrideCommandOptions & { + start?: number + count?: number + sort?: string + pluginType?: PluginType_Type + uninstalled?: boolean + }) { + const { start, count, sort, pluginType, uninstalled } = options + const path = '/api/v1/plugins' + + return this.getRequestBody>({ + ...options, + + path, + query: { + start, + count, + sort, + pluginType, + uninstalled + }, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } + + listAvailable (options: OverrideCommandOptions & { + start?: number + count?: number + sort?: string + pluginType?: PluginType_Type + currentPeerTubeEngine?: string + search?: string + expectedStatus?: HttpStatusCodeType + }) { + const { start, count, sort, pluginType, search, currentPeerTubeEngine } = options + const path = '/api/v1/plugins/available' + + const query: PeertubePluginIndexList = { + start, + count, + sort, + pluginType, + currentPeerTubeEngine, + search + } + + return this.getRequestBody>({ + ...options, + + path, + query, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } + + get (options: OverrideCommandOptions & { + npmName: string + }) { + const path = '/api/v1/plugins/' + options.npmName + + return this.getRequestBody({ + ...options, + + path, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } + + updateSettings (options: OverrideCommandOptions & { + npmName: string + settings: any + }) { + const { npmName, settings } = options + const path = '/api/v1/plugins/' + npmName + '/settings' + + return this.putBodyRequest({ + ...options, + + path, + fields: { settings }, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } + + getRegisteredSettings (options: OverrideCommandOptions & { + npmName: string + }) { + const path = '/api/v1/plugins/' + options.npmName + '/registered-settings' + + return this.getRequestBody({ + ...options, + + path, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } + + getPublicSettings (options: OverrideCommandOptions & { + npmName: string + }) { + const { npmName } = options + const path = '/api/v1/plugins/' + npmName + '/public-settings' + + return this.getRequestBody({ + ...options, + + path, + implicitToken: false, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } + + getTranslations (options: OverrideCommandOptions & { + locale: string + }) { + const { locale } = options + const path = '/plugins/translations/' + locale + '.json' + + return this.getRequestBody({ + ...options, + + path, + implicitToken: false, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } + + install (options: OverrideCommandOptions & { + path?: string + npmName?: string + pluginVersion?: string + }) { + const { npmName, path, pluginVersion } = options + const apiPath = '/api/v1/plugins/install' + + return this.postBodyRequest({ + ...options, + + path: apiPath, + fields: { npmName, path, pluginVersion }, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } + + update (options: OverrideCommandOptions & { + path?: string + npmName?: string + }) { + const { npmName, path } = options + const apiPath = '/api/v1/plugins/update' + + return this.postBodyRequest({ + ...options, + + path: apiPath, + fields: { npmName, path }, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } + + uninstall (options: OverrideCommandOptions & { + npmName: string + }) { + const { npmName } = options + const apiPath = '/api/v1/plugins/uninstall' + + return this.postBodyRequest({ + ...options, + + path: apiPath, + fields: { npmName }, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } + + getCSS (options: OverrideCommandOptions = {}) { + const path = '/plugins/global.css' + + return this.getRequestText({ + ...options, + + path, + implicitToken: false, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } + + getExternalAuth (options: OverrideCommandOptions & { + npmName: string + npmVersion: string + authName: string + query?: any + }) { + const { npmName, npmVersion, authName, query } = options + + const path = '/plugins/' + npmName + '/' + npmVersion + '/auth/' + authName + + return this.getRequest({ + ...options, + + path, + query, + implicitToken: false, + defaultExpectedStatus: HttpStatusCode.OK_200, + redirects: 0 + }) + } + + updatePackageJSON (npmName: string, json: any) { + const path = this.getPackageJSONPath(npmName) + + return writeJSON(path, json) + } + + getPackageJSON (npmName: string): Promise { + const path = this.getPackageJSONPath(npmName) + + return readJSON(path) + } + + private getPackageJSONPath (npmName: string) { + return this.server.servers.buildDirectory(join('plugins', 'node_modules', npmName, 'package.json')) + } +} diff --git a/packages/server-commands/src/server/redundancy-command.ts b/packages/server-commands/src/server/redundancy-command.ts new file mode 100644 index 000000000..a0ec3e80e --- /dev/null +++ b/packages/server-commands/src/server/redundancy-command.ts @@ -0,0 +1,80 @@ +import { HttpStatusCode, ResultList, VideoRedundanciesTarget, VideoRedundancy } from '@peertube/peertube-models' +import { AbstractCommand, OverrideCommandOptions } from '../shared/index.js' + +export class RedundancyCommand extends AbstractCommand { + + updateRedundancy (options: OverrideCommandOptions & { + host: string + redundancyAllowed: boolean + }) { + const { host, redundancyAllowed } = options + const path = '/api/v1/server/redundancy/' + host + + return this.putBodyRequest({ + ...options, + + path, + fields: { redundancyAllowed }, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } + + listVideos (options: OverrideCommandOptions & { + target: VideoRedundanciesTarget + start?: number + count?: number + sort?: string + }) { + const path = '/api/v1/server/redundancy/videos' + + const { target, start, count, sort } = options + + return this.getRequestBody>({ + ...options, + + path, + + query: { + start: start ?? 0, + count: count ?? 5, + sort: sort ?? 'name', + target + }, + + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } + + addVideo (options: OverrideCommandOptions & { + videoId: number + }) { + const path = '/api/v1/server/redundancy/videos' + const { videoId } = options + + return this.postBodyRequest({ + ...options, + + path, + fields: { videoId }, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } + + removeVideo (options: OverrideCommandOptions & { + redundancyId: number + }) { + const { redundancyId } = options + const path = '/api/v1/server/redundancy/videos/' + redundancyId + + return this.deleteRequest({ + ...options, + + path, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } +} diff --git a/packages/server-commands/src/server/server.ts b/packages/server-commands/src/server/server.ts new file mode 100644 index 000000000..57a897c17 --- /dev/null +++ b/packages/server-commands/src/server/server.ts @@ -0,0 +1,451 @@ +import { ChildProcess, fork } from 'child_process' +import { copy } from 'fs-extra/esm' +import { join } from 'path' +import { randomInt } from '@peertube/peertube-core-utils' +import { Video, VideoChannel, VideoChannelSync, VideoCreateResult, VideoDetails } from '@peertube/peertube-models' +import { parallelTests, root } from '@peertube/peertube-node-utils' +import { BulkCommand } from '../bulk/index.js' +import { CLICommand } from '../cli/index.js' +import { CustomPagesCommand } from '../custom-pages/index.js' +import { FeedCommand } from '../feeds/index.js' +import { LogsCommand } from '../logs/index.js' +import { AbusesCommand } from '../moderation/index.js' +import { OverviewsCommand } from '../overviews/index.js' +import { RunnerJobsCommand, RunnerRegistrationTokensCommand, RunnersCommand } from '../runners/index.js' +import { SearchCommand } from '../search/index.js' +import { SocketIOCommand } from '../socket/index.js' +import { + AccountsCommand, + BlocklistCommand, + LoginCommand, + NotificationsCommand, + RegistrationsCommand, + SubscriptionsCommand, + TwoFactorCommand, + UsersCommand +} from '../users/index.js' +import { + BlacklistCommand, + CaptionsCommand, + ChangeOwnershipCommand, + ChannelsCommand, + ChannelSyncsCommand, + CommentsCommand, + HistoryCommand, + ImportsCommand, + LiveCommand, + PlaylistsCommand, + ServicesCommand, + StoryboardCommand, + StreamingPlaylistsCommand, + VideoPasswordsCommand, + VideosCommand, + VideoStatsCommand, + VideoStudioCommand, + VideoTokenCommand, + ViewsCommand +} from '../videos/index.js' +import { ConfigCommand } from './config-command.js' +import { ContactFormCommand } from './contact-form-command.js' +import { DebugCommand } from './debug-command.js' +import { FollowsCommand } from './follows-command.js' +import { JobsCommand } from './jobs-command.js' +import { MetricsCommand } from './metrics-command.js' +import { PluginsCommand } from './plugins-command.js' +import { RedundancyCommand } from './redundancy-command.js' +import { ServersCommand } from './servers-command.js' +import { StatsCommand } from './stats-command.js' + +export type RunServerOptions = { + hideLogs?: boolean + nodeArgs?: string[] + peertubeArgs?: string[] + env?: { [ id: string ]: string } +} + +export class PeerTubeServer { + app?: ChildProcess + + url: string + host?: string + hostname?: string + port?: number + + rtmpPort?: number + rtmpsPort?: number + + parallel?: boolean + internalServerNumber: number + + serverNumber?: number + customConfigFile?: string + + store?: { + client?: { + id?: string + secret?: string + } + + user?: { + username: string + password: string + email?: string + } + + channel?: VideoChannel + videoChannelSync?: Partial + + video?: Video + videoCreated?: VideoCreateResult + videoDetails?: VideoDetails + + videos?: { id: number, uuid: string }[] + } + + accessToken?: string + refreshToken?: string + + bulk?: BulkCommand + cli?: CLICommand + customPage?: CustomPagesCommand + feed?: FeedCommand + logs?: LogsCommand + abuses?: AbusesCommand + overviews?: OverviewsCommand + search?: SearchCommand + contactForm?: ContactFormCommand + debug?: DebugCommand + follows?: FollowsCommand + jobs?: JobsCommand + metrics?: MetricsCommand + plugins?: PluginsCommand + redundancy?: RedundancyCommand + stats?: StatsCommand + config?: ConfigCommand + socketIO?: SocketIOCommand + accounts?: AccountsCommand + blocklist?: BlocklistCommand + subscriptions?: SubscriptionsCommand + live?: LiveCommand + services?: ServicesCommand + blacklist?: BlacklistCommand + captions?: CaptionsCommand + changeOwnership?: ChangeOwnershipCommand + playlists?: PlaylistsCommand + history?: HistoryCommand + imports?: ImportsCommand + channelSyncs?: ChannelSyncsCommand + streamingPlaylists?: StreamingPlaylistsCommand + channels?: ChannelsCommand + comments?: CommentsCommand + notifications?: NotificationsCommand + servers?: ServersCommand + login?: LoginCommand + users?: UsersCommand + videoStudio?: VideoStudioCommand + videos?: VideosCommand + videoStats?: VideoStatsCommand + views?: ViewsCommand + twoFactor?: TwoFactorCommand + videoToken?: VideoTokenCommand + registrations?: RegistrationsCommand + videoPasswords?: VideoPasswordsCommand + + storyboard?: StoryboardCommand + + runners?: RunnersCommand + runnerRegistrationTokens?: RunnerRegistrationTokensCommand + runnerJobs?: RunnerJobsCommand + + constructor (options: { serverNumber: number } | { url: string }) { + if ((options as any).url) { + this.setUrl((options as any).url) + } else { + this.setServerNumber((options as any).serverNumber) + } + + this.store = { + client: { + id: null, + secret: null + }, + user: { + username: null, + password: null + } + } + + this.assignCommands() + } + + setServerNumber (serverNumber: number) { + this.serverNumber = serverNumber + + this.parallel = parallelTests() + + this.internalServerNumber = this.parallel ? this.randomServer() : this.serverNumber + this.rtmpPort = this.parallel ? this.randomRTMP() : 1936 + this.rtmpsPort = this.parallel ? this.randomRTMP() : 1937 + this.port = 9000 + this.internalServerNumber + + this.url = `http://127.0.0.1:${this.port}` + this.host = `127.0.0.1:${this.port}` + this.hostname = '127.0.0.1' + } + + setUrl (url: string) { + const parsed = new URL(url) + + this.url = url + this.host = parsed.host + this.hostname = parsed.hostname + this.port = parseInt(parsed.port) + } + + getDirectoryPath (directoryName: string) { + const testDirectory = 'test' + this.internalServerNumber + + return join(root(), testDirectory, directoryName) + } + + async flushAndRun (configOverride?: object, options: RunServerOptions = {}) { + await ServersCommand.flushTests(this.internalServerNumber) + + return this.run(configOverride, options) + } + + async run (configOverrideArg?: any, options: RunServerOptions = {}) { + // These actions are async so we need to be sure that they have both been done + const serverRunString = { + 'HTTP server listening': false + } + const key = 'Database peertube_test' + this.internalServerNumber + ' is ready' + serverRunString[key] = false + + const regexps = { + client_id: 'Client id: (.+)', + client_secret: 'Client secret: (.+)', + user_username: 'Username: (.+)', + user_password: 'User password: (.+)' + } + + await this.assignCustomConfigFile() + + const configOverride = this.buildConfigOverride() + + if (configOverrideArg !== undefined) { + Object.assign(configOverride, configOverrideArg) + } + + // Share the environment + const env = { ...process.env } + env['NODE_ENV'] = 'test' + env['NODE_APP_INSTANCE'] = this.internalServerNumber.toString() + env['NODE_CONFIG'] = JSON.stringify(configOverride) + + if (options.env) { + Object.assign(env, options.env) + } + + const execArgv = options.nodeArgs || [] + // FIXME: too slow :/ + // execArgv.push('--enable-source-maps') + + const forkOptions = { + silent: true, + env, + detached: false, + execArgv + } + + const peertubeArgs = options.peertubeArgs || [] + + return new Promise((res, rej) => { + const self = this + let aggregatedLogs = '' + + this.app = fork(join(root(), 'dist', 'server.js'), peertubeArgs, forkOptions) + + const onPeerTubeExit = () => rej(new Error('Process exited:\n' + aggregatedLogs)) + const onParentExit = () => { + if (!this.app?.pid) return + + try { + process.kill(self.app.pid) + } catch { /* empty */ } + } + + this.app.on('exit', onPeerTubeExit) + process.on('exit', onParentExit) + + this.app.stdout.on('data', function onStdout (data) { + let dontContinue = false + + const log: string = data.toString() + aggregatedLogs += log + + // Capture things if we want to + for (const key of Object.keys(regexps)) { + const regexp = regexps[key] + const matches = log.match(regexp) + if (matches !== null) { + if (key === 'client_id') self.store.client.id = matches[1] + else if (key === 'client_secret') self.store.client.secret = matches[1] + else if (key === 'user_username') self.store.user.username = matches[1] + else if (key === 'user_password') self.store.user.password = matches[1] + } + } + + // Check if all required sentences are here + for (const key of Object.keys(serverRunString)) { + if (log.includes(key)) serverRunString[key] = true + if (serverRunString[key] === false) dontContinue = true + } + + // If no, there is maybe one thing not already initialized (client/user credentials generation...) + if (dontContinue === true) return + + if (options.hideLogs === false) { + console.log(log) + } else { + process.removeListener('exit', onParentExit) + self.app.stdout.removeListener('data', onStdout) + self.app.removeListener('exit', onPeerTubeExit) + } + + res() + }) + }) + } + + kill () { + if (!this.app) return Promise.resolve() + + process.kill(this.app.pid) + + this.app = null + + return Promise.resolve() + } + + private randomServer () { + const low = 2500 + const high = 10000 + + return randomInt(low, high) + } + + private randomRTMP () { + const low = 1900 + const high = 2100 + + return randomInt(low, high) + } + + private async assignCustomConfigFile () { + if (this.internalServerNumber === this.serverNumber) return + + const basePath = join(root(), 'config') + + const tmpConfigFile = join(basePath, `test-${this.internalServerNumber}.yaml`) + await copy(join(basePath, `test-${this.serverNumber}.yaml`), tmpConfigFile) + + this.customConfigFile = tmpConfigFile + } + + private buildConfigOverride () { + if (!this.parallel) return {} + + return { + listen: { + port: this.port + }, + webserver: { + port: this.port + }, + database: { + suffix: '_test' + this.internalServerNumber + }, + storage: { + tmp: this.getDirectoryPath('tmp') + '/', + tmp_persistent: this.getDirectoryPath('tmp-persistent') + '/', + bin: this.getDirectoryPath('bin') + '/', + avatars: this.getDirectoryPath('avatars') + '/', + web_videos: this.getDirectoryPath('web-videos') + '/', + streaming_playlists: this.getDirectoryPath('streaming-playlists') + '/', + redundancy: this.getDirectoryPath('redundancy') + '/', + logs: this.getDirectoryPath('logs') + '/', + previews: this.getDirectoryPath('previews') + '/', + thumbnails: this.getDirectoryPath('thumbnails') + '/', + storyboards: this.getDirectoryPath('storyboards') + '/', + torrents: this.getDirectoryPath('torrents') + '/', + captions: this.getDirectoryPath('captions') + '/', + cache: this.getDirectoryPath('cache') + '/', + plugins: this.getDirectoryPath('plugins') + '/', + well_known: this.getDirectoryPath('well-known') + '/' + }, + admin: { + email: `admin${this.internalServerNumber}@example.com` + }, + live: { + rtmp: { + port: this.rtmpPort + } + } + } + } + + private assignCommands () { + this.bulk = new BulkCommand(this) + this.cli = new CLICommand(this) + this.customPage = new CustomPagesCommand(this) + this.feed = new FeedCommand(this) + this.logs = new LogsCommand(this) + this.abuses = new AbusesCommand(this) + this.overviews = new OverviewsCommand(this) + this.search = new SearchCommand(this) + this.contactForm = new ContactFormCommand(this) + this.debug = new DebugCommand(this) + this.follows = new FollowsCommand(this) + this.jobs = new JobsCommand(this) + this.metrics = new MetricsCommand(this) + this.plugins = new PluginsCommand(this) + this.redundancy = new RedundancyCommand(this) + this.stats = new StatsCommand(this) + this.config = new ConfigCommand(this) + this.socketIO = new SocketIOCommand(this) + this.accounts = new AccountsCommand(this) + this.blocklist = new BlocklistCommand(this) + this.subscriptions = new SubscriptionsCommand(this) + this.live = new LiveCommand(this) + this.services = new ServicesCommand(this) + this.blacklist = new BlacklistCommand(this) + this.captions = new CaptionsCommand(this) + this.changeOwnership = new ChangeOwnershipCommand(this) + this.playlists = new PlaylistsCommand(this) + this.history = new HistoryCommand(this) + this.imports = new ImportsCommand(this) + this.channelSyncs = new ChannelSyncsCommand(this) + this.streamingPlaylists = new StreamingPlaylistsCommand(this) + this.channels = new ChannelsCommand(this) + this.comments = new CommentsCommand(this) + this.notifications = new NotificationsCommand(this) + this.servers = new ServersCommand(this) + this.login = new LoginCommand(this) + this.users = new UsersCommand(this) + this.videos = new VideosCommand(this) + this.videoStudio = new VideoStudioCommand(this) + this.videoStats = new VideoStatsCommand(this) + this.views = new ViewsCommand(this) + this.twoFactor = new TwoFactorCommand(this) + this.videoToken = new VideoTokenCommand(this) + this.registrations = new RegistrationsCommand(this) + + this.storyboard = new StoryboardCommand(this) + + this.runners = new RunnersCommand(this) + this.runnerRegistrationTokens = new RunnerRegistrationTokensCommand(this) + this.runnerJobs = new RunnerJobsCommand(this) + this.videoPasswords = new VideoPasswordsCommand(this) + } +} diff --git a/packages/server-commands/src/server/servers-command.ts b/packages/server-commands/src/server/servers-command.ts new file mode 100644 index 000000000..0b722b62f --- /dev/null +++ b/packages/server-commands/src/server/servers-command.ts @@ -0,0 +1,104 @@ +import { exec } from 'child_process' +import { copy, ensureDir, remove } from 'fs-extra/esm' +import { readdir, readFile } from 'fs/promises' +import { basename, join } from 'path' +import { wait } from '@peertube/peertube-core-utils' +import { HttpStatusCode } from '@peertube/peertube-models' +import { getFileSize, isGithubCI, root } from '@peertube/peertube-node-utils' +import { AbstractCommand, OverrideCommandOptions } from '../shared/index.js' + +export class ServersCommand extends AbstractCommand { + + static flushTests (internalServerNumber: number) { + return new Promise((res, rej) => { + const suffix = ` -- ${internalServerNumber}` + + return exec('npm run clean:server:test' + suffix, (err, _stdout, stderr) => { + if (err || stderr) return rej(err || new Error(stderr)) + + return res() + }) + }) + } + + ping (options: OverrideCommandOptions = {}) { + return this.getRequestBody({ + ...options, + + path: '/api/v1/ping', + implicitToken: false, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } + + cleanupTests () { + const promises: Promise[] = [] + + const saveGithubLogsIfNeeded = async () => { + if (!isGithubCI()) return + + await ensureDir('artifacts') + + const origin = this.buildDirectory('logs/peertube.log') + const destname = `peertube-${this.server.internalServerNumber}.log` + console.log('Saving logs %s.', destname) + + await copy(origin, join('artifacts', destname)) + } + + if (this.server.parallel) { + const promise = saveGithubLogsIfNeeded() + .then(() => ServersCommand.flushTests(this.server.internalServerNumber)) + + promises.push(promise) + } + + if (this.server.customConfigFile) { + promises.push(remove(this.server.customConfigFile)) + } + + return promises + } + + async waitUntilLog (str: string, count = 1, strictCount = true) { + const logfile = this.buildDirectory('logs/peertube.log') + + while (true) { + const buf = await readFile(logfile) + + const matches = buf.toString().match(new RegExp(str, 'g')) + if (matches && matches.length === count) return + if (matches && strictCount === false && matches.length >= count) return + + await wait(1000) + } + } + + buildDirectory (directory: string) { + return join(root(), 'test' + this.server.internalServerNumber, directory) + } + + async countFiles (directory: string) { + const files = await readdir(this.buildDirectory(directory)) + + return files.length + } + + buildWebVideoFilePath (fileUrl: string) { + return this.buildDirectory(join('web-videos', basename(fileUrl))) + } + + buildFragmentedFilePath (videoUUID: string, fileUrl: string) { + return this.buildDirectory(join('streaming-playlists', 'hls', videoUUID, basename(fileUrl))) + } + + getLogContent () { + return readFile(this.buildDirectory('logs/peertube.log')) + } + + async getServerFileSize (subPath: string) { + const path = this.server.servers.buildDirectory(subPath) + + return getFileSize(path) + } +} diff --git a/packages/server-commands/src/server/servers.ts b/packages/server-commands/src/server/servers.ts new file mode 100644 index 000000000..caf9866e1 --- /dev/null +++ b/packages/server-commands/src/server/servers.ts @@ -0,0 +1,68 @@ +import { ensureDir } from 'fs-extra/esm' +import { isGithubCI } from '@peertube/peertube-node-utils' +import { PeerTubeServer, RunServerOptions } from './server.js' + +async function createSingleServer (serverNumber: number, configOverride?: object, options: RunServerOptions = {}) { + const server = new PeerTubeServer({ serverNumber }) + + await server.flushAndRun(configOverride, options) + + return server +} + +function createMultipleServers (totalServers: number, configOverride?: object, options: RunServerOptions = {}) { + const serverPromises: Promise[] = [] + + for (let i = 1; i <= totalServers; i++) { + serverPromises.push(createSingleServer(i, configOverride, options)) + } + + return Promise.all(serverPromises) +} + +function killallServers (servers: PeerTubeServer[]) { + return Promise.all(servers.map(s => s.kill())) +} + +async function cleanupTests (servers: PeerTubeServer[]) { + await killallServers(servers) + + if (isGithubCI()) { + await ensureDir('artifacts') + } + + let p: Promise[] = [] + for (const server of servers) { + p = p.concat(server.servers.cleanupTests()) + } + + return Promise.all(p) +} + +function getServerImportConfig (mode: 'youtube-dl' | 'yt-dlp') { + return { + import: { + videos: { + http: { + youtube_dl_release: { + url: mode === 'youtube-dl' + ? 'https://yt-dl.org/downloads/latest/youtube-dl' + : 'https://api.github.com/repos/yt-dlp/yt-dlp/releases', + + name: mode + } + } + } + } + } +} + +// --------------------------------------------------------------------------- + +export { + createSingleServer, + createMultipleServers, + cleanupTests, + killallServers, + getServerImportConfig +} diff --git a/packages/server-commands/src/server/stats-command.ts b/packages/server-commands/src/server/stats-command.ts new file mode 100644 index 000000000..80acd7bdc --- /dev/null +++ b/packages/server-commands/src/server/stats-command.ts @@ -0,0 +1,25 @@ +import { HttpStatusCode, ServerStats } from '@peertube/peertube-models' +import { AbstractCommand, OverrideCommandOptions } from '../shared/index.js' + +export class StatsCommand extends AbstractCommand { + + get (options: OverrideCommandOptions & { + useCache?: boolean // default false + } = {}) { + const { useCache = false } = options + const path = '/api/v1/server/stats' + + const query = { + t: useCache ? undefined : new Date().getTime() + } + + return this.getRequestBody({ + ...options, + + path, + query, + implicitToken: false, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } +} -- cgit v1.2.3