From 3a4992633ee62d5edfbb484d9c6bcb3cf158489d Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Mon, 31 Jul 2023 14:34:36 +0200 Subject: Migrate server to ESM Sorry for the very big commit that may lead to git log issues and merge conflicts, but it's a major step forward: * Server can be faster at startup because imports() are async and we can easily lazy import big modules * Angular doesn't seem to support ES import (with .js extension), so we had to correctly organize peertube into a monorepo: * Use yarn workspace feature * Use typescript reference projects for dependencies * Shared projects have been moved into "packages", each one is now a node module (with a dedicated package.json/tsconfig.json) * server/tools have been moved into apps/ and is now a dedicated app bundled and published on NPM so users don't have to build peertube cli tools manually * server/tests have been moved into packages/ so we don't compile them every time we want to run the server * Use isolatedModule option: * Had to move from const enum to const (https://www.typescriptlang.org/docs/handbook/enums.html#objects-vs-enums) * Had to explictely specify "type" imports when used in decorators * Prefer tsx (that uses esbuild under the hood) instead of ts-node to load typescript files (tests with mocha or scripts): * To reduce test complexity as esbuild doesn't support decorator metadata, we only test server files that do not import server models * We still build tests files into js files for a faster CI * Remove unmaintained peertube CLI import script * Removed some barrels to speed up execution (less imports) --- shared/server-commands/server/config-command.ts | 576 --------------------- .../server-commands/server/contact-form-command.ts | 31 -- shared/server-commands/server/debug-command.ts | 33 -- shared/server-commands/server/follows-command.ts | 139 ----- shared/server-commands/server/follows.ts | 20 - shared/server-commands/server/index.ts | 15 - shared/server-commands/server/jobs-command.ts | 84 --- shared/server-commands/server/jobs.ts | 118 ----- shared/server-commands/server/metrics-command.ts | 18 - .../server/object-storage-command.ts | 165 ------ shared/server-commands/server/plugins-command.ts | 257 --------- .../server-commands/server/redundancy-command.ts | 80 --- shared/server-commands/server/server.ts | 450 ---------------- shared/server-commands/server/servers-command.ts | 103 ---- shared/server-commands/server/servers.ts | 68 --- shared/server-commands/server/stats-command.ts | 25 - 16 files changed, 2182 deletions(-) delete mode 100644 shared/server-commands/server/config-command.ts delete mode 100644 shared/server-commands/server/contact-form-command.ts delete mode 100644 shared/server-commands/server/debug-command.ts delete mode 100644 shared/server-commands/server/follows-command.ts delete mode 100644 shared/server-commands/server/follows.ts delete mode 100644 shared/server-commands/server/index.ts delete mode 100644 shared/server-commands/server/jobs-command.ts delete mode 100644 shared/server-commands/server/jobs.ts delete mode 100644 shared/server-commands/server/metrics-command.ts delete mode 100644 shared/server-commands/server/object-storage-command.ts delete mode 100644 shared/server-commands/server/plugins-command.ts delete mode 100644 shared/server-commands/server/redundancy-command.ts delete mode 100644 shared/server-commands/server/server.ts delete mode 100644 shared/server-commands/server/servers-command.ts delete mode 100644 shared/server-commands/server/servers.ts delete mode 100644 shared/server-commands/server/stats-command.ts (limited to 'shared/server-commands/server') diff --git a/shared/server-commands/server/config-command.ts b/shared/server-commands/server/config-command.ts deleted file mode 100644 index 5ee2fe021..000000000 --- a/shared/server-commands/server/config-command.ts +++ /dev/null @@ -1,576 +0,0 @@ -import { merge } from 'lodash' -import { About, CustomConfig, HttpStatusCode, ServerConfig } from '@shared/models' -import { DeepPartial } from '@shared/typescript-utils' -import { AbstractCommand, OverrideCommandOptions } from '../shared/abstract-command' - -export class ConfigCommand extends AbstractCommand { - - static getCustomConfigResolutions (enabled: boolean, with0p = false) { - return { - '0p': enabled && with0p, - '144p': enabled, - '240p': enabled, - '360p': enabled, - '480p': enabled, - '720p': enabled, - '1080p': enabled, - '1440p': enabled, - '2160p': enabled - } - } - - // --------------------------------------------------------------------------- - - static getEmailOverrideConfig (emailPort: number) { - return { - smtp: { - hostname: '127.0.0.1', - port: emailPort - } - } - } - - // --------------------------------------------------------------------------- - - enableSignup (requiresApproval: boolean, limit = -1) { - return this.updateExistingSubConfig({ - newConfig: { - signup: { - enabled: true, - requiresApproval, - limit - } - } - }) - } - - // --------------------------------------------------------------------------- - - disableImports () { - return this.setImportsEnabled(false) - } - - enableImports () { - return this.setImportsEnabled(true) - } - - private setImportsEnabled (enabled: boolean) { - return this.updateExistingSubConfig({ - newConfig: { - import: { - videos: { - http: { - enabled - }, - - torrent: { - enabled - } - } - } - } - }) - } - - // --------------------------------------------------------------------------- - - disableFileUpdate () { - return this.setFileUpdateEnabled(false) - } - - enableFileUpdate () { - return this.setFileUpdateEnabled(true) - } - - private setFileUpdateEnabled (enabled: boolean) { - return this.updateExistingSubConfig({ - newConfig: { - videoFile: { - update: { - enabled - } - } - } - }) - } - - // --------------------------------------------------------------------------- - - enableChannelSync () { - return this.setChannelSyncEnabled(true) - } - - disableChannelSync () { - return this.setChannelSyncEnabled(false) - } - - private setChannelSyncEnabled (enabled: boolean) { - return this.updateExistingSubConfig({ - newConfig: { - import: { - videoChannelSynchronization: { - enabled - } - } - } - }) - } - - // --------------------------------------------------------------------------- - - enableLive (options: { - allowReplay?: boolean - transcoding?: boolean - resolutions?: 'min' | 'max' // Default max - } = {}) { - const { allowReplay, transcoding, resolutions = 'max' } = options - - return this.updateExistingSubConfig({ - newConfig: { - live: { - enabled: true, - allowReplay: allowReplay ?? true, - transcoding: { - enabled: transcoding ?? true, - resolutions: ConfigCommand.getCustomConfigResolutions(resolutions === 'max') - } - } - } - }) - } - - disableTranscoding () { - return this.updateExistingSubConfig({ - newConfig: { - transcoding: { - enabled: false - }, - videoStudio: { - enabled: false - } - } - }) - } - - enableTranscoding (options: { - webVideo?: boolean // default true - hls?: boolean // default true - with0p?: boolean // default false - } = {}) { - const { webVideo = true, hls = true, with0p = false } = options - - return this.updateExistingSubConfig({ - newConfig: { - transcoding: { - enabled: true, - - allowAudioFiles: true, - allowAdditionalExtensions: true, - - resolutions: ConfigCommand.getCustomConfigResolutions(true, with0p), - - webVideos: { - enabled: webVideo - }, - hls: { - enabled: hls - } - } - } - }) - } - - enableMinimumTranscoding (options: { - webVideo?: boolean // default true - hls?: boolean // default true - } = {}) { - const { webVideo = true, hls = true } = options - - return this.updateExistingSubConfig({ - newConfig: { - transcoding: { - enabled: true, - - allowAudioFiles: true, - allowAdditionalExtensions: true, - - resolutions: { - ...ConfigCommand.getCustomConfigResolutions(false), - - '240p': true - }, - - webVideos: { - enabled: webVideo - }, - hls: { - enabled: hls - } - } - } - }) - } - - enableRemoteTranscoding () { - return this.updateExistingSubConfig({ - newConfig: { - transcoding: { - remoteRunners: { - enabled: true - } - }, - live: { - transcoding: { - remoteRunners: { - enabled: true - } - } - } - } - }) - } - - enableRemoteStudio () { - return this.updateExistingSubConfig({ - newConfig: { - videoStudio: { - remoteRunners: { - enabled: true - } - } - } - }) - } - - // --------------------------------------------------------------------------- - - enableStudio () { - return this.updateExistingSubConfig({ - newConfig: { - videoStudio: { - enabled: true - } - } - }) - } - - // --------------------------------------------------------------------------- - - getConfig (options: OverrideCommandOptions = {}) { - const path = '/api/v1/config' - - return this.getRequestBody({ - ...options, - - path, - implicitToken: false, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } - - async getIndexHTMLConfig (options: OverrideCommandOptions = {}) { - const text = await this.getRequestText({ - ...options, - - path: '/', - implicitToken: false, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - - const match = text.match('') - - // We parse the string twice, first to extract the string and then to extract the JSON - return JSON.parse(JSON.parse(match[1])) as ServerConfig - } - - getAbout (options: OverrideCommandOptions = {}) { - const path = '/api/v1/config/about' - - return this.getRequestBody({ - ...options, - - path, - implicitToken: false, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } - - getCustomConfig (options: OverrideCommandOptions = {}) { - const path = '/api/v1/config/custom' - - return this.getRequestBody({ - ...options, - - path, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } - - updateCustomConfig (options: OverrideCommandOptions & { - newCustomConfig: CustomConfig - }) { - const path = '/api/v1/config/custom' - - return this.putBodyRequest({ - ...options, - - path, - fields: options.newCustomConfig, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } - - deleteCustomConfig (options: OverrideCommandOptions = {}) { - const path = '/api/v1/config/custom' - - return this.deleteRequest({ - ...options, - - path, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } - - async updateExistingSubConfig (options: OverrideCommandOptions & { - newConfig: DeepPartial - }) { - const existing = await this.getCustomConfig({ ...options, expectedStatus: HttpStatusCode.OK_200 }) - - return this.updateCustomConfig({ ...options, newCustomConfig: merge({}, existing, options.newConfig) }) - } - - updateCustomSubConfig (options: OverrideCommandOptions & { - newConfig: DeepPartial - }) { - const newCustomConfig: CustomConfig = { - instance: { - name: 'PeerTube updated', - shortDescription: 'my short description', - description: 'my super description', - terms: 'my super terms', - codeOfConduct: 'my super coc', - - creationReason: 'my super creation reason', - moderationInformation: 'my super moderation information', - administrator: 'Kuja', - maintenanceLifetime: 'forever', - businessModel: 'my super business model', - hardwareInformation: '2vCore 3GB RAM', - - languages: [ 'en', 'es' ], - categories: [ 1, 2 ], - - isNSFW: true, - defaultNSFWPolicy: 'blur', - - defaultClientRoute: '/videos/recently-added', - - customizations: { - javascript: 'alert("coucou")', - css: 'body { background-color: red; }' - } - }, - theme: { - default: 'default' - }, - services: { - twitter: { - username: '@MySuperUsername', - whitelisted: true - } - }, - client: { - videos: { - miniature: { - preferAuthorDisplayName: false - } - }, - menu: { - login: { - redirectOnSingleExternalAuth: false - } - } - }, - cache: { - previews: { - size: 2 - }, - captions: { - size: 3 - }, - torrents: { - size: 4 - }, - storyboards: { - size: 5 - } - }, - signup: { - enabled: false, - limit: 5, - requiresApproval: true, - requiresEmailVerification: false, - minimumAge: 16 - }, - admin: { - email: 'superadmin1@example.com' - }, - contactForm: { - enabled: true - }, - user: { - history: { - videos: { - enabled: true - } - }, - videoQuota: 5242881, - videoQuotaDaily: 318742 - }, - videoChannels: { - maxPerUser: 20 - }, - transcoding: { - enabled: true, - remoteRunners: { - enabled: false - }, - allowAdditionalExtensions: true, - allowAudioFiles: true, - threads: 1, - concurrency: 3, - profile: 'default', - resolutions: { - '0p': false, - '144p': false, - '240p': false, - '360p': true, - '480p': true, - '720p': false, - '1080p': false, - '1440p': false, - '2160p': false - }, - alwaysTranscodeOriginalResolution: true, - webVideos: { - enabled: true - }, - hls: { - enabled: false - } - }, - live: { - enabled: true, - allowReplay: false, - latencySetting: { - enabled: false - }, - maxDuration: -1, - maxInstanceLives: -1, - maxUserLives: 50, - transcoding: { - enabled: true, - remoteRunners: { - enabled: false - }, - threads: 4, - profile: 'default', - resolutions: { - '144p': true, - '240p': true, - '360p': true, - '480p': true, - '720p': true, - '1080p': true, - '1440p': true, - '2160p': true - }, - alwaysTranscodeOriginalResolution: true - } - }, - videoStudio: { - enabled: false, - remoteRunners: { - enabled: false - } - }, - videoFile: { - update: { - enabled: false - } - }, - import: { - videos: { - concurrency: 3, - http: { - enabled: false - }, - torrent: { - enabled: false - } - }, - videoChannelSynchronization: { - enabled: false, - maxPerUser: 10 - } - }, - trending: { - videos: { - algorithms: { - enabled: [ 'hot', 'most-viewed', 'most-liked' ], - default: 'hot' - } - } - }, - autoBlacklist: { - videos: { - ofUsers: { - enabled: false - } - } - }, - followers: { - instance: { - enabled: true, - manualApproval: false - } - }, - followings: { - instance: { - autoFollowBack: { - enabled: false - }, - autoFollowIndex: { - indexUrl: 'https://instances.joinpeertube.org/api/v1/instances/hosts', - enabled: false - } - } - }, - broadcastMessage: { - enabled: true, - level: 'warning', - message: 'hello', - dismissable: true - }, - search: { - remoteUri: { - users: true, - anonymous: true - }, - searchIndex: { - enabled: true, - url: 'https://search.joinpeertube.org', - disableLocalSearch: true, - isDefaultSearch: true - } - } - } - - merge(newCustomConfig, options.newConfig) - - return this.updateCustomConfig({ ...options, newCustomConfig }) - } -} diff --git a/shared/server-commands/server/contact-form-command.ts b/shared/server-commands/server/contact-form-command.ts deleted file mode 100644 index 0e8fd6d84..000000000 --- a/shared/server-commands/server/contact-form-command.ts +++ /dev/null @@ -1,31 +0,0 @@ -import { HttpStatusCode } from '@shared/models' -import { ContactForm } from '../../models/server' -import { AbstractCommand, OverrideCommandOptions } from '../shared' - -export class ContactFormCommand extends AbstractCommand { - - send (options: OverrideCommandOptions & { - fromEmail: string - fromName: string - subject: string - body: string - }) { - const path = '/api/v1/server/contact' - - const body: ContactForm = { - fromEmail: options.fromEmail, - fromName: options.fromName, - subject: options.subject, - body: options.body - } - - return this.postBodyRequest({ - ...options, - - path, - fields: body, - implicitToken: false, - defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - } -} diff --git a/shared/server-commands/server/debug-command.ts b/shared/server-commands/server/debug-command.ts deleted file mode 100644 index 3c5a785bb..000000000 --- a/shared/server-commands/server/debug-command.ts +++ /dev/null @@ -1,33 +0,0 @@ -import { Debug, HttpStatusCode, SendDebugCommand } from '@shared/models' -import { AbstractCommand, OverrideCommandOptions } from '../shared' - -export class DebugCommand extends AbstractCommand { - - getDebug (options: OverrideCommandOptions = {}) { - const path = '/api/v1/server/debug' - - return this.getRequestBody({ - ...options, - - path, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } - - sendCommand (options: OverrideCommandOptions & { - body: SendDebugCommand - }) { - const { body } = options - const path = '/api/v1/server/debug/run-command' - - return this.postBodyRequest({ - ...options, - - path, - fields: body, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - } -} diff --git a/shared/server-commands/server/follows-command.ts b/shared/server-commands/server/follows-command.ts deleted file mode 100644 index 496e11df1..000000000 --- a/shared/server-commands/server/follows-command.ts +++ /dev/null @@ -1,139 +0,0 @@ -import { pick } from '@shared/core-utils' -import { ActivityPubActorType, ActorFollow, FollowState, HttpStatusCode, ResultList, ServerFollowCreate } from '@shared/models' -import { AbstractCommand, OverrideCommandOptions } from '../shared' -import { PeerTubeServer } from './server' - -export class FollowsCommand extends AbstractCommand { - - getFollowers (options: OverrideCommandOptions & { - start?: number - count?: number - sort?: string - search?: string - actorType?: ActivityPubActorType - state?: FollowState - } = {}) { - const path = '/api/v1/server/followers' - - const query = pick(options, [ 'start', 'count', 'sort', 'search', 'state', 'actorType' ]) - - return this.getRequestBody>({ - ...options, - - path, - query, - implicitToken: false, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } - - getFollowings (options: OverrideCommandOptions & { - start?: number - count?: number - sort?: string - search?: string - actorType?: ActivityPubActorType - state?: FollowState - } = {}) { - const path = '/api/v1/server/following' - - const query = pick(options, [ 'start', 'count', 'sort', 'search', 'state', 'actorType' ]) - - return this.getRequestBody>({ - ...options, - - path, - query, - implicitToken: false, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } - - follow (options: OverrideCommandOptions & { - hosts?: string[] - handles?: string[] - }) { - const path = '/api/v1/server/following' - - const fields: ServerFollowCreate = {} - - if (options.hosts) { - fields.hosts = options.hosts.map(f => f.replace(/^http:\/\//, '')) - } - - if (options.handles) { - fields.handles = options.handles - } - - return this.postBodyRequest({ - ...options, - - path, - fields, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - } - - async unfollow (options: OverrideCommandOptions & { - target: PeerTubeServer | string - }) { - const { target } = options - - const handle = typeof target === 'string' - ? target - : target.host - - const path = '/api/v1/server/following/' + handle - - return this.deleteRequest({ - ...options, - - path, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - } - - acceptFollower (options: OverrideCommandOptions & { - follower: string - }) { - const path = '/api/v1/server/followers/' + options.follower + '/accept' - - return this.postBodyRequest({ - ...options, - - path, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - } - - rejectFollower (options: OverrideCommandOptions & { - follower: string - }) { - const path = '/api/v1/server/followers/' + options.follower + '/reject' - - return this.postBodyRequest({ - ...options, - - path, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - } - - removeFollower (options: OverrideCommandOptions & { - follower: PeerTubeServer - }) { - const path = '/api/v1/server/followers/peertube@' + options.follower.host - - return this.deleteRequest({ - ...options, - - path, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - } -} diff --git a/shared/server-commands/server/follows.ts b/shared/server-commands/server/follows.ts deleted file mode 100644 index 698238f29..000000000 --- a/shared/server-commands/server/follows.ts +++ /dev/null @@ -1,20 +0,0 @@ -import { waitJobs } from './jobs' -import { PeerTubeServer } from './server' - -async function doubleFollow (server1: PeerTubeServer, server2: PeerTubeServer) { - await Promise.all([ - server1.follows.follow({ hosts: [ server2.url ] }), - server2.follows.follow({ hosts: [ server1.url ] }) - ]) - - // Wait request propagation - await waitJobs([ server1, server2 ]) - - return true -} - -// --------------------------------------------------------------------------- - -export { - doubleFollow -} diff --git a/shared/server-commands/server/index.ts b/shared/server-commands/server/index.ts deleted file mode 100644 index 9a2fbf8d3..000000000 --- a/shared/server-commands/server/index.ts +++ /dev/null @@ -1,15 +0,0 @@ -export * from './config-command' -export * from './contact-form-command' -export * from './debug-command' -export * from './follows-command' -export * from './follows' -export * from './jobs' -export * from './jobs-command' -export * from './metrics-command' -export * from './object-storage-command' -export * from './plugins-command' -export * from './redundancy-command' -export * from './server' -export * from './servers-command' -export * from './servers' -export * from './stats-command' diff --git a/shared/server-commands/server/jobs-command.ts b/shared/server-commands/server/jobs-command.ts deleted file mode 100644 index b8790ea00..000000000 --- a/shared/server-commands/server/jobs-command.ts +++ /dev/null @@ -1,84 +0,0 @@ -import { pick } from '@shared/core-utils' -import { HttpStatusCode, Job, JobState, JobType, ResultList } from '@shared/models' -import { AbstractCommand, OverrideCommandOptions } from '../shared' - -export class JobsCommand extends AbstractCommand { - - async getLatest (options: OverrideCommandOptions & { - jobType: JobType - }) { - const { data } = await this.list({ ...options, start: 0, count: 1, sort: '-createdAt' }) - - if (data.length === 0) return undefined - - return data[0] - } - - pauseJobQueue (options: OverrideCommandOptions = {}) { - const path = '/api/v1/jobs/pause' - - return this.postBodyRequest({ - ...options, - - path, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - } - - resumeJobQueue (options: OverrideCommandOptions = {}) { - const path = '/api/v1/jobs/resume' - - return this.postBodyRequest({ - ...options, - - path, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - } - - list (options: OverrideCommandOptions & { - state?: JobState - jobType?: JobType - start?: number - count?: number - sort?: string - } = {}) { - const path = this.buildJobsUrl(options.state) - - const query = pick(options, [ 'start', 'count', 'sort', 'jobType' ]) - - return this.getRequestBody>({ - ...options, - - path, - query, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } - - listFailed (options: OverrideCommandOptions & { - jobType?: JobType - }) { - const path = this.buildJobsUrl('failed') - - return this.getRequestBody>({ - ...options, - - path, - query: { start: 0, count: 50 }, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } - - private buildJobsUrl (state?: JobState) { - let path = '/api/v1/jobs' - - if (state) path += '/' + state - - return path - } -} diff --git a/shared/server-commands/server/jobs.ts b/shared/server-commands/server/jobs.ts deleted file mode 100644 index 8f131fba4..000000000 --- a/shared/server-commands/server/jobs.ts +++ /dev/null @@ -1,118 +0,0 @@ - -import { expect } from 'chai' -import { wait } from '@shared/core-utils' -import { JobState, JobType, RunnerJobState } from '../../models' -import { PeerTubeServer } from './server' - -async function waitJobs ( - serversArg: PeerTubeServer[] | PeerTubeServer, - options: { - skipDelayed?: boolean // default false - runnerJobs?: boolean // default false - } = {} -) { - const { skipDelayed = false, runnerJobs = false } = options - - const pendingJobWait = process.env.NODE_PENDING_JOB_WAIT - ? parseInt(process.env.NODE_PENDING_JOB_WAIT, 10) - : 250 - - let servers: PeerTubeServer[] - - if (Array.isArray(serversArg) === false) servers = [ serversArg as PeerTubeServer ] - else servers = serversArg as PeerTubeServer[] - - const states: JobState[] = [ 'waiting', 'active' ] - if (!skipDelayed) states.push('delayed') - - const repeatableJobs: JobType[] = [ 'videos-views-stats', 'activitypub-cleaner' ] - let pendingRequests: boolean - - function tasksBuilder () { - const tasks: Promise[] = [] - - // Check if each server has pending request - for (const server of servers) { - if (process.env.DEBUG) console.log('Checking ' + server.url) - - for (const state of states) { - - const jobPromise = server.jobs.list({ - state, - start: 0, - count: 10, - sort: '-createdAt' - }).then(body => body.data) - .then(jobs => jobs.filter(j => !repeatableJobs.includes(j.type))) - .then(jobs => { - if (jobs.length !== 0) { - pendingRequests = true - - if (process.env.DEBUG) { - console.log(jobs) - } - } - }) - - tasks.push(jobPromise) - } - - const debugPromise = server.debug.getDebug() - .then(obj => { - if (obj.activityPubMessagesWaiting !== 0) { - pendingRequests = true - - if (process.env.DEBUG) { - console.log('AP messages waiting: ' + obj.activityPubMessagesWaiting) - } - } - }) - tasks.push(debugPromise) - - if (runnerJobs) { - const runnerJobsPromise = server.runnerJobs.list({ count: 100 }) - .then(({ data }) => { - for (const job of data) { - if (job.state.id !== RunnerJobState.COMPLETED) { - pendingRequests = true - - if (process.env.DEBUG) { - console.log(job) - } - } - } - }) - tasks.push(runnerJobsPromise) - } - } - - return tasks - } - - do { - pendingRequests = false - await Promise.all(tasksBuilder()) - - // Retry, in case of new jobs were created - if (pendingRequests === false) { - await wait(pendingJobWait) - await Promise.all(tasksBuilder()) - } - - if (pendingRequests) { - await wait(pendingJobWait) - } - } while (pendingRequests) -} - -async function expectNoFailedTranscodingJob (server: PeerTubeServer) { - const { data } = await server.jobs.listFailed({ jobType: 'video-transcoding' }) - expect(data).to.have.lengthOf(0) -} - -// --------------------------------------------------------------------------- - -export { - waitJobs, - expectNoFailedTranscodingJob -} diff --git a/shared/server-commands/server/metrics-command.ts b/shared/server-commands/server/metrics-command.ts deleted file mode 100644 index d22b4833d..000000000 --- a/shared/server-commands/server/metrics-command.ts +++ /dev/null @@ -1,18 +0,0 @@ -import { HttpStatusCode, PlaybackMetricCreate } from '@shared/models' -import { AbstractCommand, OverrideCommandOptions } from '../shared' - -export class MetricsCommand extends AbstractCommand { - - addPlaybackMetric (options: OverrideCommandOptions & { metrics: PlaybackMetricCreate }) { - const path = '/api/v1/metrics/playback' - - return this.postBodyRequest({ - ...options, - - path, - fields: options.metrics, - implicitToken: false, - defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - } -} diff --git a/shared/server-commands/server/object-storage-command.ts b/shared/server-commands/server/object-storage-command.ts deleted file mode 100644 index 6bb232c36..000000000 --- a/shared/server-commands/server/object-storage-command.ts +++ /dev/null @@ -1,165 +0,0 @@ -import { randomInt } from 'crypto' -import { HttpStatusCode } from '@shared/models' -import { makePostBodyRequest } from '../requests' - -export class ObjectStorageCommand { - static readonly DEFAULT_SCALEWAY_BUCKET = 'peertube-ci-test' - - private readonly bucketsCreated: string[] = [] - private readonly seed: number - - // --------------------------------------------------------------------------- - - constructor () { - this.seed = randomInt(0, 10000) - } - - static getMockCredentialsConfig () { - return { - access_key_id: 'AKIAIOSFODNN7EXAMPLE', - secret_access_key: 'wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY' - } - } - - static getMockEndpointHost () { - return 'localhost:9444' - } - - static getMockRegion () { - return 'us-east-1' - } - - getDefaultMockConfig () { - return { - object_storage: { - enabled: true, - endpoint: 'http://' + ObjectStorageCommand.getMockEndpointHost(), - region: ObjectStorageCommand.getMockRegion(), - - credentials: ObjectStorageCommand.getMockCredentialsConfig(), - - streaming_playlists: { - bucket_name: this.getMockStreamingPlaylistsBucketName() - }, - - web_videos: { - bucket_name: this.getMockWebVideosBucketName() - } - } - } - } - - getMockWebVideosBaseUrl () { - return `http://${this.getMockWebVideosBucketName()}.${ObjectStorageCommand.getMockEndpointHost()}/` - } - - getMockPlaylistBaseUrl () { - return `http://${this.getMockStreamingPlaylistsBucketName()}.${ObjectStorageCommand.getMockEndpointHost()}/` - } - - async prepareDefaultMockBuckets () { - await this.createMockBucket(this.getMockStreamingPlaylistsBucketName()) - await this.createMockBucket(this.getMockWebVideosBucketName()) - } - - async createMockBucket (name: string) { - this.bucketsCreated.push(name) - - await this.deleteMockBucket(name) - - await makePostBodyRequest({ - url: ObjectStorageCommand.getMockEndpointHost(), - path: '/ui/' + name + '?create', - expectedStatus: HttpStatusCode.TEMPORARY_REDIRECT_307 - }) - - await makePostBodyRequest({ - url: ObjectStorageCommand.getMockEndpointHost(), - path: '/ui/' + name + '?make-public', - expectedStatus: HttpStatusCode.TEMPORARY_REDIRECT_307 - }) - } - - async cleanupMock () { - for (const name of this.bucketsCreated) { - await this.deleteMockBucket(name) - } - } - - getMockStreamingPlaylistsBucketName (name = 'streaming-playlists') { - return this.getMockBucketName(name) - } - - getMockWebVideosBucketName (name = 'web-videos') { - return this.getMockBucketName(name) - } - - getMockBucketName (name: string) { - return `${this.seed}-${name}` - } - - private async deleteMockBucket (name: string) { - await makePostBodyRequest({ - url: ObjectStorageCommand.getMockEndpointHost(), - path: '/ui/' + name + '?delete', - expectedStatus: HttpStatusCode.TEMPORARY_REDIRECT_307 - }) - } - - // --------------------------------------------------------------------------- - - static getDefaultScalewayConfig (options: { - serverNumber: number - enablePrivateProxy?: boolean // default true - privateACL?: 'private' | 'public-read' // default 'private' - }) { - const { serverNumber, enablePrivateProxy = true, privateACL = 'private' } = options - - return { - object_storage: { - enabled: true, - endpoint: this.getScalewayEndpointHost(), - region: this.getScalewayRegion(), - - credentials: this.getScalewayCredentialsConfig(), - - upload_acl: { - private: privateACL - }, - - proxy: { - proxify_private_files: enablePrivateProxy - }, - - streaming_playlists: { - bucket_name: this.DEFAULT_SCALEWAY_BUCKET, - prefix: `test:server-${serverNumber}-streaming-playlists:` - }, - - web_videos: { - bucket_name: this.DEFAULT_SCALEWAY_BUCKET, - prefix: `test:server-${serverNumber}-web-videos:` - } - } - } - } - - static getScalewayCredentialsConfig () { - return { - access_key_id: process.env.OBJECT_STORAGE_SCALEWAY_KEY_ID, - secret_access_key: process.env.OBJECT_STORAGE_SCALEWAY_ACCESS_KEY - } - } - - static getScalewayEndpointHost () { - return 's3.fr-par.scw.cloud' - } - - static getScalewayRegion () { - return 'fr-par' - } - - static getScalewayBaseUrl () { - return `https://${this.DEFAULT_SCALEWAY_BUCKET}.${this.getScalewayEndpointHost()}/` - } -} diff --git a/shared/server-commands/server/plugins-command.ts b/shared/server-commands/server/plugins-command.ts deleted file mode 100644 index bb1277a7c..000000000 --- a/shared/server-commands/server/plugins-command.ts +++ /dev/null @@ -1,257 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ - -import { readJSON, writeJSON } from 'fs-extra' -import { join } from 'path' -import { root } from '@shared/core-utils' -import { - HttpStatusCode, - PeerTubePlugin, - PeerTubePluginIndex, - PeertubePluginIndexList, - PluginPackageJSON, - PluginTranslation, - PluginType, - PublicServerSetting, - RegisteredServerSettings, - ResultList -} from '@shared/models' -import { AbstractCommand, OverrideCommandOptions } from '../shared' - -export class PluginsCommand extends AbstractCommand { - - static getPluginTestPath (suffix = '') { - return join(root(), 'server', 'tests', 'fixtures', 'peertube-plugin-test' + suffix) - } - - list (options: OverrideCommandOptions & { - start?: number - count?: number - sort?: string - pluginType?: PluginType - uninstalled?: boolean - }) { - const { start, count, sort, pluginType, uninstalled } = options - const path = '/api/v1/plugins' - - return this.getRequestBody>({ - ...options, - - path, - query: { - start, - count, - sort, - pluginType, - uninstalled - }, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } - - listAvailable (options: OverrideCommandOptions & { - start?: number - count?: number - sort?: string - pluginType?: PluginType - currentPeerTubeEngine?: string - search?: string - expectedStatus?: HttpStatusCode - }) { - const { start, count, sort, pluginType, search, currentPeerTubeEngine } = options - const path = '/api/v1/plugins/available' - - const query: PeertubePluginIndexList = { - start, - count, - sort, - pluginType, - currentPeerTubeEngine, - search - } - - return this.getRequestBody>({ - ...options, - - path, - query, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } - - get (options: OverrideCommandOptions & { - npmName: string - }) { - const path = '/api/v1/plugins/' + options.npmName - - return this.getRequestBody({ - ...options, - - path, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } - - updateSettings (options: OverrideCommandOptions & { - npmName: string - settings: any - }) { - const { npmName, settings } = options - const path = '/api/v1/plugins/' + npmName + '/settings' - - return this.putBodyRequest({ - ...options, - - path, - fields: { settings }, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - } - - getRegisteredSettings (options: OverrideCommandOptions & { - npmName: string - }) { - const path = '/api/v1/plugins/' + options.npmName + '/registered-settings' - - return this.getRequestBody({ - ...options, - - path, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } - - getPublicSettings (options: OverrideCommandOptions & { - npmName: string - }) { - const { npmName } = options - const path = '/api/v1/plugins/' + npmName + '/public-settings' - - return this.getRequestBody({ - ...options, - - path, - implicitToken: false, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } - - getTranslations (options: OverrideCommandOptions & { - locale: string - }) { - const { locale } = options - const path = '/plugins/translations/' + locale + '.json' - - return this.getRequestBody({ - ...options, - - path, - implicitToken: false, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } - - install (options: OverrideCommandOptions & { - path?: string - npmName?: string - pluginVersion?: string - }) { - const { npmName, path, pluginVersion } = options - const apiPath = '/api/v1/plugins/install' - - return this.postBodyRequest({ - ...options, - - path: apiPath, - fields: { npmName, path, pluginVersion }, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } - - update (options: OverrideCommandOptions & { - path?: string - npmName?: string - }) { - const { npmName, path } = options - const apiPath = '/api/v1/plugins/update' - - return this.postBodyRequest({ - ...options, - - path: apiPath, - fields: { npmName, path }, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } - - uninstall (options: OverrideCommandOptions & { - npmName: string - }) { - const { npmName } = options - const apiPath = '/api/v1/plugins/uninstall' - - return this.postBodyRequest({ - ...options, - - path: apiPath, - fields: { npmName }, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - } - - getCSS (options: OverrideCommandOptions = {}) { - const path = '/plugins/global.css' - - return this.getRequestText({ - ...options, - - path, - implicitToken: false, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } - - getExternalAuth (options: OverrideCommandOptions & { - npmName: string - npmVersion: string - authName: string - query?: any - }) { - const { npmName, npmVersion, authName, query } = options - - const path = '/plugins/' + npmName + '/' + npmVersion + '/auth/' + authName - - return this.getRequest({ - ...options, - - path, - query, - implicitToken: false, - defaultExpectedStatus: HttpStatusCode.OK_200, - redirects: 0 - }) - } - - updatePackageJSON (npmName: string, json: any) { - const path = this.getPackageJSONPath(npmName) - - return writeJSON(path, json) - } - - getPackageJSON (npmName: string): Promise { - const path = this.getPackageJSONPath(npmName) - - return readJSON(path) - } - - private getPackageJSONPath (npmName: string) { - return this.server.servers.buildDirectory(join('plugins', 'node_modules', npmName, 'package.json')) - } -} diff --git a/shared/server-commands/server/redundancy-command.ts b/shared/server-commands/server/redundancy-command.ts deleted file mode 100644 index e7a8b3c29..000000000 --- a/shared/server-commands/server/redundancy-command.ts +++ /dev/null @@ -1,80 +0,0 @@ -import { HttpStatusCode, ResultList, VideoRedundanciesTarget, VideoRedundancy } from '@shared/models' -import { AbstractCommand, OverrideCommandOptions } from '../shared' - -export class RedundancyCommand extends AbstractCommand { - - updateRedundancy (options: OverrideCommandOptions & { - host: string - redundancyAllowed: boolean - }) { - const { host, redundancyAllowed } = options - const path = '/api/v1/server/redundancy/' + host - - return this.putBodyRequest({ - ...options, - - path, - fields: { redundancyAllowed }, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - } - - listVideos (options: OverrideCommandOptions & { - target: VideoRedundanciesTarget - start?: number - count?: number - sort?: string - }) { - const path = '/api/v1/server/redundancy/videos' - - const { target, start, count, sort } = options - - return this.getRequestBody>({ - ...options, - - path, - - query: { - start: start ?? 0, - count: count ?? 5, - sort: sort ?? 'name', - target - }, - - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } - - addVideo (options: OverrideCommandOptions & { - videoId: number - }) { - const path = '/api/v1/server/redundancy/videos' - const { videoId } = options - - return this.postBodyRequest({ - ...options, - - path, - fields: { videoId }, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - } - - removeVideo (options: OverrideCommandOptions & { - redundancyId: number - }) { - const { redundancyId } = options - const path = '/api/v1/server/redundancy/videos/' + redundancyId - - return this.deleteRequest({ - ...options, - - path, - implicitToken: true, - defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 - }) - } -} diff --git a/shared/server-commands/server/server.ts b/shared/server-commands/server/server.ts deleted file mode 100644 index 38568a890..000000000 --- a/shared/server-commands/server/server.ts +++ /dev/null @@ -1,450 +0,0 @@ -import { ChildProcess, fork } from 'child_process' -import { copy } from 'fs-extra' -import { join } from 'path' -import { parallelTests, randomInt, root } from '@shared/core-utils' -import { Video, VideoChannel, VideoChannelSync, VideoCreateResult, VideoDetails } from '@shared/models' -import { BulkCommand } from '../bulk' -import { CLICommand } from '../cli' -import { CustomPagesCommand } from '../custom-pages' -import { FeedCommand } from '../feeds' -import { LogsCommand } from '../logs' -import { AbusesCommand } from '../moderation' -import { OverviewsCommand } from '../overviews' -import { RunnerJobsCommand, RunnerRegistrationTokensCommand, RunnersCommand } from '../runners' -import { SearchCommand } from '../search' -import { SocketIOCommand } from '../socket' -import { - AccountsCommand, - BlocklistCommand, - LoginCommand, - NotificationsCommand, - RegistrationsCommand, - SubscriptionsCommand, - TwoFactorCommand, - UsersCommand -} from '../users' -import { - BlacklistCommand, - CaptionsCommand, - ChangeOwnershipCommand, - ChannelsCommand, - ChannelSyncsCommand, - HistoryCommand, - ImportsCommand, - LiveCommand, - VideoPasswordsCommand, - PlaylistsCommand, - ServicesCommand, - StoryboardCommand, - StreamingPlaylistsCommand, - VideosCommand, - VideoStudioCommand, - VideoTokenCommand, - ViewsCommand -} from '../videos' -import { CommentsCommand } from '../videos/comments-command' -import { VideoStatsCommand } from '../videos/video-stats-command' -import { ConfigCommand } from './config-command' -import { ContactFormCommand } from './contact-form-command' -import { DebugCommand } from './debug-command' -import { FollowsCommand } from './follows-command' -import { JobsCommand } from './jobs-command' -import { MetricsCommand } from './metrics-command' -import { PluginsCommand } from './plugins-command' -import { RedundancyCommand } from './redundancy-command' -import { ServersCommand } from './servers-command' -import { StatsCommand } from './stats-command' - -export type RunServerOptions = { - hideLogs?: boolean - nodeArgs?: string[] - peertubeArgs?: string[] - env?: { [ id: string ]: string } -} - -export class PeerTubeServer { - app?: ChildProcess - - url: string - host?: string - hostname?: string - port?: number - - rtmpPort?: number - rtmpsPort?: number - - parallel?: boolean - internalServerNumber: number - - serverNumber?: number - customConfigFile?: string - - store?: { - client?: { - id?: string - secret?: string - } - - user?: { - username: string - password: string - email?: string - } - - channel?: VideoChannel - videoChannelSync?: Partial - - video?: Video - videoCreated?: VideoCreateResult - videoDetails?: VideoDetails - - videos?: { id: number, uuid: string }[] - } - - accessToken?: string - refreshToken?: string - - bulk?: BulkCommand - cli?: CLICommand - customPage?: CustomPagesCommand - feed?: FeedCommand - logs?: LogsCommand - abuses?: AbusesCommand - overviews?: OverviewsCommand - search?: SearchCommand - contactForm?: ContactFormCommand - debug?: DebugCommand - follows?: FollowsCommand - jobs?: JobsCommand - metrics?: MetricsCommand - plugins?: PluginsCommand - redundancy?: RedundancyCommand - stats?: StatsCommand - config?: ConfigCommand - socketIO?: SocketIOCommand - accounts?: AccountsCommand - blocklist?: BlocklistCommand - subscriptions?: SubscriptionsCommand - live?: LiveCommand - services?: ServicesCommand - blacklist?: BlacklistCommand - captions?: CaptionsCommand - changeOwnership?: ChangeOwnershipCommand - playlists?: PlaylistsCommand - history?: HistoryCommand - imports?: ImportsCommand - channelSyncs?: ChannelSyncsCommand - streamingPlaylists?: StreamingPlaylistsCommand - channels?: ChannelsCommand - comments?: CommentsCommand - notifications?: NotificationsCommand - servers?: ServersCommand - login?: LoginCommand - users?: UsersCommand - videoStudio?: VideoStudioCommand - videos?: VideosCommand - videoStats?: VideoStatsCommand - views?: ViewsCommand - twoFactor?: TwoFactorCommand - videoToken?: VideoTokenCommand - registrations?: RegistrationsCommand - videoPasswords?: VideoPasswordsCommand - - storyboard?: StoryboardCommand - - runners?: RunnersCommand - runnerRegistrationTokens?: RunnerRegistrationTokensCommand - runnerJobs?: RunnerJobsCommand - - constructor (options: { serverNumber: number } | { url: string }) { - if ((options as any).url) { - this.setUrl((options as any).url) - } else { - this.setServerNumber((options as any).serverNumber) - } - - this.store = { - client: { - id: null, - secret: null - }, - user: { - username: null, - password: null - } - } - - this.assignCommands() - } - - setServerNumber (serverNumber: number) { - this.serverNumber = serverNumber - - this.parallel = parallelTests() - - this.internalServerNumber = this.parallel ? this.randomServer() : this.serverNumber - this.rtmpPort = this.parallel ? this.randomRTMP() : 1936 - this.rtmpsPort = this.parallel ? this.randomRTMP() : 1937 - this.port = 9000 + this.internalServerNumber - - this.url = `http://127.0.0.1:${this.port}` - this.host = `127.0.0.1:${this.port}` - this.hostname = '127.0.0.1' - } - - setUrl (url: string) { - const parsed = new URL(url) - - this.url = url - this.host = parsed.host - this.hostname = parsed.hostname - this.port = parseInt(parsed.port) - } - - getDirectoryPath (directoryName: string) { - const testDirectory = 'test' + this.internalServerNumber - - return join(root(), testDirectory, directoryName) - } - - async flushAndRun (configOverride?: object, options: RunServerOptions = {}) { - await ServersCommand.flushTests(this.internalServerNumber) - - return this.run(configOverride, options) - } - - async run (configOverrideArg?: any, options: RunServerOptions = {}) { - // These actions are async so we need to be sure that they have both been done - const serverRunString = { - 'HTTP server listening': false - } - const key = 'Database peertube_test' + this.internalServerNumber + ' is ready' - serverRunString[key] = false - - const regexps = { - client_id: 'Client id: (.+)', - client_secret: 'Client secret: (.+)', - user_username: 'Username: (.+)', - user_password: 'User password: (.+)' - } - - await this.assignCustomConfigFile() - - const configOverride = this.buildConfigOverride() - - if (configOverrideArg !== undefined) { - Object.assign(configOverride, configOverrideArg) - } - - // Share the environment - const env = { ...process.env } - env['NODE_ENV'] = 'test' - env['NODE_APP_INSTANCE'] = this.internalServerNumber.toString() - env['NODE_CONFIG'] = JSON.stringify(configOverride) - - if (options.env) { - Object.assign(env, options.env) - } - - const execArgv = options.nodeArgs || [] - // FIXME: too slow :/ - // execArgv.push('--enable-source-maps') - - const forkOptions = { - silent: true, - env, - detached: false, - execArgv - } - - const peertubeArgs = options.peertubeArgs || [] - - return new Promise((res, rej) => { - const self = this - let aggregatedLogs = '' - - this.app = fork(join(root(), 'dist', 'server.js'), peertubeArgs, forkOptions) - - const onPeerTubeExit = () => rej(new Error('Process exited:\n' + aggregatedLogs)) - const onParentExit = () => { - if (!this.app?.pid) return - - try { - process.kill(self.app.pid) - } catch { /* empty */ } - } - - this.app.on('exit', onPeerTubeExit) - process.on('exit', onParentExit) - - this.app.stdout.on('data', function onStdout (data) { - let dontContinue = false - - const log: string = data.toString() - aggregatedLogs += log - - // Capture things if we want to - for (const key of Object.keys(regexps)) { - const regexp = regexps[key] - const matches = log.match(regexp) - if (matches !== null) { - if (key === 'client_id') self.store.client.id = matches[1] - else if (key === 'client_secret') self.store.client.secret = matches[1] - else if (key === 'user_username') self.store.user.username = matches[1] - else if (key === 'user_password') self.store.user.password = matches[1] - } - } - - // Check if all required sentences are here - for (const key of Object.keys(serverRunString)) { - if (log.includes(key)) serverRunString[key] = true - if (serverRunString[key] === false) dontContinue = true - } - - // If no, there is maybe one thing not already initialized (client/user credentials generation...) - if (dontContinue === true) return - - if (options.hideLogs === false) { - console.log(log) - } else { - process.removeListener('exit', onParentExit) - self.app.stdout.removeListener('data', onStdout) - self.app.removeListener('exit', onPeerTubeExit) - } - - res() - }) - }) - } - - kill () { - if (!this.app) return Promise.resolve() - - process.kill(this.app.pid) - - this.app = null - - return Promise.resolve() - } - - private randomServer () { - const low = 2500 - const high = 10000 - - return randomInt(low, high) - } - - private randomRTMP () { - const low = 1900 - const high = 2100 - - return randomInt(low, high) - } - - private async assignCustomConfigFile () { - if (this.internalServerNumber === this.serverNumber) return - - const basePath = join(root(), 'config') - - const tmpConfigFile = join(basePath, `test-${this.internalServerNumber}.yaml`) - await copy(join(basePath, `test-${this.serverNumber}.yaml`), tmpConfigFile) - - this.customConfigFile = tmpConfigFile - } - - private buildConfigOverride () { - if (!this.parallel) return {} - - return { - listen: { - port: this.port - }, - webserver: { - port: this.port - }, - database: { - suffix: '_test' + this.internalServerNumber - }, - storage: { - tmp: this.getDirectoryPath('tmp') + '/', - tmp_persistent: this.getDirectoryPath('tmp-persistent') + '/', - bin: this.getDirectoryPath('bin') + '/', - avatars: this.getDirectoryPath('avatars') + '/', - web_videos: this.getDirectoryPath('web-videos') + '/', - streaming_playlists: this.getDirectoryPath('streaming-playlists') + '/', - redundancy: this.getDirectoryPath('redundancy') + '/', - logs: this.getDirectoryPath('logs') + '/', - previews: this.getDirectoryPath('previews') + '/', - thumbnails: this.getDirectoryPath('thumbnails') + '/', - storyboards: this.getDirectoryPath('storyboards') + '/', - torrents: this.getDirectoryPath('torrents') + '/', - captions: this.getDirectoryPath('captions') + '/', - cache: this.getDirectoryPath('cache') + '/', - plugins: this.getDirectoryPath('plugins') + '/', - well_known: this.getDirectoryPath('well-known') + '/' - }, - admin: { - email: `admin${this.internalServerNumber}@example.com` - }, - live: { - rtmp: { - port: this.rtmpPort - } - } - } - } - - private assignCommands () { - this.bulk = new BulkCommand(this) - this.cli = new CLICommand(this) - this.customPage = new CustomPagesCommand(this) - this.feed = new FeedCommand(this) - this.logs = new LogsCommand(this) - this.abuses = new AbusesCommand(this) - this.overviews = new OverviewsCommand(this) - this.search = new SearchCommand(this) - this.contactForm = new ContactFormCommand(this) - this.debug = new DebugCommand(this) - this.follows = new FollowsCommand(this) - this.jobs = new JobsCommand(this) - this.metrics = new MetricsCommand(this) - this.plugins = new PluginsCommand(this) - this.redundancy = new RedundancyCommand(this) - this.stats = new StatsCommand(this) - this.config = new ConfigCommand(this) - this.socketIO = new SocketIOCommand(this) - this.accounts = new AccountsCommand(this) - this.blocklist = new BlocklistCommand(this) - this.subscriptions = new SubscriptionsCommand(this) - this.live = new LiveCommand(this) - this.services = new ServicesCommand(this) - this.blacklist = new BlacklistCommand(this) - this.captions = new CaptionsCommand(this) - this.changeOwnership = new ChangeOwnershipCommand(this) - this.playlists = new PlaylistsCommand(this) - this.history = new HistoryCommand(this) - this.imports = new ImportsCommand(this) - this.channelSyncs = new ChannelSyncsCommand(this) - this.streamingPlaylists = new StreamingPlaylistsCommand(this) - this.channels = new ChannelsCommand(this) - this.comments = new CommentsCommand(this) - this.notifications = new NotificationsCommand(this) - this.servers = new ServersCommand(this) - this.login = new LoginCommand(this) - this.users = new UsersCommand(this) - this.videos = new VideosCommand(this) - this.videoStudio = new VideoStudioCommand(this) - this.videoStats = new VideoStatsCommand(this) - this.views = new ViewsCommand(this) - this.twoFactor = new TwoFactorCommand(this) - this.videoToken = new VideoTokenCommand(this) - this.registrations = new RegistrationsCommand(this) - - this.storyboard = new StoryboardCommand(this) - - this.runners = new RunnersCommand(this) - this.runnerRegistrationTokens = new RunnerRegistrationTokensCommand(this) - this.runnerJobs = new RunnerJobsCommand(this) - this.videoPasswords = new VideoPasswordsCommand(this) - } -} diff --git a/shared/server-commands/server/servers-command.ts b/shared/server-commands/server/servers-command.ts deleted file mode 100644 index 54e586a18..000000000 --- a/shared/server-commands/server/servers-command.ts +++ /dev/null @@ -1,103 +0,0 @@ -import { exec } from 'child_process' -import { copy, ensureDir, readFile, readdir, remove } from 'fs-extra' -import { basename, join } from 'path' -import { isGithubCI, root, wait } from '@shared/core-utils' -import { getFileSize } from '@shared/extra-utils' -import { HttpStatusCode } from '@shared/models' -import { AbstractCommand, OverrideCommandOptions } from '../shared' - -export class ServersCommand extends AbstractCommand { - - static flushTests (internalServerNumber: number) { - return new Promise((res, rej) => { - const suffix = ` -- ${internalServerNumber}` - - return exec('npm run clean:server:test' + suffix, (err, _stdout, stderr) => { - if (err || stderr) return rej(err || new Error(stderr)) - - return res() - }) - }) - } - - ping (options: OverrideCommandOptions = {}) { - return this.getRequestBody({ - ...options, - - path: '/api/v1/ping', - implicitToken: false, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } - - cleanupTests () { - const promises: Promise[] = [] - - const saveGithubLogsIfNeeded = async () => { - if (!isGithubCI()) return - - await ensureDir('artifacts') - - const origin = this.buildDirectory('logs/peertube.log') - const destname = `peertube-${this.server.internalServerNumber}.log` - console.log('Saving logs %s.', destname) - - await copy(origin, join('artifacts', destname)) - } - - if (this.server.parallel) { - const promise = saveGithubLogsIfNeeded() - .then(() => ServersCommand.flushTests(this.server.internalServerNumber)) - - promises.push(promise) - } - - if (this.server.customConfigFile) { - promises.push(remove(this.server.customConfigFile)) - } - - return promises - } - - async waitUntilLog (str: string, count = 1, strictCount = true) { - const logfile = this.buildDirectory('logs/peertube.log') - - while (true) { - const buf = await readFile(logfile) - - const matches = buf.toString().match(new RegExp(str, 'g')) - if (matches && matches.length === count) return - if (matches && strictCount === false && matches.length >= count) return - - await wait(1000) - } - } - - buildDirectory (directory: string) { - return join(root(), 'test' + this.server.internalServerNumber, directory) - } - - async countFiles (directory: string) { - const files = await readdir(this.buildDirectory(directory)) - - return files.length - } - - buildWebVideoFilePath (fileUrl: string) { - return this.buildDirectory(join('web-videos', basename(fileUrl))) - } - - buildFragmentedFilePath (videoUUID: string, fileUrl: string) { - return this.buildDirectory(join('streaming-playlists', 'hls', videoUUID, basename(fileUrl))) - } - - getLogContent () { - return readFile(this.buildDirectory('logs/peertube.log')) - } - - async getServerFileSize (subPath: string) { - const path = this.server.servers.buildDirectory(subPath) - - return getFileSize(path) - } -} diff --git a/shared/server-commands/server/servers.ts b/shared/server-commands/server/servers.ts deleted file mode 100644 index fe9da9e63..000000000 --- a/shared/server-commands/server/servers.ts +++ /dev/null @@ -1,68 +0,0 @@ -import { ensureDir } from 'fs-extra' -import { isGithubCI } from '@shared/core-utils' -import { PeerTubeServer, RunServerOptions } from './server' - -async function createSingleServer (serverNumber: number, configOverride?: object, options: RunServerOptions = {}) { - const server = new PeerTubeServer({ serverNumber }) - - await server.flushAndRun(configOverride, options) - - return server -} - -function createMultipleServers (totalServers: number, configOverride?: object, options: RunServerOptions = {}) { - const serverPromises: Promise[] = [] - - for (let i = 1; i <= totalServers; i++) { - serverPromises.push(createSingleServer(i, configOverride, options)) - } - - return Promise.all(serverPromises) -} - -function killallServers (servers: PeerTubeServer[]) { - return Promise.all(servers.map(s => s.kill())) -} - -async function cleanupTests (servers: PeerTubeServer[]) { - await killallServers(servers) - - if (isGithubCI()) { - await ensureDir('artifacts') - } - - let p: Promise[] = [] - for (const server of servers) { - p = p.concat(server.servers.cleanupTests()) - } - - return Promise.all(p) -} - -function getServerImportConfig (mode: 'youtube-dl' | 'yt-dlp') { - return { - import: { - videos: { - http: { - youtube_dl_release: { - url: mode === 'youtube-dl' - ? 'https://yt-dl.org/downloads/latest/youtube-dl' - : 'https://api.github.com/repos/yt-dlp/yt-dlp/releases', - - name: mode - } - } - } - } - } -} - -// --------------------------------------------------------------------------- - -export { - createSingleServer, - createMultipleServers, - cleanupTests, - killallServers, - getServerImportConfig -} diff --git a/shared/server-commands/server/stats-command.ts b/shared/server-commands/server/stats-command.ts deleted file mode 100644 index 64a452306..000000000 --- a/shared/server-commands/server/stats-command.ts +++ /dev/null @@ -1,25 +0,0 @@ -import { HttpStatusCode, ServerStats } from '@shared/models' -import { AbstractCommand, OverrideCommandOptions } from '../shared' - -export class StatsCommand extends AbstractCommand { - - get (options: OverrideCommandOptions & { - useCache?: boolean // default false - } = {}) { - const { useCache = false } = options - const path = '/api/v1/server/stats' - - const query = { - t: useCache ? undefined : new Date().getTime() - } - - return this.getRequestBody({ - ...options, - - path, - query, - implicitToken: false, - defaultExpectedStatus: HttpStatusCode.OK_200 - }) - } -} -- cgit v1.2.3