From 254d3579f5338f5fd775c17d15cdfc37078bcfb4 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Fri, 16 Jul 2021 09:47:51 +0200 Subject: Use an object to represent a server --- shared/extra-utils/miscs/checks.ts | 4 +- shared/extra-utils/miscs/webtorrent.ts | 4 +- shared/extra-utils/server/directories.ts | 6 +- shared/extra-utils/server/follows-command.ts | 6 +- shared/extra-utils/server/follows.ts | 4 +- shared/extra-utils/server/index.ts | 1 + shared/extra-utils/server/jobs.ts | 10 +- shared/extra-utils/server/plugins.ts | 4 +- shared/extra-utils/server/server.ts | 376 +++++++++++++++++++++ shared/extra-utils/server/servers-command.ts | 2 +- shared/extra-utils/server/servers.ts | 400 +---------------------- shared/extra-utils/shared/abstract-command.ts | 4 +- shared/extra-utils/users/accounts.ts | 4 +- shared/extra-utils/users/login.ts | 4 +- shared/extra-utils/users/notifications.ts | 7 +- shared/extra-utils/videos/channels.ts | 4 +- shared/extra-utils/videos/live.ts | 6 +- shared/extra-utils/videos/streaming-playlists.ts | 8 +- shared/extra-utils/videos/videos-command.ts | 4 +- shared/extra-utils/videos/videos.ts | 10 +- 20 files changed, 441 insertions(+), 427 deletions(-) create mode 100644 shared/extra-utils/server/server.ts (limited to 'shared') diff --git a/shared/extra-utils/miscs/checks.ts b/shared/extra-utils/miscs/checks.ts index 8f7bdb9b5..c81460330 100644 --- a/shared/extra-utils/miscs/checks.ts +++ b/shared/extra-utils/miscs/checks.ts @@ -6,7 +6,7 @@ import { join } from 'path' import { root } from '@server/helpers/core-utils' import { HttpStatusCode } from '@shared/core-utils' import { makeGetRequest } from '../requests' -import { ServerInfo } from '../server' +import { PeerTubeServer } from '../server' // Default interval -> 5 minutes function dateIsValid (dateString: string, interval = 300000) { @@ -33,7 +33,7 @@ async function testImage (url: string, imageName: string, imagePath: string, ext expect(data.length).to.be.below(maxLength, 'the generated image is way larger than the recorded fixture') } -async function testFileExistsOrNot (server: ServerInfo, directory: string, filePath: string, exist: boolean) { +async function testFileExistsOrNot (server: PeerTubeServer, directory: string, filePath: string, exist: boolean) { const base = server.servers.buildDirectory(directory) expect(await pathExists(join(base, filePath))).to.equal(exist) diff --git a/shared/extra-utils/miscs/webtorrent.ts b/shared/extra-utils/miscs/webtorrent.ts index 84e390b2a..815ea3d56 100644 --- a/shared/extra-utils/miscs/webtorrent.ts +++ b/shared/extra-utils/miscs/webtorrent.ts @@ -2,7 +2,7 @@ import { readFile } from 'fs-extra' import * as parseTorrent from 'parse-torrent' import { join } from 'path' import * as WebTorrent from 'webtorrent' -import { ServerInfo } from '../server' +import { PeerTubeServer } from '../server' let webtorrent: WebTorrent.Instance @@ -15,7 +15,7 @@ function webtorrentAdd (torrent: string, refreshWebTorrent = false) { return new Promise(res => webtorrent.add(torrent, res)) } -async function parseTorrentVideo (server: ServerInfo, videoUUID: string, resolution: number) { +async function parseTorrentVideo (server: PeerTubeServer, videoUUID: string, resolution: number) { const torrentName = videoUUID + '-' + resolution + '.torrent' const torrentPath = server.servers.buildDirectory(join('torrents', torrentName)) diff --git a/shared/extra-utils/server/directories.ts b/shared/extra-utils/server/directories.ts index 3cd38a561..b6465cbf4 100644 --- a/shared/extra-utils/server/directories.ts +++ b/shared/extra-utils/server/directories.ts @@ -4,9 +4,9 @@ import { expect } from 'chai' import { pathExists, readdir } from 'fs-extra' import { join } from 'path' import { root } from '@server/helpers/core-utils' -import { ServerInfo } from './servers' +import { PeerTubeServer } from './server' -async function checkTmpIsEmpty (server: ServerInfo) { +async function checkTmpIsEmpty (server: PeerTubeServer) { await checkDirectoryIsEmpty(server, 'tmp', [ 'plugins-global.css', 'hls', 'resumable-uploads' ]) if (await pathExists(join('test' + server.internalServerNumber, 'tmp', 'hls'))) { @@ -14,7 +14,7 @@ async function checkTmpIsEmpty (server: ServerInfo) { } } -async function checkDirectoryIsEmpty (server: ServerInfo, directory: string, exceptions: string[] = []) { +async function checkDirectoryIsEmpty (server: PeerTubeServer, directory: string, exceptions: string[] = []) { const testDirectory = 'test' + server.internalServerNumber const directoryPath = join(root(), testDirectory, directory) diff --git a/shared/extra-utils/server/follows-command.ts b/shared/extra-utils/server/follows-command.ts index 4e1e56d7a..4e9ed9494 100644 --- a/shared/extra-utils/server/follows-command.ts +++ b/shared/extra-utils/server/follows-command.ts @@ -2,7 +2,7 @@ import { pick } from 'lodash' import { ActivityPubActorType, ActorFollow, FollowState, ResultList } from '@shared/models' import { HttpStatusCode } from '../../core-utils/miscs/http-error-codes' import { AbstractCommand, OverrideCommandOptions } from '../shared' -import { ServerInfo } from './servers' +import { PeerTubeServer } from './server' export class FollowsCommand extends AbstractCommand { @@ -70,7 +70,7 @@ export class FollowsCommand extends AbstractCommand { } async unfollow (options: OverrideCommandOptions & { - target: ServerInfo + target: PeerTubeServer }) { const path = '/api/v1/server/following/' + options.target.host @@ -112,7 +112,7 @@ export class FollowsCommand extends AbstractCommand { } removeFollower (options: OverrideCommandOptions & { - follower: ServerInfo + follower: PeerTubeServer }) { const path = '/api/v1/server/followers/peertube@' + options.follower.host diff --git a/shared/extra-utils/server/follows.ts b/shared/extra-utils/server/follows.ts index 50ae898cc..0188be1aa 100644 --- a/shared/extra-utils/server/follows.ts +++ b/shared/extra-utils/server/follows.ts @@ -1,7 +1,7 @@ import { waitJobs } from './jobs' -import { ServerInfo } from './servers' +import { PeerTubeServer } from './server' -async function doubleFollow (server1: ServerInfo, server2: ServerInfo) { +async function doubleFollow (server1: PeerTubeServer, server2: PeerTubeServer) { await Promise.all([ server1.follows.follow({ targets: [ server2.url ] }), server2.follows.follow({ targets: [ server1.url ] }) diff --git a/shared/extra-utils/server/index.ts b/shared/extra-utils/server/index.ts index 669b004cd..9055dfc57 100644 --- a/shared/extra-utils/server/index.ts +++ b/shared/extra-utils/server/index.ts @@ -9,6 +9,7 @@ export * from './jobs-command' export * from './plugins-command' export * from './plugins' export * from './redundancy-command' +export * from './server' export * from './servers-command' export * from './servers' export * from './stats-command' diff --git a/shared/extra-utils/server/jobs.ts b/shared/extra-utils/server/jobs.ts index 754530977..64a0353eb 100644 --- a/shared/extra-utils/server/jobs.ts +++ b/shared/extra-utils/server/jobs.ts @@ -1,17 +1,17 @@ import { JobState } from '../../models' import { wait } from '../miscs' -import { ServerInfo } from './servers' +import { PeerTubeServer } from './server' -async function waitJobs (serversArg: ServerInfo[] | ServerInfo) { +async function waitJobs (serversArg: PeerTubeServer[] | PeerTubeServer) { const pendingJobWait = process.env.NODE_PENDING_JOB_WAIT ? parseInt(process.env.NODE_PENDING_JOB_WAIT, 10) : 250 - let servers: ServerInfo[] + let servers: PeerTubeServer[] - if (Array.isArray(serversArg) === false) servers = [ serversArg as ServerInfo ] - else servers = serversArg as ServerInfo[] + if (Array.isArray(serversArg) === false) servers = [ serversArg as PeerTubeServer ] + else servers = serversArg as PeerTubeServer[] const states: JobState[] = [ 'waiting', 'active', 'delayed' ] const repeatableJobs = [ 'videos-views', 'activitypub-cleaner' ] diff --git a/shared/extra-utils/server/plugins.ts b/shared/extra-utils/server/plugins.ts index d1cc7e383..0f5fabd5a 100644 --- a/shared/extra-utils/server/plugins.ts +++ b/shared/extra-utils/server/plugins.ts @@ -1,9 +1,9 @@ /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ import { expect } from 'chai' -import { ServerInfo } from '../server/servers' +import { PeerTubeServer } from '../server/server' -async function testHelloWorldRegisteredSettings (server: ServerInfo) { +async function testHelloWorldRegisteredSettings (server: PeerTubeServer) { const body = await server.plugins.getRegisteredSettings({ npmName: 'peertube-plugin-hello-world' }) const registeredSettings = body.registeredSettings diff --git a/shared/extra-utils/server/server.ts b/shared/extra-utils/server/server.ts new file mode 100644 index 000000000..b1347661f --- /dev/null +++ b/shared/extra-utils/server/server.ts @@ -0,0 +1,376 @@ +import { ChildProcess, fork } from 'child_process' +import { copy } from 'fs-extra' +import { join } from 'path' +import { root } from '@server/helpers/core-utils' +import { randomInt } from '../../core-utils/miscs/miscs' +import { VideoChannel } from '../../models/videos' +import { BulkCommand } from '../bulk' +import { CLICommand } from '../cli' +import { CustomPagesCommand } from '../custom-pages' +import { FeedCommand } from '../feeds' +import { LogsCommand } from '../logs' +import { parallelTests, SQLCommand } from '../miscs' +import { AbusesCommand } from '../moderation' +import { OverviewsCommand } from '../overviews' +import { SearchCommand } from '../search' +import { SocketIOCommand } from '../socket' +import { AccountsCommand, BlocklistCommand, LoginCommand, NotificationsCommand, SubscriptionsCommand, UsersCommand } from '../users' +import { + BlacklistCommand, + CaptionsCommand, + ChangeOwnershipCommand, + ChannelsCommand, + HistoryCommand, + ImportsCommand, + LiveCommand, + PlaylistsCommand, + ServicesCommand, + StreamingPlaylistsCommand, + VideosCommand +} from '../videos' +import { CommentsCommand } from '../videos/comments-command' +import { ConfigCommand } from './config-command' +import { ContactFormCommand } from './contact-form-command' +import { DebugCommand } from './debug-command' +import { FollowsCommand } from './follows-command' +import { JobsCommand } from './jobs-command' +import { PluginsCommand } from './plugins-command' +import { RedundancyCommand } from './redundancy-command' +import { ServersCommand } from './servers-command' +import { StatsCommand } from './stats-command' + +export type RunServerOptions = { + hideLogs?: boolean + execArgv?: string[] +} + +export class PeerTubeServer { + app?: ChildProcess + + url: string + host?: string + hostname?: string + port?: number + + rtmpPort?: number + + parallel?: boolean + internalServerNumber: number + + serverNumber?: number + customConfigFile?: string + + store?: { + client?: { + id?: string + secret?: string + } + + user?: { + username: string + password: string + email?: string + } + + channel?: VideoChannel + + video?: { + id: number + uuid: string + shortUUID: string + name?: string + url?: string + + account?: { + name: string + } + + embedPath?: string + } + + videos?: { id: number, uuid: string }[] + } + + accessToken?: string + refreshToken?: string + + bulk?: BulkCommand + cli?: CLICommand + customPage?: CustomPagesCommand + feed?: FeedCommand + logs?: LogsCommand + abuses?: AbusesCommand + overviews?: OverviewsCommand + search?: SearchCommand + contactForm?: ContactFormCommand + debug?: DebugCommand + follows?: FollowsCommand + jobs?: JobsCommand + plugins?: PluginsCommand + redundancy?: RedundancyCommand + stats?: StatsCommand + config?: ConfigCommand + socketIO?: SocketIOCommand + accounts?: AccountsCommand + blocklist?: BlocklistCommand + subscriptions?: SubscriptionsCommand + live?: LiveCommand + services?: ServicesCommand + blacklist?: BlacklistCommand + captions?: CaptionsCommand + changeOwnership?: ChangeOwnershipCommand + playlists?: PlaylistsCommand + history?: HistoryCommand + imports?: ImportsCommand + streamingPlaylists?: StreamingPlaylistsCommand + channels?: ChannelsCommand + comments?: CommentsCommand + sql?: SQLCommand + notifications?: NotificationsCommand + servers?: ServersCommand + login?: LoginCommand + users?: UsersCommand + videos?: VideosCommand + + constructor (options: { serverNumber: number } | { url: string }) { + if ((options as any).url) { + this.setUrl((options as any).url) + } else { + this.setServerNumber((options as any).serverNumber) + } + + this.store = { + client: { + id: null, + secret: null + }, + user: { + username: null, + password: null + } + } + + this.assignCommands() + } + + setServerNumber (serverNumber: number) { + this.serverNumber = serverNumber + + this.parallel = parallelTests() + + this.internalServerNumber = this.parallel ? this.randomServer() : this.serverNumber + this.rtmpPort = this.parallel ? this.randomRTMP() : 1936 + this.port = 9000 + this.internalServerNumber + + this.url = `http://localhost:${this.port}` + this.host = `localhost:${this.port}` + this.hostname = 'localhost' + } + + setUrl (url: string) { + const parsed = new URL(url) + + this.url = url + this.host = parsed.host + this.hostname = parsed.hostname + this.port = parseInt(parsed.port) + } + + async flushAndRun (configOverride?: Object, args = [], options: RunServerOptions = {}) { + await ServersCommand.flushTests(this.internalServerNumber) + + return this.run(configOverride, args, options) + } + + async run (configOverrideArg?: any, args = [], options: RunServerOptions = {}) { + // These actions are async so we need to be sure that they have both been done + const serverRunString = { + 'HTTP server listening': false + } + const key = 'Database peertube_test' + this.internalServerNumber + ' is ready' + serverRunString[key] = false + + const regexps = { + client_id: 'Client id: (.+)', + client_secret: 'Client secret: (.+)', + user_username: 'Username: (.+)', + user_password: 'User password: (.+)' + } + + await this.assignCustomConfigFile() + + const configOverride = this.buildConfigOverride() + + if (configOverrideArg !== undefined) { + Object.assign(configOverride, configOverrideArg) + } + + // Share the environment + const env = Object.create(process.env) + env['NODE_ENV'] = 'test' + env['NODE_APP_INSTANCE'] = this.internalServerNumber.toString() + env['NODE_CONFIG'] = JSON.stringify(configOverride) + + const forkOptions = { + silent: true, + env, + detached: true, + execArgv: options.execArgv || [] + } + + return new Promise(res => { + this.app = fork(join(root(), 'dist', 'server.js'), args, forkOptions) + this.app.stdout.on('data', function onStdout (data) { + let dontContinue = false + + // Capture things if we want to + for (const key of Object.keys(regexps)) { + const regexp = regexps[key] + const matches = data.toString().match(regexp) + if (matches !== null) { + if (key === 'client_id') this.store.client.id = matches[1] + else if (key === 'client_secret') this.store.client.secret = matches[1] + else if (key === 'user_username') this.store.user.username = matches[1] + else if (key === 'user_password') this.store.user.password = matches[1] + } + } + + // Check if all required sentences are here + for (const key of Object.keys(serverRunString)) { + if (data.toString().indexOf(key) !== -1) serverRunString[key] = true + if (serverRunString[key] === false) dontContinue = true + } + + // If no, there is maybe one thing not already initialized (client/user credentials generation...) + if (dontContinue === true) return + + if (options.hideLogs === false) { + console.log(data.toString()) + } else { + this.app.stdout.removeListener('data', onStdout) + } + + process.on('exit', () => { + try { + process.kill(this.server.app.pid) + } catch { /* empty */ } + }) + + res() + }) + }) + } + + async kill () { + if (!this.app) return + + await this.sql.cleanup() + + process.kill(-this.app.pid) + + this.app = null + } + + private randomServer () { + const low = 10 + const high = 10000 + + return randomInt(low, high) + } + + private randomRTMP () { + const low = 1900 + const high = 2100 + + return randomInt(low, high) + } + + private async assignCustomConfigFile () { + if (this.internalServerNumber === this.serverNumber) return + + const basePath = join(root(), 'config') + + const tmpConfigFile = join(basePath, `test-${this.internalServerNumber}.yaml`) + await copy(join(basePath, `test-${this.serverNumber}.yaml`), tmpConfigFile) + + this.customConfigFile = tmpConfigFile + } + + private buildConfigOverride () { + if (!this.parallel) return {} + + return { + listen: { + port: this.port + }, + webserver: { + port: this.port + }, + database: { + suffix: '_test' + this.internalServerNumber + }, + storage: { + tmp: `test${this.internalServerNumber}/tmp/`, + avatars: `test${this.internalServerNumber}/avatars/`, + videos: `test${this.internalServerNumber}/videos/`, + streaming_playlists: `test${this.internalServerNumber}/streaming-playlists/`, + redundancy: `test${this.internalServerNumber}/redundancy/`, + logs: `test${this.internalServerNumber}/logs/`, + previews: `test${this.internalServerNumber}/previews/`, + thumbnails: `test${this.internalServerNumber}/thumbnails/`, + torrents: `test${this.internalServerNumber}/torrents/`, + captions: `test${this.internalServerNumber}/captions/`, + cache: `test${this.internalServerNumber}/cache/`, + plugins: `test${this.internalServerNumber}/plugins/` + }, + admin: { + email: `admin${this.internalServerNumber}@example.com` + }, + live: { + rtmp: { + port: this.rtmpPort + } + } + } + } + + private assignCommands () { + this.bulk = new BulkCommand(this) + this.cli = new CLICommand(this) + this.customPage = new CustomPagesCommand(this) + this.feed = new FeedCommand(this) + this.logs = new LogsCommand(this) + this.abuses = new AbusesCommand(this) + this.overviews = new OverviewsCommand(this) + this.search = new SearchCommand(this) + this.contactForm = new ContactFormCommand(this) + this.debug = new DebugCommand(this) + this.follows = new FollowsCommand(this) + this.jobs = new JobsCommand(this) + this.plugins = new PluginsCommand(this) + this.redundancy = new RedundancyCommand(this) + this.stats = new StatsCommand(this) + this.config = new ConfigCommand(this) + this.socketIO = new SocketIOCommand(this) + this.accounts = new AccountsCommand(this) + this.blocklist = new BlocklistCommand(this) + this.subscriptions = new SubscriptionsCommand(this) + this.live = new LiveCommand(this) + this.services = new ServicesCommand(this) + this.blacklist = new BlacklistCommand(this) + this.captions = new CaptionsCommand(this) + this.changeOwnership = new ChangeOwnershipCommand(this) + this.playlists = new PlaylistsCommand(this) + this.history = new HistoryCommand(this) + this.imports = new ImportsCommand(this) + this.streamingPlaylists = new StreamingPlaylistsCommand(this) + this.channels = new ChannelsCommand(this) + this.comments = new CommentsCommand(this) + this.sql = new SQLCommand(this) + this.notifications = new NotificationsCommand(this) + this.servers = new ServersCommand(this) + this.login = new LoginCommand(this) + this.users = new UsersCommand(this) + this.videos = new VideosCommand(this) + } +} diff --git a/shared/extra-utils/server/servers-command.ts b/shared/extra-utils/server/servers-command.ts index a7c5a868d..1a7b2aade 100644 --- a/shared/extra-utils/server/servers-command.ts +++ b/shared/extra-utils/server/servers-command.ts @@ -37,7 +37,7 @@ export class ServersCommand extends AbstractCommand { if (isGithubCI()) { await ensureDir('artifacts') - const origin = this.server.servers.buildDirectory('logs/peertube.log') + const origin = this.buildDirectory('logs/peertube.log') const destname = `peertube-${this.server.internalServerNumber}.log` console.log('Saving logs %s.', destname) diff --git a/shared/extra-utils/server/servers.ts b/shared/extra-utils/server/servers.ts index ea3f19a92..87d7e9449 100644 --- a/shared/extra-utils/server/servers.ts +++ b/shared/extra-utils/server/servers.ts @@ -1,391 +1,30 @@ -/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/no-floating-promises */ +import { ensureDir } from 'fs-extra' +import { isGithubCI } from '../miscs' +import { PeerTubeServer, RunServerOptions } from './server' -import { ChildProcess, fork } from 'child_process' -import { copy, ensureDir } from 'fs-extra' -import { join } from 'path' -import { root } from '@server/helpers/core-utils' -import { randomInt } from '../../core-utils/miscs/miscs' -import { VideoChannel } from '../../models/videos' -import { BulkCommand } from '../bulk' -import { CLICommand } from '../cli' -import { CustomPagesCommand } from '../custom-pages' -import { FeedCommand } from '../feeds' -import { LogsCommand } from '../logs' -import { isGithubCI, parallelTests, SQLCommand } from '../miscs' -import { AbusesCommand } from '../moderation' -import { OverviewsCommand } from '../overviews' -import { SearchCommand } from '../search' -import { SocketIOCommand } from '../socket' -import { AccountsCommand, BlocklistCommand, LoginCommand, NotificationsCommand, SubscriptionsCommand, UsersCommand } from '../users' -import { - BlacklistCommand, - CaptionsCommand, - ChangeOwnershipCommand, - ChannelsCommand, - HistoryCommand, - ImportsCommand, - LiveCommand, - PlaylistsCommand, - ServicesCommand, - StreamingPlaylistsCommand, - VideosCommand -} from '../videos' -import { CommentsCommand } from '../videos/comments-command' -import { ConfigCommand } from './config-command' -import { ContactFormCommand } from './contact-form-command' -import { DebugCommand } from './debug-command' -import { FollowsCommand } from './follows-command' -import { JobsCommand } from './jobs-command' -import { PluginsCommand } from './plugins-command' -import { RedundancyCommand } from './redundancy-command' -import { ServersCommand } from './servers-command' -import { StatsCommand } from './stats-command' +async function createSingleServer (serverNumber: number, configOverride?: Object, args = [], options: RunServerOptions = {}) { + const server = new PeerTubeServer({ serverNumber }) -interface ServerInfo { - app?: ChildProcess + await server.flushAndRun(configOverride, args, options) - url: string - host?: string - hostname?: string - port?: number - - rtmpPort?: number - - parallel?: boolean - internalServerNumber: number - - serverNumber?: number - customConfigFile?: string - - store?: { - client?: { - id?: string - secret?: string - } - - user?: { - username: string - password: string - email?: string - } - - channel?: VideoChannel - - video?: { - id: number - uuid: string - shortUUID: string - name?: string - url?: string - - account?: { - name: string - } - - embedPath?: string - } - - videos?: { id: number, uuid: string }[] - } - - accessToken?: string - refreshToken?: string - - bulk?: BulkCommand - cli?: CLICommand - customPage?: CustomPagesCommand - feed?: FeedCommand - logs?: LogsCommand - abuses?: AbusesCommand - overviews?: OverviewsCommand - search?: SearchCommand - contactForm?: ContactFormCommand - debug?: DebugCommand - follows?: FollowsCommand - jobs?: JobsCommand - plugins?: PluginsCommand - redundancy?: RedundancyCommand - stats?: StatsCommand - config?: ConfigCommand - socketIO?: SocketIOCommand - accounts?: AccountsCommand - blocklist?: BlocklistCommand - subscriptions?: SubscriptionsCommand - live?: LiveCommand - services?: ServicesCommand - blacklist?: BlacklistCommand - captions?: CaptionsCommand - changeOwnership?: ChangeOwnershipCommand - playlists?: PlaylistsCommand - history?: HistoryCommand - imports?: ImportsCommand - streamingPlaylists?: StreamingPlaylistsCommand - channels?: ChannelsCommand - comments?: CommentsCommand - sql?: SQLCommand - notifications?: NotificationsCommand - servers?: ServersCommand - login?: LoginCommand - users?: UsersCommand - videos?: VideosCommand -} - -function flushAndRunMultipleServers (totalServers: number, configOverride?: Object) { - const apps = [] - let i = 0 - - return new Promise(res => { - function anotherServerDone (serverNumber, app) { - apps[serverNumber - 1] = app - i++ - if (i === totalServers) { - return res(apps) - } - } - - for (let j = 1; j <= totalServers; j++) { - flushAndRunServer(j, configOverride).then(app => anotherServerDone(j, app)) - } - }) -} - -function randomServer () { - const low = 10 - const high = 10000 - - return randomInt(low, high) -} - -function randomRTMP () { - const low = 1900 - const high = 2100 - - return randomInt(low, high) -} - -type RunServerOptions = { - hideLogs?: boolean - execArgv?: string[] -} - -async function flushAndRunServer (serverNumber: number, configOverride?: Object, args = [], options: RunServerOptions = {}) { - const parallel = parallelTests() - - const internalServerNumber = parallel ? randomServer() : serverNumber - const rtmpPort = parallel ? randomRTMP() : 1936 - const port = 9000 + internalServerNumber - - await ServersCommand.flushTests(internalServerNumber) - - const server: ServerInfo = { - app: null, - port, - internalServerNumber, - rtmpPort, - parallel, - serverNumber, - url: `http://localhost:${port}`, - host: `localhost:${port}`, - hostname: 'localhost', - store: { - client: { - id: null, - secret: null - }, - user: { - username: null, - password: null - } - } - } - - return runServer(server, configOverride, args, options) + return server } -async function runServer (server: ServerInfo, configOverrideArg?: any, args = [], options: RunServerOptions = {}) { - // These actions are async so we need to be sure that they have both been done - const serverRunString = { - 'HTTP server listening': false - } - const key = 'Database peertube_test' + server.internalServerNumber + ' is ready' - serverRunString[key] = false - - const regexps = { - client_id: 'Client id: (.+)', - client_secret: 'Client secret: (.+)', - user_username: 'Username: (.+)', - user_password: 'User password: (.+)' - } - - if (server.internalServerNumber !== server.serverNumber) { - const basePath = join(root(), 'config') - - const tmpConfigFile = join(basePath, `test-${server.internalServerNumber}.yaml`) - await copy(join(basePath, `test-${server.serverNumber}.yaml`), tmpConfigFile) - - server.customConfigFile = tmpConfigFile - } - - const configOverride: any = {} - - if (server.parallel) { - Object.assign(configOverride, { - listen: { - port: server.port - }, - webserver: { - port: server.port - }, - database: { - suffix: '_test' + server.internalServerNumber - }, - storage: { - tmp: `test${server.internalServerNumber}/tmp/`, - avatars: `test${server.internalServerNumber}/avatars/`, - videos: `test${server.internalServerNumber}/videos/`, - streaming_playlists: `test${server.internalServerNumber}/streaming-playlists/`, - redundancy: `test${server.internalServerNumber}/redundancy/`, - logs: `test${server.internalServerNumber}/logs/`, - previews: `test${server.internalServerNumber}/previews/`, - thumbnails: `test${server.internalServerNumber}/thumbnails/`, - torrents: `test${server.internalServerNumber}/torrents/`, - captions: `test${server.internalServerNumber}/captions/`, - cache: `test${server.internalServerNumber}/cache/`, - plugins: `test${server.internalServerNumber}/plugins/` - }, - admin: { - email: `admin${server.internalServerNumber}@example.com` - }, - live: { - rtmp: { - port: server.rtmpPort - } - } - }) - } - - if (configOverrideArg !== undefined) { - Object.assign(configOverride, configOverrideArg) - } - - // Share the environment - const env = Object.create(process.env) - env['NODE_ENV'] = 'test' - env['NODE_APP_INSTANCE'] = server.internalServerNumber.toString() - env['NODE_CONFIG'] = JSON.stringify(configOverride) +function createMultipleServers (totalServers: number, configOverride?: Object) { + const serverPromises: Promise[] = [] - const forkOptions = { - silent: true, - env, - detached: true, - execArgv: options.execArgv || [] + for (let i = 1; i <= totalServers; i++) { + serverPromises.push(createSingleServer(i, configOverride)) } - return new Promise(res => { - server.app = fork(join(root(), 'dist', 'server.js'), args, forkOptions) - server.app.stdout.on('data', function onStdout (data) { - let dontContinue = false - - // Capture things if we want to - for (const key of Object.keys(regexps)) { - const regexp = regexps[key] - const matches = data.toString().match(regexp) - if (matches !== null) { - if (key === 'client_id') server.store.client.id = matches[1] - else if (key === 'client_secret') server.store.client.secret = matches[1] - else if (key === 'user_username') server.store.user.username = matches[1] - else if (key === 'user_password') server.store.user.password = matches[1] - } - } - - // Check if all required sentences are here - for (const key of Object.keys(serverRunString)) { - if (data.toString().indexOf(key) !== -1) serverRunString[key] = true - if (serverRunString[key] === false) dontContinue = true - } - - // If no, there is maybe one thing not already initialized (client/user credentials generation...) - if (dontContinue === true) return - - if (options.hideLogs === false) { - console.log(data.toString()) - } else { - server.app.stdout.removeListener('data', onStdout) - } - - process.on('exit', () => { - try { - process.kill(server.app.pid) - } catch { /* empty */ } - }) - - assignCommands(server) - - res(server) - }) - }) -} - -function assignCommands (server: ServerInfo) { - server.bulk = new BulkCommand(server) - server.cli = new CLICommand(server) - server.customPage = new CustomPagesCommand(server) - server.feed = new FeedCommand(server) - server.logs = new LogsCommand(server) - server.abuses = new AbusesCommand(server) - server.overviews = new OverviewsCommand(server) - server.search = new SearchCommand(server) - server.contactForm = new ContactFormCommand(server) - server.debug = new DebugCommand(server) - server.follows = new FollowsCommand(server) - server.jobs = new JobsCommand(server) - server.plugins = new PluginsCommand(server) - server.redundancy = new RedundancyCommand(server) - server.stats = new StatsCommand(server) - server.config = new ConfigCommand(server) - server.socketIO = new SocketIOCommand(server) - server.accounts = new AccountsCommand(server) - server.blocklist = new BlocklistCommand(server) - server.subscriptions = new SubscriptionsCommand(server) - server.live = new LiveCommand(server) - server.services = new ServicesCommand(server) - server.blacklist = new BlacklistCommand(server) - server.captions = new CaptionsCommand(server) - server.changeOwnership = new ChangeOwnershipCommand(server) - server.playlists = new PlaylistsCommand(server) - server.history = new HistoryCommand(server) - server.imports = new ImportsCommand(server) - server.streamingPlaylists = new StreamingPlaylistsCommand(server) - server.channels = new ChannelsCommand(server) - server.comments = new CommentsCommand(server) - server.sql = new SQLCommand(server) - server.notifications = new NotificationsCommand(server) - server.servers = new ServersCommand(server) - server.login = new LoginCommand(server) - server.users = new UsersCommand(server) - server.videos = new VideosCommand(server) -} - -async function reRunServer (server: ServerInfo, configOverride?: any) { - const newServer = await runServer(server, configOverride) - server.app = newServer.app - - return server + return Promise.all(serverPromises) } -async function killallServers (servers: ServerInfo[]) { - for (const server of servers) { - if (!server.app) continue - - await server.sql.cleanup() - - process.kill(-server.app.pid) - - server.app = null - } +async function killallServers (servers: PeerTubeServer[]) { + return Promise.all(servers.map(s => s.kill())) } -async function cleanupTests (servers: ServerInfo[]) { +async function cleanupTests (servers: PeerTubeServer[]) { await killallServers(servers) if (isGithubCI()) { @@ -403,11 +42,8 @@ async function cleanupTests (servers: ServerInfo[]) { // --------------------------------------------------------------------------- export { - ServerInfo, + createSingleServer, + createMultipleServers, cleanupTests, - flushAndRunMultipleServers, - flushAndRunServer, - killallServers, - reRunServer, - assignCommands + killallServers } diff --git a/shared/extra-utils/shared/abstract-command.ts b/shared/extra-utils/shared/abstract-command.ts index 5fddcf639..967f8f2ac 100644 --- a/shared/extra-utils/shared/abstract-command.ts +++ b/shared/extra-utils/shared/abstract-command.ts @@ -9,7 +9,7 @@ import { unwrapBody, unwrapText } from '../requests/requests' -import { ServerInfo } from '../server/servers' +import { PeerTubeServer } from '../server/server' export interface OverrideCommandOptions { token?: string @@ -38,7 +38,7 @@ interface InternalGetCommandOptions extends InternalCommonCommandOptions { abstract class AbstractCommand { constructor ( - protected server: ServerInfo + protected server: PeerTubeServer ) { } diff --git a/shared/extra-utils/users/accounts.ts b/shared/extra-utils/users/accounts.ts index 2c9a4b71c..9fc1bcfc4 100644 --- a/shared/extra-utils/users/accounts.ts +++ b/shared/extra-utils/users/accounts.ts @@ -4,10 +4,10 @@ import { expect } from 'chai' import { pathExists, readdir } from 'fs-extra' import { join } from 'path' import { root } from '@server/helpers/core-utils' -import { ServerInfo } from '../server' +import { PeerTubeServer } from '../server' async function expectAccountFollows (options: { - server: ServerInfo + server: PeerTubeServer handle: string followers: number following: number diff --git a/shared/extra-utils/users/login.ts b/shared/extra-utils/users/login.ts index d0c26a4f0..f1df027d3 100644 --- a/shared/extra-utils/users/login.ts +++ b/shared/extra-utils/users/login.ts @@ -1,6 +1,6 @@ -import { ServerInfo } from '../server/servers' +import { PeerTubeServer } from '../server/server' -function setAccessTokensToServers (servers: ServerInfo[]) { +function setAccessTokensToServers (servers: PeerTubeServer[]) { const tasks: Promise[] = [] for (const server of servers) { diff --git a/shared/extra-utils/users/notifications.ts b/shared/extra-utils/users/notifications.ts index 9196f0bf5..4c42fad3e 100644 --- a/shared/extra-utils/users/notifications.ts +++ b/shared/extra-utils/users/notifications.ts @@ -5,8 +5,9 @@ import { inspect } from 'util' import { AbuseState, PluginType } from '@shared/models' import { UserNotification, UserNotificationSetting, UserNotificationSettingValue, UserNotificationType } from '../../models/users' import { MockSmtpServer } from '../mock-servers/mock-email' +import { PeerTubeServer } from '../server' import { doubleFollow } from '../server/follows' -import { flushAndRunMultipleServers, ServerInfo } from '../server/servers' +import { createMultipleServers } from '../server/servers' import { setAccessTokensToServers } from './login' function getAllNotificationsSettings (): UserNotificationSetting { @@ -31,7 +32,7 @@ function getAllNotificationsSettings (): UserNotificationSetting { } type CheckerBaseParams = { - server: ServerInfo + server: PeerTubeServer emails: any[] socketNotifications: UserNotification[] token: string @@ -642,7 +643,7 @@ async function prepareNotificationsTest (serversCount = 3, overrideConfigArg: an limit: 20 } } - const servers = await flushAndRunMultipleServers(serversCount, Object.assign(overrideConfig, overrideConfigArg)) + const servers = await createMultipleServers(serversCount, Object.assign(overrideConfig, overrideConfigArg)) await setAccessTokensToServers(servers) diff --git a/shared/extra-utils/videos/channels.ts b/shared/extra-utils/videos/channels.ts index 81e818094..756c47453 100644 --- a/shared/extra-utils/videos/channels.ts +++ b/shared/extra-utils/videos/channels.ts @@ -1,6 +1,6 @@ -import { ServerInfo } from '../server/servers' +import { PeerTubeServer } from '../server/server' -function setDefaultVideoChannel (servers: ServerInfo[]) { +function setDefaultVideoChannel (servers: PeerTubeServer[]) { const tasks: Promise[] = [] for (const server of servers) { diff --git a/shared/extra-utils/videos/live.ts b/shared/extra-utils/videos/live.ts index 353595811..502964b1a 100644 --- a/shared/extra-utils/videos/live.ts +++ b/shared/extra-utils/videos/live.ts @@ -5,7 +5,7 @@ import * as ffmpeg from 'fluent-ffmpeg' import { pathExists, readdir } from 'fs-extra' import { join } from 'path' import { buildAbsoluteFixturePath, wait } from '../miscs' -import { ServerInfo } from '../server/servers' +import { PeerTubeServer } from '../server/server' function sendRTMPStream (rtmpBaseUrl: string, streamKey: string, fixtureName = 'video_short.mp4') { const fixture = buildAbsoluteFixturePath(fixtureName) @@ -70,13 +70,13 @@ async function stopFfmpeg (command: ffmpeg.FfmpegCommand) { await wait(500) } -async function waitUntilLivePublishedOnAllServers (servers: ServerInfo[], videoId: string) { +async function waitUntilLivePublishedOnAllServers (servers: PeerTubeServer[], videoId: string) { for (const server of servers) { await server.live.waitUntilPublished({ videoId }) } } -async function checkLiveCleanup (server: ServerInfo, videoUUID: string, resolutions: number[] = []) { +async function checkLiveCleanup (server: PeerTubeServer, videoUUID: string, resolutions: number[] = []) { const basePath = server.servers.buildDirectory('streaming-playlists') const hlsPath = join(basePath, 'hls', videoUUID) diff --git a/shared/extra-utils/videos/streaming-playlists.ts b/shared/extra-utils/videos/streaming-playlists.ts index e8fd2f232..002ae08b2 100644 --- a/shared/extra-utils/videos/streaming-playlists.ts +++ b/shared/extra-utils/videos/streaming-playlists.ts @@ -2,10 +2,10 @@ import { expect } from 'chai' import { sha256 } from '@server/helpers/core-utils' import { HttpStatusCode } from '@shared/core-utils' import { VideoStreamingPlaylist } from '@shared/models' -import { ServerInfo } from '../server' +import { PeerTubeServer } from '../server' async function checkSegmentHash (options: { - server: ServerInfo + server: PeerTubeServer baseUrlPlaylist: string baseUrlSegment: string videoUUID: string @@ -36,7 +36,7 @@ async function checkSegmentHash (options: { } async function checkLiveSegmentHash (options: { - server: ServerInfo + server: PeerTubeServer baseUrlSegment: string videoUUID: string segmentName: string @@ -52,7 +52,7 @@ async function checkLiveSegmentHash (options: { } async function checkResolutionsInMasterPlaylist (options: { - server: ServerInfo + server: PeerTubeServer playlistUrl: string resolutions: number[] }) { diff --git a/shared/extra-utils/videos/videos-command.ts b/shared/extra-utils/videos/videos-command.ts index 5556cddf6..feef5a771 100644 --- a/shared/extra-utils/videos/videos-command.ts +++ b/shared/extra-utils/videos/videos-command.ts @@ -22,7 +22,7 @@ import { } from '@shared/models' import { buildAbsoluteFixturePath, wait } from '../miscs' import { unwrapBody } from '../requests' -import { ServerInfo, waitJobs } from '../server' +import { PeerTubeServer, waitJobs } from '../server' import { AbstractCommand, OverrideCommandOptions } from '../shared' export type VideoEdit = Partial> & { @@ -33,7 +33,7 @@ export type VideoEdit = Partial, expectedStatus = HttpStatusCode.OK_200, @@ -52,7 +52,7 @@ function checkUploadVideoParam ( } async function completeVideoCheck ( - server: ServerInfo, + server: PeerTubeServer, video: any, attributes: { name: string @@ -197,7 +197,7 @@ async function completeVideoCheck ( // serverNumber starts from 1 async function uploadRandomVideoOnServers ( - servers: ServerInfo[], + servers: PeerTubeServer[], serverNumber: number, additionalParams?: VideoEdit & { prefixName?: string } ) { -- cgit v1.2.3