From 3a4992633ee62d5edfbb484d9c6bcb3cf158489d Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Mon, 31 Jul 2023 14:34:36 +0200 Subject: Migrate server to ESM Sorry for the very big commit that may lead to git log issues and merge conflicts, but it's a major step forward: * Server can be faster at startup because imports() are async and we can easily lazy import big modules * Angular doesn't seem to support ES import (with .js extension), so we had to correctly organize peertube into a monorepo: * Use yarn workspace feature * Use typescript reference projects for dependencies * Shared projects have been moved into "packages", each one is now a node module (with a dedicated package.json/tsconfig.json) * server/tools have been moved into apps/ and is now a dedicated app bundled and published on NPM so users don't have to build peertube cli tools manually * server/tests have been moved into packages/ so we don't compile them every time we want to run the server * Use isolatedModule option: * Had to move from const enum to const (https://www.typescriptlang.org/docs/handbook/enums.html#objects-vs-enums) * Had to explictely specify "type" imports when used in decorators * Prefer tsx (that uses esbuild under the hood) instead of ts-node to load typescript files (tests with mocha or scripts): * To reduce test complexity as esbuild doesn't support decorator metadata, we only test server files that do not import server models * We still build tests files into js files for a faster CI * Remove unmaintained peertube CLI import script * Removed some barrels to speed up execution (less imports) --- packages/server-commands/src/runners/index.ts | 3 + .../src/runners/runner-jobs-command.ts | 297 +++++++++++++++++++++ .../runners/runner-registration-tokens-command.ts | 55 ++++ .../server-commands/src/runners/runners-command.ts | 85 ++++++ 4 files changed, 440 insertions(+) create mode 100644 packages/server-commands/src/runners/index.ts create mode 100644 packages/server-commands/src/runners/runner-jobs-command.ts create mode 100644 packages/server-commands/src/runners/runner-registration-tokens-command.ts create mode 100644 packages/server-commands/src/runners/runners-command.ts (limited to 'packages/server-commands/src/runners') diff --git a/packages/server-commands/src/runners/index.ts b/packages/server-commands/src/runners/index.ts new file mode 100644 index 000000000..c868fa78e --- /dev/null +++ b/packages/server-commands/src/runners/index.ts @@ -0,0 +1,3 @@ +export * from './runner-jobs-command.js' +export * from './runner-registration-tokens-command.js' +export * from './runners-command.js' diff --git a/packages/server-commands/src/runners/runner-jobs-command.ts b/packages/server-commands/src/runners/runner-jobs-command.ts new file mode 100644 index 000000000..4e702199f --- /dev/null +++ b/packages/server-commands/src/runners/runner-jobs-command.ts @@ -0,0 +1,297 @@ +import { omit, pick, wait } from '@peertube/peertube-core-utils' +import { + AbortRunnerJobBody, + AcceptRunnerJobBody, + AcceptRunnerJobResult, + ErrorRunnerJobBody, + HttpStatusCode, + isHLSTranscodingPayloadSuccess, + isLiveRTMPHLSTranscodingUpdatePayload, + isWebVideoOrAudioMergeTranscodingPayloadSuccess, + ListRunnerJobsQuery, + RequestRunnerJobBody, + RequestRunnerJobResult, + ResultList, + RunnerJobAdmin, + RunnerJobLiveRTMPHLSTranscodingPayload, + RunnerJobPayload, + RunnerJobState, + RunnerJobStateType, + RunnerJobSuccessBody, + RunnerJobSuccessPayload, + RunnerJobType, + RunnerJobUpdateBody, + RunnerJobVODPayload, + VODHLSTranscodingSuccess, + VODWebVideoTranscodingSuccess +} from '@peertube/peertube-models' +import { unwrapBody } from '../requests/index.js' +import { waitJobs } from '../server/jobs.js' +import { AbstractCommand, OverrideCommandOptions } from '../shared/index.js' + +export class RunnerJobsCommand extends AbstractCommand { + + list (options: OverrideCommandOptions & ListRunnerJobsQuery = {}) { + const path = '/api/v1/runners/jobs' + + return this.getRequestBody>({ + ...options, + + path, + query: pick(options, [ 'start', 'count', 'sort', 'search', 'stateOneOf' ]), + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } + + cancelByAdmin (options: OverrideCommandOptions & { jobUUID: string }) { + const path = '/api/v1/runners/jobs/' + options.jobUUID + '/cancel' + + return this.postBodyRequest({ + ...options, + + path, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } + + deleteByAdmin (options: OverrideCommandOptions & { jobUUID: string }) { + const path = '/api/v1/runners/jobs/' + options.jobUUID + + return this.deleteRequest({ + ...options, + + path, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } + + // --------------------------------------------------------------------------- + + request (options: OverrideCommandOptions & RequestRunnerJobBody) { + const path = '/api/v1/runners/jobs/request' + + return unwrapBody(this.postBodyRequest({ + ...options, + + path, + fields: pick(options, [ 'runnerToken' ]), + implicitToken: false, + defaultExpectedStatus: HttpStatusCode.OK_200 + })) + } + + async requestVOD (options: OverrideCommandOptions & RequestRunnerJobBody) { + const vodTypes = new Set([ 'vod-audio-merge-transcoding', 'vod-hls-transcoding', 'vod-web-video-transcoding' ]) + + const { availableJobs } = await this.request(options) + + return { + availableJobs: availableJobs.filter(j => vodTypes.has(j.type)) + } as RequestRunnerJobResult + } + + async requestLive (options: OverrideCommandOptions & RequestRunnerJobBody) { + const vodTypes = new Set([ 'live-rtmp-hls-transcoding' ]) + + const { availableJobs } = await this.request(options) + + return { + availableJobs: availableJobs.filter(j => vodTypes.has(j.type)) + } as RequestRunnerJobResult + } + + // --------------------------------------------------------------------------- + + accept (options: OverrideCommandOptions & AcceptRunnerJobBody & { jobUUID: string }) { + const path = '/api/v1/runners/jobs/' + options.jobUUID + '/accept' + + return unwrapBody>(this.postBodyRequest({ + ...options, + + path, + fields: pick(options, [ 'runnerToken' ]), + implicitToken: false, + defaultExpectedStatus: HttpStatusCode.OK_200 + })) + } + + abort (options: OverrideCommandOptions & AbortRunnerJobBody & { jobUUID: string }) { + const path = '/api/v1/runners/jobs/' + options.jobUUID + '/abort' + + return this.postBodyRequest({ + ...options, + + path, + fields: pick(options, [ 'reason', 'jobToken', 'runnerToken' ]), + implicitToken: false, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } + + update (options: OverrideCommandOptions & RunnerJobUpdateBody & { jobUUID: string }) { + const path = '/api/v1/runners/jobs/' + options.jobUUID + '/update' + + const { payload } = options + const attaches: { [id: string]: any } = {} + let payloadWithoutFiles = payload + + if (isLiveRTMPHLSTranscodingUpdatePayload(payload)) { + if (payload.masterPlaylistFile) { + attaches[`payload[masterPlaylistFile]`] = payload.masterPlaylistFile + } + + attaches[`payload[resolutionPlaylistFile]`] = payload.resolutionPlaylistFile + attaches[`payload[videoChunkFile]`] = payload.videoChunkFile + + payloadWithoutFiles = omit(payloadWithoutFiles, [ 'masterPlaylistFile', 'resolutionPlaylistFile', 'videoChunkFile' ]) + } + + return this.postUploadRequest({ + ...options, + + path, + fields: { + ...pick(options, [ 'progress', 'jobToken', 'runnerToken' ]), + + payload: payloadWithoutFiles + }, + attaches, + implicitToken: false, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } + + error (options: OverrideCommandOptions & ErrorRunnerJobBody & { jobUUID: string }) { + const path = '/api/v1/runners/jobs/' + options.jobUUID + '/error' + + return this.postBodyRequest({ + ...options, + + path, + fields: pick(options, [ 'message', 'jobToken', 'runnerToken' ]), + implicitToken: false, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } + + success (options: OverrideCommandOptions & RunnerJobSuccessBody & { jobUUID: string }) { + const { payload } = options + + const path = '/api/v1/runners/jobs/' + options.jobUUID + '/success' + const attaches: { [id: string]: any } = {} + let payloadWithoutFiles = payload + + if ((isWebVideoOrAudioMergeTranscodingPayloadSuccess(payload) || isHLSTranscodingPayloadSuccess(payload)) && payload.videoFile) { + attaches[`payload[videoFile]`] = payload.videoFile + + payloadWithoutFiles = omit(payloadWithoutFiles as VODWebVideoTranscodingSuccess, [ 'videoFile' ]) + } + + if (isHLSTranscodingPayloadSuccess(payload) && payload.resolutionPlaylistFile) { + attaches[`payload[resolutionPlaylistFile]`] = payload.resolutionPlaylistFile + + payloadWithoutFiles = omit(payloadWithoutFiles as VODHLSTranscodingSuccess, [ 'resolutionPlaylistFile' ]) + } + + return this.postUploadRequest({ + ...options, + + path, + attaches, + fields: { + ...pick(options, [ 'jobToken', 'runnerToken' ]), + + payload: payloadWithoutFiles + }, + implicitToken: false, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } + + getJobFile (options: OverrideCommandOptions & { url: string, jobToken: string, runnerToken: string }) { + const { host, protocol, pathname } = new URL(options.url) + + return this.postBodyRequest({ + url: `${protocol}//${host}`, + path: pathname, + + fields: pick(options, [ 'jobToken', 'runnerToken' ]), + implicitToken: false, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } + + // --------------------------------------------------------------------------- + + async autoAccept (options: OverrideCommandOptions & RequestRunnerJobBody & { type?: RunnerJobType }) { + const { availableJobs } = await this.request(options) + + const job = options.type + ? availableJobs.find(j => j.type === options.type) + : availableJobs[0] + + return this.accept({ ...options, jobUUID: job.uuid }) + } + + async autoProcessWebVideoJob (runnerToken: string, jobUUIDToProcess?: string) { + let jobUUID = jobUUIDToProcess + + if (!jobUUID) { + const { availableJobs } = await this.request({ runnerToken }) + jobUUID = availableJobs[0].uuid + } + + const { job } = await this.accept({ runnerToken, jobUUID }) + const jobToken = job.jobToken + + const payload: RunnerJobSuccessPayload = { videoFile: 'video_short.mp4' } + await this.success({ runnerToken, jobUUID, jobToken, payload }) + + await waitJobs([ this.server ]) + + return job + } + + async cancelAllJobs (options: { state?: RunnerJobStateType } = {}) { + const { state } = options + + const { data } = await this.list({ count: 100 }) + + const allowedStates = new Set([ + RunnerJobState.PENDING, + RunnerJobState.PROCESSING, + RunnerJobState.WAITING_FOR_PARENT_JOB + ]) + + for (const job of data) { + if (state && job.state.id !== state) continue + else if (allowedStates.has(job.state.id) !== true) continue + + await this.cancelByAdmin({ jobUUID: job.uuid }) + } + } + + async getJob (options: OverrideCommandOptions & { uuid: string }) { + const { data } = await this.list({ ...options, count: 100, sort: '-updatedAt' }) + + return data.find(j => j.uuid === options.uuid) + } + + async requestLiveJob (runnerToken: string) { + let availableJobs: RequestRunnerJobResult['availableJobs'] = [] + + while (availableJobs.length === 0) { + const result = await this.requestLive({ runnerToken }) + availableJobs = result.availableJobs + + if (availableJobs.length === 1) break + + await wait(150) + } + + return availableJobs[0] + } +} diff --git a/packages/server-commands/src/runners/runner-registration-tokens-command.ts b/packages/server-commands/src/runners/runner-registration-tokens-command.ts new file mode 100644 index 000000000..86b6e5f93 --- /dev/null +++ b/packages/server-commands/src/runners/runner-registration-tokens-command.ts @@ -0,0 +1,55 @@ +import { pick } from '@peertube/peertube-core-utils' +import { HttpStatusCode, ResultList, RunnerRegistrationToken } from '@peertube/peertube-models' +import { AbstractCommand, OverrideCommandOptions } from '../shared/index.js' + +export class RunnerRegistrationTokensCommand extends AbstractCommand { + + list (options: OverrideCommandOptions & { + start?: number + count?: number + sort?: string + } = {}) { + const path = '/api/v1/runners/registration-tokens' + + return this.getRequestBody>({ + ...options, + + path, + query: pick(options, [ 'start', 'count', 'sort' ]), + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } + + generate (options: OverrideCommandOptions = {}) { + const path = '/api/v1/runners/registration-tokens/generate' + + return this.postBodyRequest({ + ...options, + + path, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } + + delete (options: OverrideCommandOptions & { + id: number + }) { + const path = '/api/v1/runners/registration-tokens/' + options.id + + return this.deleteRequest({ + ...options, + + path, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } + + async getFirstRegistrationToken (options: OverrideCommandOptions = {}) { + const { data } = await this.list(options) + + return data[0].registrationToken + } +} diff --git a/packages/server-commands/src/runners/runners-command.ts b/packages/server-commands/src/runners/runners-command.ts new file mode 100644 index 000000000..376a1dff9 --- /dev/null +++ b/packages/server-commands/src/runners/runners-command.ts @@ -0,0 +1,85 @@ +import { pick } from '@peertube/peertube-core-utils' +import { + HttpStatusCode, + RegisterRunnerBody, + RegisterRunnerResult, + ResultList, + Runner, + UnregisterRunnerBody +} from '@peertube/peertube-models' +import { buildUUID } from '@peertube/peertube-node-utils' +import { unwrapBody } from '../requests/index.js' +import { AbstractCommand, OverrideCommandOptions } from '../shared/index.js' + +export class RunnersCommand extends AbstractCommand { + + list (options: OverrideCommandOptions & { + start?: number + count?: number + sort?: string + } = {}) { + const path = '/api/v1/runners' + + return this.getRequestBody>({ + ...options, + + path, + query: pick(options, [ 'start', 'count', 'sort' ]), + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.OK_200 + }) + } + + register (options: OverrideCommandOptions & RegisterRunnerBody) { + const path = '/api/v1/runners/register' + + return unwrapBody(this.postBodyRequest({ + ...options, + + path, + fields: pick(options, [ 'name', 'registrationToken', 'description' ]), + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.OK_200 + })) + } + + unregister (options: OverrideCommandOptions & UnregisterRunnerBody) { + const path = '/api/v1/runners/unregister' + + return this.postBodyRequest({ + ...options, + + path, + fields: pick(options, [ 'runnerToken' ]), + implicitToken: false, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } + + delete (options: OverrideCommandOptions & { + id: number + }) { + const path = '/api/v1/runners/' + options.id + + return this.deleteRequest({ + ...options, + + path, + implicitToken: true, + defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204 + }) + } + + // --------------------------------------------------------------------------- + + async autoRegisterRunner () { + const { data } = await this.server.runnerRegistrationTokens.list({ sort: 'createdAt' }) + + const { runnerToken } = await this.register({ + name: 'runner ' + buildUUID(), + registrationToken: data[0].registrationToken + }) + + return runnerToken + } +} -- cgit v1.2.3