From 3a4992633ee62d5edfbb484d9c6bcb3cf158489d Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Mon, 31 Jul 2023 14:34:36 +0200 Subject: Migrate server to ESM Sorry for the very big commit that may lead to git log issues and merge conflicts, but it's a major step forward: * Server can be faster at startup because imports() are async and we can easily lazy import big modules * Angular doesn't seem to support ES import (with .js extension), so we had to correctly organize peertube into a monorepo: * Use yarn workspace feature * Use typescript reference projects for dependencies * Shared projects have been moved into "packages", each one is now a node module (with a dedicated package.json/tsconfig.json) * server/tools have been moved into apps/ and is now a dedicated app bundled and published on NPM so users don't have to build peertube cli tools manually * server/tests have been moved into packages/ so we don't compile them every time we want to run the server * Use isolatedModule option: * Had to move from const enum to const (https://www.typescriptlang.org/docs/handbook/enums.html#objects-vs-enums) * Had to explictely specify "type" imports when used in decorators * Prefer tsx (that uses esbuild under the hood) instead of ts-node to load typescript files (tests with mocha or scripts): * To reduce test complexity as esbuild doesn't support decorator metadata, we only test server files that do not import server models * We still build tests files into js files for a faster CI * Remove unmaintained peertube CLI import script * Removed some barrels to speed up execution (less imports) --- .../job-builders/transcoding-runner-job-builder.ts | 196 --------------------- 1 file changed, 196 deletions(-) delete mode 100644 server/lib/transcoding/shared/job-builders/transcoding-runner-job-builder.ts (limited to 'server/lib/transcoding/shared/job-builders/transcoding-runner-job-builder.ts') diff --git a/server/lib/transcoding/shared/job-builders/transcoding-runner-job-builder.ts b/server/lib/transcoding/shared/job-builders/transcoding-runner-job-builder.ts deleted file mode 100644 index f0671bd7a..000000000 --- a/server/lib/transcoding/shared/job-builders/transcoding-runner-job-builder.ts +++ /dev/null @@ -1,196 +0,0 @@ -import { computeOutputFPS } from '@server/helpers/ffmpeg' -import { logger, loggerTagsFactory } from '@server/helpers/logger' -import { CONFIG } from '@server/initializers/config' -import { DEFAULT_AUDIO_RESOLUTION, VIDEO_TRANSCODING_FPS } from '@server/initializers/constants' -import { Hooks } from '@server/lib/plugins/hooks' -import { VODAudioMergeTranscodingJobHandler, VODHLSTranscodingJobHandler, VODWebVideoTranscodingJobHandler } from '@server/lib/runners' -import { VideoPathManager } from '@server/lib/video-path-manager' -import { MUserId, MVideoFile, MVideoFullLight, MVideoWithFileThumbnail } from '@server/types/models' -import { MRunnerJob } from '@server/types/models/runners' -import { ffprobePromise, getVideoStreamDimensionsInfo, getVideoStreamFPS, hasAudioStream, isAudioFile } from '@shared/ffmpeg' -import { getTranscodingJobPriority } from '../../transcoding-priority' -import { computeResolutionsToTranscode } from '../../transcoding-resolutions' -import { AbstractJobBuilder } from './abstract-job-builder' - -/** - * - * Class to build transcoding job in the local job queue - * - */ - -const lTags = loggerTagsFactory('transcoding') - -export class TranscodingRunnerJobBuilder extends AbstractJobBuilder { - - async createOptimizeOrMergeAudioJobs (options: { - video: MVideoFullLight - videoFile: MVideoFile - isNewVideo: boolean - user: MUserId - videoFileAlreadyLocked: boolean - }) { - const { video, videoFile, isNewVideo, user, videoFileAlreadyLocked } = options - - const mutexReleaser = videoFileAlreadyLocked - ? () => {} - : await VideoPathManager.Instance.lockFiles(video.uuid) - - try { - await video.reload() - await videoFile.reload() - - await VideoPathManager.Instance.makeAvailableVideoFile(videoFile.withVideoOrPlaylist(video), async videoFilePath => { - const probe = await ffprobePromise(videoFilePath) - - const { resolution } = await getVideoStreamDimensionsInfo(videoFilePath, probe) - const hasAudio = await hasAudioStream(videoFilePath, probe) - const inputFPS = videoFile.isAudio() - ? VIDEO_TRANSCODING_FPS.AUDIO_MERGE // The first transcoding job will transcode to this FPS value - : await getVideoStreamFPS(videoFilePath, probe) - - const maxResolution = await isAudioFile(videoFilePath, probe) - ? DEFAULT_AUDIO_RESOLUTION - : resolution - - const fps = computeOutputFPS({ inputFPS, resolution: maxResolution }) - const priority = await getTranscodingJobPriority({ user, type: 'vod', fallback: 0 }) - - const mainRunnerJob = videoFile.isAudio() - ? await new VODAudioMergeTranscodingJobHandler().create({ video, resolution: maxResolution, fps, isNewVideo, priority }) - : await new VODWebVideoTranscodingJobHandler().create({ video, resolution: maxResolution, fps, isNewVideo, priority }) - - if (CONFIG.TRANSCODING.HLS.ENABLED === true) { - await new VODHLSTranscodingJobHandler().create({ - video, - deleteWebVideoFiles: CONFIG.TRANSCODING.WEB_VIDEOS.ENABLED === false, - resolution: maxResolution, - fps, - isNewVideo, - dependsOnRunnerJob: mainRunnerJob, - priority: await getTranscodingJobPriority({ user, type: 'vod', fallback: 0 }) - }) - } - - await this.buildLowerResolutionJobPayloads({ - video, - inputVideoResolution: maxResolution, - inputVideoFPS: inputFPS, - hasAudio, - isNewVideo, - mainRunnerJob, - user - }) - }) - } finally { - mutexReleaser() - } - } - - // --------------------------------------------------------------------------- - - async createTranscodingJobs (options: { - transcodingType: 'hls' | 'webtorrent' | 'web-video' // TODO: remove webtorrent in v7 - video: MVideoFullLight - resolutions: number[] - isNewVideo: boolean - user: MUserId | null - }) { - const { video, transcodingType, resolutions, isNewVideo, user } = options - - const maxResolution = Math.max(...resolutions) - const { fps: inputFPS } = await video.probeMaxQualityFile() - const maxFPS = computeOutputFPS({ inputFPS, resolution: maxResolution }) - const priority = await getTranscodingJobPriority({ user, type: 'vod', fallback: 0 }) - - const childrenResolutions = resolutions.filter(r => r !== maxResolution) - - logger.info('Manually creating transcoding jobs for %s.', transcodingType, { childrenResolutions, maxResolution }) - - // Process the last resolution before the other ones to prevent concurrency issue - // Because low resolutions use the biggest one as ffmpeg input - const mainJob = transcodingType === 'hls' - // eslint-disable-next-line max-len - ? await new VODHLSTranscodingJobHandler().create({ video, resolution: maxResolution, fps: maxFPS, isNewVideo, deleteWebVideoFiles: false, priority }) - : await new VODWebVideoTranscodingJobHandler().create({ video, resolution: maxResolution, fps: maxFPS, isNewVideo, priority }) - - for (const resolution of childrenResolutions) { - const dependsOnRunnerJob = mainJob - const fps = computeOutputFPS({ inputFPS, resolution }) - - if (transcodingType === 'hls') { - await new VODHLSTranscodingJobHandler().create({ - video, - resolution, - fps, - isNewVideo, - deleteWebVideoFiles: false, - dependsOnRunnerJob, - priority: await getTranscodingJobPriority({ user, type: 'vod', fallback: 0 }) - }) - continue - } - - if (transcodingType === 'webtorrent' || transcodingType === 'web-video') { - await new VODWebVideoTranscodingJobHandler().create({ - video, - resolution, - fps, - isNewVideo, - dependsOnRunnerJob, - priority: await getTranscodingJobPriority({ user, type: 'vod', fallback: 0 }) - }) - continue - } - - throw new Error('Unknown transcoding type') - } - } - - private async buildLowerResolutionJobPayloads (options: { - mainRunnerJob: MRunnerJob - video: MVideoWithFileThumbnail - inputVideoResolution: number - inputVideoFPS: number - hasAudio: boolean - isNewVideo: boolean - user: MUserId - }) { - const { video, inputVideoResolution, inputVideoFPS, isNewVideo, hasAudio, mainRunnerJob, user } = options - - // Create transcoding jobs if there are enabled resolutions - const resolutionsEnabled = await Hooks.wrapObject( - computeResolutionsToTranscode({ input: inputVideoResolution, type: 'vod', includeInput: false, strictLower: true, hasAudio }), - 'filter:transcoding.auto.resolutions-to-transcode.result', - options - ) - - logger.debug('Lower resolutions build for %s.', video.uuid, { resolutionsEnabled, ...lTags(video.uuid) }) - - for (const resolution of resolutionsEnabled) { - const fps = computeOutputFPS({ inputFPS: inputVideoFPS, resolution }) - - if (CONFIG.TRANSCODING.WEB_VIDEOS.ENABLED) { - await new VODWebVideoTranscodingJobHandler().create({ - video, - resolution, - fps, - isNewVideo, - dependsOnRunnerJob: mainRunnerJob, - priority: await getTranscodingJobPriority({ user, type: 'vod', fallback: 0 }) - }) - } - - if (CONFIG.TRANSCODING.HLS.ENABLED) { - await new VODHLSTranscodingJobHandler().create({ - video, - resolution, - fps, - isNewVideo, - deleteWebVideoFiles: false, - dependsOnRunnerJob: mainRunnerJob, - priority: await getTranscodingJobPriority({ user, type: 'vod', fallback: 0 }) - }) - } - } - } -} -- cgit v1.2.3