From 06aad80165d09a8863ab8103149a8ff518b10641 Mon Sep 17 00:00:00 2001 From: lutangar Date: Tue, 2 Nov 2021 19:11:20 +0100 Subject: chore(refactor): remove shared folder dependencies to the server Many files from the `shared` folder were importing files from the `server` folder. When attempting to use Typescript project references to describe dependencies, it highlighted a circular dependency beetween `shared` <-> `server`. The Typescript project forbid such usages. Using project references greatly improve performance by rebuilding only the updated project and not all source files. > see https://www.typescriptlang.org/docs/handbook/project-references.html --- server/helpers/core-utils.ts | 47 +-------- server/helpers/custom-validators/misc.ts | 2 +- server/helpers/express-utils.ts | 6 +- server/helpers/ffprobe-utils.ts | 173 +++---------------------------- server/helpers/image-utils.ts | 4 +- server/helpers/peertube-crypto.ts | 3 +- server/helpers/utils.ts | 3 +- server/helpers/uuid.ts | 32 ------ server/helpers/webtorrent.ts | 3 +- 9 files changed, 28 insertions(+), 245 deletions(-) delete mode 100644 server/helpers/uuid.ts (limited to 'server/helpers') diff --git a/server/helpers/core-utils.ts b/server/helpers/core-utils.ts index 2cbf0f8fe..531ccfba9 100644 --- a/server/helpers/core-utils.ts +++ b/server/helpers/core-utils.ts @@ -6,9 +6,8 @@ */ import { exec, ExecOptions } from 'child_process' -import { BinaryToTextEncoding, createHash, randomBytes } from 'crypto' +import { randomBytes } from 'crypto' import { truncate } from 'lodash' -import { basename, extname, isAbsolute, join, resolve } from 'path' import { createPrivateKey as createPrivateKey_1, getPublicKey as getPublicKey_1 } from 'pem' import { pipeline } from 'stream' import { URL } from 'url' @@ -159,34 +158,6 @@ function getAppNumber () { // --------------------------------------------------------------------------- -let rootPath: string - -function root () { - if (rootPath) return rootPath - - rootPath = __dirname - - if (basename(rootPath) === 'helpers') rootPath = resolve(rootPath, '..') - if (basename(rootPath) === 'server') rootPath = resolve(rootPath, '..') - if (basename(rootPath) === 'dist') rootPath = resolve(rootPath, '..') - - return rootPath -} - -function buildPath (path: string) { - if (isAbsolute(path)) return path - - return join(root(), path) -} - -function getLowercaseExtension (filename: string) { - const ext = extname(filename) || '' - - return ext.toLowerCase() -} - -// --------------------------------------------------------------------------- - // Consistent with .length, lodash truncate function is not function peertubeTruncate (str: string, options: { length: number, separator?: RegExp, omission?: string }) { const truncatedStr = truncate(str, options) @@ -221,16 +192,6 @@ function parseSemVersion (s: string) { // --------------------------------------------------------------------------- -function sha256 (str: string | Buffer, encoding: BinaryToTextEncoding = 'hex') { - return createHash('sha256').update(str).digest(encoding) -} - -function sha1 (str: string | Buffer, encoding: BinaryToTextEncoding = 'hex') { - return createHash('sha1').update(str).digest(encoding) -} - -// --------------------------------------------------------------------------- - function execShell (command: string, options?: ExecOptions) { return new Promise<{ err?: Error, stdout: string, stderr: string }>((res, rej) => { exec(command, options, (err, stdout, stderr) => { @@ -298,9 +259,6 @@ export { objectConverter, mapToJSON, - root, - buildPath, - getLowercaseExtension, sanitizeUrl, sanitizeHost, @@ -309,9 +267,6 @@ export { pageToStartAndCount, peertubeTruncate, - sha256, - sha1, - promisify0, promisify1, promisify2, diff --git a/server/helpers/custom-validators/misc.ts b/server/helpers/custom-validators/misc.ts index c19a3e5eb..eaabdbbea 100644 --- a/server/helpers/custom-validators/misc.ts +++ b/server/helpers/custom-validators/misc.ts @@ -2,7 +2,7 @@ import 'multer' import { UploadFilesForCheck } from 'express' import { sep } from 'path' import validator from 'validator' -import { isShortUUID, shortToUUID } from '../uuid' +import { isShortUUID, shortToUUID } from '@shared/core-utils/uuid' function exists (value: any) { return value !== undefined && value !== null diff --git a/server/helpers/express-utils.ts b/server/helpers/express-utils.ts index 7b81ed71b..780fd6345 100644 --- a/server/helpers/express-utils.ts +++ b/server/helpers/express-utils.ts @@ -1,9 +1,9 @@ -import express from 'express' +import express, { RequestHandler } from 'express' import multer, { diskStorage } from 'multer' import { HttpStatusCode } from '../../shared/models/http/http-error-codes' import { CONFIG } from '../initializers/config' import { REMOTE_SCHEME } from '../initializers/constants' -import { getLowercaseExtension } from './core-utils' +import { getLowercaseExtension } from '@shared/core-utils' import { isArray } from './custom-validators/misc' import { logger } from './logger' import { deleteFileAndCatch, generateRandomString } from './utils' @@ -69,7 +69,7 @@ function createReqFiles ( fieldNames: string[], mimeTypes: { [id: string]: string | string[] }, destinations: { [fieldName: string]: string } -) { +): RequestHandler { const storage = diskStorage({ destination: (req, file, cb) => { cb(null, destinations[file.fieldname]) diff --git a/server/helpers/ffprobe-utils.ts b/server/helpers/ffprobe-utils.ts index e15628e2a..595112bce 100644 --- a/server/helpers/ffprobe-utils.ts +++ b/server/helpers/ffprobe-utils.ts @@ -1,9 +1,22 @@ -import { ffprobe, FfprobeData } from 'fluent-ffmpeg' +import { FfprobeData } from 'fluent-ffmpeg' import { getMaxBitrate } from '@shared/core-utils' -import { VideoFileMetadata, VideoResolution, VideoTranscodingFPS } from '../../shared/models/videos' +import { VideoResolution, VideoTranscodingFPS } from '../../shared/models/videos' import { CONFIG } from '../initializers/config' import { VIDEO_TRANSCODING_FPS } from '../initializers/constants' import { logger } from './logger' +import { + canDoQuickAudioTranscode, + ffprobePromise, + getDurationFromVideoFile, + getAudioStream, + getMaxAudioBitrate, + getMetadataFromFile, + getVideoFileBitrate, + getVideoFileFPS, + getVideoFileResolution, + getVideoStreamFromFile, + getVideoStreamSize +} from '@shared/extra-utils/ffprobe' /** * @@ -11,79 +24,6 @@ import { logger } from './logger' * */ -function ffprobePromise (path: string) { - return new Promise((res, rej) => { - ffprobe(path, (err, data) => { - if (err) return rej(err) - - return res(data) - }) - }) -} - -async function getAudioStream (videoPath: string, existingProbe?: FfprobeData) { - // without position, ffprobe considers the last input only - // we make it consider the first input only - // if you pass a file path to pos, then ffprobe acts on that file directly - const data = existingProbe || await ffprobePromise(videoPath) - - if (Array.isArray(data.streams)) { - const audioStream = data.streams.find(stream => stream['codec_type'] === 'audio') - - if (audioStream) { - return { - absolutePath: data.format.filename, - audioStream, - bitrate: parseInt(audioStream['bit_rate'] + '', 10) - } - } - } - - return { absolutePath: data.format.filename } -} - -function getMaxAudioBitrate (type: 'aac' | 'mp3' | string, bitrate: number) { - const maxKBitrate = 384 - const kToBits = (kbits: number) => kbits * 1000 - - // If we did not manage to get the bitrate, use an average value - if (!bitrate) return 256 - - if (type === 'aac') { - switch (true) { - case bitrate > kToBits(maxKBitrate): - return maxKBitrate - - default: - return -1 // we interpret it as a signal to copy the audio stream as is - } - } - - /* - a 192kbit/sec mp3 doesn't hold as much information as a 192kbit/sec aac. - That's why, when using aac, we can go to lower kbit/sec. The equivalences - made here are not made to be accurate, especially with good mp3 encoders. - */ - switch (true) { - case bitrate <= kToBits(192): - return 128 - - case bitrate <= kToBits(384): - return 256 - - default: - return maxKBitrate - } -} - -async function getVideoStreamSize (path: string, existingProbe?: FfprobeData): Promise<{ width: number, height: number }> { - const videoStream = await getVideoStreamFromFile(path, existingProbe) - - return videoStream === null - ? { width: 0, height: 0 } - : { width: videoStream.width, height: videoStream.height } -} - async function getVideoStreamCodec (path: string) { const videoStream = await getVideoStreamFromFile(path) @@ -143,69 +83,6 @@ async function getAudioStreamCodec (path: string, existingProbe?: FfprobeData) { return 'mp4a.40.2' // Fallback } -async function getVideoFileResolution (path: string, existingProbe?: FfprobeData) { - const size = await getVideoStreamSize(path, existingProbe) - - return { - width: size.width, - height: size.height, - ratio: Math.max(size.height, size.width) / Math.min(size.height, size.width), - resolution: Math.min(size.height, size.width), - isPortraitMode: size.height > size.width - } -} - -async function getVideoFileFPS (path: string, existingProbe?: FfprobeData) { - const videoStream = await getVideoStreamFromFile(path, existingProbe) - if (videoStream === null) return 0 - - for (const key of [ 'avg_frame_rate', 'r_frame_rate' ]) { - const valuesText: string = videoStream[key] - if (!valuesText) continue - - const [ frames, seconds ] = valuesText.split('/') - if (!frames || !seconds) continue - - const result = parseInt(frames, 10) / parseInt(seconds, 10) - if (result > 0) return Math.round(result) - } - - return 0 -} - -async function getMetadataFromFile (path: string, existingProbe?: FfprobeData) { - const metadata = existingProbe || await ffprobePromise(path) - - return new VideoFileMetadata(metadata) -} - -async function getVideoFileBitrate (path: string, existingProbe?: FfprobeData): Promise { - const metadata = await getMetadataFromFile(path, existingProbe) - - let bitrate = metadata.format.bit_rate as number - if (bitrate && !isNaN(bitrate)) return bitrate - - const videoStream = await getVideoStreamFromFile(path, existingProbe) - if (!videoStream) return undefined - - bitrate = videoStream?.bit_rate - if (bitrate && !isNaN(bitrate)) return bitrate - - return undefined -} - -async function getDurationFromVideoFile (path: string, existingProbe?: FfprobeData) { - const metadata = await getMetadataFromFile(path, existingProbe) - - return Math.round(metadata.format.duration) -} - -async function getVideoStreamFromFile (path: string, existingProbe?: FfprobeData) { - const metadata = await getMetadataFromFile(path, existingProbe) - - return metadata.streams.find(s => s.codec_type === 'video') || null -} - function computeLowerResolutionsToTranscode (videoFileResolution: number, type: 'vod' | 'live') { const configResolutions = type === 'vod' ? CONFIG.TRANSCODING.RESOLUTIONS @@ -263,26 +140,6 @@ async function canDoQuickVideoTranscode (path: string, probe?: FfprobeData): Pro return true } -async function canDoQuickAudioTranscode (path: string, probe?: FfprobeData): Promise { - const parsedAudio = await getAudioStream(path, probe) - - if (!parsedAudio.audioStream) return true - - if (parsedAudio.audioStream['codec_name'] !== 'aac') return false - - const audioBitrate = parsedAudio.bitrate - if (!audioBitrate) return false - - const maxAudioBitrate = getMaxAudioBitrate('aac', audioBitrate) - if (maxAudioBitrate !== -1 && audioBitrate > maxAudioBitrate) return false - - const channelLayout = parsedAudio.audioStream['channel_layout'] - // Causes playback issues with Chrome - if (!channelLayout || channelLayout === 'unknown') return false - - return true -} - function getClosestFramerateStandard > (fps: number, type: K) { return VIDEO_TRANSCODING_FPS[type].slice(0) .sort((a, b) => fps % a - fps % b)[0] diff --git a/server/helpers/image-utils.ts b/server/helpers/image-utils.ts index 4305584d5..ced288045 100644 --- a/server/helpers/image-utils.ts +++ b/server/helpers/image-utils.ts @@ -1,9 +1,9 @@ import { copy, readFile, remove, rename } from 'fs-extra' import Jimp, { read } from 'jimp' -import { getLowercaseExtension } from './core-utils' +import { getLowercaseExtension } from '@shared/core-utils' import { convertWebPToJPG, processGIF } from './ffmpeg-utils' import { logger } from './logger' -import { buildUUID } from './uuid' +import { buildUUID } from '@shared/core-utils/uuid' function generateImageFilename (extension = '.jpg') { return buildUUID() + extension diff --git a/server/helpers/peertube-crypto.ts b/server/helpers/peertube-crypto.ts index 44d90d9f1..31705e7fa 100644 --- a/server/helpers/peertube-crypto.ts +++ b/server/helpers/peertube-crypto.ts @@ -4,7 +4,8 @@ import { Request } from 'express' import { cloneDeep } from 'lodash' import { BCRYPT_SALT_SIZE, HTTP_SIGNATURE, PRIVATE_RSA_KEY_SIZE } from '../initializers/constants' import { MActor } from '../types/models' -import { createPrivateKey, getPublicKey, promisify1, promisify2, sha256 } from './core-utils' +import { sha256 } from '@shared/core-utils/crypto' +import { createPrivateKey, getPublicKey, promisify1, promisify2 } from './core-utils' import { jsonld } from './custom-jsonld-signature' import { logger } from './logger' diff --git a/server/helpers/utils.ts b/server/helpers/utils.ts index 6c95a43b6..882f808ab 100644 --- a/server/helpers/utils.ts +++ b/server/helpers/utils.ts @@ -3,7 +3,8 @@ import { Instance as ParseTorrent } from 'parse-torrent' import { join } from 'path' import { ResultList } from '../../shared' import { CONFIG } from '../initializers/config' -import { execPromise, execPromise2, randomBytesPromise, sha256 } from './core-utils' +import { sha256 } from '@shared/core-utils/crypto' +import { execPromise, execPromise2, randomBytesPromise } from './core-utils' import { logger } from './logger' function deleteFileAndCatch (path: string) { diff --git a/server/helpers/uuid.ts b/server/helpers/uuid.ts deleted file mode 100644 index f3c80e046..000000000 --- a/server/helpers/uuid.ts +++ /dev/null @@ -1,32 +0,0 @@ -import short, { uuid } from 'short-uuid' - -const translator = short() - -function buildUUID () { - return uuid() -} - -function uuidToShort (uuid: string) { - if (!uuid) return uuid - - return translator.fromUUID(uuid) -} - -function shortToUUID (shortUUID: string) { - if (!shortUUID) return shortUUID - - return translator.toUUID(shortUUID) -} - -function isShortUUID (value: string) { - if (!value) return false - - return value.length === translator.maxLength -} - -export { - buildUUID, - uuidToShort, - shortToUUID, - isShortUUID -} diff --git a/server/helpers/webtorrent.ts b/server/helpers/webtorrent.ts index ecc703646..67cb3971d 100644 --- a/server/helpers/webtorrent.ts +++ b/server/helpers/webtorrent.ts @@ -14,7 +14,8 @@ import { MVideo } from '@server/types/models/video/video' import { MVideoFile, MVideoFileRedundanciesOpt } from '@server/types/models/video/video-file' import { MStreamingPlaylistVideo } from '@server/types/models/video/video-streaming-playlist' import { CONFIG } from '../initializers/config' -import { promisify2, sha1 } from './core-utils' +import { promisify2 } from './core-utils' +import { sha1 } from '@shared/core-utils/crypto' import { logger } from './logger' import { generateVideoImportTmpPath } from './utils' import { extractVideo } from './video' -- cgit v1.2.3