From 06aad80165d09a8863ab8103149a8ff518b10641 Mon Sep 17 00:00:00 2001 From: lutangar Date: Tue, 2 Nov 2021 19:11:20 +0100 Subject: chore(refactor): remove shared folder dependencies to the server Many files from the `shared` folder were importing files from the `server` folder. When attempting to use Typescript project references to describe dependencies, it highlighted a circular dependency beetween `shared` <-> `server`. The Typescript project forbid such usages. Using project references greatly improve performance by rebuilding only the updated project and not all source files. > see https://www.typescriptlang.org/docs/handbook/project-references.html --- shared/extra-utils/ffprobe.ts | 180 +++++++++++++++++++++++ shared/extra-utils/miscs/checks.ts | 2 +- shared/extra-utils/miscs/generate.ts | 2 +- shared/extra-utils/requests/activitypub.ts | 42 ------ shared/extra-utils/server/directories.ts | 2 +- shared/extra-utils/server/plugins-command.ts | 2 +- shared/extra-utils/server/server.ts | 3 +- shared/extra-utils/server/servers-command.ts | 2 +- shared/extra-utils/server/tracker.ts | 2 +- shared/extra-utils/users/actors.ts | 2 +- shared/extra-utils/videos/streaming-playlists.ts | 2 +- shared/extra-utils/videos/videos-command.ts | 12 +- shared/extra-utils/videos/videos.ts | 150 ------------------- 13 files changed, 191 insertions(+), 212 deletions(-) create mode 100644 shared/extra-utils/ffprobe.ts delete mode 100644 shared/extra-utils/requests/activitypub.ts (limited to 'shared/extra-utils') diff --git a/shared/extra-utils/ffprobe.ts b/shared/extra-utils/ffprobe.ts new file mode 100644 index 000000000..9257bbd5f --- /dev/null +++ b/shared/extra-utils/ffprobe.ts @@ -0,0 +1,180 @@ +import { ffprobe, FfprobeData } from 'fluent-ffmpeg' +import { VideoFileMetadata } from '@shared/models/videos' + +/** + * + * Helpers to run ffprobe and extract data from the JSON output + * + */ + +function ffprobePromise (path: string) { + return new Promise((res, rej) => { + ffprobe(path, (err, data) => { + if (err) return rej(err) + + return res(data) + }) + }) +} + +async function getAudioStream (videoPath: string, existingProbe?: FfprobeData) { + // without position, ffprobe considers the last input only + // we make it consider the first input only + // if you pass a file path to pos, then ffprobe acts on that file directly + const data = existingProbe || await ffprobePromise(videoPath) + + if (Array.isArray(data.streams)) { + const audioStream = data.streams.find(stream => stream['codec_type'] === 'audio') + + if (audioStream) { + return { + absolutePath: data.format.filename, + audioStream, + bitrate: parseInt(audioStream['bit_rate'] + '', 10) + } + } + } + + return { absolutePath: data.format.filename } +} + +function getMaxAudioBitrate (type: 'aac' | 'mp3' | string, bitrate: number) { + const maxKBitrate = 384 + const kToBits = (kbits: number) => kbits * 1000 + + // If we did not manage to get the bitrate, use an average value + if (!bitrate) return 256 + + if (type === 'aac') { + switch (true) { + case bitrate > kToBits(maxKBitrate): + return maxKBitrate + + default: + return -1 // we interpret it as a signal to copy the audio stream as is + } + } + + /* + a 192kbit/sec mp3 doesn't hold as much information as a 192kbit/sec aac. + That's why, when using aac, we can go to lower kbit/sec. The equivalences + made here are not made to be accurate, especially with good mp3 encoders. + */ + switch (true) { + case bitrate <= kToBits(192): + return 128 + + case bitrate <= kToBits(384): + return 256 + + default: + return maxKBitrate + } +} + +async function getVideoStreamSize (path: string, existingProbe?: FfprobeData): Promise<{ width: number, height: number }> { + const videoStream = await getVideoStreamFromFile(path, existingProbe) + + return videoStream === null + ? { width: 0, height: 0 } + : { width: videoStream.width, height: videoStream.height } +} + +async function getVideoFileResolution (path: string, existingProbe?: FfprobeData) { + const size = await getVideoStreamSize(path, existingProbe) + + return { + width: size.width, + height: size.height, + ratio: Math.max(size.height, size.width) / Math.min(size.height, size.width), + resolution: Math.min(size.height, size.width), + isPortraitMode: size.height > size.width + } +} + +async function getVideoFileFPS (path: string, existingProbe?: FfprobeData) { + const videoStream = await getVideoStreamFromFile(path, existingProbe) + if (videoStream === null) return 0 + + for (const key of [ 'avg_frame_rate', 'r_frame_rate' ]) { + const valuesText: string = videoStream[key] + if (!valuesText) continue + + const [ frames, seconds ] = valuesText.split('/') + if (!frames || !seconds) continue + + const result = parseInt(frames, 10) / parseInt(seconds, 10) + if (result > 0) return Math.round(result) + } + + return 0 +} + +async function getMetadataFromFile (path: string, existingProbe?: FfprobeData) { + const metadata = existingProbe || await ffprobePromise(path) + + return new VideoFileMetadata(metadata) +} + +async function getVideoFileBitrate (path: string, existingProbe?: FfprobeData): Promise { + const metadata = await getMetadataFromFile(path, existingProbe) + + let bitrate = metadata.format.bit_rate as number + if (bitrate && !isNaN(bitrate)) return bitrate + + const videoStream = await getVideoStreamFromFile(path, existingProbe) + if (!videoStream) return undefined + + bitrate = videoStream?.bit_rate + if (bitrate && !isNaN(bitrate)) return bitrate + + return undefined +} + +async function getDurationFromVideoFile (path: string, existingProbe?: FfprobeData) { + const metadata = await getMetadataFromFile(path, existingProbe) + + return Math.round(metadata.format.duration) +} + +async function getVideoStreamFromFile (path: string, existingProbe?: FfprobeData) { + const metadata = await getMetadataFromFile(path, existingProbe) + + return metadata.streams.find(s => s.codec_type === 'video') || null +} + +async function canDoQuickAudioTranscode (path: string, probe?: FfprobeData): Promise { + const parsedAudio = await getAudioStream(path, probe) + + if (!parsedAudio.audioStream) return true + + if (parsedAudio.audioStream['codec_name'] !== 'aac') return false + + const audioBitrate = parsedAudio.bitrate + if (!audioBitrate) return false + + const maxAudioBitrate = getMaxAudioBitrate('aac', audioBitrate) + if (maxAudioBitrate !== -1 && audioBitrate > maxAudioBitrate) return false + + const channelLayout = parsedAudio.audioStream['channel_layout'] + // Causes playback issues with Chrome + if (!channelLayout || channelLayout === 'unknown') return false + + return true +} + +// --------------------------------------------------------------------------- + +export { + getVideoStreamSize, + getVideoFileResolution, + getMetadataFromFile, + getMaxAudioBitrate, + getVideoStreamFromFile, + getDurationFromVideoFile, + getAudioStream, + getVideoFileFPS, + ffprobePromise, + getVideoFileBitrate, + canDoQuickAudioTranscode +} diff --git a/shared/extra-utils/miscs/checks.ts b/shared/extra-utils/miscs/checks.ts index b1be214b1..589928997 100644 --- a/shared/extra-utils/miscs/checks.ts +++ b/shared/extra-utils/miscs/checks.ts @@ -3,7 +3,7 @@ import { expect } from 'chai' import { pathExists, readFile } from 'fs-extra' import { join } from 'path' -import { root } from '@server/helpers/core-utils' +import { root } from '@shared/core-utils' import { HttpStatusCode } from '@shared/models' import { makeGetRequest } from '../requests' import { PeerTubeServer } from '../server' diff --git a/shared/extra-utils/miscs/generate.ts b/shared/extra-utils/miscs/generate.ts index 3b29c0ad4..93673a063 100644 --- a/shared/extra-utils/miscs/generate.ts +++ b/shared/extra-utils/miscs/generate.ts @@ -2,7 +2,7 @@ import { expect } from 'chai' import ffmpeg from 'fluent-ffmpeg' import { ensureDir, pathExists } from 'fs-extra' import { dirname } from 'path' -import { getVideoFileBitrate, getVideoFileFPS, getVideoFileResolution } from '@server/helpers/ffprobe-utils' +import { getVideoFileBitrate, getVideoFileFPS, getVideoFileResolution } from '@shared/extra-utils/ffprobe' import { getMaxBitrate } from '@shared/core-utils' import { buildAbsoluteFixturePath } from './tests' diff --git a/shared/extra-utils/requests/activitypub.ts b/shared/extra-utils/requests/activitypub.ts deleted file mode 100644 index 4ae878384..000000000 --- a/shared/extra-utils/requests/activitypub.ts +++ /dev/null @@ -1,42 +0,0 @@ -import { activityPubContextify } from '../../../server/helpers/activitypub' -import { doRequest } from '../../../server/helpers/requests' -import { HTTP_SIGNATURE } from '../../../server/initializers/constants' -import { buildGlobalHeaders } from '../../../server/lib/job-queue/handlers/utils/activitypub-http-utils' - -function makePOSTAPRequest (url: string, body: any, httpSignature: any, headers: any) { - const options = { - method: 'POST' as 'POST', - json: body, - httpSignature, - headers - } - - return doRequest(url, options) -} - -async function makeFollowRequest (to: { url: string }, by: { url: string, privateKey }) { - const follow = { - type: 'Follow', - id: by.url + '/' + new Date().getTime(), - actor: by.url, - object: to.url - } - - const body = activityPubContextify(follow) - - const httpSignature = { - algorithm: HTTP_SIGNATURE.ALGORITHM, - authorizationHeaderName: HTTP_SIGNATURE.HEADER_NAME, - keyId: by.url, - key: by.privateKey, - headers: HTTP_SIGNATURE.HEADERS_TO_SIGN - } - const headers = buildGlobalHeaders(body) - - return makePOSTAPRequest(to.url + '/inbox', body, httpSignature, headers) -} - -export { - makePOSTAPRequest, - makeFollowRequest -} diff --git a/shared/extra-utils/server/directories.ts b/shared/extra-utils/server/directories.ts index b6465cbf4..e6f72d6fc 100644 --- a/shared/extra-utils/server/directories.ts +++ b/shared/extra-utils/server/directories.ts @@ -3,7 +3,7 @@ import { expect } from 'chai' import { pathExists, readdir } from 'fs-extra' import { join } from 'path' -import { root } from '@server/helpers/core-utils' +import { root } from '@shared/core-utils' import { PeerTubeServer } from './server' async function checkTmpIsEmpty (server: PeerTubeServer) { diff --git a/shared/extra-utils/server/plugins-command.ts b/shared/extra-utils/server/plugins-command.ts index 9bf24afff..1c44711da 100644 --- a/shared/extra-utils/server/plugins-command.ts +++ b/shared/extra-utils/server/plugins-command.ts @@ -2,7 +2,7 @@ import { readJSON, writeJSON } from 'fs-extra' import { join } from 'path' -import { root } from '@server/helpers/core-utils' +import { root } from '@shared/core-utils' import { HttpStatusCode, PeerTubePlugin, diff --git a/shared/extra-utils/server/server.ts b/shared/extra-utils/server/server.ts index 9da293877..339b9cabb 100644 --- a/shared/extra-utils/server/server.ts +++ b/shared/extra-utils/server/server.ts @@ -1,8 +1,7 @@ import { ChildProcess, fork } from 'child_process' import { copy } from 'fs-extra' import { join } from 'path' -import { root } from '@server/helpers/core-utils' -import { randomInt } from '@shared/core-utils' +import { root, randomInt } from '@shared/core-utils' import { Video, VideoChannel, VideoCreateResult, VideoDetails } from '../../models/videos' import { BulkCommand } from '../bulk' import { CLICommand } from '../cli' diff --git a/shared/extra-utils/server/servers-command.ts b/shared/extra-utils/server/servers-command.ts index 776d2123c..47420c95f 100644 --- a/shared/extra-utils/server/servers-command.ts +++ b/shared/extra-utils/server/servers-command.ts @@ -1,7 +1,7 @@ import { exec } from 'child_process' import { copy, ensureDir, readFile, remove } from 'fs-extra' import { basename, join } from 'path' -import { root } from '@server/helpers/core-utils' +import { root } from '@shared/core-utils' import { HttpStatusCode } from '@shared/models' import { getFileSize, isGithubCI, wait } from '../miscs' import { AbstractCommand, OverrideCommandOptions } from '../shared' diff --git a/shared/extra-utils/server/tracker.ts b/shared/extra-utils/server/tracker.ts index f04e8f8a1..ed43a5924 100644 --- a/shared/extra-utils/server/tracker.ts +++ b/shared/extra-utils/server/tracker.ts @@ -1,5 +1,5 @@ import { expect } from 'chai' -import { sha1 } from '@server/helpers/core-utils' +import { sha1 } from '@shared/core-utils/crypto' import { makeGetRequest } from '../requests' async function hlsInfohashExist (serverUrl: string, masterPlaylistUrl: string, fileNumber: number) { diff --git a/shared/extra-utils/users/actors.ts b/shared/extra-utils/users/actors.ts index cfcc7d0a7..12c3e078a 100644 --- a/shared/extra-utils/users/actors.ts +++ b/shared/extra-utils/users/actors.ts @@ -3,7 +3,7 @@ import { expect } from 'chai' import { pathExists, readdir } from 'fs-extra' import { join } from 'path' -import { root } from '@server/helpers/core-utils' +import { root } from '@shared/core-utils' import { Account, VideoChannel } from '@shared/models' import { PeerTubeServer } from '../server' diff --git a/shared/extra-utils/videos/streaming-playlists.ts b/shared/extra-utils/videos/streaming-playlists.ts index 6671e3fa6..0451c0efe 100644 --- a/shared/extra-utils/videos/streaming-playlists.ts +++ b/shared/extra-utils/videos/streaming-playlists.ts @@ -1,6 +1,6 @@ import { expect } from 'chai' import { basename } from 'path' -import { sha256 } from '@server/helpers/core-utils' +import { sha256 } from '@shared/core-utils/crypto' import { removeFragmentedMP4Ext } from '@shared/core-utils' import { HttpStatusCode, VideoStreamingPlaylist } from '@shared/models' import { PeerTubeServer } from '../server' diff --git a/shared/extra-utils/videos/videos-command.ts b/shared/extra-utils/videos/videos-command.ts index 7ec9c3647..8ea828b40 100644 --- a/shared/extra-utils/videos/videos-command.ts +++ b/shared/extra-utils/videos/videos-command.ts @@ -5,8 +5,7 @@ import { createReadStream, stat } from 'fs-extra' import got, { Response as GotResponse } from 'got' import { omit } from 'lodash' import validator from 'validator' -import { buildUUID } from '@server/helpers/uuid' -import { loadLanguages } from '@server/initializers/constants' +import { buildUUID } from '@shared/core-utils/uuid' import { pick } from '@shared/core-utils' import { HttpStatusCode, @@ -23,7 +22,7 @@ import { } from '@shared/models' import { buildAbsoluteFixturePath, wait } from '../miscs' import { unwrapBody } from '../requests' -import { PeerTubeServer, waitJobs } from '../server' +import { waitJobs } from '../server' import { AbstractCommand, OverrideCommandOptions } from '../shared' export type VideoEdit = Partial> & { @@ -33,13 +32,6 @@ export type VideoEdit = Partial f.resolution.id === attributeFile.resolution) - expect(file).not.to.be.undefined - - let extension = getLowercaseExtension(attributes.fixture) - // Transcoding enabled: extension will always be .mp4 - if (attributes.files.length > 1) extension = '.mp4' - - expect(file.magnetUri).to.have.lengthOf.above(2) - - expect(file.torrentDownloadUrl).to.match(new RegExp(`http://${host}/download/torrents/${uuidRegex}-${file.resolution.id}.torrent`)) - expect(file.torrentUrl).to.match(new RegExp(`http://${host}/lazy-static/torrents/${uuidRegex}-${file.resolution.id}.torrent`)) - - expect(file.fileUrl).to.match(new RegExp(`http://${originHost}/static/webseed/${uuidRegex}-${file.resolution.id}${extension}`)) - expect(file.fileDownloadUrl).to.match(new RegExp(`http://${originHost}/download/videos/${uuidRegex}-${file.resolution.id}${extension}`)) - - await Promise.all([ - makeRawRequest(file.torrentUrl, 200), - makeRawRequest(file.torrentDownloadUrl, 200), - makeRawRequest(file.metadataUrl, 200) - ]) - - expect(file.resolution.id).to.equal(attributeFile.resolution) - expect(file.resolution.label).to.equal(attributeFile.resolution + 'p') - - const minSize = attributeFile.size - ((10 * attributeFile.size) / 100) - const maxSize = attributeFile.size + ((10 * attributeFile.size) / 100) - expect( - file.size, - 'File size for resolution ' + file.resolution.label + ' outside confidence interval (' + minSize + '> size <' + maxSize + ')' - ).to.be.above(minSize).and.below(maxSize) - - const torrent = await webtorrentAdd(file.magnetUri, true) - expect(torrent.files).to.be.an('array') - expect(torrent.files.length).to.equal(1) - expect(torrent.files[0].path).to.exist.and.to.not.equal('') - expect(torrent.files[0].name).to.equal(`${videoDetails.name} ${file.resolution.id}p${extension}`) - } - - expect(videoDetails.thumbnailPath).to.exist - await testImage(server.url, attributes.thumbnailfile || attributes.fixture, videoDetails.thumbnailPath) - - if (attributes.previewfile) { - expect(videoDetails.previewPath).to.exist - await testImage(server.url, attributes.previewfile, videoDetails.previewPath) - } -} - // serverNumber starts from 1 async function uploadRandomVideoOnServers ( servers: PeerTubeServer[], @@ -247,7 +98,6 @@ async function uploadRandomVideoOnServers ( export { checkUploadVideoParam, - completeVideoCheck, uploadRandomVideoOnServers, checkVideoFilesWereRemoved, saveVideoInServers -- cgit v1.2.3