From 3a4992633ee62d5edfbb484d9c6bcb3cf158489d Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Mon, 31 Jul 2023 14:34:36 +0200 Subject: Migrate server to ESM Sorry for the very big commit that may lead to git log issues and merge conflicts, but it's a major step forward: * Server can be faster at startup because imports() are async and we can easily lazy import big modules * Angular doesn't seem to support ES import (with .js extension), so we had to correctly organize peertube into a monorepo: * Use yarn workspace feature * Use typescript reference projects for dependencies * Shared projects have been moved into "packages", each one is now a node module (with a dedicated package.json/tsconfig.json) * server/tools have been moved into apps/ and is now a dedicated app bundled and published on NPM so users don't have to build peertube cli tools manually * server/tests have been moved into packages/ so we don't compile them every time we want to run the server * Use isolatedModule option: * Had to move from const enum to const (https://www.typescriptlang.org/docs/handbook/enums.html#objects-vs-enums) * Had to explictely specify "type" imports when used in decorators * Prefer tsx (that uses esbuild under the hood) instead of ts-node to load typescript files (tests with mocha or scripts): * To reduce test complexity as esbuild doesn't support decorator metadata, we only test server files that do not import server models * We still build tests files into js files for a faster CI * Remove unmaintained peertube CLI import script * Removed some barrels to speed up execution (less imports) --- server/lib/object-storage/index.ts | 5 - server/lib/object-storage/keys.ts | 20 -- server/lib/object-storage/pre-signed-urls.ts | 46 --- server/lib/object-storage/proxy.ts | 97 ------ server/lib/object-storage/shared/client.ts | 71 ----- server/lib/object-storage/shared/index.ts | 3 - server/lib/object-storage/shared/logger.ts | 7 - .../shared/object-storage-helpers.ts | 328 --------------------- server/lib/object-storage/urls.ts | 63 ---- server/lib/object-storage/videos.ts | 197 ------------- 10 files changed, 837 deletions(-) delete mode 100644 server/lib/object-storage/index.ts delete mode 100644 server/lib/object-storage/keys.ts delete mode 100644 server/lib/object-storage/pre-signed-urls.ts delete mode 100644 server/lib/object-storage/proxy.ts delete mode 100644 server/lib/object-storage/shared/client.ts delete mode 100644 server/lib/object-storage/shared/index.ts delete mode 100644 server/lib/object-storage/shared/logger.ts delete mode 100644 server/lib/object-storage/shared/object-storage-helpers.ts delete mode 100644 server/lib/object-storage/urls.ts delete mode 100644 server/lib/object-storage/videos.ts (limited to 'server/lib/object-storage') diff --git a/server/lib/object-storage/index.ts b/server/lib/object-storage/index.ts deleted file mode 100644 index 3ad6cab63..000000000 --- a/server/lib/object-storage/index.ts +++ /dev/null @@ -1,5 +0,0 @@ -export * from './keys' -export * from './proxy' -export * from './pre-signed-urls' -export * from './urls' -export * from './videos' diff --git a/server/lib/object-storage/keys.ts b/server/lib/object-storage/keys.ts deleted file mode 100644 index 6d2098298..000000000 --- a/server/lib/object-storage/keys.ts +++ /dev/null @@ -1,20 +0,0 @@ -import { join } from 'path' -import { MStreamingPlaylistVideo } from '@server/types/models' - -function generateHLSObjectStorageKey (playlist: MStreamingPlaylistVideo, filename: string) { - return join(generateHLSObjectBaseStorageKey(playlist), filename) -} - -function generateHLSObjectBaseStorageKey (playlist: MStreamingPlaylistVideo) { - return join(playlist.getStringType(), playlist.Video.uuid) -} - -function generateWebVideoObjectStorageKey (filename: string) { - return filename -} - -export { - generateHLSObjectStorageKey, - generateHLSObjectBaseStorageKey, - generateWebVideoObjectStorageKey -} diff --git a/server/lib/object-storage/pre-signed-urls.ts b/server/lib/object-storage/pre-signed-urls.ts deleted file mode 100644 index caf149bb8..000000000 --- a/server/lib/object-storage/pre-signed-urls.ts +++ /dev/null @@ -1,46 +0,0 @@ -import { GetObjectCommand } from '@aws-sdk/client-s3' -import { getSignedUrl } from '@aws-sdk/s3-request-presigner' -import { CONFIG } from '@server/initializers/config' -import { MStreamingPlaylistVideo, MVideoFile } from '@server/types/models' -import { generateHLSObjectStorageKey, generateWebVideoObjectStorageKey } from './keys' -import { buildKey, getClient } from './shared' -import { getHLSPublicFileUrl, getWebVideoPublicFileUrl } from './urls' - -export async function generateWebVideoPresignedUrl (options: { - file: MVideoFile - downloadFilename: string -}) { - const { file, downloadFilename } = options - - const key = generateWebVideoObjectStorageKey(file.filename) - - const command = new GetObjectCommand({ - Bucket: CONFIG.OBJECT_STORAGE.WEB_VIDEOS.BUCKET_NAME, - Key: buildKey(key, CONFIG.OBJECT_STORAGE.WEB_VIDEOS), - ResponseContentDisposition: `attachment; filename=${downloadFilename}` - }) - - const url = await getSignedUrl(getClient(), command, { expiresIn: 3600 * 24 }) - - return getWebVideoPublicFileUrl(url) -} - -export async function generateHLSFilePresignedUrl (options: { - streamingPlaylist: MStreamingPlaylistVideo - file: MVideoFile - downloadFilename: string -}) { - const { streamingPlaylist, file, downloadFilename } = options - - const key = generateHLSObjectStorageKey(streamingPlaylist, file.filename) - - const command = new GetObjectCommand({ - Bucket: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS.BUCKET_NAME, - Key: buildKey(key, CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS), - ResponseContentDisposition: `attachment; filename=${downloadFilename}` - }) - - const url = await getSignedUrl(getClient(), command, { expiresIn: 3600 * 24 }) - - return getHLSPublicFileUrl(url) -} diff --git a/server/lib/object-storage/proxy.ts b/server/lib/object-storage/proxy.ts deleted file mode 100644 index c09a0d1b0..000000000 --- a/server/lib/object-storage/proxy.ts +++ /dev/null @@ -1,97 +0,0 @@ -import express from 'express' -import { PassThrough, pipeline } from 'stream' -import { GetObjectCommandOutput } from '@aws-sdk/client-s3' -import { buildReinjectVideoFileTokenQuery } from '@server/controllers/shared/m3u8-playlist' -import { logger } from '@server/helpers/logger' -import { StreamReplacer } from '@server/helpers/stream-replacer' -import { MStreamingPlaylist, MVideo } from '@server/types/models' -import { HttpStatusCode } from '@shared/models' -import { injectQueryToPlaylistUrls } from '../hls' -import { getHLSFileReadStream, getWebVideoFileReadStream } from './videos' - -export async function proxifyWebVideoFile (options: { - req: express.Request - res: express.Response - filename: string -}) { - const { req, res, filename } = options - - logger.debug('Proxifying Web Video file %s from object storage.', filename) - - try { - const { response: s3Response, stream } = await getWebVideoFileReadStream({ - filename, - rangeHeader: req.header('range') - }) - - setS3Headers(res, s3Response) - - return stream.pipe(res) - } catch (err) { - return handleObjectStorageFailure(res, err) - } -} - -export async function proxifyHLS (options: { - req: express.Request - res: express.Response - playlist: MStreamingPlaylist - video: MVideo - filename: string - reinjectVideoFileToken: boolean -}) { - const { req, res, playlist, video, filename, reinjectVideoFileToken } = options - - logger.debug('Proxifying HLS file %s from object storage.', filename) - - try { - const { response: s3Response, stream } = await getHLSFileReadStream({ - playlist: playlist.withVideo(video), - filename, - rangeHeader: req.header('range') - }) - - setS3Headers(res, s3Response) - - const streamReplacer = reinjectVideoFileToken - ? new StreamReplacer(line => injectQueryToPlaylistUrls(line, buildReinjectVideoFileTokenQuery(req, filename.endsWith('master.m3u8')))) - : new PassThrough() - - return pipeline( - stream, - streamReplacer, - res, - err => { - if (!err) return - - handleObjectStorageFailure(res, err) - } - ) - } catch (err) { - return handleObjectStorageFailure(res, err) - } -} - -// --------------------------------------------------------------------------- -// Private -// --------------------------------------------------------------------------- - -function handleObjectStorageFailure (res: express.Response, err: Error) { - if (err.name === 'NoSuchKey') { - logger.debug('Could not find key in object storage to proxify private HLS video file.', { err }) - return res.sendStatus(HttpStatusCode.NOT_FOUND_404) - } - - return res.fail({ - status: HttpStatusCode.INTERNAL_SERVER_ERROR_500, - message: err.message, - type: err.name - }) -} - -function setS3Headers (res: express.Response, s3Response: GetObjectCommandOutput) { - if (s3Response.$metadata.httpStatusCode === HttpStatusCode.PARTIAL_CONTENT_206) { - res.setHeader('Content-Range', s3Response.ContentRange) - res.status(HttpStatusCode.PARTIAL_CONTENT_206) - } -} diff --git a/server/lib/object-storage/shared/client.ts b/server/lib/object-storage/shared/client.ts deleted file mode 100644 index d5cb074df..000000000 --- a/server/lib/object-storage/shared/client.ts +++ /dev/null @@ -1,71 +0,0 @@ -import { S3Client } from '@aws-sdk/client-s3' -import { NodeHttpHandler } from '@aws-sdk/node-http-handler' -import { logger } from '@server/helpers/logger' -import { isProxyEnabled } from '@server/helpers/proxy' -import { getAgent } from '@server/helpers/requests' -import { CONFIG } from '@server/initializers/config' -import { lTags } from './logger' - -function getProxyRequestHandler () { - if (!isProxyEnabled()) return null - - const { agent } = getAgent() - - return new NodeHttpHandler({ - httpAgent: agent.http, - httpsAgent: agent.https - }) -} - -let endpointParsed: URL -function getEndpointParsed () { - if (endpointParsed) return endpointParsed - - endpointParsed = new URL(getEndpoint()) - - return endpointParsed -} - -let s3Client: S3Client -function getClient () { - if (s3Client) return s3Client - - const OBJECT_STORAGE = CONFIG.OBJECT_STORAGE - - s3Client = new S3Client({ - endpoint: getEndpoint(), - region: OBJECT_STORAGE.REGION, - credentials: OBJECT_STORAGE.CREDENTIALS.ACCESS_KEY_ID - ? { - accessKeyId: OBJECT_STORAGE.CREDENTIALS.ACCESS_KEY_ID, - secretAccessKey: OBJECT_STORAGE.CREDENTIALS.SECRET_ACCESS_KEY - } - : undefined, - requestHandler: getProxyRequestHandler() - }) - - logger.info('Initialized S3 client %s with region %s.', getEndpoint(), OBJECT_STORAGE.REGION, lTags()) - - return s3Client -} - -// --------------------------------------------------------------------------- - -export { - getEndpointParsed, - getClient -} - -// --------------------------------------------------------------------------- - -let endpoint: string -function getEndpoint () { - if (endpoint) return endpoint - - const endpointConfig = CONFIG.OBJECT_STORAGE.ENDPOINT - endpoint = endpointConfig.startsWith('http://') || endpointConfig.startsWith('https://') - ? CONFIG.OBJECT_STORAGE.ENDPOINT - : 'https://' + CONFIG.OBJECT_STORAGE.ENDPOINT - - return endpoint -} diff --git a/server/lib/object-storage/shared/index.ts b/server/lib/object-storage/shared/index.ts deleted file mode 100644 index 11e10aa9f..000000000 --- a/server/lib/object-storage/shared/index.ts +++ /dev/null @@ -1,3 +0,0 @@ -export * from './client' -export * from './logger' -export * from './object-storage-helpers' diff --git a/server/lib/object-storage/shared/logger.ts b/server/lib/object-storage/shared/logger.ts deleted file mode 100644 index 8ab7cbd71..000000000 --- a/server/lib/object-storage/shared/logger.ts +++ /dev/null @@ -1,7 +0,0 @@ -import { loggerTagsFactory } from '@server/helpers/logger' - -const lTags = loggerTagsFactory('object-storage') - -export { - lTags -} diff --git a/server/lib/object-storage/shared/object-storage-helpers.ts b/server/lib/object-storage/shared/object-storage-helpers.ts deleted file mode 100644 index 0d8878bd2..000000000 --- a/server/lib/object-storage/shared/object-storage-helpers.ts +++ /dev/null @@ -1,328 +0,0 @@ -import { map } from 'bluebird' -import { createReadStream, createWriteStream, ensureDir, ReadStream } from 'fs-extra' -import { dirname } from 'path' -import { Readable } from 'stream' -import { - _Object, - CompleteMultipartUploadCommandOutput, - DeleteObjectCommand, - GetObjectCommand, - ListObjectsV2Command, - PutObjectAclCommand, - PutObjectCommandInput, - S3Client -} from '@aws-sdk/client-s3' -import { Upload } from '@aws-sdk/lib-storage' -import { pipelinePromise } from '@server/helpers/core-utils' -import { isArray } from '@server/helpers/custom-validators/misc' -import { logger } from '@server/helpers/logger' -import { CONFIG } from '@server/initializers/config' -import { getInternalUrl } from '../urls' -import { getClient } from './client' -import { lTags } from './logger' - -type BucketInfo = { - BUCKET_NAME: string - PREFIX?: string -} - -async function listKeysOfPrefix (prefix: string, bucketInfo: BucketInfo) { - const s3Client = getClient() - - const commandPrefix = bucketInfo.PREFIX + prefix - const listCommand = new ListObjectsV2Command({ - Bucket: bucketInfo.BUCKET_NAME, - Prefix: commandPrefix - }) - - const listedObjects = await s3Client.send(listCommand) - - if (isArray(listedObjects.Contents) !== true) return [] - - return listedObjects.Contents.map(c => c.Key) -} - -// --------------------------------------------------------------------------- - -async function storeObject (options: { - inputPath: string - objectStorageKey: string - bucketInfo: BucketInfo - isPrivate: boolean -}): Promise { - const { inputPath, objectStorageKey, bucketInfo, isPrivate } = options - - logger.debug('Uploading file %s to %s%s in bucket %s', inputPath, bucketInfo.PREFIX, objectStorageKey, bucketInfo.BUCKET_NAME, lTags()) - - const fileStream = createReadStream(inputPath) - - return uploadToStorage({ objectStorageKey, content: fileStream, bucketInfo, isPrivate }) -} - -async function storeContent (options: { - content: string - inputPath: string - objectStorageKey: string - bucketInfo: BucketInfo - isPrivate: boolean -}): Promise { - const { content, objectStorageKey, bucketInfo, inputPath, isPrivate } = options - - logger.debug('Uploading %s content to %s%s in bucket %s', inputPath, bucketInfo.PREFIX, objectStorageKey, bucketInfo.BUCKET_NAME, lTags()) - - return uploadToStorage({ objectStorageKey, content, bucketInfo, isPrivate }) -} - -// --------------------------------------------------------------------------- - -async function updateObjectACL (options: { - objectStorageKey: string - bucketInfo: BucketInfo - isPrivate: boolean -}) { - const { objectStorageKey, bucketInfo, isPrivate } = options - - const acl = getACL(isPrivate) - if (!acl) return - - const key = buildKey(objectStorageKey, bucketInfo) - - logger.debug('Updating ACL file %s in bucket %s', key, bucketInfo.BUCKET_NAME, lTags()) - - const command = new PutObjectAclCommand({ - Bucket: bucketInfo.BUCKET_NAME, - Key: key, - ACL: acl - }) - - await getClient().send(command) -} - -function updatePrefixACL (options: { - prefix: string - bucketInfo: BucketInfo - isPrivate: boolean -}) { - const { prefix, bucketInfo, isPrivate } = options - - const acl = getACL(isPrivate) - if (!acl) return - - logger.debug('Updating ACL of files in prefix %s in bucket %s', prefix, bucketInfo.BUCKET_NAME, lTags()) - - return applyOnPrefix({ - prefix, - bucketInfo, - commandBuilder: obj => { - logger.debug('Updating ACL of %s inside prefix %s in bucket %s', obj.Key, prefix, bucketInfo.BUCKET_NAME, lTags()) - - return new PutObjectAclCommand({ - Bucket: bucketInfo.BUCKET_NAME, - Key: obj.Key, - ACL: acl - }) - } - }) -} - -// --------------------------------------------------------------------------- - -function removeObject (objectStorageKey: string, bucketInfo: BucketInfo) { - const key = buildKey(objectStorageKey, bucketInfo) - - return removeObjectByFullKey(key, bucketInfo) -} - -function removeObjectByFullKey (fullKey: string, bucketInfo: BucketInfo) { - logger.debug('Removing file %s in bucket %s', fullKey, bucketInfo.BUCKET_NAME, lTags()) - - const command = new DeleteObjectCommand({ - Bucket: bucketInfo.BUCKET_NAME, - Key: fullKey - }) - - return getClient().send(command) -} - -async function removePrefix (prefix: string, bucketInfo: BucketInfo) { - logger.debug('Removing prefix %s in bucket %s', prefix, bucketInfo.BUCKET_NAME, lTags()) - - return applyOnPrefix({ - prefix, - bucketInfo, - commandBuilder: obj => { - logger.debug('Removing %s inside prefix %s in bucket %s', obj.Key, prefix, bucketInfo.BUCKET_NAME, lTags()) - - return new DeleteObjectCommand({ - Bucket: bucketInfo.BUCKET_NAME, - Key: obj.Key - }) - } - }) -} - -// --------------------------------------------------------------------------- - -async function makeAvailable (options: { - key: string - destination: string - bucketInfo: BucketInfo -}) { - const { key, destination, bucketInfo } = options - - await ensureDir(dirname(options.destination)) - - const command = new GetObjectCommand({ - Bucket: bucketInfo.BUCKET_NAME, - Key: buildKey(key, bucketInfo) - }) - const response = await getClient().send(command) - - const file = createWriteStream(destination) - await pipelinePromise(response.Body as Readable, file) - - file.close() -} - -function buildKey (key: string, bucketInfo: BucketInfo) { - return bucketInfo.PREFIX + key -} - -// --------------------------------------------------------------------------- - -async function createObjectReadStream (options: { - key: string - bucketInfo: BucketInfo - rangeHeader: string -}) { - const { key, bucketInfo, rangeHeader } = options - - const command = new GetObjectCommand({ - Bucket: bucketInfo.BUCKET_NAME, - Key: buildKey(key, bucketInfo), - Range: rangeHeader - }) - - const response = await getClient().send(command) - - return { - response, - stream: response.Body as Readable - } -} - -// --------------------------------------------------------------------------- - -export { - BucketInfo, - buildKey, - - storeObject, - storeContent, - - removeObject, - removeObjectByFullKey, - removePrefix, - - makeAvailable, - - updateObjectACL, - updatePrefixACL, - - listKeysOfPrefix, - createObjectReadStream -} - -// --------------------------------------------------------------------------- - -async function uploadToStorage (options: { - content: ReadStream | string - objectStorageKey: string - bucketInfo: BucketInfo - isPrivate: boolean -}) { - const { content, objectStorageKey, bucketInfo, isPrivate } = options - - const input: PutObjectCommandInput = { - Body: content, - Bucket: bucketInfo.BUCKET_NAME, - Key: buildKey(objectStorageKey, bucketInfo) - } - - const acl = getACL(isPrivate) - if (acl) input.ACL = acl - - const parallelUploads3 = new Upload({ - client: getClient(), - queueSize: 4, - partSize: CONFIG.OBJECT_STORAGE.MAX_UPLOAD_PART, - - // `leavePartsOnError` must be set to `true` to avoid silently dropping failed parts - // More detailed explanation: - // https://github.com/aws/aws-sdk-js-v3/blob/v3.164.0/lib/lib-storage/src/Upload.ts#L274 - // https://github.com/aws/aws-sdk-js-v3/issues/2311#issuecomment-939413928 - leavePartsOnError: true, - params: input - }) - - const response = (await parallelUploads3.done()) as CompleteMultipartUploadCommandOutput - // Check is needed even if the HTTP status code is 200 OK - // For more information, see https://docs.aws.amazon.com/AmazonS3/latest/API/API_CompleteMultipartUpload.html - if (!response.Bucket) { - const message = `Error uploading ${objectStorageKey} to bucket ${bucketInfo.BUCKET_NAME}` - logger.error(message, { response, ...lTags() }) - throw new Error(message) - } - - logger.debug( - 'Completed %s%s in bucket %s', - bucketInfo.PREFIX, objectStorageKey, bucketInfo.BUCKET_NAME, { ...lTags(), reseponseMetadata: response.$metadata } - ) - - return getInternalUrl(bucketInfo, objectStorageKey) -} - -async function applyOnPrefix (options: { - prefix: string - bucketInfo: BucketInfo - commandBuilder: (obj: _Object) => Parameters[0] - - continuationToken?: string -}) { - const { prefix, bucketInfo, commandBuilder, continuationToken } = options - - const s3Client = getClient() - - const commandPrefix = buildKey(prefix, bucketInfo) - const listCommand = new ListObjectsV2Command({ - Bucket: bucketInfo.BUCKET_NAME, - Prefix: commandPrefix, - ContinuationToken: continuationToken - }) - - const listedObjects = await s3Client.send(listCommand) - - if (isArray(listedObjects.Contents) !== true) { - const message = `Cannot apply function on ${commandPrefix} prefix in bucket ${bucketInfo.BUCKET_NAME}: no files listed.` - - logger.error(message, { response: listedObjects, ...lTags() }) - throw new Error(message) - } - - await map(listedObjects.Contents, object => { - const command = commandBuilder(object) - - return s3Client.send(command) - }, { concurrency: 10 }) - - // Repeat if not all objects could be listed at once (limit of 1000?) - if (listedObjects.IsTruncated) { - await applyOnPrefix({ ...options, continuationToken: listedObjects.ContinuationToken }) - } -} - -function getACL (isPrivate: boolean) { - return isPrivate - ? CONFIG.OBJECT_STORAGE.UPLOAD_ACL.PRIVATE - : CONFIG.OBJECT_STORAGE.UPLOAD_ACL.PUBLIC -} diff --git a/server/lib/object-storage/urls.ts b/server/lib/object-storage/urls.ts deleted file mode 100644 index 40619cd5a..000000000 --- a/server/lib/object-storage/urls.ts +++ /dev/null @@ -1,63 +0,0 @@ -import { CONFIG } from '@server/initializers/config' -import { OBJECT_STORAGE_PROXY_PATHS, WEBSERVER } from '@server/initializers/constants' -import { MVideoUUID } from '@server/types/models' -import { BucketInfo, buildKey, getEndpointParsed } from './shared' - -function getInternalUrl (config: BucketInfo, keyWithoutPrefix: string) { - return getBaseUrl(config) + buildKey(keyWithoutPrefix, config) -} - -// --------------------------------------------------------------------------- - -function getWebVideoPublicFileUrl (fileUrl: string) { - const baseUrl = CONFIG.OBJECT_STORAGE.WEB_VIDEOS.BASE_URL - if (!baseUrl) return fileUrl - - return replaceByBaseUrl(fileUrl, baseUrl) -} - -function getHLSPublicFileUrl (fileUrl: string) { - const baseUrl = CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS.BASE_URL - if (!baseUrl) return fileUrl - - return replaceByBaseUrl(fileUrl, baseUrl) -} - -// --------------------------------------------------------------------------- - -function getHLSPrivateFileUrl (video: MVideoUUID, filename: string) { - return WEBSERVER.URL + OBJECT_STORAGE_PROXY_PATHS.STREAMING_PLAYLISTS.PRIVATE_HLS + video.uuid + `/${filename}` -} - -function getWebVideoPrivateFileUrl (filename: string) { - return WEBSERVER.URL + OBJECT_STORAGE_PROXY_PATHS.PRIVATE_WEB_VIDEOS + filename -} - -// --------------------------------------------------------------------------- - -export { - getInternalUrl, - - getWebVideoPublicFileUrl, - getHLSPublicFileUrl, - - getHLSPrivateFileUrl, - getWebVideoPrivateFileUrl, - - replaceByBaseUrl -} - -// --------------------------------------------------------------------------- - -function getBaseUrl (bucketInfo: BucketInfo, baseUrl?: string) { - if (baseUrl) return baseUrl - - return `${getEndpointParsed().protocol}//${bucketInfo.BUCKET_NAME}.${getEndpointParsed().host}/` -} - -const regex = new RegExp('https?://[^/]+') -function replaceByBaseUrl (fileUrl: string, baseUrl: string) { - if (!fileUrl) return fileUrl - - return fileUrl.replace(regex, baseUrl) -} diff --git a/server/lib/object-storage/videos.ts b/server/lib/object-storage/videos.ts deleted file mode 100644 index 891e9ff76..000000000 --- a/server/lib/object-storage/videos.ts +++ /dev/null @@ -1,197 +0,0 @@ -import { basename, join } from 'path' -import { logger } from '@server/helpers/logger' -import { CONFIG } from '@server/initializers/config' -import { MStreamingPlaylistVideo, MVideo, MVideoFile } from '@server/types/models' -import { getHLSDirectory } from '../paths' -import { VideoPathManager } from '../video-path-manager' -import { generateHLSObjectBaseStorageKey, generateHLSObjectStorageKey, generateWebVideoObjectStorageKey } from './keys' -import { - createObjectReadStream, - listKeysOfPrefix, - lTags, - makeAvailable, - removeObject, - removeObjectByFullKey, - removePrefix, - storeContent, - storeObject, - updateObjectACL, - updatePrefixACL -} from './shared' - -function listHLSFileKeysOf (playlist: MStreamingPlaylistVideo) { - return listKeysOfPrefix(generateHLSObjectBaseStorageKey(playlist), CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS) -} - -// --------------------------------------------------------------------------- - -function storeHLSFileFromFilename (playlist: MStreamingPlaylistVideo, filename: string) { - return storeObject({ - inputPath: join(getHLSDirectory(playlist.Video), filename), - objectStorageKey: generateHLSObjectStorageKey(playlist, filename), - bucketInfo: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS, - isPrivate: playlist.Video.hasPrivateStaticPath() - }) -} - -function storeHLSFileFromPath (playlist: MStreamingPlaylistVideo, path: string) { - return storeObject({ - inputPath: path, - objectStorageKey: generateHLSObjectStorageKey(playlist, basename(path)), - bucketInfo: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS, - isPrivate: playlist.Video.hasPrivateStaticPath() - }) -} - -function storeHLSFileFromContent (playlist: MStreamingPlaylistVideo, path: string, content: string) { - return storeContent({ - content, - inputPath: path, - objectStorageKey: generateHLSObjectStorageKey(playlist, basename(path)), - bucketInfo: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS, - isPrivate: playlist.Video.hasPrivateStaticPath() - }) -} - -// --------------------------------------------------------------------------- - -function storeWebVideoFile (video: MVideo, file: MVideoFile) { - return storeObject({ - inputPath: VideoPathManager.Instance.getFSVideoFileOutputPath(video, file), - objectStorageKey: generateWebVideoObjectStorageKey(file.filename), - bucketInfo: CONFIG.OBJECT_STORAGE.WEB_VIDEOS, - isPrivate: video.hasPrivateStaticPath() - }) -} - -// --------------------------------------------------------------------------- - -async function updateWebVideoFileACL (video: MVideo, file: MVideoFile) { - await updateObjectACL({ - objectStorageKey: generateWebVideoObjectStorageKey(file.filename), - bucketInfo: CONFIG.OBJECT_STORAGE.WEB_VIDEOS, - isPrivate: video.hasPrivateStaticPath() - }) -} - -async function updateHLSFilesACL (playlist: MStreamingPlaylistVideo) { - await updatePrefixACL({ - prefix: generateHLSObjectBaseStorageKey(playlist), - bucketInfo: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS, - isPrivate: playlist.Video.hasPrivateStaticPath() - }) -} - -// --------------------------------------------------------------------------- - -function removeHLSObjectStorage (playlist: MStreamingPlaylistVideo) { - return removePrefix(generateHLSObjectBaseStorageKey(playlist), CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS) -} - -function removeHLSFileObjectStorageByFilename (playlist: MStreamingPlaylistVideo, filename: string) { - return removeObject(generateHLSObjectStorageKey(playlist, filename), CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS) -} - -function removeHLSFileObjectStorageByPath (playlist: MStreamingPlaylistVideo, path: string) { - return removeObject(generateHLSObjectStorageKey(playlist, basename(path)), CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS) -} - -function removeHLSFileObjectStorageByFullKey (key: string) { - return removeObjectByFullKey(key, CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS) -} - -// --------------------------------------------------------------------------- - -function removeWebVideoObjectStorage (videoFile: MVideoFile) { - return removeObject(generateWebVideoObjectStorageKey(videoFile.filename), CONFIG.OBJECT_STORAGE.WEB_VIDEOS) -} - -// --------------------------------------------------------------------------- - -async function makeHLSFileAvailable (playlist: MStreamingPlaylistVideo, filename: string, destination: string) { - const key = generateHLSObjectStorageKey(playlist, filename) - - logger.info('Fetching HLS file %s from object storage to %s.', key, destination, lTags()) - - await makeAvailable({ - key, - destination, - bucketInfo: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS - }) - - return destination -} - -async function makeWebVideoFileAvailable (filename: string, destination: string) { - const key = generateWebVideoObjectStorageKey(filename) - - logger.info('Fetching Web Video file %s from object storage to %s.', key, destination, lTags()) - - await makeAvailable({ - key, - destination, - bucketInfo: CONFIG.OBJECT_STORAGE.WEB_VIDEOS - }) - - return destination -} - -// --------------------------------------------------------------------------- - -function getWebVideoFileReadStream (options: { - filename: string - rangeHeader: string -}) { - const { filename, rangeHeader } = options - - const key = generateWebVideoObjectStorageKey(filename) - - return createObjectReadStream({ - key, - bucketInfo: CONFIG.OBJECT_STORAGE.WEB_VIDEOS, - rangeHeader - }) -} - -function getHLSFileReadStream (options: { - playlist: MStreamingPlaylistVideo - filename: string - rangeHeader: string -}) { - const { playlist, filename, rangeHeader } = options - - const key = generateHLSObjectStorageKey(playlist, filename) - - return createObjectReadStream({ - key, - bucketInfo: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS, - rangeHeader - }) -} - -// --------------------------------------------------------------------------- - -export { - listHLSFileKeysOf, - - storeWebVideoFile, - storeHLSFileFromFilename, - storeHLSFileFromPath, - storeHLSFileFromContent, - - updateWebVideoFileACL, - updateHLSFilesACL, - - removeHLSObjectStorage, - removeHLSFileObjectStorageByFilename, - removeHLSFileObjectStorageByPath, - removeHLSFileObjectStorageByFullKey, - - removeWebVideoObjectStorage, - - makeWebVideoFileAvailable, - makeHLSFileAvailable, - - getWebVideoFileReadStream, - getHLSFileReadStream -} -- cgit v1.2.3