From 5a122dddc5aab1b2ae1843411032d5f392bdd216 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Wed, 26 Oct 2022 16:23:39 +0200 Subject: [PATCH] Option to disable static files auth check/s3 proxy --- config/default.yaml | 14 +- server/controllers/object-storage-proxy.ts | 8 + server/controllers/static.ts | 20 ++- server/initializers/config.ts | 6 + server/middlewares/validators/index.ts | 1 + .../validators/object-storage-proxy.ts | 20 +++ server/models/video/video-file.ts | 3 +- .../models/video/video-streaming-playlist.ts | 5 +- .../video-static-file-privacy.ts | 166 ++++++++++++------ .../api/videos/video-static-file-privacy.ts | 33 ++++ .../server/object-storage-command.ts | 16 +- 11 files changed, 232 insertions(+), 60 deletions(-) create mode 100644 server/middlewares/validators/object-storage-proxy.ts diff --git a/config/default.yaml b/config/default.yaml index 7753821da..2e249cc76 100644 --- a/config/default.yaml +++ b/config/default.yaml @@ -140,6 +140,10 @@ storage: # If not, peertube will fallback to the default file client_overrides: 'storage/client-overrides/' +static_files: + # Require and check user authentication when accessing private files (internal/private video files) + private_files_require_auth: true + object_storage: enabled: false @@ -151,9 +155,17 @@ object_storage: upload_acl: # Set this ACL on each uploaded object of public/unlisted videos public: 'public-read' - # Set this ACL on each uploaded object of private/internal videos + # Set this ACL on each uploaded object of private/internal videos + # PeerTube can proxify requests to private objects so your users can access them private: 'private' + proxy: + # If private files (private/internal video files) have a private ACL, users can't access directly the ressource + # PeerTube can proxify requests between your object storage service and your users + # If you disable PeerTube proxy, ensure you use your own proxy that is able to access the private files + # Or you can also set a public ACL for private files in object storage if you don't want to use a proxy + proxify_private_files: true + credentials: # You can also use AWS_ACCESS_KEY_ID env variable access_key_id: '' diff --git a/server/controllers/object-storage-proxy.ts b/server/controllers/object-storage-proxy.ts index 6fedcfd8f..3ce279671 100644 --- a/server/controllers/object-storage-proxy.ts +++ b/server/controllers/object-storage-proxy.ts @@ -1,11 +1,13 @@ import cors from 'cors' import express from 'express' +import { logger } from '@server/helpers/logger' import { OBJECT_STORAGE_PROXY_PATHS } from '@server/initializers/constants' import { getHLSFileReadStream, getWebTorrentFileReadStream } from '@server/lib/object-storage' import { asyncMiddleware, ensureCanAccessPrivateVideoHLSFiles, ensureCanAccessVideoPrivateWebTorrentFiles, + ensurePrivateObjectStorageProxyIsEnabled, optionalAuthenticate } from '@server/middlewares' import { HttpStatusCode } from '@shared/models' @@ -15,12 +17,14 @@ const objectStorageProxyRouter = express.Router() objectStorageProxyRouter.use(cors()) objectStorageProxyRouter.get(OBJECT_STORAGE_PROXY_PATHS.PRIVATE_WEBSEED + ':filename', + ensurePrivateObjectStorageProxyIsEnabled, optionalAuthenticate, asyncMiddleware(ensureCanAccessVideoPrivateWebTorrentFiles), asyncMiddleware(proxifyWebTorrent) ) objectStorageProxyRouter.get(OBJECT_STORAGE_PROXY_PATHS.STREAMING_PLAYLISTS.PRIVATE_HLS + ':videoUUID/:filename', + ensurePrivateObjectStorageProxyIsEnabled, optionalAuthenticate, asyncMiddleware(ensureCanAccessPrivateVideoHLSFiles), asyncMiddleware(proxifyHLS) @@ -35,6 +39,8 @@ export { async function proxifyWebTorrent (req: express.Request, res: express.Response) { const filename = req.params.filename + logger.debug('Proxifying WebTorrent file %s from object storage.', filename) + try { const stream = await getWebTorrentFileReadStream({ filename, @@ -52,6 +58,8 @@ async function proxifyHLS (req: express.Request, res: express.Response) { const video = res.locals.onlyVideo const filename = req.params.filename + logger.debug('Proxifying HLS file %s from object storage.', filename) + try { const stream = await getHLSFileReadStream({ playlist: playlist.withVideo(video), diff --git a/server/controllers/static.ts b/server/controllers/static.ts index dc091455a..6ef9154b9 100644 --- a/server/controllers/static.ts +++ b/server/controllers/static.ts @@ -15,11 +15,17 @@ const staticRouter = express.Router() // Cors is very important to let other servers access torrent and video files staticRouter.use(cors()) +// --------------------------------------------------------------------------- // WebTorrent/Classic videos +// --------------------------------------------------------------------------- + +const privateWebTorrentStaticMiddlewares = CONFIG.STATIC_FILES.PRIVATE_FILES_REQUIRE_AUTH === true + ? [ optionalAuthenticate, asyncMiddleware(ensureCanAccessVideoPrivateWebTorrentFiles) ] + : [] + staticRouter.use( STATIC_PATHS.PRIVATE_WEBSEED, - optionalAuthenticate, - asyncMiddleware(ensureCanAccessVideoPrivateWebTorrentFiles), + ...privateWebTorrentStaticMiddlewares, express.static(DIRECTORIES.VIDEOS.PRIVATE, { fallthrough: false }), handleStaticError ) @@ -35,11 +41,17 @@ staticRouter.use( handleStaticError ) +// --------------------------------------------------------------------------- // HLS +// --------------------------------------------------------------------------- + +const privateHLSStaticMiddlewares = CONFIG.STATIC_FILES.PRIVATE_FILES_REQUIRE_AUTH === true + ? [ optionalAuthenticate, asyncMiddleware(ensureCanAccessPrivateVideoHLSFiles) ] + : [] + staticRouter.use( STATIC_PATHS.STREAMING_PLAYLISTS.PRIVATE_HLS, - optionalAuthenticate, - asyncMiddleware(ensureCanAccessPrivateVideoHLSFiles), + ...privateHLSStaticMiddlewares, express.static(DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE, { fallthrough: false }), handleStaticError ) diff --git a/server/initializers/config.ts b/server/initializers/config.ts index ab5e645ad..7652da24a 100644 --- a/server/initializers/config.ts +++ b/server/initializers/config.ts @@ -113,6 +113,9 @@ const CONFIG = { CLIENT_OVERRIDES_DIR: buildPath(config.get('storage.client_overrides')), WELL_KNOWN_DIR: buildPath(config.get('storage.well_known')) }, + STATIC_FILES: { + PRIVATE_FILES_REQUIRE_AUTH: config.get('static_files.private_files_require_auth') + }, OBJECT_STORAGE: { ENABLED: config.get('object_storage.enabled'), MAX_UPLOAD_PART: bytes.parse(config.get('object_storage.max_upload_part')), @@ -126,6 +129,9 @@ const CONFIG = { ACCESS_KEY_ID: config.get('object_storage.credentials.access_key_id'), SECRET_ACCESS_KEY: config.get('object_storage.credentials.secret_access_key') }, + PROXY: { + PROXIFY_PRIVATE_FILES: config.get('object_storage.proxy.proxify_private_files') + }, VIDEOS: { BUCKET_NAME: config.get('object_storage.videos.bucket_name'), PREFIX: config.get('object_storage.videos.prefix'), diff --git a/server/middlewares/validators/index.ts b/server/middlewares/validators/index.ts index 899da229a..9bc8887ff 100644 --- a/server/middlewares/validators/index.ts +++ b/server/middlewares/validators/index.ts @@ -11,6 +11,7 @@ export * from './follows' export * from './jobs' export * from './logs' export * from './metrics' +export * from './object-storage-proxy' export * from './oembed' export * from './pagination' export * from './plugins' diff --git a/server/middlewares/validators/object-storage-proxy.ts b/server/middlewares/validators/object-storage-proxy.ts new file mode 100644 index 000000000..bbd77f262 --- /dev/null +++ b/server/middlewares/validators/object-storage-proxy.ts @@ -0,0 +1,20 @@ +import express from 'express' +import { CONFIG } from '@server/initializers/config' +import { HttpStatusCode } from '@shared/models' + +const ensurePrivateObjectStorageProxyIsEnabled = [ + (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (CONFIG.OBJECT_STORAGE.PROXY.PROXIFY_PRIVATE_FILES !== true) { + return res.fail({ + message: 'Private object storage proxy is not enabled', + status: HttpStatusCode.BAD_REQUEST_400 + }) + } + + return next() + } +] + +export { + ensurePrivateObjectStorageProxyIsEnabled +} diff --git a/server/models/video/video-file.ts b/server/models/video/video-file.ts index c20c90c1b..9c4e6d078 100644 --- a/server/models/video/video-file.ts +++ b/server/models/video/video-file.ts @@ -54,6 +54,7 @@ import { doesExist } from '../shared' import { parseAggregateResult, throwIfNotValid } from '../utils' import { VideoModel } from './video' import { VideoStreamingPlaylistModel } from './video-streaming-playlist' +import { CONFIG } from '@server/initializers/config' export enum ScopeNames { WITH_VIDEO = 'WITH_VIDEO', @@ -511,7 +512,7 @@ export class VideoFileModel extends Model // --------------------------------------------------------------------------- getObjectStorageUrl (video: MVideo) { - if (video.hasPrivateStaticPath()) { + if (video.hasPrivateStaticPath() && CONFIG.OBJECT_STORAGE.PROXY.PROXIFY_PRIVATE_FILES === true) { return this.getPrivateObjectStorageUrl(video) } diff --git a/server/models/video/video-streaming-playlist.ts b/server/models/video/video-streaming-playlist.ts index 1318a4dae..0386edf28 100644 --- a/server/models/video/video-streaming-playlist.ts +++ b/server/models/video/video-streaming-playlist.ts @@ -15,6 +15,7 @@ import { Table, UpdatedAt } from 'sequelize-typescript' +import { CONFIG } from '@server/initializers/config' import { getHLSPrivateFileUrl, getHLSPublicFileUrl } from '@server/lib/object-storage' import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename } from '@server/lib/paths' import { isVideoInPrivateDirectory } from '@server/lib/video-privacy' @@ -260,7 +261,7 @@ export class VideoStreamingPlaylistModel extends Model