From 9ab330b90decf4edf152ff8e1d2948c065766b2c Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Wed, 19 Oct 2022 10:43:53 +0200 Subject: Use private ACL for private videos in s3 --- server/controllers/download.ts | 22 +- server/controllers/index.ts | 11 +- server/controllers/object-storage-proxy.ts | 78 +++++ server/helpers/webtorrent.ts | 4 +- server/initializers/checker-after-init.ts | 8 + server/initializers/config.ts | 5 +- server/initializers/constants.ts | 8 + server/lib/live/live-segment-sha-store.ts | 27 +- .../shared/object-storage-helpers.ts | 192 +++++++++--- server/lib/object-storage/urls.ts | 29 +- server/lib/object-storage/videos.ts | 80 ++++- server/lib/video-privacy.ts | 89 ++++-- server/middlewares/validators/shared/videos.ts | 6 +- server/middlewares/validators/static.ts | 72 +++-- server/models/video/video-file.ts | 62 +++- server/models/video/video-streaming-playlist.ts | 28 +- server/models/video/video.ts | 34 ++- server/tests/api/object-storage/index.ts | 1 + server/tests/api/object-storage/live.ts | 10 +- server/tests/api/object-storage/video-imports.ts | 14 +- .../object-storage/video-static-file-privacy.ts | 336 +++++++++++++++++++++ server/tests/api/object-storage/videos.ts | 36 +-- server/tests/api/server/proxy.ts | 14 +- server/tests/api/transcoding/create-transcoding.ts | 16 +- server/tests/api/transcoding/hls.ts | 10 +- .../api/transcoding/update-while-transcoding.ts | 10 +- server/tests/api/transcoding/video-studio.ts | 12 +- .../tests/api/videos/video-static-file-privacy.ts | 14 +- server/tests/cli/create-import-video-file-job.ts | 10 +- server/tests/cli/create-move-video-storage-job.ts | 16 +- server/tests/cli/create-transcoding-job.ts | 12 +- server/tests/shared/live.ts | 6 +- .../shared/mock-servers/mock-object-storage.ts | 2 +- server/types/express.d.ts | 2 +- 34 files changed, 1036 insertions(+), 240 deletions(-) create mode 100644 server/controllers/object-storage-proxy.ts create mode 100644 server/tests/api/object-storage/video-static-file-privacy.ts (limited to 'server') diff --git a/server/controllers/download.ts b/server/controllers/download.ts index abd1df26f..d9f34109f 100644 --- a/server/controllers/download.ts +++ b/server/controllers/download.ts @@ -5,6 +5,7 @@ import { VideosTorrentCache } from '@server/lib/files-cache/videos-torrent-cache import { Hooks } from '@server/lib/plugins/hooks' import { VideoPathManager } from '@server/lib/video-path-manager' import { MStreamingPlaylist, MVideo, MVideoFile, MVideoFullLight } from '@server/types/models' +import { addQueryParams } from '@shared/core-utils' import { HttpStatusCode, VideoStorage, VideoStreamingPlaylistType } from '@shared/models' import { STATIC_DOWNLOAD_PATHS } from '../initializers/constants' import { asyncMiddleware, optionalAuthenticate, videosDownloadValidator } from '../middlewares' @@ -84,7 +85,7 @@ async function downloadVideoFile (req: express.Request, res: express.Response) { if (!checkAllowResult(res, allowParameters, allowedResult)) return if (videoFile.storage === VideoStorage.OBJECT_STORAGE) { - return res.redirect(videoFile.getObjectStorageUrl()) + return redirectToObjectStorage({ req, res, video, file: videoFile }) } await VideoPathManager.Instance.makeAvailableVideoFile(videoFile.withVideoOrPlaylist(video), path => { @@ -120,7 +121,7 @@ async function downloadHLSVideoFile (req: express.Request, res: express.Response if (!checkAllowResult(res, allowParameters, allowedResult)) return if (videoFile.storage === VideoStorage.OBJECT_STORAGE) { - return res.redirect(videoFile.getObjectStorageUrl()) + return redirectToObjectStorage({ req, res, video, file: videoFile }) } await VideoPathManager.Instance.makeAvailableVideoFile(videoFile.withVideoOrPlaylist(streamingPlaylist), path => { @@ -174,3 +175,20 @@ function checkAllowResult (res: express.Response, allowParameters: any, result?: return true } + +function redirectToObjectStorage (options: { + req: express.Request + res: express.Response + video: MVideo + file: MVideoFile +}) { + const { req, res, video, file } = options + + const baseUrl = file.getObjectStorageUrl(video) + + const url = video.hasPrivateStaticPath() && req.query.videoFileToken + ? addQueryParams(baseUrl, { videoFileToken: req.query.videoFileToken }) + : baseUrl + + return res.redirect(url) +} diff --git a/server/controllers/index.ts b/server/controllers/index.ts index 8574a9e7b..eaa2dd7c8 100644 --- a/server/controllers/index.ts +++ b/server/controllers/index.ts @@ -1,14 +1,15 @@ export * from './activitypub' export * from './api' +export * from './bots' export * from './client' export * from './download' export * from './feeds' -export * from './services' -export * from './static' export * from './lazy-static' export * from './misc' -export * from './webfinger' -export * from './tracker' -export * from './bots' +export * from './object-storage-proxy' export * from './plugins' +export * from './services' +export * from './static' +export * from './tracker' +export * from './webfinger' export * from './well-known' diff --git a/server/controllers/object-storage-proxy.ts b/server/controllers/object-storage-proxy.ts new file mode 100644 index 000000000..6fedcfd8f --- /dev/null +++ b/server/controllers/object-storage-proxy.ts @@ -0,0 +1,78 @@ +import cors from 'cors' +import express from 'express' +import { OBJECT_STORAGE_PROXY_PATHS } from '@server/initializers/constants' +import { getHLSFileReadStream, getWebTorrentFileReadStream } from '@server/lib/object-storage' +import { + asyncMiddleware, + ensureCanAccessPrivateVideoHLSFiles, + ensureCanAccessVideoPrivateWebTorrentFiles, + optionalAuthenticate +} from '@server/middlewares' +import { HttpStatusCode } from '@shared/models' + +const objectStorageProxyRouter = express.Router() + +objectStorageProxyRouter.use(cors()) + +objectStorageProxyRouter.get(OBJECT_STORAGE_PROXY_PATHS.PRIVATE_WEBSEED + ':filename', + optionalAuthenticate, + asyncMiddleware(ensureCanAccessVideoPrivateWebTorrentFiles), + asyncMiddleware(proxifyWebTorrent) +) + +objectStorageProxyRouter.get(OBJECT_STORAGE_PROXY_PATHS.STREAMING_PLAYLISTS.PRIVATE_HLS + ':videoUUID/:filename', + optionalAuthenticate, + asyncMiddleware(ensureCanAccessPrivateVideoHLSFiles), + asyncMiddleware(proxifyHLS) +) + +// --------------------------------------------------------------------------- + +export { + objectStorageProxyRouter +} + +async function proxifyWebTorrent (req: express.Request, res: express.Response) { + const filename = req.params.filename + + try { + const stream = await getWebTorrentFileReadStream({ + filename, + rangeHeader: req.header('range') + }) + + return stream.pipe(res) + } catch (err) { + return handleObjectStorageFailure(res, err) + } +} + +async function proxifyHLS (req: express.Request, res: express.Response) { + const playlist = res.locals.videoStreamingPlaylist + const video = res.locals.onlyVideo + const filename = req.params.filename + + try { + const stream = await getHLSFileReadStream({ + playlist: playlist.withVideo(video), + filename, + rangeHeader: req.header('range') + }) + + return stream.pipe(res) + } catch (err) { + return handleObjectStorageFailure(res, err) + } +} + +function handleObjectStorageFailure (res: express.Response, err: Error) { + if (err.name === 'NoSuchKey') { + return res.sendStatus(HttpStatusCode.NOT_FOUND_404) + } + + return res.fail({ + status: HttpStatusCode.INTERNAL_SERVER_ERROR_500, + message: err.message, + type: err.name + }) +} diff --git a/server/helpers/webtorrent.ts b/server/helpers/webtorrent.ts index 6d87c74f7..b458e86d2 100644 --- a/server/helpers/webtorrent.ts +++ b/server/helpers/webtorrent.ts @@ -165,7 +165,7 @@ function generateMagnetUri ( const xs = videoFile.getTorrentUrl() const announce = trackerUrls - let urlList = video.requiresAuth(video.uuid) + let urlList = video.hasPrivateStaticPath() ? [] : [ videoFile.getFileUrl(video) ] @@ -243,7 +243,7 @@ function buildAnnounceList () { } function buildUrlList (video: MVideo, videoFile: MVideoFile) { - if (video.requiresAuth(video.uuid)) return [] + if (video.hasPrivateStaticPath()) return [] return [ videoFile.getFileUrl(video) ] } diff --git a/server/initializers/checker-after-init.ts b/server/initializers/checker-after-init.ts index c83fef425..09e878eee 100644 --- a/server/initializers/checker-after-init.ts +++ b/server/initializers/checker-after-init.ts @@ -278,6 +278,14 @@ function checkObjectStorageConfig () { 'Object storage bucket prefixes should be set to different values when the same bucket is used for both types of video.' ) } + + if (!CONFIG.OBJECT_STORAGE.UPLOAD_ACL.PUBLIC) { + throw new Error('object_storage.upload_acl.public must be set') + } + + if (!CONFIG.OBJECT_STORAGE.UPLOAD_ACL.PRIVATE) { + throw new Error('object_storage.upload_acl.private must be set') + } } } diff --git a/server/initializers/config.ts b/server/initializers/config.ts index a5a0d4e46..ab5e645ad 100644 --- a/server/initializers/config.ts +++ b/server/initializers/config.ts @@ -118,7 +118,10 @@ const CONFIG = { MAX_UPLOAD_PART: bytes.parse(config.get('object_storage.max_upload_part')), ENDPOINT: config.get('object_storage.endpoint'), REGION: config.get('object_storage.region'), - UPLOAD_ACL: config.get('object_storage.upload_acl'), + UPLOAD_ACL: { + PUBLIC: config.get('object_storage.upload_acl.public'), + PRIVATE: config.get('object_storage.upload_acl.private') + }, CREDENTIALS: { ACCESS_KEY_ID: config.get('object_storage.credentials.access_key_id'), SECRET_ACCESS_KEY: config.get('object_storage.credentials.secret_access_key') diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index 88bdd07fe..66eb31230 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts @@ -685,6 +685,13 @@ const LAZY_STATIC_PATHS = { VIDEO_CAPTIONS: '/lazy-static/video-captions/', TORRENTS: '/lazy-static/torrents/' } +const OBJECT_STORAGE_PROXY_PATHS = { + PRIVATE_WEBSEED: '/object-storage-proxy/webseed/private/', + + STREAMING_PLAYLISTS: { + PRIVATE_HLS: '/object-storage-proxy/streaming-playlists/hls/private/' + } +} // Cache control const STATIC_MAX_AGE = { @@ -995,6 +1002,7 @@ export { VIDEO_LIVE, PEERTUBE_VERSION, LAZY_STATIC_PATHS, + OBJECT_STORAGE_PROXY_PATHS, SEARCH_INDEX, DIRECTORIES, RESUMABLE_UPLOAD_SESSION_LIFETIME, diff --git a/server/lib/live/live-segment-sha-store.ts b/server/lib/live/live-segment-sha-store.ts index faf03dccf..4d03754a9 100644 --- a/server/lib/live/live-segment-sha-store.ts +++ b/server/lib/live/live-segment-sha-store.ts @@ -5,6 +5,7 @@ import { logger, loggerTagsFactory } from '@server/helpers/logger' import { MStreamingPlaylistVideo } from '@server/types/models' import { buildSha256Segment } from '../hls' import { storeHLSFileFromPath } from '../object-storage' +import PQueue from 'p-queue' const lTags = loggerTagsFactory('live') @@ -16,6 +17,7 @@ class LiveSegmentShaStore { private readonly sha256Path: string private readonly streamingPlaylist: MStreamingPlaylistVideo private readonly sendToObjectStorage: boolean + private readonly writeQueue = new PQueue({ concurrency: 1 }) constructor (options: { videoUUID: string @@ -37,7 +39,11 @@ class LiveSegmentShaStore { const segmentName = basename(segmentPath) this.segmentsSha256.set(segmentName, shaResult) - await this.writeToDisk() + try { + await this.writeToDisk() + } catch (err) { + logger.error('Cannot write sha segments to disk.', { err }) + } } async removeSegmentSha (segmentPath: string) { @@ -55,19 +61,20 @@ class LiveSegmentShaStore { await this.writeToDisk() } - private async writeToDisk () { - await writeJson(this.sha256Path, mapToJSON(this.segmentsSha256)) + private writeToDisk () { + return this.writeQueue.add(async () => { + await writeJson(this.sha256Path, mapToJSON(this.segmentsSha256)) - if (this.sendToObjectStorage) { - const url = await storeHLSFileFromPath(this.streamingPlaylist, this.sha256Path) + if (this.sendToObjectStorage) { + const url = await storeHLSFileFromPath(this.streamingPlaylist, this.sha256Path) - if (this.streamingPlaylist.segmentsSha256Url !== url) { - this.streamingPlaylist.segmentsSha256Url = url - await this.streamingPlaylist.save() + if (this.streamingPlaylist.segmentsSha256Url !== url) { + this.streamingPlaylist.segmentsSha256Url = url + await this.streamingPlaylist.save() + } } - } + }) } - } export { diff --git a/server/lib/object-storage/shared/object-storage-helpers.ts b/server/lib/object-storage/shared/object-storage-helpers.ts index c131977e8..05b52f412 100644 --- a/server/lib/object-storage/shared/object-storage-helpers.ts +++ b/server/lib/object-storage/shared/object-storage-helpers.ts @@ -2,18 +2,21 @@ import { createReadStream, createWriteStream, ensureDir, ReadStream } from 'fs-e import { dirname } from 'path' import { Readable } from 'stream' import { + _Object, CompleteMultipartUploadCommandOutput, DeleteObjectCommand, GetObjectCommand, ListObjectsV2Command, - PutObjectCommandInput + PutObjectAclCommand, + PutObjectCommandInput, + S3Client } from '@aws-sdk/client-s3' import { Upload } from '@aws-sdk/lib-storage' import { pipelinePromise } from '@server/helpers/core-utils' import { isArray } from '@server/helpers/custom-validators/misc' import { logger } from '@server/helpers/logger' import { CONFIG } from '@server/initializers/config' -import { getPrivateUrl } from '../urls' +import { getInternalUrl } from '../urls' import { getClient } from './client' import { lTags } from './logger' @@ -44,69 +47,91 @@ async function storeObject (options: { inputPath: string objectStorageKey: string bucketInfo: BucketInfo + isPrivate: boolean }): Promise { - const { inputPath, objectStorageKey, bucketInfo } = options + const { inputPath, objectStorageKey, bucketInfo, isPrivate } = options logger.debug('Uploading file %s to %s%s in bucket %s', inputPath, bucketInfo.PREFIX, objectStorageKey, bucketInfo.BUCKET_NAME, lTags()) const fileStream = createReadStream(inputPath) - return uploadToStorage({ objectStorageKey, content: fileStream, bucketInfo }) + return uploadToStorage({ objectStorageKey, content: fileStream, bucketInfo, isPrivate }) } // --------------------------------------------------------------------------- -async function removeObject (filename: string, bucketInfo: BucketInfo) { - const command = new DeleteObjectCommand({ +function updateObjectACL (options: { + objectStorageKey: string + bucketInfo: BucketInfo + isPrivate: boolean +}) { + const { objectStorageKey, bucketInfo, isPrivate } = options + + const key = buildKey(objectStorageKey, bucketInfo) + + logger.debug('Updating ACL file %s in bucket %s', key, bucketInfo.BUCKET_NAME, lTags()) + + const command = new PutObjectAclCommand({ Bucket: bucketInfo.BUCKET_NAME, - Key: buildKey(filename, bucketInfo) + Key: key, + ACL: getACL(isPrivate) }) return getClient().send(command) } -async function removePrefix (prefix: string, bucketInfo: BucketInfo) { - const s3Client = getClient() - - const commandPrefix = bucketInfo.PREFIX + prefix - const listCommand = new ListObjectsV2Command({ - Bucket: bucketInfo.BUCKET_NAME, - Prefix: commandPrefix +function updatePrefixACL (options: { + prefix: string + bucketInfo: BucketInfo + isPrivate: boolean +}) { + const { prefix, bucketInfo, isPrivate } = options + + logger.debug('Updating ACL of files in prefix %s in bucket %s', prefix, bucketInfo.BUCKET_NAME, lTags()) + + return applyOnPrefix({ + prefix, + bucketInfo, + commandBuilder: obj => { + return new PutObjectAclCommand({ + Bucket: bucketInfo.BUCKET_NAME, + Key: obj.Key, + ACL: getACL(isPrivate) + }) + } }) +} - const listedObjects = await s3Client.send(listCommand) +// --------------------------------------------------------------------------- - // FIXME: use bulk delete when s3ninja will support this operation - // const deleteParams = { - // Bucket: bucketInfo.BUCKET_NAME, - // Delete: { Objects: [] } - // } +function removeObject (objectStorageKey: string, bucketInfo: BucketInfo) { + const key = buildKey(objectStorageKey, bucketInfo) - if (isArray(listedObjects.Contents) !== true) { - const message = `Cannot remove ${commandPrefix} prefix in bucket ${bucketInfo.BUCKET_NAME}: no files listed.` + logger.debug('Removing file %s in bucket %s', key, bucketInfo.BUCKET_NAME, lTags()) - logger.error(message, { response: listedObjects, ...lTags() }) - throw new Error(message) - } - - for (const object of listedObjects.Contents) { - const command = new DeleteObjectCommand({ - Bucket: bucketInfo.BUCKET_NAME, - Key: object.Key - }) - - await s3Client.send(command) + const command = new DeleteObjectCommand({ + Bucket: bucketInfo.BUCKET_NAME, + Key: key + }) - // FIXME: use bulk delete when s3ninja will support this operation - // deleteParams.Delete.Objects.push({ Key: object.Key }) - } + return getClient().send(command) +} +function removePrefix (prefix: string, bucketInfo: BucketInfo) { // FIXME: use bulk delete when s3ninja will support this operation - // const deleteCommand = new DeleteObjectsCommand(deleteParams) - // await s3Client.send(deleteCommand) - // Repeat if not all objects could be listed at once (limit of 1000?) - if (listedObjects.IsTruncated) await removePrefix(prefix, bucketInfo) + logger.debug('Removing prefix %s in bucket %s', prefix, bucketInfo.BUCKET_NAME, lTags()) + + return applyOnPrefix({ + prefix, + bucketInfo, + commandBuilder: obj => { + return new DeleteObjectCommand({ + Bucket: bucketInfo.BUCKET_NAME, + Key: obj.Key + }) + } + }) } // --------------------------------------------------------------------------- @@ -138,14 +163,42 @@ function buildKey (key: string, bucketInfo: BucketInfo) { // --------------------------------------------------------------------------- +async function createObjectReadStream (options: { + key: string + bucketInfo: BucketInfo + rangeHeader: string +}) { + const { key, bucketInfo, rangeHeader } = options + + const command = new GetObjectCommand({ + Bucket: bucketInfo.BUCKET_NAME, + Key: buildKey(key, bucketInfo), + Range: rangeHeader + }) + + const response = await getClient().send(command) + + return response.Body as Readable +} + +// --------------------------------------------------------------------------- + export { BucketInfo, buildKey, + storeObject, + removeObject, removePrefix, + makeAvailable, - listKeysOfPrefix + + updateObjectACL, + updatePrefixACL, + + listKeysOfPrefix, + createObjectReadStream } // --------------------------------------------------------------------------- @@ -154,17 +207,15 @@ async function uploadToStorage (options: { content: ReadStream objectStorageKey: string bucketInfo: BucketInfo + isPrivate: boolean }) { - const { content, objectStorageKey, bucketInfo } = options + const { content, objectStorageKey, bucketInfo, isPrivate } = options const input: PutObjectCommandInput = { Body: content, Bucket: bucketInfo.BUCKET_NAME, - Key: buildKey(objectStorageKey, bucketInfo) - } - - if (CONFIG.OBJECT_STORAGE.UPLOAD_ACL) { - input.ACL = CONFIG.OBJECT_STORAGE.UPLOAD_ACL + Key: buildKey(objectStorageKey, bucketInfo), + ACL: getACL(isPrivate) } const parallelUploads3 = new Upload({ @@ -194,5 +245,50 @@ async function uploadToStorage (options: { bucketInfo.PREFIX, objectStorageKey, bucketInfo.BUCKET_NAME, lTags() ) - return getPrivateUrl(bucketInfo, objectStorageKey) + return getInternalUrl(bucketInfo, objectStorageKey) +} + +async function applyOnPrefix (options: { + prefix: string + bucketInfo: BucketInfo + commandBuilder: (obj: _Object) => Parameters[0] + + continuationToken?: string +}) { + const { prefix, bucketInfo, commandBuilder, continuationToken } = options + + const s3Client = getClient() + + const commandPrefix = bucketInfo.PREFIX + prefix + const listCommand = new ListObjectsV2Command({ + Bucket: bucketInfo.BUCKET_NAME, + Prefix: commandPrefix, + ContinuationToken: continuationToken + }) + + const listedObjects = await s3Client.send(listCommand) + + if (isArray(listedObjects.Contents) !== true) { + const message = `Cannot apply function on ${commandPrefix} prefix in bucket ${bucketInfo.BUCKET_NAME}: no files listed.` + + logger.error(message, { response: listedObjects, ...lTags() }) + throw new Error(message) + } + + for (const object of listedObjects.Contents) { + const command = commandBuilder(object) + + await s3Client.send(command) + } + + // Repeat if not all objects could be listed at once (limit of 1000?) + if (listedObjects.IsTruncated) { + await applyOnPrefix({ ...options, continuationToken: listedObjects.ContinuationToken }) + } +} + +function getACL (isPrivate: boolean) { + return isPrivate + ? CONFIG.OBJECT_STORAGE.UPLOAD_ACL.PRIVATE + : CONFIG.OBJECT_STORAGE.UPLOAD_ACL.PUBLIC } diff --git a/server/lib/object-storage/urls.ts b/server/lib/object-storage/urls.ts index 2a889190b..a47a98b98 100644 --- a/server/lib/object-storage/urls.ts +++ b/server/lib/object-storage/urls.ts @@ -1,10 +1,14 @@ import { CONFIG } from '@server/initializers/config' +import { OBJECT_STORAGE_PROXY_PATHS, WEBSERVER } from '@server/initializers/constants' +import { MVideoUUID } from '@server/types/models' import { BucketInfo, buildKey, getEndpointParsed } from './shared' -function getPrivateUrl (config: BucketInfo, keyWithoutPrefix: string) { +function getInternalUrl (config: BucketInfo, keyWithoutPrefix: string) { return getBaseUrl(config) + buildKey(keyWithoutPrefix, config) } +// --------------------------------------------------------------------------- + function getWebTorrentPublicFileUrl (fileUrl: string) { const baseUrl = CONFIG.OBJECT_STORAGE.VIDEOS.BASE_URL if (!baseUrl) return fileUrl @@ -19,11 +23,28 @@ function getHLSPublicFileUrl (fileUrl: string) { return replaceByBaseUrl(fileUrl, baseUrl) } +// --------------------------------------------------------------------------- + +function getHLSPrivateFileUrl (video: MVideoUUID, filename: string) { + return WEBSERVER.URL + OBJECT_STORAGE_PROXY_PATHS.STREAMING_PLAYLISTS.PRIVATE_HLS + video.uuid + `/${filename}` +} + +function getWebTorrentPrivateFileUrl (filename: string) { + return WEBSERVER.URL + OBJECT_STORAGE_PROXY_PATHS.PRIVATE_WEBSEED + filename +} + +// --------------------------------------------------------------------------- + export { - getPrivateUrl, + getInternalUrl, + getWebTorrentPublicFileUrl, - replaceByBaseUrl, - getHLSPublicFileUrl + getHLSPublicFileUrl, + + getHLSPrivateFileUrl, + getWebTorrentPrivateFileUrl, + + replaceByBaseUrl } // --------------------------------------------------------------------------- diff --git a/server/lib/object-storage/videos.ts b/server/lib/object-storage/videos.ts index e323baaa2..003807826 100644 --- a/server/lib/object-storage/videos.ts +++ b/server/lib/object-storage/videos.ts @@ -5,7 +5,17 @@ import { MStreamingPlaylistVideo, MVideo, MVideoFile } from '@server/types/model import { getHLSDirectory } from '../paths' import { VideoPathManager } from '../video-path-manager' import { generateHLSObjectBaseStorageKey, generateHLSObjectStorageKey, generateWebTorrentObjectStorageKey } from './keys' -import { listKeysOfPrefix, lTags, makeAvailable, removeObject, removePrefix, storeObject } from './shared' +import { + createObjectReadStream, + listKeysOfPrefix, + lTags, + makeAvailable, + removeObject, + removePrefix, + storeObject, + updateObjectACL, + updatePrefixACL +} from './shared' function listHLSFileKeysOf (playlist: MStreamingPlaylistVideo) { return listKeysOfPrefix(generateHLSObjectBaseStorageKey(playlist), CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS) @@ -17,7 +27,8 @@ function storeHLSFileFromFilename (playlist: MStreamingPlaylistVideo, filename: return storeObject({ inputPath: join(getHLSDirectory(playlist.Video), filename), objectStorageKey: generateHLSObjectStorageKey(playlist, filename), - bucketInfo: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS + bucketInfo: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS, + isPrivate: playlist.Video.hasPrivateStaticPath() }) } @@ -25,7 +36,8 @@ function storeHLSFileFromPath (playlist: MStreamingPlaylistVideo, path: string) return storeObject({ inputPath: path, objectStorageKey: generateHLSObjectStorageKey(playlist, basename(path)), - bucketInfo: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS + bucketInfo: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS, + isPrivate: playlist.Video.hasPrivateStaticPath() }) } @@ -35,7 +47,26 @@ function storeWebTorrentFile (video: MVideo, file: MVideoFile) { return storeObject({ inputPath: VideoPathManager.Instance.getFSVideoFileOutputPath(video, file), objectStorageKey: generateWebTorrentObjectStorageKey(file.filename), - bucketInfo: CONFIG.OBJECT_STORAGE.VIDEOS + bucketInfo: CONFIG.OBJECT_STORAGE.VIDEOS, + isPrivate: video.hasPrivateStaticPath() + }) +} + +// --------------------------------------------------------------------------- + +function updateWebTorrentFileACL (video: MVideo, file: MVideoFile) { + return updateObjectACL({ + objectStorageKey: generateWebTorrentObjectStorageKey(file.filename), + bucketInfo: CONFIG.OBJECT_STORAGE.VIDEOS, + isPrivate: video.hasPrivateStaticPath() + }) +} + +function updateHLSFilesACL (playlist: MStreamingPlaylistVideo) { + return updatePrefixACL({ + prefix: generateHLSObjectBaseStorageKey(playlist), + bucketInfo: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS, + isPrivate: playlist.Video.hasPrivateStaticPath() }) } @@ -87,6 +118,39 @@ async function makeWebTorrentFileAvailable (filename: string, destination: strin // --------------------------------------------------------------------------- +function getWebTorrentFileReadStream (options: { + filename: string + rangeHeader: string +}) { + const { filename, rangeHeader } = options + + const key = generateWebTorrentObjectStorageKey(filename) + + return createObjectReadStream({ + key, + bucketInfo: CONFIG.OBJECT_STORAGE.VIDEOS, + rangeHeader + }) +} + +function getHLSFileReadStream (options: { + playlist: MStreamingPlaylistVideo + filename: string + rangeHeader: string +}) { + const { playlist, filename, rangeHeader } = options + + const key = generateHLSObjectStorageKey(playlist, filename) + + return createObjectReadStream({ + key, + bucketInfo: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS, + rangeHeader + }) +} + +// --------------------------------------------------------------------------- + export { listHLSFileKeysOf, @@ -94,10 +158,16 @@ export { storeHLSFileFromFilename, storeHLSFileFromPath, + updateWebTorrentFileACL, + updateHLSFilesACL, + removeHLSObjectStorage, removeHLSFileObjectStorage, removeWebTorrentObjectStorage, makeWebTorrentFileAvailable, - makeHLSFileAvailable + makeHLSFileAvailable, + + getWebTorrentFileReadStream, + getHLSFileReadStream } diff --git a/server/lib/video-privacy.ts b/server/lib/video-privacy.ts index 1a4a5a22d..41f9d62b3 100644 --- a/server/lib/video-privacy.ts +++ b/server/lib/video-privacy.ts @@ -2,8 +2,9 @@ import { move } from 'fs-extra' import { join } from 'path' import { logger } from '@server/helpers/logger' import { DIRECTORIES } from '@server/initializers/constants' -import { MVideo, MVideoFullLight } from '@server/types/models' -import { VideoPrivacy } from '@shared/models' +import { MVideo, MVideoFile, MVideoFullLight } from '@server/types/models' +import { VideoPrivacy, VideoStorage } from '@shared/models' +import { updateHLSFilesACL, updateWebTorrentFileACL } from './object-storage' function setVideoPrivacy (video: MVideo, newPrivacy: VideoPrivacy) { if (video.privacy === VideoPrivacy.PRIVATE && newPrivacy !== VideoPrivacy.PRIVATE) { @@ -50,47 +51,77 @@ export { // --------------------------------------------------------------------------- +type MoveType = 'private-to-public' | 'public-to-private' + async function moveFiles (options: { - type: 'private-to-public' | 'public-to-private' + type: MoveType video: MVideoFullLight }) { const { type, video } = options - const directories = type === 'private-to-public' - ? { - webtorrent: { old: DIRECTORIES.VIDEOS.PRIVATE, new: DIRECTORIES.VIDEOS.PUBLIC }, - hls: { old: DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE, new: DIRECTORIES.HLS_STREAMING_PLAYLIST.PUBLIC } + for (const file of video.VideoFiles) { + if (file.storage === VideoStorage.FILE_SYSTEM) { + await moveWebTorrentFileOnFS(type, video, file) + } else { + await updateWebTorrentFileACL(video, file) } - : { - webtorrent: { old: DIRECTORIES.VIDEOS.PUBLIC, new: DIRECTORIES.VIDEOS.PRIVATE }, - hls: { old: DIRECTORIES.HLS_STREAMING_PLAYLIST.PUBLIC, new: DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE } + } + + const hls = video.getHLSPlaylist() + + if (hls) { + if (hls.storage === VideoStorage.FILE_SYSTEM) { + await moveHLSFilesOnFS(type, video) + } else { + await updateHLSFilesACL(hls) } + } +} - for (const file of video.VideoFiles) { - const source = join(directories.webtorrent.old, file.filename) - const destination = join(directories.webtorrent.new, file.filename) +async function moveWebTorrentFileOnFS (type: MoveType, video: MVideo, file: MVideoFile) { + const directories = getWebTorrentDirectories(type) - try { - logger.info('Moving WebTorrent files of %s after privacy change (%s -> %s).', video.uuid, source, destination) + const source = join(directories.old, file.filename) + const destination = join(directories.new, file.filename) - await move(source, destination) - } catch (err) { - logger.error('Cannot move webtorrent file %s to %s after privacy change', source, destination, { err }) - } + try { + logger.info('Moving WebTorrent files of %s after privacy change (%s -> %s).', video.uuid, source, destination) + + await move(source, destination) + } catch (err) { + logger.error('Cannot move webtorrent file %s to %s after privacy change', source, destination, { err }) + } +} + +function getWebTorrentDirectories (moveType: MoveType) { + if (moveType === 'private-to-public') { + return { old: DIRECTORIES.VIDEOS.PRIVATE, new: DIRECTORIES.VIDEOS.PUBLIC } } - const hls = video.getHLSPlaylist() + return { old: DIRECTORIES.VIDEOS.PUBLIC, new: DIRECTORIES.VIDEOS.PRIVATE } +} - if (hls) { - const source = join(directories.hls.old, video.uuid) - const destination = join(directories.hls.new, video.uuid) +// --------------------------------------------------------------------------- - try { - logger.info('Moving HLS files of %s after privacy change (%s -> %s).', video.uuid, source, destination) +async function moveHLSFilesOnFS (type: MoveType, video: MVideo) { + const directories = getHLSDirectories(type) - await move(source, destination) - } catch (err) { - logger.error('Cannot move HLS file %s to %s after privacy change', source, destination, { err }) - } + const source = join(directories.old, video.uuid) + const destination = join(directories.new, video.uuid) + + try { + logger.info('Moving HLS files of %s after privacy change (%s -> %s).', video.uuid, source, destination) + + await move(source, destination) + } catch (err) { + logger.error('Cannot move HLS file %s to %s after privacy change', source, destination, { err }) + } +} + +function getHLSDirectories (moveType: MoveType) { + if (moveType === 'private-to-public') { + return { old: DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE, new: DIRECTORIES.HLS_STREAMING_PLAYLIST.PUBLIC } } + + return { old: DIRECTORIES.HLS_STREAMING_PLAYLIST.PUBLIC, new: DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE } } diff --git a/server/middlewares/validators/shared/videos.ts b/server/middlewares/validators/shared/videos.ts index c29751eca..ebbfc0a0a 100644 --- a/server/middlewares/validators/shared/videos.ts +++ b/server/middlewares/validators/shared/videos.ts @@ -111,7 +111,7 @@ async function checkCanSeeVideo (options: { }) { const { req, res, video, paramId } = options - if (video.requiresAuth(paramId)) { + if (video.requiresAuth({ urlParamId: paramId, checkBlacklist: true })) { return checkCanSeeAuthVideo(req, res, video) } @@ -174,13 +174,13 @@ async function checkCanAccessVideoStaticFiles (options: { res: Response paramId: string }) { - const { video, req, res, paramId } = options + const { video, req, res } = options if (res.locals.oauth?.token.User) { return checkCanSeeVideo(options) } - if (!video.requiresAuth(paramId)) return true + if (!video.hasPrivateStaticPath()) return true const videoFileToken = req.query.videoFileToken if (!videoFileToken) { diff --git a/server/middlewares/validators/static.ts b/server/middlewares/validators/static.ts index ff9e6ae6e..13fde6dd1 100644 --- a/server/middlewares/validators/static.ts +++ b/server/middlewares/validators/static.ts @@ -7,10 +7,17 @@ import { logger } from '@server/helpers/logger' import { LRU_CACHE } from '@server/initializers/constants' import { VideoModel } from '@server/models/video/video' import { VideoFileModel } from '@server/models/video/video-file' +import { MStreamingPlaylist, MVideoFile, MVideoThumbnail } from '@server/types/models' import { HttpStatusCode } from '@shared/models' import { areValidationErrors, checkCanAccessVideoStaticFiles } from './shared' -const staticFileTokenBypass = new LRUCache({ +type LRUValue = { + allowed: boolean + video?: MVideoThumbnail + file?: MVideoFile + playlist?: MStreamingPlaylist } + +const staticFileTokenBypass = new LRUCache({ max: LRU_CACHE.STATIC_VIDEO_FILES_RIGHTS_CHECK.MAX_SIZE, ttl: LRU_CACHE.STATIC_VIDEO_FILES_RIGHTS_CHECK.TTL }) @@ -27,18 +34,26 @@ const ensureCanAccessVideoPrivateWebTorrentFiles = [ const cacheKey = token + '-' + req.originalUrl if (staticFileTokenBypass.has(cacheKey)) { - const allowedFromCache = staticFileTokenBypass.get(cacheKey) + const { allowed, file, video } = staticFileTokenBypass.get(cacheKey) + + if (allowed === true) { + res.locals.onlyVideo = video + res.locals.videoFile = file - if (allowedFromCache === true) return next() + return next() + } return res.sendStatus(HttpStatusCode.FORBIDDEN_403) } - const allowed = await isWebTorrentAllowed(req, res) + const result = await isWebTorrentAllowed(req, res) + + staticFileTokenBypass.set(cacheKey, result) - staticFileTokenBypass.set(cacheKey, allowed) + if (result.allowed !== true) return - if (allowed !== true) return + res.locals.onlyVideo = result.video + res.locals.videoFile = result.file return next() } @@ -64,18 +79,28 @@ const ensureCanAccessPrivateVideoHLSFiles = [ const cacheKey = token + '-' + videoUUID if (staticFileTokenBypass.has(cacheKey)) { - const allowedFromCache = staticFileTokenBypass.get(cacheKey) + const { allowed, file, playlist, video } = staticFileTokenBypass.get(cacheKey) - if (allowedFromCache === true) return next() + if (allowed === true) { + res.locals.onlyVideo = video + res.locals.videoFile = file + res.locals.videoStreamingPlaylist = playlist + + return next() + } return res.sendStatus(HttpStatusCode.FORBIDDEN_403) } - const allowed = await isHLSAllowed(req, res, videoUUID) + const result = await isHLSAllowed(req, res, videoUUID) + + staticFileTokenBypass.set(cacheKey, result) - staticFileTokenBypass.set(cacheKey, allowed) + if (result.allowed !== true) return - if (allowed !== true) return + res.locals.onlyVideo = result.video + res.locals.videoFile = result.file + res.locals.videoStreamingPlaylist = result.playlist return next() } @@ -96,25 +121,38 @@ async function isWebTorrentAllowed (req: express.Request, res: express.Response) logger.debug('Unknown static file %s to serve', req.originalUrl, { filename }) res.sendStatus(HttpStatusCode.FORBIDDEN_403) - return false + return { allowed: false } } - const video = file.getVideo() + const video = await VideoModel.load(file.getVideo().id) - return checkCanAccessVideoStaticFiles({ req, res, video, paramId: video.uuid }) + return { + file, + video, + allowed: await checkCanAccessVideoStaticFiles({ req, res, video, paramId: video.uuid }) + } } async function isHLSAllowed (req: express.Request, res: express.Response, videoUUID: string) { - const video = await VideoModel.load(videoUUID) + const filename = basename(req.path) + + const video = await VideoModel.loadWithFiles(videoUUID) if (!video) { logger.debug('Unknown static file %s to serve', req.originalUrl, { videoUUID }) res.sendStatus(HttpStatusCode.FORBIDDEN_403) - return false + return { allowed: false } } - return checkCanAccessVideoStaticFiles({ req, res, video, paramId: video.uuid }) + const file = await VideoFileModel.loadByFilename(filename) + + return { + file, + video, + playlist: video.getHLSPlaylist(), + allowed: await checkCanAccessVideoStaticFiles({ req, res, video, paramId: video.uuid }) + } } function extractTokenOrDie (req: express.Request, res: express.Response) { diff --git a/server/models/video/video-file.ts b/server/models/video/video-file.ts index 1a608932f..c20c90c1b 100644 --- a/server/models/video/video-file.ts +++ b/server/models/video/video-file.ts @@ -22,7 +22,12 @@ import validator from 'validator' import { logger } from '@server/helpers/logger' import { extractVideo } from '@server/helpers/video' import { buildRemoteVideoBaseUrl } from '@server/lib/activitypub/url' -import { getHLSPublicFileUrl, getWebTorrentPublicFileUrl } from '@server/lib/object-storage' +import { + getHLSPrivateFileUrl, + getHLSPublicFileUrl, + getWebTorrentPrivateFileUrl, + getWebTorrentPublicFileUrl +} from '@server/lib/object-storage' import { getFSTorrentFilePath } from '@server/lib/paths' import { isVideoInPrivateDirectory } from '@server/lib/video-privacy' import { isStreamingPlaylist, MStreamingPlaylistVideo, MVideo, MVideoWithHost } from '@server/types/models' @@ -503,7 +508,25 @@ export class VideoFileModel extends Model return !!this.videoStreamingPlaylistId } - getObjectStorageUrl () { + // --------------------------------------------------------------------------- + + getObjectStorageUrl (video: MVideo) { + if (video.hasPrivateStaticPath()) { + return this.getPrivateObjectStorageUrl(video) + } + + return this.getPublicObjectStorageUrl() + } + + private getPrivateObjectStorageUrl (video: MVideo) { + if (this.isHLS()) { + return getHLSPrivateFileUrl(video, this.filename) + } + + return getWebTorrentPrivateFileUrl(this.filename) + } + + private getPublicObjectStorageUrl () { if (this.isHLS()) { return getHLSPublicFileUrl(this.fileUrl) } @@ -511,26 +534,29 @@ export class VideoFileModel extends Model return getWebTorrentPublicFileUrl(this.fileUrl) } + // --------------------------------------------------------------------------- + getFileUrl (video: MVideo) { - if (this.storage === VideoStorage.OBJECT_STORAGE) { - return this.getObjectStorageUrl() - } + if (video.isOwned()) { + if (this.storage === VideoStorage.OBJECT_STORAGE) { + return this.getObjectStorageUrl(video) + } - if (!this.Video) this.Video = video as VideoModel - if (video.isOwned()) return WEBSERVER.URL + this.getFileStaticPath(video) + return WEBSERVER.URL + this.getFileStaticPath(video) + } return this.fileUrl } + // --------------------------------------------------------------------------- + getFileStaticPath (video: MVideo) { - if (this.isHLS()) { - if (isVideoInPrivateDirectory(video.privacy)) { - return join(STATIC_PATHS.STREAMING_PLAYLISTS.PRIVATE_HLS, video.uuid, this.filename) - } + if (this.isHLS()) return this.getHLSFileStaticPath(video) - return join(STATIC_PATHS.STREAMING_PLAYLISTS.HLS, video.uuid, this.filename) - } + return this.getWebTorrentFileStaticPath(video) + } + private getWebTorrentFileStaticPath (video: MVideo) { if (isVideoInPrivateDirectory(video.privacy)) { return join(STATIC_PATHS.PRIVATE_WEBSEED, this.filename) } @@ -538,6 +564,16 @@ export class VideoFileModel extends Model return join(STATIC_PATHS.WEBSEED, this.filename) } + private getHLSFileStaticPath (video: MVideo) { + if (isVideoInPrivateDirectory(video.privacy)) { + return join(STATIC_PATHS.STREAMING_PLAYLISTS.PRIVATE_HLS, video.uuid, this.filename) + } + + return join(STATIC_PATHS.STREAMING_PLAYLISTS.HLS, video.uuid, this.filename) + } + + // --------------------------------------------------------------------------- + getFileDownloadUrl (video: MVideoWithHost) { const path = this.isHLS() ? join(STATIC_DOWNLOAD_PATHS.HLS_VIDEOS, `${video.uuid}-${this.resolution}-fragmented${this.extname}`) diff --git a/server/models/video/video-streaming-playlist.ts b/server/models/video/video-streaming-playlist.ts index b919046ed..1318a4dae 100644 --- a/server/models/video/video-streaming-playlist.ts +++ b/server/models/video/video-streaming-playlist.ts @@ -15,7 +15,7 @@ import { Table, UpdatedAt } from 'sequelize-typescript' -import { getHLSPublicFileUrl } from '@server/lib/object-storage' +import { getHLSPrivateFileUrl, getHLSPublicFileUrl } from '@server/lib/object-storage' import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename } from '@server/lib/paths' import { isVideoInPrivateDirectory } from '@server/lib/video-privacy' import { VideoFileModel } from '@server/models/video/video-file' @@ -245,10 +245,12 @@ export class VideoStreamingPlaylistModel extends Model>> { const playlist = this.VideoStreamingPlaylists.find(p => p.type === VideoStreamingPlaylistType.HLS) if (!playlist) return undefined - playlist.Video = this - - return playlist + return playlist.withVideo(this) } setHLSPlaylist (playlist: MStreamingPlaylist) { @@ -1868,16 +1867,39 @@ export class VideoModel extends Model>> { return setAsUpdated('video', this.id, transaction) } - requiresAuth (paramId: string) { + // --------------------------------------------------------------------------- + + requiresAuth (options: { + urlParamId: string + checkBlacklist: boolean + }) { + const { urlParamId, checkBlacklist } = options + + if (this.privacy === VideoPrivacy.PRIVATE || this.privacy === VideoPrivacy.INTERNAL) { + return true + } + if (this.privacy === VideoPrivacy.UNLISTED) { - if (!isUUIDValid(paramId)) return true + if (urlParamId && !isUUIDValid(urlParamId)) return true return false } - return this.privacy === VideoPrivacy.PRIVATE || this.privacy === VideoPrivacy.INTERNAL || !!this.VideoBlacklist + if (checkBlacklist && this.VideoBlacklist) return true + + if (this.privacy !== VideoPrivacy.PUBLIC) { + throw new Error(`Unknown video privacy ${this.privacy} to know if the video requires auth`) + } + + return false } + hasPrivateStaticPath () { + return isVideoInPrivateDirectory(this.privacy) + } + + // --------------------------------------------------------------------------- + async setNewState (newState: VideoState, isNewVideo: boolean, transaction: Transaction) { if (this.state === newState) throw new Error('Cannot use same state ' + newState) diff --git a/server/tests/api/object-storage/index.ts b/server/tests/api/object-storage/index.ts index f319d6ef5..1f4489fa3 100644 --- a/server/tests/api/object-storage/index.ts +++ b/server/tests/api/object-storage/index.ts @@ -1,3 +1,4 @@ export * from './live' export * from './video-imports' +export * from './video-static-file-privacy' export * from './videos' diff --git a/server/tests/api/object-storage/live.ts b/server/tests/api/object-storage/live.ts index 77f3a8066..ad2b554b7 100644 --- a/server/tests/api/object-storage/live.ts +++ b/server/tests/api/object-storage/live.ts @@ -2,7 +2,7 @@ import { expect } from 'chai' import { expectStartWith, testVideoResolutions } from '@server/tests/shared' -import { areObjectStorageTestsDisabled } from '@shared/core-utils' +import { areMockObjectStorageTestsDisabled } from '@shared/core-utils' import { HttpStatusCode, LiveVideoCreate, VideoPrivacy } from '@shared/models' import { createMultipleServers, @@ -46,7 +46,7 @@ async function checkFilesExist (servers: PeerTubeServer[], videoUUID: string, nu expect(files).to.have.lengthOf(numberOfFiles) for (const file of files) { - expectStartWith(file.fileUrl, ObjectStorageCommand.getPlaylistBaseUrl()) + expectStartWith(file.fileUrl, ObjectStorageCommand.getMockPlaylistBaseUrl()) await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 }) } @@ -75,16 +75,16 @@ async function checkFilesCleanup (server: PeerTubeServer, videoUUID: string, res } describe('Object storage for lives', function () { - if (areObjectStorageTestsDisabled()) return + if (areMockObjectStorageTestsDisabled()) return let servers: PeerTubeServer[] before(async function () { this.timeout(120000) - await ObjectStorageCommand.prepareDefaultBuckets() + await ObjectStorageCommand.prepareDefaultMockBuckets() - servers = await createMultipleServers(2, ObjectStorageCommand.getDefaultConfig()) + servers = await createMultipleServers(2, ObjectStorageCommand.getDefaultMockConfig()) await setAccessTokensToServers(servers) await setDefaultVideoChannel(servers) diff --git a/server/tests/api/object-storage/video-imports.ts b/server/tests/api/object-storage/video-imports.ts index 90988ea9a..11c866411 100644 --- a/server/tests/api/object-storage/video-imports.ts +++ b/server/tests/api/object-storage/video-imports.ts @@ -2,7 +2,7 @@ import { expect } from 'chai' import { expectStartWith, FIXTURE_URLS } from '@server/tests/shared' -import { areObjectStorageTestsDisabled } from '@shared/core-utils' +import { areMockObjectStorageTestsDisabled } from '@shared/core-utils' import { HttpStatusCode, VideoPrivacy } from '@shared/models' import { createSingleServer, @@ -29,16 +29,16 @@ async function importVideo (server: PeerTubeServer) { } describe('Object storage for video import', function () { - if (areObjectStorageTestsDisabled()) return + if (areMockObjectStorageTestsDisabled()) return let server: PeerTubeServer before(async function () { this.timeout(120000) - await ObjectStorageCommand.prepareDefaultBuckets() + await ObjectStorageCommand.prepareDefaultMockBuckets() - server = await createSingleServer(1, ObjectStorageCommand.getDefaultConfig()) + server = await createSingleServer(1, ObjectStorageCommand.getDefaultMockConfig()) await setAccessTokensToServers([ server ]) await setDefaultVideoChannel([ server ]) @@ -64,7 +64,7 @@ describe('Object storage for video import', function () { expect(video.streamingPlaylists).to.have.lengthOf(0) const fileUrl = video.files[0].fileUrl - expectStartWith(fileUrl, ObjectStorageCommand.getWebTorrentBaseUrl()) + expectStartWith(fileUrl, ObjectStorageCommand.getMockWebTorrentBaseUrl()) await makeRawRequest({ url: fileUrl, expectedStatus: HttpStatusCode.OK_200 }) }) @@ -89,13 +89,13 @@ describe('Object storage for video import', function () { expect(video.streamingPlaylists[0].files).to.have.lengthOf(5) for (const file of video.files) { - expectStartWith(file.fileUrl, ObjectStorageCommand.getWebTorrentBaseUrl()) + expectStartWith(file.fileUrl, ObjectStorageCommand.getMockWebTorrentBaseUrl()) await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 }) } for (const file of video.streamingPlaylists[0].files) { - expectStartWith(file.fileUrl, ObjectStorageCommand.getPlaylistBaseUrl()) + expectStartWith(file.fileUrl, ObjectStorageCommand.getMockPlaylistBaseUrl()) await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 }) } diff --git a/server/tests/api/object-storage/video-static-file-privacy.ts b/server/tests/api/object-storage/video-static-file-privacy.ts new file mode 100644 index 000000000..981bbaa16 --- /dev/null +++ b/server/tests/api/object-storage/video-static-file-privacy.ts @@ -0,0 +1,336 @@ +/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ + +import { expect } from 'chai' +import { basename } from 'path' +import { expectStartWith } from '@server/tests/shared' +import { areScalewayObjectStorageTestsDisabled, getAllFiles, getHLS } from '@shared/core-utils' +import { HttpStatusCode, LiveVideo, VideoDetails, VideoPrivacy } from '@shared/models' +import { + cleanupTests, + createSingleServer, + findExternalSavedVideo, + makeRawRequest, + ObjectStorageCommand, + PeerTubeServer, + sendRTMPStream, + setAccessTokensToServers, + setDefaultVideoChannel, + stopFfmpeg, + waitJobs +} from '@shared/server-commands' + +describe('Object storage for video static file privacy', function () { + // We need real world object storage to check ACL + if (areScalewayObjectStorageTestsDisabled()) return + + let server: PeerTubeServer + let userToken: string + + before(async function () { + this.timeout(120000) + + server = await createSingleServer(1, ObjectStorageCommand.getDefaultScalewayConfig(1)) + await setAccessTokensToServers([ server ]) + await setDefaultVideoChannel([ server ]) + + await server.config.enableMinimumTranscoding() + + userToken = await server.users.generateUserAndToken('user1') + }) + + describe('VOD', function () { + let privateVideoUUID: string + let publicVideoUUID: string + let userPrivateVideoUUID: string + + async function checkPrivateFiles (uuid: string) { + const video = await server.videos.getWithToken({ id: uuid }) + + for (const file of video.files) { + expectStartWith(file.fileUrl, server.url + '/object-storage-proxy/webseed/private/') + + await makeRawRequest({ url: file.fileUrl, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 }) + } + + for (const file of getAllFiles(video)) { + const internalFileUrl = await server.sql.getInternalFileUrl(file.id) + expectStartWith(internalFileUrl, ObjectStorageCommand.getScalewayBaseUrl()) + await makeRawRequest({ url: internalFileUrl, token: server.accessToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) + } + + const hls = getHLS(video) + + if (hls) { + for (const url of [ hls.playlistUrl, hls.segmentsSha256Url ]) { + expectStartWith(url, server.url + '/object-storage-proxy/streaming-playlists/hls/private/') + } + + await makeRawRequest({ url: hls.playlistUrl, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 }) + await makeRawRequest({ url: hls.segmentsSha256Url, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 }) + + for (const file of hls.files) { + expectStartWith(file.fileUrl, server.url + '/object-storage-proxy/streaming-playlists/hls/private/') + + await makeRawRequest({ url: file.fileUrl, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 }) + } + } + } + + async function checkPublicFiles (uuid: string) { + const video = await server.videos.getWithToken({ id: uuid }) + + for (const file of getAllFiles(video)) { + expectStartWith(file.fileUrl, ObjectStorageCommand.getScalewayBaseUrl()) + + await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 }) + } + + const hls = getHLS(video) + + if (hls) { + expectStartWith(hls.playlistUrl, ObjectStorageCommand.getScalewayBaseUrl()) + expectStartWith(hls.segmentsSha256Url, ObjectStorageCommand.getScalewayBaseUrl()) + + await makeRawRequest({ url: hls.playlistUrl, expectedStatus: HttpStatusCode.OK_200 }) + await makeRawRequest({ url: hls.segmentsSha256Url, expectedStatus: HttpStatusCode.OK_200 }) + } + } + + async function getSampleFileUrls (videoId: string) { + const video = await server.videos.getWithToken({ id: videoId }) + + return { + webTorrentFile: video.files[0].fileUrl, + hlsFile: getHLS(video).files[0].fileUrl + } + } + + it('Should upload a private video and have appropriate object storage ACL', async function () { + this.timeout(60000) + + { + const { uuid } = await server.videos.quickUpload({ name: 'video', privacy: VideoPrivacy.PRIVATE }) + privateVideoUUID = uuid + } + + { + const { uuid } = await server.videos.quickUpload({ name: 'user video', token: userToken, privacy: VideoPrivacy.PRIVATE }) + userPrivateVideoUUID = uuid + } + + await waitJobs([ server ]) + + await checkPrivateFiles(privateVideoUUID) + }) + + it('Should upload a public video and have appropriate object storage ACL', async function () { + this.timeout(60000) + + const { uuid } = await server.videos.quickUpload({ name: 'video', privacy: VideoPrivacy.UNLISTED }) + await waitJobs([ server ]) + + publicVideoUUID = uuid + + await checkPublicFiles(publicVideoUUID) + }) + + it('Should not get files without appropriate OAuth token', async function () { + this.timeout(60000) + + const { webTorrentFile, hlsFile } = await getSampleFileUrls(privateVideoUUID) + + await makeRawRequest({ url: webTorrentFile, token: userToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) + await makeRawRequest({ url: webTorrentFile, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 }) + + await makeRawRequest({ url: hlsFile, token: userToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) + await makeRawRequest({ url: hlsFile, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 }) + }) + + it('Should not get HLS file of another video', async function () { + this.timeout(60000) + + const privateVideo = await server.videos.getWithToken({ id: privateVideoUUID }) + const hlsFilename = basename(getHLS(privateVideo).files[0].fileUrl) + + const badUrl = server.url + '/object-storage-proxy/streaming-playlists/hls/private/' + userPrivateVideoUUID + '/' + hlsFilename + const goodUrl = server.url + '/object-storage-proxy/streaming-playlists/hls/private/' + privateVideoUUID + '/' + hlsFilename + + await makeRawRequest({ url: badUrl, token: server.accessToken, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) + await makeRawRequest({ url: goodUrl, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 }) + }) + + it('Should correctly check OAuth or video file token', async function () { + this.timeout(60000) + + const badVideoFileToken = await server.videoToken.getVideoFileToken({ token: userToken, videoId: userPrivateVideoUUID }) + const goodVideoFileToken = await server.videoToken.getVideoFileToken({ videoId: privateVideoUUID }) + + const { webTorrentFile, hlsFile } = await getSampleFileUrls(privateVideoUUID) + + for (const url of [ webTorrentFile, hlsFile ]) { + await makeRawRequest({ url, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) + await makeRawRequest({ url, token: userToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) + await makeRawRequest({ url, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 }) + + await makeRawRequest({ url, query: { videoFileToken: badVideoFileToken }, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) + await makeRawRequest({ url, query: { videoFileToken: goodVideoFileToken }, expectedStatus: HttpStatusCode.OK_200 }) + } + }) + + it('Should update public video to private', async function () { + this.timeout(60000) + + await server.videos.update({ id: publicVideoUUID, attributes: { privacy: VideoPrivacy.INTERNAL } }) + + await checkPrivateFiles(publicVideoUUID) + }) + + it('Should update private video to public', async function () { + this.timeout(60000) + + await server.videos.update({ id: publicVideoUUID, attributes: { privacy: VideoPrivacy.PUBLIC } }) + + await checkPublicFiles(publicVideoUUID) + }) + + after(async function () { + this.timeout(30000) + + if (privateVideoUUID) await server.videos.remove({ id: privateVideoUUID }) + if (publicVideoUUID) await server.videos.remove({ id: publicVideoUUID }) + if (userPrivateVideoUUID) await server.videos.remove({ id: userPrivateVideoUUID }) + + await waitJobs([ server ]) + }) + }) + + describe('Live', function () { + let normalLiveId: string + let normalLive: LiveVideo + + let permanentLiveId: string + let permanentLive: LiveVideo + + let unrelatedFileToken: string + + async function checkLiveFiles (live: LiveVideo, liveId: string) { + const ffmpegCommand = sendRTMPStream({ rtmpBaseUrl: live.rtmpUrl, streamKey: live.streamKey }) + await server.live.waitUntilPublished({ videoId: liveId }) + + const video = await server.videos.getWithToken({ id: liveId }) + const fileToken = await server.videoToken.getVideoFileToken({ videoId: video.uuid }) + + const hls = video.streamingPlaylists[0] + + for (const url of [ hls.playlistUrl, hls.segmentsSha256Url ]) { + expectStartWith(url, server.url + '/object-storage-proxy/streaming-playlists/hls/private/') + + await makeRawRequest({ url: hls.playlistUrl, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 }) + await makeRawRequest({ url: hls.segmentsSha256Url, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 }) + + await makeRawRequest({ url, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 }) + await makeRawRequest({ url, query: { videoFileToken: fileToken }, expectedStatus: HttpStatusCode.OK_200 }) + + await makeRawRequest({ url, token: userToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) + await makeRawRequest({ url, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) + await makeRawRequest({ url, query: { videoFileToken: unrelatedFileToken }, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) + } + + await stopFfmpeg(ffmpegCommand) + } + + async function checkReplay (replay: VideoDetails) { + const fileToken = await server.videoToken.getVideoFileToken({ videoId: replay.uuid }) + + const hls = replay.streamingPlaylists[0] + expect(hls.files).to.not.have.lengthOf(0) + + for (const file of hls.files) { + await makeRawRequest({ url: file.fileUrl, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 }) + await makeRawRequest({ url: file.fileUrl, query: { videoFileToken: fileToken }, expectedStatus: HttpStatusCode.OK_200 }) + + await makeRawRequest({ url: file.fileUrl, token: userToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) + await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) + await makeRawRequest({ + url: file.fileUrl, + query: { videoFileToken: unrelatedFileToken }, + expectedStatus: HttpStatusCode.FORBIDDEN_403 + }) + } + + for (const url of [ hls.playlistUrl, hls.segmentsSha256Url ]) { + expectStartWith(url, server.url + '/object-storage-proxy/streaming-playlists/hls/private/') + + await makeRawRequest({ url, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 }) + await makeRawRequest({ url, query: { videoFileToken: fileToken }, expectedStatus: HttpStatusCode.OK_200 }) + + await makeRawRequest({ url, token: userToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) + await makeRawRequest({ url, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) + await makeRawRequest({ url, query: { videoFileToken: unrelatedFileToken }, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) + } + } + + before(async function () { + await server.config.enableMinimumTranscoding() + + const { uuid } = await server.videos.quickUpload({ name: 'another video' }) + unrelatedFileToken = await server.videoToken.getVideoFileToken({ videoId: uuid }) + + await server.config.enableLive({ + allowReplay: true, + transcoding: true, + resolutions: 'min' + }) + + { + const { video, live } = await server.live.quickCreate({ saveReplay: true, permanentLive: false, privacy: VideoPrivacy.PRIVATE }) + normalLiveId = video.uuid + normalLive = live + } + + { + const { video, live } = await server.live.quickCreate({ saveReplay: true, permanentLive: true, privacy: VideoPrivacy.PRIVATE }) + permanentLiveId = video.uuid + permanentLive = live + } + }) + + it('Should create a private normal live and have a private static path', async function () { + this.timeout(240000) + + await checkLiveFiles(normalLive, normalLiveId) + }) + + it('Should create a private permanent live and have a private static path', async function () { + this.timeout(240000) + + await checkLiveFiles(permanentLive, permanentLiveId) + }) + + it('Should have created a replay of the normal live with a private static path', async function () { + this.timeout(240000) + + await server.live.waitUntilReplacedByReplay({ videoId: normalLiveId }) + + const replay = await server.videos.getWithToken({ id: normalLiveId }) + await checkReplay(replay) + }) + + it('Should have created a replay of the permanent live with a private static path', async function () { + this.timeout(240000) + + await server.live.waitUntilWaiting({ videoId: permanentLiveId }) + await waitJobs([ server ]) + + const live = await server.videos.getWithToken({ id: permanentLiveId }) + const replayFromList = await findExternalSavedVideo(server, live) + const replay = await server.videos.getWithToken({ id: replayFromList.id }) + + await checkReplay(replay) + }) + }) + + after(async function () { + await cleanupTests([ server ]) + }) +}) diff --git a/server/tests/api/object-storage/videos.ts b/server/tests/api/object-storage/videos.ts index 63f5179c7..af3db9334 100644 --- a/server/tests/api/object-storage/videos.ts +++ b/server/tests/api/object-storage/videos.ts @@ -11,7 +11,7 @@ import { generateHighBitrateVideo, MockObjectStorage } from '@server/tests/shared' -import { areObjectStorageTestsDisabled } from '@shared/core-utils' +import { areMockObjectStorageTestsDisabled } from '@shared/core-utils' import { HttpStatusCode, VideoDetails } from '@shared/models' import { cleanupTests, @@ -52,7 +52,7 @@ async function checkFiles (options: { for (const file of video.files) { const baseUrl = baseMockUrl ? `${baseMockUrl}/${webtorrentBucket}/` - : `http://${webtorrentBucket}.${ObjectStorageCommand.getEndpointHost()}/` + : `http://${webtorrentBucket}.${ObjectStorageCommand.getMockEndpointHost()}/` const prefix = webtorrentPrefix || '' const start = baseUrl + prefix @@ -73,7 +73,7 @@ async function checkFiles (options: { const baseUrl = baseMockUrl ? `${baseMockUrl}/${playlistBucket}/` - : `http://${playlistBucket}.${ObjectStorageCommand.getEndpointHost()}/` + : `http://${playlistBucket}.${ObjectStorageCommand.getMockEndpointHost()}/` const prefix = playlistPrefix || '' const start = baseUrl + prefix @@ -141,16 +141,16 @@ function runTestSuite (options: { const port = await mockObjectStorage.initialize() baseMockUrl = options.useMockBaseUrl ? `http://localhost:${port}` : undefined - await ObjectStorageCommand.createBucket(options.playlistBucket) - await ObjectStorageCommand.createBucket(options.webtorrentBucket) + await ObjectStorageCommand.createMockBucket(options.playlistBucket) + await ObjectStorageCommand.createMockBucket(options.webtorrentBucket) const config = { object_storage: { enabled: true, - endpoint: 'http://' + ObjectStorageCommand.getEndpointHost(), - region: ObjectStorageCommand.getRegion(), + endpoint: 'http://' + ObjectStorageCommand.getMockEndpointHost(), + region: ObjectStorageCommand.getMockRegion(), - credentials: ObjectStorageCommand.getCredentialsConfig(), + credentials: ObjectStorageCommand.getMockCredentialsConfig(), max_upload_part: options.maxUploadPart || '5MB', @@ -261,7 +261,7 @@ function runTestSuite (options: { } describe('Object storage for videos', function () { - if (areObjectStorageTestsDisabled()) return + if (areMockObjectStorageTestsDisabled()) return describe('Test config', function () { let server: PeerTubeServer @@ -269,17 +269,17 @@ describe('Object storage for videos', function () { const baseConfig = { object_storage: { enabled: true, - endpoint: 'http://' + ObjectStorageCommand.getEndpointHost(), - region: ObjectStorageCommand.getRegion(), + endpoint: 'http://' + ObjectStorageCommand.getMockEndpointHost(), + region: ObjectStorageCommand.getMockRegion(), - credentials: ObjectStorageCommand.getCredentialsConfig(), + credentials: ObjectStorageCommand.getMockCredentialsConfig(), streaming_playlists: { - bucket_name: ObjectStorageCommand.DEFAULT_PLAYLIST_BUCKET + bucket_name: ObjectStorageCommand.DEFAULT_PLAYLIST_MOCK_BUCKET }, videos: { - bucket_name: ObjectStorageCommand.DEFAULT_WEBTORRENT_BUCKET + bucket_name: ObjectStorageCommand.DEFAULT_WEBTORRENT_MOCK_BUCKET } } } @@ -310,7 +310,7 @@ describe('Object storage for videos', function () { it('Should fail with bad credentials', async function () { this.timeout(60000) - await ObjectStorageCommand.prepareDefaultBuckets() + await ObjectStorageCommand.prepareDefaultMockBuckets() const config = merge({}, baseConfig, { object_storage: { @@ -334,7 +334,7 @@ describe('Object storage for videos', function () { it('Should succeed with credentials from env', async function () { this.timeout(60000) - await ObjectStorageCommand.prepareDefaultBuckets() + await ObjectStorageCommand.prepareDefaultMockBuckets() const config = merge({}, baseConfig, { object_storage: { @@ -345,7 +345,7 @@ describe('Object storage for videos', function () { } }) - const goodCredentials = ObjectStorageCommand.getCredentialsConfig() + const goodCredentials = ObjectStorageCommand.getMockCredentialsConfig() server = await createSingleServer(1, config, { env: { @@ -361,7 +361,7 @@ describe('Object storage for videos', function () { await waitJobs([ server ], true) const video = await server.videos.get({ id: uuid }) - expectStartWith(video.files[0].fileUrl, ObjectStorageCommand.getWebTorrentBaseUrl()) + expectStartWith(video.files[0].fileUrl, ObjectStorageCommand.getMockWebTorrentBaseUrl()) }) after(async function () { diff --git a/server/tests/api/server/proxy.ts b/server/tests/api/server/proxy.ts index a4151ebdd..3700b0746 100644 --- a/server/tests/api/server/proxy.ts +++ b/server/tests/api/server/proxy.ts @@ -2,7 +2,7 @@ import { expect } from 'chai' import { expectNotStartWith, expectStartWith, FIXTURE_URLS, MockProxy } from '@server/tests/shared' -import { areObjectStorageTestsDisabled } from '@shared/core-utils' +import { areMockObjectStorageTestsDisabled } from '@shared/core-utils' import { HttpStatusCode, VideoPrivacy } from '@shared/models' import { cleanupTests, @@ -120,40 +120,40 @@ describe('Test proxy', function () { }) describe('Object storage', function () { - if (areObjectStorageTestsDisabled()) return + if (areMockObjectStorageTestsDisabled()) return before(async function () { this.timeout(30000) - await ObjectStorageCommand.prepareDefaultBuckets() + await ObjectStorageCommand.prepareDefaultMockBuckets() }) it('Should succeed to upload to object storage with the appropriate proxy config', async function () { this.timeout(120000) await servers[0].kill() - await servers[0].run(ObjectStorageCommand.getDefaultConfig(), { env: goodEnv }) + await servers[0].run(ObjectStorageCommand.getDefaultMockConfig(), { env: goodEnv }) const { uuid } = await servers[0].videos.quickUpload({ name: 'video' }) await waitJobs(servers) const video = await servers[0].videos.get({ id: uuid }) - expectStartWith(video.files[0].fileUrl, ObjectStorageCommand.getWebTorrentBaseUrl()) + expectStartWith(video.files[0].fileUrl, ObjectStorageCommand.getMockWebTorrentBaseUrl()) }) it('Should fail to upload to object storage with a wrong proxy config', async function () { this.timeout(120000) await servers[0].kill() - await servers[0].run(ObjectStorageCommand.getDefaultConfig(), { env: badEnv }) + await servers[0].run(ObjectStorageCommand.getDefaultMockConfig(), { env: badEnv }) const { uuid } = await servers[0].videos.quickUpload({ name: 'video' }) await waitJobs(servers) const video = await servers[0].videos.get({ id: uuid }) - expectNotStartWith(video.files[0].fileUrl, ObjectStorageCommand.getWebTorrentBaseUrl()) + expectNotStartWith(video.files[0].fileUrl, ObjectStorageCommand.getMockWebTorrentBaseUrl()) }) }) diff --git a/server/tests/api/transcoding/create-transcoding.ts b/server/tests/api/transcoding/create-transcoding.ts index 372f5332a..85389a949 100644 --- a/server/tests/api/transcoding/create-transcoding.ts +++ b/server/tests/api/transcoding/create-transcoding.ts @@ -2,7 +2,7 @@ import { expect } from 'chai' import { checkResolutionsInMasterPlaylist, expectStartWith } from '@server/tests/shared' -import { areObjectStorageTestsDisabled } from '@shared/core-utils' +import { areMockObjectStorageTestsDisabled } from '@shared/core-utils' import { HttpStatusCode, VideoDetails } from '@shared/models' import { cleanupTests, @@ -19,7 +19,7 @@ import { async function checkFilesInObjectStorage (video: VideoDetails) { for (const file of video.files) { - expectStartWith(file.fileUrl, ObjectStorageCommand.getWebTorrentBaseUrl()) + expectStartWith(file.fileUrl, ObjectStorageCommand.getMockWebTorrentBaseUrl()) await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 }) } @@ -27,14 +27,14 @@ async function checkFilesInObjectStorage (video: VideoDetails) { const hlsPlaylist = video.streamingPlaylists[0] for (const file of hlsPlaylist.files) { - expectStartWith(file.fileUrl, ObjectStorageCommand.getPlaylistBaseUrl()) + expectStartWith(file.fileUrl, ObjectStorageCommand.getMockPlaylistBaseUrl()) await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 }) } - expectStartWith(hlsPlaylist.playlistUrl, ObjectStorageCommand.getPlaylistBaseUrl()) + expectStartWith(hlsPlaylist.playlistUrl, ObjectStorageCommand.getMockPlaylistBaseUrl()) await makeRawRequest({ url: hlsPlaylist.playlistUrl, expectedStatus: HttpStatusCode.OK_200 }) - expectStartWith(hlsPlaylist.segmentsSha256Url, ObjectStorageCommand.getPlaylistBaseUrl()) + expectStartWith(hlsPlaylist.segmentsSha256Url, ObjectStorageCommand.getMockPlaylistBaseUrl()) await makeRawRequest({ url: hlsPlaylist.segmentsSha256Url, expectedStatus: HttpStatusCode.OK_200 }) } @@ -49,7 +49,7 @@ function runTests (objectStorage: boolean) { this.timeout(120000) const config = objectStorage - ? ObjectStorageCommand.getDefaultConfig() + ? ObjectStorageCommand.getDefaultMockConfig() : {} // Run server 2 to have transcoding enabled @@ -60,7 +60,7 @@ function runTests (objectStorage: boolean) { await doubleFollow(servers[0], servers[1]) - if (objectStorage) await ObjectStorageCommand.prepareDefaultBuckets() + if (objectStorage) await ObjectStorageCommand.prepareDefaultMockBuckets() const { shortUUID } = await servers[0].videos.quickUpload({ name: 'video' }) videoUUID = shortUUID @@ -256,7 +256,7 @@ describe('Test create transcoding jobs from API', function () { }) describe('On object storage', function () { - if (areObjectStorageTestsDisabled()) return + if (areMockObjectStorageTestsDisabled()) return runTests(true) }) diff --git a/server/tests/api/transcoding/hls.ts b/server/tests/api/transcoding/hls.ts index 7b5492cd4..84a53c0bd 100644 --- a/server/tests/api/transcoding/hls.ts +++ b/server/tests/api/transcoding/hls.ts @@ -2,7 +2,7 @@ import { join } from 'path' import { checkDirectoryIsEmpty, checkTmpIsEmpty, completeCheckHlsPlaylist } from '@server/tests/shared' -import { areObjectStorageTestsDisabled } from '@shared/core-utils' +import { areMockObjectStorageTestsDisabled } from '@shared/core-utils' import { HttpStatusCode } from '@shared/models' import { cleanupTests, @@ -150,19 +150,19 @@ describe('Test HLS videos', function () { }) describe('With object storage enabled', function () { - if (areObjectStorageTestsDisabled()) return + if (areMockObjectStorageTestsDisabled()) return before(async function () { this.timeout(120000) - const configOverride = ObjectStorageCommand.getDefaultConfig() - await ObjectStorageCommand.prepareDefaultBuckets() + const configOverride = ObjectStorageCommand.getDefaultMockConfig() + await ObjectStorageCommand.prepareDefaultMockBuckets() await servers[0].kill() await servers[0].run(configOverride) }) - runTestSuite(true, ObjectStorageCommand.getPlaylistBaseUrl()) + runTestSuite(true, ObjectStorageCommand.getMockPlaylistBaseUrl()) }) after(async function () { diff --git a/server/tests/api/transcoding/update-while-transcoding.ts b/server/tests/api/transcoding/update-while-transcoding.ts index 5ca923392..8e32ea069 100644 --- a/server/tests/api/transcoding/update-while-transcoding.ts +++ b/server/tests/api/transcoding/update-while-transcoding.ts @@ -1,7 +1,7 @@ /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ import { completeCheckHlsPlaylist } from '@server/tests/shared' -import { areObjectStorageTestsDisabled, wait } from '@shared/core-utils' +import { areMockObjectStorageTestsDisabled, wait } from '@shared/core-utils' import { VideoPrivacy } from '@shared/models' import { cleanupTests, @@ -130,19 +130,19 @@ describe('Test update video privacy while transcoding', function () { }) describe('With object storage enabled', function () { - if (areObjectStorageTestsDisabled()) return + if (areMockObjectStorageTestsDisabled()) return before(async function () { this.timeout(120000) - const configOverride = ObjectStorageCommand.getDefaultConfig() - await ObjectStorageCommand.prepareDefaultBuckets() + const configOverride = ObjectStorageCommand.getDefaultMockConfig() + await ObjectStorageCommand.prepareDefaultMockBuckets() await servers[0].kill() await servers[0].run(configOverride) }) - runTestSuite(true, ObjectStorageCommand.getPlaylistBaseUrl()) + runTestSuite(true, ObjectStorageCommand.getMockPlaylistBaseUrl()) }) after(async function () { diff --git a/server/tests/api/transcoding/video-studio.ts b/server/tests/api/transcoding/video-studio.ts index 9613111b5..ab08e8fb6 100644 --- a/server/tests/api/transcoding/video-studio.ts +++ b/server/tests/api/transcoding/video-studio.ts @@ -1,6 +1,6 @@ import { expect } from 'chai' import { expectStartWith } from '@server/tests/shared' -import { areObjectStorageTestsDisabled, getAllFiles } from '@shared/core-utils' +import { areMockObjectStorageTestsDisabled, getAllFiles } from '@shared/core-utils' import { VideoStudioTask } from '@shared/models' import { cleanupTests, @@ -315,13 +315,13 @@ describe('Test video studio', function () { }) describe('Object storage video edition', function () { - if (areObjectStorageTestsDisabled()) return + if (areMockObjectStorageTestsDisabled()) return before(async function () { - await ObjectStorageCommand.prepareDefaultBuckets() + await ObjectStorageCommand.prepareDefaultMockBuckets() await servers[0].kill() - await servers[0].run(ObjectStorageCommand.getDefaultConfig()) + await servers[0].run(ObjectStorageCommand.getDefaultMockConfig()) await servers[0].config.enableMinimumTranscoding() }) @@ -344,11 +344,11 @@ describe('Test video studio', function () { } for (const webtorrentFile of video.files) { - expectStartWith(webtorrentFile.fileUrl, ObjectStorageCommand.getWebTorrentBaseUrl()) + expectStartWith(webtorrentFile.fileUrl, ObjectStorageCommand.getMockWebTorrentBaseUrl()) } for (const hlsFile of video.streamingPlaylists[0].files) { - expectStartWith(hlsFile.fileUrl, ObjectStorageCommand.getPlaylistBaseUrl()) + expectStartWith(hlsFile.fileUrl, ObjectStorageCommand.getMockPlaylistBaseUrl()) } await checkDuration(server, 9) diff --git a/server/tests/api/videos/video-static-file-privacy.ts b/server/tests/api/videos/video-static-file-privacy.ts index e38fdec6e..bdbe85127 100644 --- a/server/tests/api/videos/video-static-file-privacy.ts +++ b/server/tests/api/videos/video-static-file-privacy.ts @@ -37,7 +37,7 @@ describe('Test video static file privacy', function () { function runSuite () { - async function checkPrivateWebTorrentFiles (uuid: string) { + async function checkPrivateFiles (uuid: string) { const video = await server.videos.getWithToken({ id: uuid }) for (const file of video.files) { @@ -63,7 +63,7 @@ describe('Test video static file privacy', function () { } } - async function checkPublicWebTorrentFiles (uuid: string) { + async function checkPublicFiles (uuid: string) { const video = await server.videos.get({ id: uuid }) for (const file of getAllFiles(video)) { @@ -98,7 +98,7 @@ describe('Test video static file privacy', function () { const { uuid } = await server.videos.quickUpload({ name: 'video', privacy }) await waitJobs([ server ]) - await checkPrivateWebTorrentFiles(uuid) + await checkPrivateFiles(uuid) } }) @@ -112,7 +112,7 @@ describe('Test video static file privacy', function () { await server.videos.update({ id: uuid, attributes: { privacy } }) await waitJobs([ server ]) - await checkPrivateWebTorrentFiles(uuid) + await checkPrivateFiles(uuid) } }) @@ -125,7 +125,7 @@ describe('Test video static file privacy', function () { await server.videos.update({ id: uuid, attributes: { privacy: VideoPrivacy.UNLISTED } }) await waitJobs([ server ]) - await checkPublicWebTorrentFiles(uuid) + await checkPublicFiles(uuid) }) it('Should upload an internal video and update it to public to have a public static path', async function () { @@ -137,7 +137,7 @@ describe('Test video static file privacy', function () { await server.videos.update({ id: uuid, attributes: { privacy: VideoPrivacy.PUBLIC } }) await waitJobs([ server ]) - await checkPublicWebTorrentFiles(uuid) + await checkPublicFiles(uuid) }) it('Should upload an internal video and schedule a public publish', async function () { @@ -160,7 +160,7 @@ describe('Test video static file privacy', function () { await waitJobs([ server ]) - await checkPublicWebTorrentFiles(uuid) + await checkPublicFiles(uuid) }) } diff --git a/server/tests/cli/create-import-video-file-job.ts b/server/tests/cli/create-import-video-file-job.ts index a4aa5f699..43f53035b 100644 --- a/server/tests/cli/create-import-video-file-job.ts +++ b/server/tests/cli/create-import-video-file-job.ts @@ -1,7 +1,7 @@ /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ import { expect } from 'chai' -import { areObjectStorageTestsDisabled } from '@shared/core-utils' +import { areMockObjectStorageTestsDisabled } from '@shared/core-utils' import { HttpStatusCode, VideoDetails, VideoFile, VideoInclude } from '@shared/models' import { cleanupTests, @@ -27,7 +27,7 @@ function assertVideoProperties (video: VideoFile, resolution: number, extname: s async function checkFiles (video: VideoDetails, objectStorage: boolean) { for (const file of video.files) { - if (objectStorage) expectStartWith(file.fileUrl, ObjectStorageCommand.getWebTorrentBaseUrl()) + if (objectStorage) expectStartWith(file.fileUrl, ObjectStorageCommand.getMockWebTorrentBaseUrl()) await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 }) } @@ -43,7 +43,7 @@ function runTests (objectStorage: boolean) { this.timeout(90000) const config = objectStorage - ? ObjectStorageCommand.getDefaultConfig() + ? ObjectStorageCommand.getDefaultMockConfig() : {} // Run server 2 to have transcoding enabled @@ -52,7 +52,7 @@ function runTests (objectStorage: boolean) { await doubleFollow(servers[0], servers[1]) - if (objectStorage) await ObjectStorageCommand.prepareDefaultBuckets() + if (objectStorage) await ObjectStorageCommand.prepareDefaultMockBuckets() // Upload two videos for our needs { @@ -157,7 +157,7 @@ describe('Test create import video jobs', function () { }) describe('On object storage', function () { - if (areObjectStorageTestsDisabled()) return + if (areMockObjectStorageTestsDisabled()) return runTests(true) }) diff --git a/server/tests/cli/create-move-video-storage-job.ts b/server/tests/cli/create-move-video-storage-job.ts index ecdd75b76..c357f501b 100644 --- a/server/tests/cli/create-move-video-storage-job.ts +++ b/server/tests/cli/create-move-video-storage-job.ts @@ -1,6 +1,6 @@ /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ -import { areObjectStorageTestsDisabled } from '@shared/core-utils' +import { areMockObjectStorageTestsDisabled } from '@shared/core-utils' import { HttpStatusCode, VideoDetails } from '@shared/models' import { cleanupTests, @@ -17,7 +17,7 @@ import { expectStartWith } from '../shared' async function checkFiles (origin: PeerTubeServer, video: VideoDetails, inObjectStorage: boolean) { for (const file of video.files) { const start = inObjectStorage - ? ObjectStorageCommand.getWebTorrentBaseUrl() + ? ObjectStorageCommand.getMockWebTorrentBaseUrl() : origin.url expectStartWith(file.fileUrl, start) @@ -26,7 +26,7 @@ async function checkFiles (origin: PeerTubeServer, video: VideoDetails, inObject } const start = inObjectStorage - ? ObjectStorageCommand.getPlaylistBaseUrl() + ? ObjectStorageCommand.getMockPlaylistBaseUrl() : origin.url const hls = video.streamingPlaylists[0] @@ -41,7 +41,7 @@ async function checkFiles (origin: PeerTubeServer, video: VideoDetails, inObject } describe('Test create move video storage job', function () { - if (areObjectStorageTestsDisabled()) return + if (areMockObjectStorageTestsDisabled()) return let servers: PeerTubeServer[] = [] const uuids: string[] = [] @@ -55,7 +55,7 @@ describe('Test create move video storage job', function () { await doubleFollow(servers[0], servers[1]) - await ObjectStorageCommand.prepareDefaultBuckets() + await ObjectStorageCommand.prepareDefaultMockBuckets() await servers[0].config.enableTranscoding() @@ -67,14 +67,14 @@ describe('Test create move video storage job', function () { await waitJobs(servers) await servers[0].kill() - await servers[0].run(ObjectStorageCommand.getDefaultConfig()) + await servers[0].run(ObjectStorageCommand.getDefaultMockConfig()) }) it('Should move only one file', async function () { this.timeout(120000) const command = `npm run create-move-video-storage-job -- --to-object-storage -v ${uuids[1]}` - await servers[0].cli.execWithEnv(command, ObjectStorageCommand.getDefaultConfig()) + await servers[0].cli.execWithEnv(command, ObjectStorageCommand.getDefaultMockConfig()) await waitJobs(servers) for (const server of servers) { @@ -94,7 +94,7 @@ describe('Test create move video storage job', function () { this.timeout(120000) const command = `npm run create-move-video-storage-job -- --to-object-storage --all-videos` - await servers[0].cli.execWithEnv(command, ObjectStorageCommand.getDefaultConfig()) + await servers[0].cli.execWithEnv(command, ObjectStorageCommand.getDefaultMockConfig()) await waitJobs(servers) for (const server of servers) { diff --git a/server/tests/cli/create-transcoding-job.ts b/server/tests/cli/create-transcoding-job.ts index 51bf04a80..38b737829 100644 --- a/server/tests/cli/create-transcoding-job.ts +++ b/server/tests/cli/create-transcoding-job.ts @@ -1,7 +1,7 @@ /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ import { expect } from 'chai' -import { areObjectStorageTestsDisabled } from '@shared/core-utils' +import { areMockObjectStorageTestsDisabled } from '@shared/core-utils' import { HttpStatusCode, VideoFile } from '@shared/models' import { cleanupTests, @@ -18,8 +18,8 @@ import { checkResolutionsInMasterPlaylist, expectStartWith } from '../shared' async function checkFilesInObjectStorage (files: VideoFile[], type: 'webtorrent' | 'playlist') { for (const file of files) { const shouldStartWith = type === 'webtorrent' - ? ObjectStorageCommand.getWebTorrentBaseUrl() - : ObjectStorageCommand.getPlaylistBaseUrl() + ? ObjectStorageCommand.getMockWebTorrentBaseUrl() + : ObjectStorageCommand.getMockPlaylistBaseUrl() expectStartWith(file.fileUrl, shouldStartWith) @@ -36,7 +36,7 @@ function runTests (objectStorage: boolean) { this.timeout(120000) const config = objectStorage - ? ObjectStorageCommand.getDefaultConfig() + ? ObjectStorageCommand.getDefaultMockConfig() : {} // Run server 2 to have transcoding enabled @@ -47,7 +47,7 @@ function runTests (objectStorage: boolean) { await doubleFollow(servers[0], servers[1]) - if (objectStorage) await ObjectStorageCommand.prepareDefaultBuckets() + if (objectStorage) await ObjectStorageCommand.prepareDefaultMockBuckets() for (let i = 1; i <= 5; i++) { const { uuid, shortUUID } = await servers[0].videos.upload({ attributes: { name: 'video' + i } }) @@ -255,7 +255,7 @@ describe('Test create transcoding jobs', function () { }) describe('On object storage', function () { - if (areObjectStorageTestsDisabled()) return + if (areMockObjectStorageTestsDisabled()) return runTests(true) }) diff --git a/server/tests/shared/live.ts b/server/tests/shared/live.ts index da3691711..78e29f575 100644 --- a/server/tests/shared/live.ts +++ b/server/tests/shared/live.ts @@ -50,7 +50,7 @@ async function testVideoResolutions (options: { }) if (objectStorage) { - expect(hlsPlaylist.playlistUrl).to.contain(ObjectStorageCommand.getPlaylistBaseUrl()) + expect(hlsPlaylist.playlistUrl).to.contain(ObjectStorageCommand.getMockPlaylistBaseUrl()) } for (let i = 0; i < resolutions.length; i++) { @@ -65,11 +65,11 @@ async function testVideoResolutions (options: { }) const baseUrl = objectStorage - ? ObjectStorageCommand.getPlaylistBaseUrl() + 'hls' + ? ObjectStorageCommand.getMockPlaylistBaseUrl() + 'hls' : originServer.url + '/static/streaming-playlists/hls' if (objectStorage) { - expect(hlsPlaylist.segmentsSha256Url).to.contain(ObjectStorageCommand.getPlaylistBaseUrl()) + expect(hlsPlaylist.segmentsSha256Url).to.contain(ObjectStorageCommand.getMockPlaylistBaseUrl()) } const subPlaylist = await originServer.streamingPlaylists.get({ diff --git a/server/tests/shared/mock-servers/mock-object-storage.ts b/server/tests/shared/mock-servers/mock-object-storage.ts index 99d68e014..8c325bf11 100644 --- a/server/tests/shared/mock-servers/mock-object-storage.ts +++ b/server/tests/shared/mock-servers/mock-object-storage.ts @@ -12,7 +12,7 @@ export class MockObjectStorage { const app = express() app.get('/:bucketName/:path(*)', (req: express.Request, res: express.Response, next: express.NextFunction) => { - const url = `http://${req.params.bucketName}.${ObjectStorageCommand.getEndpointHost()}/${req.params.path}` + const url = `http://${req.params.bucketName}.${ObjectStorageCommand.getMockEndpointHost()}/${req.params.path}` if (process.env.DEBUG) { console.log('Receiving request on mocked server %s.', req.url) diff --git a/server/types/express.d.ts b/server/types/express.d.ts index 27d60da72..3738ffc47 100644 --- a/server/types/express.d.ts +++ b/server/types/express.d.ts @@ -97,7 +97,7 @@ declare module 'express' { title?: string status?: number - type?: ServerErrorCode + type?: ServerErrorCode | string instance?: string data?: PeerTubeProblemDocumentData -- cgit v1.2.3