aboutsummaryrefslogtreecommitdiffhomepage
path: root/server
diff options
context:
space:
mode:
authorChocobozzz <me@florianbigard.com>2022-10-19 10:43:53 +0200
committerChocobozzz <chocobozzz@cpy.re>2022-10-24 14:48:24 +0200
commit9ab330b90decf4edf152ff8e1d2948c065766b2c (patch)
tree29d924f50f7307e8e828a57ecb9ea78623487ce0 /server
parent3545e72c686ff1725bbdfd8d16d693e2f4aa75a3 (diff)
downloadPeerTube-9ab330b90decf4edf152ff8e1d2948c065766b2c.tar.gz
PeerTube-9ab330b90decf4edf152ff8e1d2948c065766b2c.tar.zst
PeerTube-9ab330b90decf4edf152ff8e1d2948c065766b2c.zip
Use private ACL for private videos in s3
Diffstat (limited to 'server')
-rw-r--r--server/controllers/download.ts22
-rw-r--r--server/controllers/index.ts11
-rw-r--r--server/controllers/object-storage-proxy.ts78
-rw-r--r--server/helpers/webtorrent.ts4
-rw-r--r--server/initializers/checker-after-init.ts8
-rw-r--r--server/initializers/config.ts5
-rw-r--r--server/initializers/constants.ts8
-rw-r--r--server/lib/live/live-segment-sha-store.ts27
-rw-r--r--server/lib/object-storage/shared/object-storage-helpers.ts192
-rw-r--r--server/lib/object-storage/urls.ts29
-rw-r--r--server/lib/object-storage/videos.ts80
-rw-r--r--server/lib/video-privacy.ts89
-rw-r--r--server/middlewares/validators/shared/videos.ts6
-rw-r--r--server/middlewares/validators/static.ts72
-rw-r--r--server/models/video/video-file.ts62
-rw-r--r--server/models/video/video-streaming-playlist.ts28
-rw-r--r--server/models/video/video.ts34
-rw-r--r--server/tests/api/object-storage/index.ts1
-rw-r--r--server/tests/api/object-storage/live.ts10
-rw-r--r--server/tests/api/object-storage/video-imports.ts14
-rw-r--r--server/tests/api/object-storage/video-static-file-privacy.ts336
-rw-r--r--server/tests/api/object-storage/videos.ts36
-rw-r--r--server/tests/api/server/proxy.ts14
-rw-r--r--server/tests/api/transcoding/create-transcoding.ts16
-rw-r--r--server/tests/api/transcoding/hls.ts10
-rw-r--r--server/tests/api/transcoding/update-while-transcoding.ts10
-rw-r--r--server/tests/api/transcoding/video-studio.ts12
-rw-r--r--server/tests/api/videos/video-static-file-privacy.ts14
-rw-r--r--server/tests/cli/create-import-video-file-job.ts10
-rw-r--r--server/tests/cli/create-move-video-storage-job.ts16
-rw-r--r--server/tests/cli/create-transcoding-job.ts12
-rw-r--r--server/tests/shared/live.ts6
-rw-r--r--server/tests/shared/mock-servers/mock-object-storage.ts2
-rw-r--r--server/types/express.d.ts2
34 files changed, 1036 insertions, 240 deletions
diff --git a/server/controllers/download.ts b/server/controllers/download.ts
index abd1df26f..d9f34109f 100644
--- a/server/controllers/download.ts
+++ b/server/controllers/download.ts
@@ -5,6 +5,7 @@ import { VideosTorrentCache } from '@server/lib/files-cache/videos-torrent-cache
5import { Hooks } from '@server/lib/plugins/hooks' 5import { Hooks } from '@server/lib/plugins/hooks'
6import { VideoPathManager } from '@server/lib/video-path-manager' 6import { VideoPathManager } from '@server/lib/video-path-manager'
7import { MStreamingPlaylist, MVideo, MVideoFile, MVideoFullLight } from '@server/types/models' 7import { MStreamingPlaylist, MVideo, MVideoFile, MVideoFullLight } from '@server/types/models'
8import { addQueryParams } from '@shared/core-utils'
8import { HttpStatusCode, VideoStorage, VideoStreamingPlaylistType } from '@shared/models' 9import { HttpStatusCode, VideoStorage, VideoStreamingPlaylistType } from '@shared/models'
9import { STATIC_DOWNLOAD_PATHS } from '../initializers/constants' 10import { STATIC_DOWNLOAD_PATHS } from '../initializers/constants'
10import { asyncMiddleware, optionalAuthenticate, videosDownloadValidator } from '../middlewares' 11import { asyncMiddleware, optionalAuthenticate, videosDownloadValidator } from '../middlewares'
@@ -84,7 +85,7 @@ async function downloadVideoFile (req: express.Request, res: express.Response) {
84 if (!checkAllowResult(res, allowParameters, allowedResult)) return 85 if (!checkAllowResult(res, allowParameters, allowedResult)) return
85 86
86 if (videoFile.storage === VideoStorage.OBJECT_STORAGE) { 87 if (videoFile.storage === VideoStorage.OBJECT_STORAGE) {
87 return res.redirect(videoFile.getObjectStorageUrl()) 88 return redirectToObjectStorage({ req, res, video, file: videoFile })
88 } 89 }
89 90
90 await VideoPathManager.Instance.makeAvailableVideoFile(videoFile.withVideoOrPlaylist(video), path => { 91 await VideoPathManager.Instance.makeAvailableVideoFile(videoFile.withVideoOrPlaylist(video), path => {
@@ -120,7 +121,7 @@ async function downloadHLSVideoFile (req: express.Request, res: express.Response
120 if (!checkAllowResult(res, allowParameters, allowedResult)) return 121 if (!checkAllowResult(res, allowParameters, allowedResult)) return
121 122
122 if (videoFile.storage === VideoStorage.OBJECT_STORAGE) { 123 if (videoFile.storage === VideoStorage.OBJECT_STORAGE) {
123 return res.redirect(videoFile.getObjectStorageUrl()) 124 return redirectToObjectStorage({ req, res, video, file: videoFile })
124 } 125 }
125 126
126 await VideoPathManager.Instance.makeAvailableVideoFile(videoFile.withVideoOrPlaylist(streamingPlaylist), path => { 127 await VideoPathManager.Instance.makeAvailableVideoFile(videoFile.withVideoOrPlaylist(streamingPlaylist), path => {
@@ -174,3 +175,20 @@ function checkAllowResult (res: express.Response, allowParameters: any, result?:
174 175
175 return true 176 return true
176} 177}
178
179function redirectToObjectStorage (options: {
180 req: express.Request
181 res: express.Response
182 video: MVideo
183 file: MVideoFile
184}) {
185 const { req, res, video, file } = options
186
187 const baseUrl = file.getObjectStorageUrl(video)
188
189 const url = video.hasPrivateStaticPath() && req.query.videoFileToken
190 ? addQueryParams(baseUrl, { videoFileToken: req.query.videoFileToken })
191 : baseUrl
192
193 return res.redirect(url)
194}
diff --git a/server/controllers/index.ts b/server/controllers/index.ts
index 8574a9e7b..eaa2dd7c8 100644
--- a/server/controllers/index.ts
+++ b/server/controllers/index.ts
@@ -1,14 +1,15 @@
1export * from './activitypub' 1export * from './activitypub'
2export * from './api' 2export * from './api'
3export * from './bots'
3export * from './client' 4export * from './client'
4export * from './download' 5export * from './download'
5export * from './feeds' 6export * from './feeds'
6export * from './services'
7export * from './static'
8export * from './lazy-static' 7export * from './lazy-static'
9export * from './misc' 8export * from './misc'
10export * from './webfinger' 9export * from './object-storage-proxy'
11export * from './tracker'
12export * from './bots'
13export * from './plugins' 10export * from './plugins'
11export * from './services'
12export * from './static'
13export * from './tracker'
14export * from './webfinger'
14export * from './well-known' 15export * from './well-known'
diff --git a/server/controllers/object-storage-proxy.ts b/server/controllers/object-storage-proxy.ts
new file mode 100644
index 000000000..6fedcfd8f
--- /dev/null
+++ b/server/controllers/object-storage-proxy.ts
@@ -0,0 +1,78 @@
1import cors from 'cors'
2import express from 'express'
3import { OBJECT_STORAGE_PROXY_PATHS } from '@server/initializers/constants'
4import { getHLSFileReadStream, getWebTorrentFileReadStream } from '@server/lib/object-storage'
5import {
6 asyncMiddleware,
7 ensureCanAccessPrivateVideoHLSFiles,
8 ensureCanAccessVideoPrivateWebTorrentFiles,
9 optionalAuthenticate
10} from '@server/middlewares'
11import { HttpStatusCode } from '@shared/models'
12
13const objectStorageProxyRouter = express.Router()
14
15objectStorageProxyRouter.use(cors())
16
17objectStorageProxyRouter.get(OBJECT_STORAGE_PROXY_PATHS.PRIVATE_WEBSEED + ':filename',
18 optionalAuthenticate,
19 asyncMiddleware(ensureCanAccessVideoPrivateWebTorrentFiles),
20 asyncMiddleware(proxifyWebTorrent)
21)
22
23objectStorageProxyRouter.get(OBJECT_STORAGE_PROXY_PATHS.STREAMING_PLAYLISTS.PRIVATE_HLS + ':videoUUID/:filename',
24 optionalAuthenticate,
25 asyncMiddleware(ensureCanAccessPrivateVideoHLSFiles),
26 asyncMiddleware(proxifyHLS)
27)
28
29// ---------------------------------------------------------------------------
30
31export {
32 objectStorageProxyRouter
33}
34
35async function proxifyWebTorrent (req: express.Request, res: express.Response) {
36 const filename = req.params.filename
37
38 try {
39 const stream = await getWebTorrentFileReadStream({
40 filename,
41 rangeHeader: req.header('range')
42 })
43
44 return stream.pipe(res)
45 } catch (err) {
46 return handleObjectStorageFailure(res, err)
47 }
48}
49
50async function proxifyHLS (req: express.Request, res: express.Response) {
51 const playlist = res.locals.videoStreamingPlaylist
52 const video = res.locals.onlyVideo
53 const filename = req.params.filename
54
55 try {
56 const stream = await getHLSFileReadStream({
57 playlist: playlist.withVideo(video),
58 filename,
59 rangeHeader: req.header('range')
60 })
61
62 return stream.pipe(res)
63 } catch (err) {
64 return handleObjectStorageFailure(res, err)
65 }
66}
67
68function handleObjectStorageFailure (res: express.Response, err: Error) {
69 if (err.name === 'NoSuchKey') {
70 return res.sendStatus(HttpStatusCode.NOT_FOUND_404)
71 }
72
73 return res.fail({
74 status: HttpStatusCode.INTERNAL_SERVER_ERROR_500,
75 message: err.message,
76 type: err.name
77 })
78}
diff --git a/server/helpers/webtorrent.ts b/server/helpers/webtorrent.ts
index 6d87c74f7..b458e86d2 100644
--- a/server/helpers/webtorrent.ts
+++ b/server/helpers/webtorrent.ts
@@ -165,7 +165,7 @@ function generateMagnetUri (
165 const xs = videoFile.getTorrentUrl() 165 const xs = videoFile.getTorrentUrl()
166 const announce = trackerUrls 166 const announce = trackerUrls
167 167
168 let urlList = video.requiresAuth(video.uuid) 168 let urlList = video.hasPrivateStaticPath()
169 ? [] 169 ? []
170 : [ videoFile.getFileUrl(video) ] 170 : [ videoFile.getFileUrl(video) ]
171 171
@@ -243,7 +243,7 @@ function buildAnnounceList () {
243} 243}
244 244
245function buildUrlList (video: MVideo, videoFile: MVideoFile) { 245function buildUrlList (video: MVideo, videoFile: MVideoFile) {
246 if (video.requiresAuth(video.uuid)) return [] 246 if (video.hasPrivateStaticPath()) return []
247 247
248 return [ videoFile.getFileUrl(video) ] 248 return [ videoFile.getFileUrl(video) ]
249} 249}
diff --git a/server/initializers/checker-after-init.ts b/server/initializers/checker-after-init.ts
index c83fef425..09e878eee 100644
--- a/server/initializers/checker-after-init.ts
+++ b/server/initializers/checker-after-init.ts
@@ -278,6 +278,14 @@ function checkObjectStorageConfig () {
278 'Object storage bucket prefixes should be set to different values when the same bucket is used for both types of video.' 278 'Object storage bucket prefixes should be set to different values when the same bucket is used for both types of video.'
279 ) 279 )
280 } 280 }
281
282 if (!CONFIG.OBJECT_STORAGE.UPLOAD_ACL.PUBLIC) {
283 throw new Error('object_storage.upload_acl.public must be set')
284 }
285
286 if (!CONFIG.OBJECT_STORAGE.UPLOAD_ACL.PRIVATE) {
287 throw new Error('object_storage.upload_acl.private must be set')
288 }
281 } 289 }
282} 290}
283 291
diff --git a/server/initializers/config.ts b/server/initializers/config.ts
index a5a0d4e46..ab5e645ad 100644
--- a/server/initializers/config.ts
+++ b/server/initializers/config.ts
@@ -118,7 +118,10 @@ const CONFIG = {
118 MAX_UPLOAD_PART: bytes.parse(config.get<string>('object_storage.max_upload_part')), 118 MAX_UPLOAD_PART: bytes.parse(config.get<string>('object_storage.max_upload_part')),
119 ENDPOINT: config.get<string>('object_storage.endpoint'), 119 ENDPOINT: config.get<string>('object_storage.endpoint'),
120 REGION: config.get<string>('object_storage.region'), 120 REGION: config.get<string>('object_storage.region'),
121 UPLOAD_ACL: config.get<string>('object_storage.upload_acl'), 121 UPLOAD_ACL: {
122 PUBLIC: config.get<string>('object_storage.upload_acl.public'),
123 PRIVATE: config.get<string>('object_storage.upload_acl.private')
124 },
122 CREDENTIALS: { 125 CREDENTIALS: {
123 ACCESS_KEY_ID: config.get<string>('object_storage.credentials.access_key_id'), 126 ACCESS_KEY_ID: config.get<string>('object_storage.credentials.access_key_id'),
124 SECRET_ACCESS_KEY: config.get<string>('object_storage.credentials.secret_access_key') 127 SECRET_ACCESS_KEY: config.get<string>('object_storage.credentials.secret_access_key')
diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts
index 88bdd07fe..66eb31230 100644
--- a/server/initializers/constants.ts
+++ b/server/initializers/constants.ts
@@ -685,6 +685,13 @@ const LAZY_STATIC_PATHS = {
685 VIDEO_CAPTIONS: '/lazy-static/video-captions/', 685 VIDEO_CAPTIONS: '/lazy-static/video-captions/',
686 TORRENTS: '/lazy-static/torrents/' 686 TORRENTS: '/lazy-static/torrents/'
687} 687}
688const OBJECT_STORAGE_PROXY_PATHS = {
689 PRIVATE_WEBSEED: '/object-storage-proxy/webseed/private/',
690
691 STREAMING_PLAYLISTS: {
692 PRIVATE_HLS: '/object-storage-proxy/streaming-playlists/hls/private/'
693 }
694}
688 695
689// Cache control 696// Cache control
690const STATIC_MAX_AGE = { 697const STATIC_MAX_AGE = {
@@ -995,6 +1002,7 @@ export {
995 VIDEO_LIVE, 1002 VIDEO_LIVE,
996 PEERTUBE_VERSION, 1003 PEERTUBE_VERSION,
997 LAZY_STATIC_PATHS, 1004 LAZY_STATIC_PATHS,
1005 OBJECT_STORAGE_PROXY_PATHS,
998 SEARCH_INDEX, 1006 SEARCH_INDEX,
999 DIRECTORIES, 1007 DIRECTORIES,
1000 RESUMABLE_UPLOAD_SESSION_LIFETIME, 1008 RESUMABLE_UPLOAD_SESSION_LIFETIME,
diff --git a/server/lib/live/live-segment-sha-store.ts b/server/lib/live/live-segment-sha-store.ts
index faf03dccf..4d03754a9 100644
--- a/server/lib/live/live-segment-sha-store.ts
+++ b/server/lib/live/live-segment-sha-store.ts
@@ -5,6 +5,7 @@ import { logger, loggerTagsFactory } from '@server/helpers/logger'
5import { MStreamingPlaylistVideo } from '@server/types/models' 5import { MStreamingPlaylistVideo } from '@server/types/models'
6import { buildSha256Segment } from '../hls' 6import { buildSha256Segment } from '../hls'
7import { storeHLSFileFromPath } from '../object-storage' 7import { storeHLSFileFromPath } from '../object-storage'
8import PQueue from 'p-queue'
8 9
9const lTags = loggerTagsFactory('live') 10const lTags = loggerTagsFactory('live')
10 11
@@ -16,6 +17,7 @@ class LiveSegmentShaStore {
16 private readonly sha256Path: string 17 private readonly sha256Path: string
17 private readonly streamingPlaylist: MStreamingPlaylistVideo 18 private readonly streamingPlaylist: MStreamingPlaylistVideo
18 private readonly sendToObjectStorage: boolean 19 private readonly sendToObjectStorage: boolean
20 private readonly writeQueue = new PQueue({ concurrency: 1 })
19 21
20 constructor (options: { 22 constructor (options: {
21 videoUUID: string 23 videoUUID: string
@@ -37,7 +39,11 @@ class LiveSegmentShaStore {
37 const segmentName = basename(segmentPath) 39 const segmentName = basename(segmentPath)
38 this.segmentsSha256.set(segmentName, shaResult) 40 this.segmentsSha256.set(segmentName, shaResult)
39 41
40 await this.writeToDisk() 42 try {
43 await this.writeToDisk()
44 } catch (err) {
45 logger.error('Cannot write sha segments to disk.', { err })
46 }
41 } 47 }
42 48
43 async removeSegmentSha (segmentPath: string) { 49 async removeSegmentSha (segmentPath: string) {
@@ -55,19 +61,20 @@ class LiveSegmentShaStore {
55 await this.writeToDisk() 61 await this.writeToDisk()
56 } 62 }
57 63
58 private async writeToDisk () { 64 private writeToDisk () {
59 await writeJson(this.sha256Path, mapToJSON(this.segmentsSha256)) 65 return this.writeQueue.add(async () => {
66 await writeJson(this.sha256Path, mapToJSON(this.segmentsSha256))
60 67
61 if (this.sendToObjectStorage) { 68 if (this.sendToObjectStorage) {
62 const url = await storeHLSFileFromPath(this.streamingPlaylist, this.sha256Path) 69 const url = await storeHLSFileFromPath(this.streamingPlaylist, this.sha256Path)
63 70
64 if (this.streamingPlaylist.segmentsSha256Url !== url) { 71 if (this.streamingPlaylist.segmentsSha256Url !== url) {
65 this.streamingPlaylist.segmentsSha256Url = url 72 this.streamingPlaylist.segmentsSha256Url = url
66 await this.streamingPlaylist.save() 73 await this.streamingPlaylist.save()
74 }
67 } 75 }
68 } 76 })
69 } 77 }
70
71} 78}
72 79
73export { 80export {
diff --git a/server/lib/object-storage/shared/object-storage-helpers.ts b/server/lib/object-storage/shared/object-storage-helpers.ts
index c131977e8..05b52f412 100644
--- a/server/lib/object-storage/shared/object-storage-helpers.ts
+++ b/server/lib/object-storage/shared/object-storage-helpers.ts
@@ -2,18 +2,21 @@ import { createReadStream, createWriteStream, ensureDir, ReadStream } from 'fs-e
2import { dirname } from 'path' 2import { dirname } from 'path'
3import { Readable } from 'stream' 3import { Readable } from 'stream'
4import { 4import {
5 _Object,
5 CompleteMultipartUploadCommandOutput, 6 CompleteMultipartUploadCommandOutput,
6 DeleteObjectCommand, 7 DeleteObjectCommand,
7 GetObjectCommand, 8 GetObjectCommand,
8 ListObjectsV2Command, 9 ListObjectsV2Command,
9 PutObjectCommandInput 10 PutObjectAclCommand,
11 PutObjectCommandInput,
12 S3Client
10} from '@aws-sdk/client-s3' 13} from '@aws-sdk/client-s3'
11import { Upload } from '@aws-sdk/lib-storage' 14import { Upload } from '@aws-sdk/lib-storage'
12import { pipelinePromise } from '@server/helpers/core-utils' 15import { pipelinePromise } from '@server/helpers/core-utils'
13import { isArray } from '@server/helpers/custom-validators/misc' 16import { isArray } from '@server/helpers/custom-validators/misc'
14import { logger } from '@server/helpers/logger' 17import { logger } from '@server/helpers/logger'
15import { CONFIG } from '@server/initializers/config' 18import { CONFIG } from '@server/initializers/config'
16import { getPrivateUrl } from '../urls' 19import { getInternalUrl } from '../urls'
17import { getClient } from './client' 20import { getClient } from './client'
18import { lTags } from './logger' 21import { lTags } from './logger'
19 22
@@ -44,69 +47,91 @@ async function storeObject (options: {
44 inputPath: string 47 inputPath: string
45 objectStorageKey: string 48 objectStorageKey: string
46 bucketInfo: BucketInfo 49 bucketInfo: BucketInfo
50 isPrivate: boolean
47}): Promise<string> { 51}): Promise<string> {
48 const { inputPath, objectStorageKey, bucketInfo } = options 52 const { inputPath, objectStorageKey, bucketInfo, isPrivate } = options
49 53
50 logger.debug('Uploading file %s to %s%s in bucket %s', inputPath, bucketInfo.PREFIX, objectStorageKey, bucketInfo.BUCKET_NAME, lTags()) 54 logger.debug('Uploading file %s to %s%s in bucket %s', inputPath, bucketInfo.PREFIX, objectStorageKey, bucketInfo.BUCKET_NAME, lTags())
51 55
52 const fileStream = createReadStream(inputPath) 56 const fileStream = createReadStream(inputPath)
53 57
54 return uploadToStorage({ objectStorageKey, content: fileStream, bucketInfo }) 58 return uploadToStorage({ objectStorageKey, content: fileStream, bucketInfo, isPrivate })
55} 59}
56 60
57// --------------------------------------------------------------------------- 61// ---------------------------------------------------------------------------
58 62
59async function removeObject (filename: string, bucketInfo: BucketInfo) { 63function updateObjectACL (options: {
60 const command = new DeleteObjectCommand({ 64 objectStorageKey: string
65 bucketInfo: BucketInfo
66 isPrivate: boolean
67}) {
68 const { objectStorageKey, bucketInfo, isPrivate } = options
69
70 const key = buildKey(objectStorageKey, bucketInfo)
71
72 logger.debug('Updating ACL file %s in bucket %s', key, bucketInfo.BUCKET_NAME, lTags())
73
74 const command = new PutObjectAclCommand({
61 Bucket: bucketInfo.BUCKET_NAME, 75 Bucket: bucketInfo.BUCKET_NAME,
62 Key: buildKey(filename, bucketInfo) 76 Key: key,
77 ACL: getACL(isPrivate)
63 }) 78 })
64 79
65 return getClient().send(command) 80 return getClient().send(command)
66} 81}
67 82
68async function removePrefix (prefix: string, bucketInfo: BucketInfo) { 83function updatePrefixACL (options: {
69 const s3Client = getClient() 84 prefix: string
70 85 bucketInfo: BucketInfo
71 const commandPrefix = bucketInfo.PREFIX + prefix 86 isPrivate: boolean
72 const listCommand = new ListObjectsV2Command({ 87}) {
73 Bucket: bucketInfo.BUCKET_NAME, 88 const { prefix, bucketInfo, isPrivate } = options
74 Prefix: commandPrefix 89
90 logger.debug('Updating ACL of files in prefix %s in bucket %s', prefix, bucketInfo.BUCKET_NAME, lTags())
91
92 return applyOnPrefix({
93 prefix,
94 bucketInfo,
95 commandBuilder: obj => {
96 return new PutObjectAclCommand({
97 Bucket: bucketInfo.BUCKET_NAME,
98 Key: obj.Key,
99 ACL: getACL(isPrivate)
100 })
101 }
75 }) 102 })
103}
76 104
77 const listedObjects = await s3Client.send(listCommand) 105// ---------------------------------------------------------------------------
78 106
79 // FIXME: use bulk delete when s3ninja will support this operation 107function removeObject (objectStorageKey: string, bucketInfo: BucketInfo) {
80 // const deleteParams = { 108 const key = buildKey(objectStorageKey, bucketInfo)
81 // Bucket: bucketInfo.BUCKET_NAME,
82 // Delete: { Objects: [] }
83 // }
84 109
85 if (isArray(listedObjects.Contents) !== true) { 110 logger.debug('Removing file %s in bucket %s', key, bucketInfo.BUCKET_NAME, lTags())
86 const message = `Cannot remove ${commandPrefix} prefix in bucket ${bucketInfo.BUCKET_NAME}: no files listed.`
87 111
88 logger.error(message, { response: listedObjects, ...lTags() }) 112 const command = new DeleteObjectCommand({
89 throw new Error(message) 113 Bucket: bucketInfo.BUCKET_NAME,
90 } 114 Key: key
91 115 })
92 for (const object of listedObjects.Contents) {
93 const command = new DeleteObjectCommand({
94 Bucket: bucketInfo.BUCKET_NAME,
95 Key: object.Key
96 })
97
98 await s3Client.send(command)
99 116
100 // FIXME: use bulk delete when s3ninja will support this operation 117 return getClient().send(command)
101 // deleteParams.Delete.Objects.push({ Key: object.Key }) 118}
102 }
103 119
120function removePrefix (prefix: string, bucketInfo: BucketInfo) {
104 // FIXME: use bulk delete when s3ninja will support this operation 121 // FIXME: use bulk delete when s3ninja will support this operation
105 // const deleteCommand = new DeleteObjectsCommand(deleteParams)
106 // await s3Client.send(deleteCommand)
107 122
108 // Repeat if not all objects could be listed at once (limit of 1000?) 123 logger.debug('Removing prefix %s in bucket %s', prefix, bucketInfo.BUCKET_NAME, lTags())
109 if (listedObjects.IsTruncated) await removePrefix(prefix, bucketInfo) 124
125 return applyOnPrefix({
126 prefix,
127 bucketInfo,
128 commandBuilder: obj => {
129 return new DeleteObjectCommand({
130 Bucket: bucketInfo.BUCKET_NAME,
131 Key: obj.Key
132 })
133 }
134 })
110} 135}
111 136
112// --------------------------------------------------------------------------- 137// ---------------------------------------------------------------------------
@@ -138,14 +163,42 @@ function buildKey (key: string, bucketInfo: BucketInfo) {
138 163
139// --------------------------------------------------------------------------- 164// ---------------------------------------------------------------------------
140 165
166async function createObjectReadStream (options: {
167 key: string
168 bucketInfo: BucketInfo
169 rangeHeader: string
170}) {
171 const { key, bucketInfo, rangeHeader } = options
172
173 const command = new GetObjectCommand({
174 Bucket: bucketInfo.BUCKET_NAME,
175 Key: buildKey(key, bucketInfo),
176 Range: rangeHeader
177 })
178
179 const response = await getClient().send(command)
180
181 return response.Body as Readable
182}
183
184// ---------------------------------------------------------------------------
185
141export { 186export {
142 BucketInfo, 187 BucketInfo,
143 buildKey, 188 buildKey,
189
144 storeObject, 190 storeObject,
191
145 removeObject, 192 removeObject,
146 removePrefix, 193 removePrefix,
194
147 makeAvailable, 195 makeAvailable,
148 listKeysOfPrefix 196
197 updateObjectACL,
198 updatePrefixACL,
199
200 listKeysOfPrefix,
201 createObjectReadStream
149} 202}
150 203
151// --------------------------------------------------------------------------- 204// ---------------------------------------------------------------------------
@@ -154,17 +207,15 @@ async function uploadToStorage (options: {
154 content: ReadStream 207 content: ReadStream
155 objectStorageKey: string 208 objectStorageKey: string
156 bucketInfo: BucketInfo 209 bucketInfo: BucketInfo
210 isPrivate: boolean
157}) { 211}) {
158 const { content, objectStorageKey, bucketInfo } = options 212 const { content, objectStorageKey, bucketInfo, isPrivate } = options
159 213
160 const input: PutObjectCommandInput = { 214 const input: PutObjectCommandInput = {
161 Body: content, 215 Body: content,
162 Bucket: bucketInfo.BUCKET_NAME, 216 Bucket: bucketInfo.BUCKET_NAME,
163 Key: buildKey(objectStorageKey, bucketInfo) 217 Key: buildKey(objectStorageKey, bucketInfo),
164 } 218 ACL: getACL(isPrivate)
165
166 if (CONFIG.OBJECT_STORAGE.UPLOAD_ACL) {
167 input.ACL = CONFIG.OBJECT_STORAGE.UPLOAD_ACL
168 } 219 }
169 220
170 const parallelUploads3 = new Upload({ 221 const parallelUploads3 = new Upload({
@@ -194,5 +245,50 @@ async function uploadToStorage (options: {
194 bucketInfo.PREFIX, objectStorageKey, bucketInfo.BUCKET_NAME, lTags() 245 bucketInfo.PREFIX, objectStorageKey, bucketInfo.BUCKET_NAME, lTags()
195 ) 246 )
196 247
197 return getPrivateUrl(bucketInfo, objectStorageKey) 248 return getInternalUrl(bucketInfo, objectStorageKey)
249}
250
251async function applyOnPrefix (options: {
252 prefix: string
253 bucketInfo: BucketInfo
254 commandBuilder: (obj: _Object) => Parameters<S3Client['send']>[0]
255
256 continuationToken?: string
257}) {
258 const { prefix, bucketInfo, commandBuilder, continuationToken } = options
259
260 const s3Client = getClient()
261
262 const commandPrefix = bucketInfo.PREFIX + prefix
263 const listCommand = new ListObjectsV2Command({
264 Bucket: bucketInfo.BUCKET_NAME,
265 Prefix: commandPrefix,
266 ContinuationToken: continuationToken
267 })
268
269 const listedObjects = await s3Client.send(listCommand)
270
271 if (isArray(listedObjects.Contents) !== true) {
272 const message = `Cannot apply function on ${commandPrefix} prefix in bucket ${bucketInfo.BUCKET_NAME}: no files listed.`
273
274 logger.error(message, { response: listedObjects, ...lTags() })
275 throw new Error(message)
276 }
277
278 for (const object of listedObjects.Contents) {
279 const command = commandBuilder(object)
280
281 await s3Client.send(command)
282 }
283
284 // Repeat if not all objects could be listed at once (limit of 1000?)
285 if (listedObjects.IsTruncated) {
286 await applyOnPrefix({ ...options, continuationToken: listedObjects.ContinuationToken })
287 }
288}
289
290function getACL (isPrivate: boolean) {
291 return isPrivate
292 ? CONFIG.OBJECT_STORAGE.UPLOAD_ACL.PRIVATE
293 : CONFIG.OBJECT_STORAGE.UPLOAD_ACL.PUBLIC
198} 294}
diff --git a/server/lib/object-storage/urls.ts b/server/lib/object-storage/urls.ts
index 2a889190b..a47a98b98 100644
--- a/server/lib/object-storage/urls.ts
+++ b/server/lib/object-storage/urls.ts
@@ -1,10 +1,14 @@
1import { CONFIG } from '@server/initializers/config' 1import { CONFIG } from '@server/initializers/config'
2import { OBJECT_STORAGE_PROXY_PATHS, WEBSERVER } from '@server/initializers/constants'
3import { MVideoUUID } from '@server/types/models'
2import { BucketInfo, buildKey, getEndpointParsed } from './shared' 4import { BucketInfo, buildKey, getEndpointParsed } from './shared'
3 5
4function getPrivateUrl (config: BucketInfo, keyWithoutPrefix: string) { 6function getInternalUrl (config: BucketInfo, keyWithoutPrefix: string) {
5 return getBaseUrl(config) + buildKey(keyWithoutPrefix, config) 7 return getBaseUrl(config) + buildKey(keyWithoutPrefix, config)
6} 8}
7 9
10// ---------------------------------------------------------------------------
11
8function getWebTorrentPublicFileUrl (fileUrl: string) { 12function getWebTorrentPublicFileUrl (fileUrl: string) {
9 const baseUrl = CONFIG.OBJECT_STORAGE.VIDEOS.BASE_URL 13 const baseUrl = CONFIG.OBJECT_STORAGE.VIDEOS.BASE_URL
10 if (!baseUrl) return fileUrl 14 if (!baseUrl) return fileUrl
@@ -19,11 +23,28 @@ function getHLSPublicFileUrl (fileUrl: string) {
19 return replaceByBaseUrl(fileUrl, baseUrl) 23 return replaceByBaseUrl(fileUrl, baseUrl)
20} 24}
21 25
26// ---------------------------------------------------------------------------
27
28function getHLSPrivateFileUrl (video: MVideoUUID, filename: string) {
29 return WEBSERVER.URL + OBJECT_STORAGE_PROXY_PATHS.STREAMING_PLAYLISTS.PRIVATE_HLS + video.uuid + `/${filename}`
30}
31
32function getWebTorrentPrivateFileUrl (filename: string) {
33 return WEBSERVER.URL + OBJECT_STORAGE_PROXY_PATHS.PRIVATE_WEBSEED + filename
34}
35
36// ---------------------------------------------------------------------------
37
22export { 38export {
23 getPrivateUrl, 39 getInternalUrl,
40
24 getWebTorrentPublicFileUrl, 41 getWebTorrentPublicFileUrl,
25 replaceByBaseUrl, 42 getHLSPublicFileUrl,
26 getHLSPublicFileUrl 43
44 getHLSPrivateFileUrl,
45 getWebTorrentPrivateFileUrl,
46
47 replaceByBaseUrl
27} 48}
28 49
29// --------------------------------------------------------------------------- 50// ---------------------------------------------------------------------------
diff --git a/server/lib/object-storage/videos.ts b/server/lib/object-storage/videos.ts
index e323baaa2..003807826 100644
--- a/server/lib/object-storage/videos.ts
+++ b/server/lib/object-storage/videos.ts
@@ -5,7 +5,17 @@ import { MStreamingPlaylistVideo, MVideo, MVideoFile } from '@server/types/model
5import { getHLSDirectory } from '../paths' 5import { getHLSDirectory } from '../paths'
6import { VideoPathManager } from '../video-path-manager' 6import { VideoPathManager } from '../video-path-manager'
7import { generateHLSObjectBaseStorageKey, generateHLSObjectStorageKey, generateWebTorrentObjectStorageKey } from './keys' 7import { generateHLSObjectBaseStorageKey, generateHLSObjectStorageKey, generateWebTorrentObjectStorageKey } from './keys'
8import { listKeysOfPrefix, lTags, makeAvailable, removeObject, removePrefix, storeObject } from './shared' 8import {
9 createObjectReadStream,
10 listKeysOfPrefix,
11 lTags,
12 makeAvailable,
13 removeObject,
14 removePrefix,
15 storeObject,
16 updateObjectACL,
17 updatePrefixACL
18} from './shared'
9 19
10function listHLSFileKeysOf (playlist: MStreamingPlaylistVideo) { 20function listHLSFileKeysOf (playlist: MStreamingPlaylistVideo) {
11 return listKeysOfPrefix(generateHLSObjectBaseStorageKey(playlist), CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS) 21 return listKeysOfPrefix(generateHLSObjectBaseStorageKey(playlist), CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS)
@@ -17,7 +27,8 @@ function storeHLSFileFromFilename (playlist: MStreamingPlaylistVideo, filename:
17 return storeObject({ 27 return storeObject({
18 inputPath: join(getHLSDirectory(playlist.Video), filename), 28 inputPath: join(getHLSDirectory(playlist.Video), filename),
19 objectStorageKey: generateHLSObjectStorageKey(playlist, filename), 29 objectStorageKey: generateHLSObjectStorageKey(playlist, filename),
20 bucketInfo: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS 30 bucketInfo: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS,
31 isPrivate: playlist.Video.hasPrivateStaticPath()
21 }) 32 })
22} 33}
23 34
@@ -25,7 +36,8 @@ function storeHLSFileFromPath (playlist: MStreamingPlaylistVideo, path: string)
25 return storeObject({ 36 return storeObject({
26 inputPath: path, 37 inputPath: path,
27 objectStorageKey: generateHLSObjectStorageKey(playlist, basename(path)), 38 objectStorageKey: generateHLSObjectStorageKey(playlist, basename(path)),
28 bucketInfo: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS 39 bucketInfo: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS,
40 isPrivate: playlist.Video.hasPrivateStaticPath()
29 }) 41 })
30} 42}
31 43
@@ -35,7 +47,26 @@ function storeWebTorrentFile (video: MVideo, file: MVideoFile) {
35 return storeObject({ 47 return storeObject({
36 inputPath: VideoPathManager.Instance.getFSVideoFileOutputPath(video, file), 48 inputPath: VideoPathManager.Instance.getFSVideoFileOutputPath(video, file),
37 objectStorageKey: generateWebTorrentObjectStorageKey(file.filename), 49 objectStorageKey: generateWebTorrentObjectStorageKey(file.filename),
38 bucketInfo: CONFIG.OBJECT_STORAGE.VIDEOS 50 bucketInfo: CONFIG.OBJECT_STORAGE.VIDEOS,
51 isPrivate: video.hasPrivateStaticPath()
52 })
53}
54
55// ---------------------------------------------------------------------------
56
57function updateWebTorrentFileACL (video: MVideo, file: MVideoFile) {
58 return updateObjectACL({
59 objectStorageKey: generateWebTorrentObjectStorageKey(file.filename),
60 bucketInfo: CONFIG.OBJECT_STORAGE.VIDEOS,
61 isPrivate: video.hasPrivateStaticPath()
62 })
63}
64
65function updateHLSFilesACL (playlist: MStreamingPlaylistVideo) {
66 return updatePrefixACL({
67 prefix: generateHLSObjectBaseStorageKey(playlist),
68 bucketInfo: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS,
69 isPrivate: playlist.Video.hasPrivateStaticPath()
39 }) 70 })
40} 71}
41 72
@@ -87,6 +118,39 @@ async function makeWebTorrentFileAvailable (filename: string, destination: strin
87 118
88// --------------------------------------------------------------------------- 119// ---------------------------------------------------------------------------
89 120
121function getWebTorrentFileReadStream (options: {
122 filename: string
123 rangeHeader: string
124}) {
125 const { filename, rangeHeader } = options
126
127 const key = generateWebTorrentObjectStorageKey(filename)
128
129 return createObjectReadStream({
130 key,
131 bucketInfo: CONFIG.OBJECT_STORAGE.VIDEOS,
132 rangeHeader
133 })
134}
135
136function getHLSFileReadStream (options: {
137 playlist: MStreamingPlaylistVideo
138 filename: string
139 rangeHeader: string
140}) {
141 const { playlist, filename, rangeHeader } = options
142
143 const key = generateHLSObjectStorageKey(playlist, filename)
144
145 return createObjectReadStream({
146 key,
147 bucketInfo: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS,
148 rangeHeader
149 })
150}
151
152// ---------------------------------------------------------------------------
153
90export { 154export {
91 listHLSFileKeysOf, 155 listHLSFileKeysOf,
92 156
@@ -94,10 +158,16 @@ export {
94 storeHLSFileFromFilename, 158 storeHLSFileFromFilename,
95 storeHLSFileFromPath, 159 storeHLSFileFromPath,
96 160
161 updateWebTorrentFileACL,
162 updateHLSFilesACL,
163
97 removeHLSObjectStorage, 164 removeHLSObjectStorage,
98 removeHLSFileObjectStorage, 165 removeHLSFileObjectStorage,
99 removeWebTorrentObjectStorage, 166 removeWebTorrentObjectStorage,
100 167
101 makeWebTorrentFileAvailable, 168 makeWebTorrentFileAvailable,
102 makeHLSFileAvailable 169 makeHLSFileAvailable,
170
171 getWebTorrentFileReadStream,
172 getHLSFileReadStream
103} 173}
diff --git a/server/lib/video-privacy.ts b/server/lib/video-privacy.ts
index 1a4a5a22d..41f9d62b3 100644
--- a/server/lib/video-privacy.ts
+++ b/server/lib/video-privacy.ts
@@ -2,8 +2,9 @@ import { move } from 'fs-extra'
2import { join } from 'path' 2import { join } from 'path'
3import { logger } from '@server/helpers/logger' 3import { logger } from '@server/helpers/logger'
4import { DIRECTORIES } from '@server/initializers/constants' 4import { DIRECTORIES } from '@server/initializers/constants'
5import { MVideo, MVideoFullLight } from '@server/types/models' 5import { MVideo, MVideoFile, MVideoFullLight } from '@server/types/models'
6import { VideoPrivacy } from '@shared/models' 6import { VideoPrivacy, VideoStorage } from '@shared/models'
7import { updateHLSFilesACL, updateWebTorrentFileACL } from './object-storage'
7 8
8function setVideoPrivacy (video: MVideo, newPrivacy: VideoPrivacy) { 9function setVideoPrivacy (video: MVideo, newPrivacy: VideoPrivacy) {
9 if (video.privacy === VideoPrivacy.PRIVATE && newPrivacy !== VideoPrivacy.PRIVATE) { 10 if (video.privacy === VideoPrivacy.PRIVATE && newPrivacy !== VideoPrivacy.PRIVATE) {
@@ -50,47 +51,77 @@ export {
50 51
51// --------------------------------------------------------------------------- 52// ---------------------------------------------------------------------------
52 53
54type MoveType = 'private-to-public' | 'public-to-private'
55
53async function moveFiles (options: { 56async function moveFiles (options: {
54 type: 'private-to-public' | 'public-to-private' 57 type: MoveType
55 video: MVideoFullLight 58 video: MVideoFullLight
56}) { 59}) {
57 const { type, video } = options 60 const { type, video } = options
58 61
59 const directories = type === 'private-to-public' 62 for (const file of video.VideoFiles) {
60 ? { 63 if (file.storage === VideoStorage.FILE_SYSTEM) {
61 webtorrent: { old: DIRECTORIES.VIDEOS.PRIVATE, new: DIRECTORIES.VIDEOS.PUBLIC }, 64 await moveWebTorrentFileOnFS(type, video, file)
62 hls: { old: DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE, new: DIRECTORIES.HLS_STREAMING_PLAYLIST.PUBLIC } 65 } else {
66 await updateWebTorrentFileACL(video, file)
63 } 67 }
64 : { 68 }
65 webtorrent: { old: DIRECTORIES.VIDEOS.PUBLIC, new: DIRECTORIES.VIDEOS.PRIVATE }, 69
66 hls: { old: DIRECTORIES.HLS_STREAMING_PLAYLIST.PUBLIC, new: DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE } 70 const hls = video.getHLSPlaylist()
71
72 if (hls) {
73 if (hls.storage === VideoStorage.FILE_SYSTEM) {
74 await moveHLSFilesOnFS(type, video)
75 } else {
76 await updateHLSFilesACL(hls)
67 } 77 }
78 }
79}
68 80
69 for (const file of video.VideoFiles) { 81async function moveWebTorrentFileOnFS (type: MoveType, video: MVideo, file: MVideoFile) {
70 const source = join(directories.webtorrent.old, file.filename) 82 const directories = getWebTorrentDirectories(type)
71 const destination = join(directories.webtorrent.new, file.filename)
72 83
73 try { 84 const source = join(directories.old, file.filename)
74 logger.info('Moving WebTorrent files of %s after privacy change (%s -> %s).', video.uuid, source, destination) 85 const destination = join(directories.new, file.filename)
75 86
76 await move(source, destination) 87 try {
77 } catch (err) { 88 logger.info('Moving WebTorrent files of %s after privacy change (%s -> %s).', video.uuid, source, destination)
78 logger.error('Cannot move webtorrent file %s to %s after privacy change', source, destination, { err }) 89
79 } 90 await move(source, destination)
91 } catch (err) {
92 logger.error('Cannot move webtorrent file %s to %s after privacy change', source, destination, { err })
93 }
94}
95
96function getWebTorrentDirectories (moveType: MoveType) {
97 if (moveType === 'private-to-public') {
98 return { old: DIRECTORIES.VIDEOS.PRIVATE, new: DIRECTORIES.VIDEOS.PUBLIC }
80 } 99 }
81 100
82 const hls = video.getHLSPlaylist() 101 return { old: DIRECTORIES.VIDEOS.PUBLIC, new: DIRECTORIES.VIDEOS.PRIVATE }
102}
83 103
84 if (hls) { 104// ---------------------------------------------------------------------------
85 const source = join(directories.hls.old, video.uuid)
86 const destination = join(directories.hls.new, video.uuid)
87 105
88 try { 106async function moveHLSFilesOnFS (type: MoveType, video: MVideo) {
89 logger.info('Moving HLS files of %s after privacy change (%s -> %s).', video.uuid, source, destination) 107 const directories = getHLSDirectories(type)
90 108
91 await move(source, destination) 109 const source = join(directories.old, video.uuid)
92 } catch (err) { 110 const destination = join(directories.new, video.uuid)
93 logger.error('Cannot move HLS file %s to %s after privacy change', source, destination, { err }) 111
94 } 112 try {
113 logger.info('Moving HLS files of %s after privacy change (%s -> %s).', video.uuid, source, destination)
114
115 await move(source, destination)
116 } catch (err) {
117 logger.error('Cannot move HLS file %s to %s after privacy change', source, destination, { err })
118 }
119}
120
121function getHLSDirectories (moveType: MoveType) {
122 if (moveType === 'private-to-public') {
123 return { old: DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE, new: DIRECTORIES.HLS_STREAMING_PLAYLIST.PUBLIC }
95 } 124 }
125
126 return { old: DIRECTORIES.HLS_STREAMING_PLAYLIST.PUBLIC, new: DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE }
96} 127}
diff --git a/server/middlewares/validators/shared/videos.ts b/server/middlewares/validators/shared/videos.ts
index c29751eca..ebbfc0a0a 100644
--- a/server/middlewares/validators/shared/videos.ts
+++ b/server/middlewares/validators/shared/videos.ts
@@ -111,7 +111,7 @@ async function checkCanSeeVideo (options: {
111}) { 111}) {
112 const { req, res, video, paramId } = options 112 const { req, res, video, paramId } = options
113 113
114 if (video.requiresAuth(paramId)) { 114 if (video.requiresAuth({ urlParamId: paramId, checkBlacklist: true })) {
115 return checkCanSeeAuthVideo(req, res, video) 115 return checkCanSeeAuthVideo(req, res, video)
116 } 116 }
117 117
@@ -174,13 +174,13 @@ async function checkCanAccessVideoStaticFiles (options: {
174 res: Response 174 res: Response
175 paramId: string 175 paramId: string
176}) { 176}) {
177 const { video, req, res, paramId } = options 177 const { video, req, res } = options
178 178
179 if (res.locals.oauth?.token.User) { 179 if (res.locals.oauth?.token.User) {
180 return checkCanSeeVideo(options) 180 return checkCanSeeVideo(options)
181 } 181 }
182 182
183 if (!video.requiresAuth(paramId)) return true 183 if (!video.hasPrivateStaticPath()) return true
184 184
185 const videoFileToken = req.query.videoFileToken 185 const videoFileToken = req.query.videoFileToken
186 if (!videoFileToken) { 186 if (!videoFileToken) {
diff --git a/server/middlewares/validators/static.ts b/server/middlewares/validators/static.ts
index ff9e6ae6e..13fde6dd1 100644
--- a/server/middlewares/validators/static.ts
+++ b/server/middlewares/validators/static.ts
@@ -7,10 +7,17 @@ import { logger } from '@server/helpers/logger'
7import { LRU_CACHE } from '@server/initializers/constants' 7import { LRU_CACHE } from '@server/initializers/constants'
8import { VideoModel } from '@server/models/video/video' 8import { VideoModel } from '@server/models/video/video'
9import { VideoFileModel } from '@server/models/video/video-file' 9import { VideoFileModel } from '@server/models/video/video-file'
10import { MStreamingPlaylist, MVideoFile, MVideoThumbnail } from '@server/types/models'
10import { HttpStatusCode } from '@shared/models' 11import { HttpStatusCode } from '@shared/models'
11import { areValidationErrors, checkCanAccessVideoStaticFiles } from './shared' 12import { areValidationErrors, checkCanAccessVideoStaticFiles } from './shared'
12 13
13const staticFileTokenBypass = new LRUCache<string, boolean>({ 14type LRUValue = {
15 allowed: boolean
16 video?: MVideoThumbnail
17 file?: MVideoFile
18 playlist?: MStreamingPlaylist }
19
20const staticFileTokenBypass = new LRUCache<string, LRUValue>({
14 max: LRU_CACHE.STATIC_VIDEO_FILES_RIGHTS_CHECK.MAX_SIZE, 21 max: LRU_CACHE.STATIC_VIDEO_FILES_RIGHTS_CHECK.MAX_SIZE,
15 ttl: LRU_CACHE.STATIC_VIDEO_FILES_RIGHTS_CHECK.TTL 22 ttl: LRU_CACHE.STATIC_VIDEO_FILES_RIGHTS_CHECK.TTL
16}) 23})
@@ -27,18 +34,26 @@ const ensureCanAccessVideoPrivateWebTorrentFiles = [
27 const cacheKey = token + '-' + req.originalUrl 34 const cacheKey = token + '-' + req.originalUrl
28 35
29 if (staticFileTokenBypass.has(cacheKey)) { 36 if (staticFileTokenBypass.has(cacheKey)) {
30 const allowedFromCache = staticFileTokenBypass.get(cacheKey) 37 const { allowed, file, video } = staticFileTokenBypass.get(cacheKey)
38
39 if (allowed === true) {
40 res.locals.onlyVideo = video
41 res.locals.videoFile = file
31 42
32 if (allowedFromCache === true) return next() 43 return next()
44 }
33 45
34 return res.sendStatus(HttpStatusCode.FORBIDDEN_403) 46 return res.sendStatus(HttpStatusCode.FORBIDDEN_403)
35 } 47 }
36 48
37 const allowed = await isWebTorrentAllowed(req, res) 49 const result = await isWebTorrentAllowed(req, res)
50
51 staticFileTokenBypass.set(cacheKey, result)
38 52
39 staticFileTokenBypass.set(cacheKey, allowed) 53 if (result.allowed !== true) return
40 54
41 if (allowed !== true) return 55 res.locals.onlyVideo = result.video
56 res.locals.videoFile = result.file
42 57
43 return next() 58 return next()
44 } 59 }
@@ -64,18 +79,28 @@ const ensureCanAccessPrivateVideoHLSFiles = [
64 const cacheKey = token + '-' + videoUUID 79 const cacheKey = token + '-' + videoUUID
65 80
66 if (staticFileTokenBypass.has(cacheKey)) { 81 if (staticFileTokenBypass.has(cacheKey)) {
67 const allowedFromCache = staticFileTokenBypass.get(cacheKey) 82 const { allowed, file, playlist, video } = staticFileTokenBypass.get(cacheKey)
68 83
69 if (allowedFromCache === true) return next() 84 if (allowed === true) {
85 res.locals.onlyVideo = video
86 res.locals.videoFile = file
87 res.locals.videoStreamingPlaylist = playlist
88
89 return next()
90 }
70 91
71 return res.sendStatus(HttpStatusCode.FORBIDDEN_403) 92 return res.sendStatus(HttpStatusCode.FORBIDDEN_403)
72 } 93 }
73 94
74 const allowed = await isHLSAllowed(req, res, videoUUID) 95 const result = await isHLSAllowed(req, res, videoUUID)
96
97 staticFileTokenBypass.set(cacheKey, result)
75 98
76 staticFileTokenBypass.set(cacheKey, allowed) 99 if (result.allowed !== true) return
77 100
78 if (allowed !== true) return 101 res.locals.onlyVideo = result.video
102 res.locals.videoFile = result.file
103 res.locals.videoStreamingPlaylist = result.playlist
79 104
80 return next() 105 return next()
81 } 106 }
@@ -96,25 +121,38 @@ async function isWebTorrentAllowed (req: express.Request, res: express.Response)
96 logger.debug('Unknown static file %s to serve', req.originalUrl, { filename }) 121 logger.debug('Unknown static file %s to serve', req.originalUrl, { filename })
97 122
98 res.sendStatus(HttpStatusCode.FORBIDDEN_403) 123 res.sendStatus(HttpStatusCode.FORBIDDEN_403)
99 return false 124 return { allowed: false }
100 } 125 }
101 126
102 const video = file.getVideo() 127 const video = await VideoModel.load(file.getVideo().id)
103 128
104 return checkCanAccessVideoStaticFiles({ req, res, video, paramId: video.uuid }) 129 return {
130 file,
131 video,
132 allowed: await checkCanAccessVideoStaticFiles({ req, res, video, paramId: video.uuid })
133 }
105} 134}
106 135
107async function isHLSAllowed (req: express.Request, res: express.Response, videoUUID: string) { 136async function isHLSAllowed (req: express.Request, res: express.Response, videoUUID: string) {
108 const video = await VideoModel.load(videoUUID) 137 const filename = basename(req.path)
138
139 const video = await VideoModel.loadWithFiles(videoUUID)
109 140
110 if (!video) { 141 if (!video) {
111 logger.debug('Unknown static file %s to serve', req.originalUrl, { videoUUID }) 142 logger.debug('Unknown static file %s to serve', req.originalUrl, { videoUUID })
112 143
113 res.sendStatus(HttpStatusCode.FORBIDDEN_403) 144 res.sendStatus(HttpStatusCode.FORBIDDEN_403)
114 return false 145 return { allowed: false }
115 } 146 }
116 147
117 return checkCanAccessVideoStaticFiles({ req, res, video, paramId: video.uuid }) 148 const file = await VideoFileModel.loadByFilename(filename)
149
150 return {
151 file,
152 video,
153 playlist: video.getHLSPlaylist(),
154 allowed: await checkCanAccessVideoStaticFiles({ req, res, video, paramId: video.uuid })
155 }
118} 156}
119 157
120function extractTokenOrDie (req: express.Request, res: express.Response) { 158function extractTokenOrDie (req: express.Request, res: express.Response) {
diff --git a/server/models/video/video-file.ts b/server/models/video/video-file.ts
index 1a608932f..c20c90c1b 100644
--- a/server/models/video/video-file.ts
+++ b/server/models/video/video-file.ts
@@ -22,7 +22,12 @@ import validator from 'validator'
22import { logger } from '@server/helpers/logger' 22import { logger } from '@server/helpers/logger'
23import { extractVideo } from '@server/helpers/video' 23import { extractVideo } from '@server/helpers/video'
24import { buildRemoteVideoBaseUrl } from '@server/lib/activitypub/url' 24import { buildRemoteVideoBaseUrl } from '@server/lib/activitypub/url'
25import { getHLSPublicFileUrl, getWebTorrentPublicFileUrl } from '@server/lib/object-storage' 25import {
26 getHLSPrivateFileUrl,
27 getHLSPublicFileUrl,
28 getWebTorrentPrivateFileUrl,
29 getWebTorrentPublicFileUrl
30} from '@server/lib/object-storage'
26import { getFSTorrentFilePath } from '@server/lib/paths' 31import { getFSTorrentFilePath } from '@server/lib/paths'
27import { isVideoInPrivateDirectory } from '@server/lib/video-privacy' 32import { isVideoInPrivateDirectory } from '@server/lib/video-privacy'
28import { isStreamingPlaylist, MStreamingPlaylistVideo, MVideo, MVideoWithHost } from '@server/types/models' 33import { isStreamingPlaylist, MStreamingPlaylistVideo, MVideo, MVideoWithHost } from '@server/types/models'
@@ -503,7 +508,25 @@ export class VideoFileModel extends Model<Partial<AttributesOnly<VideoFileModel>
503 return !!this.videoStreamingPlaylistId 508 return !!this.videoStreamingPlaylistId
504 } 509 }
505 510
506 getObjectStorageUrl () { 511 // ---------------------------------------------------------------------------
512
513 getObjectStorageUrl (video: MVideo) {
514 if (video.hasPrivateStaticPath()) {
515 return this.getPrivateObjectStorageUrl(video)
516 }
517
518 return this.getPublicObjectStorageUrl()
519 }
520
521 private getPrivateObjectStorageUrl (video: MVideo) {
522 if (this.isHLS()) {
523 return getHLSPrivateFileUrl(video, this.filename)
524 }
525
526 return getWebTorrentPrivateFileUrl(this.filename)
527 }
528
529 private getPublicObjectStorageUrl () {
507 if (this.isHLS()) { 530 if (this.isHLS()) {
508 return getHLSPublicFileUrl(this.fileUrl) 531 return getHLSPublicFileUrl(this.fileUrl)
509 } 532 }
@@ -511,26 +534,29 @@ export class VideoFileModel extends Model<Partial<AttributesOnly<VideoFileModel>
511 return getWebTorrentPublicFileUrl(this.fileUrl) 534 return getWebTorrentPublicFileUrl(this.fileUrl)
512 } 535 }
513 536
537 // ---------------------------------------------------------------------------
538
514 getFileUrl (video: MVideo) { 539 getFileUrl (video: MVideo) {
515 if (this.storage === VideoStorage.OBJECT_STORAGE) { 540 if (video.isOwned()) {
516 return this.getObjectStorageUrl() 541 if (this.storage === VideoStorage.OBJECT_STORAGE) {
517 } 542 return this.getObjectStorageUrl(video)
543 }
518 544
519 if (!this.Video) this.Video = video as VideoModel 545 return WEBSERVER.URL + this.getFileStaticPath(video)
520 if (video.isOwned()) return WEBSERVER.URL + this.getFileStaticPath(video) 546 }
521 547
522 return this.fileUrl 548 return this.fileUrl
523 } 549 }
524 550
551 // ---------------------------------------------------------------------------
552
525 getFileStaticPath (video: MVideo) { 553 getFileStaticPath (video: MVideo) {
526 if (this.isHLS()) { 554 if (this.isHLS()) return this.getHLSFileStaticPath(video)
527 if (isVideoInPrivateDirectory(video.privacy)) {
528 return join(STATIC_PATHS.STREAMING_PLAYLISTS.PRIVATE_HLS, video.uuid, this.filename)
529 }
530 555
531 return join(STATIC_PATHS.STREAMING_PLAYLISTS.HLS, video.uuid, this.filename) 556 return this.getWebTorrentFileStaticPath(video)
532 } 557 }
533 558
559 private getWebTorrentFileStaticPath (video: MVideo) {
534 if (isVideoInPrivateDirectory(video.privacy)) { 560 if (isVideoInPrivateDirectory(video.privacy)) {
535 return join(STATIC_PATHS.PRIVATE_WEBSEED, this.filename) 561 return join(STATIC_PATHS.PRIVATE_WEBSEED, this.filename)
536 } 562 }
@@ -538,6 +564,16 @@ export class VideoFileModel extends Model<Partial<AttributesOnly<VideoFileModel>
538 return join(STATIC_PATHS.WEBSEED, this.filename) 564 return join(STATIC_PATHS.WEBSEED, this.filename)
539 } 565 }
540 566
567 private getHLSFileStaticPath (video: MVideo) {
568 if (isVideoInPrivateDirectory(video.privacy)) {
569 return join(STATIC_PATHS.STREAMING_PLAYLISTS.PRIVATE_HLS, video.uuid, this.filename)
570 }
571
572 return join(STATIC_PATHS.STREAMING_PLAYLISTS.HLS, video.uuid, this.filename)
573 }
574
575 // ---------------------------------------------------------------------------
576
541 getFileDownloadUrl (video: MVideoWithHost) { 577 getFileDownloadUrl (video: MVideoWithHost) {
542 const path = this.isHLS() 578 const path = this.isHLS()
543 ? join(STATIC_DOWNLOAD_PATHS.HLS_VIDEOS, `${video.uuid}-${this.resolution}-fragmented${this.extname}`) 579 ? join(STATIC_DOWNLOAD_PATHS.HLS_VIDEOS, `${video.uuid}-${this.resolution}-fragmented${this.extname}`)
diff --git a/server/models/video/video-streaming-playlist.ts b/server/models/video/video-streaming-playlist.ts
index b919046ed..1318a4dae 100644
--- a/server/models/video/video-streaming-playlist.ts
+++ b/server/models/video/video-streaming-playlist.ts
@@ -15,7 +15,7 @@ import {
15 Table, 15 Table,
16 UpdatedAt 16 UpdatedAt
17} from 'sequelize-typescript' 17} from 'sequelize-typescript'
18import { getHLSPublicFileUrl } from '@server/lib/object-storage' 18import { getHLSPrivateFileUrl, getHLSPublicFileUrl } from '@server/lib/object-storage'
19import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename } from '@server/lib/paths' 19import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename } from '@server/lib/paths'
20import { isVideoInPrivateDirectory } from '@server/lib/video-privacy' 20import { isVideoInPrivateDirectory } from '@server/lib/video-privacy'
21import { VideoFileModel } from '@server/models/video/video-file' 21import { VideoFileModel } from '@server/models/video/video-file'
@@ -245,10 +245,12 @@ export class VideoStreamingPlaylistModel extends Model<Partial<AttributesOnly<Vi
245 this.p2pMediaLoaderInfohashes = VideoStreamingPlaylistModel.buildP2PMediaLoaderInfoHashes(masterPlaylistUrl, files) 245 this.p2pMediaLoaderInfohashes = VideoStreamingPlaylistModel.buildP2PMediaLoaderInfoHashes(masterPlaylistUrl, files)
246 } 246 }
247 247
248 // ---------------------------------------------------------------------------
249
248 getMasterPlaylistUrl (video: MVideo) { 250 getMasterPlaylistUrl (video: MVideo) {
249 if (video.isOwned()) { 251 if (video.isOwned()) {
250 if (this.storage === VideoStorage.OBJECT_STORAGE) { 252 if (this.storage === VideoStorage.OBJECT_STORAGE) {
251 return getHLSPublicFileUrl(this.playlistUrl) 253 return this.getMasterPlaylistObjectStorageUrl(video)
252 } 254 }
253 255
254 return WEBSERVER.URL + this.getMasterPlaylistStaticPath(video) 256 return WEBSERVER.URL + this.getMasterPlaylistStaticPath(video)
@@ -257,10 +259,20 @@ export class VideoStreamingPlaylistModel extends Model<Partial<AttributesOnly<Vi
257 return this.playlistUrl 259 return this.playlistUrl
258 } 260 }
259 261
262 private getMasterPlaylistObjectStorageUrl (video: MVideo) {
263 if (video.hasPrivateStaticPath()) {
264 return getHLSPrivateFileUrl(video, this.playlistFilename)
265 }
266
267 return getHLSPublicFileUrl(this.playlistUrl)
268 }
269
270 // ---------------------------------------------------------------------------
271
260 getSha256SegmentsUrl (video: MVideo) { 272 getSha256SegmentsUrl (video: MVideo) {
261 if (video.isOwned()) { 273 if (video.isOwned()) {
262 if (this.storage === VideoStorage.OBJECT_STORAGE) { 274 if (this.storage === VideoStorage.OBJECT_STORAGE) {
263 return getHLSPublicFileUrl(this.segmentsSha256Url) 275 return this.getSha256SegmentsObjectStorageUrl(video)
264 } 276 }
265 277
266 return WEBSERVER.URL + this.getSha256SegmentsStaticPath(video) 278 return WEBSERVER.URL + this.getSha256SegmentsStaticPath(video)
@@ -269,6 +281,16 @@ export class VideoStreamingPlaylistModel extends Model<Partial<AttributesOnly<Vi
269 return this.segmentsSha256Url 281 return this.segmentsSha256Url
270 } 282 }
271 283
284 private getSha256SegmentsObjectStorageUrl (video: MVideo) {
285 if (video.hasPrivateStaticPath()) {
286 return getHLSPrivateFileUrl(video, this.segmentsSha256Filename)
287 }
288
289 return getHLSPublicFileUrl(this.segmentsSha256Url)
290 }
291
292 // ---------------------------------------------------------------------------
293
272 getStringType () { 294 getStringType () {
273 if (this.type === VideoStreamingPlaylistType.HLS) return 'hls' 295 if (this.type === VideoStreamingPlaylistType.HLS) return 'hls'
274 296
diff --git a/server/models/video/video.ts b/server/models/video/video.ts
index 82362917e..c568075d8 100644
--- a/server/models/video/video.ts
+++ b/server/models/video/video.ts
@@ -30,6 +30,7 @@ import { removeHLSFileObjectStorage, removeHLSObjectStorage, removeWebTorrentObj
30import { tracer } from '@server/lib/opentelemetry/tracing' 30import { tracer } from '@server/lib/opentelemetry/tracing'
31import { getHLSDirectory, getHLSRedundancyDirectory, getHlsResolutionPlaylistFilename } from '@server/lib/paths' 31import { getHLSDirectory, getHLSRedundancyDirectory, getHlsResolutionPlaylistFilename } from '@server/lib/paths'
32import { VideoPathManager } from '@server/lib/video-path-manager' 32import { VideoPathManager } from '@server/lib/video-path-manager'
33import { isVideoInPrivateDirectory } from '@server/lib/video-privacy'
33import { getServerActor } from '@server/models/application/application' 34import { getServerActor } from '@server/models/application/application'
34import { ModelCache } from '@server/models/model-cache' 35import { ModelCache } from '@server/models/model-cache'
35import { buildVideoEmbedPath, buildVideoWatchPath, pick } from '@shared/core-utils' 36import { buildVideoEmbedPath, buildVideoWatchPath, pick } from '@shared/core-utils'
@@ -1764,9 +1765,7 @@ export class VideoModel extends Model<Partial<AttributesOnly<VideoModel>>> {
1764 const playlist = this.VideoStreamingPlaylists.find(p => p.type === VideoStreamingPlaylistType.HLS) 1765 const playlist = this.VideoStreamingPlaylists.find(p => p.type === VideoStreamingPlaylistType.HLS)
1765 if (!playlist) return undefined 1766 if (!playlist) return undefined
1766 1767
1767 playlist.Video = this 1768 return playlist.withVideo(this)
1768
1769 return playlist
1770 } 1769 }
1771 1770
1772 setHLSPlaylist (playlist: MStreamingPlaylist) { 1771 setHLSPlaylist (playlist: MStreamingPlaylist) {
@@ -1868,16 +1867,39 @@ export class VideoModel extends Model<Partial<AttributesOnly<VideoModel>>> {
1868 return setAsUpdated('video', this.id, transaction) 1867 return setAsUpdated('video', this.id, transaction)
1869 } 1868 }
1870 1869
1871 requiresAuth (paramId: string) { 1870 // ---------------------------------------------------------------------------
1871
1872 requiresAuth (options: {
1873 urlParamId: string
1874 checkBlacklist: boolean
1875 }) {
1876 const { urlParamId, checkBlacklist } = options
1877
1878 if (this.privacy === VideoPrivacy.PRIVATE || this.privacy === VideoPrivacy.INTERNAL) {
1879 return true
1880 }
1881
1872 if (this.privacy === VideoPrivacy.UNLISTED) { 1882 if (this.privacy === VideoPrivacy.UNLISTED) {
1873 if (!isUUIDValid(paramId)) return true 1883 if (urlParamId && !isUUIDValid(urlParamId)) return true
1874 1884
1875 return false 1885 return false
1876 } 1886 }
1877 1887
1878 return this.privacy === VideoPrivacy.PRIVATE || this.privacy === VideoPrivacy.INTERNAL || !!this.VideoBlacklist 1888 if (checkBlacklist && this.VideoBlacklist) return true
1889
1890 if (this.privacy !== VideoPrivacy.PUBLIC) {
1891 throw new Error(`Unknown video privacy ${this.privacy} to know if the video requires auth`)
1892 }
1893
1894 return false
1879 } 1895 }
1880 1896
1897 hasPrivateStaticPath () {
1898 return isVideoInPrivateDirectory(this.privacy)
1899 }
1900
1901 // ---------------------------------------------------------------------------
1902
1881 async setNewState (newState: VideoState, isNewVideo: boolean, transaction: Transaction) { 1903 async setNewState (newState: VideoState, isNewVideo: boolean, transaction: Transaction) {
1882 if (this.state === newState) throw new Error('Cannot use same state ' + newState) 1904 if (this.state === newState) throw new Error('Cannot use same state ' + newState)
1883 1905
diff --git a/server/tests/api/object-storage/index.ts b/server/tests/api/object-storage/index.ts
index f319d6ef5..1f4489fa3 100644
--- a/server/tests/api/object-storage/index.ts
+++ b/server/tests/api/object-storage/index.ts
@@ -1,3 +1,4 @@
1export * from './live' 1export * from './live'
2export * from './video-imports' 2export * from './video-imports'
3export * from './video-static-file-privacy'
3export * from './videos' 4export * from './videos'
diff --git a/server/tests/api/object-storage/live.ts b/server/tests/api/object-storage/live.ts
index 77f3a8066..ad2b554b7 100644
--- a/server/tests/api/object-storage/live.ts
+++ b/server/tests/api/object-storage/live.ts
@@ -2,7 +2,7 @@
2 2
3import { expect } from 'chai' 3import { expect } from 'chai'
4import { expectStartWith, testVideoResolutions } from '@server/tests/shared' 4import { expectStartWith, testVideoResolutions } from '@server/tests/shared'
5import { areObjectStorageTestsDisabled } from '@shared/core-utils' 5import { areMockObjectStorageTestsDisabled } from '@shared/core-utils'
6import { HttpStatusCode, LiveVideoCreate, VideoPrivacy } from '@shared/models' 6import { HttpStatusCode, LiveVideoCreate, VideoPrivacy } from '@shared/models'
7import { 7import {
8 createMultipleServers, 8 createMultipleServers,
@@ -46,7 +46,7 @@ async function checkFilesExist (servers: PeerTubeServer[], videoUUID: string, nu
46 expect(files).to.have.lengthOf(numberOfFiles) 46 expect(files).to.have.lengthOf(numberOfFiles)
47 47
48 for (const file of files) { 48 for (const file of files) {
49 expectStartWith(file.fileUrl, ObjectStorageCommand.getPlaylistBaseUrl()) 49 expectStartWith(file.fileUrl, ObjectStorageCommand.getMockPlaylistBaseUrl())
50 50
51 await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 }) 51 await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 })
52 } 52 }
@@ -75,16 +75,16 @@ async function checkFilesCleanup (server: PeerTubeServer, videoUUID: string, res
75} 75}
76 76
77describe('Object storage for lives', function () { 77describe('Object storage for lives', function () {
78 if (areObjectStorageTestsDisabled()) return 78 if (areMockObjectStorageTestsDisabled()) return
79 79
80 let servers: PeerTubeServer[] 80 let servers: PeerTubeServer[]
81 81
82 before(async function () { 82 before(async function () {
83 this.timeout(120000) 83 this.timeout(120000)
84 84
85 await ObjectStorageCommand.prepareDefaultBuckets() 85 await ObjectStorageCommand.prepareDefaultMockBuckets()
86 86
87 servers = await createMultipleServers(2, ObjectStorageCommand.getDefaultConfig()) 87 servers = await createMultipleServers(2, ObjectStorageCommand.getDefaultMockConfig())
88 88
89 await setAccessTokensToServers(servers) 89 await setAccessTokensToServers(servers)
90 await setDefaultVideoChannel(servers) 90 await setDefaultVideoChannel(servers)
diff --git a/server/tests/api/object-storage/video-imports.ts b/server/tests/api/object-storage/video-imports.ts
index 90988ea9a..11c866411 100644
--- a/server/tests/api/object-storage/video-imports.ts
+++ b/server/tests/api/object-storage/video-imports.ts
@@ -2,7 +2,7 @@
2 2
3import { expect } from 'chai' 3import { expect } from 'chai'
4import { expectStartWith, FIXTURE_URLS } from '@server/tests/shared' 4import { expectStartWith, FIXTURE_URLS } from '@server/tests/shared'
5import { areObjectStorageTestsDisabled } from '@shared/core-utils' 5import { areMockObjectStorageTestsDisabled } from '@shared/core-utils'
6import { HttpStatusCode, VideoPrivacy } from '@shared/models' 6import { HttpStatusCode, VideoPrivacy } from '@shared/models'
7import { 7import {
8 createSingleServer, 8 createSingleServer,
@@ -29,16 +29,16 @@ async function importVideo (server: PeerTubeServer) {
29} 29}
30 30
31describe('Object storage for video import', function () { 31describe('Object storage for video import', function () {
32 if (areObjectStorageTestsDisabled()) return 32 if (areMockObjectStorageTestsDisabled()) return
33 33
34 let server: PeerTubeServer 34 let server: PeerTubeServer
35 35
36 before(async function () { 36 before(async function () {
37 this.timeout(120000) 37 this.timeout(120000)
38 38
39 await ObjectStorageCommand.prepareDefaultBuckets() 39 await ObjectStorageCommand.prepareDefaultMockBuckets()
40 40
41 server = await createSingleServer(1, ObjectStorageCommand.getDefaultConfig()) 41 server = await createSingleServer(1, ObjectStorageCommand.getDefaultMockConfig())
42 42
43 await setAccessTokensToServers([ server ]) 43 await setAccessTokensToServers([ server ])
44 await setDefaultVideoChannel([ server ]) 44 await setDefaultVideoChannel([ server ])
@@ -64,7 +64,7 @@ describe('Object storage for video import', function () {
64 expect(video.streamingPlaylists).to.have.lengthOf(0) 64 expect(video.streamingPlaylists).to.have.lengthOf(0)
65 65
66 const fileUrl = video.files[0].fileUrl 66 const fileUrl = video.files[0].fileUrl
67 expectStartWith(fileUrl, ObjectStorageCommand.getWebTorrentBaseUrl()) 67 expectStartWith(fileUrl, ObjectStorageCommand.getMockWebTorrentBaseUrl())
68 68
69 await makeRawRequest({ url: fileUrl, expectedStatus: HttpStatusCode.OK_200 }) 69 await makeRawRequest({ url: fileUrl, expectedStatus: HttpStatusCode.OK_200 })
70 }) 70 })
@@ -89,13 +89,13 @@ describe('Object storage for video import', function () {
89 expect(video.streamingPlaylists[0].files).to.have.lengthOf(5) 89 expect(video.streamingPlaylists[0].files).to.have.lengthOf(5)
90 90
91 for (const file of video.files) { 91 for (const file of video.files) {
92 expectStartWith(file.fileUrl, ObjectStorageCommand.getWebTorrentBaseUrl()) 92 expectStartWith(file.fileUrl, ObjectStorageCommand.getMockWebTorrentBaseUrl())
93 93
94 await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 }) 94 await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 })
95 } 95 }
96 96
97 for (const file of video.streamingPlaylists[0].files) { 97 for (const file of video.streamingPlaylists[0].files) {
98 expectStartWith(file.fileUrl, ObjectStorageCommand.getPlaylistBaseUrl()) 98 expectStartWith(file.fileUrl, ObjectStorageCommand.getMockPlaylistBaseUrl())
99 99
100 await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 }) 100 await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 })
101 } 101 }
diff --git a/server/tests/api/object-storage/video-static-file-privacy.ts b/server/tests/api/object-storage/video-static-file-privacy.ts
new file mode 100644
index 000000000..981bbaa16
--- /dev/null
+++ b/server/tests/api/object-storage/video-static-file-privacy.ts
@@ -0,0 +1,336 @@
1/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
2
3import { expect } from 'chai'
4import { basename } from 'path'
5import { expectStartWith } from '@server/tests/shared'
6import { areScalewayObjectStorageTestsDisabled, getAllFiles, getHLS } from '@shared/core-utils'
7import { HttpStatusCode, LiveVideo, VideoDetails, VideoPrivacy } from '@shared/models'
8import {
9 cleanupTests,
10 createSingleServer,
11 findExternalSavedVideo,
12 makeRawRequest,
13 ObjectStorageCommand,
14 PeerTubeServer,
15 sendRTMPStream,
16 setAccessTokensToServers,
17 setDefaultVideoChannel,
18 stopFfmpeg,
19 waitJobs
20} from '@shared/server-commands'
21
22describe('Object storage for video static file privacy', function () {
23 // We need real world object storage to check ACL
24 if (areScalewayObjectStorageTestsDisabled()) return
25
26 let server: PeerTubeServer
27 let userToken: string
28
29 before(async function () {
30 this.timeout(120000)
31
32 server = await createSingleServer(1, ObjectStorageCommand.getDefaultScalewayConfig(1))
33 await setAccessTokensToServers([ server ])
34 await setDefaultVideoChannel([ server ])
35
36 await server.config.enableMinimumTranscoding()
37
38 userToken = await server.users.generateUserAndToken('user1')
39 })
40
41 describe('VOD', function () {
42 let privateVideoUUID: string
43 let publicVideoUUID: string
44 let userPrivateVideoUUID: string
45
46 async function checkPrivateFiles (uuid: string) {
47 const video = await server.videos.getWithToken({ id: uuid })
48
49 for (const file of video.files) {
50 expectStartWith(file.fileUrl, server.url + '/object-storage-proxy/webseed/private/')
51
52 await makeRawRequest({ url: file.fileUrl, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 })
53 }
54
55 for (const file of getAllFiles(video)) {
56 const internalFileUrl = await server.sql.getInternalFileUrl(file.id)
57 expectStartWith(internalFileUrl, ObjectStorageCommand.getScalewayBaseUrl())
58 await makeRawRequest({ url: internalFileUrl, token: server.accessToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
59 }
60
61 const hls = getHLS(video)
62
63 if (hls) {
64 for (const url of [ hls.playlistUrl, hls.segmentsSha256Url ]) {
65 expectStartWith(url, server.url + '/object-storage-proxy/streaming-playlists/hls/private/')
66 }
67
68 await makeRawRequest({ url: hls.playlistUrl, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 })
69 await makeRawRequest({ url: hls.segmentsSha256Url, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 })
70
71 for (const file of hls.files) {
72 expectStartWith(file.fileUrl, server.url + '/object-storage-proxy/streaming-playlists/hls/private/')
73
74 await makeRawRequest({ url: file.fileUrl, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 })
75 }
76 }
77 }
78
79 async function checkPublicFiles (uuid: string) {
80 const video = await server.videos.getWithToken({ id: uuid })
81
82 for (const file of getAllFiles(video)) {
83 expectStartWith(file.fileUrl, ObjectStorageCommand.getScalewayBaseUrl())
84
85 await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 })
86 }
87
88 const hls = getHLS(video)
89
90 if (hls) {
91 expectStartWith(hls.playlistUrl, ObjectStorageCommand.getScalewayBaseUrl())
92 expectStartWith(hls.segmentsSha256Url, ObjectStorageCommand.getScalewayBaseUrl())
93
94 await makeRawRequest({ url: hls.playlistUrl, expectedStatus: HttpStatusCode.OK_200 })
95 await makeRawRequest({ url: hls.segmentsSha256Url, expectedStatus: HttpStatusCode.OK_200 })
96 }
97 }
98
99 async function getSampleFileUrls (videoId: string) {
100 const video = await server.videos.getWithToken({ id: videoId })
101
102 return {
103 webTorrentFile: video.files[0].fileUrl,
104 hlsFile: getHLS(video).files[0].fileUrl
105 }
106 }
107
108 it('Should upload a private video and have appropriate object storage ACL', async function () {
109 this.timeout(60000)
110
111 {
112 const { uuid } = await server.videos.quickUpload({ name: 'video', privacy: VideoPrivacy.PRIVATE })
113 privateVideoUUID = uuid
114 }
115
116 {
117 const { uuid } = await server.videos.quickUpload({ name: 'user video', token: userToken, privacy: VideoPrivacy.PRIVATE })
118 userPrivateVideoUUID = uuid
119 }
120
121 await waitJobs([ server ])
122
123 await checkPrivateFiles(privateVideoUUID)
124 })
125
126 it('Should upload a public video and have appropriate object storage ACL', async function () {
127 this.timeout(60000)
128
129 const { uuid } = await server.videos.quickUpload({ name: 'video', privacy: VideoPrivacy.UNLISTED })
130 await waitJobs([ server ])
131
132 publicVideoUUID = uuid
133
134 await checkPublicFiles(publicVideoUUID)
135 })
136
137 it('Should not get files without appropriate OAuth token', async function () {
138 this.timeout(60000)
139
140 const { webTorrentFile, hlsFile } = await getSampleFileUrls(privateVideoUUID)
141
142 await makeRawRequest({ url: webTorrentFile, token: userToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
143 await makeRawRequest({ url: webTorrentFile, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 })
144
145 await makeRawRequest({ url: hlsFile, token: userToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
146 await makeRawRequest({ url: hlsFile, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 })
147 })
148
149 it('Should not get HLS file of another video', async function () {
150 this.timeout(60000)
151
152 const privateVideo = await server.videos.getWithToken({ id: privateVideoUUID })
153 const hlsFilename = basename(getHLS(privateVideo).files[0].fileUrl)
154
155 const badUrl = server.url + '/object-storage-proxy/streaming-playlists/hls/private/' + userPrivateVideoUUID + '/' + hlsFilename
156 const goodUrl = server.url + '/object-storage-proxy/streaming-playlists/hls/private/' + privateVideoUUID + '/' + hlsFilename
157
158 await makeRawRequest({ url: badUrl, token: server.accessToken, expectedStatus: HttpStatusCode.NOT_FOUND_404 })
159 await makeRawRequest({ url: goodUrl, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 })
160 })
161
162 it('Should correctly check OAuth or video file token', async function () {
163 this.timeout(60000)
164
165 const badVideoFileToken = await server.videoToken.getVideoFileToken({ token: userToken, videoId: userPrivateVideoUUID })
166 const goodVideoFileToken = await server.videoToken.getVideoFileToken({ videoId: privateVideoUUID })
167
168 const { webTorrentFile, hlsFile } = await getSampleFileUrls(privateVideoUUID)
169
170 for (const url of [ webTorrentFile, hlsFile ]) {
171 await makeRawRequest({ url, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
172 await makeRawRequest({ url, token: userToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
173 await makeRawRequest({ url, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 })
174
175 await makeRawRequest({ url, query: { videoFileToken: badVideoFileToken }, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
176 await makeRawRequest({ url, query: { videoFileToken: goodVideoFileToken }, expectedStatus: HttpStatusCode.OK_200 })
177 }
178 })
179
180 it('Should update public video to private', async function () {
181 this.timeout(60000)
182
183 await server.videos.update({ id: publicVideoUUID, attributes: { privacy: VideoPrivacy.INTERNAL } })
184
185 await checkPrivateFiles(publicVideoUUID)
186 })
187
188 it('Should update private video to public', async function () {
189 this.timeout(60000)
190
191 await server.videos.update({ id: publicVideoUUID, attributes: { privacy: VideoPrivacy.PUBLIC } })
192
193 await checkPublicFiles(publicVideoUUID)
194 })
195
196 after(async function () {
197 this.timeout(30000)
198
199 if (privateVideoUUID) await server.videos.remove({ id: privateVideoUUID })
200 if (publicVideoUUID) await server.videos.remove({ id: publicVideoUUID })
201 if (userPrivateVideoUUID) await server.videos.remove({ id: userPrivateVideoUUID })
202
203 await waitJobs([ server ])
204 })
205 })
206
207 describe('Live', function () {
208 let normalLiveId: string
209 let normalLive: LiveVideo
210
211 let permanentLiveId: string
212 let permanentLive: LiveVideo
213
214 let unrelatedFileToken: string
215
216 async function checkLiveFiles (live: LiveVideo, liveId: string) {
217 const ffmpegCommand = sendRTMPStream({ rtmpBaseUrl: live.rtmpUrl, streamKey: live.streamKey })
218 await server.live.waitUntilPublished({ videoId: liveId })
219
220 const video = await server.videos.getWithToken({ id: liveId })
221 const fileToken = await server.videoToken.getVideoFileToken({ videoId: video.uuid })
222
223 const hls = video.streamingPlaylists[0]
224
225 for (const url of [ hls.playlistUrl, hls.segmentsSha256Url ]) {
226 expectStartWith(url, server.url + '/object-storage-proxy/streaming-playlists/hls/private/')
227
228 await makeRawRequest({ url: hls.playlistUrl, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 })
229 await makeRawRequest({ url: hls.segmentsSha256Url, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 })
230
231 await makeRawRequest({ url, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 })
232 await makeRawRequest({ url, query: { videoFileToken: fileToken }, expectedStatus: HttpStatusCode.OK_200 })
233
234 await makeRawRequest({ url, token: userToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
235 await makeRawRequest({ url, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
236 await makeRawRequest({ url, query: { videoFileToken: unrelatedFileToken }, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
237 }
238
239 await stopFfmpeg(ffmpegCommand)
240 }
241
242 async function checkReplay (replay: VideoDetails) {
243 const fileToken = await server.videoToken.getVideoFileToken({ videoId: replay.uuid })
244
245 const hls = replay.streamingPlaylists[0]
246 expect(hls.files).to.not.have.lengthOf(0)
247
248 for (const file of hls.files) {
249 await makeRawRequest({ url: file.fileUrl, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 })
250 await makeRawRequest({ url: file.fileUrl, query: { videoFileToken: fileToken }, expectedStatus: HttpStatusCode.OK_200 })
251
252 await makeRawRequest({ url: file.fileUrl, token: userToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
253 await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
254 await makeRawRequest({
255 url: file.fileUrl,
256 query: { videoFileToken: unrelatedFileToken },
257 expectedStatus: HttpStatusCode.FORBIDDEN_403
258 })
259 }
260
261 for (const url of [ hls.playlistUrl, hls.segmentsSha256Url ]) {
262 expectStartWith(url, server.url + '/object-storage-proxy/streaming-playlists/hls/private/')
263
264 await makeRawRequest({ url, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 })
265 await makeRawRequest({ url, query: { videoFileToken: fileToken }, expectedStatus: HttpStatusCode.OK_200 })
266
267 await makeRawRequest({ url, token: userToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
268 await makeRawRequest({ url, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
269 await makeRawRequest({ url, query: { videoFileToken: unrelatedFileToken }, expectedStatus: HttpStatusCode.FORBIDDEN_403 })
270 }
271 }
272
273 before(async function () {
274 await server.config.enableMinimumTranscoding()
275
276 const { uuid } = await server.videos.quickUpload({ name: 'another video' })
277 unrelatedFileToken = await server.videoToken.getVideoFileToken({ videoId: uuid })
278
279 await server.config.enableLive({
280 allowReplay: true,
281 transcoding: true,
282 resolutions: 'min'
283 })
284
285 {
286 const { video, live } = await server.live.quickCreate({ saveReplay: true, permanentLive: false, privacy: VideoPrivacy.PRIVATE })
287 normalLiveId = video.uuid
288 normalLive = live
289 }
290
291 {
292 const { video, live } = await server.live.quickCreate({ saveReplay: true, permanentLive: true, privacy: VideoPrivacy.PRIVATE })
293 permanentLiveId = video.uuid
294 permanentLive = live
295 }
296 })
297
298 it('Should create a private normal live and have a private static path', async function () {
299 this.timeout(240000)
300
301 await checkLiveFiles(normalLive, normalLiveId)
302 })
303
304 it('Should create a private permanent live and have a private static path', async function () {
305 this.timeout(240000)
306
307 await checkLiveFiles(permanentLive, permanentLiveId)
308 })
309
310 it('Should have created a replay of the normal live with a private static path', async function () {
311 this.timeout(240000)
312
313 await server.live.waitUntilReplacedByReplay({ videoId: normalLiveId })
314
315 const replay = await server.videos.getWithToken({ id: normalLiveId })
316 await checkReplay(replay)
317 })
318
319 it('Should have created a replay of the permanent live with a private static path', async function () {
320 this.timeout(240000)
321
322 await server.live.waitUntilWaiting({ videoId: permanentLiveId })
323 await waitJobs([ server ])
324
325 const live = await server.videos.getWithToken({ id: permanentLiveId })
326 const replayFromList = await findExternalSavedVideo(server, live)
327 const replay = await server.videos.getWithToken({ id: replayFromList.id })
328
329 await checkReplay(replay)
330 })
331 })
332
333 after(async function () {
334 await cleanupTests([ server ])
335 })
336})
diff --git a/server/tests/api/object-storage/videos.ts b/server/tests/api/object-storage/videos.ts
index 63f5179c7..af3db9334 100644
--- a/server/tests/api/object-storage/videos.ts
+++ b/server/tests/api/object-storage/videos.ts
@@ -11,7 +11,7 @@ import {
11 generateHighBitrateVideo, 11 generateHighBitrateVideo,
12 MockObjectStorage 12 MockObjectStorage
13} from '@server/tests/shared' 13} from '@server/tests/shared'
14import { areObjectStorageTestsDisabled } from '@shared/core-utils' 14import { areMockObjectStorageTestsDisabled } from '@shared/core-utils'
15import { HttpStatusCode, VideoDetails } from '@shared/models' 15import { HttpStatusCode, VideoDetails } from '@shared/models'
16import { 16import {
17 cleanupTests, 17 cleanupTests,
@@ -52,7 +52,7 @@ async function checkFiles (options: {
52 for (const file of video.files) { 52 for (const file of video.files) {
53 const baseUrl = baseMockUrl 53 const baseUrl = baseMockUrl
54 ? `${baseMockUrl}/${webtorrentBucket}/` 54 ? `${baseMockUrl}/${webtorrentBucket}/`
55 : `http://${webtorrentBucket}.${ObjectStorageCommand.getEndpointHost()}/` 55 : `http://${webtorrentBucket}.${ObjectStorageCommand.getMockEndpointHost()}/`
56 56
57 const prefix = webtorrentPrefix || '' 57 const prefix = webtorrentPrefix || ''
58 const start = baseUrl + prefix 58 const start = baseUrl + prefix
@@ -73,7 +73,7 @@ async function checkFiles (options: {
73 73
74 const baseUrl = baseMockUrl 74 const baseUrl = baseMockUrl
75 ? `${baseMockUrl}/${playlistBucket}/` 75 ? `${baseMockUrl}/${playlistBucket}/`
76 : `http://${playlistBucket}.${ObjectStorageCommand.getEndpointHost()}/` 76 : `http://${playlistBucket}.${ObjectStorageCommand.getMockEndpointHost()}/`
77 77
78 const prefix = playlistPrefix || '' 78 const prefix = playlistPrefix || ''
79 const start = baseUrl + prefix 79 const start = baseUrl + prefix
@@ -141,16 +141,16 @@ function runTestSuite (options: {
141 const port = await mockObjectStorage.initialize() 141 const port = await mockObjectStorage.initialize()
142 baseMockUrl = options.useMockBaseUrl ? `http://localhost:${port}` : undefined 142 baseMockUrl = options.useMockBaseUrl ? `http://localhost:${port}` : undefined
143 143
144 await ObjectStorageCommand.createBucket(options.playlistBucket) 144 await ObjectStorageCommand.createMockBucket(options.playlistBucket)
145 await ObjectStorageCommand.createBucket(options.webtorrentBucket) 145 await ObjectStorageCommand.createMockBucket(options.webtorrentBucket)
146 146
147 const config = { 147 const config = {
148 object_storage: { 148 object_storage: {
149 enabled: true, 149 enabled: true,
150 endpoint: 'http://' + ObjectStorageCommand.getEndpointHost(), 150 endpoint: 'http://' + ObjectStorageCommand.getMockEndpointHost(),
151 region: ObjectStorageCommand.getRegion(), 151 region: ObjectStorageCommand.getMockRegion(),
152 152
153 credentials: ObjectStorageCommand.getCredentialsConfig(), 153 credentials: ObjectStorageCommand.getMockCredentialsConfig(),
154 154
155 max_upload_part: options.maxUploadPart || '5MB', 155 max_upload_part: options.maxUploadPart || '5MB',
156 156
@@ -261,7 +261,7 @@ function runTestSuite (options: {
261} 261}
262 262
263describe('Object storage for videos', function () { 263describe('Object storage for videos', function () {
264 if (areObjectStorageTestsDisabled()) return 264 if (areMockObjectStorageTestsDisabled()) return
265 265
266 describe('Test config', function () { 266 describe('Test config', function () {
267 let server: PeerTubeServer 267 let server: PeerTubeServer
@@ -269,17 +269,17 @@ describe('Object storage for videos', function () {
269 const baseConfig = { 269 const baseConfig = {
270 object_storage: { 270 object_storage: {
271 enabled: true, 271 enabled: true,
272 endpoint: 'http://' + ObjectStorageCommand.getEndpointHost(), 272 endpoint: 'http://' + ObjectStorageCommand.getMockEndpointHost(),
273 region: ObjectStorageCommand.getRegion(), 273 region: ObjectStorageCommand.getMockRegion(),
274 274
275 credentials: ObjectStorageCommand.getCredentialsConfig(), 275 credentials: ObjectStorageCommand.getMockCredentialsConfig(),
276 276
277 streaming_playlists: { 277 streaming_playlists: {
278 bucket_name: ObjectStorageCommand.DEFAULT_PLAYLIST_BUCKET 278 bucket_name: ObjectStorageCommand.DEFAULT_PLAYLIST_MOCK_BUCKET
279 }, 279 },
280 280
281 videos: { 281 videos: {
282 bucket_name: ObjectStorageCommand.DEFAULT_WEBTORRENT_BUCKET 282 bucket_name: ObjectStorageCommand.DEFAULT_WEBTORRENT_MOCK_BUCKET
283 } 283 }
284 } 284 }
285 } 285 }
@@ -310,7 +310,7 @@ describe('Object storage for videos', function () {
310 it('Should fail with bad credentials', async function () { 310 it('Should fail with bad credentials', async function () {
311 this.timeout(60000) 311 this.timeout(60000)
312 312
313 await ObjectStorageCommand.prepareDefaultBuckets() 313 await ObjectStorageCommand.prepareDefaultMockBuckets()
314 314
315 const config = merge({}, baseConfig, { 315 const config = merge({}, baseConfig, {
316 object_storage: { 316 object_storage: {
@@ -334,7 +334,7 @@ describe('Object storage for videos', function () {
334 it('Should succeed with credentials from env', async function () { 334 it('Should succeed with credentials from env', async function () {
335 this.timeout(60000) 335 this.timeout(60000)
336 336
337 await ObjectStorageCommand.prepareDefaultBuckets() 337 await ObjectStorageCommand.prepareDefaultMockBuckets()
338 338
339 const config = merge({}, baseConfig, { 339 const config = merge({}, baseConfig, {
340 object_storage: { 340 object_storage: {
@@ -345,7 +345,7 @@ describe('Object storage for videos', function () {
345 } 345 }
346 }) 346 })
347 347
348 const goodCredentials = ObjectStorageCommand.getCredentialsConfig() 348 const goodCredentials = ObjectStorageCommand.getMockCredentialsConfig()
349 349
350 server = await createSingleServer(1, config, { 350 server = await createSingleServer(1, config, {
351 env: { 351 env: {
@@ -361,7 +361,7 @@ describe('Object storage for videos', function () {
361 await waitJobs([ server ], true) 361 await waitJobs([ server ], true)
362 const video = await server.videos.get({ id: uuid }) 362 const video = await server.videos.get({ id: uuid })
363 363
364 expectStartWith(video.files[0].fileUrl, ObjectStorageCommand.getWebTorrentBaseUrl()) 364 expectStartWith(video.files[0].fileUrl, ObjectStorageCommand.getMockWebTorrentBaseUrl())
365 }) 365 })
366 366
367 after(async function () { 367 after(async function () {
diff --git a/server/tests/api/server/proxy.ts b/server/tests/api/server/proxy.ts
index a4151ebdd..3700b0746 100644
--- a/server/tests/api/server/proxy.ts
+++ b/server/tests/api/server/proxy.ts
@@ -2,7 +2,7 @@
2 2
3import { expect } from 'chai' 3import { expect } from 'chai'
4import { expectNotStartWith, expectStartWith, FIXTURE_URLS, MockProxy } from '@server/tests/shared' 4import { expectNotStartWith, expectStartWith, FIXTURE_URLS, MockProxy } from '@server/tests/shared'
5import { areObjectStorageTestsDisabled } from '@shared/core-utils' 5import { areMockObjectStorageTestsDisabled } from '@shared/core-utils'
6import { HttpStatusCode, VideoPrivacy } from '@shared/models' 6import { HttpStatusCode, VideoPrivacy } from '@shared/models'
7import { 7import {
8 cleanupTests, 8 cleanupTests,
@@ -120,40 +120,40 @@ describe('Test proxy', function () {
120 }) 120 })
121 121
122 describe('Object storage', function () { 122 describe('Object storage', function () {
123 if (areObjectStorageTestsDisabled()) return 123 if (areMockObjectStorageTestsDisabled()) return
124 124
125 before(async function () { 125 before(async function () {
126 this.timeout(30000) 126 this.timeout(30000)
127 127
128 await ObjectStorageCommand.prepareDefaultBuckets() 128 await ObjectStorageCommand.prepareDefaultMockBuckets()
129 }) 129 })
130 130
131 it('Should succeed to upload to object storage with the appropriate proxy config', async function () { 131 it('Should succeed to upload to object storage with the appropriate proxy config', async function () {
132 this.timeout(120000) 132 this.timeout(120000)
133 133
134 await servers[0].kill() 134 await servers[0].kill()
135 await servers[0].run(ObjectStorageCommand.getDefaultConfig(), { env: goodEnv }) 135 await servers[0].run(ObjectStorageCommand.getDefaultMockConfig(), { env: goodEnv })
136 136
137 const { uuid } = await servers[0].videos.quickUpload({ name: 'video' }) 137 const { uuid } = await servers[0].videos.quickUpload({ name: 'video' })
138 await waitJobs(servers) 138 await waitJobs(servers)
139 139
140 const video = await servers[0].videos.get({ id: uuid }) 140 const video = await servers[0].videos.get({ id: uuid })
141 141
142 expectStartWith(video.files[0].fileUrl, ObjectStorageCommand.getWebTorrentBaseUrl()) 142 expectStartWith(video.files[0].fileUrl, ObjectStorageCommand.getMockWebTorrentBaseUrl())
143 }) 143 })
144 144
145 it('Should fail to upload to object storage with a wrong proxy config', async function () { 145 it('Should fail to upload to object storage with a wrong proxy config', async function () {
146 this.timeout(120000) 146 this.timeout(120000)
147 147
148 await servers[0].kill() 148 await servers[0].kill()
149 await servers[0].run(ObjectStorageCommand.getDefaultConfig(), { env: badEnv }) 149 await servers[0].run(ObjectStorageCommand.getDefaultMockConfig(), { env: badEnv })
150 150
151 const { uuid } = await servers[0].videos.quickUpload({ name: 'video' }) 151 const { uuid } = await servers[0].videos.quickUpload({ name: 'video' })
152 await waitJobs(servers) 152 await waitJobs(servers)
153 153
154 const video = await servers[0].videos.get({ id: uuid }) 154 const video = await servers[0].videos.get({ id: uuid })
155 155
156 expectNotStartWith(video.files[0].fileUrl, ObjectStorageCommand.getWebTorrentBaseUrl()) 156 expectNotStartWith(video.files[0].fileUrl, ObjectStorageCommand.getMockWebTorrentBaseUrl())
157 }) 157 })
158 }) 158 })
159 159
diff --git a/server/tests/api/transcoding/create-transcoding.ts b/server/tests/api/transcoding/create-transcoding.ts
index 372f5332a..85389a949 100644
--- a/server/tests/api/transcoding/create-transcoding.ts
+++ b/server/tests/api/transcoding/create-transcoding.ts
@@ -2,7 +2,7 @@
2 2
3import { expect } from 'chai' 3import { expect } from 'chai'
4import { checkResolutionsInMasterPlaylist, expectStartWith } from '@server/tests/shared' 4import { checkResolutionsInMasterPlaylist, expectStartWith } from '@server/tests/shared'
5import { areObjectStorageTestsDisabled } from '@shared/core-utils' 5import { areMockObjectStorageTestsDisabled } from '@shared/core-utils'
6import { HttpStatusCode, VideoDetails } from '@shared/models' 6import { HttpStatusCode, VideoDetails } from '@shared/models'
7import { 7import {
8 cleanupTests, 8 cleanupTests,
@@ -19,7 +19,7 @@ import {
19 19
20async function checkFilesInObjectStorage (video: VideoDetails) { 20async function checkFilesInObjectStorage (video: VideoDetails) {
21 for (const file of video.files) { 21 for (const file of video.files) {
22 expectStartWith(file.fileUrl, ObjectStorageCommand.getWebTorrentBaseUrl()) 22 expectStartWith(file.fileUrl, ObjectStorageCommand.getMockWebTorrentBaseUrl())
23 await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 }) 23 await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 })
24 } 24 }
25 25
@@ -27,14 +27,14 @@ async function checkFilesInObjectStorage (video: VideoDetails) {
27 27
28 const hlsPlaylist = video.streamingPlaylists[0] 28 const hlsPlaylist = video.streamingPlaylists[0]
29 for (const file of hlsPlaylist.files) { 29 for (const file of hlsPlaylist.files) {
30 expectStartWith(file.fileUrl, ObjectStorageCommand.getPlaylistBaseUrl()) 30 expectStartWith(file.fileUrl, ObjectStorageCommand.getMockPlaylistBaseUrl())
31 await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 }) 31 await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 })
32 } 32 }
33 33
34 expectStartWith(hlsPlaylist.playlistUrl, ObjectStorageCommand.getPlaylistBaseUrl()) 34 expectStartWith(hlsPlaylist.playlistUrl, ObjectStorageCommand.getMockPlaylistBaseUrl())
35 await makeRawRequest({ url: hlsPlaylist.playlistUrl, expectedStatus: HttpStatusCode.OK_200 }) 35 await makeRawRequest({ url: hlsPlaylist.playlistUrl, expectedStatus: HttpStatusCode.OK_200 })
36 36
37 expectStartWith(hlsPlaylist.segmentsSha256Url, ObjectStorageCommand.getPlaylistBaseUrl()) 37 expectStartWith(hlsPlaylist.segmentsSha256Url, ObjectStorageCommand.getMockPlaylistBaseUrl())
38 await makeRawRequest({ url: hlsPlaylist.segmentsSha256Url, expectedStatus: HttpStatusCode.OK_200 }) 38 await makeRawRequest({ url: hlsPlaylist.segmentsSha256Url, expectedStatus: HttpStatusCode.OK_200 })
39} 39}
40 40
@@ -49,7 +49,7 @@ function runTests (objectStorage: boolean) {
49 this.timeout(120000) 49 this.timeout(120000)
50 50
51 const config = objectStorage 51 const config = objectStorage
52 ? ObjectStorageCommand.getDefaultConfig() 52 ? ObjectStorageCommand.getDefaultMockConfig()
53 : {} 53 : {}
54 54
55 // Run server 2 to have transcoding enabled 55 // Run server 2 to have transcoding enabled
@@ -60,7 +60,7 @@ function runTests (objectStorage: boolean) {
60 60
61 await doubleFollow(servers[0], servers[1]) 61 await doubleFollow(servers[0], servers[1])
62 62
63 if (objectStorage) await ObjectStorageCommand.prepareDefaultBuckets() 63 if (objectStorage) await ObjectStorageCommand.prepareDefaultMockBuckets()
64 64
65 const { shortUUID } = await servers[0].videos.quickUpload({ name: 'video' }) 65 const { shortUUID } = await servers[0].videos.quickUpload({ name: 'video' })
66 videoUUID = shortUUID 66 videoUUID = shortUUID
@@ -256,7 +256,7 @@ describe('Test create transcoding jobs from API', function () {
256 }) 256 })
257 257
258 describe('On object storage', function () { 258 describe('On object storage', function () {
259 if (areObjectStorageTestsDisabled()) return 259 if (areMockObjectStorageTestsDisabled()) return
260 260
261 runTests(true) 261 runTests(true)
262 }) 262 })
diff --git a/server/tests/api/transcoding/hls.ts b/server/tests/api/transcoding/hls.ts
index 7b5492cd4..84a53c0bd 100644
--- a/server/tests/api/transcoding/hls.ts
+++ b/server/tests/api/transcoding/hls.ts
@@ -2,7 +2,7 @@
2 2
3import { join } from 'path' 3import { join } from 'path'
4import { checkDirectoryIsEmpty, checkTmpIsEmpty, completeCheckHlsPlaylist } from '@server/tests/shared' 4import { checkDirectoryIsEmpty, checkTmpIsEmpty, completeCheckHlsPlaylist } from '@server/tests/shared'
5import { areObjectStorageTestsDisabled } from '@shared/core-utils' 5import { areMockObjectStorageTestsDisabled } from '@shared/core-utils'
6import { HttpStatusCode } from '@shared/models' 6import { HttpStatusCode } from '@shared/models'
7import { 7import {
8 cleanupTests, 8 cleanupTests,
@@ -150,19 +150,19 @@ describe('Test HLS videos', function () {
150 }) 150 })
151 151
152 describe('With object storage enabled', function () { 152 describe('With object storage enabled', function () {
153 if (areObjectStorageTestsDisabled()) return 153 if (areMockObjectStorageTestsDisabled()) return
154 154
155 before(async function () { 155 before(async function () {
156 this.timeout(120000) 156 this.timeout(120000)
157 157
158 const configOverride = ObjectStorageCommand.getDefaultConfig() 158 const configOverride = ObjectStorageCommand.getDefaultMockConfig()
159 await ObjectStorageCommand.prepareDefaultBuckets() 159 await ObjectStorageCommand.prepareDefaultMockBuckets()
160 160
161 await servers[0].kill() 161 await servers[0].kill()
162 await servers[0].run(configOverride) 162 await servers[0].run(configOverride)
163 }) 163 })
164 164
165 runTestSuite(true, ObjectStorageCommand.getPlaylistBaseUrl()) 165 runTestSuite(true, ObjectStorageCommand.getMockPlaylistBaseUrl())
166 }) 166 })
167 167
168 after(async function () { 168 after(async function () {
diff --git a/server/tests/api/transcoding/update-while-transcoding.ts b/server/tests/api/transcoding/update-while-transcoding.ts
index 5ca923392..8e32ea069 100644
--- a/server/tests/api/transcoding/update-while-transcoding.ts
+++ b/server/tests/api/transcoding/update-while-transcoding.ts
@@ -1,7 +1,7 @@
1/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ 1/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
2 2
3import { completeCheckHlsPlaylist } from '@server/tests/shared' 3import { completeCheckHlsPlaylist } from '@server/tests/shared'
4import { areObjectStorageTestsDisabled, wait } from '@shared/core-utils' 4import { areMockObjectStorageTestsDisabled, wait } from '@shared/core-utils'
5import { VideoPrivacy } from '@shared/models' 5import { VideoPrivacy } from '@shared/models'
6import { 6import {
7 cleanupTests, 7 cleanupTests,
@@ -130,19 +130,19 @@ describe('Test update video privacy while transcoding', function () {
130 }) 130 })
131 131
132 describe('With object storage enabled', function () { 132 describe('With object storage enabled', function () {
133 if (areObjectStorageTestsDisabled()) return 133 if (areMockObjectStorageTestsDisabled()) return
134 134
135 before(async function () { 135 before(async function () {
136 this.timeout(120000) 136 this.timeout(120000)
137 137
138 const configOverride = ObjectStorageCommand.getDefaultConfig() 138 const configOverride = ObjectStorageCommand.getDefaultMockConfig()
139 await ObjectStorageCommand.prepareDefaultBuckets() 139 await ObjectStorageCommand.prepareDefaultMockBuckets()
140 140
141 await servers[0].kill() 141 await servers[0].kill()
142 await servers[0].run(configOverride) 142 await servers[0].run(configOverride)
143 }) 143 })
144 144
145 runTestSuite(true, ObjectStorageCommand.getPlaylistBaseUrl()) 145 runTestSuite(true, ObjectStorageCommand.getMockPlaylistBaseUrl())
146 }) 146 })
147 147
148 after(async function () { 148 after(async function () {
diff --git a/server/tests/api/transcoding/video-studio.ts b/server/tests/api/transcoding/video-studio.ts
index 9613111b5..ab08e8fb6 100644
--- a/server/tests/api/transcoding/video-studio.ts
+++ b/server/tests/api/transcoding/video-studio.ts
@@ -1,6 +1,6 @@
1import { expect } from 'chai' 1import { expect } from 'chai'
2import { expectStartWith } from '@server/tests/shared' 2import { expectStartWith } from '@server/tests/shared'
3import { areObjectStorageTestsDisabled, getAllFiles } from '@shared/core-utils' 3import { areMockObjectStorageTestsDisabled, getAllFiles } from '@shared/core-utils'
4import { VideoStudioTask } from '@shared/models' 4import { VideoStudioTask } from '@shared/models'
5import { 5import {
6 cleanupTests, 6 cleanupTests,
@@ -315,13 +315,13 @@ describe('Test video studio', function () {
315 }) 315 })
316 316
317 describe('Object storage video edition', function () { 317 describe('Object storage video edition', function () {
318 if (areObjectStorageTestsDisabled()) return 318 if (areMockObjectStorageTestsDisabled()) return
319 319
320 before(async function () { 320 before(async function () {
321 await ObjectStorageCommand.prepareDefaultBuckets() 321 await ObjectStorageCommand.prepareDefaultMockBuckets()
322 322
323 await servers[0].kill() 323 await servers[0].kill()
324 await servers[0].run(ObjectStorageCommand.getDefaultConfig()) 324 await servers[0].run(ObjectStorageCommand.getDefaultMockConfig())
325 325
326 await servers[0].config.enableMinimumTranscoding() 326 await servers[0].config.enableMinimumTranscoding()
327 }) 327 })
@@ -344,11 +344,11 @@ describe('Test video studio', function () {
344 } 344 }
345 345
346 for (const webtorrentFile of video.files) { 346 for (const webtorrentFile of video.files) {
347 expectStartWith(webtorrentFile.fileUrl, ObjectStorageCommand.getWebTorrentBaseUrl()) 347 expectStartWith(webtorrentFile.fileUrl, ObjectStorageCommand.getMockWebTorrentBaseUrl())
348 } 348 }
349 349
350 for (const hlsFile of video.streamingPlaylists[0].files) { 350 for (const hlsFile of video.streamingPlaylists[0].files) {
351 expectStartWith(hlsFile.fileUrl, ObjectStorageCommand.getPlaylistBaseUrl()) 351 expectStartWith(hlsFile.fileUrl, ObjectStorageCommand.getMockPlaylistBaseUrl())
352 } 352 }
353 353
354 await checkDuration(server, 9) 354 await checkDuration(server, 9)
diff --git a/server/tests/api/videos/video-static-file-privacy.ts b/server/tests/api/videos/video-static-file-privacy.ts
index e38fdec6e..bdbe85127 100644
--- a/server/tests/api/videos/video-static-file-privacy.ts
+++ b/server/tests/api/videos/video-static-file-privacy.ts
@@ -37,7 +37,7 @@ describe('Test video static file privacy', function () {
37 37
38 function runSuite () { 38 function runSuite () {
39 39
40 async function checkPrivateWebTorrentFiles (uuid: string) { 40 async function checkPrivateFiles (uuid: string) {
41 const video = await server.videos.getWithToken({ id: uuid }) 41 const video = await server.videos.getWithToken({ id: uuid })
42 42
43 for (const file of video.files) { 43 for (const file of video.files) {
@@ -63,7 +63,7 @@ describe('Test video static file privacy', function () {
63 } 63 }
64 } 64 }
65 65
66 async function checkPublicWebTorrentFiles (uuid: string) { 66 async function checkPublicFiles (uuid: string) {
67 const video = await server.videos.get({ id: uuid }) 67 const video = await server.videos.get({ id: uuid })
68 68
69 for (const file of getAllFiles(video)) { 69 for (const file of getAllFiles(video)) {
@@ -98,7 +98,7 @@ describe('Test video static file privacy', function () {
98 const { uuid } = await server.videos.quickUpload({ name: 'video', privacy }) 98 const { uuid } = await server.videos.quickUpload({ name: 'video', privacy })
99 await waitJobs([ server ]) 99 await waitJobs([ server ])
100 100
101 await checkPrivateWebTorrentFiles(uuid) 101 await checkPrivateFiles(uuid)
102 } 102 }
103 }) 103 })
104 104
@@ -112,7 +112,7 @@ describe('Test video static file privacy', function () {
112 await server.videos.update({ id: uuid, attributes: { privacy } }) 112 await server.videos.update({ id: uuid, attributes: { privacy } })
113 await waitJobs([ server ]) 113 await waitJobs([ server ])
114 114
115 await checkPrivateWebTorrentFiles(uuid) 115 await checkPrivateFiles(uuid)
116 } 116 }
117 }) 117 })
118 118
@@ -125,7 +125,7 @@ describe('Test video static file privacy', function () {
125 await server.videos.update({ id: uuid, attributes: { privacy: VideoPrivacy.UNLISTED } }) 125 await server.videos.update({ id: uuid, attributes: { privacy: VideoPrivacy.UNLISTED } })
126 await waitJobs([ server ]) 126 await waitJobs([ server ])
127 127
128 await checkPublicWebTorrentFiles(uuid) 128 await checkPublicFiles(uuid)
129 }) 129 })
130 130
131 it('Should upload an internal video and update it to public to have a public static path', async function () { 131 it('Should upload an internal video and update it to public to have a public static path', async function () {
@@ -137,7 +137,7 @@ describe('Test video static file privacy', function () {
137 await server.videos.update({ id: uuid, attributes: { privacy: VideoPrivacy.PUBLIC } }) 137 await server.videos.update({ id: uuid, attributes: { privacy: VideoPrivacy.PUBLIC } })
138 await waitJobs([ server ]) 138 await waitJobs([ server ])
139 139
140 await checkPublicWebTorrentFiles(uuid) 140 await checkPublicFiles(uuid)
141 }) 141 })
142 142
143 it('Should upload an internal video and schedule a public publish', async function () { 143 it('Should upload an internal video and schedule a public publish', async function () {
@@ -160,7 +160,7 @@ describe('Test video static file privacy', function () {
160 160
161 await waitJobs([ server ]) 161 await waitJobs([ server ])
162 162
163 await checkPublicWebTorrentFiles(uuid) 163 await checkPublicFiles(uuid)
164 }) 164 })
165 } 165 }
166 166
diff --git a/server/tests/cli/create-import-video-file-job.ts b/server/tests/cli/create-import-video-file-job.ts
index a4aa5f699..43f53035b 100644
--- a/server/tests/cli/create-import-video-file-job.ts
+++ b/server/tests/cli/create-import-video-file-job.ts
@@ -1,7 +1,7 @@
1/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ 1/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
2 2
3import { expect } from 'chai' 3import { expect } from 'chai'
4import { areObjectStorageTestsDisabled } from '@shared/core-utils' 4import { areMockObjectStorageTestsDisabled } from '@shared/core-utils'
5import { HttpStatusCode, VideoDetails, VideoFile, VideoInclude } from '@shared/models' 5import { HttpStatusCode, VideoDetails, VideoFile, VideoInclude } from '@shared/models'
6import { 6import {
7 cleanupTests, 7 cleanupTests,
@@ -27,7 +27,7 @@ function assertVideoProperties (video: VideoFile, resolution: number, extname: s
27 27
28async function checkFiles (video: VideoDetails, objectStorage: boolean) { 28async function checkFiles (video: VideoDetails, objectStorage: boolean) {
29 for (const file of video.files) { 29 for (const file of video.files) {
30 if (objectStorage) expectStartWith(file.fileUrl, ObjectStorageCommand.getWebTorrentBaseUrl()) 30 if (objectStorage) expectStartWith(file.fileUrl, ObjectStorageCommand.getMockWebTorrentBaseUrl())
31 31
32 await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 }) 32 await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 })
33 } 33 }
@@ -43,7 +43,7 @@ function runTests (objectStorage: boolean) {
43 this.timeout(90000) 43 this.timeout(90000)
44 44
45 const config = objectStorage 45 const config = objectStorage
46 ? ObjectStorageCommand.getDefaultConfig() 46 ? ObjectStorageCommand.getDefaultMockConfig()
47 : {} 47 : {}
48 48
49 // Run server 2 to have transcoding enabled 49 // Run server 2 to have transcoding enabled
@@ -52,7 +52,7 @@ function runTests (objectStorage: boolean) {
52 52
53 await doubleFollow(servers[0], servers[1]) 53 await doubleFollow(servers[0], servers[1])
54 54
55 if (objectStorage) await ObjectStorageCommand.prepareDefaultBuckets() 55 if (objectStorage) await ObjectStorageCommand.prepareDefaultMockBuckets()
56 56
57 // Upload two videos for our needs 57 // Upload two videos for our needs
58 { 58 {
@@ -157,7 +157,7 @@ describe('Test create import video jobs', function () {
157 }) 157 })
158 158
159 describe('On object storage', function () { 159 describe('On object storage', function () {
160 if (areObjectStorageTestsDisabled()) return 160 if (areMockObjectStorageTestsDisabled()) return
161 161
162 runTests(true) 162 runTests(true)
163 }) 163 })
diff --git a/server/tests/cli/create-move-video-storage-job.ts b/server/tests/cli/create-move-video-storage-job.ts
index ecdd75b76..c357f501b 100644
--- a/server/tests/cli/create-move-video-storage-job.ts
+++ b/server/tests/cli/create-move-video-storage-job.ts
@@ -1,6 +1,6 @@
1/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ 1/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
2 2
3import { areObjectStorageTestsDisabled } from '@shared/core-utils' 3import { areMockObjectStorageTestsDisabled } from '@shared/core-utils'
4import { HttpStatusCode, VideoDetails } from '@shared/models' 4import { HttpStatusCode, VideoDetails } from '@shared/models'
5import { 5import {
6 cleanupTests, 6 cleanupTests,
@@ -17,7 +17,7 @@ import { expectStartWith } from '../shared'
17async function checkFiles (origin: PeerTubeServer, video: VideoDetails, inObjectStorage: boolean) { 17async function checkFiles (origin: PeerTubeServer, video: VideoDetails, inObjectStorage: boolean) {
18 for (const file of video.files) { 18 for (const file of video.files) {
19 const start = inObjectStorage 19 const start = inObjectStorage
20 ? ObjectStorageCommand.getWebTorrentBaseUrl() 20 ? ObjectStorageCommand.getMockWebTorrentBaseUrl()
21 : origin.url 21 : origin.url
22 22
23 expectStartWith(file.fileUrl, start) 23 expectStartWith(file.fileUrl, start)
@@ -26,7 +26,7 @@ async function checkFiles (origin: PeerTubeServer, video: VideoDetails, inObject
26 } 26 }
27 27
28 const start = inObjectStorage 28 const start = inObjectStorage
29 ? ObjectStorageCommand.getPlaylistBaseUrl() 29 ? ObjectStorageCommand.getMockPlaylistBaseUrl()
30 : origin.url 30 : origin.url
31 31
32 const hls = video.streamingPlaylists[0] 32 const hls = video.streamingPlaylists[0]
@@ -41,7 +41,7 @@ async function checkFiles (origin: PeerTubeServer, video: VideoDetails, inObject
41} 41}
42 42
43describe('Test create move video storage job', function () { 43describe('Test create move video storage job', function () {
44 if (areObjectStorageTestsDisabled()) return 44 if (areMockObjectStorageTestsDisabled()) return
45 45
46 let servers: PeerTubeServer[] = [] 46 let servers: PeerTubeServer[] = []
47 const uuids: string[] = [] 47 const uuids: string[] = []
@@ -55,7 +55,7 @@ describe('Test create move video storage job', function () {
55 55
56 await doubleFollow(servers[0], servers[1]) 56 await doubleFollow(servers[0], servers[1])
57 57
58 await ObjectStorageCommand.prepareDefaultBuckets() 58 await ObjectStorageCommand.prepareDefaultMockBuckets()
59 59
60 await servers[0].config.enableTranscoding() 60 await servers[0].config.enableTranscoding()
61 61
@@ -67,14 +67,14 @@ describe('Test create move video storage job', function () {
67 await waitJobs(servers) 67 await waitJobs(servers)
68 68
69 await servers[0].kill() 69 await servers[0].kill()
70 await servers[0].run(ObjectStorageCommand.getDefaultConfig()) 70 await servers[0].run(ObjectStorageCommand.getDefaultMockConfig())
71 }) 71 })
72 72
73 it('Should move only one file', async function () { 73 it('Should move only one file', async function () {
74 this.timeout(120000) 74 this.timeout(120000)
75 75
76 const command = `npm run create-move-video-storage-job -- --to-object-storage -v ${uuids[1]}` 76 const command = `npm run create-move-video-storage-job -- --to-object-storage -v ${uuids[1]}`
77 await servers[0].cli.execWithEnv(command, ObjectStorageCommand.getDefaultConfig()) 77 await servers[0].cli.execWithEnv(command, ObjectStorageCommand.getDefaultMockConfig())
78 await waitJobs(servers) 78 await waitJobs(servers)
79 79
80 for (const server of servers) { 80 for (const server of servers) {
@@ -94,7 +94,7 @@ describe('Test create move video storage job', function () {
94 this.timeout(120000) 94 this.timeout(120000)
95 95
96 const command = `npm run create-move-video-storage-job -- --to-object-storage --all-videos` 96 const command = `npm run create-move-video-storage-job -- --to-object-storage --all-videos`
97 await servers[0].cli.execWithEnv(command, ObjectStorageCommand.getDefaultConfig()) 97 await servers[0].cli.execWithEnv(command, ObjectStorageCommand.getDefaultMockConfig())
98 await waitJobs(servers) 98 await waitJobs(servers)
99 99
100 for (const server of servers) { 100 for (const server of servers) {
diff --git a/server/tests/cli/create-transcoding-job.ts b/server/tests/cli/create-transcoding-job.ts
index 51bf04a80..38b737829 100644
--- a/server/tests/cli/create-transcoding-job.ts
+++ b/server/tests/cli/create-transcoding-job.ts
@@ -1,7 +1,7 @@
1/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ 1/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
2 2
3import { expect } from 'chai' 3import { expect } from 'chai'
4import { areObjectStorageTestsDisabled } from '@shared/core-utils' 4import { areMockObjectStorageTestsDisabled } from '@shared/core-utils'
5import { HttpStatusCode, VideoFile } from '@shared/models' 5import { HttpStatusCode, VideoFile } from '@shared/models'
6import { 6import {
7 cleanupTests, 7 cleanupTests,
@@ -18,8 +18,8 @@ import { checkResolutionsInMasterPlaylist, expectStartWith } from '../shared'
18async function checkFilesInObjectStorage (files: VideoFile[], type: 'webtorrent' | 'playlist') { 18async function checkFilesInObjectStorage (files: VideoFile[], type: 'webtorrent' | 'playlist') {
19 for (const file of files) { 19 for (const file of files) {
20 const shouldStartWith = type === 'webtorrent' 20 const shouldStartWith = type === 'webtorrent'
21 ? ObjectStorageCommand.getWebTorrentBaseUrl() 21 ? ObjectStorageCommand.getMockWebTorrentBaseUrl()
22 : ObjectStorageCommand.getPlaylistBaseUrl() 22 : ObjectStorageCommand.getMockPlaylistBaseUrl()
23 23
24 expectStartWith(file.fileUrl, shouldStartWith) 24 expectStartWith(file.fileUrl, shouldStartWith)
25 25
@@ -36,7 +36,7 @@ function runTests (objectStorage: boolean) {
36 this.timeout(120000) 36 this.timeout(120000)
37 37
38 const config = objectStorage 38 const config = objectStorage
39 ? ObjectStorageCommand.getDefaultConfig() 39 ? ObjectStorageCommand.getDefaultMockConfig()
40 : {} 40 : {}
41 41
42 // Run server 2 to have transcoding enabled 42 // Run server 2 to have transcoding enabled
@@ -47,7 +47,7 @@ function runTests (objectStorage: boolean) {
47 47
48 await doubleFollow(servers[0], servers[1]) 48 await doubleFollow(servers[0], servers[1])
49 49
50 if (objectStorage) await ObjectStorageCommand.prepareDefaultBuckets() 50 if (objectStorage) await ObjectStorageCommand.prepareDefaultMockBuckets()
51 51
52 for (let i = 1; i <= 5; i++) { 52 for (let i = 1; i <= 5; i++) {
53 const { uuid, shortUUID } = await servers[0].videos.upload({ attributes: { name: 'video' + i } }) 53 const { uuid, shortUUID } = await servers[0].videos.upload({ attributes: { name: 'video' + i } })
@@ -255,7 +255,7 @@ describe('Test create transcoding jobs', function () {
255 }) 255 })
256 256
257 describe('On object storage', function () { 257 describe('On object storage', function () {
258 if (areObjectStorageTestsDisabled()) return 258 if (areMockObjectStorageTestsDisabled()) return
259 259
260 runTests(true) 260 runTests(true)
261 }) 261 })
diff --git a/server/tests/shared/live.ts b/server/tests/shared/live.ts
index da3691711..78e29f575 100644
--- a/server/tests/shared/live.ts
+++ b/server/tests/shared/live.ts
@@ -50,7 +50,7 @@ async function testVideoResolutions (options: {
50 }) 50 })
51 51
52 if (objectStorage) { 52 if (objectStorage) {
53 expect(hlsPlaylist.playlistUrl).to.contain(ObjectStorageCommand.getPlaylistBaseUrl()) 53 expect(hlsPlaylist.playlistUrl).to.contain(ObjectStorageCommand.getMockPlaylistBaseUrl())
54 } 54 }
55 55
56 for (let i = 0; i < resolutions.length; i++) { 56 for (let i = 0; i < resolutions.length; i++) {
@@ -65,11 +65,11 @@ async function testVideoResolutions (options: {
65 }) 65 })
66 66
67 const baseUrl = objectStorage 67 const baseUrl = objectStorage
68 ? ObjectStorageCommand.getPlaylistBaseUrl() + 'hls' 68 ? ObjectStorageCommand.getMockPlaylistBaseUrl() + 'hls'
69 : originServer.url + '/static/streaming-playlists/hls' 69 : originServer.url + '/static/streaming-playlists/hls'
70 70
71 if (objectStorage) { 71 if (objectStorage) {
72 expect(hlsPlaylist.segmentsSha256Url).to.contain(ObjectStorageCommand.getPlaylistBaseUrl()) 72 expect(hlsPlaylist.segmentsSha256Url).to.contain(ObjectStorageCommand.getMockPlaylistBaseUrl())
73 } 73 }
74 74
75 const subPlaylist = await originServer.streamingPlaylists.get({ 75 const subPlaylist = await originServer.streamingPlaylists.get({
diff --git a/server/tests/shared/mock-servers/mock-object-storage.ts b/server/tests/shared/mock-servers/mock-object-storage.ts
index 99d68e014..8c325bf11 100644
--- a/server/tests/shared/mock-servers/mock-object-storage.ts
+++ b/server/tests/shared/mock-servers/mock-object-storage.ts
@@ -12,7 +12,7 @@ export class MockObjectStorage {
12 const app = express() 12 const app = express()
13 13
14 app.get('/:bucketName/:path(*)', (req: express.Request, res: express.Response, next: express.NextFunction) => { 14 app.get('/:bucketName/:path(*)', (req: express.Request, res: express.Response, next: express.NextFunction) => {
15 const url = `http://${req.params.bucketName}.${ObjectStorageCommand.getEndpointHost()}/${req.params.path}` 15 const url = `http://${req.params.bucketName}.${ObjectStorageCommand.getMockEndpointHost()}/${req.params.path}`
16 16
17 if (process.env.DEBUG) { 17 if (process.env.DEBUG) {
18 console.log('Receiving request on mocked server %s.', req.url) 18 console.log('Receiving request on mocked server %s.', req.url)
diff --git a/server/types/express.d.ts b/server/types/express.d.ts
index 27d60da72..3738ffc47 100644
--- a/server/types/express.d.ts
+++ b/server/types/express.d.ts
@@ -97,7 +97,7 @@ declare module 'express' {
97 97
98 title?: string 98 title?: string
99 status?: number 99 status?: number
100 type?: ServerErrorCode 100 type?: ServerErrorCode | string
101 instance?: string 101 instance?: string
102 102
103 data?: PeerTubeProblemDocumentData 103 data?: PeerTubeProblemDocumentData