diff options
author | Chocobozzz <me@florianbigard.com> | 2022-10-26 16:23:39 +0200 |
---|---|---|
committer | Chocobozzz <me@florianbigard.com> | 2022-10-27 09:13:10 +0200 |
commit | 5a122dddc5aab1b2ae1843411032d5f392bdd216 (patch) | |
tree | 39778c4372c4b371f1294c94bf22d8a6e7a26456 | |
parent | 49e7e4d9ffd16cba7a721f6d3d3203decf4f4b2c (diff) | |
download | PeerTube-5a122dddc5aab1b2ae1843411032d5f392bdd216.tar.gz PeerTube-5a122dddc5aab1b2ae1843411032d5f392bdd216.tar.zst PeerTube-5a122dddc5aab1b2ae1843411032d5f392bdd216.zip |
Option to disable static files auth check/s3 proxy
-rw-r--r-- | config/default.yaml | 14 | ||||
-rw-r--r-- | server/controllers/object-storage-proxy.ts | 8 | ||||
-rw-r--r-- | server/controllers/static.ts | 20 | ||||
-rw-r--r-- | server/initializers/config.ts | 6 | ||||
-rw-r--r-- | server/middlewares/validators/index.ts | 1 | ||||
-rw-r--r-- | server/middlewares/validators/object-storage-proxy.ts | 20 | ||||
-rw-r--r-- | server/models/video/video-file.ts | 3 | ||||
-rw-r--r-- | server/models/video/video-streaming-playlist.ts | 5 | ||||
-rw-r--r-- | server/tests/api/object-storage/video-static-file-privacy.ts | 166 | ||||
-rw-r--r-- | server/tests/api/videos/video-static-file-privacy.ts | 33 | ||||
-rw-r--r-- | shared/server-commands/server/object-storage-command.ts | 16 |
11 files changed, 232 insertions, 60 deletions
diff --git a/config/default.yaml b/config/default.yaml index 7753821da..2e249cc76 100644 --- a/config/default.yaml +++ b/config/default.yaml | |||
@@ -140,6 +140,10 @@ storage: | |||
140 | # If not, peertube will fallback to the default file | 140 | # If not, peertube will fallback to the default file |
141 | client_overrides: 'storage/client-overrides/' | 141 | client_overrides: 'storage/client-overrides/' |
142 | 142 | ||
143 | static_files: | ||
144 | # Require and check user authentication when accessing private files (internal/private video files) | ||
145 | private_files_require_auth: true | ||
146 | |||
143 | object_storage: | 147 | object_storage: |
144 | enabled: false | 148 | enabled: false |
145 | 149 | ||
@@ -151,9 +155,17 @@ object_storage: | |||
151 | upload_acl: | 155 | upload_acl: |
152 | # Set this ACL on each uploaded object of public/unlisted videos | 156 | # Set this ACL on each uploaded object of public/unlisted videos |
153 | public: 'public-read' | 157 | public: 'public-read' |
154 | # Set this ACL on each uploaded object of private/internal videos | 158 | # Set this ACL on each uploaded object of private/internal videos |
159 | # PeerTube can proxify requests to private objects so your users can access them | ||
155 | private: 'private' | 160 | private: 'private' |
156 | 161 | ||
162 | proxy: | ||
163 | # If private files (private/internal video files) have a private ACL, users can't access directly the ressource | ||
164 | # PeerTube can proxify requests between your object storage service and your users | ||
165 | # If you disable PeerTube proxy, ensure you use your own proxy that is able to access the private files | ||
166 | # Or you can also set a public ACL for private files in object storage if you don't want to use a proxy | ||
167 | proxify_private_files: true | ||
168 | |||
157 | credentials: | 169 | credentials: |
158 | # You can also use AWS_ACCESS_KEY_ID env variable | 170 | # You can also use AWS_ACCESS_KEY_ID env variable |
159 | access_key_id: '' | 171 | access_key_id: '' |
diff --git a/server/controllers/object-storage-proxy.ts b/server/controllers/object-storage-proxy.ts index 6fedcfd8f..3ce279671 100644 --- a/server/controllers/object-storage-proxy.ts +++ b/server/controllers/object-storage-proxy.ts | |||
@@ -1,11 +1,13 @@ | |||
1 | import cors from 'cors' | 1 | import cors from 'cors' |
2 | import express from 'express' | 2 | import express from 'express' |
3 | import { logger } from '@server/helpers/logger' | ||
3 | import { OBJECT_STORAGE_PROXY_PATHS } from '@server/initializers/constants' | 4 | import { OBJECT_STORAGE_PROXY_PATHS } from '@server/initializers/constants' |
4 | import { getHLSFileReadStream, getWebTorrentFileReadStream } from '@server/lib/object-storage' | 5 | import { getHLSFileReadStream, getWebTorrentFileReadStream } from '@server/lib/object-storage' |
5 | import { | 6 | import { |
6 | asyncMiddleware, | 7 | asyncMiddleware, |
7 | ensureCanAccessPrivateVideoHLSFiles, | 8 | ensureCanAccessPrivateVideoHLSFiles, |
8 | ensureCanAccessVideoPrivateWebTorrentFiles, | 9 | ensureCanAccessVideoPrivateWebTorrentFiles, |
10 | ensurePrivateObjectStorageProxyIsEnabled, | ||
9 | optionalAuthenticate | 11 | optionalAuthenticate |
10 | } from '@server/middlewares' | 12 | } from '@server/middlewares' |
11 | import { HttpStatusCode } from '@shared/models' | 13 | import { HttpStatusCode } from '@shared/models' |
@@ -15,12 +17,14 @@ const objectStorageProxyRouter = express.Router() | |||
15 | objectStorageProxyRouter.use(cors()) | 17 | objectStorageProxyRouter.use(cors()) |
16 | 18 | ||
17 | objectStorageProxyRouter.get(OBJECT_STORAGE_PROXY_PATHS.PRIVATE_WEBSEED + ':filename', | 19 | objectStorageProxyRouter.get(OBJECT_STORAGE_PROXY_PATHS.PRIVATE_WEBSEED + ':filename', |
20 | ensurePrivateObjectStorageProxyIsEnabled, | ||
18 | optionalAuthenticate, | 21 | optionalAuthenticate, |
19 | asyncMiddleware(ensureCanAccessVideoPrivateWebTorrentFiles), | 22 | asyncMiddleware(ensureCanAccessVideoPrivateWebTorrentFiles), |
20 | asyncMiddleware(proxifyWebTorrent) | 23 | asyncMiddleware(proxifyWebTorrent) |
21 | ) | 24 | ) |
22 | 25 | ||
23 | objectStorageProxyRouter.get(OBJECT_STORAGE_PROXY_PATHS.STREAMING_PLAYLISTS.PRIVATE_HLS + ':videoUUID/:filename', | 26 | objectStorageProxyRouter.get(OBJECT_STORAGE_PROXY_PATHS.STREAMING_PLAYLISTS.PRIVATE_HLS + ':videoUUID/:filename', |
27 | ensurePrivateObjectStorageProxyIsEnabled, | ||
24 | optionalAuthenticate, | 28 | optionalAuthenticate, |
25 | asyncMiddleware(ensureCanAccessPrivateVideoHLSFiles), | 29 | asyncMiddleware(ensureCanAccessPrivateVideoHLSFiles), |
26 | asyncMiddleware(proxifyHLS) | 30 | asyncMiddleware(proxifyHLS) |
@@ -35,6 +39,8 @@ export { | |||
35 | async function proxifyWebTorrent (req: express.Request, res: express.Response) { | 39 | async function proxifyWebTorrent (req: express.Request, res: express.Response) { |
36 | const filename = req.params.filename | 40 | const filename = req.params.filename |
37 | 41 | ||
42 | logger.debug('Proxifying WebTorrent file %s from object storage.', filename) | ||
43 | |||
38 | try { | 44 | try { |
39 | const stream = await getWebTorrentFileReadStream({ | 45 | const stream = await getWebTorrentFileReadStream({ |
40 | filename, | 46 | filename, |
@@ -52,6 +58,8 @@ async function proxifyHLS (req: express.Request, res: express.Response) { | |||
52 | const video = res.locals.onlyVideo | 58 | const video = res.locals.onlyVideo |
53 | const filename = req.params.filename | 59 | const filename = req.params.filename |
54 | 60 | ||
61 | logger.debug('Proxifying HLS file %s from object storage.', filename) | ||
62 | |||
55 | try { | 63 | try { |
56 | const stream = await getHLSFileReadStream({ | 64 | const stream = await getHLSFileReadStream({ |
57 | playlist: playlist.withVideo(video), | 65 | playlist: playlist.withVideo(video), |
diff --git a/server/controllers/static.ts b/server/controllers/static.ts index dc091455a..6ef9154b9 100644 --- a/server/controllers/static.ts +++ b/server/controllers/static.ts | |||
@@ -15,11 +15,17 @@ const staticRouter = express.Router() | |||
15 | // Cors is very important to let other servers access torrent and video files | 15 | // Cors is very important to let other servers access torrent and video files |
16 | staticRouter.use(cors()) | 16 | staticRouter.use(cors()) |
17 | 17 | ||
18 | // --------------------------------------------------------------------------- | ||
18 | // WebTorrent/Classic videos | 19 | // WebTorrent/Classic videos |
20 | // --------------------------------------------------------------------------- | ||
21 | |||
22 | const privateWebTorrentStaticMiddlewares = CONFIG.STATIC_FILES.PRIVATE_FILES_REQUIRE_AUTH === true | ||
23 | ? [ optionalAuthenticate, asyncMiddleware(ensureCanAccessVideoPrivateWebTorrentFiles) ] | ||
24 | : [] | ||
25 | |||
19 | staticRouter.use( | 26 | staticRouter.use( |
20 | STATIC_PATHS.PRIVATE_WEBSEED, | 27 | STATIC_PATHS.PRIVATE_WEBSEED, |
21 | optionalAuthenticate, | 28 | ...privateWebTorrentStaticMiddlewares, |
22 | asyncMiddleware(ensureCanAccessVideoPrivateWebTorrentFiles), | ||
23 | express.static(DIRECTORIES.VIDEOS.PRIVATE, { fallthrough: false }), | 29 | express.static(DIRECTORIES.VIDEOS.PRIVATE, { fallthrough: false }), |
24 | handleStaticError | 30 | handleStaticError |
25 | ) | 31 | ) |
@@ -35,11 +41,17 @@ staticRouter.use( | |||
35 | handleStaticError | 41 | handleStaticError |
36 | ) | 42 | ) |
37 | 43 | ||
44 | // --------------------------------------------------------------------------- | ||
38 | // HLS | 45 | // HLS |
46 | // --------------------------------------------------------------------------- | ||
47 | |||
48 | const privateHLSStaticMiddlewares = CONFIG.STATIC_FILES.PRIVATE_FILES_REQUIRE_AUTH === true | ||
49 | ? [ optionalAuthenticate, asyncMiddleware(ensureCanAccessPrivateVideoHLSFiles) ] | ||
50 | : [] | ||
51 | |||
39 | staticRouter.use( | 52 | staticRouter.use( |
40 | STATIC_PATHS.STREAMING_PLAYLISTS.PRIVATE_HLS, | 53 | STATIC_PATHS.STREAMING_PLAYLISTS.PRIVATE_HLS, |
41 | optionalAuthenticate, | 54 | ...privateHLSStaticMiddlewares, |
42 | asyncMiddleware(ensureCanAccessPrivateVideoHLSFiles), | ||
43 | express.static(DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE, { fallthrough: false }), | 55 | express.static(DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE, { fallthrough: false }), |
44 | handleStaticError | 56 | handleStaticError |
45 | ) | 57 | ) |
diff --git a/server/initializers/config.ts b/server/initializers/config.ts index ab5e645ad..7652da24a 100644 --- a/server/initializers/config.ts +++ b/server/initializers/config.ts | |||
@@ -113,6 +113,9 @@ const CONFIG = { | |||
113 | CLIENT_OVERRIDES_DIR: buildPath(config.get<string>('storage.client_overrides')), | 113 | CLIENT_OVERRIDES_DIR: buildPath(config.get<string>('storage.client_overrides')), |
114 | WELL_KNOWN_DIR: buildPath(config.get<string>('storage.well_known')) | 114 | WELL_KNOWN_DIR: buildPath(config.get<string>('storage.well_known')) |
115 | }, | 115 | }, |
116 | STATIC_FILES: { | ||
117 | PRIVATE_FILES_REQUIRE_AUTH: config.get<boolean>('static_files.private_files_require_auth') | ||
118 | }, | ||
116 | OBJECT_STORAGE: { | 119 | OBJECT_STORAGE: { |
117 | ENABLED: config.get<boolean>('object_storage.enabled'), | 120 | ENABLED: config.get<boolean>('object_storage.enabled'), |
118 | MAX_UPLOAD_PART: bytes.parse(config.get<string>('object_storage.max_upload_part')), | 121 | MAX_UPLOAD_PART: bytes.parse(config.get<string>('object_storage.max_upload_part')), |
@@ -126,6 +129,9 @@ const CONFIG = { | |||
126 | ACCESS_KEY_ID: config.get<string>('object_storage.credentials.access_key_id'), | 129 | ACCESS_KEY_ID: config.get<string>('object_storage.credentials.access_key_id'), |
127 | SECRET_ACCESS_KEY: config.get<string>('object_storage.credentials.secret_access_key') | 130 | SECRET_ACCESS_KEY: config.get<string>('object_storage.credentials.secret_access_key') |
128 | }, | 131 | }, |
132 | PROXY: { | ||
133 | PROXIFY_PRIVATE_FILES: config.get<boolean>('object_storage.proxy.proxify_private_files') | ||
134 | }, | ||
129 | VIDEOS: { | 135 | VIDEOS: { |
130 | BUCKET_NAME: config.get<string>('object_storage.videos.bucket_name'), | 136 | BUCKET_NAME: config.get<string>('object_storage.videos.bucket_name'), |
131 | PREFIX: config.get<string>('object_storage.videos.prefix'), | 137 | PREFIX: config.get<string>('object_storage.videos.prefix'), |
diff --git a/server/middlewares/validators/index.ts b/server/middlewares/validators/index.ts index 899da229a..9bc8887ff 100644 --- a/server/middlewares/validators/index.ts +++ b/server/middlewares/validators/index.ts | |||
@@ -11,6 +11,7 @@ export * from './follows' | |||
11 | export * from './jobs' | 11 | export * from './jobs' |
12 | export * from './logs' | 12 | export * from './logs' |
13 | export * from './metrics' | 13 | export * from './metrics' |
14 | export * from './object-storage-proxy' | ||
14 | export * from './oembed' | 15 | export * from './oembed' |
15 | export * from './pagination' | 16 | export * from './pagination' |
16 | export * from './plugins' | 17 | export * from './plugins' |
diff --git a/server/middlewares/validators/object-storage-proxy.ts b/server/middlewares/validators/object-storage-proxy.ts new file mode 100644 index 000000000..bbd77f262 --- /dev/null +++ b/server/middlewares/validators/object-storage-proxy.ts | |||
@@ -0,0 +1,20 @@ | |||
1 | import express from 'express' | ||
2 | import { CONFIG } from '@server/initializers/config' | ||
3 | import { HttpStatusCode } from '@shared/models' | ||
4 | |||
5 | const ensurePrivateObjectStorageProxyIsEnabled = [ | ||
6 | (req: express.Request, res: express.Response, next: express.NextFunction) => { | ||
7 | if (CONFIG.OBJECT_STORAGE.PROXY.PROXIFY_PRIVATE_FILES !== true) { | ||
8 | return res.fail({ | ||
9 | message: 'Private object storage proxy is not enabled', | ||
10 | status: HttpStatusCode.BAD_REQUEST_400 | ||
11 | }) | ||
12 | } | ||
13 | |||
14 | return next() | ||
15 | } | ||
16 | ] | ||
17 | |||
18 | export { | ||
19 | ensurePrivateObjectStorageProxyIsEnabled | ||
20 | } | ||
diff --git a/server/models/video/video-file.ts b/server/models/video/video-file.ts index c20c90c1b..9c4e6d078 100644 --- a/server/models/video/video-file.ts +++ b/server/models/video/video-file.ts | |||
@@ -54,6 +54,7 @@ import { doesExist } from '../shared' | |||
54 | import { parseAggregateResult, throwIfNotValid } from '../utils' | 54 | import { parseAggregateResult, throwIfNotValid } from '../utils' |
55 | import { VideoModel } from './video' | 55 | import { VideoModel } from './video' |
56 | import { VideoStreamingPlaylistModel } from './video-streaming-playlist' | 56 | import { VideoStreamingPlaylistModel } from './video-streaming-playlist' |
57 | import { CONFIG } from '@server/initializers/config' | ||
57 | 58 | ||
58 | export enum ScopeNames { | 59 | export enum ScopeNames { |
59 | WITH_VIDEO = 'WITH_VIDEO', | 60 | WITH_VIDEO = 'WITH_VIDEO', |
@@ -511,7 +512,7 @@ export class VideoFileModel extends Model<Partial<AttributesOnly<VideoFileModel> | |||
511 | // --------------------------------------------------------------------------- | 512 | // --------------------------------------------------------------------------- |
512 | 513 | ||
513 | getObjectStorageUrl (video: MVideo) { | 514 | getObjectStorageUrl (video: MVideo) { |
514 | if (video.hasPrivateStaticPath()) { | 515 | if (video.hasPrivateStaticPath() && CONFIG.OBJECT_STORAGE.PROXY.PROXIFY_PRIVATE_FILES === true) { |
515 | return this.getPrivateObjectStorageUrl(video) | 516 | return this.getPrivateObjectStorageUrl(video) |
516 | } | 517 | } |
517 | 518 | ||
diff --git a/server/models/video/video-streaming-playlist.ts b/server/models/video/video-streaming-playlist.ts index 1318a4dae..0386edf28 100644 --- a/server/models/video/video-streaming-playlist.ts +++ b/server/models/video/video-streaming-playlist.ts | |||
@@ -15,6 +15,7 @@ import { | |||
15 | Table, | 15 | Table, |
16 | UpdatedAt | 16 | UpdatedAt |
17 | } from 'sequelize-typescript' | 17 | } from 'sequelize-typescript' |
18 | import { CONFIG } from '@server/initializers/config' | ||
18 | import { getHLSPrivateFileUrl, getHLSPublicFileUrl } from '@server/lib/object-storage' | 19 | import { getHLSPrivateFileUrl, getHLSPublicFileUrl } from '@server/lib/object-storage' |
19 | import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename } from '@server/lib/paths' | 20 | import { generateHLSMasterPlaylistFilename, generateHlsSha256SegmentsFilename } from '@server/lib/paths' |
20 | import { isVideoInPrivateDirectory } from '@server/lib/video-privacy' | 21 | import { isVideoInPrivateDirectory } from '@server/lib/video-privacy' |
@@ -260,7 +261,7 @@ export class VideoStreamingPlaylistModel extends Model<Partial<AttributesOnly<Vi | |||
260 | } | 261 | } |
261 | 262 | ||
262 | private getMasterPlaylistObjectStorageUrl (video: MVideo) { | 263 | private getMasterPlaylistObjectStorageUrl (video: MVideo) { |
263 | if (video.hasPrivateStaticPath()) { | 264 | if (video.hasPrivateStaticPath() && CONFIG.OBJECT_STORAGE.PROXY.PROXIFY_PRIVATE_FILES === true) { |
264 | return getHLSPrivateFileUrl(video, this.playlistFilename) | 265 | return getHLSPrivateFileUrl(video, this.playlistFilename) |
265 | } | 266 | } |
266 | 267 | ||
@@ -282,7 +283,7 @@ export class VideoStreamingPlaylistModel extends Model<Partial<AttributesOnly<Vi | |||
282 | } | 283 | } |
283 | 284 | ||
284 | private getSha256SegmentsObjectStorageUrl (video: MVideo) { | 285 | private getSha256SegmentsObjectStorageUrl (video: MVideo) { |
285 | if (video.hasPrivateStaticPath()) { | 286 | if (video.hasPrivateStaticPath() && CONFIG.OBJECT_STORAGE.PROXY.PROXIFY_PRIVATE_FILES === true) { |
286 | return getHLSPrivateFileUrl(video, this.segmentsSha256Filename) | 287 | return getHLSPrivateFileUrl(video, this.segmentsSha256Filename) |
287 | } | 288 | } |
288 | 289 | ||
diff --git a/server/tests/api/object-storage/video-static-file-privacy.ts b/server/tests/api/object-storage/video-static-file-privacy.ts index c6d7a1a2c..ed8855b3b 100644 --- a/server/tests/api/object-storage/video-static-file-privacy.ts +++ b/server/tests/api/object-storage/video-static-file-privacy.ts | |||
@@ -19,6 +19,12 @@ import { | |||
19 | waitJobs | 19 | waitJobs |
20 | } from '@shared/server-commands' | 20 | } from '@shared/server-commands' |
21 | 21 | ||
22 | function extractFilenameFromUrl (url: string) { | ||
23 | const parts = basename(url).split(':') | ||
24 | |||
25 | return parts[parts.length - 1] | ||
26 | } | ||
27 | |||
22 | describe('Object storage for video static file privacy', function () { | 28 | describe('Object storage for video static file privacy', function () { |
23 | // We need real world object storage to check ACL | 29 | // We need real world object storage to check ACL |
24 | if (areScalewayObjectStorageTestsDisabled()) return | 30 | if (areScalewayObjectStorageTestsDisabled()) return |
@@ -26,75 +32,81 @@ describe('Object storage for video static file privacy', function () { | |||
26 | let server: PeerTubeServer | 32 | let server: PeerTubeServer |
27 | let userToken: string | 33 | let userToken: string |
28 | 34 | ||
29 | before(async function () { | 35 | // --------------------------------------------------------------------------- |
30 | this.timeout(120000) | ||
31 | 36 | ||
32 | server = await createSingleServer(1, ObjectStorageCommand.getDefaultScalewayConfig(1)) | 37 | async function checkPrivateVODFiles (uuid: string) { |
33 | await setAccessTokensToServers([ server ]) | 38 | const video = await server.videos.getWithToken({ id: uuid }) |
34 | await setDefaultVideoChannel([ server ]) | ||
35 | 39 | ||
36 | await server.config.enableMinimumTranscoding() | 40 | for (const file of video.files) { |
41 | expectStartWith(file.fileUrl, server.url + '/object-storage-proxy/webseed/private/') | ||
37 | 42 | ||
38 | userToken = await server.users.generateUserAndToken('user1') | 43 | await makeRawRequest({ url: file.fileUrl, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 }) |
39 | }) | 44 | } |
40 | 45 | ||
41 | describe('VOD', function () { | 46 | for (const file of getAllFiles(video)) { |
42 | let privateVideoUUID: string | 47 | const internalFileUrl = await server.sql.getInternalFileUrl(file.id) |
43 | let publicVideoUUID: string | 48 | expectStartWith(internalFileUrl, ObjectStorageCommand.getScalewayBaseUrl()) |
44 | let userPrivateVideoUUID: string | 49 | await makeRawRequest({ url: internalFileUrl, token: server.accessToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) |
50 | } | ||
51 | |||
52 | const hls = getHLS(video) | ||
53 | |||
54 | if (hls) { | ||
55 | for (const url of [ hls.playlistUrl, hls.segmentsSha256Url ]) { | ||
56 | expectStartWith(url, server.url + '/object-storage-proxy/streaming-playlists/hls/private/') | ||
57 | } | ||
45 | 58 | ||
46 | async function checkPrivateFiles (uuid: string) { | 59 | await makeRawRequest({ url: hls.playlistUrl, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 }) |
47 | const video = await server.videos.getWithToken({ id: uuid }) | 60 | await makeRawRequest({ url: hls.segmentsSha256Url, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 }) |
48 | 61 | ||
49 | for (const file of video.files) { | 62 | for (const file of hls.files) { |
50 | expectStartWith(file.fileUrl, server.url + '/object-storage-proxy/webseed/private/') | 63 | expectStartWith(file.fileUrl, server.url + '/object-storage-proxy/streaming-playlists/hls/private/') |
51 | 64 | ||
52 | await makeRawRequest({ url: file.fileUrl, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 }) | 65 | await makeRawRequest({ url: file.fileUrl, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 }) |
53 | } | 66 | } |
67 | } | ||
68 | } | ||
54 | 69 | ||
55 | for (const file of getAllFiles(video)) { | 70 | async function checkPublicVODFiles (uuid: string) { |
56 | const internalFileUrl = await server.sql.getInternalFileUrl(file.id) | 71 | const video = await server.videos.getWithToken({ id: uuid }) |
57 | expectStartWith(internalFileUrl, ObjectStorageCommand.getScalewayBaseUrl()) | ||
58 | await makeRawRequest({ url: internalFileUrl, token: server.accessToken, expectedStatus: HttpStatusCode.FORBIDDEN_403 }) | ||
59 | } | ||
60 | 72 | ||
61 | const hls = getHLS(video) | 73 | for (const file of getAllFiles(video)) { |
74 | expectStartWith(file.fileUrl, ObjectStorageCommand.getScalewayBaseUrl()) | ||
62 | 75 | ||
63 | if (hls) { | 76 | await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 }) |
64 | for (const url of [ hls.playlistUrl, hls.segmentsSha256Url ]) { | 77 | } |
65 | expectStartWith(url, server.url + '/object-storage-proxy/streaming-playlists/hls/private/') | ||
66 | } | ||
67 | 78 | ||
68 | await makeRawRequest({ url: hls.playlistUrl, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 }) | 79 | const hls = getHLS(video) |
69 | await makeRawRequest({ url: hls.segmentsSha256Url, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 }) | ||
70 | 80 | ||
71 | for (const file of hls.files) { | 81 | if (hls) { |
72 | expectStartWith(file.fileUrl, server.url + '/object-storage-proxy/streaming-playlists/hls/private/') | 82 | expectStartWith(hls.playlistUrl, ObjectStorageCommand.getScalewayBaseUrl()) |
83 | expectStartWith(hls.segmentsSha256Url, ObjectStorageCommand.getScalewayBaseUrl()) | ||
73 | 84 | ||
74 | await makeRawRequest({ url: file.fileUrl, token: server.accessToken, expectedStatus: HttpStatusCode.OK_200 }) | 85 | await makeRawRequest({ url: hls.playlistUrl, expectedStatus: HttpStatusCode.OK_200 }) |
75 | } | 86 | await makeRawRequest({ url: hls.segmentsSha256Url, expectedStatus: HttpStatusCode.OK_200 }) |
76 | } | ||
77 | } | 87 | } |
88 | } | ||
78 | 89 | ||
79 | async function checkPublicFiles (uuid: string) { | 90 | // --------------------------------------------------------------------------- |
80 | const video = await server.videos.getWithToken({ id: uuid }) | ||
81 | 91 | ||
82 | for (const file of getAllFiles(video)) { | 92 | before(async function () { |
83 | expectStartWith(file.fileUrl, ObjectStorageCommand.getScalewayBaseUrl()) | 93 | this.timeout(120000) |
84 | 94 | ||
85 | await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 }) | 95 | server = await createSingleServer(1, ObjectStorageCommand.getDefaultScalewayConfig({ serverNumber: 1 })) |
86 | } | 96 | await setAccessTokensToServers([ server ]) |
97 | await setDefaultVideoChannel([ server ]) | ||
87 | 98 | ||
88 | const hls = getHLS(video) | 99 | await server.config.enableMinimumTranscoding() |
89 | 100 | ||
90 | if (hls) { | 101 | userToken = await server.users.generateUserAndToken('user1') |
91 | expectStartWith(hls.playlistUrl, ObjectStorageCommand.getScalewayBaseUrl()) | 102 | }) |
92 | expectStartWith(hls.segmentsSha256Url, ObjectStorageCommand.getScalewayBaseUrl()) | ||
93 | 103 | ||
94 | await makeRawRequest({ url: hls.playlistUrl, expectedStatus: HttpStatusCode.OK_200 }) | 104 | describe('VOD', function () { |
95 | await makeRawRequest({ url: hls.segmentsSha256Url, expectedStatus: HttpStatusCode.OK_200 }) | 105 | let privateVideoUUID: string |
96 | } | 106 | let publicVideoUUID: string |
97 | } | 107 | let userPrivateVideoUUID: string |
108 | |||
109 | // --------------------------------------------------------------------------- | ||
98 | 110 | ||
99 | async function getSampleFileUrls (videoId: string) { | 111 | async function getSampleFileUrls (videoId: string) { |
100 | const video = await server.videos.getWithToken({ id: videoId }) | 112 | const video = await server.videos.getWithToken({ id: videoId }) |
@@ -105,6 +117,8 @@ describe('Object storage for video static file privacy', function () { | |||
105 | } | 117 | } |
106 | } | 118 | } |
107 | 119 | ||
120 | // --------------------------------------------------------------------------- | ||
121 | |||
108 | it('Should upload a private video and have appropriate object storage ACL', async function () { | 122 | it('Should upload a private video and have appropriate object storage ACL', async function () { |
109 | this.timeout(60000) | 123 | this.timeout(60000) |
110 | 124 | ||
@@ -120,7 +134,7 @@ describe('Object storage for video static file privacy', function () { | |||
120 | 134 | ||
121 | await waitJobs([ server ]) | 135 | await waitJobs([ server ]) |
122 | 136 | ||
123 | await checkPrivateFiles(privateVideoUUID) | 137 | await checkPrivateVODFiles(privateVideoUUID) |
124 | }) | 138 | }) |
125 | 139 | ||
126 | it('Should upload a public video and have appropriate object storage ACL', async function () { | 140 | it('Should upload a public video and have appropriate object storage ACL', async function () { |
@@ -131,7 +145,7 @@ describe('Object storage for video static file privacy', function () { | |||
131 | 145 | ||
132 | publicVideoUUID = uuid | 146 | publicVideoUUID = uuid |
133 | 147 | ||
134 | await checkPublicFiles(publicVideoUUID) | 148 | await checkPublicVODFiles(publicVideoUUID) |
135 | }) | 149 | }) |
136 | 150 | ||
137 | it('Should not get files without appropriate OAuth token', async function () { | 151 | it('Should not get files without appropriate OAuth token', async function () { |
@@ -182,7 +196,7 @@ describe('Object storage for video static file privacy', function () { | |||
182 | 196 | ||
183 | await server.videos.update({ id: publicVideoUUID, attributes: { privacy: VideoPrivacy.INTERNAL } }) | 197 | await server.videos.update({ id: publicVideoUUID, attributes: { privacy: VideoPrivacy.INTERNAL } }) |
184 | 198 | ||
185 | await checkPrivateFiles(publicVideoUUID) | 199 | await checkPrivateVODFiles(publicVideoUUID) |
186 | }) | 200 | }) |
187 | 201 | ||
188 | it('Should update private video to public', async function () { | 202 | it('Should update private video to public', async function () { |
@@ -190,7 +204,7 @@ describe('Object storage for video static file privacy', function () { | |||
190 | 204 | ||
191 | await server.videos.update({ id: publicVideoUUID, attributes: { privacy: VideoPrivacy.PUBLIC } }) | 205 | await server.videos.update({ id: publicVideoUUID, attributes: { privacy: VideoPrivacy.PUBLIC } }) |
192 | 206 | ||
193 | await checkPublicFiles(publicVideoUUID) | 207 | await checkPublicVODFiles(publicVideoUUID) |
194 | }) | 208 | }) |
195 | }) | 209 | }) |
196 | 210 | ||
@@ -203,6 +217,8 @@ describe('Object storage for video static file privacy', function () { | |||
203 | 217 | ||
204 | let unrelatedFileToken: string | 218 | let unrelatedFileToken: string |
205 | 219 | ||
220 | // --------------------------------------------------------------------------- | ||
221 | |||
206 | async function checkLiveFiles (live: LiveVideo, liveId: string) { | 222 | async function checkLiveFiles (live: LiveVideo, liveId: string) { |
207 | const ffmpegCommand = sendRTMPStream({ rtmpBaseUrl: live.rtmpUrl, streamKey: live.streamKey }) | 223 | const ffmpegCommand = sendRTMPStream({ rtmpBaseUrl: live.rtmpUrl, streamKey: live.streamKey }) |
208 | await server.live.waitUntilPublished({ videoId: liveId }) | 224 | await server.live.waitUntilPublished({ videoId: liveId }) |
@@ -260,6 +276,8 @@ describe('Object storage for video static file privacy', function () { | |||
260 | } | 276 | } |
261 | } | 277 | } |
262 | 278 | ||
279 | // --------------------------------------------------------------------------- | ||
280 | |||
263 | before(async function () { | 281 | before(async function () { |
264 | await server.config.enableMinimumTranscoding() | 282 | await server.config.enableMinimumTranscoding() |
265 | 283 | ||
@@ -320,6 +338,52 @@ describe('Object storage for video static file privacy', function () { | |||
320 | }) | 338 | }) |
321 | }) | 339 | }) |
322 | 340 | ||
341 | describe('With private files proxy disabled and public ACL for private files', function () { | ||
342 | let videoUUID: string | ||
343 | |||
344 | before(async function () { | ||
345 | this.timeout(240000) | ||
346 | |||
347 | await server.kill() | ||
348 | |||
349 | const config = ObjectStorageCommand.getDefaultScalewayConfig({ | ||
350 | serverNumber: server.internalServerNumber, | ||
351 | enablePrivateProxy: false, | ||
352 | privateACL: 'public-read' | ||
353 | }) | ||
354 | await server.run(config) | ||
355 | |||
356 | const { uuid } = await server.videos.quickUpload({ name: 'video', privacy: VideoPrivacy.PRIVATE }) | ||
357 | videoUUID = uuid | ||
358 | |||
359 | await waitJobs([ server ]) | ||
360 | }) | ||
361 | |||
362 | it('Should display object storage path for a private video and be able to access them', async function () { | ||
363 | this.timeout(60000) | ||
364 | |||
365 | await checkPublicVODFiles(videoUUID) | ||
366 | }) | ||
367 | |||
368 | it('Should not be able to access object storage proxy', async function () { | ||
369 | const privateVideo = await server.videos.getWithToken({ id: videoUUID }) | ||
370 | const webtorrentFilename = extractFilenameFromUrl(privateVideo.files[0].fileUrl) | ||
371 | const hlsFilename = extractFilenameFromUrl(getHLS(privateVideo).files[0].fileUrl) | ||
372 | |||
373 | await makeRawRequest({ | ||
374 | url: server.url + '/object-storage-proxy/webseed/private/' + webtorrentFilename, | ||
375 | token: server.accessToken, | ||
376 | expectedStatus: HttpStatusCode.BAD_REQUEST_400 | ||
377 | }) | ||
378 | |||
379 | await makeRawRequest({ | ||
380 | url: server.url + '/object-storage-proxy/streaming-playlists/hls/private/' + videoUUID + '/' + hlsFilename, | ||
381 | token: server.accessToken, | ||
382 | expectedStatus: HttpStatusCode.BAD_REQUEST_400 | ||
383 | }) | ||
384 | }) | ||
385 | }) | ||
386 | |||
323 | after(async function () { | 387 | after(async function () { |
324 | this.timeout(60000) | 388 | this.timeout(60000) |
325 | 389 | ||
diff --git a/server/tests/api/videos/video-static-file-privacy.ts b/server/tests/api/videos/video-static-file-privacy.ts index bdbe85127..eaaed5aad 100644 --- a/server/tests/api/videos/video-static-file-privacy.ts +++ b/server/tests/api/videos/video-static-file-privacy.ts | |||
@@ -383,6 +383,39 @@ describe('Test video static file privacy', function () { | |||
383 | }) | 383 | }) |
384 | }) | 384 | }) |
385 | 385 | ||
386 | describe('With static file right check disabled', function () { | ||
387 | let videoUUID: string | ||
388 | |||
389 | before(async function () { | ||
390 | this.timeout(240000) | ||
391 | |||
392 | await server.kill() | ||
393 | |||
394 | await server.run({ | ||
395 | static_files: { | ||
396 | private_files_require_auth: false | ||
397 | } | ||
398 | }) | ||
399 | |||
400 | const { uuid } = await server.videos.quickUpload({ name: 'video', privacy: VideoPrivacy.INTERNAL }) | ||
401 | videoUUID = uuid | ||
402 | |||
403 | await waitJobs([ server ]) | ||
404 | }) | ||
405 | |||
406 | it('Should not check auth for private static files', async function () { | ||
407 | const video = await server.videos.getWithToken({ id: videoUUID }) | ||
408 | |||
409 | for (const file of getAllFiles(video)) { | ||
410 | await makeRawRequest({ url: file.fileUrl, expectedStatus: HttpStatusCode.OK_200 }) | ||
411 | } | ||
412 | |||
413 | const hls = video.streamingPlaylists[0] | ||
414 | await makeRawRequest({ url: hls.playlistUrl, expectedStatus: HttpStatusCode.OK_200 }) | ||
415 | await makeRawRequest({ url: hls.segmentsSha256Url, expectedStatus: HttpStatusCode.OK_200 }) | ||
416 | }) | ||
417 | }) | ||
418 | |||
386 | after(async function () { | 419 | after(async function () { |
387 | await cleanupTests([ server ]) | 420 | await cleanupTests([ server ]) |
388 | }) | 421 | }) |
diff --git a/shared/server-commands/server/object-storage-command.ts b/shared/server-commands/server/object-storage-command.ts index 405e1b043..a1fe4f0f7 100644 --- a/shared/server-commands/server/object-storage-command.ts +++ b/shared/server-commands/server/object-storage-command.ts | |||
@@ -81,7 +81,13 @@ export class ObjectStorageCommand extends AbstractCommand { | |||
81 | 81 | ||
82 | // --------------------------------------------------------------------------- | 82 | // --------------------------------------------------------------------------- |
83 | 83 | ||
84 | static getDefaultScalewayConfig (serverNumber: number) { | 84 | static getDefaultScalewayConfig (options: { |
85 | serverNumber: number | ||
86 | enablePrivateProxy?: boolean // default true | ||
87 | privateACL?: 'private' | 'public-read' // default 'private' | ||
88 | }) { | ||
89 | const { serverNumber, enablePrivateProxy = true, privateACL = 'private' } = options | ||
90 | |||
85 | return { | 91 | return { |
86 | object_storage: { | 92 | object_storage: { |
87 | enabled: true, | 93 | enabled: true, |
@@ -90,6 +96,14 @@ export class ObjectStorageCommand extends AbstractCommand { | |||
90 | 96 | ||
91 | credentials: this.getScalewayCredentialsConfig(), | 97 | credentials: this.getScalewayCredentialsConfig(), |
92 | 98 | ||
99 | upload_acl: { | ||
100 | private: privateACL | ||
101 | }, | ||
102 | |||
103 | proxy: { | ||
104 | proxify_private_files: enablePrivateProxy | ||
105 | }, | ||
106 | |||
93 | streaming_playlists: { | 107 | streaming_playlists: { |
94 | bucket_name: this.DEFAULT_SCALEWAY_BUCKET, | 108 | bucket_name: this.DEFAULT_SCALEWAY_BUCKET, |
95 | prefix: `test:server-${serverNumber}-streaming-playlists:` | 109 | prefix: `test:server-${serverNumber}-streaming-playlists:` |