diff options
Diffstat (limited to 'server/lib')
-rw-r--r-- | server/lib/live/live-segment-sha-store.ts | 27 | ||||
-rw-r--r-- | server/lib/object-storage/shared/object-storage-helpers.ts | 192 | ||||
-rw-r--r-- | server/lib/object-storage/urls.ts | 29 | ||||
-rw-r--r-- | server/lib/object-storage/videos.ts | 80 | ||||
-rw-r--r-- | server/lib/video-privacy.ts | 89 |
5 files changed, 321 insertions, 96 deletions
diff --git a/server/lib/live/live-segment-sha-store.ts b/server/lib/live/live-segment-sha-store.ts index faf03dccf..4d03754a9 100644 --- a/server/lib/live/live-segment-sha-store.ts +++ b/server/lib/live/live-segment-sha-store.ts | |||
@@ -5,6 +5,7 @@ import { logger, loggerTagsFactory } from '@server/helpers/logger' | |||
5 | import { MStreamingPlaylistVideo } from '@server/types/models' | 5 | import { MStreamingPlaylistVideo } from '@server/types/models' |
6 | import { buildSha256Segment } from '../hls' | 6 | import { buildSha256Segment } from '../hls' |
7 | import { storeHLSFileFromPath } from '../object-storage' | 7 | import { storeHLSFileFromPath } from '../object-storage' |
8 | import PQueue from 'p-queue' | ||
8 | 9 | ||
9 | const lTags = loggerTagsFactory('live') | 10 | const lTags = loggerTagsFactory('live') |
10 | 11 | ||
@@ -16,6 +17,7 @@ class LiveSegmentShaStore { | |||
16 | private readonly sha256Path: string | 17 | private readonly sha256Path: string |
17 | private readonly streamingPlaylist: MStreamingPlaylistVideo | 18 | private readonly streamingPlaylist: MStreamingPlaylistVideo |
18 | private readonly sendToObjectStorage: boolean | 19 | private readonly sendToObjectStorage: boolean |
20 | private readonly writeQueue = new PQueue({ concurrency: 1 }) | ||
19 | 21 | ||
20 | constructor (options: { | 22 | constructor (options: { |
21 | videoUUID: string | 23 | videoUUID: string |
@@ -37,7 +39,11 @@ class LiveSegmentShaStore { | |||
37 | const segmentName = basename(segmentPath) | 39 | const segmentName = basename(segmentPath) |
38 | this.segmentsSha256.set(segmentName, shaResult) | 40 | this.segmentsSha256.set(segmentName, shaResult) |
39 | 41 | ||
40 | await this.writeToDisk() | 42 | try { |
43 | await this.writeToDisk() | ||
44 | } catch (err) { | ||
45 | logger.error('Cannot write sha segments to disk.', { err }) | ||
46 | } | ||
41 | } | 47 | } |
42 | 48 | ||
43 | async removeSegmentSha (segmentPath: string) { | 49 | async removeSegmentSha (segmentPath: string) { |
@@ -55,19 +61,20 @@ class LiveSegmentShaStore { | |||
55 | await this.writeToDisk() | 61 | await this.writeToDisk() |
56 | } | 62 | } |
57 | 63 | ||
58 | private async writeToDisk () { | 64 | private writeToDisk () { |
59 | await writeJson(this.sha256Path, mapToJSON(this.segmentsSha256)) | 65 | return this.writeQueue.add(async () => { |
66 | await writeJson(this.sha256Path, mapToJSON(this.segmentsSha256)) | ||
60 | 67 | ||
61 | if (this.sendToObjectStorage) { | 68 | if (this.sendToObjectStorage) { |
62 | const url = await storeHLSFileFromPath(this.streamingPlaylist, this.sha256Path) | 69 | const url = await storeHLSFileFromPath(this.streamingPlaylist, this.sha256Path) |
63 | 70 | ||
64 | if (this.streamingPlaylist.segmentsSha256Url !== url) { | 71 | if (this.streamingPlaylist.segmentsSha256Url !== url) { |
65 | this.streamingPlaylist.segmentsSha256Url = url | 72 | this.streamingPlaylist.segmentsSha256Url = url |
66 | await this.streamingPlaylist.save() | 73 | await this.streamingPlaylist.save() |
74 | } | ||
67 | } | 75 | } |
68 | } | 76 | }) |
69 | } | 77 | } |
70 | |||
71 | } | 78 | } |
72 | 79 | ||
73 | export { | 80 | export { |
diff --git a/server/lib/object-storage/shared/object-storage-helpers.ts b/server/lib/object-storage/shared/object-storage-helpers.ts index c131977e8..05b52f412 100644 --- a/server/lib/object-storage/shared/object-storage-helpers.ts +++ b/server/lib/object-storage/shared/object-storage-helpers.ts | |||
@@ -2,18 +2,21 @@ import { createReadStream, createWriteStream, ensureDir, ReadStream } from 'fs-e | |||
2 | import { dirname } from 'path' | 2 | import { dirname } from 'path' |
3 | import { Readable } from 'stream' | 3 | import { Readable } from 'stream' |
4 | import { | 4 | import { |
5 | _Object, | ||
5 | CompleteMultipartUploadCommandOutput, | 6 | CompleteMultipartUploadCommandOutput, |
6 | DeleteObjectCommand, | 7 | DeleteObjectCommand, |
7 | GetObjectCommand, | 8 | GetObjectCommand, |
8 | ListObjectsV2Command, | 9 | ListObjectsV2Command, |
9 | PutObjectCommandInput | 10 | PutObjectAclCommand, |
11 | PutObjectCommandInput, | ||
12 | S3Client | ||
10 | } from '@aws-sdk/client-s3' | 13 | } from '@aws-sdk/client-s3' |
11 | import { Upload } from '@aws-sdk/lib-storage' | 14 | import { Upload } from '@aws-sdk/lib-storage' |
12 | import { pipelinePromise } from '@server/helpers/core-utils' | 15 | import { pipelinePromise } from '@server/helpers/core-utils' |
13 | import { isArray } from '@server/helpers/custom-validators/misc' | 16 | import { isArray } from '@server/helpers/custom-validators/misc' |
14 | import { logger } from '@server/helpers/logger' | 17 | import { logger } from '@server/helpers/logger' |
15 | import { CONFIG } from '@server/initializers/config' | 18 | import { CONFIG } from '@server/initializers/config' |
16 | import { getPrivateUrl } from '../urls' | 19 | import { getInternalUrl } from '../urls' |
17 | import { getClient } from './client' | 20 | import { getClient } from './client' |
18 | import { lTags } from './logger' | 21 | import { lTags } from './logger' |
19 | 22 | ||
@@ -44,69 +47,91 @@ async function storeObject (options: { | |||
44 | inputPath: string | 47 | inputPath: string |
45 | objectStorageKey: string | 48 | objectStorageKey: string |
46 | bucketInfo: BucketInfo | 49 | bucketInfo: BucketInfo |
50 | isPrivate: boolean | ||
47 | }): Promise<string> { | 51 | }): Promise<string> { |
48 | const { inputPath, objectStorageKey, bucketInfo } = options | 52 | const { inputPath, objectStorageKey, bucketInfo, isPrivate } = options |
49 | 53 | ||
50 | logger.debug('Uploading file %s to %s%s in bucket %s', inputPath, bucketInfo.PREFIX, objectStorageKey, bucketInfo.BUCKET_NAME, lTags()) | 54 | logger.debug('Uploading file %s to %s%s in bucket %s', inputPath, bucketInfo.PREFIX, objectStorageKey, bucketInfo.BUCKET_NAME, lTags()) |
51 | 55 | ||
52 | const fileStream = createReadStream(inputPath) | 56 | const fileStream = createReadStream(inputPath) |
53 | 57 | ||
54 | return uploadToStorage({ objectStorageKey, content: fileStream, bucketInfo }) | 58 | return uploadToStorage({ objectStorageKey, content: fileStream, bucketInfo, isPrivate }) |
55 | } | 59 | } |
56 | 60 | ||
57 | // --------------------------------------------------------------------------- | 61 | // --------------------------------------------------------------------------- |
58 | 62 | ||
59 | async function removeObject (filename: string, bucketInfo: BucketInfo) { | 63 | function updateObjectACL (options: { |
60 | const command = new DeleteObjectCommand({ | 64 | objectStorageKey: string |
65 | bucketInfo: BucketInfo | ||
66 | isPrivate: boolean | ||
67 | }) { | ||
68 | const { objectStorageKey, bucketInfo, isPrivate } = options | ||
69 | |||
70 | const key = buildKey(objectStorageKey, bucketInfo) | ||
71 | |||
72 | logger.debug('Updating ACL file %s in bucket %s', key, bucketInfo.BUCKET_NAME, lTags()) | ||
73 | |||
74 | const command = new PutObjectAclCommand({ | ||
61 | Bucket: bucketInfo.BUCKET_NAME, | 75 | Bucket: bucketInfo.BUCKET_NAME, |
62 | Key: buildKey(filename, bucketInfo) | 76 | Key: key, |
77 | ACL: getACL(isPrivate) | ||
63 | }) | 78 | }) |
64 | 79 | ||
65 | return getClient().send(command) | 80 | return getClient().send(command) |
66 | } | 81 | } |
67 | 82 | ||
68 | async function removePrefix (prefix: string, bucketInfo: BucketInfo) { | 83 | function updatePrefixACL (options: { |
69 | const s3Client = getClient() | 84 | prefix: string |
70 | 85 | bucketInfo: BucketInfo | |
71 | const commandPrefix = bucketInfo.PREFIX + prefix | 86 | isPrivate: boolean |
72 | const listCommand = new ListObjectsV2Command({ | 87 | }) { |
73 | Bucket: bucketInfo.BUCKET_NAME, | 88 | const { prefix, bucketInfo, isPrivate } = options |
74 | Prefix: commandPrefix | 89 | |
90 | logger.debug('Updating ACL of files in prefix %s in bucket %s', prefix, bucketInfo.BUCKET_NAME, lTags()) | ||
91 | |||
92 | return applyOnPrefix({ | ||
93 | prefix, | ||
94 | bucketInfo, | ||
95 | commandBuilder: obj => { | ||
96 | return new PutObjectAclCommand({ | ||
97 | Bucket: bucketInfo.BUCKET_NAME, | ||
98 | Key: obj.Key, | ||
99 | ACL: getACL(isPrivate) | ||
100 | }) | ||
101 | } | ||
75 | }) | 102 | }) |
103 | } | ||
76 | 104 | ||
77 | const listedObjects = await s3Client.send(listCommand) | 105 | // --------------------------------------------------------------------------- |
78 | 106 | ||
79 | // FIXME: use bulk delete when s3ninja will support this operation | 107 | function removeObject (objectStorageKey: string, bucketInfo: BucketInfo) { |
80 | // const deleteParams = { | 108 | const key = buildKey(objectStorageKey, bucketInfo) |
81 | // Bucket: bucketInfo.BUCKET_NAME, | ||
82 | // Delete: { Objects: [] } | ||
83 | // } | ||
84 | 109 | ||
85 | if (isArray(listedObjects.Contents) !== true) { | 110 | logger.debug('Removing file %s in bucket %s', key, bucketInfo.BUCKET_NAME, lTags()) |
86 | const message = `Cannot remove ${commandPrefix} prefix in bucket ${bucketInfo.BUCKET_NAME}: no files listed.` | ||
87 | 111 | ||
88 | logger.error(message, { response: listedObjects, ...lTags() }) | 112 | const command = new DeleteObjectCommand({ |
89 | throw new Error(message) | 113 | Bucket: bucketInfo.BUCKET_NAME, |
90 | } | 114 | Key: key |
91 | 115 | }) | |
92 | for (const object of listedObjects.Contents) { | ||
93 | const command = new DeleteObjectCommand({ | ||
94 | Bucket: bucketInfo.BUCKET_NAME, | ||
95 | Key: object.Key | ||
96 | }) | ||
97 | |||
98 | await s3Client.send(command) | ||
99 | 116 | ||
100 | // FIXME: use bulk delete when s3ninja will support this operation | 117 | return getClient().send(command) |
101 | // deleteParams.Delete.Objects.push({ Key: object.Key }) | 118 | } |
102 | } | ||
103 | 119 | ||
120 | function removePrefix (prefix: string, bucketInfo: BucketInfo) { | ||
104 | // FIXME: use bulk delete when s3ninja will support this operation | 121 | // FIXME: use bulk delete when s3ninja will support this operation |
105 | // const deleteCommand = new DeleteObjectsCommand(deleteParams) | ||
106 | // await s3Client.send(deleteCommand) | ||
107 | 122 | ||
108 | // Repeat if not all objects could be listed at once (limit of 1000?) | 123 | logger.debug('Removing prefix %s in bucket %s', prefix, bucketInfo.BUCKET_NAME, lTags()) |
109 | if (listedObjects.IsTruncated) await removePrefix(prefix, bucketInfo) | 124 | |
125 | return applyOnPrefix({ | ||
126 | prefix, | ||
127 | bucketInfo, | ||
128 | commandBuilder: obj => { | ||
129 | return new DeleteObjectCommand({ | ||
130 | Bucket: bucketInfo.BUCKET_NAME, | ||
131 | Key: obj.Key | ||
132 | }) | ||
133 | } | ||
134 | }) | ||
110 | } | 135 | } |
111 | 136 | ||
112 | // --------------------------------------------------------------------------- | 137 | // --------------------------------------------------------------------------- |
@@ -138,14 +163,42 @@ function buildKey (key: string, bucketInfo: BucketInfo) { | |||
138 | 163 | ||
139 | // --------------------------------------------------------------------------- | 164 | // --------------------------------------------------------------------------- |
140 | 165 | ||
166 | async function createObjectReadStream (options: { | ||
167 | key: string | ||
168 | bucketInfo: BucketInfo | ||
169 | rangeHeader: string | ||
170 | }) { | ||
171 | const { key, bucketInfo, rangeHeader } = options | ||
172 | |||
173 | const command = new GetObjectCommand({ | ||
174 | Bucket: bucketInfo.BUCKET_NAME, | ||
175 | Key: buildKey(key, bucketInfo), | ||
176 | Range: rangeHeader | ||
177 | }) | ||
178 | |||
179 | const response = await getClient().send(command) | ||
180 | |||
181 | return response.Body as Readable | ||
182 | } | ||
183 | |||
184 | // --------------------------------------------------------------------------- | ||
185 | |||
141 | export { | 186 | export { |
142 | BucketInfo, | 187 | BucketInfo, |
143 | buildKey, | 188 | buildKey, |
189 | |||
144 | storeObject, | 190 | storeObject, |
191 | |||
145 | removeObject, | 192 | removeObject, |
146 | removePrefix, | 193 | removePrefix, |
194 | |||
147 | makeAvailable, | 195 | makeAvailable, |
148 | listKeysOfPrefix | 196 | |
197 | updateObjectACL, | ||
198 | updatePrefixACL, | ||
199 | |||
200 | listKeysOfPrefix, | ||
201 | createObjectReadStream | ||
149 | } | 202 | } |
150 | 203 | ||
151 | // --------------------------------------------------------------------------- | 204 | // --------------------------------------------------------------------------- |
@@ -154,17 +207,15 @@ async function uploadToStorage (options: { | |||
154 | content: ReadStream | 207 | content: ReadStream |
155 | objectStorageKey: string | 208 | objectStorageKey: string |
156 | bucketInfo: BucketInfo | 209 | bucketInfo: BucketInfo |
210 | isPrivate: boolean | ||
157 | }) { | 211 | }) { |
158 | const { content, objectStorageKey, bucketInfo } = options | 212 | const { content, objectStorageKey, bucketInfo, isPrivate } = options |
159 | 213 | ||
160 | const input: PutObjectCommandInput = { | 214 | const input: PutObjectCommandInput = { |
161 | Body: content, | 215 | Body: content, |
162 | Bucket: bucketInfo.BUCKET_NAME, | 216 | Bucket: bucketInfo.BUCKET_NAME, |
163 | Key: buildKey(objectStorageKey, bucketInfo) | 217 | Key: buildKey(objectStorageKey, bucketInfo), |
164 | } | 218 | ACL: getACL(isPrivate) |
165 | |||
166 | if (CONFIG.OBJECT_STORAGE.UPLOAD_ACL) { | ||
167 | input.ACL = CONFIG.OBJECT_STORAGE.UPLOAD_ACL | ||
168 | } | 219 | } |
169 | 220 | ||
170 | const parallelUploads3 = new Upload({ | 221 | const parallelUploads3 = new Upload({ |
@@ -194,5 +245,50 @@ async function uploadToStorage (options: { | |||
194 | bucketInfo.PREFIX, objectStorageKey, bucketInfo.BUCKET_NAME, lTags() | 245 | bucketInfo.PREFIX, objectStorageKey, bucketInfo.BUCKET_NAME, lTags() |
195 | ) | 246 | ) |
196 | 247 | ||
197 | return getPrivateUrl(bucketInfo, objectStorageKey) | 248 | return getInternalUrl(bucketInfo, objectStorageKey) |
249 | } | ||
250 | |||
251 | async function applyOnPrefix (options: { | ||
252 | prefix: string | ||
253 | bucketInfo: BucketInfo | ||
254 | commandBuilder: (obj: _Object) => Parameters<S3Client['send']>[0] | ||
255 | |||
256 | continuationToken?: string | ||
257 | }) { | ||
258 | const { prefix, bucketInfo, commandBuilder, continuationToken } = options | ||
259 | |||
260 | const s3Client = getClient() | ||
261 | |||
262 | const commandPrefix = bucketInfo.PREFIX + prefix | ||
263 | const listCommand = new ListObjectsV2Command({ | ||
264 | Bucket: bucketInfo.BUCKET_NAME, | ||
265 | Prefix: commandPrefix, | ||
266 | ContinuationToken: continuationToken | ||
267 | }) | ||
268 | |||
269 | const listedObjects = await s3Client.send(listCommand) | ||
270 | |||
271 | if (isArray(listedObjects.Contents) !== true) { | ||
272 | const message = `Cannot apply function on ${commandPrefix} prefix in bucket ${bucketInfo.BUCKET_NAME}: no files listed.` | ||
273 | |||
274 | logger.error(message, { response: listedObjects, ...lTags() }) | ||
275 | throw new Error(message) | ||
276 | } | ||
277 | |||
278 | for (const object of listedObjects.Contents) { | ||
279 | const command = commandBuilder(object) | ||
280 | |||
281 | await s3Client.send(command) | ||
282 | } | ||
283 | |||
284 | // Repeat if not all objects could be listed at once (limit of 1000?) | ||
285 | if (listedObjects.IsTruncated) { | ||
286 | await applyOnPrefix({ ...options, continuationToken: listedObjects.ContinuationToken }) | ||
287 | } | ||
288 | } | ||
289 | |||
290 | function getACL (isPrivate: boolean) { | ||
291 | return isPrivate | ||
292 | ? CONFIG.OBJECT_STORAGE.UPLOAD_ACL.PRIVATE | ||
293 | : CONFIG.OBJECT_STORAGE.UPLOAD_ACL.PUBLIC | ||
198 | } | 294 | } |
diff --git a/server/lib/object-storage/urls.ts b/server/lib/object-storage/urls.ts index 2a889190b..a47a98b98 100644 --- a/server/lib/object-storage/urls.ts +++ b/server/lib/object-storage/urls.ts | |||
@@ -1,10 +1,14 @@ | |||
1 | import { CONFIG } from '@server/initializers/config' | 1 | import { CONFIG } from '@server/initializers/config' |
2 | import { OBJECT_STORAGE_PROXY_PATHS, WEBSERVER } from '@server/initializers/constants' | ||
3 | import { MVideoUUID } from '@server/types/models' | ||
2 | import { BucketInfo, buildKey, getEndpointParsed } from './shared' | 4 | import { BucketInfo, buildKey, getEndpointParsed } from './shared' |
3 | 5 | ||
4 | function getPrivateUrl (config: BucketInfo, keyWithoutPrefix: string) { | 6 | function getInternalUrl (config: BucketInfo, keyWithoutPrefix: string) { |
5 | return getBaseUrl(config) + buildKey(keyWithoutPrefix, config) | 7 | return getBaseUrl(config) + buildKey(keyWithoutPrefix, config) |
6 | } | 8 | } |
7 | 9 | ||
10 | // --------------------------------------------------------------------------- | ||
11 | |||
8 | function getWebTorrentPublicFileUrl (fileUrl: string) { | 12 | function getWebTorrentPublicFileUrl (fileUrl: string) { |
9 | const baseUrl = CONFIG.OBJECT_STORAGE.VIDEOS.BASE_URL | 13 | const baseUrl = CONFIG.OBJECT_STORAGE.VIDEOS.BASE_URL |
10 | if (!baseUrl) return fileUrl | 14 | if (!baseUrl) return fileUrl |
@@ -19,11 +23,28 @@ function getHLSPublicFileUrl (fileUrl: string) { | |||
19 | return replaceByBaseUrl(fileUrl, baseUrl) | 23 | return replaceByBaseUrl(fileUrl, baseUrl) |
20 | } | 24 | } |
21 | 25 | ||
26 | // --------------------------------------------------------------------------- | ||
27 | |||
28 | function getHLSPrivateFileUrl (video: MVideoUUID, filename: string) { | ||
29 | return WEBSERVER.URL + OBJECT_STORAGE_PROXY_PATHS.STREAMING_PLAYLISTS.PRIVATE_HLS + video.uuid + `/${filename}` | ||
30 | } | ||
31 | |||
32 | function getWebTorrentPrivateFileUrl (filename: string) { | ||
33 | return WEBSERVER.URL + OBJECT_STORAGE_PROXY_PATHS.PRIVATE_WEBSEED + filename | ||
34 | } | ||
35 | |||
36 | // --------------------------------------------------------------------------- | ||
37 | |||
22 | export { | 38 | export { |
23 | getPrivateUrl, | 39 | getInternalUrl, |
40 | |||
24 | getWebTorrentPublicFileUrl, | 41 | getWebTorrentPublicFileUrl, |
25 | replaceByBaseUrl, | 42 | getHLSPublicFileUrl, |
26 | getHLSPublicFileUrl | 43 | |
44 | getHLSPrivateFileUrl, | ||
45 | getWebTorrentPrivateFileUrl, | ||
46 | |||
47 | replaceByBaseUrl | ||
27 | } | 48 | } |
28 | 49 | ||
29 | // --------------------------------------------------------------------------- | 50 | // --------------------------------------------------------------------------- |
diff --git a/server/lib/object-storage/videos.ts b/server/lib/object-storage/videos.ts index e323baaa2..003807826 100644 --- a/server/lib/object-storage/videos.ts +++ b/server/lib/object-storage/videos.ts | |||
@@ -5,7 +5,17 @@ import { MStreamingPlaylistVideo, MVideo, MVideoFile } from '@server/types/model | |||
5 | import { getHLSDirectory } from '../paths' | 5 | import { getHLSDirectory } from '../paths' |
6 | import { VideoPathManager } from '../video-path-manager' | 6 | import { VideoPathManager } from '../video-path-manager' |
7 | import { generateHLSObjectBaseStorageKey, generateHLSObjectStorageKey, generateWebTorrentObjectStorageKey } from './keys' | 7 | import { generateHLSObjectBaseStorageKey, generateHLSObjectStorageKey, generateWebTorrentObjectStorageKey } from './keys' |
8 | import { listKeysOfPrefix, lTags, makeAvailable, removeObject, removePrefix, storeObject } from './shared' | 8 | import { |
9 | createObjectReadStream, | ||
10 | listKeysOfPrefix, | ||
11 | lTags, | ||
12 | makeAvailable, | ||
13 | removeObject, | ||
14 | removePrefix, | ||
15 | storeObject, | ||
16 | updateObjectACL, | ||
17 | updatePrefixACL | ||
18 | } from './shared' | ||
9 | 19 | ||
10 | function listHLSFileKeysOf (playlist: MStreamingPlaylistVideo) { | 20 | function listHLSFileKeysOf (playlist: MStreamingPlaylistVideo) { |
11 | return listKeysOfPrefix(generateHLSObjectBaseStorageKey(playlist), CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS) | 21 | return listKeysOfPrefix(generateHLSObjectBaseStorageKey(playlist), CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS) |
@@ -17,7 +27,8 @@ function storeHLSFileFromFilename (playlist: MStreamingPlaylistVideo, filename: | |||
17 | return storeObject({ | 27 | return storeObject({ |
18 | inputPath: join(getHLSDirectory(playlist.Video), filename), | 28 | inputPath: join(getHLSDirectory(playlist.Video), filename), |
19 | objectStorageKey: generateHLSObjectStorageKey(playlist, filename), | 29 | objectStorageKey: generateHLSObjectStorageKey(playlist, filename), |
20 | bucketInfo: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS | 30 | bucketInfo: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS, |
31 | isPrivate: playlist.Video.hasPrivateStaticPath() | ||
21 | }) | 32 | }) |
22 | } | 33 | } |
23 | 34 | ||
@@ -25,7 +36,8 @@ function storeHLSFileFromPath (playlist: MStreamingPlaylistVideo, path: string) | |||
25 | return storeObject({ | 36 | return storeObject({ |
26 | inputPath: path, | 37 | inputPath: path, |
27 | objectStorageKey: generateHLSObjectStorageKey(playlist, basename(path)), | 38 | objectStorageKey: generateHLSObjectStorageKey(playlist, basename(path)), |
28 | bucketInfo: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS | 39 | bucketInfo: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS, |
40 | isPrivate: playlist.Video.hasPrivateStaticPath() | ||
29 | }) | 41 | }) |
30 | } | 42 | } |
31 | 43 | ||
@@ -35,7 +47,26 @@ function storeWebTorrentFile (video: MVideo, file: MVideoFile) { | |||
35 | return storeObject({ | 47 | return storeObject({ |
36 | inputPath: VideoPathManager.Instance.getFSVideoFileOutputPath(video, file), | 48 | inputPath: VideoPathManager.Instance.getFSVideoFileOutputPath(video, file), |
37 | objectStorageKey: generateWebTorrentObjectStorageKey(file.filename), | 49 | objectStorageKey: generateWebTorrentObjectStorageKey(file.filename), |
38 | bucketInfo: CONFIG.OBJECT_STORAGE.VIDEOS | 50 | bucketInfo: CONFIG.OBJECT_STORAGE.VIDEOS, |
51 | isPrivate: video.hasPrivateStaticPath() | ||
52 | }) | ||
53 | } | ||
54 | |||
55 | // --------------------------------------------------------------------------- | ||
56 | |||
57 | function updateWebTorrentFileACL (video: MVideo, file: MVideoFile) { | ||
58 | return updateObjectACL({ | ||
59 | objectStorageKey: generateWebTorrentObjectStorageKey(file.filename), | ||
60 | bucketInfo: CONFIG.OBJECT_STORAGE.VIDEOS, | ||
61 | isPrivate: video.hasPrivateStaticPath() | ||
62 | }) | ||
63 | } | ||
64 | |||
65 | function updateHLSFilesACL (playlist: MStreamingPlaylistVideo) { | ||
66 | return updatePrefixACL({ | ||
67 | prefix: generateHLSObjectBaseStorageKey(playlist), | ||
68 | bucketInfo: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS, | ||
69 | isPrivate: playlist.Video.hasPrivateStaticPath() | ||
39 | }) | 70 | }) |
40 | } | 71 | } |
41 | 72 | ||
@@ -87,6 +118,39 @@ async function makeWebTorrentFileAvailable (filename: string, destination: strin | |||
87 | 118 | ||
88 | // --------------------------------------------------------------------------- | 119 | // --------------------------------------------------------------------------- |
89 | 120 | ||
121 | function getWebTorrentFileReadStream (options: { | ||
122 | filename: string | ||
123 | rangeHeader: string | ||
124 | }) { | ||
125 | const { filename, rangeHeader } = options | ||
126 | |||
127 | const key = generateWebTorrentObjectStorageKey(filename) | ||
128 | |||
129 | return createObjectReadStream({ | ||
130 | key, | ||
131 | bucketInfo: CONFIG.OBJECT_STORAGE.VIDEOS, | ||
132 | rangeHeader | ||
133 | }) | ||
134 | } | ||
135 | |||
136 | function getHLSFileReadStream (options: { | ||
137 | playlist: MStreamingPlaylistVideo | ||
138 | filename: string | ||
139 | rangeHeader: string | ||
140 | }) { | ||
141 | const { playlist, filename, rangeHeader } = options | ||
142 | |||
143 | const key = generateHLSObjectStorageKey(playlist, filename) | ||
144 | |||
145 | return createObjectReadStream({ | ||
146 | key, | ||
147 | bucketInfo: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS, | ||
148 | rangeHeader | ||
149 | }) | ||
150 | } | ||
151 | |||
152 | // --------------------------------------------------------------------------- | ||
153 | |||
90 | export { | 154 | export { |
91 | listHLSFileKeysOf, | 155 | listHLSFileKeysOf, |
92 | 156 | ||
@@ -94,10 +158,16 @@ export { | |||
94 | storeHLSFileFromFilename, | 158 | storeHLSFileFromFilename, |
95 | storeHLSFileFromPath, | 159 | storeHLSFileFromPath, |
96 | 160 | ||
161 | updateWebTorrentFileACL, | ||
162 | updateHLSFilesACL, | ||
163 | |||
97 | removeHLSObjectStorage, | 164 | removeHLSObjectStorage, |
98 | removeHLSFileObjectStorage, | 165 | removeHLSFileObjectStorage, |
99 | removeWebTorrentObjectStorage, | 166 | removeWebTorrentObjectStorage, |
100 | 167 | ||
101 | makeWebTorrentFileAvailable, | 168 | makeWebTorrentFileAvailable, |
102 | makeHLSFileAvailable | 169 | makeHLSFileAvailable, |
170 | |||
171 | getWebTorrentFileReadStream, | ||
172 | getHLSFileReadStream | ||
103 | } | 173 | } |
diff --git a/server/lib/video-privacy.ts b/server/lib/video-privacy.ts index 1a4a5a22d..41f9d62b3 100644 --- a/server/lib/video-privacy.ts +++ b/server/lib/video-privacy.ts | |||
@@ -2,8 +2,9 @@ import { move } from 'fs-extra' | |||
2 | import { join } from 'path' | 2 | import { join } from 'path' |
3 | import { logger } from '@server/helpers/logger' | 3 | import { logger } from '@server/helpers/logger' |
4 | import { DIRECTORIES } from '@server/initializers/constants' | 4 | import { DIRECTORIES } from '@server/initializers/constants' |
5 | import { MVideo, MVideoFullLight } from '@server/types/models' | 5 | import { MVideo, MVideoFile, MVideoFullLight } from '@server/types/models' |
6 | import { VideoPrivacy } from '@shared/models' | 6 | import { VideoPrivacy, VideoStorage } from '@shared/models' |
7 | import { updateHLSFilesACL, updateWebTorrentFileACL } from './object-storage' | ||
7 | 8 | ||
8 | function setVideoPrivacy (video: MVideo, newPrivacy: VideoPrivacy) { | 9 | function setVideoPrivacy (video: MVideo, newPrivacy: VideoPrivacy) { |
9 | if (video.privacy === VideoPrivacy.PRIVATE && newPrivacy !== VideoPrivacy.PRIVATE) { | 10 | if (video.privacy === VideoPrivacy.PRIVATE && newPrivacy !== VideoPrivacy.PRIVATE) { |
@@ -50,47 +51,77 @@ export { | |||
50 | 51 | ||
51 | // --------------------------------------------------------------------------- | 52 | // --------------------------------------------------------------------------- |
52 | 53 | ||
54 | type MoveType = 'private-to-public' | 'public-to-private' | ||
55 | |||
53 | async function moveFiles (options: { | 56 | async function moveFiles (options: { |
54 | type: 'private-to-public' | 'public-to-private' | 57 | type: MoveType |
55 | video: MVideoFullLight | 58 | video: MVideoFullLight |
56 | }) { | 59 | }) { |
57 | const { type, video } = options | 60 | const { type, video } = options |
58 | 61 | ||
59 | const directories = type === 'private-to-public' | 62 | for (const file of video.VideoFiles) { |
60 | ? { | 63 | if (file.storage === VideoStorage.FILE_SYSTEM) { |
61 | webtorrent: { old: DIRECTORIES.VIDEOS.PRIVATE, new: DIRECTORIES.VIDEOS.PUBLIC }, | 64 | await moveWebTorrentFileOnFS(type, video, file) |
62 | hls: { old: DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE, new: DIRECTORIES.HLS_STREAMING_PLAYLIST.PUBLIC } | 65 | } else { |
66 | await updateWebTorrentFileACL(video, file) | ||
63 | } | 67 | } |
64 | : { | 68 | } |
65 | webtorrent: { old: DIRECTORIES.VIDEOS.PUBLIC, new: DIRECTORIES.VIDEOS.PRIVATE }, | 69 | |
66 | hls: { old: DIRECTORIES.HLS_STREAMING_PLAYLIST.PUBLIC, new: DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE } | 70 | const hls = video.getHLSPlaylist() |
71 | |||
72 | if (hls) { | ||
73 | if (hls.storage === VideoStorage.FILE_SYSTEM) { | ||
74 | await moveHLSFilesOnFS(type, video) | ||
75 | } else { | ||
76 | await updateHLSFilesACL(hls) | ||
67 | } | 77 | } |
78 | } | ||
79 | } | ||
68 | 80 | ||
69 | for (const file of video.VideoFiles) { | 81 | async function moveWebTorrentFileOnFS (type: MoveType, video: MVideo, file: MVideoFile) { |
70 | const source = join(directories.webtorrent.old, file.filename) | 82 | const directories = getWebTorrentDirectories(type) |
71 | const destination = join(directories.webtorrent.new, file.filename) | ||
72 | 83 | ||
73 | try { | 84 | const source = join(directories.old, file.filename) |
74 | logger.info('Moving WebTorrent files of %s after privacy change (%s -> %s).', video.uuid, source, destination) | 85 | const destination = join(directories.new, file.filename) |
75 | 86 | ||
76 | await move(source, destination) | 87 | try { |
77 | } catch (err) { | 88 | logger.info('Moving WebTorrent files of %s after privacy change (%s -> %s).', video.uuid, source, destination) |
78 | logger.error('Cannot move webtorrent file %s to %s after privacy change', source, destination, { err }) | 89 | |
79 | } | 90 | await move(source, destination) |
91 | } catch (err) { | ||
92 | logger.error('Cannot move webtorrent file %s to %s after privacy change', source, destination, { err }) | ||
93 | } | ||
94 | } | ||
95 | |||
96 | function getWebTorrentDirectories (moveType: MoveType) { | ||
97 | if (moveType === 'private-to-public') { | ||
98 | return { old: DIRECTORIES.VIDEOS.PRIVATE, new: DIRECTORIES.VIDEOS.PUBLIC } | ||
80 | } | 99 | } |
81 | 100 | ||
82 | const hls = video.getHLSPlaylist() | 101 | return { old: DIRECTORIES.VIDEOS.PUBLIC, new: DIRECTORIES.VIDEOS.PRIVATE } |
102 | } | ||
83 | 103 | ||
84 | if (hls) { | 104 | // --------------------------------------------------------------------------- |
85 | const source = join(directories.hls.old, video.uuid) | ||
86 | const destination = join(directories.hls.new, video.uuid) | ||
87 | 105 | ||
88 | try { | 106 | async function moveHLSFilesOnFS (type: MoveType, video: MVideo) { |
89 | logger.info('Moving HLS files of %s after privacy change (%s -> %s).', video.uuid, source, destination) | 107 | const directories = getHLSDirectories(type) |
90 | 108 | ||
91 | await move(source, destination) | 109 | const source = join(directories.old, video.uuid) |
92 | } catch (err) { | 110 | const destination = join(directories.new, video.uuid) |
93 | logger.error('Cannot move HLS file %s to %s after privacy change', source, destination, { err }) | 111 | |
94 | } | 112 | try { |
113 | logger.info('Moving HLS files of %s after privacy change (%s -> %s).', video.uuid, source, destination) | ||
114 | |||
115 | await move(source, destination) | ||
116 | } catch (err) { | ||
117 | logger.error('Cannot move HLS file %s to %s after privacy change', source, destination, { err }) | ||
118 | } | ||
119 | } | ||
120 | |||
121 | function getHLSDirectories (moveType: MoveType) { | ||
122 | if (moveType === 'private-to-public') { | ||
123 | return { old: DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE, new: DIRECTORIES.HLS_STREAMING_PLAYLIST.PUBLIC } | ||
95 | } | 124 | } |
125 | |||
126 | return { old: DIRECTORIES.HLS_STREAMING_PLAYLIST.PUBLIC, new: DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE } | ||
96 | } | 127 | } |