3 files changed, 325 insertions, 65 deletions

diff --git a/server/lib/object-storage/shared/object-storage-helpers.ts b/server/lib/object-storage/shared/object-storage-helpers.ts
index 16161362c..3046d76bc 100644
--- a/server/lib/object-storage/shared/object-storage-helpers.ts
+++ b/server/lib/object-storage/shared/object-storage-helpers.ts
@@ -1,19 +1,23 @@
+import { map } from 'bluebird'
 import { createReadStream, createWriteStream, ensureDir, ReadStream } from 'fs-extra'
 import { dirname } from 'path'
 import { Readable } from 'stream'
 import {
+  _Object,
   CompleteMultipartUploadCommandOutput,
   DeleteObjectCommand,
   GetObjectCommand,
   ListObjectsV2Command,
-  PutObjectCommandInput
+  PutObjectAclCommand,
+  PutObjectCommandInput,
+  S3Client
 } from '@aws-sdk/client-s3'
 import { Upload } from '@aws-sdk/lib-storage'
 import { pipelinePromise } from '@server/helpers/core-utils'
 import { isArray } from '@server/helpers/custom-validators/misc'
 import { logger } from '@server/helpers/logger'
 import { CONFIG } from '@server/initializers/config'
-import { getPrivateUrl } from '../urls'
+import { getInternalUrl } from '../urls'
 import { getClient } from './client'
 import { lTags } from './logger'
 
@@ -22,73 +26,125 @@ type BucketInfo = {
   PREFIX?: string
 }
 
+async function listKeysOfPrefix (prefix: string, bucketInfo: BucketInfo) {
+  const s3Client = getClient()
+
+  const commandPrefix = bucketInfo.PREFIX + prefix
+  const listCommand = new ListObjectsV2Command({
+    Bucket: bucketInfo.BUCKET_NAME,
+    Prefix: commandPrefix
+  })
+
+  const listedObjects = await s3Client.send(listCommand)
+
+  if (isArray(listedObjects.Contents) !== true) return []
+
+  return listedObjects.Contents.map(c => c.Key)
+}
+
+// ---------------------------------------------------------------------------
+
 async function storeObject (options: {
   inputPath: string
   objectStorageKey: string
   bucketInfo: BucketInfo
+  isPrivate: boolean
 }): Promise<string> {
-  const { inputPath, objectStorageKey, bucketInfo } = options
+  const { inputPath, objectStorageKey, bucketInfo, isPrivate } = options
 
   logger.debug('Uploading file %s to %s%s in bucket %s', inputPath, bucketInfo.PREFIX, objectStorageKey, bucketInfo.BUCKET_NAME, lTags())
 
   const fileStream = createReadStream(inputPath)
 
-  return uploadToStorage({ objectStorageKey, content: fileStream, bucketInfo })
+  return uploadToStorage({ objectStorageKey, content: fileStream, bucketInfo, isPrivate })
 }
 
-async function removeObject (filename: string, bucketInfo: BucketInfo) {
-  const command = new DeleteObjectCommand({
+// ---------------------------------------------------------------------------
+
+function updateObjectACL (options: {
+  objectStorageKey: string
+  bucketInfo: BucketInfo
+  isPrivate: boolean
+}) {
+  const { objectStorageKey, bucketInfo, isPrivate } = options
+
+  const key = buildKey(objectStorageKey, bucketInfo)
+
+  logger.debug('Updating ACL file %s in bucket %s', key, bucketInfo.BUCKET_NAME, lTags())
+
+  const command = new PutObjectAclCommand({
     Bucket: bucketInfo.BUCKET_NAME,
-    Key: buildKey(filename, bucketInfo)
+    Key: key,
+    ACL: getACL(isPrivate)
   })
 
   return getClient().send(command)
 }
 
-async function removePrefix (prefix: string, bucketInfo: BucketInfo) {
-  const s3Client = getClient()
-
-  const commandPrefix = bucketInfo.PREFIX + prefix
-  const listCommand = new ListObjectsV2Command({
-    Bucket: bucketInfo.BUCKET_NAME,
-    Prefix: commandPrefix
+function updatePrefixACL (options: {
+  prefix: string
+  bucketInfo: BucketInfo
+  isPrivate: boolean
+}) {
+  const { prefix, bucketInfo, isPrivate } = options
+
+  logger.debug('Updating ACL of files in prefix %s in bucket %s', prefix, bucketInfo.BUCKET_NAME, lTags())
+
+  return applyOnPrefix({
+    prefix,
+    bucketInfo,
+    commandBuilder: obj => {
+      logger.debug('Updating ACL of %s inside prefix %s in bucket %s', obj.Key, prefix, bucketInfo.BUCKET_NAME, lTags())
+
+      return new PutObjectAclCommand({
+        Bucket: bucketInfo.BUCKET_NAME,
+        Key: obj.Key,
+        ACL: getACL(isPrivate)
+      })
+    }
   })
+}
 
-  const listedObjects = await s3Client.send(listCommand)
-
-  // FIXME: use bulk delete when s3ninja will support this operation
-  // const deleteParams = {
-  //   Bucket: bucketInfo.BUCKET_NAME,
-  //   Delete: { Objects: [] }
-  // }
+// ---------------------------------------------------------------------------
 
-  if (isArray(listedObjects.Contents) !== true) {
-    const message = `Cannot remove ${commandPrefix} prefix in bucket ${bucketInfo.BUCKET_NAME}: no files listed.`
+function removeObject (objectStorageKey: string, bucketInfo: BucketInfo) {
+  const key = buildKey(objectStorageKey, bucketInfo)
 
-    logger.error(message, { response: listedObjects, ...lTags() })
-    throw new Error(message)
-  }
+  return removeObjectByFullKey(key, bucketInfo)
+}
 
-  for (const object of listedObjects.Contents) {
-    const command = new DeleteObjectCommand({
-      Bucket: bucketInfo.BUCKET_NAME,
-      Key: object.Key
-    })
+function removeObjectByFullKey (fullKey: string, bucketInfo: BucketInfo) {
+  logger.debug('Removing file %s in bucket %s', fullKey, bucketInfo.BUCKET_NAME, lTags())
 
-    await s3Client.send(command)
+  const command = new DeleteObjectCommand({
+    Bucket: bucketInfo.BUCKET_NAME,
+    Key: fullKey
+  })
 
-    // FIXME: use bulk delete when s3ninja will support this operation
-    // deleteParams.Delete.Objects.push({ Key: object.Key })
-  }
+  return getClient().send(command)
+}
 
+async function removePrefix (prefix: string, bucketInfo: BucketInfo) {
   // FIXME: use bulk delete when s3ninja will support this operation
-  // const deleteCommand = new DeleteObjectsCommand(deleteParams)
-  // await s3Client.send(deleteCommand)
 
-  // Repeat if not all objects could be listed at once (limit of 1000?)
-  if (listedObjects.IsTruncated) await removePrefix(prefix, bucketInfo)
+  logger.debug('Removing prefix %s in bucket %s', prefix, bucketInfo.BUCKET_NAME, lTags())
+
+  return applyOnPrefix({
+    prefix,
+    bucketInfo,
+    commandBuilder: obj => {
+      logger.debug('Removing %s inside prefix %s in bucket %s', obj.Key, prefix, bucketInfo.BUCKET_NAME, lTags())
+
+      return new DeleteObjectCommand({
+        Bucket: bucketInfo.BUCKET_NAME,
+        Key: obj.Key
+      })
+    }
+  })
 }
 
+// ---------------------------------------------------------------------------
+
 async function makeAvailable (options: {
   key: string
   destination: string
@@ -116,13 +172,43 @@ function buildKey (key: string, bucketInfo: BucketInfo) {
 
 // ---------------------------------------------------------------------------
 
+async function createObjectReadStream (options: {
+  key: string
+  bucketInfo: BucketInfo
+  rangeHeader: string
+}) {
+  const { key, bucketInfo, rangeHeader } = options
+
+  const command = new GetObjectCommand({
+    Bucket: bucketInfo.BUCKET_NAME,
+    Key: buildKey(key, bucketInfo),
+    Range: rangeHeader
+  })
+
+  const response = await getClient().send(command)
+
+  return response.Body as Readable
+}
+
+// ---------------------------------------------------------------------------
+
 export {
   BucketInfo,
   buildKey,
+
   storeObject,
+
   removeObject,
+  removeObjectByFullKey,
   removePrefix,
-  makeAvailable
+
+  makeAvailable,
+
+  updateObjectACL,
+  updatePrefixACL,
+
+  listKeysOfPrefix,
+  createObjectReadStream
 }
 
 // ---------------------------------------------------------------------------
@@ -131,17 +217,15 @@ async function uploadToStorage (options: {
   content: ReadStream
   objectStorageKey: string
   bucketInfo: BucketInfo
+  isPrivate: boolean
 }) {
-  const { content, objectStorageKey, bucketInfo } = options
+  const { content, objectStorageKey, bucketInfo, isPrivate } = options
 
   const input: PutObjectCommandInput = {
     Body: content,
     Bucket: bucketInfo.BUCKET_NAME,
-    Key: buildKey(objectStorageKey, bucketInfo)
-  }
-
-  if (CONFIG.OBJECT_STORAGE.UPLOAD_ACL) {
-    input.ACL = CONFIG.OBJECT_STORAGE.UPLOAD_ACL
+    Key: buildKey(objectStorageKey, bucketInfo),
+    ACL: getACL(isPrivate)
   }
 
   const parallelUploads3 = new Upload({
@@ -171,5 +255,50 @@ async function uploadToStorage (options: {
     bucketInfo.PREFIX, objectStorageKey, bucketInfo.BUCKET_NAME, lTags()
   )
 
-  return getPrivateUrl(bucketInfo, objectStorageKey)
+  return getInternalUrl(bucketInfo, objectStorageKey)
+}
+
+async function applyOnPrefix (options: {
+  prefix: string
+  bucketInfo: BucketInfo
+  commandBuilder: (obj: _Object) => Parameters<S3Client['send']>[0]
+
+  continuationToken?: string
+}) {
+  const { prefix, bucketInfo, commandBuilder, continuationToken } = options
+
+  const s3Client = getClient()
+
+  const commandPrefix = buildKey(prefix, bucketInfo)
+  const listCommand = new ListObjectsV2Command({
+    Bucket: bucketInfo.BUCKET_NAME,
+    Prefix: commandPrefix,
+    ContinuationToken: continuationToken
+  })
+
+  const listedObjects = await s3Client.send(listCommand)
+
+  if (isArray(listedObjects.Contents) !== true) {
+    const message = `Cannot apply function on ${commandPrefix} prefix in bucket ${bucketInfo.BUCKET_NAME}: no files listed.`
+
+    logger.error(message, { response: listedObjects, ...lTags() })
+    throw new Error(message)
+  }
+
+  await map(listedObjects.Contents, object => {
+    const command = commandBuilder(object)
+
+    return s3Client.send(command)
+  }, { concurrency: 10 })
+
+  // Repeat if not all objects could be listed at once (limit of 1000?)
+  if (listedObjects.IsTruncated) {
+    await applyOnPrefix({ ...options, continuationToken: listedObjects.ContinuationToken })
+  }
+}
+
+function getACL (isPrivate: boolean) {
+  return isPrivate
+    ? CONFIG.OBJECT_STORAGE.UPLOAD_ACL.PRIVATE
+    : CONFIG.OBJECT_STORAGE.UPLOAD_ACL.PUBLIC
 }
diff --git a/server/lib/object-storage/urls.ts b/server/lib/object-storage/urls.ts
index 2a889190b..a47a98b98 100644
--- a/server/lib/object-storage/urls.ts
+++ b/server/lib/object-storage/urls.ts
@@ -1,10 +1,14 @@
 import { CONFIG } from '@server/initializers/config'
+import { OBJECT_STORAGE_PROXY_PATHS, WEBSERVER } from '@server/initializers/constants'
+import { MVideoUUID } from '@server/types/models'
 import { BucketInfo, buildKey, getEndpointParsed } from './shared'
 
-function getPrivateUrl (config: BucketInfo, keyWithoutPrefix: string) {
+function getInternalUrl (config: BucketInfo, keyWithoutPrefix: string) {
   return getBaseUrl(config) + buildKey(keyWithoutPrefix, config)
 }
 
+// ---------------------------------------------------------------------------
+
 function getWebTorrentPublicFileUrl (fileUrl: string) {
   const baseUrl = CONFIG.OBJECT_STORAGE.VIDEOS.BASE_URL
   if (!baseUrl) return fileUrl
@@ -19,11 +23,28 @@ function getHLSPublicFileUrl (fileUrl: string) {
   return replaceByBaseUrl(fileUrl, baseUrl)
 }
 
+// ---------------------------------------------------------------------------
+
+function getHLSPrivateFileUrl (video: MVideoUUID, filename: string) {
+  return WEBSERVER.URL + OBJECT_STORAGE_PROXY_PATHS.STREAMING_PLAYLISTS.PRIVATE_HLS + video.uuid + `/${filename}`
+}
+
+function getWebTorrentPrivateFileUrl (filename: string) {
+  return WEBSERVER.URL + OBJECT_STORAGE_PROXY_PATHS.PRIVATE_WEBSEED + filename
+}
+
+// ---------------------------------------------------------------------------
+
 export {
-  getPrivateUrl,
+  getInternalUrl,
+
   getWebTorrentPublicFileUrl,
-  replaceByBaseUrl,
-  getHLSPublicFileUrl
+  getHLSPublicFileUrl,
+
+  getHLSPrivateFileUrl,
+  getWebTorrentPrivateFileUrl,
+
+  replaceByBaseUrl
 }
 
 // ---------------------------------------------------------------------------
diff --git a/server/lib/object-storage/videos.ts b/server/lib/object-storage/videos.ts
index 66e738200..b764e4b22 100644
--- a/server/lib/object-storage/videos.ts
+++ b/server/lib/object-storage/videos.ts
@@ -1,39 +1,102 @@
-import { join } from 'path'
+import { basename, join } from 'path'
 import { logger } from '@server/helpers/logger'
 import { CONFIG } from '@server/initializers/config'
-import { MStreamingPlaylistVideo, MVideoFile } from '@server/types/models'
+import { MStreamingPlaylistVideo, MVideo, MVideoFile } from '@server/types/models'
 import { getHLSDirectory } from '../paths'
+import { VideoPathManager } from '../video-path-manager'
 import { generateHLSObjectBaseStorageKey, generateHLSObjectStorageKey, generateWebTorrentObjectStorageKey } from './keys'
-import { lTags, makeAvailable, removeObject, removePrefix, storeObject } from './shared'
+import {
+  createObjectReadStream,
+  listKeysOfPrefix,
+  lTags,
+  makeAvailable,
+  removeObject,
+  removeObjectByFullKey,
+  removePrefix,
+  storeObject,
+  updateObjectACL,
+  updatePrefixACL
+} from './shared'
+
+function listHLSFileKeysOf (playlist: MStreamingPlaylistVideo) {
+  return listKeysOfPrefix(generateHLSObjectBaseStorageKey(playlist), CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS)
+}
+
+// ---------------------------------------------------------------------------
 
-function storeHLSFile (playlist: MStreamingPlaylistVideo, filename: string, path?: string) {
+function storeHLSFileFromFilename (playlist: MStreamingPlaylistVideo, filename: string) {
   return storeObject({
-    inputPath: path ?? join(getHLSDirectory(playlist.Video), filename),
+    inputPath: join(getHLSDirectory(playlist.Video), filename),
     objectStorageKey: generateHLSObjectStorageKey(playlist, filename),
-    bucketInfo: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS
+    bucketInfo: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS,
+    isPrivate: playlist.Video.hasPrivateStaticPath()
   })
 }
 
-function storeWebTorrentFile (filename: string) {
+function storeHLSFileFromPath (playlist: MStreamingPlaylistVideo, path: string) {
   return storeObject({
-    inputPath: join(CONFIG.STORAGE.VIDEOS_DIR, filename),
-    objectStorageKey: generateWebTorrentObjectStorageKey(filename),
-    bucketInfo: CONFIG.OBJECT_STORAGE.VIDEOS
+    inputPath: path,
+    objectStorageKey: generateHLSObjectStorageKey(playlist, basename(path)),
+    bucketInfo: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS,
+    isPrivate: playlist.Video.hasPrivateStaticPath()
+  })
+}
+
+// ---------------------------------------------------------------------------
+
+function storeWebTorrentFile (video: MVideo, file: MVideoFile) {
+  return storeObject({
+    inputPath: VideoPathManager.Instance.getFSVideoFileOutputPath(video, file),
+    objectStorageKey: generateWebTorrentObjectStorageKey(file.filename),
+    bucketInfo: CONFIG.OBJECT_STORAGE.VIDEOS,
+    isPrivate: video.hasPrivateStaticPath()
+  })
+}
+
+// ---------------------------------------------------------------------------
+
+function updateWebTorrentFileACL (video: MVideo, file: MVideoFile) {
+  return updateObjectACL({
+    objectStorageKey: generateWebTorrentObjectStorageKey(file.filename),
+    bucketInfo: CONFIG.OBJECT_STORAGE.VIDEOS,
+    isPrivate: video.hasPrivateStaticPath()
+  })
+}
+
+function updateHLSFilesACL (playlist: MStreamingPlaylistVideo) {
+  return updatePrefixACL({
+    prefix: generateHLSObjectBaseStorageKey(playlist),
+    bucketInfo: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS,
+    isPrivate: playlist.Video.hasPrivateStaticPath()
   })
 }
 
+// ---------------------------------------------------------------------------
+
 function removeHLSObjectStorage (playlist: MStreamingPlaylistVideo) {
   return removePrefix(generateHLSObjectBaseStorageKey(playlist), CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS)
 }
 
-function removeHLSFileObjectStorage (playlist: MStreamingPlaylistVideo, filename: string) {
+function removeHLSFileObjectStorageByFilename (playlist: MStreamingPlaylistVideo, filename: string) {
   return removeObject(generateHLSObjectStorageKey(playlist, filename), CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS)
 }
 
+function removeHLSFileObjectStorageByPath (playlist: MStreamingPlaylistVideo, path: string) {
+  return removeObject(generateHLSObjectStorageKey(playlist, basename(path)), CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS)
+}
+
+function removeHLSFileObjectStorageByFullKey (key: string) {
+  return removeObjectByFullKey(key, CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS)
+}
+
+// ---------------------------------------------------------------------------
+
 function removeWebTorrentObjectStorage (videoFile: MVideoFile) {
   return removeObject(generateWebTorrentObjectStorageKey(videoFile.filename), CONFIG.OBJECT_STORAGE.VIDEOS)
 }
 
+// ---------------------------------------------------------------------------
+
 async function makeHLSFileAvailable (playlist: MStreamingPlaylistVideo, filename: string, destination: string) {
   const key = generateHLSObjectStorageKey(playlist, filename)
 
@@ -62,14 +125,61 @@ async function makeWebTorrentFileAvailable (filename: string, destination: strin
   return destination
 }
 
+// ---------------------------------------------------------------------------
+
+function getWebTorrentFileReadStream (options: {
+  filename: string
+  rangeHeader: string
+}) {
+  const { filename, rangeHeader } = options
+
+  const key = generateWebTorrentObjectStorageKey(filename)
+
+  return createObjectReadStream({
+    key,
+    bucketInfo: CONFIG.OBJECT_STORAGE.VIDEOS,
+    rangeHeader
+  })
+}
+
+function getHLSFileReadStream (options: {
+  playlist: MStreamingPlaylistVideo
+  filename: string
+  rangeHeader: string
+}) {
+  const { playlist, filename, rangeHeader } = options
+
+  const key = generateHLSObjectStorageKey(playlist, filename)
+
+  return createObjectReadStream({
+    key,
+    bucketInfo: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS,
+    rangeHeader
+  })
+}
+
+// ---------------------------------------------------------------------------
+
 export {
+  listHLSFileKeysOf,
+
   storeWebTorrentFile,
-  storeHLSFile,
+  storeHLSFileFromFilename,
+  storeHLSFileFromPath,
+
+  updateWebTorrentFileACL,
+  updateHLSFilesACL,
 
   removeHLSObjectStorage,
-  removeHLSFileObjectStorage,
+  removeHLSFileObjectStorageByFilename,
+  removeHLSFileObjectStorageByPath,
+  removeHLSFileObjectStorageByFullKey,
+
   removeWebTorrentObjectStorage,
 
   makeWebTorrentFileAvailable,
-  makeHLSFileAvailable
+  makeHLSFileAvailable,
+
+  getWebTorrentFileReadStream,
+  getHLSFileReadStream
 }