Use private ACL for private videos in s3

6 files changed, 106 insertions, 33 deletions

diff --git a/shared/core-utils/common/env.ts b/shared/core-utils/common/env.ts
index 38c96b152..973f895d4 100644
--- a/shared/core-utils/common/env.ts
+++ b/shared/core-utils/common/env.ts
@@ -14,7 +14,7 @@ function areHttpImportTestsDisabled () {
   return disabled
 }
 
-function areObjectStorageTestsDisabled () {
+function areMockObjectStorageTestsDisabled () {
   const disabled = process.env.ENABLE_OBJECT_STORAGE_TESTS !== 'true'
 
   if (disabled) console.log('ENABLE_OBJECT_STORAGE_TESTS env is not set to "true" so object storage tests are disabled')
@@ -22,9 +22,25 @@ function areObjectStorageTestsDisabled () {
   return disabled
 }
 
+function areScalewayObjectStorageTestsDisabled () {
+  if (areMockObjectStorageTestsDisabled()) return true
+
+  const enabled = process.env.OBJECT_STORAGE_SCALEWAY_KEY_ID && process.env.OBJECT_STORAGE_SCALEWAY_ACCESS_KEY
+  if (!enabled) {
+    console.log(
+      'OBJECT_STORAGE_SCALEWAY_KEY_ID and/or OBJECT_STORAGE_SCALEWAY_ACCESS_KEY are not set, so scaleway object storage tests are disabled'
+    )
+
+    return true
+  }
+
+  return false
+}
+
 export {
   parallelTests,
   isGithubCI,
   areHttpImportTestsDisabled,
-  areObjectStorageTestsDisabled
+  areMockObjectStorageTestsDisabled,
+  areScalewayObjectStorageTestsDisabled
 }
diff --git a/shared/core-utils/videos/privacy.ts b/shared/core-utils/videos/common.ts
index f33487b49..2c6efdb7f 100644
--- a/shared/core-utils/videos/privacy.ts
+++ b/shared/core-utils/videos/common.ts
@@ -1,5 +1,6 @@
-import { VideoDetails } from '../../models/videos/video.model'
+import { VideoStreamingPlaylistType } from '@shared/models'
 import { VideoPrivacy } from '../../models/videos/video-privacy.enum'
+import { VideoDetails } from '../../models/videos/video.model'
 
 function getAllPrivacies () {
   return [ VideoPrivacy.PUBLIC, VideoPrivacy.INTERNAL, VideoPrivacy.PRIVATE, VideoPrivacy.UNLISTED ]
@@ -8,14 +9,18 @@ function getAllPrivacies () {
 function getAllFiles (video: Partial<Pick<VideoDetails, 'files' | 'streamingPlaylists'>>) {
   const files = video.files
 
-  if (video.streamingPlaylists[0]) {
-    return files.concat(video.streamingPlaylists[0].files)
-  }
+  const hls = getHLS(video)
+  if (hls) return files.concat(hls.files)
 
   return files
 }
 
+function getHLS (video: Partial<Pick<VideoDetails, 'streamingPlaylists'>>) {
+  return video.streamingPlaylists.find(p => p.type === VideoStreamingPlaylistType.HLS)
+}
+
 export {
   getAllPrivacies,
-  getAllFiles
+  getAllFiles,
+  getHLS
 }
diff --git a/shared/core-utils/videos/index.ts b/shared/core-utils/videos/index.ts
index 620e3a716..2cf319395 100644
--- a/shared/core-utils/videos/index.ts
+++ b/shared/core-utils/videos/index.ts
@@ -1,2 +1,2 @@
 export * from './bitrate'
-export * from './privacy'
+export * from './common'
diff --git a/shared/server-commands/miscs/sql-command.ts b/shared/server-commands/miscs/sql-command.ts
index 09a99f834..b0d9ce56d 100644
--- a/shared/server-commands/miscs/sql-command.ts
+++ b/shared/server-commands/miscs/sql-command.ts
@@ -23,6 +23,11 @@ export class SQLCommand extends AbstractCommand {
     return parseInt(total, 10)
   }
 
+  async getInternalFileUrl (fileId: number) {
+    return this.selectQuery(`SELECT "fileUrl" FROM "videoFile" WHERE id = ${fileId}`)
+      .then(rows => rows[0].fileUrl as string)
+  }
+
   setActorField (to: string, field: string, value: string) {
     const seq = this.getSequelize()
 
diff --git a/shared/server-commands/server/object-storage-command.ts b/shared/server-commands/server/object-storage-command.ts
index b4de8f4cb..405e1b043 100644
--- a/shared/server-commands/server/object-storage-command.ts
+++ b/shared/server-commands/server/object-storage-command.ts
@@ -4,74 +4,121 @@ import { makePostBodyRequest } from '../requests'
 import { AbstractCommand } from '../shared'
 
 export class ObjectStorageCommand extends AbstractCommand {
-  static readonly DEFAULT_PLAYLIST_BUCKET = 'streaming-playlists'
-  static readonly DEFAULT_WEBTORRENT_BUCKET = 'videos'
+  static readonly DEFAULT_PLAYLIST_MOCK_BUCKET = 'streaming-playlists'
+  static readonly DEFAULT_WEBTORRENT_MOCK_BUCKET = 'videos'
 
-  static getDefaultConfig () {
+  static readonly DEFAULT_SCALEWAY_BUCKET = 'peertube-ci-test'
+
+  // ---------------------------------------------------------------------------
+
+  static getDefaultMockConfig () {
     return {
       object_storage: {
         enabled: true,
-        endpoint: 'http://' + this.getEndpointHost(),
-        region: this.getRegion(),
+        endpoint: 'http://' + this.getMockEndpointHost(),
+        region: this.getMockRegion(),
 
-        credentials: this.getCredentialsConfig(),
+        credentials: this.getMockCredentialsConfig(),
 
         streaming_playlists: {
-          bucket_name: this.DEFAULT_PLAYLIST_BUCKET
+          bucket_name: this.DEFAULT_PLAYLIST_MOCK_BUCKET
         },
 
         videos: {
-          bucket_name: this.DEFAULT_WEBTORRENT_BUCKET
+          bucket_name: this.DEFAULT_WEBTORRENT_MOCK_BUCKET
         }
       }
     }
   }
 
-  static getCredentialsConfig () {
+  static getMockCredentialsConfig () {
     return {
       access_key_id: 'AKIAIOSFODNN7EXAMPLE',
       secret_access_key: 'wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY'
     }
   }
 
-  static getEndpointHost () {
+  static getMockEndpointHost () {
     return 'localhost:9444'
   }
 
-  static getRegion () {
+  static getMockRegion () {
     return 'us-east-1'
   }
 
-  static getWebTorrentBaseUrl () {
-    return `http://${this.DEFAULT_WEBTORRENT_BUCKET}.${this.getEndpointHost()}/`
+  static getMockWebTorrentBaseUrl () {
+    return `http://${this.DEFAULT_WEBTORRENT_MOCK_BUCKET}.${this.getMockEndpointHost()}/`
   }
 
-  static getPlaylistBaseUrl () {
-    return `http://${this.DEFAULT_PLAYLIST_BUCKET}.${this.getEndpointHost()}/`
+  static getMockPlaylistBaseUrl () {
+    return `http://${this.DEFAULT_PLAYLIST_MOCK_BUCKET}.${this.getMockEndpointHost()}/`
   }
 
-  static async prepareDefaultBuckets () {
-    await this.createBucket(this.DEFAULT_PLAYLIST_BUCKET)
-    await this.createBucket(this.DEFAULT_WEBTORRENT_BUCKET)
+  static async prepareDefaultMockBuckets () {
+    await this.createMockBucket(this.DEFAULT_PLAYLIST_MOCK_BUCKET)
+    await this.createMockBucket(this.DEFAULT_WEBTORRENT_MOCK_BUCKET)
   }
 
-  static async createBucket (name: string) {
+  static async createMockBucket (name: string) {
     await makePostBodyRequest({
-      url: this.getEndpointHost(),
+      url: this.getMockEndpointHost(),
       path: '/ui/' + name + '?delete',
       expectedStatus: HttpStatusCode.TEMPORARY_REDIRECT_307
     })
 
     await makePostBodyRequest({
-      url: this.getEndpointHost(),
+      url: this.getMockEndpointHost(),
       path: '/ui/' + name + '?create',
       expectedStatus: HttpStatusCode.TEMPORARY_REDIRECT_307
     })
 
     await makePostBodyRequest({
-      url: this.getEndpointHost(),
+      url: this.getMockEndpointHost(),
       path: '/ui/' + name + '?make-public',
       expectedStatus: HttpStatusCode.TEMPORARY_REDIRECT_307
     })
   }
+
+  // ---------------------------------------------------------------------------
+
+  static getDefaultScalewayConfig (serverNumber: number) {
+    return {
+      object_storage: {
+        enabled: true,
+        endpoint: this.getScalewayEndpointHost(),
+        region: this.getScalewayRegion(),
+
+        credentials: this.getScalewayCredentialsConfig(),
+
+        streaming_playlists: {
+          bucket_name: this.DEFAULT_SCALEWAY_BUCKET,
+          prefix: `test:server-${serverNumber}-streaming-playlists:`
+        },
+
+        videos: {
+          bucket_name: this.DEFAULT_SCALEWAY_BUCKET,
+          prefix: `test:server-${serverNumber}-videos:`
+        }
+      }
+    }
+  }
+
+  static getScalewayCredentialsConfig () {
+    return {
+      access_key_id: process.env.OBJECT_STORAGE_SCALEWAY_KEY_ID,
+      secret_access_key: process.env.OBJECT_STORAGE_SCALEWAY_ACCESS_KEY
+    }
+  }
+
+  static getScalewayEndpointHost () {
+    return 's3.fr-par.scw.cloud'
+  }
+
+  static getScalewayRegion () {
+    return 'fr-par'
+  }
+
+  static getScalewayBaseUrl () {
+    return `https://${this.DEFAULT_SCALEWAY_BUCKET}.${this.getScalewayEndpointHost()}/`
+  }
 }
diff --git a/shared/server-commands/videos/live-command.ts b/shared/server-commands/videos/live-command.ts
index de193fa49..cc9502c6f 100644
--- a/shared/server-commands/videos/live-command.ts
+++ b/shared/server-commands/videos/live-command.ts
@@ -197,7 +197,7 @@ export class LiveCommand extends AbstractCommand {
 
     const segmentName = `${playlistNumber}-00000${segment}.ts`
     const baseUrl = objectStorage
-      ? ObjectStorageCommand.getPlaylistBaseUrl() + 'hls'
+      ? ObjectStorageCommand.getMockPlaylistBaseUrl() + 'hls'
       : server.url + '/static/streaming-playlists/hls'
 
     let error = true
@@ -253,7 +253,7 @@ export class LiveCommand extends AbstractCommand {
 
     const segmentName = `${playlistNumber}-00000${segment}.ts`
     const baseUrl = objectStorage
-      ? ObjectStorageCommand.getPlaylistBaseUrl()
+      ? ObjectStorageCommand.getMockPlaylistBaseUrl()
       : `${this.server.url}/static/streaming-playlists/hls`
 
     const url = `${baseUrl}/${videoUUID}/${segmentName}`
@@ -275,7 +275,7 @@ export class LiveCommand extends AbstractCommand {
     const { playlistName, videoUUID, objectStorage = false } = options
 
     const baseUrl = objectStorage
-      ? ObjectStorageCommand.getPlaylistBaseUrl()
+      ? ObjectStorageCommand.getMockPlaylistBaseUrl()
       : `${this.server.url}/static/streaming-playlists/hls`
 
     const url = `${baseUrl}/${videoUUID}/${playlistName}`