From 9ab330b90decf4edf152ff8e1d2948c065766b2c Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Wed, 19 Oct 2022 10:43:53 +0200 Subject: Use private ACL for private videos in s3 --- shared/core-utils/common/env.ts | 20 ++++- shared/core-utils/videos/common.ts | 26 +++++++ shared/core-utils/videos/index.ts | 2 +- shared/core-utils/videos/privacy.ts | 21 ----- shared/server-commands/miscs/sql-command.ts | 5 ++ .../server/object-storage-command.ts | 91 ++++++++++++++++------ shared/server-commands/videos/live-command.ts | 6 +- 7 files changed, 122 insertions(+), 49 deletions(-) create mode 100644 shared/core-utils/videos/common.ts delete mode 100644 shared/core-utils/videos/privacy.ts (limited to 'shared') diff --git a/shared/core-utils/common/env.ts b/shared/core-utils/common/env.ts index 38c96b152..973f895d4 100644 --- a/shared/core-utils/common/env.ts +++ b/shared/core-utils/common/env.ts @@ -14,7 +14,7 @@ function areHttpImportTestsDisabled () { return disabled } -function areObjectStorageTestsDisabled () { +function areMockObjectStorageTestsDisabled () { const disabled = process.env.ENABLE_OBJECT_STORAGE_TESTS !== 'true' if (disabled) console.log('ENABLE_OBJECT_STORAGE_TESTS env is not set to "true" so object storage tests are disabled') @@ -22,9 +22,25 @@ function areObjectStorageTestsDisabled () { return disabled } +function areScalewayObjectStorageTestsDisabled () { + if (areMockObjectStorageTestsDisabled()) return true + + const enabled = process.env.OBJECT_STORAGE_SCALEWAY_KEY_ID && process.env.OBJECT_STORAGE_SCALEWAY_ACCESS_KEY + if (!enabled) { + console.log( + 'OBJECT_STORAGE_SCALEWAY_KEY_ID and/or OBJECT_STORAGE_SCALEWAY_ACCESS_KEY are not set, so scaleway object storage tests are disabled' + ) + + return true + } + + return false +} + export { parallelTests, isGithubCI, areHttpImportTestsDisabled, - areObjectStorageTestsDisabled + areMockObjectStorageTestsDisabled, + areScalewayObjectStorageTestsDisabled } diff --git a/shared/core-utils/videos/common.ts b/shared/core-utils/videos/common.ts new file mode 100644 index 000000000..2c6efdb7f --- /dev/null +++ b/shared/core-utils/videos/common.ts @@ -0,0 +1,26 @@ +import { VideoStreamingPlaylistType } from '@shared/models' +import { VideoPrivacy } from '../../models/videos/video-privacy.enum' +import { VideoDetails } from '../../models/videos/video.model' + +function getAllPrivacies () { + return [ VideoPrivacy.PUBLIC, VideoPrivacy.INTERNAL, VideoPrivacy.PRIVATE, VideoPrivacy.UNLISTED ] +} + +function getAllFiles (video: Partial>) { + const files = video.files + + const hls = getHLS(video) + if (hls) return files.concat(hls.files) + + return files +} + +function getHLS (video: Partial>) { + return video.streamingPlaylists.find(p => p.type === VideoStreamingPlaylistType.HLS) +} + +export { + getAllPrivacies, + getAllFiles, + getHLS +} diff --git a/shared/core-utils/videos/index.ts b/shared/core-utils/videos/index.ts index 620e3a716..2cf319395 100644 --- a/shared/core-utils/videos/index.ts +++ b/shared/core-utils/videos/index.ts @@ -1,2 +1,2 @@ export * from './bitrate' -export * from './privacy' +export * from './common' diff --git a/shared/core-utils/videos/privacy.ts b/shared/core-utils/videos/privacy.ts deleted file mode 100644 index f33487b49..000000000 --- a/shared/core-utils/videos/privacy.ts +++ /dev/null @@ -1,21 +0,0 @@ -import { VideoDetails } from '../../models/videos/video.model' -import { VideoPrivacy } from '../../models/videos/video-privacy.enum' - -function getAllPrivacies () { - return [ VideoPrivacy.PUBLIC, VideoPrivacy.INTERNAL, VideoPrivacy.PRIVATE, VideoPrivacy.UNLISTED ] -} - -function getAllFiles (video: Partial>) { - const files = video.files - - if (video.streamingPlaylists[0]) { - return files.concat(video.streamingPlaylists[0].files) - } - - return files -} - -export { - getAllPrivacies, - getAllFiles -} diff --git a/shared/server-commands/miscs/sql-command.ts b/shared/server-commands/miscs/sql-command.ts index 09a99f834..b0d9ce56d 100644 --- a/shared/server-commands/miscs/sql-command.ts +++ b/shared/server-commands/miscs/sql-command.ts @@ -23,6 +23,11 @@ export class SQLCommand extends AbstractCommand { return parseInt(total, 10) } + async getInternalFileUrl (fileId: number) { + return this.selectQuery(`SELECT "fileUrl" FROM "videoFile" WHERE id = ${fileId}`) + .then(rows => rows[0].fileUrl as string) + } + setActorField (to: string, field: string, value: string) { const seq = this.getSequelize() diff --git a/shared/server-commands/server/object-storage-command.ts b/shared/server-commands/server/object-storage-command.ts index b4de8f4cb..405e1b043 100644 --- a/shared/server-commands/server/object-storage-command.ts +++ b/shared/server-commands/server/object-storage-command.ts @@ -4,74 +4,121 @@ import { makePostBodyRequest } from '../requests' import { AbstractCommand } from '../shared' export class ObjectStorageCommand extends AbstractCommand { - static readonly DEFAULT_PLAYLIST_BUCKET = 'streaming-playlists' - static readonly DEFAULT_WEBTORRENT_BUCKET = 'videos' + static readonly DEFAULT_PLAYLIST_MOCK_BUCKET = 'streaming-playlists' + static readonly DEFAULT_WEBTORRENT_MOCK_BUCKET = 'videos' - static getDefaultConfig () { + static readonly DEFAULT_SCALEWAY_BUCKET = 'peertube-ci-test' + + // --------------------------------------------------------------------------- + + static getDefaultMockConfig () { return { object_storage: { enabled: true, - endpoint: 'http://' + this.getEndpointHost(), - region: this.getRegion(), + endpoint: 'http://' + this.getMockEndpointHost(), + region: this.getMockRegion(), - credentials: this.getCredentialsConfig(), + credentials: this.getMockCredentialsConfig(), streaming_playlists: { - bucket_name: this.DEFAULT_PLAYLIST_BUCKET + bucket_name: this.DEFAULT_PLAYLIST_MOCK_BUCKET }, videos: { - bucket_name: this.DEFAULT_WEBTORRENT_BUCKET + bucket_name: this.DEFAULT_WEBTORRENT_MOCK_BUCKET } } } } - static getCredentialsConfig () { + static getMockCredentialsConfig () { return { access_key_id: 'AKIAIOSFODNN7EXAMPLE', secret_access_key: 'wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY' } } - static getEndpointHost () { + static getMockEndpointHost () { return 'localhost:9444' } - static getRegion () { + static getMockRegion () { return 'us-east-1' } - static getWebTorrentBaseUrl () { - return `http://${this.DEFAULT_WEBTORRENT_BUCKET}.${this.getEndpointHost()}/` + static getMockWebTorrentBaseUrl () { + return `http://${this.DEFAULT_WEBTORRENT_MOCK_BUCKET}.${this.getMockEndpointHost()}/` } - static getPlaylistBaseUrl () { - return `http://${this.DEFAULT_PLAYLIST_BUCKET}.${this.getEndpointHost()}/` + static getMockPlaylistBaseUrl () { + return `http://${this.DEFAULT_PLAYLIST_MOCK_BUCKET}.${this.getMockEndpointHost()}/` } - static async prepareDefaultBuckets () { - await this.createBucket(this.DEFAULT_PLAYLIST_BUCKET) - await this.createBucket(this.DEFAULT_WEBTORRENT_BUCKET) + static async prepareDefaultMockBuckets () { + await this.createMockBucket(this.DEFAULT_PLAYLIST_MOCK_BUCKET) + await this.createMockBucket(this.DEFAULT_WEBTORRENT_MOCK_BUCKET) } - static async createBucket (name: string) { + static async createMockBucket (name: string) { await makePostBodyRequest({ - url: this.getEndpointHost(), + url: this.getMockEndpointHost(), path: '/ui/' + name + '?delete', expectedStatus: HttpStatusCode.TEMPORARY_REDIRECT_307 }) await makePostBodyRequest({ - url: this.getEndpointHost(), + url: this.getMockEndpointHost(), path: '/ui/' + name + '?create', expectedStatus: HttpStatusCode.TEMPORARY_REDIRECT_307 }) await makePostBodyRequest({ - url: this.getEndpointHost(), + url: this.getMockEndpointHost(), path: '/ui/' + name + '?make-public', expectedStatus: HttpStatusCode.TEMPORARY_REDIRECT_307 }) } + + // --------------------------------------------------------------------------- + + static getDefaultScalewayConfig (serverNumber: number) { + return { + object_storage: { + enabled: true, + endpoint: this.getScalewayEndpointHost(), + region: this.getScalewayRegion(), + + credentials: this.getScalewayCredentialsConfig(), + + streaming_playlists: { + bucket_name: this.DEFAULT_SCALEWAY_BUCKET, + prefix: `test:server-${serverNumber}-streaming-playlists:` + }, + + videos: { + bucket_name: this.DEFAULT_SCALEWAY_BUCKET, + prefix: `test:server-${serverNumber}-videos:` + } + } + } + } + + static getScalewayCredentialsConfig () { + return { + access_key_id: process.env.OBJECT_STORAGE_SCALEWAY_KEY_ID, + secret_access_key: process.env.OBJECT_STORAGE_SCALEWAY_ACCESS_KEY + } + } + + static getScalewayEndpointHost () { + return 's3.fr-par.scw.cloud' + } + + static getScalewayRegion () { + return 'fr-par' + } + + static getScalewayBaseUrl () { + return `https://${this.DEFAULT_SCALEWAY_BUCKET}.${this.getScalewayEndpointHost()}/` + } } diff --git a/shared/server-commands/videos/live-command.ts b/shared/server-commands/videos/live-command.ts index de193fa49..cc9502c6f 100644 --- a/shared/server-commands/videos/live-command.ts +++ b/shared/server-commands/videos/live-command.ts @@ -197,7 +197,7 @@ export class LiveCommand extends AbstractCommand { const segmentName = `${playlistNumber}-00000${segment}.ts` const baseUrl = objectStorage - ? ObjectStorageCommand.getPlaylistBaseUrl() + 'hls' + ? ObjectStorageCommand.getMockPlaylistBaseUrl() + 'hls' : server.url + '/static/streaming-playlists/hls' let error = true @@ -253,7 +253,7 @@ export class LiveCommand extends AbstractCommand { const segmentName = `${playlistNumber}-00000${segment}.ts` const baseUrl = objectStorage - ? ObjectStorageCommand.getPlaylistBaseUrl() + ? ObjectStorageCommand.getMockPlaylistBaseUrl() : `${this.server.url}/static/streaming-playlists/hls` const url = `${baseUrl}/${videoUUID}/${segmentName}` @@ -275,7 +275,7 @@ export class LiveCommand extends AbstractCommand { const { playlistName, videoUUID, objectStorage = false } = options const baseUrl = objectStorage - ? ObjectStorageCommand.getPlaylistBaseUrl() + ? ObjectStorageCommand.getMockPlaylistBaseUrl() : `${this.server.url}/static/streaming-playlists/hls` const url = `${baseUrl}/${videoUUID}/${playlistName}` -- cgit v1.2.3