aboutsummaryrefslogtreecommitdiffhomepage
path: root/shared
diff options
context:
space:
mode:
authorChocobozzz <me@florianbigard.com>2022-10-19 10:43:53 +0200
committerChocobozzz <chocobozzz@cpy.re>2022-10-24 14:48:24 +0200
commit9ab330b90decf4edf152ff8e1d2948c065766b2c (patch)
tree29d924f50f7307e8e828a57ecb9ea78623487ce0 /shared
parent3545e72c686ff1725bbdfd8d16d693e2f4aa75a3 (diff)
downloadPeerTube-9ab330b90decf4edf152ff8e1d2948c065766b2c.tar.gz
PeerTube-9ab330b90decf4edf152ff8e1d2948c065766b2c.tar.zst
PeerTube-9ab330b90decf4edf152ff8e1d2948c065766b2c.zip
Use private ACL for private videos in s3
Diffstat (limited to 'shared')
-rw-r--r--shared/core-utils/common/env.ts20
-rw-r--r--shared/core-utils/videos/common.ts (renamed from shared/core-utils/videos/privacy.ts)15
-rw-r--r--shared/core-utils/videos/index.ts2
-rw-r--r--shared/server-commands/miscs/sql-command.ts5
-rw-r--r--shared/server-commands/server/object-storage-command.ts91
-rw-r--r--shared/server-commands/videos/live-command.ts6
6 files changed, 106 insertions, 33 deletions
diff --git a/shared/core-utils/common/env.ts b/shared/core-utils/common/env.ts
index 38c96b152..973f895d4 100644
--- a/shared/core-utils/common/env.ts
+++ b/shared/core-utils/common/env.ts
@@ -14,7 +14,7 @@ function areHttpImportTestsDisabled () {
14 return disabled 14 return disabled
15} 15}
16 16
17function areObjectStorageTestsDisabled () { 17function areMockObjectStorageTestsDisabled () {
18 const disabled = process.env.ENABLE_OBJECT_STORAGE_TESTS !== 'true' 18 const disabled = process.env.ENABLE_OBJECT_STORAGE_TESTS !== 'true'
19 19
20 if (disabled) console.log('ENABLE_OBJECT_STORAGE_TESTS env is not set to "true" so object storage tests are disabled') 20 if (disabled) console.log('ENABLE_OBJECT_STORAGE_TESTS env is not set to "true" so object storage tests are disabled')
@@ -22,9 +22,25 @@ function areObjectStorageTestsDisabled () {
22 return disabled 22 return disabled
23} 23}
24 24
25function areScalewayObjectStorageTestsDisabled () {
26 if (areMockObjectStorageTestsDisabled()) return true
27
28 const enabled = process.env.OBJECT_STORAGE_SCALEWAY_KEY_ID && process.env.OBJECT_STORAGE_SCALEWAY_ACCESS_KEY
29 if (!enabled) {
30 console.log(
31 'OBJECT_STORAGE_SCALEWAY_KEY_ID and/or OBJECT_STORAGE_SCALEWAY_ACCESS_KEY are not set, so scaleway object storage tests are disabled'
32 )
33
34 return true
35 }
36
37 return false
38}
39
25export { 40export {
26 parallelTests, 41 parallelTests,
27 isGithubCI, 42 isGithubCI,
28 areHttpImportTestsDisabled, 43 areHttpImportTestsDisabled,
29 areObjectStorageTestsDisabled 44 areMockObjectStorageTestsDisabled,
45 areScalewayObjectStorageTestsDisabled
30} 46}
diff --git a/shared/core-utils/videos/privacy.ts b/shared/core-utils/videos/common.ts
index f33487b49..2c6efdb7f 100644
--- a/shared/core-utils/videos/privacy.ts
+++ b/shared/core-utils/videos/common.ts
@@ -1,5 +1,6 @@
1import { VideoDetails } from '../../models/videos/video.model' 1import { VideoStreamingPlaylistType } from '@shared/models'
2import { VideoPrivacy } from '../../models/videos/video-privacy.enum' 2import { VideoPrivacy } from '../../models/videos/video-privacy.enum'
3import { VideoDetails } from '../../models/videos/video.model'
3 4
4function getAllPrivacies () { 5function getAllPrivacies () {
5 return [ VideoPrivacy.PUBLIC, VideoPrivacy.INTERNAL, VideoPrivacy.PRIVATE, VideoPrivacy.UNLISTED ] 6 return [ VideoPrivacy.PUBLIC, VideoPrivacy.INTERNAL, VideoPrivacy.PRIVATE, VideoPrivacy.UNLISTED ]
@@ -8,14 +9,18 @@ function getAllPrivacies () {
8function getAllFiles (video: Partial<Pick<VideoDetails, 'files' | 'streamingPlaylists'>>) { 9function getAllFiles (video: Partial<Pick<VideoDetails, 'files' | 'streamingPlaylists'>>) {
9 const files = video.files 10 const files = video.files
10 11
11 if (video.streamingPlaylists[0]) { 12 const hls = getHLS(video)
12 return files.concat(video.streamingPlaylists[0].files) 13 if (hls) return files.concat(hls.files)
13 }
14 14
15 return files 15 return files
16} 16}
17 17
18function getHLS (video: Partial<Pick<VideoDetails, 'streamingPlaylists'>>) {
19 return video.streamingPlaylists.find(p => p.type === VideoStreamingPlaylistType.HLS)
20}
21
18export { 22export {
19 getAllPrivacies, 23 getAllPrivacies,
20 getAllFiles 24 getAllFiles,
25 getHLS
21} 26}
diff --git a/shared/core-utils/videos/index.ts b/shared/core-utils/videos/index.ts
index 620e3a716..2cf319395 100644
--- a/shared/core-utils/videos/index.ts
+++ b/shared/core-utils/videos/index.ts
@@ -1,2 +1,2 @@
1export * from './bitrate' 1export * from './bitrate'
2export * from './privacy' 2export * from './common'
diff --git a/shared/server-commands/miscs/sql-command.ts b/shared/server-commands/miscs/sql-command.ts
index 09a99f834..b0d9ce56d 100644
--- a/shared/server-commands/miscs/sql-command.ts
+++ b/shared/server-commands/miscs/sql-command.ts
@@ -23,6 +23,11 @@ export class SQLCommand extends AbstractCommand {
23 return parseInt(total, 10) 23 return parseInt(total, 10)
24 } 24 }
25 25
26 async getInternalFileUrl (fileId: number) {
27 return this.selectQuery(`SELECT "fileUrl" FROM "videoFile" WHERE id = ${fileId}`)
28 .then(rows => rows[0].fileUrl as string)
29 }
30
26 setActorField (to: string, field: string, value: string) { 31 setActorField (to: string, field: string, value: string) {
27 const seq = this.getSequelize() 32 const seq = this.getSequelize()
28 33
diff --git a/shared/server-commands/server/object-storage-command.ts b/shared/server-commands/server/object-storage-command.ts
index b4de8f4cb..405e1b043 100644
--- a/shared/server-commands/server/object-storage-command.ts
+++ b/shared/server-commands/server/object-storage-command.ts
@@ -4,74 +4,121 @@ import { makePostBodyRequest } from '../requests'
4import { AbstractCommand } from '../shared' 4import { AbstractCommand } from '../shared'
5 5
6export class ObjectStorageCommand extends AbstractCommand { 6export class ObjectStorageCommand extends AbstractCommand {
7 static readonly DEFAULT_PLAYLIST_BUCKET = 'streaming-playlists' 7 static readonly DEFAULT_PLAYLIST_MOCK_BUCKET = 'streaming-playlists'
8 static readonly DEFAULT_WEBTORRENT_BUCKET = 'videos' 8 static readonly DEFAULT_WEBTORRENT_MOCK_BUCKET = 'videos'
9 9
10 static getDefaultConfig () { 10 static readonly DEFAULT_SCALEWAY_BUCKET = 'peertube-ci-test'
11
12 // ---------------------------------------------------------------------------
13
14 static getDefaultMockConfig () {
11 return { 15 return {
12 object_storage: { 16 object_storage: {
13 enabled: true, 17 enabled: true,
14 endpoint: 'http://' + this.getEndpointHost(), 18 endpoint: 'http://' + this.getMockEndpointHost(),
15 region: this.getRegion(), 19 region: this.getMockRegion(),
16 20
17 credentials: this.getCredentialsConfig(), 21 credentials: this.getMockCredentialsConfig(),
18 22
19 streaming_playlists: { 23 streaming_playlists: {
20 bucket_name: this.DEFAULT_PLAYLIST_BUCKET 24 bucket_name: this.DEFAULT_PLAYLIST_MOCK_BUCKET
21 }, 25 },
22 26
23 videos: { 27 videos: {
24 bucket_name: this.DEFAULT_WEBTORRENT_BUCKET 28 bucket_name: this.DEFAULT_WEBTORRENT_MOCK_BUCKET
25 } 29 }
26 } 30 }
27 } 31 }
28 } 32 }
29 33
30 static getCredentialsConfig () { 34 static getMockCredentialsConfig () {
31 return { 35 return {
32 access_key_id: 'AKIAIOSFODNN7EXAMPLE', 36 access_key_id: 'AKIAIOSFODNN7EXAMPLE',
33 secret_access_key: 'wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY' 37 secret_access_key: 'wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY'
34 } 38 }
35 } 39 }
36 40
37 static getEndpointHost () { 41 static getMockEndpointHost () {
38 return 'localhost:9444' 42 return 'localhost:9444'
39 } 43 }
40 44
41 static getRegion () { 45 static getMockRegion () {
42 return 'us-east-1' 46 return 'us-east-1'
43 } 47 }
44 48
45 static getWebTorrentBaseUrl () { 49 static getMockWebTorrentBaseUrl () {
46 return `http://${this.DEFAULT_WEBTORRENT_BUCKET}.${this.getEndpointHost()}/` 50 return `http://${this.DEFAULT_WEBTORRENT_MOCK_BUCKET}.${this.getMockEndpointHost()}/`
47 } 51 }
48 52
49 static getPlaylistBaseUrl () { 53 static getMockPlaylistBaseUrl () {
50 return `http://${this.DEFAULT_PLAYLIST_BUCKET}.${this.getEndpointHost()}/` 54 return `http://${this.DEFAULT_PLAYLIST_MOCK_BUCKET}.${this.getMockEndpointHost()}/`
51 } 55 }
52 56
53 static async prepareDefaultBuckets () { 57 static async prepareDefaultMockBuckets () {
54 await this.createBucket(this.DEFAULT_PLAYLIST_BUCKET) 58 await this.createMockBucket(this.DEFAULT_PLAYLIST_MOCK_BUCKET)
55 await this.createBucket(this.DEFAULT_WEBTORRENT_BUCKET) 59 await this.createMockBucket(this.DEFAULT_WEBTORRENT_MOCK_BUCKET)
56 } 60 }
57 61
58 static async createBucket (name: string) { 62 static async createMockBucket (name: string) {
59 await makePostBodyRequest({ 63 await makePostBodyRequest({
60 url: this.getEndpointHost(), 64 url: this.getMockEndpointHost(),
61 path: '/ui/' + name + '?delete', 65 path: '/ui/' + name + '?delete',
62 expectedStatus: HttpStatusCode.TEMPORARY_REDIRECT_307 66 expectedStatus: HttpStatusCode.TEMPORARY_REDIRECT_307
63 }) 67 })
64 68
65 await makePostBodyRequest({ 69 await makePostBodyRequest({
66 url: this.getEndpointHost(), 70 url: this.getMockEndpointHost(),
67 path: '/ui/' + name + '?create', 71 path: '/ui/' + name + '?create',
68 expectedStatus: HttpStatusCode.TEMPORARY_REDIRECT_307 72 expectedStatus: HttpStatusCode.TEMPORARY_REDIRECT_307
69 }) 73 })
70 74
71 await makePostBodyRequest({ 75 await makePostBodyRequest({
72 url: this.getEndpointHost(), 76 url: this.getMockEndpointHost(),
73 path: '/ui/' + name + '?make-public', 77 path: '/ui/' + name + '?make-public',
74 expectedStatus: HttpStatusCode.TEMPORARY_REDIRECT_307 78 expectedStatus: HttpStatusCode.TEMPORARY_REDIRECT_307
75 }) 79 })
76 } 80 }
81
82 // ---------------------------------------------------------------------------
83
84 static getDefaultScalewayConfig (serverNumber: number) {
85 return {
86 object_storage: {
87 enabled: true,
88 endpoint: this.getScalewayEndpointHost(),
89 region: this.getScalewayRegion(),
90
91 credentials: this.getScalewayCredentialsConfig(),
92
93 streaming_playlists: {
94 bucket_name: this.DEFAULT_SCALEWAY_BUCKET,
95 prefix: `test:server-${serverNumber}-streaming-playlists:`
96 },
97
98 videos: {
99 bucket_name: this.DEFAULT_SCALEWAY_BUCKET,
100 prefix: `test:server-${serverNumber}-videos:`
101 }
102 }
103 }
104 }
105
106 static getScalewayCredentialsConfig () {
107 return {
108 access_key_id: process.env.OBJECT_STORAGE_SCALEWAY_KEY_ID,
109 secret_access_key: process.env.OBJECT_STORAGE_SCALEWAY_ACCESS_KEY
110 }
111 }
112
113 static getScalewayEndpointHost () {
114 return 's3.fr-par.scw.cloud'
115 }
116
117 static getScalewayRegion () {
118 return 'fr-par'
119 }
120
121 static getScalewayBaseUrl () {
122 return `https://${this.DEFAULT_SCALEWAY_BUCKET}.${this.getScalewayEndpointHost()}/`
123 }
77} 124}
diff --git a/shared/server-commands/videos/live-command.ts b/shared/server-commands/videos/live-command.ts
index de193fa49..cc9502c6f 100644
--- a/shared/server-commands/videos/live-command.ts
+++ b/shared/server-commands/videos/live-command.ts
@@ -197,7 +197,7 @@ export class LiveCommand extends AbstractCommand {
197 197
198 const segmentName = `${playlistNumber}-00000${segment}.ts` 198 const segmentName = `${playlistNumber}-00000${segment}.ts`
199 const baseUrl = objectStorage 199 const baseUrl = objectStorage
200 ? ObjectStorageCommand.getPlaylistBaseUrl() + 'hls' 200 ? ObjectStorageCommand.getMockPlaylistBaseUrl() + 'hls'
201 : server.url + '/static/streaming-playlists/hls' 201 : server.url + '/static/streaming-playlists/hls'
202 202
203 let error = true 203 let error = true
@@ -253,7 +253,7 @@ export class LiveCommand extends AbstractCommand {
253 253
254 const segmentName = `${playlistNumber}-00000${segment}.ts` 254 const segmentName = `${playlistNumber}-00000${segment}.ts`
255 const baseUrl = objectStorage 255 const baseUrl = objectStorage
256 ? ObjectStorageCommand.getPlaylistBaseUrl() 256 ? ObjectStorageCommand.getMockPlaylistBaseUrl()
257 : `${this.server.url}/static/streaming-playlists/hls` 257 : `${this.server.url}/static/streaming-playlists/hls`
258 258
259 const url = `${baseUrl}/${videoUUID}/${segmentName}` 259 const url = `${baseUrl}/${videoUUID}/${segmentName}`
@@ -275,7 +275,7 @@ export class LiveCommand extends AbstractCommand {
275 const { playlistName, videoUUID, objectStorage = false } = options 275 const { playlistName, videoUUID, objectStorage = false } = options
276 276
277 const baseUrl = objectStorage 277 const baseUrl = objectStorage
278 ? ObjectStorageCommand.getPlaylistBaseUrl() 278 ? ObjectStorageCommand.getMockPlaylistBaseUrl()
279 : `${this.server.url}/static/streaming-playlists/hls` 279 : `${this.server.url}/static/streaming-playlists/hls`
280 280
281 const url = `${baseUrl}/${videoUUID}/${playlistName}` 281 const url = `${baseUrl}/${videoUUID}/${playlistName}`