aboutsummaryrefslogtreecommitdiffhomepage
diff options
context:
space:
mode:
-rw-r--r--package.json1
-rw-r--r--server/helpers/requests.ts1
-rw-r--r--server/lib/object-storage/shared/client.ts17
-rw-r--r--server/tests/api/server/proxy.ts42
-rw-r--r--server/tests/shared/checks.ts5
-rw-r--r--yarn.lock41
6 files changed, 105 insertions, 2 deletions
diff --git a/package.json b/package.json
index 360bd781f..ec9a2a632 100644
--- a/package.json
+++ b/package.json
@@ -79,6 +79,7 @@
79 "dependencies": { 79 "dependencies": {
80 "@aws-sdk/client-s3": "^3.23.0", 80 "@aws-sdk/client-s3": "^3.23.0",
81 "@aws-sdk/lib-storage": "^3.72.0", 81 "@aws-sdk/lib-storage": "^3.72.0",
82 "@aws-sdk/node-http-handler": "^3.82.0",
82 "@babel/parser": "7.17.8", 83 "@babel/parser": "7.17.8",
83 "@peertube/feed": "^5.0.1", 84 "@peertube/feed": "^5.0.1",
84 "@peertube/http-signature": "^1.4.0", 85 "@peertube/http-signature": "^1.4.0",
diff --git a/server/helpers/requests.ts b/server/helpers/requests.ts
index 327610558..a9869e987 100644
--- a/server/helpers/requests.ts
+++ b/server/helpers/requests.ts
@@ -212,6 +212,7 @@ export {
212 doRequestAndSaveToFile, 212 doRequestAndSaveToFile,
213 isBinaryResponse, 213 isBinaryResponse,
214 downloadImage, 214 downloadImage,
215 getAgent,
215 findLatestRedirection, 216 findLatestRedirection,
216 peertubeGot 217 peertubeGot
217} 218}
diff --git a/server/lib/object-storage/shared/client.ts b/server/lib/object-storage/shared/client.ts
index c9a614593..d5cb074df 100644
--- a/server/lib/object-storage/shared/client.ts
+++ b/server/lib/object-storage/shared/client.ts
@@ -1,8 +1,22 @@
1import { S3Client } from '@aws-sdk/client-s3' 1import { S3Client } from '@aws-sdk/client-s3'
2import { NodeHttpHandler } from '@aws-sdk/node-http-handler'
2import { logger } from '@server/helpers/logger' 3import { logger } from '@server/helpers/logger'
4import { isProxyEnabled } from '@server/helpers/proxy'
5import { getAgent } from '@server/helpers/requests'
3import { CONFIG } from '@server/initializers/config' 6import { CONFIG } from '@server/initializers/config'
4import { lTags } from './logger' 7import { lTags } from './logger'
5 8
9function getProxyRequestHandler () {
10 if (!isProxyEnabled()) return null
11
12 const { agent } = getAgent()
13
14 return new NodeHttpHandler({
15 httpAgent: agent.http,
16 httpsAgent: agent.https
17 })
18}
19
6let endpointParsed: URL 20let endpointParsed: URL
7function getEndpointParsed () { 21function getEndpointParsed () {
8 if (endpointParsed) return endpointParsed 22 if (endpointParsed) return endpointParsed
@@ -26,7 +40,8 @@ function getClient () {
26 accessKeyId: OBJECT_STORAGE.CREDENTIALS.ACCESS_KEY_ID, 40 accessKeyId: OBJECT_STORAGE.CREDENTIALS.ACCESS_KEY_ID,
27 secretAccessKey: OBJECT_STORAGE.CREDENTIALS.SECRET_ACCESS_KEY 41 secretAccessKey: OBJECT_STORAGE.CREDENTIALS.SECRET_ACCESS_KEY
28 } 42 }
29 : undefined 43 : undefined,
44 requestHandler: getProxyRequestHandler()
30 }) 45 })
31 46
32 logger.info('Initialized S3 client %s with region %s.', getEndpoint(), OBJECT_STORAGE.REGION, lTags()) 47 logger.info('Initialized S3 client %s with region %s.', getEndpoint(), OBJECT_STORAGE.REGION, lTags())
diff --git a/server/tests/api/server/proxy.ts b/server/tests/api/server/proxy.ts
index 2a8ff56d2..e238edaf4 100644
--- a/server/tests/api/server/proxy.ts
+++ b/server/tests/api/server/proxy.ts
@@ -2,12 +2,14 @@
2 2
3import 'mocha' 3import 'mocha'
4import * as chai from 'chai' 4import * as chai from 'chai'
5import { FIXTURE_URLS, MockProxy } from '@server/tests/shared' 5import { expectNotStartWith, expectStartWith, FIXTURE_URLS, MockProxy } from '@server/tests/shared'
6import { areObjectStorageTestsDisabled } from '@shared/core-utils'
6import { HttpStatusCode, VideoPrivacy } from '@shared/models' 7import { HttpStatusCode, VideoPrivacy } from '@shared/models'
7import { 8import {
8 cleanupTests, 9 cleanupTests,
9 createMultipleServers, 10 createMultipleServers,
10 doubleFollow, 11 doubleFollow,
12 ObjectStorageCommand,
11 PeerTubeServer, 13 PeerTubeServer,
12 setAccessTokensToServers, 14 setAccessTokensToServers,
13 setDefaultVideoChannel, 15 setDefaultVideoChannel,
@@ -120,6 +122,44 @@ describe('Test proxy', function () {
120 }) 122 })
121 }) 123 })
122 124
125 describe('Object storage', function () {
126 if (areObjectStorageTestsDisabled()) return
127
128 before(async function () {
129 this.timeout(30000)
130
131 await ObjectStorageCommand.prepareDefaultBuckets()
132 })
133
134 it('Should succeed to upload to object storage with the appropriate proxy config', async function () {
135 this.timeout(120000)
136
137 await servers[0].kill()
138 await servers[0].run(ObjectStorageCommand.getDefaultConfig(), { env: goodEnv })
139
140 const { uuid } = await servers[0].videos.quickUpload({ name: 'video' })
141 await waitJobs(servers)
142
143 const video = await servers[0].videos.get({ id: uuid })
144
145 expectStartWith(video.files[0].fileUrl, ObjectStorageCommand.getWebTorrentBaseUrl())
146 })
147
148 it('Should fail to upload to object storage with a wrong proxy config', async function () {
149 this.timeout(120000)
150
151 await servers[0].kill()
152 await servers[0].run(ObjectStorageCommand.getDefaultConfig(), { env: badEnv })
153
154 const { uuid } = await servers[0].videos.quickUpload({ name: 'video' })
155 await waitJobs(servers)
156
157 const video = await servers[0].videos.get({ id: uuid })
158
159 expectNotStartWith(video.files[0].fileUrl, ObjectStorageCommand.getWebTorrentBaseUrl())
160 })
161 })
162
123 after(async function () { 163 after(async function () {
124 await proxy.terminate() 164 await proxy.terminate()
125 165
diff --git a/server/tests/shared/checks.ts b/server/tests/shared/checks.ts
index dcc16d7ea..33b917f31 100644
--- a/server/tests/shared/checks.ts
+++ b/server/tests/shared/checks.ts
@@ -19,6 +19,10 @@ function expectStartWith (str: string, start: string) {
19 expect(str.startsWith(start), `${str} does not start with ${start}`).to.be.true 19 expect(str.startsWith(start), `${str} does not start with ${start}`).to.be.true
20} 20}
21 21
22function expectNotStartWith (str: string, start: string) {
23 expect(str.startsWith(start), `${str} does not start with ${start}`).to.be.false
24}
25
22async function expectLogDoesNotContain (server: PeerTubeServer, str: string) { 26async function expectLogDoesNotContain (server: PeerTubeServer, str: string) {
23 const content = await server.servers.getLogContent() 27 const content = await server.servers.getLogContent()
24 28
@@ -92,6 +96,7 @@ export {
92 expectLogDoesNotContain, 96 expectLogDoesNotContain,
93 testFileExistsOrNot, 97 testFileExistsOrNot,
94 expectStartWith, 98 expectStartWith,
99 expectNotStartWith,
95 checkBadStartPagination, 100 checkBadStartPagination,
96 checkBadCountPagination, 101 checkBadCountPagination,
97 checkBadSortPagination 102 checkBadSortPagination
diff --git a/yarn.lock b/yarn.lock
index 104f4c241..bfc4102a8 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -142,6 +142,14 @@
142 "@aws-sdk/types" "3.55.0" 142 "@aws-sdk/types" "3.55.0"
143 tslib "^2.3.1" 143 tslib "^2.3.1"
144 144
145"@aws-sdk/abort-controller@3.78.0":
146 version "3.78.0"
147 resolved "https://registry.yarnpkg.com/@aws-sdk/abort-controller/-/abort-controller-3.78.0.tgz#f2b0f8d63954afe51136254f389a18dd24a8f6f3"
148 integrity sha512-iz1YLwM2feJUj/y97yO4XmDeTxs+yZ1XJwQgoawKuc8IDBKUutnJNCHL5jL04WUKU7Nrlq+Hr2fCTScFh2z9zg==
149 dependencies:
150 "@aws-sdk/types" "3.78.0"
151 tslib "^2.3.1"
152
145"@aws-sdk/chunked-blob-reader-native@3.58.0": 153"@aws-sdk/chunked-blob-reader-native@3.58.0":
146 version "3.58.0" 154 version "3.58.0"
147 resolved "https://registry.yarnpkg.com/@aws-sdk/chunked-blob-reader-native/-/chunked-blob-reader-native-3.58.0.tgz#1db413c5c80b32e24f1b62b22e15e9ad74d75cda" 155 resolved "https://registry.yarnpkg.com/@aws-sdk/chunked-blob-reader-native/-/chunked-blob-reader-native-3.58.0.tgz#1db413c5c80b32e24f1b62b22e15e9ad74d75cda"
@@ -678,6 +686,17 @@
678 "@aws-sdk/types" "3.55.0" 686 "@aws-sdk/types" "3.55.0"
679 tslib "^2.3.1" 687 tslib "^2.3.1"
680 688
689"@aws-sdk/node-http-handler@^3.82.0":
690 version "3.82.0"
691 resolved "https://registry.yarnpkg.com/@aws-sdk/node-http-handler/-/node-http-handler-3.82.0.tgz#e28064815c6c6caf22a16bb7fee4e9e7e73ef3bb"
692 integrity sha512-yyq/DA/IMzL4fLJhV7zVfP7aUQWPHfOKTCJjWB3KeV5YPiviJtSKb/KyzNi+gQyO7SmsL/8vQbQrf3/s7N/2OA==
693 dependencies:
694 "@aws-sdk/abort-controller" "3.78.0"
695 "@aws-sdk/protocol-http" "3.78.0"
696 "@aws-sdk/querystring-builder" "3.78.0"
697 "@aws-sdk/types" "3.78.0"
698 tslib "^2.3.1"
699
681"@aws-sdk/property-provider@3.55.0": 700"@aws-sdk/property-provider@3.55.0":
682 version "3.55.0" 701 version "3.55.0"
683 resolved "https://registry.yarnpkg.com/@aws-sdk/property-provider/-/property-provider-3.55.0.tgz#0eabe5e84d9258c85c2c5e44bcb09379ae9429d2" 702 resolved "https://registry.yarnpkg.com/@aws-sdk/property-provider/-/property-provider-3.55.0.tgz#0eabe5e84d9258c85c2c5e44bcb09379ae9429d2"
@@ -694,6 +713,14 @@
694 "@aws-sdk/types" "3.55.0" 713 "@aws-sdk/types" "3.55.0"
695 tslib "^2.3.1" 714 tslib "^2.3.1"
696 715
716"@aws-sdk/protocol-http@3.78.0":
717 version "3.78.0"
718 resolved "https://registry.yarnpkg.com/@aws-sdk/protocol-http/-/protocol-http-3.78.0.tgz#8a30db90e3373fe94e2b0007c3cba47b5c9e08bd"
719 integrity sha512-SQB26MhEK96yDxyXd3UAaxLz1Y/ZvgE4pzv7V3wZiokdEedM0kawHKEn1UQJlqJLEZcQI9QYyysh3rTvHZ3fyg==
720 dependencies:
721 "@aws-sdk/types" "3.78.0"
722 tslib "^2.3.1"
723
697"@aws-sdk/querystring-builder@3.55.0": 724"@aws-sdk/querystring-builder@3.55.0":
698 version "3.55.0" 725 version "3.55.0"
699 resolved "https://registry.yarnpkg.com/@aws-sdk/querystring-builder/-/querystring-builder-3.55.0.tgz#7d6d4e2c597eb3d636bd3a368b494dac175ba329" 726 resolved "https://registry.yarnpkg.com/@aws-sdk/querystring-builder/-/querystring-builder-3.55.0.tgz#7d6d4e2c597eb3d636bd3a368b494dac175ba329"
@@ -703,6 +730,15 @@
703 "@aws-sdk/util-uri-escape" "3.55.0" 730 "@aws-sdk/util-uri-escape" "3.55.0"
704 tslib "^2.3.1" 731 tslib "^2.3.1"
705 732
733"@aws-sdk/querystring-builder@3.78.0":
734 version "3.78.0"
735 resolved "https://registry.yarnpkg.com/@aws-sdk/querystring-builder/-/querystring-builder-3.78.0.tgz#29068c4d1fad056e26f848779a31335469cb0038"
736 integrity sha512-aib6RW1WAaTQDqVgRU1Ku9idkhm90gJKbCxVaGId+as6QHNUqMChEfK2v+0afuKiPNOs5uWmqvOXI9+Gt+UGDg==
737 dependencies:
738 "@aws-sdk/types" "3.78.0"
739 "@aws-sdk/util-uri-escape" "3.55.0"
740 tslib "^2.3.1"
741
706"@aws-sdk/querystring-parser@3.55.0": 742"@aws-sdk/querystring-parser@3.55.0":
707 version "3.55.0" 743 version "3.55.0"
708 resolved "https://registry.yarnpkg.com/@aws-sdk/querystring-parser/-/querystring-parser-3.55.0.tgz#ea35642c1b8324dd896d45185f99ad9d6c3af6d2" 744 resolved "https://registry.yarnpkg.com/@aws-sdk/querystring-parser/-/querystring-parser-3.55.0.tgz#ea35642c1b8324dd896d45185f99ad9d6c3af6d2"
@@ -760,6 +796,11 @@
760 resolved "https://registry.yarnpkg.com/@aws-sdk/types/-/types-3.55.0.tgz#d524d567e2b2722f2d6be83e2417dd6d46ce1490" 796 resolved "https://registry.yarnpkg.com/@aws-sdk/types/-/types-3.55.0.tgz#d524d567e2b2722f2d6be83e2417dd6d46ce1490"
761 integrity sha512-wrDZjuy1CVAYxDCbm3bWQIKMGfNs7XXmG0eG4858Ixgqmq2avsIn5TORy8ynBxcXn9aekV/+tGEQ7BBSYzIVNQ== 797 integrity sha512-wrDZjuy1CVAYxDCbm3bWQIKMGfNs7XXmG0eG4858Ixgqmq2avsIn5TORy8ynBxcXn9aekV/+tGEQ7BBSYzIVNQ==
762 798
799"@aws-sdk/types@3.78.0":
800 version "3.78.0"
801 resolved "https://registry.yarnpkg.com/@aws-sdk/types/-/types-3.78.0.tgz#51dc80b2142ee20821fb9f476bdca6e541021443"
802 integrity sha512-I9PTlVNSbwhIgMfmDM5as1tqRIkVZunjVmfogb2WVVPp4CaX0Ll01S0FSMSLL9k6tcQLXqh45pFRjrxCl9WKdQ==
803
763"@aws-sdk/url-parser@3.55.0": 804"@aws-sdk/url-parser@3.55.0":
764 version "3.55.0" 805 version "3.55.0"
765 resolved "https://registry.yarnpkg.com/@aws-sdk/url-parser/-/url-parser-3.55.0.tgz#03b47a45c591d52c9d00dc40c630b91094991fe7" 806 resolved "https://registry.yarnpkg.com/@aws-sdk/url-parser/-/url-parser-3.55.0.tgz#03b47a45c591d52c9d00dc40c630b91094991fe7"