From ca3d5912e16b39697bdeeda35d10b44ed8f711aa Mon Sep 17 00:00:00 2001
From: Chocobozzz <me@florianbigard.com>
Date: Mon, 9 May 2022 11:49:25 +0200
Subject: Add use proxy for s3 (#4973)

* Fix object storage to be accessible via proxy

* fix lint

* Use hpagent

* Fix lint

* Fix PR

Co-authored-by: noellabo <noel.yoshiba@gmail.com>
---
 server/helpers/requests.ts                 |  1 +
 server/lib/object-storage/shared/client.ts | 17 +++++++++++-
 server/tests/api/server/proxy.ts           | 42 +++++++++++++++++++++++++++++-
 server/tests/shared/checks.ts              |  5 ++++
 4 files changed, 63 insertions(+), 2 deletions(-)

(limited to 'server')

diff --git a/server/helpers/requests.ts b/server/helpers/requests.ts
index 327610558..a9869e987 100644
--- a/server/helpers/requests.ts
+++ b/server/helpers/requests.ts
@@ -212,6 +212,7 @@ export {
   doRequestAndSaveToFile,
   isBinaryResponse,
   downloadImage,
+  getAgent,
   findLatestRedirection,
   peertubeGot
 }
diff --git a/server/lib/object-storage/shared/client.ts b/server/lib/object-storage/shared/client.ts
index c9a614593..d5cb074df 100644
--- a/server/lib/object-storage/shared/client.ts
+++ b/server/lib/object-storage/shared/client.ts
@@ -1,8 +1,22 @@
 import { S3Client } from '@aws-sdk/client-s3'
+import { NodeHttpHandler } from '@aws-sdk/node-http-handler'
 import { logger } from '@server/helpers/logger'
+import { isProxyEnabled } from '@server/helpers/proxy'
+import { getAgent } from '@server/helpers/requests'
 import { CONFIG } from '@server/initializers/config'
 import { lTags } from './logger'
 
+function getProxyRequestHandler () {
+  if (!isProxyEnabled()) return null
+
+  const { agent } = getAgent()
+
+  return new NodeHttpHandler({
+    httpAgent: agent.http,
+    httpsAgent: agent.https
+  })
+}
+
 let endpointParsed: URL
 function getEndpointParsed () {
   if (endpointParsed) return endpointParsed
@@ -26,7 +40,8 @@ function getClient () {
         accessKeyId: OBJECT_STORAGE.CREDENTIALS.ACCESS_KEY_ID,
         secretAccessKey: OBJECT_STORAGE.CREDENTIALS.SECRET_ACCESS_KEY
       }
-      : undefined
+      : undefined,
+    requestHandler: getProxyRequestHandler()
   })
 
   logger.info('Initialized S3 client %s with region %s.', getEndpoint(), OBJECT_STORAGE.REGION, lTags())
diff --git a/server/tests/api/server/proxy.ts b/server/tests/api/server/proxy.ts
index 2a8ff56d2..e238edaf4 100644
--- a/server/tests/api/server/proxy.ts
+++ b/server/tests/api/server/proxy.ts
@@ -2,12 +2,14 @@
 
 import 'mocha'
 import * as chai from 'chai'
-import { FIXTURE_URLS, MockProxy } from '@server/tests/shared'
+import { expectNotStartWith, expectStartWith, FIXTURE_URLS, MockProxy } from '@server/tests/shared'
+import { areObjectStorageTestsDisabled } from '@shared/core-utils'
 import { HttpStatusCode, VideoPrivacy } from '@shared/models'
 import {
   cleanupTests,
   createMultipleServers,
   doubleFollow,
+  ObjectStorageCommand,
   PeerTubeServer,
   setAccessTokensToServers,
   setDefaultVideoChannel,
@@ -120,6 +122,44 @@ describe('Test proxy', function () {
     })
   })
 
+  describe('Object storage', function () {
+    if (areObjectStorageTestsDisabled()) return
+
+    before(async function () {
+      this.timeout(30000)
+
+      await ObjectStorageCommand.prepareDefaultBuckets()
+    })
+
+    it('Should succeed to upload to object storage with the appropriate proxy config', async function () {
+      this.timeout(120000)
+
+      await servers[0].kill()
+      await servers[0].run(ObjectStorageCommand.getDefaultConfig(), { env: goodEnv })
+
+      const { uuid } = await servers[0].videos.quickUpload({ name: 'video' })
+      await waitJobs(servers)
+
+      const video = await servers[0].videos.get({ id: uuid })
+
+      expectStartWith(video.files[0].fileUrl, ObjectStorageCommand.getWebTorrentBaseUrl())
+    })
+
+    it('Should fail to upload to object storage with a wrong proxy config', async function () {
+      this.timeout(120000)
+
+      await servers[0].kill()
+      await servers[0].run(ObjectStorageCommand.getDefaultConfig(), { env: badEnv })
+
+      const { uuid } = await servers[0].videos.quickUpload({ name: 'video' })
+      await waitJobs(servers)
+
+      const video = await servers[0].videos.get({ id: uuid })
+
+      expectNotStartWith(video.files[0].fileUrl, ObjectStorageCommand.getWebTorrentBaseUrl())
+    })
+  })
+
   after(async function () {
     await proxy.terminate()
 
diff --git a/server/tests/shared/checks.ts b/server/tests/shared/checks.ts
index dcc16d7ea..33b917f31 100644
--- a/server/tests/shared/checks.ts
+++ b/server/tests/shared/checks.ts
@@ -19,6 +19,10 @@ function expectStartWith (str: string, start: string) {
   expect(str.startsWith(start), `${str} does not start with ${start}`).to.be.true
 }
 
+function expectNotStartWith (str: string, start: string) {
+  expect(str.startsWith(start), `${str} does not start with ${start}`).to.be.false
+}
+
 async function expectLogDoesNotContain (server: PeerTubeServer, str: string) {
   const content = await server.servers.getLogContent()
 
@@ -92,6 +96,7 @@ export {
   expectLogDoesNotContain,
   testFileExistsOrNot,
   expectStartWith,
+  expectNotStartWith,
   checkBadStartPagination,
   checkBadCountPagination,
   checkBadSortPagination
-- 
cgit v1.2.3