Support proxies for PeerTube (#4346)

* Updated with latest code

* Updated Support proxies for PeerTube

* Support Proxies for PeerTube (Updated with change request)

* Cleanup proxy PR

Co-authored-by: Chocobozzz <me@florianbigard.com>

7 files changed, 191 insertions, 3 deletions

diff --git a/package.json b/package.json
index 18bce1123..dc15405ce 100644
--- a/package.json
+++ b/package.json
@@ -99,6 +99,7 @@
     "fs-extra": "^10.0.0",
     "got": "^11.8.2",
     "helmet": "^4.1.0",
+    "hpagent": "^0.1.2",
     "http-problem-details": "^0.1.5",
     "http-signature": "1.3.5",
     "ip-anonymize": "^0.1.0",
@@ -199,6 +200,7 @@
     "marked-man": "^0.7.0",
     "mocha": "^9.0.0",
     "nodemon": "^2.0.1",
+    "proxy": "^1.0.2",
     "socket.io-client": "^4.0.1",
     "source-map-support": "^0.5.0",
     "supertest": "^6.0.1",
diff --git a/server/helpers/proxy.ts b/server/helpers/proxy.ts
new file mode 100644
index 000000000..8b82ccae0
--- /dev/null
+++ b/server/helpers/proxy.ts
@@ -0,0 +1,14 @@
+function getProxy () {
+  return process.env.HTTPS_PROXY ||
+         process.env.HTTP_PROXY ||
+         undefined
+}
+
+function isProxyEnabled () {
+  return !!getProxy()
+}
+
+export {
+  getProxy,
+  isProxyEnabled
+}
diff --git a/server/helpers/requests.ts b/server/helpers/requests.ts
index 36e69458e..e09e23086 100644
--- a/server/helpers/requests.ts
+++ b/server/helpers/requests.ts
@@ -1,19 +1,21 @@
 import { createWriteStream, remove } from 'fs-extra'
 import got, { CancelableRequest, Options as GotOptions, RequestError } from 'got'
+import { HttpProxyAgent, HttpsProxyAgent } from 'hpagent'
 import { join } from 'path'
 import { CONFIG } from '../initializers/config'
 import { ACTIVITY_PUB, PEERTUBE_VERSION, REQUEST_TIMEOUT, WEBSERVER } from '../initializers/constants'
 import { pipelinePromise } from './core-utils'
 import { processImage } from './image-utils'
 import { logger } from './logger'
+import { getProxy, isProxyEnabled } from './proxy'
+
+const httpSignature = require('http-signature')
 
 export interface PeerTubeRequestError extends Error {
   statusCode?: number
   responseBody?: any
 }
 
-const httpSignature = require('http-signature')
-
 type PeerTubeRequestOptions = {
   activityPub?: boolean
   bodyKBLimit?: number // 1MB
@@ -29,6 +31,8 @@ type PeerTubeRequestOptions = {
 } & Pick<GotOptions, 'headers' | 'json' | 'method' | 'searchParams'>
 
 const peertubeGot = got.extend({
+  ...getAgent(),
+
   headers: {
     'user-agent': getUserAgent()
   },
@@ -153,6 +157,30 @@ async function downloadImage (url: string, destDir: string, destName: string, si
   }
 }
 
+function getAgent () {
+  if (!isProxyEnabled()) return {}
+
+  const proxy = getProxy()
+
+  logger.info('Using proxy %s.', proxy)
+
+  const proxyAgentOptions = {
+    keepAlive: true,
+    keepAliveMsecs: 1000,
+    maxSockets: 256,
+    maxFreeSockets: 256,
+    scheduling: 'lifo' as 'lifo',
+    proxy
+  }
+
+  return {
+    agent: {
+      http: new HttpProxyAgent(proxyAgentOptions),
+      https: new HttpsProxyAgent(proxyAgentOptions)
+    }
+  }
+}
+
 function getUserAgent () {
   return `PeerTube/${PEERTUBE_VERSION} (+${WEBSERVER.URL})`
 }
diff --git a/server/tests/api/server/index.ts b/server/tests/api/server/index.ts
index 56e6eb5da..b16a22ee7 100644
--- a/server/tests/api/server/index.ts
+++ b/server/tests/api/server/index.ts
@@ -15,3 +15,4 @@ import './stats'
 import './tracker'
 import './no-client'
 import './plugins'
+import './proxy'
diff --git a/server/tests/api/server/proxy.ts b/server/tests/api/server/proxy.ts
new file mode 100644
index 000000000..d5042ef27
--- /dev/null
+++ b/server/tests/api/server/proxy.ts
@@ -0,0 +1,72 @@
+/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
+
+import 'mocha'
+import * as chai from 'chai'
+import { cleanupTests, createMultipleServers, doubleFollow, PeerTubeServer, setAccessTokensToServers, waitJobs } from '@shared/extra-utils'
+import { MockProxy } from '@shared/extra-utils/mock-servers/mock-proxy'
+
+const expect = chai.expect
+
+describe('Test proxy', function () {
+  let servers: PeerTubeServer[] = []
+  let proxy: MockProxy
+
+  const goodEnv = { HTTP_PROXY: '' }
+  const badEnv = { HTTP_PROXY: 'http://localhost:9000' }
+
+  before(async function () {
+    this.timeout(120000)
+
+    proxy = new MockProxy()
+
+    const proxyPort = await proxy.initialize()
+    servers = await createMultipleServers(2)
+
+    goodEnv.HTTP_PROXY = 'http://localhost:' + proxyPort
+
+    await setAccessTokensToServers(servers)
+    await doubleFollow(servers[0], servers[1])
+  })
+
+  it('Should succeed federation with the appropriate proxy config', async function () {
+    await servers[0].kill()
+    await servers[0].run({}, { env: goodEnv })
+
+    await servers[0].videos.quickUpload({ name: 'video 1' })
+
+    await waitJobs(servers)
+
+    for (const server of servers) {
+      const { total, data } = await server.videos.list()
+      expect(total).to.equal(1)
+      expect(data).to.have.lengthOf(1)
+    }
+  })
+
+  it('Should fail federation with a wrong proxy config', async function () {
+    await servers[0].kill()
+    await servers[0].run({}, { env: badEnv })
+
+    await servers[0].videos.quickUpload({ name: 'video 2' })
+
+    await waitJobs(servers)
+
+    {
+      const { total, data } = await servers[0].videos.list()
+      expect(total).to.equal(2)
+      expect(data).to.have.lengthOf(2)
+    }
+
+    {
+      const { total, data } = await servers[1].videos.list()
+      expect(total).to.equal(1)
+      expect(data).to.have.lengthOf(1)
+    }
+  })
+
+  after(async function () {
+    proxy.terminate()
+
+    await cleanupTests(servers)
+  })
+})
diff --git a/shared/extra-utils/mock-servers/mock-proxy.ts b/shared/extra-utils/mock-servers/mock-proxy.ts
new file mode 100644
index 000000000..5365f87d1
--- /dev/null
+++ b/shared/extra-utils/mock-servers/mock-proxy.ts
@@ -0,0 +1,27 @@
+
+import { createServer, Server } from 'http'
+import * as proxy from 'proxy'
+import { randomInt } from '@shared/core-utils'
+
+class MockProxy {
+  private server: Server
+
+  initialize () {
+    return new Promise<number>(res => {
+      const port = 42501 + randomInt(1, 100)
+
+      this.server = proxy(createServer())
+      this.server.listen(port, () => res(port))
+    })
+  }
+
+  terminate () {
+    if (this.server) this.server.close()
+  }
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+  MockProxy
+}
diff --git a/yarn.lock b/yarn.lock
index 0f4fe3938..1179f273b 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -2002,6 +2002,16 @@ argparse@^2.0.1:
   resolved "https://registry.yarnpkg.com/argparse/-/argparse-2.0.1.tgz#246f50f3ca78a3240f6c997e8a9bd1eac49e4b38"
   integrity sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==
 
+args@5.0.1:
+  version "5.0.1"
+  resolved "https://registry.yarnpkg.com/args/-/args-5.0.1.tgz#4bf298df90a4799a09521362c579278cc2fdd761"
+  integrity sha512-1kqmFCFsPffavQFGt8OxJdIcETti99kySRUPMpOhaGjL6mRJn8HFU1OxKY5bMqfZKUwTQc1mZkAjmGYaVOHFtQ==
+  dependencies:
+    camelcase "5.0.0"
+    chalk "2.4.2"
+    leven "2.1.0"
+    mri "1.1.4"
+
 array-differ@^3.0.0:
   version "3.0.0"
   resolved "https://registry.yarnpkg.com/array-differ/-/array-differ-3.0.0.tgz#3cbb3d0f316810eafcc47624734237d6aee4ae6b"
@@ -2200,6 +2210,11 @@ basic-auth-connect@^1.0.0:
   resolved "https://registry.yarnpkg.com/basic-auth-connect/-/basic-auth-connect-1.0.0.tgz#fdb0b43962ca7b40456a7c2bb48fe173da2d2122"
   integrity sha1-/bC0OWLKe0BFanwrtI/hc9otISI=
 
+basic-auth-parser@0.0.2:
+  version "0.0.2"
+  resolved "https://registry.yarnpkg.com/basic-auth-parser/-/basic-auth-parser-0.0.2.tgz#ce9e71a77f23c1279eecd2659b2a46244c156e41"
+  integrity sha1-zp5xp38jwSee7NJlmypGJEwVbkE=
+
 basic-auth@2.0.1, basic-auth@~2.0.1:
   version "2.0.1"
   resolved "https://registry.yarnpkg.com/basic-auth/-/basic-auth-2.0.1.tgz#b998279bf47ce38344b4f3cf916d4679bbf51e3a"
@@ -2616,6 +2631,11 @@ camelcase-keys@^4.0.0:
     map-obj "^2.0.0"
     quick-lru "^1.0.0"
 
+camelcase@5.0.0:
+  version "5.0.0"
+  resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-5.0.0.tgz#03295527d58bd3cd4aa75363f35b2e8d97be2f42"
+  integrity sha512-faqwZqnWxbxn+F1d399ygeamQNy3lPp/H9H6rNrqYh4FSVCtcY+3cub1MxA8o9mDd55mM8Aghuu/kuyYA6VTsA==
+
 camelcase@^4.1.0:
   version "4.1.0"
   resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-4.1.0.tgz#d545635be1e33c542649c69173e5de6acfae34dd"
@@ -2663,7 +2683,7 @@ chai@^4.1.1:
     pathval "^1.1.1"
     type-detect "^4.0.5"
 
-chalk@^2.0.0, chalk@^2.4.2:
+chalk@2.4.2, chalk@^2.0.0, chalk@^2.4.2:
   version "2.4.2"
   resolved "https://registry.yarnpkg.com/chalk/-/chalk-2.4.2.tgz#cd42541677a54333cf541a49108c1432b44c9424"
   integrity sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==
@@ -4692,6 +4712,11 @@ hosted-git-info@^2.1.4:
   resolved "https://registry.yarnpkg.com/hosted-git-info/-/hosted-git-info-2.8.9.tgz#dffc0bf9a21c02209090f2aa69429e1414daf3f9"
   integrity sha512-mxIDAb9Lsm6DoOJ7xH+5+X4y1LU/4Hi50L9C5sIswK3JzULS4bwk1FvjdBgvYR4bzT4tuUQiC15FE2f5HbLvYw==
 
+hpagent@^0.1.2:
+  version "0.1.2"
+  resolved "https://registry.yarnpkg.com/hpagent/-/hpagent-0.1.2.tgz#cab39c66d4df2d4377dbd212295d878deb9bdaa9"
+  integrity sha512-ePqFXHtSQWAFXYmj+JtOTHr84iNrII4/QRlAAPPE+zqnKy4xJo7Ie1Y4kC7AdB+LxLxSTTzBMASsEcy0q8YyvQ==
+
 html-to-text@8.0.0:
   version "8.0.0"
   resolved "https://registry.yarnpkg.com/html-to-text/-/html-to-text-8.0.0.tgz#5848681a5a38d657a7bb58cf5006d1c29fe64ce3"
@@ -5548,6 +5573,11 @@ latest-version@^5.0.0:
   dependencies:
     package-json "^6.3.0"
 
+leven@2.1.0:
+  version "2.1.0"
+  resolved "https://registry.yarnpkg.com/leven/-/leven-2.1.0.tgz#c2e7a9f772094dee9d34202ae8acce4687875580"
+  integrity sha1-wuep93IJTe6dNCAq6KzORoeHVYA=
+
 levn@^0.4.1:
   version "0.4.1"
   resolved "https://registry.yarnpkg.com/levn/-/levn-0.4.1.tgz#ae4562c007473b932a6200d403268dd2fffc6ade"
@@ -6196,6 +6226,11 @@ mp4-stream@^3.0.0:
     queue-microtask "^1.2.2"
     readable-stream "^3.0.6"
 
+mri@1.1.4:
+  version "1.1.4"
+  resolved "https://registry.yarnpkg.com/mri/-/mri-1.1.4.tgz#7cb1dd1b9b40905f1fac053abe25b6720f44744a"
+  integrity sha512-6y7IjGPm8AzlvoUrwAaw1tLnUBudaS3752vcd8JtrpGGQn+rXIe63LFVHm/YMwtqAuh+LJPCFdlLYPWM1nYn6w==
+
 ms@2.0.0:
   version "2.0.0"
   resolved "https://registry.yarnpkg.com/ms/-/ms-2.0.0.tgz#5608aeadfc00be6c2901df5f9861788de0d597c8"
@@ -7131,6 +7166,15 @@ proxy-addr@~2.0.5:
     forwarded "0.2.0"
     ipaddr.js "1.9.1"
 
+proxy@^1.0.2:
+  version "1.0.2"
+  resolved "https://registry.yarnpkg.com/proxy/-/proxy-1.0.2.tgz#e0cfbe11c0a7a8b238fd2d7134de4e2867578e7f"
+  integrity sha512-KNac2ueWRpjbUh77OAFPZuNdfEqNynm9DD4xHT14CccGpW8wKZwEkN0yjlb7X9G9Z9F55N0Q+1z+WfgAhwYdzQ==
+  dependencies:
+    args "5.0.1"
+    basic-auth-parser "0.0.2"
+    debug "^4.1.1"
+
 pseudomap@^1.0.2:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/pseudomap/-/pseudomap-1.0.2.tgz#f052a28da70e618917ef0a8ac34c1ae5a68286b3"