Correctly cleanup sql command

[github/Chocobozzz/PeerTube.git] / server / helpers / requests.ts

diff --git a/server/helpers/requests.ts b/server/helpers/requests.ts
index 36e69458e533064a584e93a061d6361b57dc4db6..1a3cc1b5b6f38492aeae086b1e42d0d01315f7bf 100644 (file)
--- a/server/helpers/requests.ts
+++ b/server/helpers/requests.ts
@@ -1,20 +1,23 @@
 import { createWriteStream, remove } from 'fs-extra'
-import got, { CancelableRequest, Options as GotOptions, RequestError } from 'got'
-import { join } from 'path'
-import { CONFIG } from '../initializers/config'
-import { ACTIVITY_PUB, PEERTUBE_VERSION, REQUEST_TIMEOUT, WEBSERVER } from '../initializers/constants'
+import got, { CancelableRequest, NormalizedOptions, Options as GotOptions, RequestError, Response } from 'got'
+import { HttpProxyAgent, HttpsProxyAgent } from 'hpagent'
+import { ACTIVITY_PUB, BINARY_CONTENT_TYPES, PEERTUBE_VERSION, REQUEST_TIMEOUTS, WEBSERVER } from '../initializers/constants'
 import { pipelinePromise } from './core-utils'
-import { processImage } from './image-utils'
-import { logger } from './logger'
+import { logger, loggerTagsFactory } from './logger'
+import { getProxy, isProxyEnabled } from './proxy'
+
+const lTags = loggerTagsFactory('request')
+
+const httpSignature = require('@peertube/http-signature')
 
 export interface PeerTubeRequestError extends Error {
   statusCode?: number
   responseBody?: any
+  responseHeaders?: any
 }
 
-const httpSignature = require('http-signature')
-
 type PeerTubeRequestOptions = {
+  timeout?: number
   activityPub?: boolean
   bodyKBLimit?: number // 1MB
   httpSignature?: {
@@ -24,11 +27,12 @@ type PeerTubeRequestOptions = {
     key: string
     headers: string[]
   }
-  timeout?: number
   jsonResponse?: boolean
 } & Pick<GotOptions, 'headers' | 'json' | 'method' | 'searchParams'>
 
 const peertubeGot = got.extend({
+  ...getAgent(),
+
   headers: {
     'user-agent': getUserAgent()
   },
@@ -45,7 +49,7 @@ const peertubeGot = got.extend({
       promiseOrStream.on('downloadProgress', progress => {
         if (progress.transferred > bodyLimit && progress.percent !== 1) {
           const message = `Exceeded the download limit of ${bodyLimit} B`
-          logger.warn(message)
+          logger.warn(message, lTags())
 
           // CancelableRequest
           if (promiseOrStream.cancel) {
@@ -81,11 +85,14 @@ const peertubeGot = got.extend({
           }
 
           httpSignature.signRequest({
-            getHeader: function (header) {
-              return options.headers[header]
+            getHeader: function (header: string) {
+              const value = options.headers[header.toLowerCase()]
+
+              if (!value) logger.warn('Unknown header requested by http-signature.', { headers: options.headers, header })
+              return value
             },
 
-            setHeader: function (header, value) {
+            setHeader: function (header: string, value: string) {
               options.headers[header] = value
             },
 
@@ -93,10 +100,12 @@ const peertubeGot = got.extend({
             path
           }, httpSignatureOptions)
         }
-      },
+      }
+    ],
 
-      (options: GotOptions) => {
-        options.timeout = REQUEST_TIMEOUT
+    beforeRetry: [
+      (_options: NormalizedOptions, error: RequestError, retryCount: number) => {
+        logger.debug('Retrying request to %s.', error.request.requestUrl, { retryCount, error: buildRequestError(error), ...lTags() })
       }
     ]
   }
@@ -121,7 +130,7 @@ async function doRequestAndSaveToFile (
   destPath: string,
   options: PeerTubeRequestOptions = {}
 ) {
-  const gotOptions = buildGotOptions(options)
+  const gotOptions = buildGotOptions({ ...options, timeout: options.timeout ?? REQUEST_TIMEOUTS.FILE })
 
   const outFile = createWriteStream(destPath)
 
@@ -132,24 +141,33 @@ async function doRequestAndSaveToFile (
     )
   } catch (err) {
     remove(destPath)
-      .catch(err => logger.error('Cannot remove %s after request failure.', destPath, { err }))
+      .catch(err => logger.error('Cannot remove %s after request failure.', destPath, { err, ...lTags() }))
 
     throw buildRequestError(err)
   }
 }
 
-async function downloadImage (url: string, destDir: string, destName: string, size: { width: number, height: number }) {
-  const tmpPath = join(CONFIG.STORAGE.TMP_DIR, 'pending-' + destName)
-  await doRequestAndSaveToFile(url, tmpPath)
+function getAgent () {
+  if (!isProxyEnabled()) return {}
 
-  const destPath = join(destDir, destName)
+  const proxy = getProxy()
 
-  try {
-    await processImage(tmpPath, destPath, size)
-  } catch (err) {
-    await remove(tmpPath)
+  logger.info('Using proxy %s.', proxy, lTags())
 
-    throw err
+  const proxyAgentOptions = {
+    keepAlive: true,
+    keepAliveMsecs: 1000,
+    maxSockets: 256,
+    maxFreeSockets: 256,
+    scheduling: 'lifo' as 'lifo',
+    proxy
+  }
+
+  return {
+    agent: {
+      http: new HttpProxyAgent(proxyAgentOptions),
+      https: new HttpsProxyAgent(proxyAgentOptions)
+    }
   }
 }
 
@@ -157,13 +175,32 @@ function getUserAgent () {
   return `PeerTube/${PEERTUBE_VERSION} (+${WEBSERVER.URL})`
 }
 
+function isBinaryResponse (result: Response<any>) {
+  return BINARY_CONTENT_TYPES.has(result.headers['content-type'])
+}
+
+async function findLatestRedirection (url: string, options: PeerTubeRequestOptions, iteration = 1) {
+  if (iteration > 10) throw new Error('Too much iterations to find final URL ' + url)
+
+  const { headers } = await peertubeGot(url, { followRedirect: false, ...buildGotOptions(options) })
+
+  if (headers.location) return findLatestRedirection(headers.location, options, iteration + 1)
+
+  return url
+}
+
 // ---------------------------------------------------------------------------
 
 export {
+  PeerTubeRequestOptions,
+
   doRequest,
   doJSONRequest,
   doRequestAndSaveToFile,
-  downloadImage
+  isBinaryResponse,
+  getAgent,
+  findLatestRedirection,
+  peertubeGot
 }
 
 // ---------------------------------------------------------------------------
@@ -186,9 +223,10 @@ function buildGotOptions (options: PeerTubeRequestOptions) {
   return {
     method: options.method,
     dnsCache: true,
+    timeout: options.timeout ?? REQUEST_TIMEOUTS.DEFAULT,
     json: options.json,
     searchParams: options.searchParams,
-    timeout: options.timeout ?? REQUEST_TIMEOUT,
+    retry: 2,
     headers,
     context
   }
@@ -201,6 +239,7 @@ function buildRequestError (error: RequestError) {
 
   if (error.response) {
     newError.responseBody = error.response.body
+    newError.responseHeaders = error.response.headers
     newError.statusCode = error.response.statusCode
   }