server/helpers/requests.ts

   1 import { createWriteStream, remove } from 'fs-extra'
   2 import got, { CancelableRequest, NormalizedOptions, Options as GotOptions, RequestError, Response } from 'got'
   3 import { HttpProxyAgent, HttpsProxyAgent } from 'hpagent'
   4 import { ACTIVITY_PUB, BINARY_CONTENT_TYPES, PEERTUBE_VERSION, REQUEST_TIMEOUTS, WEBSERVER } from '../initializers/constants'
   5 import { pipelinePromise } from './core-utils'
   6 import { logger, loggerTagsFactory } from './logger'
   7 import { getProxy, isProxyEnabled } from './proxy'
   8
   9 const lTags = loggerTagsFactory('request')
  10
  11 const httpSignature = require('@peertube/http-signature')
  12
  13 export interface PeerTubeRequestError extends Error {
  14   statusCode?: number
  15   responseBody?: any
  16   responseHeaders?: any
  17 }
  18
  19 type PeerTubeRequestOptions = {
  20   timeout?: number
  21   activityPub?: boolean
  22   bodyKBLimit?: number // 1MB
  23   httpSignature?: {
  24     algorithm: string
  25     authorizationHeaderName: string
  26     keyId: string
  27     key: string
  28     headers: string[]
  29   }
  30   jsonResponse?: boolean
  31 } & Pick<GotOptions, 'headers' | 'json' | 'method' | 'searchParams'>
  32
  33 const peertubeGot = got.extend({
  34   ...getAgent(),
  35
  36   headers: {
  37     'user-agent': getUserAgent()
  38   },
  39
  40   handlers: [
  41     (options, next) => {
  42       const promiseOrStream = next(options) as CancelableRequest<any>
  43       const bodyKBLimit = options.context?.bodyKBLimit as number
  44       if (!bodyKBLimit) throw new Error('No KB limit for this request')
  45
  46       const bodyLimit = bodyKBLimit * 1000
  47
  48       /* eslint-disable @typescript-eslint/no-floating-promises */
  49       promiseOrStream.on('downloadProgress', progress => {
  50         if (progress.transferred > bodyLimit && progress.percent !== 1) {
  51           const message = `Exceeded the download limit of ${bodyLimit} B`
  52           logger.warn(message, lTags())
  53
  54           // CancelableRequest
  55           if (promiseOrStream.cancel) {
  56             promiseOrStream.cancel()
  57             return
  58           }
  59
  60           // Stream
  61           (promiseOrStream as any).destroy()
  62         }
  63       })
  64
  65       return promiseOrStream
  66     }
  67   ],
  68
  69   hooks: {
  70     beforeRequest: [
  71       options => {
  72         const headers = options.headers || {}
  73         headers['host'] = options.url.host
  74       },
  75
  76       options => {
  77         const httpSignatureOptions = options.context?.httpSignature
  78
  79         if (httpSignatureOptions) {
  80           const method = options.method ?? 'GET'
  81           const path = options.path ?? options.url.pathname
  82
  83           if (!method || !path) {
  84             throw new Error(`Cannot sign request without method (${method}) or path (${path}) ${options}`)
  85           }
  86
  87           httpSignature.signRequest({
  88             getHeader: function (header) {
  89               return options.headers[header]
  90             },
  91
  92             setHeader: function (header, value) {
  93               options.headers[header] = value
  94             },
  95
  96             method,
  97             path
  98           }, httpSignatureOptions)
  99         }
 100       }
 101     ],
 102
 103     beforeRetry: [
 104       (_options: NormalizedOptions, error: RequestError, retryCount: number) => {
 105         logger.debug('Retrying request to %s.', error.request.requestUrl, { retryCount, error: buildRequestError(error), ...lTags() })
 106       }
 107     ]
 108   }
 109 })
 110
 111 function doRequest (url: string, options: PeerTubeRequestOptions = {}) {
 112   const gotOptions = buildGotOptions(options)
 113
 114   return peertubeGot(url, gotOptions)
 115     .catch(err => { throw buildRequestError(err) })
 116 }
 117
 118 function doJSONRequest <T> (url: string, options: PeerTubeRequestOptions = {}) {
 119   const gotOptions = buildGotOptions(options)
 120
 121   return peertubeGot<T>(url, { ...gotOptions, responseType: 'json' })
 122     .catch(err => { throw buildRequestError(err) })
 123 }
 124
 125 async function doRequestAndSaveToFile (
 126   url: string,
 127   destPath: string,
 128   options: PeerTubeRequestOptions = {}
 129 ) {
 130   const gotOptions = buildGotOptions({ ...options, timeout: options.timeout ?? REQUEST_TIMEOUTS.FILE })
 131
 132   const outFile = createWriteStream(destPath)
 133
 134   try {
 135     await pipelinePromise(
 136       peertubeGot.stream(url, gotOptions),
 137       outFile
 138     )
 139   } catch (err) {
 140     remove(destPath)
 141       .catch(err => logger.error('Cannot remove %s after request failure.', destPath, { err, ...lTags() }))
 142
 143     throw buildRequestError(err)
 144   }
 145 }
 146
 147 function getAgent () {
 148   if (!isProxyEnabled()) return {}
 149
 150   const proxy = getProxy()
 151
 152   logger.info('Using proxy %s.', proxy, lTags())
 153
 154   const proxyAgentOptions = {
 155     keepAlive: true,
 156     keepAliveMsecs: 1000,
 157     maxSockets: 256,
 158     maxFreeSockets: 256,
 159     scheduling: 'lifo' as 'lifo',
 160     proxy
 161   }
 162
 163   return {
 164     agent: {
 165       http: new HttpProxyAgent(proxyAgentOptions),
 166       https: new HttpsProxyAgent(proxyAgentOptions)
 167     }
 168   }
 169 }
 170
 171 function getUserAgent () {
 172   return `PeerTube/${PEERTUBE_VERSION} (+${WEBSERVER.URL})`
 173 }
 174
 175 function isBinaryResponse (result: Response<any>) {
 176   return BINARY_CONTENT_TYPES.has(result.headers['content-type'])
 177 }
 178
 179 async function findLatestRedirection (url: string, options: PeerTubeRequestOptions, iteration = 1) {
 180   if (iteration > 10) throw new Error('Too much iterations to find final URL ' + url)
 181
 182   const { headers } = await peertubeGot(url, { followRedirect: false, ...buildGotOptions(options) })
 183
 184   if (headers.location) return findLatestRedirection(headers.location, options, iteration + 1)
 185
 186   return url
 187 }
 188
 189 // ---------------------------------------------------------------------------
 190
 191 export {
 192   doRequest,
 193   doJSONRequest,
 194   doRequestAndSaveToFile,
 195   isBinaryResponse,
 196   getAgent,
 197   findLatestRedirection,
 198   peertubeGot
 199 }
 200
 201 // ---------------------------------------------------------------------------
 202
 203 function buildGotOptions (options: PeerTubeRequestOptions) {
 204   const { activityPub, bodyKBLimit = 1000 } = options
 205
 206   const context = { bodyKBLimit, httpSignature: options.httpSignature }
 207
 208   let headers = options.headers || {}
 209
 210   if (!headers.date) {
 211     headers = { ...headers, date: new Date().toUTCString() }
 212   }
 213
 214   if (activityPub && !headers.accept) {
 215     headers = { ...headers, accept: ACTIVITY_PUB.ACCEPT_HEADER }
 216   }
 217
 218   return {
 219     method: options.method,
 220     dnsCache: true,
 221     timeout: options.timeout ?? REQUEST_TIMEOUTS.DEFAULT,
 222     json: options.json,
 223     searchParams: options.searchParams,
 224     retry: 2,
 225     headers,
 226     context
 227   }
 228 }
 229
 230 function buildRequestError (error: RequestError) {
 231   const newError: PeerTubeRequestError = new Error(error.message)
 232   newError.name = error.name
 233   newError.stack = error.stack
 234
 235   if (error.response) {
 236     newError.responseBody = error.response.body
 237     newError.responseHeaders = error.response.headers
 238     newError.statusCode = error.response.statusCode
 239   }
 240
 241   return newError
 242 }