server/helpers/requests.ts

   1 import { createWriteStream, remove } from 'fs-extra'
   2 import got, { CancelableRequest, NormalizedOptions, Options as GotOptions, RequestError, Response } from 'got'
   3 import { HttpProxyAgent, HttpsProxyAgent } from 'hpagent'
   4 import { join } from 'path'
   5 import { CONFIG } from '../initializers/config'
   6 import { ACTIVITY_PUB, BINARY_CONTENT_TYPES, PEERTUBE_VERSION, REQUEST_TIMEOUTS, WEBSERVER } from '../initializers/constants'
   7 import { pipelinePromise } from './core-utils'
   8 import { processImage } from './image-utils'
   9 import { logger, loggerTagsFactory } from './logger'
  10 import { getProxy, isProxyEnabled } from './proxy'
  11
  12 const lTags = loggerTagsFactory('request')
  13
  14 const httpSignature = require('@peertube/http-signature')
  15
  16 export interface PeerTubeRequestError extends Error {
  17   statusCode?: number
  18   responseBody?: any
  19   responseHeaders?: any
  20 }
  21
  22 type PeerTubeRequestOptions = {
  23   timeout?: number
  24   activityPub?: boolean
  25   bodyKBLimit?: number // 1MB
  26   httpSignature?: {
  27     algorithm: string
  28     authorizationHeaderName: string
  29     keyId: string
  30     key: string
  31     headers: string[]
  32   }
  33   jsonResponse?: boolean
  34 } & Pick<GotOptions, 'headers' | 'json' | 'method' | 'searchParams'>
  35
  36 const peertubeGot = got.extend({
  37   ...getAgent(),
  38
  39   headers: {
  40     'user-agent': getUserAgent()
  41   },
  42
  43   handlers: [
  44     (options, next) => {
  45       const promiseOrStream = next(options) as CancelableRequest<any>
  46       const bodyKBLimit = options.context?.bodyKBLimit as number
  47       if (!bodyKBLimit) throw new Error('No KB limit for this request')
  48
  49       const bodyLimit = bodyKBLimit * 1000
  50
  51       /* eslint-disable @typescript-eslint/no-floating-promises */
  52       promiseOrStream.on('downloadProgress', progress => {
  53         if (progress.transferred > bodyLimit && progress.percent !== 1) {
  54           const message = `Exceeded the download limit of ${bodyLimit} B`
  55           logger.warn(message, lTags())
  56
  57           // CancelableRequest
  58           if (promiseOrStream.cancel) {
  59             promiseOrStream.cancel()
  60             return
  61           }
  62
  63           // Stream
  64           (promiseOrStream as any).destroy()
  65         }
  66       })
  67
  68       return promiseOrStream
  69     }
  70   ],
  71
  72   hooks: {
  73     beforeRequest: [
  74       options => {
  75         const headers = options.headers || {}
  76         headers['host'] = options.url.host
  77       },
  78
  79       options => {
  80         const httpSignatureOptions = options.context?.httpSignature
  81
  82         if (httpSignatureOptions) {
  83           const method = options.method ?? 'GET'
  84           const path = options.path ?? options.url.pathname
  85
  86           if (!method || !path) {
  87             throw new Error(`Cannot sign request without method (${method}) or path (${path}) ${options}`)
  88           }
  89
  90           httpSignature.signRequest({
  91             getHeader: function (header) {
  92               return options.headers[header]
  93             },
  94
  95             setHeader: function (header, value) {
  96               options.headers[header] = value
  97             },
  98
  99             method,
 100             path
 101           }, httpSignatureOptions)
 102         }
 103       }
 104     ],
 105
 106     beforeRetry: [
 107       (_options: NormalizedOptions, error: RequestError, retryCount: number) => {
 108         logger.debug('Retrying request to %s.', error.request.requestUrl, { retryCount, error: buildRequestError(error), ...lTags() })
 109       }
 110     ]
 111   }
 112 })
 113
 114 function doRequest (url: string, options: PeerTubeRequestOptions = {}) {
 115   const gotOptions = buildGotOptions(options)
 116
 117   return peertubeGot(url, gotOptions)
 118     .catch(err => { throw buildRequestError(err) })
 119 }
 120
 121 function doJSONRequest <T> (url: string, options: PeerTubeRequestOptions = {}) {
 122   const gotOptions = buildGotOptions(options)
 123
 124   return peertubeGot<T>(url, { ...gotOptions, responseType: 'json' })
 125     .catch(err => { throw buildRequestError(err) })
 126 }
 127
 128 async function doRequestAndSaveToFile (
 129   url: string,
 130   destPath: string,
 131   options: PeerTubeRequestOptions = {}
 132 ) {
 133   const gotOptions = buildGotOptions({ ...options, timeout: options.timeout ?? REQUEST_TIMEOUTS.FILE })
 134
 135   const outFile = createWriteStream(destPath)
 136
 137   try {
 138     await pipelinePromise(
 139       peertubeGot.stream(url, gotOptions),
 140       outFile
 141     )
 142   } catch (err) {
 143     remove(destPath)
 144       .catch(err => logger.error('Cannot remove %s after request failure.', destPath, { err, ...lTags() }))
 145
 146     throw buildRequestError(err)
 147   }
 148 }
 149
 150 async function downloadImage (url: string, destDir: string, destName: string, size: { width: number, height: number }) {
 151   const tmpPath = join(CONFIG.STORAGE.TMP_DIR, 'pending-' + destName)
 152   await doRequestAndSaveToFile(url, tmpPath)
 153
 154   const destPath = join(destDir, destName)
 155
 156   try {
 157     await processImage(tmpPath, destPath, size)
 158   } catch (err) {
 159     await remove(tmpPath)
 160
 161     throw err
 162   }
 163 }
 164
 165 function getAgent () {
 166   if (!isProxyEnabled()) return {}
 167
 168   const proxy = getProxy()
 169
 170   logger.info('Using proxy %s.', proxy, lTags())
 171
 172   const proxyAgentOptions = {
 173     keepAlive: true,
 174     keepAliveMsecs: 1000,
 175     maxSockets: 256,
 176     maxFreeSockets: 256,
 177     scheduling: 'lifo' as 'lifo',
 178     proxy
 179   }
 180
 181   return {
 182     agent: {
 183       http: new HttpProxyAgent(proxyAgentOptions),
 184       https: new HttpsProxyAgent(proxyAgentOptions)
 185     }
 186   }
 187 }
 188
 189 function getUserAgent () {
 190   return `PeerTube/${PEERTUBE_VERSION} (+${WEBSERVER.URL})`
 191 }
 192
 193 function isBinaryResponse (result: Response<any>) {
 194   return BINARY_CONTENT_TYPES.has(result.headers['content-type'])
 195 }
 196
 197 async function findLatestRedirection (url: string, options: PeerTubeRequestOptions, iteration = 1) {
 198   if (iteration > 10) throw new Error('Too much iterations to find final URL ' + url)
 199
 200   const { headers } = await peertubeGot(url, { followRedirect: false, ...buildGotOptions(options) })
 201
 202   if (headers.location) return findLatestRedirection(headers.location, options, iteration + 1)
 203
 204   return url
 205 }
 206
 207 // ---------------------------------------------------------------------------
 208
 209 export {
 210   doRequest,
 211   doJSONRequest,
 212   doRequestAndSaveToFile,
 213   isBinaryResponse,
 214   downloadImage,
 215   findLatestRedirection,
 216   peertubeGot
 217 }
 218
 219 // ---------------------------------------------------------------------------
 220
 221 function buildGotOptions (options: PeerTubeRequestOptions) {
 222   const { activityPub, bodyKBLimit = 1000 } = options
 223
 224   const context = { bodyKBLimit, httpSignature: options.httpSignature }
 225
 226   let headers = options.headers || {}
 227
 228   if (!headers.date) {
 229     headers = { ...headers, date: new Date().toUTCString() }
 230   }
 231
 232   if (activityPub && !headers.accept) {
 233     headers = { ...headers, accept: ACTIVITY_PUB.ACCEPT_HEADER }
 234   }
 235
 236   return {
 237     method: options.method,
 238     dnsCache: true,
 239     timeout: options.timeout ?? REQUEST_TIMEOUTS.DEFAULT,
 240     json: options.json,
 241     searchParams: options.searchParams,
 242     retry: 2,
 243     headers,
 244     context
 245   }
 246 }
 247
 248 function buildRequestError (error: RequestError) {
 249   const newError: PeerTubeRequestError = new Error(error.message)
 250   newError.name = error.name
 251   newError.stack = error.stack
 252
 253   if (error.response) {
 254     newError.responseBody = error.response.body
 255     newError.responseHeaders = error.response.headers
 256     newError.statusCode = error.response.statusCode
 257   }
 258
 259   return newError
 260 }