server/helpers/requests.ts

   1 import { createWriteStream, remove } from 'fs-extra'
   2 import got, { CancelableRequest, Options as GotOptions, RequestError, Response } from 'got'
   3 import { HttpProxyAgent, HttpsProxyAgent } from 'hpagent'
   4 import { join } from 'path'
   5 import { CONFIG } from '../initializers/config'
   6 import { ACTIVITY_PUB, BINARY_CONTENT_TYPES, PEERTUBE_VERSION, REQUEST_TIMEOUT, WEBSERVER } from '../initializers/constants'
   7 import { pipelinePromise } from './core-utils'
   8 import { processImage } from './image-utils'
   9 import { logger, loggerTagsFactory } from './logger'
  10 import { getProxy, isProxyEnabled } from './proxy'
  11
  12 const lTags = loggerTagsFactory('request')
  13
  14 const httpSignature = require('@peertube/http-signature')
  15
  16 export interface PeerTubeRequestError extends Error {
  17   statusCode?: number
  18   responseBody?: any
  19 }
  20
  21 type PeerTubeRequestOptions = {
  22   activityPub?: boolean
  23   bodyKBLimit?: number // 1MB
  24   httpSignature?: {
  25     algorithm: string
  26     authorizationHeaderName: string
  27     keyId: string
  28     key: string
  29     headers: string[]
  30   }
  31   jsonResponse?: boolean
  32 } & Pick<GotOptions, 'headers' | 'json' | 'method' | 'searchParams'>
  33
  34 const peertubeGot = got.extend({
  35   ...getAgent(),
  36
  37   headers: {
  38     'user-agent': getUserAgent()
  39   },
  40
  41   handlers: [
  42     (options, next) => {
  43       const promiseOrStream = next(options) as CancelableRequest<any>
  44       const bodyKBLimit = options.context?.bodyKBLimit as number
  45       if (!bodyKBLimit) throw new Error('No KB limit for this request')
  46
  47       const bodyLimit = bodyKBLimit * 1000
  48
  49       /* eslint-disable @typescript-eslint/no-floating-promises */
  50       promiseOrStream.on('downloadProgress', progress => {
  51         if (progress.transferred > bodyLimit && progress.percent !== 1) {
  52           const message = `Exceeded the download limit of ${bodyLimit} B`
  53           logger.warn(message, lTags())
  54
  55           // CancelableRequest
  56           if (promiseOrStream.cancel) {
  57             promiseOrStream.cancel()
  58             return
  59           }
  60
  61           // Stream
  62           (promiseOrStream as any).destroy()
  63         }
  64       })
  65
  66       return promiseOrStream
  67     }
  68   ],
  69
  70   hooks: {
  71     beforeRequest: [
  72       options => {
  73         const headers = options.headers || {}
  74         headers['host'] = options.url.host
  75       },
  76
  77       options => {
  78         const httpSignatureOptions = options.context?.httpSignature
  79
  80         if (httpSignatureOptions) {
  81           const method = options.method ?? 'GET'
  82           const path = options.path ?? options.url.pathname
  83
  84           if (!method || !path) {
  85             throw new Error(`Cannot sign request without method (${method}) or path (${path}) ${options}`)
  86           }
  87
  88           httpSignature.signRequest({
  89             getHeader: function (header) {
  90               return options.headers[header]
  91             },
  92
  93             setHeader: function (header, value) {
  94               options.headers[header] = value
  95             },
  96
  97             method,
  98             path
  99           }, httpSignatureOptions)
 100         }
 101       }
 102     ]
 103   }
 104 })
 105
 106 function doRequest (url: string, options: PeerTubeRequestOptions = {}) {
 107   const gotOptions = buildGotOptions(options)
 108
 109   return peertubeGot(url, gotOptions)
 110     .on('retry', logRetryFactory(url))
 111     .catch(err => { throw buildRequestError(err) })
 112 }
 113
 114 function doJSONRequest <T> (url: string, options: PeerTubeRequestOptions = {}) {
 115   const gotOptions = buildGotOptions(options)
 116
 117   return peertubeGot<T>(url, { ...gotOptions, responseType: 'json' })
 118     .on('retry', logRetryFactory(url))
 119     .catch(err => { throw buildRequestError(err) })
 120 }
 121
 122 async function doRequestAndSaveToFile (
 123   url: string,
 124   destPath: string,
 125   options: PeerTubeRequestOptions = {}
 126 ) {
 127   const gotOptions = buildGotOptions(options)
 128
 129   const outFile = createWriteStream(destPath)
 130
 131   try {
 132     await pipelinePromise(
 133       peertubeGot.stream(url, gotOptions),
 134       outFile
 135     )
 136   } catch (err) {
 137     remove(destPath)
 138       .catch(err => logger.error('Cannot remove %s after request failure.', destPath, { err, ...lTags() }))
 139
 140     throw buildRequestError(err)
 141   }
 142 }
 143
 144 async function downloadImage (url: string, destDir: string, destName: string, size: { width: number, height: number }) {
 145   const tmpPath = join(CONFIG.STORAGE.TMP_DIR, 'pending-' + destName)
 146   await doRequestAndSaveToFile(url, tmpPath)
 147
 148   const destPath = join(destDir, destName)
 149
 150   try {
 151     await processImage(tmpPath, destPath, size)
 152   } catch (err) {
 153     await remove(tmpPath)
 154
 155     throw err
 156   }
 157 }
 158
 159 function getAgent () {
 160   if (!isProxyEnabled()) return {}
 161
 162   const proxy = getProxy()
 163
 164   logger.info('Using proxy %s.', proxy, lTags())
 165
 166   const proxyAgentOptions = {
 167     keepAlive: true,
 168     keepAliveMsecs: 1000,
 169     maxSockets: 256,
 170     maxFreeSockets: 256,
 171     scheduling: 'lifo' as 'lifo',
 172     proxy
 173   }
 174
 175   return {
 176     agent: {
 177       http: new HttpProxyAgent(proxyAgentOptions),
 178       https: new HttpsProxyAgent(proxyAgentOptions)
 179     }
 180   }
 181 }
 182
 183 function getUserAgent () {
 184   return `PeerTube/${PEERTUBE_VERSION} (+${WEBSERVER.URL})`
 185 }
 186
 187 function isBinaryResponse (result: Response<any>) {
 188   return BINARY_CONTENT_TYPES.has(result.headers['content-type'])
 189 }
 190
 191 async function findLatestRedirection (url: string, options: PeerTubeRequestOptions, iteration = 1) {
 192   if (iteration > 10) throw new Error('Too much iterations to find final URL ' + url)
 193
 194   const { headers } = await peertubeGot(url, { followRedirect: false, ...buildGotOptions(options) })
 195
 196   if (headers.location) return findLatestRedirection(headers.location, options, iteration + 1)
 197
 198   return url
 199 }
 200
 201 // ---------------------------------------------------------------------------
 202
 203 export {
 204   doRequest,
 205   doJSONRequest,
 206   doRequestAndSaveToFile,
 207   isBinaryResponse,
 208   downloadImage,
 209   findLatestRedirection,
 210   peertubeGot
 211 }
 212
 213 // ---------------------------------------------------------------------------
 214
 215 function buildGotOptions (options: PeerTubeRequestOptions) {
 216   const { activityPub, bodyKBLimit = 1000 } = options
 217
 218   const context = { bodyKBLimit, httpSignature: options.httpSignature }
 219
 220   let headers = options.headers || {}
 221
 222   if (!headers.date) {
 223     headers = { ...headers, date: new Date().toUTCString() }
 224   }
 225
 226   if (activityPub && !headers.accept) {
 227     headers = { ...headers, accept: ACTIVITY_PUB.ACCEPT_HEADER }
 228   }
 229
 230   return {
 231     method: options.method,
 232     dnsCache: true,
 233     timeout: REQUEST_TIMEOUT,
 234     json: options.json,
 235     searchParams: options.searchParams,
 236     retry: 2,
 237     headers,
 238     context
 239   }
 240 }
 241
 242 function buildRequestError (error: RequestError) {
 243   const newError: PeerTubeRequestError = new Error(error.message)
 244   newError.name = error.name
 245   newError.stack = error.stack
 246
 247   if (error.response) {
 248     newError.responseBody = error.response.body
 249     newError.statusCode = error.response.statusCode
 250   }
 251
 252   return newError
 253 }
 254
 255 function logRetryFactory (url: string) {
 256   return (retryCount: number, error: RequestError) => {
 257     logger.debug('Retrying request to %s.', url, { retryCount, error, ...lTags() })
 258   }
 259 }