server/helpers/requests.ts

   1 import { createWriteStream, remove } from 'fs-extra'
   2 import got, { CancelableRequest, Options as GotOptions, RequestError, Response } from 'got'
   3 import { HttpProxyAgent, HttpsProxyAgent } from 'hpagent'
   4 import { join } from 'path'
   5 import { CONFIG } from '../initializers/config'
   6 import { ACTIVITY_PUB, BINARY_CONTENT_TYPES, PEERTUBE_VERSION, REQUEST_TIMEOUT, WEBSERVER } from '../initializers/constants'
   7 import { pipelinePromise } from './core-utils'
   8 import { processImage } from './image-utils'
   9 import { logger } from './logger'
  10 import { getProxy, isProxyEnabled } from './proxy'
  11
  12 const httpSignature = require('@peertube/http-signature')
  13
  14 export interface PeerTubeRequestError extends Error {
  15   statusCode?: number
  16   responseBody?: any
  17 }
  18
  19 type PeerTubeRequestOptions = {
  20   activityPub?: boolean
  21   bodyKBLimit?: number // 1MB
  22   httpSignature?: {
  23     algorithm: string
  24     authorizationHeaderName: string
  25     keyId: string
  26     key: string
  27     headers: string[]
  28   }
  29   jsonResponse?: boolean
  30 } & Pick<GotOptions, 'headers' | 'json' | 'method' | 'searchParams'>
  31
  32 const peertubeGot = got.extend({
  33   ...getAgent(),
  34
  35   headers: {
  36     'user-agent': getUserAgent()
  37   },
  38
  39   handlers: [
  40     (options, next) => {
  41       const promiseOrStream = next(options) as CancelableRequest<any>
  42       const bodyKBLimit = options.context?.bodyKBLimit as number
  43       if (!bodyKBLimit) throw new Error('No KB limit for this request')
  44
  45       const bodyLimit = bodyKBLimit * 1000
  46
  47       /* eslint-disable @typescript-eslint/no-floating-promises */
  48       promiseOrStream.on('downloadProgress', progress => {
  49         if (progress.transferred > bodyLimit && progress.percent !== 1) {
  50           const message = `Exceeded the download limit of ${bodyLimit} B`
  51           logger.warn(message)
  52
  53           // CancelableRequest
  54           if (promiseOrStream.cancel) {
  55             promiseOrStream.cancel()
  56             return
  57           }
  58
  59           // Stream
  60           (promiseOrStream as any).destroy()
  61         }
  62       })
  63
  64       return promiseOrStream
  65     }
  66   ],
  67
  68   hooks: {
  69     beforeRequest: [
  70       options => {
  71         const headers = options.headers || {}
  72         headers['host'] = options.url.host
  73       },
  74
  75       options => {
  76         const httpSignatureOptions = options.context?.httpSignature
  77
  78         if (httpSignatureOptions) {
  79           const method = options.method ?? 'GET'
  80           const path = options.path ?? options.url.pathname
  81
  82           if (!method || !path) {
  83             throw new Error(`Cannot sign request without method (${method}) or path (${path}) ${options}`)
  84           }
  85
  86           httpSignature.signRequest({
  87             getHeader: function (header) {
  88               return options.headers[header]
  89             },
  90
  91             setHeader: function (header, value) {
  92               options.headers[header] = value
  93             },
  94
  95             method,
  96             path
  97           }, httpSignatureOptions)
  98         }
  99       }
 100     ]
 101   }
 102 })
 103
 104 function doRequest (url: string, options: PeerTubeRequestOptions = {}) {
 105   const gotOptions = buildGotOptions(options)
 106
 107   return peertubeGot(url, gotOptions)
 108     .catch(err => { throw buildRequestError(err) })
 109 }
 110
 111 function doJSONRequest <T> (url: string, options: PeerTubeRequestOptions = {}) {
 112   const gotOptions = buildGotOptions(options)
 113
 114   return peertubeGot<T>(url, { ...gotOptions, responseType: 'json' })
 115     .catch(err => { throw buildRequestError(err) })
 116 }
 117
 118 async function doRequestAndSaveToFile (
 119   url: string,
 120   destPath: string,
 121   options: PeerTubeRequestOptions = {}
 122 ) {
 123   const gotOptions = buildGotOptions(options)
 124
 125   const outFile = createWriteStream(destPath)
 126
 127   try {
 128     await pipelinePromise(
 129       peertubeGot.stream(url, gotOptions),
 130       outFile
 131     )
 132   } catch (err) {
 133     remove(destPath)
 134       .catch(err => logger.error('Cannot remove %s after request failure.', destPath, { err }))
 135
 136     throw buildRequestError(err)
 137   }
 138 }
 139
 140 async function downloadImage (url: string, destDir: string, destName: string, size: { width: number, height: number }) {
 141   const tmpPath = join(CONFIG.STORAGE.TMP_DIR, 'pending-' + destName)
 142   await doRequestAndSaveToFile(url, tmpPath)
 143
 144   const destPath = join(destDir, destName)
 145
 146   try {
 147     await processImage(tmpPath, destPath, size)
 148   } catch (err) {
 149     await remove(tmpPath)
 150
 151     throw err
 152   }
 153 }
 154
 155 function getAgent () {
 156   if (!isProxyEnabled()) return {}
 157
 158   const proxy = getProxy()
 159
 160   logger.info('Using proxy %s.', proxy)
 161
 162   const proxyAgentOptions = {
 163     keepAlive: true,
 164     keepAliveMsecs: 1000,
 165     maxSockets: 256,
 166     maxFreeSockets: 256,
 167     scheduling: 'lifo' as 'lifo',
 168     proxy
 169   }
 170
 171   return {
 172     agent: {
 173       http: new HttpProxyAgent(proxyAgentOptions),
 174       https: new HttpsProxyAgent(proxyAgentOptions)
 175     }
 176   }
 177 }
 178
 179 function getUserAgent () {
 180   return `PeerTube/${PEERTUBE_VERSION} (+${WEBSERVER.URL})`
 181 }
 182
 183 function isBinaryResponse (result: Response<any>) {
 184   return BINARY_CONTENT_TYPES.has(result.headers['content-type'])
 185 }
 186
 187 async function findLatestRedirection (url: string, options: PeerTubeRequestOptions, iteration = 1) {
 188   if (iteration > 10) throw new Error('Too much iterations to find final URL ' + url)
 189
 190   const { headers } = await peertubeGot(url, { followRedirect: false, ...buildGotOptions(options) })
 191
 192   if (headers.location) return findLatestRedirection(headers.location, options, iteration + 1)
 193
 194   return url
 195 }
 196
 197 // ---------------------------------------------------------------------------
 198
 199 export {
 200   doRequest,
 201   doJSONRequest,
 202   doRequestAndSaveToFile,
 203   isBinaryResponse,
 204   downloadImage,
 205   findLatestRedirection,
 206   peertubeGot
 207 }
 208
 209 // ---------------------------------------------------------------------------
 210
 211 function buildGotOptions (options: PeerTubeRequestOptions) {
 212   const { activityPub, bodyKBLimit = 1000 } = options
 213
 214   const context = { bodyKBLimit, httpSignature: options.httpSignature }
 215
 216   let headers = options.headers || {}
 217
 218   if (!headers.date) {
 219     headers = { ...headers, date: new Date().toUTCString() }
 220   }
 221
 222   if (activityPub && !headers.accept) {
 223     headers = { ...headers, accept: ACTIVITY_PUB.ACCEPT_HEADER }
 224   }
 225
 226   return {
 227     method: options.method,
 228     dnsCache: true,
 229     timeout: REQUEST_TIMEOUT,
 230     json: options.json,
 231     searchParams: options.searchParams,
 232     headers,
 233     context
 234   }
 235 }
 236
 237 function buildRequestError (error: RequestError) {
 238   const newError: PeerTubeRequestError = new Error(error.message)
 239   newError.name = error.name
 240   newError.stack = error.stack
 241
 242   if (error.response) {
 243     newError.responseBody = error.response.body
 244     newError.statusCode = error.response.statusCode
 245   }
 246
 247   return newError
 248 }