server/helpers/requests.ts

   1 import { createWriteStream, remove } from 'fs-extra'
   2 import got, { CancelableRequest, Options as GotOptions, RequestError } from 'got'
   3 import { HttpProxyAgent, HttpsProxyAgent } from 'hpagent'
   4 import { join } from 'path'
   5 import { CONFIG } from '../initializers/config'
   6 import { ACTIVITY_PUB, PEERTUBE_VERSION, REQUEST_TIMEOUT, WEBSERVER } from '../initializers/constants'
   7 import { pipelinePromise } from './core-utils'
   8 import { processImage } from './image-utils'
   9 import { logger } from './logger'
  10 import { getProxy, isProxyEnabled } from './proxy'
  11
  12 const httpSignature = require('http-signature')
  13
  14 export interface PeerTubeRequestError extends Error {
  15   statusCode?: number
  16   responseBody?: any
  17 }
  18
  19 type PeerTubeRequestOptions = {
  20   activityPub?: boolean
  21   bodyKBLimit?: number // 1MB
  22   httpSignature?: {
  23     algorithm: string
  24     authorizationHeaderName: string
  25     keyId: string
  26     key: string
  27     headers: string[]
  28   }
  29   timeout?: number
  30   jsonResponse?: boolean
  31 } & Pick<GotOptions, 'headers' | 'json' | 'method' | 'searchParams'>
  32
  33 const peertubeGot = got.extend({
  34   ...getAgent(),
  35
  36   headers: {
  37     'user-agent': getUserAgent()
  38   },
  39
  40   handlers: [
  41     (options, next) => {
  42       const promiseOrStream = next(options) as CancelableRequest<any>
  43       const bodyKBLimit = options.context?.bodyKBLimit as number
  44       if (!bodyKBLimit) throw new Error('No KB limit for this request')
  45
  46       const bodyLimit = bodyKBLimit * 1000
  47
  48       /* eslint-disable @typescript-eslint/no-floating-promises */
  49       promiseOrStream.on('downloadProgress', progress => {
  50         if (progress.transferred > bodyLimit && progress.percent !== 1) {
  51           const message = `Exceeded the download limit of ${bodyLimit} B`
  52           logger.warn(message)
  53
  54           // CancelableRequest
  55           if (promiseOrStream.cancel) {
  56             promiseOrStream.cancel()
  57             return
  58           }
  59
  60           // Stream
  61           (promiseOrStream as any).destroy()
  62         }
  63       })
  64
  65       return promiseOrStream
  66     }
  67   ],
  68
  69   hooks: {
  70     beforeRequest: [
  71       options => {
  72         const headers = options.headers || {}
  73         headers['host'] = options.url.host
  74       },
  75
  76       options => {
  77         const httpSignatureOptions = options.context?.httpSignature
  78
  79         if (httpSignatureOptions) {
  80           const method = options.method ?? 'GET'
  81           const path = options.path ?? options.url.pathname
  82
  83           if (!method || !path) {
  84             throw new Error(`Cannot sign request without method (${method}) or path (${path}) ${options}`)
  85           }
  86
  87           httpSignature.signRequest({
  88             getHeader: function (header) {
  89               return options.headers[header]
  90             },
  91
  92             setHeader: function (header, value) {
  93               options.headers[header] = value
  94             },
  95
  96             method,
  97             path
  98           }, httpSignatureOptions)
  99         }
 100       },
 101
 102       (options: GotOptions) => {
 103         options.timeout = REQUEST_TIMEOUT
 104       }
 105     ]
 106   }
 107 })
 108
 109 function doRequest (url: string, options: PeerTubeRequestOptions = {}) {
 110   const gotOptions = buildGotOptions(options)
 111
 112   return peertubeGot(url, gotOptions)
 113     .catch(err => { throw buildRequestError(err) })
 114 }
 115
 116 function doJSONRequest <T> (url: string, options: PeerTubeRequestOptions = {}) {
 117   const gotOptions = buildGotOptions(options)
 118
 119   return peertubeGot<T>(url, { ...gotOptions, responseType: 'json' })
 120     .catch(err => { throw buildRequestError(err) })
 121 }
 122
 123 async function doRequestAndSaveToFile (
 124   url: string,
 125   destPath: string,
 126   options: PeerTubeRequestOptions = {}
 127 ) {
 128   const gotOptions = buildGotOptions(options)
 129
 130   const outFile = createWriteStream(destPath)
 131
 132   try {
 133     await pipelinePromise(
 134       peertubeGot.stream(url, gotOptions),
 135       outFile
 136     )
 137   } catch (err) {
 138     remove(destPath)
 139       .catch(err => logger.error('Cannot remove %s after request failure.', destPath, { err }))
 140
 141     throw buildRequestError(err)
 142   }
 143 }
 144
 145 async function downloadImage (url: string, destDir: string, destName: string, size: { width: number, height: number }) {
 146   const tmpPath = join(CONFIG.STORAGE.TMP_DIR, 'pending-' + destName)
 147   await doRequestAndSaveToFile(url, tmpPath)
 148
 149   const destPath = join(destDir, destName)
 150
 151   try {
 152     await processImage(tmpPath, destPath, size)
 153   } catch (err) {
 154     await remove(tmpPath)
 155
 156     throw err
 157   }
 158 }
 159
 160 function getAgent () {
 161   if (!isProxyEnabled()) return {}
 162
 163   const proxy = getProxy()
 164
 165   logger.info('Using proxy %s.', proxy)
 166
 167   const proxyAgentOptions = {
 168     keepAlive: true,
 169     keepAliveMsecs: 1000,
 170     maxSockets: 256,
 171     maxFreeSockets: 256,
 172     scheduling: 'lifo' as 'lifo',
 173     proxy
 174   }
 175
 176   return {
 177     agent: {
 178       http: new HttpProxyAgent(proxyAgentOptions),
 179       https: new HttpsProxyAgent(proxyAgentOptions)
 180     }
 181   }
 182 }
 183
 184 function getUserAgent () {
 185   return `PeerTube/${PEERTUBE_VERSION} (+${WEBSERVER.URL})`
 186 }
 187
 188 // ---------------------------------------------------------------------------
 189
 190 export {
 191   doRequest,
 192   doJSONRequest,
 193   doRequestAndSaveToFile,
 194   downloadImage
 195 }
 196
 197 // ---------------------------------------------------------------------------
 198
 199 function buildGotOptions (options: PeerTubeRequestOptions) {
 200   const { activityPub, bodyKBLimit = 1000 } = options
 201
 202   const context = { bodyKBLimit, httpSignature: options.httpSignature }
 203
 204   let headers = options.headers || {}
 205
 206   if (!headers.date) {
 207     headers = { ...headers, date: new Date().toUTCString() }
 208   }
 209
 210   if (activityPub && !headers.accept) {
 211     headers = { ...headers, accept: ACTIVITY_PUB.ACCEPT_HEADER }
 212   }
 213
 214   return {
 215     method: options.method,
 216     dnsCache: true,
 217     json: options.json,
 218     searchParams: options.searchParams,
 219     timeout: options.timeout ?? REQUEST_TIMEOUT,
 220     headers,
 221     context
 222   }
 223 }
 224
 225 function buildRequestError (error: RequestError) {
 226   const newError: PeerTubeRequestError = new Error(error.message)
 227   newError.name = error.name
 228   newError.stack = error.stack
 229
 230   if (error.response) {
 231     newError.responseBody = error.response.body
 232     newError.statusCode = error.response.statusCode
 233   }
 234
 235   return newError
 236 }