server/helpers/requests.ts

   1 import { createWriteStream, remove } from 'fs-extra'
   2 import got, { CancelableRequest, Options as GotOptions, RequestError } from 'got'
   3 import { HttpProxyAgent, HttpsProxyAgent } from 'hpagent'
   4 import { join } from 'path'
   5 import { CONFIG } from '../initializers/config'
   6 import { ACTIVITY_PUB, PEERTUBE_VERSION, REQUEST_TIMEOUT, WEBSERVER } from '../initializers/constants'
   7 import { pipelinePromise } from './core-utils'
   8 import { processImage } from './image-utils'
   9 import { logger } from './logger'
  10 import { getProxy, isProxyEnabled } from './proxy'
  11
  12 const httpSignature = require('http-signature')
  13
  14 export interface PeerTubeRequestError extends Error {
  15   statusCode?: number
  16   responseBody?: any
  17 }
  18
  19 type PeerTubeRequestOptions = {
  20   activityPub?: boolean
  21   bodyKBLimit?: number // 1MB
  22   httpSignature?: {
  23     algorithm: string
  24     authorizationHeaderName: string
  25     keyId: string
  26     key: string
  27     headers: string[]
  28   }
  29   jsonResponse?: boolean
  30 } & Pick<GotOptions, 'headers' | 'json' | 'method' | 'searchParams'>
  31
  32 const peertubeGot = got.extend({
  33   ...getAgent(),
  34
  35   headers: {
  36     'user-agent': getUserAgent()
  37   },
  38
  39   handlers: [
  40     (options, next) => {
  41       const promiseOrStream = next(options) as CancelableRequest<any>
  42       const bodyKBLimit = options.context?.bodyKBLimit as number
  43       if (!bodyKBLimit) throw new Error('No KB limit for this request')
  44
  45       const bodyLimit = bodyKBLimit * 1000
  46
  47       /* eslint-disable @typescript-eslint/no-floating-promises */
  48       promiseOrStream.on('downloadProgress', progress => {
  49         if (progress.transferred > bodyLimit && progress.percent !== 1) {
  50           const message = `Exceeded the download limit of ${bodyLimit} B`
  51           logger.warn(message)
  52
  53           // CancelableRequest
  54           if (promiseOrStream.cancel) {
  55             promiseOrStream.cancel()
  56             return
  57           }
  58
  59           // Stream
  60           (promiseOrStream as any).destroy()
  61         }
  62       })
  63
  64       return promiseOrStream
  65     }
  66   ],
  67
  68   hooks: {
  69     beforeRequest: [
  70       options => {
  71         const headers = options.headers || {}
  72         headers['host'] = options.url.host
  73       },
  74
  75       options => {
  76         const httpSignatureOptions = options.context?.httpSignature
  77
  78         if (httpSignatureOptions) {
  79           const method = options.method ?? 'GET'
  80           const path = options.path ?? options.url.pathname
  81
  82           if (!method || !path) {
  83             throw new Error(`Cannot sign request without method (${method}) or path (${path}) ${options}`)
  84           }
  85
  86           httpSignature.signRequest({
  87             getHeader: function (header) {
  88               return options.headers[header]
  89             },
  90
  91             setHeader: function (header, value) {
  92               options.headers[header] = value
  93             },
  94
  95             method,
  96             path
  97           }, httpSignatureOptions)
  98         }
  99       },
 100
 101       (options: GotOptions) => {
 102         options.timeout = REQUEST_TIMEOUT
 103       }
 104     ]
 105   }
 106 })
 107
 108 function doRequest (url: string, options: PeerTubeRequestOptions = {}) {
 109   const gotOptions = buildGotOptions(options)
 110
 111   return peertubeGot(url, gotOptions)
 112     .catch(err => { throw buildRequestError(err) })
 113 }
 114
 115 function doJSONRequest <T> (url: string, options: PeerTubeRequestOptions = {}) {
 116   const gotOptions = buildGotOptions(options)
 117
 118   return peertubeGot<T>(url, { ...gotOptions, responseType: 'json' })
 119     .catch(err => { throw buildRequestError(err) })
 120 }
 121
 122 async function doRequestAndSaveToFile (
 123   url: string,
 124   destPath: string,
 125   options: PeerTubeRequestOptions = {}
 126 ) {
 127   const gotOptions = buildGotOptions(options)
 128
 129   const outFile = createWriteStream(destPath)
 130
 131   try {
 132     await pipelinePromise(
 133       peertubeGot.stream(url, gotOptions),
 134       outFile
 135     )
 136   } catch (err) {
 137     remove(destPath)
 138       .catch(err => logger.error('Cannot remove %s after request failure.', destPath, { err }))
 139
 140     throw buildRequestError(err)
 141   }
 142 }
 143
 144 async function downloadImage (url: string, destDir: string, destName: string, size: { width: number, height: number }) {
 145   const tmpPath = join(CONFIG.STORAGE.TMP_DIR, 'pending-' + destName)
 146   await doRequestAndSaveToFile(url, tmpPath)
 147
 148   const destPath = join(destDir, destName)
 149
 150   try {
 151     await processImage(tmpPath, destPath, size)
 152   } catch (err) {
 153     await remove(tmpPath)
 154
 155     throw err
 156   }
 157 }
 158
 159 function getAgent () {
 160   if (!isProxyEnabled()) return {}
 161
 162   const proxy = getProxy()
 163
 164   logger.info('Using proxy %s.', proxy)
 165
 166   const proxyAgentOptions = {
 167     keepAlive: true,
 168     keepAliveMsecs: 1000,
 169     maxSockets: 256,
 170     maxFreeSockets: 256,
 171     scheduling: 'lifo' as 'lifo',
 172     proxy
 173   }
 174
 175   return {
 176     agent: {
 177       http: new HttpProxyAgent(proxyAgentOptions),
 178       https: new HttpsProxyAgent(proxyAgentOptions)
 179     }
 180   }
 181 }
 182
 183 function getUserAgent () {
 184   return `PeerTube/${PEERTUBE_VERSION} (+${WEBSERVER.URL})`
 185 }
 186
 187 // ---------------------------------------------------------------------------
 188
 189 export {
 190   doRequest,
 191   doJSONRequest,
 192   doRequestAndSaveToFile,
 193   downloadImage
 194 }
 195
 196 // ---------------------------------------------------------------------------
 197
 198 function buildGotOptions (options: PeerTubeRequestOptions) {
 199   const { activityPub, bodyKBLimit = 1000 } = options
 200
 201   const context = { bodyKBLimit, httpSignature: options.httpSignature }
 202
 203   let headers = options.headers || {}
 204
 205   if (!headers.date) {
 206     headers = { ...headers, date: new Date().toUTCString() }
 207   }
 208
 209   if (activityPub && !headers.accept) {
 210     headers = { ...headers, accept: ACTIVITY_PUB.ACCEPT_HEADER }
 211   }
 212
 213   return {
 214     method: options.method,
 215     dnsCache: true,
 216     json: options.json,
 217     searchParams: options.searchParams,
 218     headers,
 219     context
 220   }
 221 }
 222
 223 function buildRequestError (error: RequestError) {
 224   const newError: PeerTubeRequestError = new Error(error.message)
 225   newError.name = error.name
 226   newError.stack = error.stack
 227
 228   if (error.response) {
 229     newError.responseBody = error.response.body
 230     newError.statusCode = error.response.statusCode
 231   }
 232
 233   return newError
 234 }