server/helpers/requests.ts

   1 import { createWriteStream, remove } from 'fs-extra'
   2 import got, { CancelableRequest, Options as GotOptions, RequestError } from 'got'
   3 import { join } from 'path'
   4 import { CONFIG } from '../initializers/config'
   5 import { ACTIVITY_PUB, PEERTUBE_VERSION, REQUEST_TIMEOUT, WEBSERVER } from '../initializers/constants'
   6 import { pipelinePromise } from './core-utils'
   7 import { processImage } from './image-utils'
   8 import { logger } from './logger'
   9
  10 export interface PeerTubeRequestError extends Error {
  11   statusCode?: number
  12   responseBody?: any
  13 }
  14
  15 const httpSignature = require('http-signature')
  16
  17 type PeerTubeRequestOptions = {
  18   activityPub?: boolean
  19   bodyKBLimit?: number // 1MB
  20   httpSignature?: {
  21     algorithm: string
  22     authorizationHeaderName: string
  23     keyId: string
  24     key: string
  25     headers: string[]
  26   }
  27   timeout?: number
  28   jsonResponse?: boolean
  29 } & Pick<GotOptions, 'headers' | 'json' | 'method' | 'searchParams'>
  30
  31 const peertubeGot = got.extend({
  32   headers: {
  33     'user-agent': getUserAgent()
  34   },
  35
  36   handlers: [
  37     (options, next) => {
  38       const promiseOrStream = next(options) as CancelableRequest<any>
  39       const bodyKBLimit = options.context?.bodyKBLimit as number
  40       if (!bodyKBLimit) throw new Error('No KB limit for this request')
  41
  42       const bodyLimit = bodyKBLimit * 1000
  43
  44       /* eslint-disable @typescript-eslint/no-floating-promises */
  45       promiseOrStream.on('downloadProgress', progress => {
  46         if (progress.transferred > bodyLimit && progress.percent !== 1) {
  47           const message = `Exceeded the download limit of ${bodyLimit} B`
  48           logger.warn(message)
  49
  50           // CancelableRequest
  51           if (promiseOrStream.cancel) {
  52             promiseOrStream.cancel()
  53             return
  54           }
  55
  56           // Stream
  57           (promiseOrStream as any).destroy()
  58         }
  59       })
  60
  61       return promiseOrStream
  62     }
  63   ],
  64
  65   hooks: {
  66     beforeRequest: [
  67       options => {
  68         const headers = options.headers || {}
  69         headers['host'] = options.url.host
  70       },
  71
  72       options => {
  73         const httpSignatureOptions = options.context?.httpSignature
  74
  75         if (httpSignatureOptions) {
  76           const method = options.method ?? 'GET'
  77           const path = options.path ?? options.url.pathname
  78
  79           if (!method || !path) {
  80             throw new Error(`Cannot sign request without method (${method}) or path (${path}) ${options}`)
  81           }
  82
  83           httpSignature.signRequest({
  84             getHeader: function (header) {
  85               return options.headers[header]
  86             },
  87
  88             setHeader: function (header, value) {
  89               options.headers[header] = value
  90             },
  91
  92             method,
  93             path
  94           }, httpSignatureOptions)
  95         }
  96       },
  97
  98       (options: GotOptions) => {
  99         options.timeout = REQUEST_TIMEOUT
 100       }
 101     ]
 102   }
 103 })
 104
 105 function doRequest (url: string, options: PeerTubeRequestOptions = {}) {
 106   const gotOptions = buildGotOptions(options)
 107
 108   return peertubeGot(url, gotOptions)
 109     .catch(err => { throw buildRequestError(err) })
 110 }
 111
 112 function doJSONRequest <T> (url: string, options: PeerTubeRequestOptions = {}) {
 113   const gotOptions = buildGotOptions(options)
 114
 115   return peertubeGot<T>(url, { ...gotOptions, responseType: 'json' })
 116     .catch(err => { throw buildRequestError(err) })
 117 }
 118
 119 async function doRequestAndSaveToFile (
 120   url: string,
 121   destPath: string,
 122   options: PeerTubeRequestOptions = {}
 123 ) {
 124   const gotOptions = buildGotOptions(options)
 125
 126   const outFile = createWriteStream(destPath)
 127
 128   try {
 129     await pipelinePromise(
 130       peertubeGot.stream(url, gotOptions),
 131       outFile
 132     )
 133   } catch (err) {
 134     remove(destPath)
 135       .catch(err => logger.error('Cannot remove %s after request failure.', destPath, { err }))
 136
 137     throw buildRequestError(err)
 138   }
 139 }
 140
 141 async function downloadImage (url: string, destDir: string, destName: string, size: { width: number, height: number }) {
 142   const tmpPath = join(CONFIG.STORAGE.TMP_DIR, 'pending-' + destName)
 143   await doRequestAndSaveToFile(url, tmpPath)
 144
 145   const destPath = join(destDir, destName)
 146
 147   try {
 148     await processImage(tmpPath, destPath, size)
 149   } catch (err) {
 150     await remove(tmpPath)
 151
 152     throw err
 153   }
 154 }
 155
 156 function getUserAgent () {
 157   return `PeerTube/${PEERTUBE_VERSION} (+${WEBSERVER.URL})`
 158 }
 159
 160 // ---------------------------------------------------------------------------
 161
 162 export {
 163   doRequest,
 164   doJSONRequest,
 165   doRequestAndSaveToFile,
 166   downloadImage
 167 }
 168
 169 // ---------------------------------------------------------------------------
 170
 171 function buildGotOptions (options: PeerTubeRequestOptions) {
 172   const { activityPub, bodyKBLimit = 1000 } = options
 173
 174   const context = { bodyKBLimit, httpSignature: options.httpSignature }
 175
 176   let headers = options.headers || {}
 177
 178   if (!headers.date) {
 179     headers = { ...headers, date: new Date().toUTCString() }
 180   }
 181
 182   if (activityPub && !headers.accept) {
 183     headers = { ...headers, accept: ACTIVITY_PUB.ACCEPT_HEADER }
 184   }
 185
 186   return {
 187     method: options.method,
 188     dnsCache: true,
 189     json: options.json,
 190     searchParams: options.searchParams,
 191     timeout: options.timeout ?? REQUEST_TIMEOUT,
 192     headers,
 193     context
 194   }
 195 }
 196
 197 function buildRequestError (error: RequestError) {
 198   const newError: PeerTubeRequestError = new Error(error.message)
 199   newError.name = error.name
 200   newError.stack = error.stack
 201
 202   if (error.response) {
 203     newError.responseBody = error.response.body
 204     newError.statusCode = error.response.statusCode
 205   }
 206
 207   return newError
 208 }