1 import { createWriteStream, remove } from 'fs-extra'
2 import got, { CancelableRequest, Options as GotOptions, RequestError, Response } from 'got'
3 import { HttpProxyAgent, HttpsProxyAgent } from 'hpagent'
4 import { join } from 'path'
5 import { CONFIG } from '../initializers/config'
6 import { ACTIVITY_PUB, BINARY_CONTENT_TYPES, PEERTUBE_VERSION, REQUEST_TIMEOUT, WEBSERVER } from '../initializers/constants'
7 import { pipelinePromise } from './core-utils'
8 import { processImage } from './image-utils'
9 import { logger } from './logger'
10 import { getProxy, isProxyEnabled } from './proxy'
12 const httpSignature = require('http-signature')
14 export interface PeerTubeRequestError extends Error {
19 type PeerTubeRequestOptions = {
21 bodyKBLimit?: number // 1MB
24 authorizationHeaderName: string
29 jsonResponse?: boolean
30 } & Pick<GotOptions, 'headers' | 'json' | 'method' | 'searchParams'>
32 const peertubeGot = got.extend({
36 'user-agent': getUserAgent()
41 const promiseOrStream = next(options) as CancelableRequest<any>
42 const bodyKBLimit = options.context?.bodyKBLimit as number
43 if (!bodyKBLimit) throw new Error('No KB limit for this request')
45 const bodyLimit = bodyKBLimit * 1000
47 /* eslint-disable @typescript-eslint/no-floating-promises */
48 promiseOrStream.on('downloadProgress', progress => {
49 if (progress.transferred > bodyLimit && progress.percent !== 1) {
50 const message = `Exceeded the download limit of ${bodyLimit} B`
54 if (promiseOrStream.cancel) {
55 promiseOrStream.cancel()
60 (promiseOrStream as any).destroy()
64 return promiseOrStream
71 const headers = options.headers || {}
72 headers['host'] = options.url.host
76 const httpSignatureOptions = options.context?.httpSignature
78 if (httpSignatureOptions) {
79 const method = options.method ?? 'GET'
80 const path = options.path ?? options.url.pathname
82 if (!method || !path) {
83 throw new Error(`Cannot sign request without method (${method}) or path (${path}) ${options}`)
86 httpSignature.signRequest({
87 getHeader: function (header) {
88 return options.headers[header]
91 setHeader: function (header, value) {
92 options.headers[header] = value
97 }, httpSignatureOptions)
104 function doRequest (url: string, options: PeerTubeRequestOptions = {}) {
105 const gotOptions = buildGotOptions(options)
107 return peertubeGot(url, gotOptions)
108 .catch(err => { throw buildRequestError(err) })
111 function doJSONRequest <T> (url: string, options: PeerTubeRequestOptions = {}) {
112 const gotOptions = buildGotOptions(options)
114 return peertubeGot<T>(url, { ...gotOptions, responseType: 'json' })
115 .catch(err => { throw buildRequestError(err) })
118 async function doRequestAndSaveToFile (
121 options: PeerTubeRequestOptions = {}
123 const gotOptions = buildGotOptions(options)
125 const outFile = createWriteStream(destPath)
128 await pipelinePromise(
129 peertubeGot.stream(url, gotOptions),
134 .catch(err => logger.error('Cannot remove %s after request failure.', destPath, { err }))
136 throw buildRequestError(err)
140 async function downloadImage (url: string, destDir: string, destName: string, size: { width: number, height: number }) {
141 const tmpPath = join(CONFIG.STORAGE.TMP_DIR, 'pending-' + destName)
142 await doRequestAndSaveToFile(url, tmpPath)
144 const destPath = join(destDir, destName)
147 await processImage(tmpPath, destPath, size)
149 await remove(tmpPath)
155 function getAgent () {
156 if (!isProxyEnabled()) return {}
158 const proxy = getProxy()
160 logger.info('Using proxy %s.', proxy)
162 const proxyAgentOptions = {
164 keepAliveMsecs: 1000,
167 scheduling: 'lifo' as 'lifo',
173 http: new HttpProxyAgent(proxyAgentOptions),
174 https: new HttpsProxyAgent(proxyAgentOptions)
179 function getUserAgent () {
180 return `PeerTube/${PEERTUBE_VERSION} (+${WEBSERVER.URL})`
183 function isBinaryResponse (result: Response<any>) {
184 return BINARY_CONTENT_TYPES.has(result.headers['content-type'])
187 // ---------------------------------------------------------------------------
192 doRequestAndSaveToFile,
198 // ---------------------------------------------------------------------------
200 function buildGotOptions (options: PeerTubeRequestOptions) {
201 const { activityPub, bodyKBLimit = 1000 } = options
203 const context = { bodyKBLimit, httpSignature: options.httpSignature }
205 let headers = options.headers || {}
208 headers = { ...headers, date: new Date().toUTCString() }
211 if (activityPub && !headers.accept) {
212 headers = { ...headers, accept: ACTIVITY_PUB.ACCEPT_HEADER }
216 method: options.method,
218 timeout: REQUEST_TIMEOUT,
220 searchParams: options.searchParams,
226 function buildRequestError (error: RequestError) {
227 const newError: PeerTubeRequestError = new Error(error.message)
228 newError.name = error.name
229 newError.stack = error.stack
231 if (error.response) {
232 newError.responseBody = error.response.body
233 newError.statusCode = error.response.statusCode