1 import { createWriteStream, remove } from 'fs-extra'
2 import got, { CancelableRequest, NormalizedOptions, Options as GotOptions, RequestError, Response } from 'got'
3 import { HttpProxyAgent, HttpsProxyAgent } from 'hpagent'
4 import { ACTIVITY_PUB, BINARY_CONTENT_TYPES, PEERTUBE_VERSION, REQUEST_TIMEOUTS, WEBSERVER } from '../initializers/constants'
5 import { pipelinePromise } from './core-utils'
6 import { logger, loggerTagsFactory } from './logger'
7 import { getProxy, isProxyEnabled } from './proxy'
9 const lTags = loggerTagsFactory('request')
11 const httpSignature = require('@peertube/http-signature')
13 export interface PeerTubeRequestError extends Error {
19 type PeerTubeRequestOptions = {
22 bodyKBLimit?: number // 1MB
25 authorizationHeaderName: string
30 jsonResponse?: boolean
31 } & Pick<GotOptions, 'headers' | 'json' | 'method' | 'searchParams'>
33 const peertubeGot = got.extend({
37 'user-agent': getUserAgent()
42 const promiseOrStream = next(options) as CancelableRequest<any>
43 const bodyKBLimit = options.context?.bodyKBLimit as number
44 if (!bodyKBLimit) throw new Error('No KB limit for this request')
46 const bodyLimit = bodyKBLimit * 1000
48 /* eslint-disable @typescript-eslint/no-floating-promises */
49 promiseOrStream.on('downloadProgress', progress => {
50 if (progress.transferred > bodyLimit && progress.percent !== 1) {
51 const message = `Exceeded the download limit of ${bodyLimit} B`
52 logger.warn(message, lTags())
55 if (promiseOrStream.cancel) {
56 promiseOrStream.cancel()
61 (promiseOrStream as any).destroy()
65 return promiseOrStream
72 const headers = options.headers || {}
73 headers['host'] = options.url.host
77 const httpSignatureOptions = options.context?.httpSignature
79 if (httpSignatureOptions) {
80 const method = options.method ?? 'GET'
81 const path = options.path ?? options.url.pathname
83 if (!method || !path) {
84 throw new Error(`Cannot sign request without method (${method}) or path (${path}) ${options}`)
87 httpSignature.signRequest({
88 getHeader: function (header: string) {
89 const value = options.headers[header.toLowerCase()]
91 if (!value) logger.warn('Unknown header requested by http-signature.', { headers: options.headers, header })
95 setHeader: function (header: string, value: string) {
96 options.headers[header] = value
101 }, httpSignatureOptions)
107 (_options: NormalizedOptions, error: RequestError, retryCount: number) => {
108 logger.debug('Retrying request to %s.', error.request.requestUrl, { retryCount, error: buildRequestError(error), ...lTags() })
114 function doRequest (url: string, options: PeerTubeRequestOptions = {}) {
115 const gotOptions = buildGotOptions(options)
117 return peertubeGot(url, gotOptions)
118 .catch(err => { throw buildRequestError(err) })
121 function doJSONRequest <T> (url: string, options: PeerTubeRequestOptions = {}) {
122 const gotOptions = buildGotOptions(options)
124 return peertubeGot<T>(url, { ...gotOptions, responseType: 'json' })
125 .catch(err => { throw buildRequestError(err) })
128 async function doRequestAndSaveToFile (
131 options: PeerTubeRequestOptions = {}
133 const gotOptions = buildGotOptions({ ...options, timeout: options.timeout ?? REQUEST_TIMEOUTS.FILE })
135 const outFile = createWriteStream(destPath)
138 await pipelinePromise(
139 peertubeGot.stream(url, gotOptions),
144 .catch(err => logger.error('Cannot remove %s after request failure.', destPath, { err, ...lTags() }))
146 throw buildRequestError(err)
150 function getAgent () {
151 if (!isProxyEnabled()) return {}
153 const proxy = getProxy()
155 logger.info('Using proxy %s.', proxy, lTags())
157 const proxyAgentOptions = {
159 keepAliveMsecs: 1000,
162 scheduling: 'lifo' as 'lifo',
168 http: new HttpProxyAgent(proxyAgentOptions),
169 https: new HttpsProxyAgent(proxyAgentOptions)
174 function getUserAgent () {
175 return `PeerTube/${PEERTUBE_VERSION} (+${WEBSERVER.URL})`
178 function isBinaryResponse (result: Response<any>) {
179 return BINARY_CONTENT_TYPES.has(result.headers['content-type'])
182 async function findLatestRedirection (url: string, options: PeerTubeRequestOptions, iteration = 1) {
183 if (iteration > 10) throw new Error('Too much iterations to find final URL ' + url)
185 const { headers } = await peertubeGot(url, { followRedirect: false, ...buildGotOptions(options) })
187 if (headers.location) return findLatestRedirection(headers.location, options, iteration + 1)
192 // ---------------------------------------------------------------------------
195 PeerTubeRequestOptions,
199 doRequestAndSaveToFile,
202 findLatestRedirection,
206 // ---------------------------------------------------------------------------
208 function buildGotOptions (options: PeerTubeRequestOptions) {
209 const { activityPub, bodyKBLimit = 1000 } = options
211 const context = { bodyKBLimit, httpSignature: options.httpSignature }
213 let headers = options.headers || {}
216 headers = { ...headers, date: new Date().toUTCString() }
219 if (activityPub && !headers.accept) {
220 headers = { ...headers, accept: ACTIVITY_PUB.ACCEPT_HEADER }
224 method: options.method,
226 timeout: options.timeout ?? REQUEST_TIMEOUTS.DEFAULT,
228 searchParams: options.searchParams,
235 function buildRequestError (error: RequestError) {
236 const newError: PeerTubeRequestError = new Error(error.message)
237 newError.name = error.name
238 newError.stack = error.stack
240 if (error.response) {
241 newError.responseBody = error.response.body
242 newError.responseHeaders = error.response.headers
243 newError.statusCode = error.response.statusCode