import { createWriteStream, remove } from 'fs-extra'
-import got, { CancelableRequest, Options as GotOptions } from 'got'
-import { join } from 'path'
-import { CONFIG } from '../initializers/config'
-import { ACTIVITY_PUB, PEERTUBE_VERSION, WEBSERVER } from '../initializers/constants'
+import got, { CancelableRequest, NormalizedOptions, Options as GotOptions, RequestError, Response } from 'got'
+import { HttpProxyAgent, HttpsProxyAgent } from 'hpagent'
+import { ACTIVITY_PUB, BINARY_CONTENT_TYPES, PEERTUBE_VERSION, REQUEST_TIMEOUTS, WEBSERVER } from '../initializers/constants'
import { pipelinePromise } from './core-utils'
-import { processImage } from './image-utils'
-import { logger } from './logger'
+import { logger, loggerTagsFactory } from './logger'
+import { getProxy, isProxyEnabled } from './proxy'
-const httpSignature = require('http-signature')
+const lTags = loggerTagsFactory('request')
+
+const httpSignature = require('@peertube/http-signature')
+
+export interface PeerTubeRequestError extends Error {
+ statusCode?: number
+ responseBody?: any
+ responseHeaders?: any
+}
type PeerTubeRequestOptions = {
+ timeout?: number
activityPub?: boolean
bodyKBLimit?: number // 1MB
httpSignature?: {
} & Pick<GotOptions, 'headers' | 'json' | 'method' | 'searchParams'>
const peertubeGot = got.extend({
+ ...getAgent(),
+
headers: {
'user-agent': getUserAgent()
},
handlers: [
(options, next) => {
const promiseOrStream = next(options) as CancelableRequest<any>
- const bodyKBLimit = options.context?.bodyKBLimit
+ const bodyKBLimit = options.context?.bodyKBLimit as number
if (!bodyKBLimit) throw new Error('No KB limit for this request')
+ const bodyLimit = bodyKBLimit * 1000
+
/* eslint-disable @typescript-eslint/no-floating-promises */
promiseOrStream.on('downloadProgress', progress => {
- if (progress.transferred * 1000 > bodyKBLimit && progress.percent !== 1) {
- promiseOrStream.cancel(`Exceeded the download limit of ${bodyKBLimit} bytes`)
+ if (progress.transferred > bodyLimit && progress.percent !== 1) {
+ const message = `Exceeded the download limit of ${bodyLimit} B`
+ logger.warn(message, lTags())
+
+ // CancelableRequest
+ if (promiseOrStream.cancel) {
+ promiseOrStream.cancel()
+ return
+ }
+
+ // Stream
+ (promiseOrStream as any).destroy()
}
})
}
httpSignature.signRequest({
- getHeader: function (header) {
- return options.headers[header]
+ getHeader: function (header: string) {
+ const value = options.headers[header.toLowerCase()]
+
+ if (!value) logger.warn('Unknown header requested by http-signature.', { headers: options.headers, header })
+ return value
},
- setHeader: function (header, value) {
+ setHeader: function (header: string, value: string) {
options.headers[header] = value
},
}, httpSignatureOptions)
}
}
+ ],
+
+ beforeRetry: [
+ (_options: NormalizedOptions, error: RequestError, retryCount: number) => {
+ logger.debug('Retrying request to %s.', error.request.requestUrl, { retryCount, error: buildRequestError(error), ...lTags() })
+ }
]
}
})
destPath: string,
options: PeerTubeRequestOptions = {}
) {
- const gotOptions = buildGotOptions(options)
+ const gotOptions = buildGotOptions({ ...options, timeout: options.timeout ?? REQUEST_TIMEOUTS.FILE })
const outFile = createWriteStream(destPath)
)
} catch (err) {
remove(destPath)
- .catch(err => logger.error('Cannot remove %s after request failure.', destPath, { err }))
+ .catch(err => logger.error('Cannot remove %s after request failure.', destPath, { err, ...lTags() }))
throw buildRequestError(err)
}
}
-async function downloadImage (url: string, destDir: string, destName: string, size: { width: number, height: number }) {
- const tmpPath = join(CONFIG.STORAGE.TMP_DIR, 'pending-' + destName)
- await doRequestAndSaveToFile(url, tmpPath)
+function getAgent () {
+ if (!isProxyEnabled()) return {}
- const destPath = join(destDir, destName)
+ const proxy = getProxy()
- try {
- await processImage(tmpPath, destPath, size)
- } catch (err) {
- await remove(tmpPath)
+ logger.info('Using proxy %s.', proxy, lTags())
- throw err
+ const proxyAgentOptions = {
+ keepAlive: true,
+ keepAliveMsecs: 1000,
+ maxSockets: 256,
+ maxFreeSockets: 256,
+ scheduling: 'lifo' as 'lifo',
+ proxy
+ }
+
+ return {
+ agent: {
+ http: new HttpProxyAgent(proxyAgentOptions),
+ https: new HttpsProxyAgent(proxyAgentOptions)
+ }
}
}
return `PeerTube/${PEERTUBE_VERSION} (+${WEBSERVER.URL})`
}
+function isBinaryResponse (result: Response<any>) {
+ return BINARY_CONTENT_TYPES.has(result.headers['content-type'])
+}
+
+async function findLatestRedirection (url: string, options: PeerTubeRequestOptions, iteration = 1) {
+ if (iteration > 10) throw new Error('Too much iterations to find final URL ' + url)
+
+ const { headers } = await peertubeGot(url, { followRedirect: false, ...buildGotOptions(options) })
+
+ if (headers.location) return findLatestRedirection(headers.location, options, iteration + 1)
+
+ return url
+}
+
// ---------------------------------------------------------------------------
export {
+ PeerTubeRequestOptions,
+
doRequest,
doJSONRequest,
doRequestAndSaveToFile,
- downloadImage
+ isBinaryResponse,
+ getAgent,
+ findLatestRedirection,
+ peertubeGot
}
// ---------------------------------------------------------------------------
let headers = options.headers || {}
- headers = { ...headers, date: new Date().toUTCString() }
+ if (!headers.date) {
+ headers = { ...headers, date: new Date().toUTCString() }
+ }
- if (activityPub) {
+ if (activityPub && !headers.accept) {
headers = { ...headers, accept: ACTIVITY_PUB.ACCEPT_HEADER }
}
return {
method: options.method,
+ dnsCache: true,
+ timeout: options.timeout ?? REQUEST_TIMEOUTS.DEFAULT,
json: options.json,
searchParams: options.searchParams,
+ retry: 2,
headers,
context
}
}
-function buildRequestError (error: any) {
- const newError = new Error(error.message)
+function buildRequestError (error: RequestError) {
+ const newError: PeerTubeRequestError = new Error(error.message)
newError.name = error.name
newError.stack = error.stack
- if (error.response?.body) {
- error.responseBody = error.response.body
+ if (error.response) {
+ newError.responseBody = error.response.body
+ newError.responseHeaders = error.response.headers
+ newError.statusCode = error.response.statusCode
}
return newError