X-Git-Url: https://git.immae.eu/?a=blobdiff_plain;f=server%2Fhelpers%2Frequests.ts;h=0756beb16fb1c226f4f860a83a6dd8ff2f35fe2b;hb=6c38f40d966eb25e53ac38174153cb1fef0475f5;hp=8c4c983f7ac0f61ddb4e9ea207f0f049e890e1a6;hpb=e4f97babf701481b55cc10fb3448feab5f97c867;p=github%2FChocobozzz%2FPeerTube.git diff --git a/server/helpers/requests.ts b/server/helpers/requests.ts index 8c4c983f7..0756beb16 100644 --- a/server/helpers/requests.ts +++ b/server/helpers/requests.ts @@ -1,93 +1,245 @@ -import * as replay from 'request-replay' -import * as request from 'request' -import * as Promise from 'bluebird' - -import { - RETRY_REQUESTS, - REMOTE_SCHEME, - CONFIG -} from '../initializers' -import { PodInstance } from '../models' -import { PodSignature } from '../../shared' -import { signObject } from './peertube-crypto' - -function doRequest (requestOptions: request.CoreOptions & request.UriOptions) { - return new Promise<{ response: request.RequestResponse, body: any }>((res, rej) => { - request(requestOptions, (err, response, body) => err ? rej(err) : res({ response, body })) - }) -} +import { createWriteStream, remove } from 'fs-extra' +import got, { CancelableRequest, NormalizedOptions, Options as GotOptions, RequestError, Response } from 'got' +import { HttpProxyAgent, HttpsProxyAgent } from 'hpagent' +import { ACTIVITY_PUB, BINARY_CONTENT_TYPES, PEERTUBE_VERSION, REQUEST_TIMEOUTS, WEBSERVER } from '../initializers/constants' +import { pipelinePromise } from './core-utils' +import { logger, loggerTagsFactory } from './logger' +import { getProxy, isProxyEnabled } from './proxy' -type MakeRetryRequestParams = { - url: string, - method: 'GET' | 'POST', - json: Object +const lTags = loggerTagsFactory('request') + +const httpSignature = require('@peertube/http-signature') + +export interface PeerTubeRequestError extends Error { + statusCode?: number + responseBody?: any + responseHeaders?: any } -function makeRetryRequest (params: MakeRetryRequestParams) { - return new Promise<{ response: request.RequestResponse, body: any }>((res, rej) => { - replay( - request(params, (err, response, body) => err ? rej(err) : res({ response, body })), - { - retries: RETRY_REQUESTS, - factor: 3, - maxTimeout: Infinity, - errorCodes: [ 'EADDRINFO', 'ETIMEDOUT', 'ECONNRESET', 'ESOCKETTIMEDOUT', 'ENOTFOUND', 'ECONNREFUSED' ] + +type PeerTubeRequestOptions = { + timeout?: number + activityPub?: boolean + bodyKBLimit?: number // 1MB + httpSignature?: { + algorithm: string + authorizationHeaderName: string + keyId: string + key: string + headers: string[] + } + jsonResponse?: boolean +} & Pick + +const peertubeGot = got.extend({ + ...getAgent(), + + headers: { + 'user-agent': getUserAgent() + }, + + handlers: [ + (options, next) => { + const promiseOrStream = next(options) as CancelableRequest + const bodyKBLimit = options.context?.bodyKBLimit as number + if (!bodyKBLimit) throw new Error('No KB limit for this request') + + const bodyLimit = bodyKBLimit * 1000 + + /* eslint-disable @typescript-eslint/no-floating-promises */ + promiseOrStream.on('downloadProgress', progress => { + if (progress.transferred > bodyLimit && progress.percent !== 1) { + const message = `Exceeded the download limit of ${bodyLimit} B` + logger.warn(message, lTags()) + + // CancelableRequest + if (promiseOrStream.cancel) { + promiseOrStream.cancel() + return + } + + // Stream + (promiseOrStream as any).destroy() + } + }) + + return promiseOrStream + } + ], + + hooks: { + beforeRequest: [ + options => { + const headers = options.headers || {} + headers['host'] = options.url.host + }, + + options => { + const httpSignatureOptions = options.context?.httpSignature + + if (httpSignatureOptions) { + const method = options.method ?? 'GET' + const path = options.path ?? options.url.pathname + + if (!method || !path) { + throw new Error(`Cannot sign request without method (${method}) or path (${path}) ${options}`) + } + + httpSignature.signRequest({ + getHeader: function (header: string) { + const value = options.headers[header.toLowerCase()] + + if (!value) logger.warn('Unknown header requested by http-signature.', { headers: options.headers, header }) + return value + }, + + setHeader: function (header: string, value: string) { + options.headers[header] = value + }, + + method, + path + }, httpSignatureOptions) + } } - ) - }) + ], + + beforeRetry: [ + (_options: NormalizedOptions, error: RequestError, retryCount: number) => { + logger.debug('Retrying request to %s.', error.request.requestUrl, { retryCount, error: buildRequestError(error), ...lTags() }) + } + ] + } +}) + +function doRequest (url: string, options: PeerTubeRequestOptions = {}) { + const gotOptions = buildGotOptions(options) + + return peertubeGot(url, gotOptions) + .catch(err => { throw buildRequestError(err) }) } -type MakeSecureRequestParams = { - toPod: PodInstance - path: string - data?: Object +function doJSONRequest (url: string, options: PeerTubeRequestOptions = {}) { + const gotOptions = buildGotOptions(options) + + return peertubeGot(url, { ...gotOptions, responseType: 'json' }) + .catch(err => { throw buildRequestError(err) }) } -function makeSecureRequest (params: MakeSecureRequestParams) { - const requestParams: { - method: 'POST', - uri: string, - json: { - signature: PodSignature, - data: any - } - } = { - method: 'POST', - uri: REMOTE_SCHEME.HTTP + '://' + params.toPod.host + params.path, - json: { - signature: null, - data: null - } + +async function doRequestAndSaveToFile ( + url: string, + destPath: string, + options: PeerTubeRequestOptions = {} +) { + const gotOptions = buildGotOptions({ ...options, timeout: options.timeout ?? REQUEST_TIMEOUTS.FILE }) + + const outFile = createWriteStream(destPath) + + try { + await pipelinePromise( + peertubeGot.stream(url, gotOptions), + outFile + ) + } catch (err) { + remove(destPath) + .catch(err => logger.error('Cannot remove %s after request failure.', destPath, { err, ...lTags() })) + + throw buildRequestError(err) } +} - const host = CONFIG.WEBSERVER.HOST +function getAgent () { + if (!isProxyEnabled()) return {} - let dataToSign - if (params.data) { - dataToSign = params.data - } else { - // We do not have data to sign so we just take our host - // It is not ideal but the connection should be in HTTPS - dataToSign = host + const proxy = getProxy() + + logger.info('Using proxy %s.', proxy, lTags()) + + const proxyAgentOptions = { + keepAlive: true, + keepAliveMsecs: 1000, + maxSockets: 256, + maxFreeSockets: 256, + scheduling: 'lifo' as 'lifo', + proxy } - sign(dataToSign).then(signature => { - requestParams.json.signature = { - host, // Which host we pretend to be - signature + return { + agent: { + http: new HttpProxyAgent(proxyAgentOptions), + https: new HttpsProxyAgent(proxyAgentOptions) } + } +} - // If there are data information - if (params.data) { - requestParams.json.data = params.data - } +function getUserAgent () { + return `PeerTube/${PEERTUBE_VERSION} (+${WEBSERVER.URL})` +} + +function isBinaryResponse (result: Response) { + return BINARY_CONTENT_TYPES.has(result.headers['content-type']) +} - return doRequest(requestParams) - }) +async function findLatestRedirection (url: string, options: PeerTubeRequestOptions, iteration = 1) { + if (iteration > 10) throw new Error('Too much iterations to find final URL ' + url) + + const { headers } = await peertubeGot(url, { followRedirect: false, ...buildGotOptions(options) }) + + if (headers.location) return findLatestRedirection(headers.location, options, iteration + 1) + + return url } // --------------------------------------------------------------------------- export { doRequest, - makeRetryRequest, - makeSecureRequest + doJSONRequest, + doRequestAndSaveToFile, + isBinaryResponse, + getAgent, + findLatestRedirection, + peertubeGot +} + +// --------------------------------------------------------------------------- + +function buildGotOptions (options: PeerTubeRequestOptions) { + const { activityPub, bodyKBLimit = 1000 } = options + + const context = { bodyKBLimit, httpSignature: options.httpSignature } + + let headers = options.headers || {} + + if (!headers.date) { + headers = { ...headers, date: new Date().toUTCString() } + } + + if (activityPub && !headers.accept) { + headers = { ...headers, accept: ACTIVITY_PUB.ACCEPT_HEADER } + } + + return { + method: options.method, + dnsCache: true, + timeout: options.timeout ?? REQUEST_TIMEOUTS.DEFAULT, + json: options.json, + searchParams: options.searchParams, + retry: 2, + headers, + context + } +} + +function buildRequestError (error: RequestError) { + const newError: PeerTubeRequestError = new Error(error.message) + newError.name = error.name + newError.stack = error.stack + + if (error.response) { + newError.responseBody = error.response.body + newError.responseHeaders = error.response.headers + newError.statusCode = error.response.statusCode + } + + return newError }