X-Git-Url: https://git.immae.eu/?a=blobdiff_plain;f=server%2Fhelpers%2Frequests.ts;h=327610558008ef439bf7e5b76abeba2a20415ccb;hb=75b7117f078461d2507572ba9da6527894e1b734;hp=3762e4d3c5c2cf981766e757e1eae46a0af9305c;hpb=bfe2ef6bfae03444a232883fc7c449206cf3bee4;p=github%2FChocobozzz%2FPeerTube.git diff --git a/server/helpers/requests.ts b/server/helpers/requests.ts index 3762e4d3c..327610558 100644 --- a/server/helpers/requests.ts +++ b/server/helpers/requests.ts @@ -1,79 +1,260 @@ -import * as Bluebird from 'bluebird' import { createWriteStream, remove } from 'fs-extra' -import * as request from 'request' -import { ACTIVITY_PUB, CONFIG } from '../initializers' -import { processImage } from './image-utils' +import got, { CancelableRequest, NormalizedOptions, Options as GotOptions, RequestError, Response } from 'got' +import { HttpProxyAgent, HttpsProxyAgent } from 'hpagent' import { join } from 'path' -import { logger } from './logger' - -function doRequest ( - requestOptions: request.CoreOptions & request.UriOptions & { activityPub?: boolean }, - bodyKBLimit = 1000 // 1MB -): Bluebird<{ response: request.RequestResponse, body: T }> { - if (requestOptions.activityPub === true) { - if (!Array.isArray(requestOptions.headers)) requestOptions.headers = {} - requestOptions.headers['accept'] = ACTIVITY_PUB.ACCEPT_HEADER +import { CONFIG } from '../initializers/config' +import { ACTIVITY_PUB, BINARY_CONTENT_TYPES, PEERTUBE_VERSION, REQUEST_TIMEOUTS, WEBSERVER } from '../initializers/constants' +import { pipelinePromise } from './core-utils' +import { processImage } from './image-utils' +import { logger, loggerTagsFactory } from './logger' +import { getProxy, isProxyEnabled } from './proxy' + +const lTags = loggerTagsFactory('request') + +const httpSignature = require('@peertube/http-signature') + +export interface PeerTubeRequestError extends Error { + statusCode?: number + responseBody?: any + responseHeaders?: any +} + +type PeerTubeRequestOptions = { + timeout?: number + activityPub?: boolean + bodyKBLimit?: number // 1MB + httpSignature?: { + algorithm: string + authorizationHeaderName: string + keyId: string + key: string + headers: string[] + } + jsonResponse?: boolean +} & Pick + +const peertubeGot = got.extend({ + ...getAgent(), + + headers: { + 'user-agent': getUserAgent() + }, + + handlers: [ + (options, next) => { + const promiseOrStream = next(options) as CancelableRequest + const bodyKBLimit = options.context?.bodyKBLimit as number + if (!bodyKBLimit) throw new Error('No KB limit for this request') + + const bodyLimit = bodyKBLimit * 1000 + + /* eslint-disable @typescript-eslint/no-floating-promises */ + promiseOrStream.on('downloadProgress', progress => { + if (progress.transferred > bodyLimit && progress.percent !== 1) { + const message = `Exceeded the download limit of ${bodyLimit} B` + logger.warn(message, lTags()) + + // CancelableRequest + if (promiseOrStream.cancel) { + promiseOrStream.cancel() + return + } + + // Stream + (promiseOrStream as any).destroy() + } + }) + + return promiseOrStream + } + ], + + hooks: { + beforeRequest: [ + options => { + const headers = options.headers || {} + headers['host'] = options.url.host + }, + + options => { + const httpSignatureOptions = options.context?.httpSignature + + if (httpSignatureOptions) { + const method = options.method ?? 'GET' + const path = options.path ?? options.url.pathname + + if (!method || !path) { + throw new Error(`Cannot sign request without method (${method}) or path (${path}) ${options}`) + } + + httpSignature.signRequest({ + getHeader: function (header) { + return options.headers[header] + }, + + setHeader: function (header, value) { + options.headers[header] = value + }, + + method, + path + }, httpSignatureOptions) + } + } + ], + + beforeRetry: [ + (_options: NormalizedOptions, error: RequestError, retryCount: number) => { + logger.debug('Retrying request to %s.', error.request.requestUrl, { retryCount, error: buildRequestError(error), ...lTags() }) + } + ] } +}) + +function doRequest (url: string, options: PeerTubeRequestOptions = {}) { + const gotOptions = buildGotOptions(options) + + return peertubeGot(url, gotOptions) + .catch(err => { throw buildRequestError(err) }) +} + +function doJSONRequest (url: string, options: PeerTubeRequestOptions = {}) { + const gotOptions = buildGotOptions(options) - return new Bluebird<{ response: request.RequestResponse, body: T }>((res, rej) => { - request(requestOptions, (err, response, body) => err ? rej(err) : res({ response, body })) - .on('data', onRequestDataLengthCheck(bodyKBLimit)) - }) + return peertubeGot(url, { ...gotOptions, responseType: 'json' }) + .catch(err => { throw buildRequestError(err) }) } -function doRequestAndSaveToFile ( - requestOptions: request.CoreOptions & request.UriOptions, +async function doRequestAndSaveToFile ( + url: string, destPath: string, - bodyKBLimit = 10000 // 10MB + options: PeerTubeRequestOptions = {} ) { - return new Bluebird((res, rej) => { - const file = createWriteStream(destPath) - file.on('finish', () => res()) + const gotOptions = buildGotOptions({ ...options, timeout: options.timeout ?? REQUEST_TIMEOUTS.FILE }) - request(requestOptions) - .on('data', onRequestDataLengthCheck(bodyKBLimit)) - .on('error', err => { - file.close() + const outFile = createWriteStream(destPath) - remove(destPath) - .catch(err => logger.error('Cannot remove %s after request failure.', destPath, { err })) + try { + await pipelinePromise( + peertubeGot.stream(url, gotOptions), + outFile + ) + } catch (err) { + remove(destPath) + .catch(err => logger.error('Cannot remove %s after request failure.', destPath, { err, ...lTags() })) - return rej(err) - }) - .pipe(file) - }) + throw buildRequestError(err) + } } async function downloadImage (url: string, destDir: string, destName: string, size: { width: number, height: number }) { const tmpPath = join(CONFIG.STORAGE.TMP_DIR, 'pending-' + destName) - await doRequestAndSaveToFile({ method: 'GET', uri: url }, tmpPath) + await doRequestAndSaveToFile(url, tmpPath) const destPath = join(destDir, destName) - await processImage({ path: tmpPath }, destPath, size) + + try { + await processImage(tmpPath, destPath, size) + } catch (err) { + await remove(tmpPath) + + throw err + } +} + +function getAgent () { + if (!isProxyEnabled()) return {} + + const proxy = getProxy() + + logger.info('Using proxy %s.', proxy, lTags()) + + const proxyAgentOptions = { + keepAlive: true, + keepAliveMsecs: 1000, + maxSockets: 256, + maxFreeSockets: 256, + scheduling: 'lifo' as 'lifo', + proxy + } + + return { + agent: { + http: new HttpProxyAgent(proxyAgentOptions), + https: new HttpsProxyAgent(proxyAgentOptions) + } + } +} + +function getUserAgent () { + return `PeerTube/${PEERTUBE_VERSION} (+${WEBSERVER.URL})` +} + +function isBinaryResponse (result: Response) { + return BINARY_CONTENT_TYPES.has(result.headers['content-type']) +} + +async function findLatestRedirection (url: string, options: PeerTubeRequestOptions, iteration = 1) { + if (iteration > 10) throw new Error('Too much iterations to find final URL ' + url) + + const { headers } = await peertubeGot(url, { followRedirect: false, ...buildGotOptions(options) }) + + if (headers.location) return findLatestRedirection(headers.location, options, iteration + 1) + + return url } // --------------------------------------------------------------------------- export { doRequest, + doJSONRequest, doRequestAndSaveToFile, - downloadImage + isBinaryResponse, + downloadImage, + findLatestRedirection, + peertubeGot } // --------------------------------------------------------------------------- -// Thanks to https://github.com/request/request/issues/2470#issuecomment-268929907 <3 -function onRequestDataLengthCheck (bodyKBLimit: number) { - let bufferLength = 0 - const bytesLimit = bodyKBLimit * 1000 +function buildGotOptions (options: PeerTubeRequestOptions) { + const { activityPub, bodyKBLimit = 1000 } = options - return function (chunk) { - bufferLength += chunk.length - if (bufferLength > bytesLimit) { - this.abort() + const context = { bodyKBLimit, httpSignature: options.httpSignature } - const error = new Error(`Response was too large - aborted after ${bytesLimit} bytes.`) - this.emit('error', error) - } + let headers = options.headers || {} + + if (!headers.date) { + headers = { ...headers, date: new Date().toUTCString() } + } + + if (activityPub && !headers.accept) { + headers = { ...headers, accept: ACTIVITY_PUB.ACCEPT_HEADER } + } + + return { + method: options.method, + dnsCache: true, + timeout: options.timeout ?? REQUEST_TIMEOUTS.DEFAULT, + json: options.json, + searchParams: options.searchParams, + retry: 2, + headers, + context + } +} + +function buildRequestError (error: RequestError) { + const newError: PeerTubeRequestError = new Error(error.message) + newError.name = error.name + newError.stack = error.stack + + if (error.response) { + newError.responseBody = error.response.body + newError.responseHeaders = error.response.headers + newError.statusCode = error.response.statusCode } + + return newError }