From db4b15f21fbf4e33434e930ffc7fb768cdcf9d42 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Mon, 8 Mar 2021 14:24:11 +0100 Subject: Use got instead of request --- server/helpers/activitypub.ts | 5 +- server/helpers/core-utils.ts | 6 +- .../custom-validators/activitypub/activity.ts | 2 +- server/helpers/peertube-crypto.ts | 2 +- server/helpers/requests.ts | 179 +++++++++++++++------ server/helpers/youtube-dl.ts | 71 +++----- 6 files changed, 165 insertions(+), 100 deletions(-) (limited to 'server/helpers') diff --git a/server/helpers/activitypub.ts b/server/helpers/activitypub.ts index 08aef2908..e0754b501 100644 --- a/server/helpers/activitypub.ts +++ b/server/helpers/activitypub.ts @@ -3,7 +3,6 @@ import { URL } from 'url' import validator from 'validator' import { ContextType } from '@shared/models/activitypub/context' import { ResultList } from '../../shared/models' -import { Activity } from '../../shared/models/activitypub' import { ACTIVITY_PUB, REMOTE_SCHEME } from '../initializers/constants' import { MActor, MVideoWithHost } from '../types/models' import { pageToStartAndCount } from './core-utils' @@ -182,10 +181,10 @@ async function activityPubCollectionPagination ( } -function buildSignedActivity (byActor: MActor, data: Object, contextType?: ContextType) { +function buildSignedActivity (byActor: MActor, data: T, contextType?: ContextType) { const activity = activityPubContextify(data, contextType) - return signJsonLDObject(byActor, activity) as Promise + return signJsonLDObject(byActor, activity) } function getAPId (activity: string | { id: string }) { diff --git a/server/helpers/core-utils.ts b/server/helpers/core-utils.ts index 935fd22d9..7ba7d865a 100644 --- a/server/helpers/core-utils.ts +++ b/server/helpers/core-utils.ts @@ -10,7 +10,9 @@ import { BinaryToTextEncoding, createHash, randomBytes } from 'crypto' import { truncate } from 'lodash' import { basename, isAbsolute, join, resolve } from 'path' import * as pem from 'pem' +import { pipeline } from 'stream' import { URL } from 'url' +import { promisify } from 'util' const objectConverter = (oldObject: any, keyConverter: (e: string) => string, valueConverter: (e: any) => any) => { if (!oldObject || typeof oldObject !== 'object') { @@ -254,6 +256,7 @@ const createPrivateKey = promisify1(pem.createPrivateKe const getPublicKey = promisify1(pem.getPublicKey) const execPromise2 = promisify2(exec) const execPromise = promisify1(exec) +const pipelinePromise = promisify(pipeline) // --------------------------------------------------------------------------- @@ -284,5 +287,6 @@ export { createPrivateKey, getPublicKey, execPromise2, - execPromise + execPromise, + pipelinePromise } diff --git a/server/helpers/custom-validators/activitypub/activity.ts b/server/helpers/custom-validators/activitypub/activity.ts index 46126da57..69558e358 100644 --- a/server/helpers/custom-validators/activitypub/activity.ts +++ b/server/helpers/custom-validators/activitypub/activity.ts @@ -41,7 +41,7 @@ const activityCheckers: { [ P in ActivityType ]: (activity: Activity) => boolean } function isActivityValid (activity: any) { - const checker = activityCheckers[activity.tswype] + const checker = activityCheckers[activity.type] // Unknown activity type if (!checker) return false diff --git a/server/helpers/peertube-crypto.ts b/server/helpers/peertube-crypto.ts index 994f725d8..bc6f1d074 100644 --- a/server/helpers/peertube-crypto.ts +++ b/server/helpers/peertube-crypto.ts @@ -84,7 +84,7 @@ async function isJsonLDRSA2017Verified (fromActor: MActor, signedDocument: any) return verify.verify(fromActor.publicKey, signedDocument.signature.signatureValue, 'base64') } -async function signJsonLDObject (byActor: MActor, data: any) { +async function signJsonLDObject (byActor: MActor, data: T) { const signature = { type: 'RsaSignature2017', creator: byActor.url, diff --git a/server/helpers/requests.ts b/server/helpers/requests.ts index b556c392e..2c9da213c 100644 --- a/server/helpers/requests.ts +++ b/server/helpers/requests.ts @@ -1,58 +1,124 @@ -import * as Bluebird from 'bluebird' import { createWriteStream, remove } from 'fs-extra' -import * as request from 'request' +import got, { CancelableRequest, Options as GotOptions } from 'got' +import { join } from 'path' +import { CONFIG } from '../initializers/config' import { ACTIVITY_PUB, PEERTUBE_VERSION, WEBSERVER } from '../initializers/constants' +import { pipelinePromise } from './core-utils' import { processImage } from './image-utils' -import { join } from 'path' import { logger } from './logger' -import { CONFIG } from '../initializers/config' -function doRequest ( - requestOptions: request.CoreOptions & request.UriOptions & { activityPub?: boolean }, - bodyKBLimit = 1000 // 1MB -): Bluebird<{ response: request.RequestResponse, body: T }> { - if (!(requestOptions.headers)) requestOptions.headers = {} - requestOptions.headers['User-Agent'] = getUserAgent() +const httpSignature = require('http-signature') + +type PeerTubeRequestOptions = { + activityPub?: boolean + bodyKBLimit?: number // 1MB + httpSignature?: { + algorithm: string + authorizationHeaderName: string + keyId: string + key: string + headers: string[] + } + jsonResponse?: boolean +} & Pick + +const peertubeGot = got.extend({ + headers: { + 'user-agent': getUserAgent() + }, + + handlers: [ + (options, next) => { + const promiseOrStream = next(options) as CancelableRequest + const bodyKBLimit = options.context?.bodyKBLimit + if (!bodyKBLimit) throw new Error('No KB limit for this request') + + /* eslint-disable @typescript-eslint/no-floating-promises */ + promiseOrStream.on('downloadProgress', progress => { + if (progress.transferred * 1000 > bodyKBLimit && progress.percent !== 1) { + promiseOrStream.cancel(`Exceeded the download limit of ${bodyKBLimit} bytes`) + } + }) - if (requestOptions.activityPub === true) { - requestOptions.headers['accept'] = ACTIVITY_PUB.ACCEPT_HEADER + return promiseOrStream + } + ], + + hooks: { + beforeRequest: [ + options => { + const headers = options.headers || {} + headers['host'] = options.url.host + }, + + options => { + const httpSignatureOptions = options.context?.httpSignature + + if (httpSignatureOptions) { + const method = options.method ?? 'GET' + const path = options.path ?? options.url.pathname + + if (!method || !path) { + throw new Error(`Cannot sign request without method (${method}) or path (${path}) ${options}`) + } + + httpSignature.signRequest({ + getHeader: function (header) { + return options.headers[header] + }, + + setHeader: function (header, value) { + options.headers[header] = value + }, + + method, + path + }, httpSignatureOptions) + } + } + ] } +}) - return new Bluebird<{ response: request.RequestResponse, body: T }>((res, rej) => { - request(requestOptions, (err, response, body) => err ? rej(err) : res({ response, body })) - .on('data', onRequestDataLengthCheck(bodyKBLimit)) - }) +function doRequest (url: string, options: PeerTubeRequestOptions = {}) { + const gotOptions = buildGotOptions(options) + + return peertubeGot(url, gotOptions) + .catch(err => { throw buildRequestError(err) }) +} + +function doJSONRequest (url: string, options: PeerTubeRequestOptions = {}) { + const gotOptions = buildGotOptions(options) + + return peertubeGot(url, { ...gotOptions, responseType: 'json' }) + .catch(err => { throw buildRequestError(err) }) } -function doRequestAndSaveToFile ( - requestOptions: request.CoreOptions & request.UriOptions, +async function doRequestAndSaveToFile ( + url: string, destPath: string, - bodyKBLimit = 10000 // 10MB + options: PeerTubeRequestOptions = {} ) { - if (!requestOptions.headers) requestOptions.headers = {} - requestOptions.headers['User-Agent'] = getUserAgent() - - return new Bluebird((res, rej) => { - const file = createWriteStream(destPath) - file.on('finish', () => res()) + const gotOptions = buildGotOptions(options) - request(requestOptions) - .on('data', onRequestDataLengthCheck(bodyKBLimit)) - .on('error', err => { - file.close() + const outFile = createWriteStream(destPath) - remove(destPath) - .catch(err => logger.error('Cannot remove %s after request failure.', destPath, { err })) + try { + await pipelinePromise( + peertubeGot.stream(url, gotOptions), + outFile + ) + } catch (err) { + remove(destPath) + .catch(err => logger.error('Cannot remove %s after request failure.', destPath, { err })) - return rej(err) - }) - .pipe(file) - }) + throw buildRequestError(err) + } } async function downloadImage (url: string, destDir: string, destName: string, size: { width: number, height: number }) { const tmpPath = join(CONFIG.STORAGE.TMP_DIR, 'pending-' + destName) - await doRequestAndSaveToFile({ method: 'GET', uri: url }, tmpPath) + await doRequestAndSaveToFile(url, tmpPath) const destPath = join(destDir, destName) @@ -73,24 +139,43 @@ function getUserAgent () { export { doRequest, + doJSONRequest, doRequestAndSaveToFile, downloadImage } // --------------------------------------------------------------------------- -// Thanks to https://github.com/request/request/issues/2470#issuecomment-268929907 <3 -function onRequestDataLengthCheck (bodyKBLimit: number) { - let bufferLength = 0 - const bytesLimit = bodyKBLimit * 1000 +function buildGotOptions (options: PeerTubeRequestOptions) { + const { activityPub, bodyKBLimit = 1000 } = options - return function (chunk) { - bufferLength += chunk.length - if (bufferLength > bytesLimit) { - this.abort() + const context = { bodyKBLimit, httpSignature: options.httpSignature } - const error = new Error(`Response was too large - aborted after ${bytesLimit} bytes.`) - this.emit('error', error) - } + let headers = options.headers || {} + + headers = { ...headers, date: new Date().toUTCString() } + + if (activityPub) { + headers = { ...headers, accept: ACTIVITY_PUB.ACCEPT_HEADER } } + + return { + method: options.method, + json: options.json, + searchParams: options.searchParams, + headers, + context + } +} + +function buildRequestError (error: any) { + const newError = new Error(error.message) + newError.name = error.name + newError.stack = error.stack + + if (error.response?.body) { + error.responseBody = error.response.body + } + + return newError } diff --git a/server/helpers/youtube-dl.ts b/server/helpers/youtube-dl.ts index 8537a5772..9d2e54fb5 100644 --- a/server/helpers/youtube-dl.ts +++ b/server/helpers/youtube-dl.ts @@ -1,13 +1,13 @@ import { createWriteStream } from 'fs' import { ensureDir, move, pathExists, remove, writeFile } from 'fs-extra' +import got from 'got' import { join } from 'path' -import * as request from 'request' import { CONFIG } from '@server/initializers/config' import { HttpStatusCode } from '../../shared/core-utils/miscs/http-error-codes' import { VideoResolution } from '../../shared/models/videos' import { CONSTRAINTS_FIELDS, VIDEO_CATEGORIES, VIDEO_LANGUAGES, VIDEO_LICENCES } from '../initializers/constants' import { getEnabledResolutions } from '../lib/video-transcoding' -import { peertubeTruncate, root } from './core-utils' +import { peertubeTruncate, pipelinePromise, root } from './core-utils' import { isVideoFileExtnameValid } from './custom-validators/videos' import { logger } from './logger' import { generateVideoImportTmpPath } from './utils' @@ -195,55 +195,32 @@ async function updateYoutubeDLBinary () { await ensureDir(binDirectory) - return new Promise(res => { - request.get(url, { followRedirect: false }, (err, result) => { - if (err) { - logger.error('Cannot update youtube-dl.', { err }) - return res() - } - - if (result.statusCode !== HttpStatusCode.FOUND_302) { - logger.error('youtube-dl update error: did not get redirect for the latest version link. Status %d', result.statusCode) - return res() - } - - const url = result.headers.location - const downloadFile = request.get(url) - const newVersion = /yt-dl\.org\/downloads\/(\d{4}\.\d\d\.\d\d(\.\d)?)\/youtube-dl/.exec(url)[1] - - downloadFile.on('response', result => { - if (result.statusCode !== HttpStatusCode.OK_200) { - logger.error('Cannot update youtube-dl: new version response is not 200, it\'s %d.', result.statusCode) - return res() - } - - const writeStream = createWriteStream(bin, { mode: 493 }).on('error', err => { - logger.error('youtube-dl update error in write stream', { err }) - return res() - }) + try { + const result = await got(url, { followRedirect: false }) - downloadFile.pipe(writeStream) - }) + if (result.statusCode !== HttpStatusCode.FOUND_302) { + logger.error('youtube-dl update error: did not get redirect for the latest version link. Status %d', result.statusCode) + return + } - downloadFile.on('error', err => { - logger.error('youtube-dl update error.', { err }) - return res() - }) + const newUrl = result.headers.location + const newVersion = /yt-dl\.org\/downloads\/(\d{4}\.\d\d\.\d\d(\.\d)?)\/youtube-dl/.exec(newUrl)[1] - downloadFile.on('end', () => { - const details = JSON.stringify({ version: newVersion, path: bin, exec: 'youtube-dl' }) - writeFile(detailsPath, details, { encoding: 'utf8' }, err => { - if (err) { - logger.error('youtube-dl update error: cannot write details.', { err }) - return res() - } + const downloadFileStream = got.stream(newUrl) + const writeStream = createWriteStream(bin, { mode: 493 }) - logger.info('youtube-dl updated to version %s.', newVersion) - return res() - }) - }) - }) - }) + await pipelinePromise( + downloadFileStream, + writeStream + ) + + const details = JSON.stringify({ version: newVersion, path: bin, exec: 'youtube-dl' }) + await writeFile(detailsPath, details, { encoding: 'utf8' }) + + logger.info('youtube-dl updated to version %s.', newVersion) + } catch (err) { + logger.error('Cannot update youtube-dl.', { err }) + } } async function safeGetYoutubeDL () { -- cgit v1.2.3