From db4b15f21fbf4e33434e930ffc7fb768cdcf9d42 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Mon, 8 Mar 2021 14:24:11 +0100 Subject: [PATCH] Use got instead of request --- package.json | 5 +- server/controllers/api/search.ts | 10 +- server/helpers/activitypub.ts | 5 +- server/helpers/core-utils.ts | 6 +- .../custom-validators/activitypub/activity.ts | 2 +- server/helpers/peertube-crypto.ts | 2 +- server/helpers/requests.ts | 179 +++++++++++++----- server/helpers/youtube-dl.ts | 71 +++---- server/initializers/constants.ts | 2 +- server/lib/activitypub/actor.ts | 41 ++-- server/lib/activitypub/crawl.ts | 25 ++- server/lib/activitypub/playlist.ts | 54 ++---- server/lib/activitypub/share.ts | 9 +- server/lib/activitypub/video-comments.ts | 14 +- server/lib/activitypub/video-rates.ts | 22 +-- server/lib/activitypub/videos.ts | 31 +-- .../lib/files-cache/videos-caption-cache.ts | 2 +- .../lib/files-cache/videos-preview-cache.ts | 2 +- .../lib/files-cache/videos-torrent-cache.ts | 2 +- server/lib/hls.ts | 4 +- .../job-queue/handlers/activitypub-cleaner.ts | 11 +- .../handlers/activitypub-http-broadcast.ts | 5 +- .../handlers/activitypub-http-unicast.ts | 5 +- .../handlers/utils/activitypub-http-utils.ts | 9 +- server/lib/plugins/plugin-index.ts | 26 +-- .../schedulers/auto-follow-index-instances.ts | 8 +- server/tests/api/activitypub/security.ts | 40 ++-- server/tests/api/server/handle-down.ts | 2 +- server/tests/helpers/request.ts | 16 +- shared/extra-utils/requests/activitypub.ts | 5 +- shared/models/server/job.model.ts | 2 +- yarn.lock | 49 +++-- 32 files changed, 342 insertions(+), 324 deletions(-) diff --git a/package.json b/package.json index 67a54a57f..323275faf 100644 --- a/package.json +++ b/package.json @@ -83,8 +83,7 @@ "sass-lint": "sass-lint" }, "resolutions": { - "oauth2-server": "3.1.0-beta.1", - "http-signature": "1.3.5" + "oauth2-server": "3.1.0-beta.1" }, "dependencies": { "apicache": "1.6.2", @@ -111,6 +110,7 @@ "flat": "^5.0.0", "fluent-ffmpeg": "^2.1.0", "fs-extra": "^9.0.0", + "got": "^11.8.2", "helmet": "^4.1.0", "http-signature": "1.3.5", "ip-anonymize": "^0.1.0", @@ -140,7 +140,6 @@ "pug": "^3.0.0", "redis": "^3.0.2", "reflect-metadata": "^0.1.12", - "request": "^2.81.0", "sanitize-html": "2.x", "scripty": "^2.0.0", "sequelize": "6.5.0", diff --git a/server/controllers/api/search.ts b/server/controllers/api/search.ts index 7e1b7b230..317b24fe9 100644 --- a/server/controllers/api/search.ts +++ b/server/controllers/api/search.ts @@ -1,6 +1,6 @@ import * as express from 'express' import { sanitizeUrl } from '@server/helpers/core-utils' -import { doRequest } from '@server/helpers/requests' +import { doJSONRequest } from '@server/helpers/requests' import { CONFIG } from '@server/initializers/config' import { getOrCreateVideoAndAccountAndChannel } from '@server/lib/activitypub/videos' import { AccountBlocklistModel } from '@server/models/account/account-blocklist' @@ -94,9 +94,9 @@ async function searchVideoChannelsIndex (query: VideoChannelsSearchQuery, res: e try { logger.debug('Doing video channels search index request on %s.', url, { body }) - const searchIndexResult = await doRequest>({ uri: url, body, json: true }) + const searchIndexResult = await doJSONRequest>(url, { json: body }) - return res.json(searchIndexResult.body) + return res.json(searchIndexResult) } catch (err) { logger.warn('Cannot use search index to make video channels search.', { err }) @@ -186,9 +186,9 @@ async function searchVideosIndex (query: VideosSearchQuery, res: express.Respons try { logger.debug('Doing videos search index request on %s.', url, { body }) - const searchIndexResult = await doRequest>({ uri: url, body, json: true }) + const searchIndexResult = await doJSONRequest>(url, { json: body }) - return res.json(searchIndexResult.body) + return res.json(searchIndexResult) } catch (err) { logger.warn('Cannot use search index to make video search.', { err }) diff --git a/server/helpers/activitypub.ts b/server/helpers/activitypub.ts index 08aef2908..e0754b501 100644 --- a/server/helpers/activitypub.ts +++ b/server/helpers/activitypub.ts @@ -3,7 +3,6 @@ import { URL } from 'url' import validator from 'validator' import { ContextType } from '@shared/models/activitypub/context' import { ResultList } from '../../shared/models' -import { Activity } from '../../shared/models/activitypub' import { ACTIVITY_PUB, REMOTE_SCHEME } from '../initializers/constants' import { MActor, MVideoWithHost } from '../types/models' import { pageToStartAndCount } from './core-utils' @@ -182,10 +181,10 @@ async function activityPubCollectionPagination ( } -function buildSignedActivity (byActor: MActor, data: Object, contextType?: ContextType) { +function buildSignedActivity (byActor: MActor, data: T, contextType?: ContextType) { const activity = activityPubContextify(data, contextType) - return signJsonLDObject(byActor, activity) as Promise + return signJsonLDObject(byActor, activity) } function getAPId (activity: string | { id: string }) { diff --git a/server/helpers/core-utils.ts b/server/helpers/core-utils.ts index 935fd22d9..7ba7d865a 100644 --- a/server/helpers/core-utils.ts +++ b/server/helpers/core-utils.ts @@ -10,7 +10,9 @@ import { BinaryToTextEncoding, createHash, randomBytes } from 'crypto' import { truncate } from 'lodash' import { basename, isAbsolute, join, resolve } from 'path' import * as pem from 'pem' +import { pipeline } from 'stream' import { URL } from 'url' +import { promisify } from 'util' const objectConverter = (oldObject: any, keyConverter: (e: string) => string, valueConverter: (e: any) => any) => { if (!oldObject || typeof oldObject !== 'object') { @@ -254,6 +256,7 @@ const createPrivateKey = promisify1(pem.createPrivateKe const getPublicKey = promisify1(pem.getPublicKey) const execPromise2 = promisify2(exec) const execPromise = promisify1(exec) +const pipelinePromise = promisify(pipeline) // --------------------------------------------------------------------------- @@ -284,5 +287,6 @@ export { createPrivateKey, getPublicKey, execPromise2, - execPromise + execPromise, + pipelinePromise } diff --git a/server/helpers/custom-validators/activitypub/activity.ts b/server/helpers/custom-validators/activitypub/activity.ts index 46126da57..69558e358 100644 --- a/server/helpers/custom-validators/activitypub/activity.ts +++ b/server/helpers/custom-validators/activitypub/activity.ts @@ -41,7 +41,7 @@ const activityCheckers: { [ P in ActivityType ]: (activity: Activity) => boolean } function isActivityValid (activity: any) { - const checker = activityCheckers[activity.tswype] + const checker = activityCheckers[activity.type] // Unknown activity type if (!checker) return false diff --git a/server/helpers/peertube-crypto.ts b/server/helpers/peertube-crypto.ts index 994f725d8..bc6f1d074 100644 --- a/server/helpers/peertube-crypto.ts +++ b/server/helpers/peertube-crypto.ts @@ -84,7 +84,7 @@ async function isJsonLDRSA2017Verified (fromActor: MActor, signedDocument: any) return verify.verify(fromActor.publicKey, signedDocument.signature.signatureValue, 'base64') } -async function signJsonLDObject (byActor: MActor, data: any) { +async function signJsonLDObject (byActor: MActor, data: T) { const signature = { type: 'RsaSignature2017', creator: byActor.url, diff --git a/server/helpers/requests.ts b/server/helpers/requests.ts index b556c392e..2c9da213c 100644 --- a/server/helpers/requests.ts +++ b/server/helpers/requests.ts @@ -1,58 +1,124 @@ -import * as Bluebird from 'bluebird' import { createWriteStream, remove } from 'fs-extra' -import * as request from 'request' +import got, { CancelableRequest, Options as GotOptions } from 'got' +import { join } from 'path' +import { CONFIG } from '../initializers/config' import { ACTIVITY_PUB, PEERTUBE_VERSION, WEBSERVER } from '../initializers/constants' +import { pipelinePromise } from './core-utils' import { processImage } from './image-utils' -import { join } from 'path' import { logger } from './logger' -import { CONFIG } from '../initializers/config' -function doRequest ( - requestOptions: request.CoreOptions & request.UriOptions & { activityPub?: boolean }, - bodyKBLimit = 1000 // 1MB -): Bluebird<{ response: request.RequestResponse, body: T }> { - if (!(requestOptions.headers)) requestOptions.headers = {} - requestOptions.headers['User-Agent'] = getUserAgent() +const httpSignature = require('http-signature') + +type PeerTubeRequestOptions = { + activityPub?: boolean + bodyKBLimit?: number // 1MB + httpSignature?: { + algorithm: string + authorizationHeaderName: string + keyId: string + key: string + headers: string[] + } + jsonResponse?: boolean +} & Pick + +const peertubeGot = got.extend({ + headers: { + 'user-agent': getUserAgent() + }, + + handlers: [ + (options, next) => { + const promiseOrStream = next(options) as CancelableRequest + const bodyKBLimit = options.context?.bodyKBLimit + if (!bodyKBLimit) throw new Error('No KB limit for this request') + + /* eslint-disable @typescript-eslint/no-floating-promises */ + promiseOrStream.on('downloadProgress', progress => { + if (progress.transferred * 1000 > bodyKBLimit && progress.percent !== 1) { + promiseOrStream.cancel(`Exceeded the download limit of ${bodyKBLimit} bytes`) + } + }) - if (requestOptions.activityPub === true) { - requestOptions.headers['accept'] = ACTIVITY_PUB.ACCEPT_HEADER + return promiseOrStream + } + ], + + hooks: { + beforeRequest: [ + options => { + const headers = options.headers || {} + headers['host'] = options.url.host + }, + + options => { + const httpSignatureOptions = options.context?.httpSignature + + if (httpSignatureOptions) { + const method = options.method ?? 'GET' + const path = options.path ?? options.url.pathname + + if (!method || !path) { + throw new Error(`Cannot sign request without method (${method}) or path (${path}) ${options}`) + } + + httpSignature.signRequest({ + getHeader: function (header) { + return options.headers[header] + }, + + setHeader: function (header, value) { + options.headers[header] = value + }, + + method, + path + }, httpSignatureOptions) + } + } + ] } +}) - return new Bluebird<{ response: request.RequestResponse, body: T }>((res, rej) => { - request(requestOptions, (err, response, body) => err ? rej(err) : res({ response, body })) - .on('data', onRequestDataLengthCheck(bodyKBLimit)) - }) +function doRequest (url: string, options: PeerTubeRequestOptions = {}) { + const gotOptions = buildGotOptions(options) + + return peertubeGot(url, gotOptions) + .catch(err => { throw buildRequestError(err) }) +} + +function doJSONRequest (url: string, options: PeerTubeRequestOptions = {}) { + const gotOptions = buildGotOptions(options) + + return peertubeGot(url, { ...gotOptions, responseType: 'json' }) + .catch(err => { throw buildRequestError(err) }) } -function doRequestAndSaveToFile ( - requestOptions: request.CoreOptions & request.UriOptions, +async function doRequestAndSaveToFile ( + url: string, destPath: string, - bodyKBLimit = 10000 // 10MB + options: PeerTubeRequestOptions = {} ) { - if (!requestOptions.headers) requestOptions.headers = {} - requestOptions.headers['User-Agent'] = getUserAgent() - - return new Bluebird((res, rej) => { - const file = createWriteStream(destPath) - file.on('finish', () => res()) + const gotOptions = buildGotOptions(options) - request(requestOptions) - .on('data', onRequestDataLengthCheck(bodyKBLimit)) - .on('error', err => { - file.close() + const outFile = createWriteStream(destPath) - remove(destPath) - .catch(err => logger.error('Cannot remove %s after request failure.', destPath, { err })) + try { + await pipelinePromise( + peertubeGot.stream(url, gotOptions), + outFile + ) + } catch (err) { + remove(destPath) + .catch(err => logger.error('Cannot remove %s after request failure.', destPath, { err })) - return rej(err) - }) - .pipe(file) - }) + throw buildRequestError(err) + } } async function downloadImage (url: string, destDir: string, destName: string, size: { width: number, height: number }) { const tmpPath = join(CONFIG.STORAGE.TMP_DIR, 'pending-' + destName) - await doRequestAndSaveToFile({ method: 'GET', uri: url }, tmpPath) + await doRequestAndSaveToFile(url, tmpPath) const destPath = join(destDir, destName) @@ -73,24 +139,43 @@ function getUserAgent () { export { doRequest, + doJSONRequest, doRequestAndSaveToFile, downloadImage } // --------------------------------------------------------------------------- -// Thanks to https://github.com/request/request/issues/2470#issuecomment-268929907 <3 -function onRequestDataLengthCheck (bodyKBLimit: number) { - let bufferLength = 0 - const bytesLimit = bodyKBLimit * 1000 +function buildGotOptions (options: PeerTubeRequestOptions) { + const { activityPub, bodyKBLimit = 1000 } = options - return function (chunk) { - bufferLength += chunk.length - if (bufferLength > bytesLimit) { - this.abort() + const context = { bodyKBLimit, httpSignature: options.httpSignature } - const error = new Error(`Response was too large - aborted after ${bytesLimit} bytes.`) - this.emit('error', error) - } + let headers = options.headers || {} + + headers = { ...headers, date: new Date().toUTCString() } + + if (activityPub) { + headers = { ...headers, accept: ACTIVITY_PUB.ACCEPT_HEADER } } + + return { + method: options.method, + json: options.json, + searchParams: options.searchParams, + headers, + context + } +} + +function buildRequestError (error: any) { + const newError = new Error(error.message) + newError.name = error.name + newError.stack = error.stack + + if (error.response?.body) { + error.responseBody = error.response.body + } + + return newError } diff --git a/server/helpers/youtube-dl.ts b/server/helpers/youtube-dl.ts index 8537a5772..9d2e54fb5 100644 --- a/server/helpers/youtube-dl.ts +++ b/server/helpers/youtube-dl.ts @@ -1,13 +1,13 @@ import { createWriteStream } from 'fs' import { ensureDir, move, pathExists, remove, writeFile } from 'fs-extra' +import got from 'got' import { join } from 'path' -import * as request from 'request' import { CONFIG } from '@server/initializers/config' import { HttpStatusCode } from '../../shared/core-utils/miscs/http-error-codes' import { VideoResolution } from '../../shared/models/videos' import { CONSTRAINTS_FIELDS, VIDEO_CATEGORIES, VIDEO_LANGUAGES, VIDEO_LICENCES } from '../initializers/constants' import { getEnabledResolutions } from '../lib/video-transcoding' -import { peertubeTruncate, root } from './core-utils' +import { peertubeTruncate, pipelinePromise, root } from './core-utils' import { isVideoFileExtnameValid } from './custom-validators/videos' import { logger } from './logger' import { generateVideoImportTmpPath } from './utils' @@ -195,55 +195,32 @@ async function updateYoutubeDLBinary () { await ensureDir(binDirectory) - return new Promise(res => { - request.get(url, { followRedirect: false }, (err, result) => { - if (err) { - logger.error('Cannot update youtube-dl.', { err }) - return res() - } - - if (result.statusCode !== HttpStatusCode.FOUND_302) { - logger.error('youtube-dl update error: did not get redirect for the latest version link. Status %d', result.statusCode) - return res() - } - - const url = result.headers.location - const downloadFile = request.get(url) - const newVersion = /yt-dl\.org\/downloads\/(\d{4}\.\d\d\.\d\d(\.\d)?)\/youtube-dl/.exec(url)[1] - - downloadFile.on('response', result => { - if (result.statusCode !== HttpStatusCode.OK_200) { - logger.error('Cannot update youtube-dl: new version response is not 200, it\'s %d.', result.statusCode) - return res() - } - - const writeStream = createWriteStream(bin, { mode: 493 }).on('error', err => { - logger.error('youtube-dl update error in write stream', { err }) - return res() - }) + try { + const result = await got(url, { followRedirect: false }) - downloadFile.pipe(writeStream) - }) + if (result.statusCode !== HttpStatusCode.FOUND_302) { + logger.error('youtube-dl update error: did not get redirect for the latest version link. Status %d', result.statusCode) + return + } - downloadFile.on('error', err => { - logger.error('youtube-dl update error.', { err }) - return res() - }) + const newUrl = result.headers.location + const newVersion = /yt-dl\.org\/downloads\/(\d{4}\.\d\d\.\d\d(\.\d)?)\/youtube-dl/.exec(newUrl)[1] - downloadFile.on('end', () => { - const details = JSON.stringify({ version: newVersion, path: bin, exec: 'youtube-dl' }) - writeFile(detailsPath, details, { encoding: 'utf8' }, err => { - if (err) { - logger.error('youtube-dl update error: cannot write details.', { err }) - return res() - } + const downloadFileStream = got.stream(newUrl) + const writeStream = createWriteStream(bin, { mode: 493 }) - logger.info('youtube-dl updated to version %s.', newVersion) - return res() - }) - }) - }) - }) + await pipelinePromise( + downloadFileStream, + writeStream + ) + + const details = JSON.stringify({ version: newVersion, path: bin, exec: 'youtube-dl' }) + await writeFile(detailsPath, details, { encoding: 'utf8' }) + + logger.info('youtube-dl updated to version %s.', newVersion) + } catch (err) { + logger.error('Cannot update youtube-dl.', { err }) + } } async function safeGetYoutubeDL () { diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index 1623e6f42..ea98e8a38 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts @@ -29,7 +29,7 @@ const LAST_MIGRATION_VERSION = 610 // --------------------------------------------------------------------------- const API_VERSION = 'v1' -const PEERTUBE_VERSION = require(join(root(), 'package.json')).version +const PEERTUBE_VERSION: string = require(join(root(), 'package.json')).version const PAGINATION = { GLOBAL: { diff --git a/server/lib/activitypub/actor.ts b/server/lib/activitypub/actor.ts index a726f9e20..52b6c1f56 100644 --- a/server/lib/activitypub/actor.ts +++ b/server/lib/activitypub/actor.ts @@ -1,26 +1,28 @@ import * as Bluebird from 'bluebird' +import { extname } from 'path' import { Op, Transaction } from 'sequelize' import { URL } from 'url' import { v4 as uuidv4 } from 'uuid' +import { getServerActor } from '@server/models/application/application' +import { HttpStatusCode } from '../../../shared/core-utils/miscs/http-error-codes' import { ActivityPubActor, ActivityPubActorType, ActivityPubOrderedCollection } from '../../../shared/models/activitypub' import { ActivityPubAttributedTo } from '../../../shared/models/activitypub/objects' import { checkUrlsSameHost, getAPId } from '../../helpers/activitypub' +import { ActorFetchByUrlType, fetchActorByUrl } from '../../helpers/actor' import { sanitizeAndCheckActorObject } from '../../helpers/custom-validators/activitypub/actor' import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc' import { retryTransactionWrapper, updateInstanceWithAnother } from '../../helpers/database-utils' import { logger } from '../../helpers/logger' import { createPrivateAndPublicKeys } from '../../helpers/peertube-crypto' -import { doRequest } from '../../helpers/requests' +import { doJSONRequest } from '../../helpers/requests' import { getUrlFromWebfinger } from '../../helpers/webfinger' import { MIMETYPES, WEBSERVER } from '../../initializers/constants' +import { sequelizeTypescript } from '../../initializers/database' import { AccountModel } from '../../models/account/account' import { ActorModel } from '../../models/activitypub/actor' import { AvatarModel } from '../../models/avatar/avatar' import { ServerModel } from '../../models/server/server' import { VideoChannelModel } from '../../models/video/video-channel' -import { JobQueue } from '../job-queue' -import { ActorFetchByUrlType, fetchActorByUrl } from '../../helpers/actor' -import { sequelizeTypescript } from '../../initializers/database' import { MAccount, MAccountDefault, @@ -34,9 +36,7 @@ import { MActorId, MChannel } from '../../types/models' -import { extname } from 'path' -import { getServerActor } from '@server/models/application/application' -import { HttpStatusCode } from '../../../shared/core-utils/miscs/http-error-codes' +import { JobQueue } from '../job-queue' // Set account keys, this could be long so process after the account creation and do not block the client async function generateAndSaveActorKeys (actor: T) { @@ -209,16 +209,10 @@ async function deleteActorAvatarInstance (actor: MActorDefault, t: Transaction) } async function fetchActorTotalItems (url: string) { - const options = { - uri: url, - method: 'GET', - json: true, - activityPub: true - } - try { - const { body } = await doRequest>(options) - return body.totalItems ? body.totalItems : 0 + const { body } = await doJSONRequest>(url, { activityPub: true }) + + return body.totalItems || 0 } catch (err) { logger.warn('Cannot fetch remote actor count %s.', url, { err }) return 0 @@ -449,26 +443,19 @@ type FetchRemoteActorResult = { attributedTo: ActivityPubAttributedTo[] } async function fetchRemoteActor (actorUrl: string): Promise<{ statusCode?: number, result: FetchRemoteActorResult }> { - const options = { - uri: actorUrl, - method: 'GET', - json: true, - activityPub: true - } - logger.info('Fetching remote actor %s.', actorUrl) - const requestResult = await doRequest(options) + const requestResult = await doJSONRequest(actorUrl, { activityPub: true }) const actorJSON = requestResult.body if (sanitizeAndCheckActorObject(actorJSON) === false) { logger.debug('Remote actor JSON is not valid.', { actorJSON }) - return { result: undefined, statusCode: requestResult.response.statusCode } + return { result: undefined, statusCode: requestResult.statusCode } } if (checkUrlsSameHost(actorJSON.id, actorUrl) !== true) { logger.warn('Actor url %s has not the same host than its AP id %s', actorUrl, actorJSON.id) - return { result: undefined, statusCode: requestResult.response.statusCode } + return { result: undefined, statusCode: requestResult.statusCode } } const followersCount = await fetchActorTotalItems(actorJSON.followers) @@ -496,7 +483,7 @@ async function fetchRemoteActor (actorUrl: string): Promise<{ statusCode?: numbe const name = actorJSON.name || actorJSON.preferredUsername return { - statusCode: requestResult.response.statusCode, + statusCode: requestResult.statusCode, result: { actor, name, diff --git a/server/lib/activitypub/crawl.ts b/server/lib/activitypub/crawl.ts index 1ed105bbe..278abf7de 100644 --- a/server/lib/activitypub/crawl.ts +++ b/server/lib/activitypub/crawl.ts @@ -1,27 +1,26 @@ -import { ACTIVITY_PUB, REQUEST_TIMEOUT, WEBSERVER } from '../../initializers/constants' -import { doRequest } from '../../helpers/requests' -import { logger } from '../../helpers/logger' import * as Bluebird from 'bluebird' -import { ActivityPubOrderedCollection } from '../../../shared/models/activitypub' import { URL } from 'url' +import { ActivityPubOrderedCollection } from '../../../shared/models/activitypub' +import { logger } from '../../helpers/logger' +import { doJSONRequest } from '../../helpers/requests' +import { ACTIVITY_PUB, REQUEST_TIMEOUT, WEBSERVER } from '../../initializers/constants' type HandlerFunction = (items: T[]) => (Promise | Bluebird) type CleanerFunction = (startedDate: Date) => (Promise | Bluebird) -async function crawlCollectionPage (uri: string, handler: HandlerFunction, cleaner?: CleanerFunction) { - logger.info('Crawling ActivityPub data on %s.', uri) +async function crawlCollectionPage (argUrl: string, handler: HandlerFunction, cleaner?: CleanerFunction) { + let url = argUrl + + logger.info('Crawling ActivityPub data on %s.', url) const options = { - method: 'GET', - uri, - json: true, activityPub: true, timeout: REQUEST_TIMEOUT } const startDate = new Date() - const response = await doRequest>(options) + const response = await doJSONRequest>(url, options) const firstBody = response.body const limit = ACTIVITY_PUB.FETCH_PAGE_LIMIT @@ -35,9 +34,9 @@ async function crawlCollectionPage (uri: string, handler: HandlerFunction const remoteHost = new URL(nextLink).host if (remoteHost === WEBSERVER.HOST) continue - options.uri = nextLink + url = nextLink - const res = await doRequest>(options) + const res = await doJSONRequest>(url, options) body = res.body } else { // nextLink is already the object we want @@ -49,7 +48,7 @@ async function crawlCollectionPage (uri: string, handler: HandlerFunction if (Array.isArray(body.orderedItems)) { const items = body.orderedItems - logger.info('Processing %i ActivityPub items for %s.', items.length, options.uri) + logger.info('Processing %i ActivityPub items for %s.', items.length, url) await handler(items) } diff --git a/server/lib/activitypub/playlist.ts b/server/lib/activitypub/playlist.ts index d5a3ef7c8..795be60d7 100644 --- a/server/lib/activitypub/playlist.ts +++ b/server/lib/activitypub/playlist.ts @@ -1,24 +1,24 @@ +import * as Bluebird from 'bluebird' +import { HttpStatusCode } from '../../../shared/core-utils/miscs/http-error-codes' +import { PlaylistElementObject } from '../../../shared/models/activitypub/objects/playlist-element-object' import { PlaylistObject } from '../../../shared/models/activitypub/objects/playlist-object' -import { crawlCollectionPage } from './crawl' -import { ACTIVITY_PUB, CRAWL_REQUEST_CONCURRENCY } from '../../initializers/constants' +import { VideoPlaylistPrivacy } from '../../../shared/models/videos/playlist/video-playlist-privacy.model' +import { checkUrlsSameHost } from '../../helpers/activitypub' +import { isPlaylistElementObjectValid, isPlaylistObjectValid } from '../../helpers/custom-validators/activitypub/playlist' import { isArray } from '../../helpers/custom-validators/misc' -import { getOrCreateActorAndServerAndModel } from './actor' import { logger } from '../../helpers/logger' +import { doJSONRequest } from '../../helpers/requests' +import { ACTIVITY_PUB, CRAWL_REQUEST_CONCURRENCY } from '../../initializers/constants' +import { sequelizeTypescript } from '../../initializers/database' import { VideoPlaylistModel } from '../../models/video/video-playlist' -import { doRequest } from '../../helpers/requests' -import { checkUrlsSameHost } from '../../helpers/activitypub' -import * as Bluebird from 'bluebird' -import { PlaylistElementObject } from '../../../shared/models/activitypub/objects/playlist-element-object' -import { getOrCreateVideoAndAccountAndChannel } from './videos' -import { isPlaylistElementObjectValid, isPlaylistObjectValid } from '../../helpers/custom-validators/activitypub/playlist' import { VideoPlaylistElementModel } from '../../models/video/video-playlist-element' -import { VideoPlaylistPrivacy } from '../../../shared/models/videos/playlist/video-playlist-privacy.model' -import { sequelizeTypescript } from '../../initializers/database' -import { createPlaylistMiniatureFromUrl } from '../thumbnail' -import { FilteredModelAttributes } from '../../types/sequelize' import { MAccountDefault, MAccountId, MVideoId } from '../../types/models' import { MVideoPlaylist, MVideoPlaylistId, MVideoPlaylistOwner } from '../../types/models/video/video-playlist' -import { HttpStatusCode } from '../../../shared/core-utils/miscs/http-error-codes' +import { FilteredModelAttributes } from '../../types/sequelize' +import { createPlaylistMiniatureFromUrl } from '../thumbnail' +import { getOrCreateActorAndServerAndModel } from './actor' +import { crawlCollectionPage } from './crawl' +import { getOrCreateVideoAndAccountAndChannel } from './videos' function playlistObjectToDBAttributes (playlistObject: PlaylistObject, byAccount: MAccountId, to: string[]) { const privacy = to.includes(ACTIVITY_PUB.PUBLIC) @@ -56,11 +56,7 @@ async function createAccountPlaylists (playlistUrls: string[], account: MAccount if (exists === true) return // Fetch url - const { body } = await doRequest({ - uri: playlistUrl, - json: true, - activityPub: true - }) + const { body } = await doJSONRequest(playlistUrl, { activityPub: true }) if (!isPlaylistObjectValid(body)) { throw new Error(`Invalid playlist object when fetch account playlists: ${JSON.stringify(body)}`) @@ -164,12 +160,7 @@ async function resetVideoPlaylistElements (elementUrls: string[], playlist: MVid await Bluebird.map(elementUrls, async elementUrl => { try { - // Fetch url - const { body } = await doRequest({ - uri: elementUrl, - json: true, - activityPub: true - }) + const { body } = await doJSONRequest(elementUrl, { activityPub: true }) if (!isPlaylistElementObjectValid(body)) throw new Error(`Invalid body in video get playlist element ${elementUrl}`) @@ -199,21 +190,14 @@ async function resetVideoPlaylistElements (elementUrls: string[], playlist: MVid } async function fetchRemoteVideoPlaylist (playlistUrl: string): Promise<{ statusCode: number, playlistObject: PlaylistObject }> { - const options = { - uri: playlistUrl, - method: 'GET', - json: true, - activityPub: true - } - logger.info('Fetching remote playlist %s.', playlistUrl) - const { response, body } = await doRequest(options) + const { body, statusCode } = await doJSONRequest(playlistUrl, { activityPub: true }) if (isPlaylistObjectValid(body) === false || checkUrlsSameHost(body.id, playlistUrl) !== true) { logger.debug('Remote video playlist JSON is not valid.', { body }) - return { statusCode: response.statusCode, playlistObject: undefined } + return { statusCode, playlistObject: undefined } } - return { statusCode: response.statusCode, playlistObject: body } + return { statusCode, playlistObject: body } } diff --git a/server/lib/activitypub/share.ts b/server/lib/activitypub/share.ts index dde0c628e..c22fa0893 100644 --- a/server/lib/activitypub/share.ts +++ b/server/lib/activitypub/share.ts @@ -3,7 +3,7 @@ import { Transaction } from 'sequelize' import { getServerActor } from '@server/models/application/application' import { checkUrlsSameHost, getAPId } from '../../helpers/activitypub' import { logger, loggerTagsFactory } from '../../helpers/logger' -import { doRequest } from '../../helpers/requests' +import { doJSONRequest } from '../../helpers/requests' import { CRAWL_REQUEST_CONCURRENCY } from '../../initializers/constants' import { VideoShareModel } from '../../models/video/video-share' import { MChannelActorLight, MVideo, MVideoAccountLight, MVideoId } from '../../types/models/video' @@ -40,12 +40,7 @@ async function changeVideoChannelShare ( async function addVideoShares (shareUrls: string[], video: MVideoId) { await Bluebird.map(shareUrls, async shareUrl => { try { - // Fetch url - const { body } = await doRequest({ - uri: shareUrl, - json: true, - activityPub: true - }) + const { body } = await doJSONRequest(shareUrl, { activityPub: true }) if (!body || !body.actor) throw new Error('Body or body actor is invalid') const actorUrl = getAPId(body.actor) diff --git a/server/lib/activitypub/video-comments.ts b/server/lib/activitypub/video-comments.ts index d025ed7f1..f1edfb0ac 100644 --- a/server/lib/activitypub/video-comments.ts +++ b/server/lib/activitypub/video-comments.ts @@ -1,13 +1,13 @@ +import * as Bluebird from 'bluebird' +import { checkUrlsSameHost } from '../../helpers/activitypub' import { sanitizeAndCheckVideoCommentObject } from '../../helpers/custom-validators/activitypub/video-comments' import { logger } from '../../helpers/logger' -import { doRequest } from '../../helpers/requests' +import { doJSONRequest } from '../../helpers/requests' import { ACTIVITY_PUB, CRAWL_REQUEST_CONCURRENCY } from '../../initializers/constants' import { VideoCommentModel } from '../../models/video/video-comment' +import { MCommentOwner, MCommentOwnerVideo, MVideoAccountLightBlacklistAllFiles } from '../../types/models/video' import { getOrCreateActorAndServerAndModel } from './actor' import { getOrCreateVideoAndAccountAndChannel } from './videos' -import * as Bluebird from 'bluebird' -import { checkUrlsSameHost } from '../../helpers/activitypub' -import { MCommentOwner, MCommentOwnerVideo, MVideoAccountLightBlacklistAllFiles } from '../../types/models/video' type ResolveThreadParams = { url: string @@ -126,11 +126,7 @@ async function resolveRemoteParentComment (params: ResolveThreadParams) { throw new Error('Recursion limit reached when resolving a thread') } - const { body } = await doRequest({ - uri: url, - json: true, - activityPub: true - }) + const { body } = await doJSONRequest(url, { activityPub: true }) if (sanitizeAndCheckVideoCommentObject(body) === false) { throw new Error(`Remote video comment JSON ${url} is not valid:` + JSON.stringify(body)) diff --git a/server/lib/activitypub/video-rates.ts b/server/lib/activitypub/video-rates.ts index e246b1313..f40c07fea 100644 --- a/server/lib/activitypub/video-rates.ts +++ b/server/lib/activitypub/video-rates.ts @@ -1,26 +1,22 @@ +import * as Bluebird from 'bluebird' import { Transaction } from 'sequelize' -import { sendLike, sendUndoDislike, sendUndoLike } from './send' +import { doJSONRequest } from '@server/helpers/requests' import { VideoRateType } from '../../../shared/models/videos' -import * as Bluebird from 'bluebird' -import { getOrCreateActorAndServerAndModel } from './actor' -import { AccountVideoRateModel } from '../../models/account/account-video-rate' +import { checkUrlsSameHost, getAPId } from '../../helpers/activitypub' import { logger } from '../../helpers/logger' import { CRAWL_REQUEST_CONCURRENCY } from '../../initializers/constants' -import { doRequest } from '../../helpers/requests' -import { checkUrlsSameHost, getAPId } from '../../helpers/activitypub' -import { getVideoDislikeActivityPubUrlByLocalActor, getVideoLikeActivityPubUrlByLocalActor } from './url' -import { sendDislike } from './send/send-dislike' +import { AccountVideoRateModel } from '../../models/account/account-video-rate' import { MAccountActor, MActorUrl, MVideo, MVideoAccountLight, MVideoId } from '../../types/models' +import { getOrCreateActorAndServerAndModel } from './actor' +import { sendLike, sendUndoDislike, sendUndoLike } from './send' +import { sendDislike } from './send/send-dislike' +import { getVideoDislikeActivityPubUrlByLocalActor, getVideoLikeActivityPubUrlByLocalActor } from './url' async function createRates (ratesUrl: string[], video: MVideo, rate: VideoRateType) { await Bluebird.map(ratesUrl, async rateUrl => { try { // Fetch url - const { body } = await doRequest({ - uri: rateUrl, - json: true, - activityPub: true - }) + const { body } = await doJSONRequest(rateUrl, { activityPub: true }) if (!body || !body.actor) throw new Error('Body or body actor is invalid') const actorUrl = getAPId(body.actor) diff --git a/server/lib/activitypub/videos.ts b/server/lib/activitypub/videos.ts index c02578aad..a5f58dd01 100644 --- a/server/lib/activitypub/videos.ts +++ b/server/lib/activitypub/videos.ts @@ -2,7 +2,6 @@ import * as Bluebird from 'bluebird' import { maxBy, minBy } from 'lodash' import * as magnetUtil from 'magnet-uri' import { basename, join } from 'path' -import * as request from 'request' import { Transaction } from 'sequelize/types' import { TrackerModel } from '@server/models/server/tracker' import { VideoLiveModel } from '@server/models/video/video-live' @@ -31,7 +30,7 @@ import { isArray } from '../../helpers/custom-validators/misc' import { isVideoFileInfoHashValid } from '../../helpers/custom-validators/videos' import { deleteNonExistingModels, resetSequelizeInstance, retryTransactionWrapper } from '../../helpers/database-utils' import { logger } from '../../helpers/logger' -import { doRequest } from '../../helpers/requests' +import { doJSONRequest } from '../../helpers/requests' import { fetchVideoByUrl, getExtFromMimetype, VideoFetchByUrlType } from '../../helpers/video' import { ACTIVITY_PUB, @@ -115,36 +114,26 @@ async function federateVideoIfNeeded (videoArg: MVideoAPWithoutCaption, isNewVid } } -async function fetchRemoteVideo (videoUrl: string): Promise<{ response: request.RequestResponse, videoObject: VideoObject }> { - const options = { - uri: videoUrl, - method: 'GET', - json: true, - activityPub: true - } - +async function fetchRemoteVideo (videoUrl: string): Promise<{ statusCode: number, videoObject: VideoObject }> { logger.info('Fetching remote video %s.', videoUrl) - const { response, body } = await doRequest(options) + const { statusCode, body } = await doJSONRequest(videoUrl, { activityPub: true }) if (sanitizeAndCheckVideoTorrentObject(body) === false || checkUrlsSameHost(body.id, videoUrl) !== true) { logger.debug('Remote video JSON is not valid.', { body }) - return { response, videoObject: undefined } + return { statusCode, videoObject: undefined } } - return { response, videoObject: body } + return { statusCode, videoObject: body } } async function fetchRemoteVideoDescription (video: MVideoAccountLight) { const host = video.VideoChannel.Account.Actor.Server.host const path = video.getDescriptionAPIPath() - const options = { - uri: REMOTE_SCHEME.HTTP + '://' + host + path, - json: true - } + const url = REMOTE_SCHEME.HTTP + '://' + host + path - const { body } = await doRequest(options) - return body.description ? body.description : '' + const { body } = await doJSONRequest(url) + return body.description || '' } function getOrCreateVideoChannelFromVideoObject (videoObject: VideoObject) { @@ -534,8 +523,8 @@ async function refreshVideoIfNeeded (options: { : await VideoModel.loadByUrlAndPopulateAccount(options.video.url) try { - const { response, videoObject } = await fetchRemoteVideo(video.url) - if (response.statusCode === HttpStatusCode.NOT_FOUND_404) { + const { statusCode, videoObject } = await fetchRemoteVideo(video.url) + if (statusCode === HttpStatusCode.NOT_FOUND_404) { logger.info('Cannot refresh remote video %s: video does not exist anymore. Deleting it.', video.url) // Video does not exist anymore diff --git a/server/lib/files-cache/videos-caption-cache.ts b/server/lib/files-cache/videos-caption-cache.ts index ee0447010..58e2260b6 100644 --- a/server/lib/files-cache/videos-caption-cache.ts +++ b/server/lib/files-cache/videos-caption-cache.ts @@ -41,7 +41,7 @@ class VideosCaptionCache extends AbstractVideoStaticFileCache { const remoteUrl = videoCaption.getFileUrl(video) const destPath = join(FILES_CACHE.VIDEO_CAPTIONS.DIRECTORY, videoCaption.filename) - await doRequestAndSaveToFile({ uri: remoteUrl }, destPath) + await doRequestAndSaveToFile(remoteUrl, destPath) return { isOwned: false, path: destPath } } diff --git a/server/lib/files-cache/videos-preview-cache.ts b/server/lib/files-cache/videos-preview-cache.ts index ee72cd3f9..dd3a84aca 100644 --- a/server/lib/files-cache/videos-preview-cache.ts +++ b/server/lib/files-cache/videos-preview-cache.ts @@ -39,7 +39,7 @@ class VideosPreviewCache extends AbstractVideoStaticFileCache { const destPath = join(FILES_CACHE.PREVIEWS.DIRECTORY, preview.filename) const remoteUrl = preview.getFileUrl(video) - await doRequestAndSaveToFile({ uri: remoteUrl }, destPath) + await doRequestAndSaveToFile(remoteUrl, destPath) logger.debug('Fetched remote preview %s to %s.', remoteUrl, destPath) diff --git a/server/lib/files-cache/videos-torrent-cache.ts b/server/lib/files-cache/videos-torrent-cache.ts index ca0e1770d..881fa9ced 100644 --- a/server/lib/files-cache/videos-torrent-cache.ts +++ b/server/lib/files-cache/videos-torrent-cache.ts @@ -41,7 +41,7 @@ class VideosTorrentCache extends AbstractVideoStaticFileCache { const remoteUrl = file.getRemoteTorrentUrl(video) const destPath = join(FILES_CACHE.TORRENTS.DIRECTORY, file.torrentFilename) - await doRequestAndSaveToFile({ uri: remoteUrl }, destPath) + await doRequestAndSaveToFile(remoteUrl, destPath) const downloadName = `${video.name}-${file.resolution}p.torrent` diff --git a/server/lib/hls.ts b/server/lib/hls.ts index 04187668c..84539e2c1 100644 --- a/server/lib/hls.ts +++ b/server/lib/hls.ts @@ -135,7 +135,7 @@ function downloadPlaylistSegments (playlistUrl: string, destinationDir: string, const destPath = join(tmpDirectory, basename(fileUrl)) const bodyKBLimit = 10 * 1000 * 1000 // 10GB - await doRequestAndSaveToFile({ uri: fileUrl }, destPath, bodyKBLimit) + await doRequestAndSaveToFile(fileUrl, destPath, { bodyKBLimit }) } clearTimeout(timer) @@ -156,7 +156,7 @@ function downloadPlaylistSegments (playlistUrl: string, destinationDir: string, } async function fetchUniqUrls (playlistUrl: string) { - const { body } = await doRequest({ uri: playlistUrl }) + const { body } = await doRequest(playlistUrl) if (!body) return [] diff --git a/server/lib/job-queue/handlers/activitypub-cleaner.ts b/server/lib/job-queue/handlers/activitypub-cleaner.ts index 0e75b0a6e..9dcc778fa 100644 --- a/server/lib/job-queue/handlers/activitypub-cleaner.ts +++ b/server/lib/job-queue/handlers/activitypub-cleaner.ts @@ -7,7 +7,7 @@ import { isLikeActivityValid } from '@server/helpers/custom-validators/activitypub/activity' import { sanitizeAndCheckVideoCommentObject } from '@server/helpers/custom-validators/activitypub/video-comments' -import { doRequest } from '@server/helpers/requests' +import { doJSONRequest } from '@server/helpers/requests' import { AP_CLEANER_CONCURRENCY } from '@server/initializers/constants' import { VideoModel } from '@server/models/video/video' import { VideoCommentModel } from '@server/models/video/video-comment' @@ -81,15 +81,10 @@ async function updateObjectIfNeeded ( updater: (url: string, newUrl: string) => Promise, deleter: (url: string) => Promise ): Promise<{ data: T, status: 'deleted' | 'updated' } | null> { - // Fetch url - const { response, body } = await doRequest({ - uri: url, - json: true, - activityPub: true - }) + const { statusCode, body } = await doJSONRequest(url, { activityPub: true }) // Does not exist anymore, remove entry - if (response.statusCode === HttpStatusCode.NOT_FOUND_404) { + if (statusCode === HttpStatusCode.NOT_FOUND_404) { logger.info('Removing remote AP object %s.', url) const data = await deleter(url) diff --git a/server/lib/job-queue/handlers/activitypub-http-broadcast.ts b/server/lib/job-queue/handlers/activitypub-http-broadcast.ts index 7174786d6..c69ff9e83 100644 --- a/server/lib/job-queue/handlers/activitypub-http-broadcast.ts +++ b/server/lib/job-queue/handlers/activitypub-http-broadcast.ts @@ -16,8 +16,7 @@ async function processActivityPubHttpBroadcast (job: Bull.Job) { const httpSignatureOptions = await buildSignedRequestOptions(payload) const options = { - method: 'POST', - uri: '', + method: 'POST' as 'POST', json: body, httpSignature: httpSignatureOptions, timeout: REQUEST_TIMEOUT, @@ -28,7 +27,7 @@ async function processActivityPubHttpBroadcast (job: Bull.Job) { const goodUrls: string[] = [] await Bluebird.map(payload.uris, uri => { - return doRequest(Object.assign({}, options, { uri })) + return doRequest(uri, options) .then(() => goodUrls.push(uri)) .catch(() => badUrls.push(uri)) }, { concurrency: BROADCAST_CONCURRENCY }) diff --git a/server/lib/job-queue/handlers/activitypub-http-unicast.ts b/server/lib/job-queue/handlers/activitypub-http-unicast.ts index 74989d62e..585dad671 100644 --- a/server/lib/job-queue/handlers/activitypub-http-unicast.ts +++ b/server/lib/job-queue/handlers/activitypub-http-unicast.ts @@ -16,8 +16,7 @@ async function processActivityPubHttpUnicast (job: Bull.Job) { const httpSignatureOptions = await buildSignedRequestOptions(payload) const options = { - method: 'POST', - uri, + method: 'POST' as 'POST', json: body, httpSignature: httpSignatureOptions, timeout: REQUEST_TIMEOUT, @@ -25,7 +24,7 @@ async function processActivityPubHttpUnicast (job: Bull.Job) { } try { - await doRequest(options) + await doRequest(uri, options) ActorFollowScoreCache.Instance.updateActorFollowsScore([ uri ], []) } catch (err) { ActorFollowScoreCache.Instance.updateActorFollowsScore([], [ uri ]) diff --git a/server/lib/job-queue/handlers/utils/activitypub-http-utils.ts b/server/lib/job-queue/handlers/utils/activitypub-http-utils.ts index c030d31ef..4116a9c0e 100644 --- a/server/lib/job-queue/handlers/utils/activitypub-http-utils.ts +++ b/server/lib/job-queue/handlers/utils/activitypub-http-utils.ts @@ -6,21 +6,24 @@ import { getServerActor } from '@server/models/application/application' import { buildDigest } from '@server/helpers/peertube-crypto' import { ContextType } from '@shared/models/activitypub/context' -type Payload = { body: any, contextType?: ContextType, signatureActorId?: number } +type Payload = { body: T, contextType?: ContextType, signatureActorId?: number } -async function computeBody (payload: Payload) { +async function computeBody ( + payload: Payload +): Promise { let body = payload.body if (payload.signatureActorId) { const actorSignature = await ActorModel.load(payload.signatureActorId) if (!actorSignature) throw new Error('Unknown signature actor id.') + body = await buildSignedActivity(actorSignature, payload.body, payload.contextType) } return body } -async function buildSignedRequestOptions (payload: Payload) { +async function buildSignedRequestOptions (payload: Payload) { let actor: MActor | null if (payload.signatureActorId) { diff --git a/server/lib/plugins/plugin-index.ts b/server/lib/plugins/plugin-index.ts index 7bcb6ed4c..624f5da1d 100644 --- a/server/lib/plugins/plugin-index.ts +++ b/server/lib/plugins/plugin-index.ts @@ -1,22 +1,22 @@ -import { doRequest } from '../../helpers/requests' -import { CONFIG } from '../../initializers/config' +import { sanitizeUrl } from '@server/helpers/core-utils' +import { ResultList } from '../../../shared/models' +import { PeertubePluginIndexList } from '../../../shared/models/plugins/peertube-plugin-index-list.model' +import { PeerTubePluginIndex } from '../../../shared/models/plugins/peertube-plugin-index.model' import { PeertubePluginLatestVersionRequest, PeertubePluginLatestVersionResponse } from '../../../shared/models/plugins/peertube-plugin-latest-version.model' -import { PeertubePluginIndexList } from '../../../shared/models/plugins/peertube-plugin-index-list.model' -import { ResultList } from '../../../shared/models' -import { PeerTubePluginIndex } from '../../../shared/models/plugins/peertube-plugin-index.model' -import { PluginModel } from '../../models/server/plugin' -import { PluginManager } from './plugin-manager' import { logger } from '../../helpers/logger' +import { doJSONRequest } from '../../helpers/requests' +import { CONFIG } from '../../initializers/config' import { PEERTUBE_VERSION } from '../../initializers/constants' -import { sanitizeUrl } from '@server/helpers/core-utils' +import { PluginModel } from '../../models/server/plugin' +import { PluginManager } from './plugin-manager' async function listAvailablePluginsFromIndex (options: PeertubePluginIndexList) { const { start = 0, count = 20, search, sort = 'npmName', pluginType } = options - const qs: PeertubePluginIndexList = { + const searchParams: PeertubePluginIndexList & Record = { start, count, sort, @@ -28,7 +28,7 @@ async function listAvailablePluginsFromIndex (options: PeertubePluginIndexList) const uri = CONFIG.PLUGINS.INDEX.URL + '/api/v1/plugins' try { - const { body } = await doRequest({ uri, qs, json: true }) + const { body } = await doJSONRequest(uri, { searchParams }) logger.debug('Got result from PeerTube index.', { body }) @@ -58,7 +58,11 @@ async function getLatestPluginsVersion (npmNames: string[]): Promise({ uri, body: bodyRequest, json: true, method: 'POST' }) + const options = { + json: bodyRequest, + method: 'POST' as 'POST' + } + const { body } = await doJSONRequest(uri, options) return body } diff --git a/server/lib/schedulers/auto-follow-index-instances.ts b/server/lib/schedulers/auto-follow-index-instances.ts index f62f52f9c..0b8cd1389 100644 --- a/server/lib/schedulers/auto-follow-index-instances.ts +++ b/server/lib/schedulers/auto-follow-index-instances.ts @@ -1,5 +1,5 @@ import { chunk } from 'lodash' -import { doRequest } from '@server/helpers/requests' +import { doJSONRequest } from '@server/helpers/requests' import { JobQueue } from '@server/lib/job-queue' import { ActorFollowModel } from '@server/models/activitypub/actor-follow' import { getServerActor } from '@server/models/application/application' @@ -34,12 +34,12 @@ export class AutoFollowIndexInstances extends AbstractScheduler { try { const serverActor = await getServerActor() - const qs = { count: 1000 } - if (this.lastCheck) Object.assign(qs, { since: this.lastCheck.toISOString() }) + const searchParams = { count: 1000 } + if (this.lastCheck) Object.assign(searchParams, { since: this.lastCheck.toISOString() }) this.lastCheck = new Date() - const { body } = await doRequest({ uri: indexUrl, qs, json: true }) + const { body } = await doJSONRequest(indexUrl, { searchParams }) if (!body.data || Array.isArray(body.data) === false) { logger.error('Cannot auto follow instances of index %s. Please check the auto follow URL.', indexUrl, { body }) return diff --git a/server/tests/api/activitypub/security.ts b/server/tests/api/activitypub/security.ts index 8bde54a40..26b4545ac 100644 --- a/server/tests/api/activitypub/security.ts +++ b/server/tests/api/activitypub/security.ts @@ -79,9 +79,9 @@ describe('Test ActivityPub security', function () { Digest: buildDigest({ hello: 'coucou' }) } - const { response } = await makePOSTAPRequest(url, body, baseHttpSignature(), headers) + const { statusCode } = await makePOSTAPRequest(url, body, baseHttpSignature(), headers) - expect(response.statusCode).to.equal(HttpStatusCode.FORBIDDEN_403) + expect(statusCode).to.equal(HttpStatusCode.FORBIDDEN_403) }) it('Should fail with an invalid date', async function () { @@ -89,9 +89,9 @@ describe('Test ActivityPub security', function () { const headers = buildGlobalHeaders(body) headers['date'] = 'Wed, 21 Oct 2015 07:28:00 GMT' - const { response } = await makePOSTAPRequest(url, body, baseHttpSignature(), headers) + const { statusCode } = await makePOSTAPRequest(url, body, baseHttpSignature(), headers) - expect(response.statusCode).to.equal(HttpStatusCode.FORBIDDEN_403) + expect(statusCode).to.equal(HttpStatusCode.FORBIDDEN_403) }) it('Should fail with bad keys', async function () { @@ -101,9 +101,9 @@ describe('Test ActivityPub security', function () { const body = activityPubContextify(getAnnounceWithoutContext(servers[1])) const headers = buildGlobalHeaders(body) - const { response } = await makePOSTAPRequest(url, body, baseHttpSignature(), headers) + const { statusCode } = await makePOSTAPRequest(url, body, baseHttpSignature(), headers) - expect(response.statusCode).to.equal(HttpStatusCode.FORBIDDEN_403) + expect(statusCode).to.equal(HttpStatusCode.FORBIDDEN_403) }) it('Should reject requests without appropriate signed headers', async function () { @@ -123,8 +123,8 @@ describe('Test ActivityPub security', function () { for (const badHeaders of badHeadersMatrix) { signatureOptions.headers = badHeaders - const { response } = await makePOSTAPRequest(url, body, signatureOptions, headers) - expect(response.statusCode).to.equal(HttpStatusCode.FORBIDDEN_403) + const { statusCode } = await makePOSTAPRequest(url, body, signatureOptions, headers) + expect(statusCode).to.equal(HttpStatusCode.FORBIDDEN_403) } }) @@ -132,9 +132,9 @@ describe('Test ActivityPub security', function () { const body = activityPubContextify(getAnnounceWithoutContext(servers[1])) const headers = buildGlobalHeaders(body) - const { response } = await makePOSTAPRequest(url, body, baseHttpSignature(), headers) + const { statusCode } = await makePOSTAPRequest(url, body, baseHttpSignature(), headers) - expect(response.statusCode).to.equal(HttpStatusCode.NO_CONTENT_204) + expect(statusCode).to.equal(HttpStatusCode.NO_CONTENT_204) }) it('Should refresh the actor keys', async function () { @@ -150,9 +150,9 @@ describe('Test ActivityPub security', function () { const body = activityPubContextify(getAnnounceWithoutContext(servers[1])) const headers = buildGlobalHeaders(body) - const { response } = await makePOSTAPRequest(url, body, baseHttpSignature(), headers) + const { statusCode } = await makePOSTAPRequest(url, body, baseHttpSignature(), headers) - expect(response.statusCode).to.equal(HttpStatusCode.FORBIDDEN_403) + expect(statusCode).to.equal(HttpStatusCode.FORBIDDEN_403) }) }) @@ -183,9 +183,9 @@ describe('Test ActivityPub security', function () { const headers = buildGlobalHeaders(signedBody) - const { response } = await makePOSTAPRequest(url, signedBody, baseHttpSignature(), headers) + const { statusCode } = await makePOSTAPRequest(url, signedBody, baseHttpSignature(), headers) - expect(response.statusCode).to.equal(HttpStatusCode.FORBIDDEN_403) + expect(statusCode).to.equal(HttpStatusCode.FORBIDDEN_403) }) it('Should fail with an altered body', async function () { @@ -204,9 +204,9 @@ describe('Test ActivityPub security', function () { const headers = buildGlobalHeaders(signedBody) - const { response } = await makePOSTAPRequest(url, signedBody, baseHttpSignature(), headers) + const { statusCode } = await makePOSTAPRequest(url, signedBody, baseHttpSignature(), headers) - expect(response.statusCode).to.equal(HttpStatusCode.FORBIDDEN_403) + expect(statusCode).to.equal(HttpStatusCode.FORBIDDEN_403) }) it('Should succeed with a valid signature', async function () { @@ -220,9 +220,9 @@ describe('Test ActivityPub security', function () { const headers = buildGlobalHeaders(signedBody) - const { response } = await makePOSTAPRequest(url, signedBody, baseHttpSignature(), headers) + const { statusCode } = await makePOSTAPRequest(url, signedBody, baseHttpSignature(), headers) - expect(response.statusCode).to.equal(HttpStatusCode.NO_CONTENT_204) + expect(statusCode).to.equal(HttpStatusCode.NO_CONTENT_204) }) it('Should refresh the actor keys', async function () { @@ -243,9 +243,9 @@ describe('Test ActivityPub security', function () { const headers = buildGlobalHeaders(signedBody) - const { response } = await makePOSTAPRequest(url, signedBody, baseHttpSignature(), headers) + const { statusCode } = await makePOSTAPRequest(url, signedBody, baseHttpSignature(), headers) - expect(response.statusCode).to.equal(HttpStatusCode.FORBIDDEN_403) + expect(statusCode).to.equal(HttpStatusCode.FORBIDDEN_403) }) }) diff --git a/server/tests/api/server/handle-down.ts b/server/tests/api/server/handle-down.ts index 043754e70..f3ba11950 100644 --- a/server/tests/api/server/handle-down.ts +++ b/server/tests/api/server/handle-down.ts @@ -348,8 +348,8 @@ describe('Test handle downs', function () { for (let i = 0; i < 3; i++) { await getVideo(servers[1].url, videoIdsServer1[i]) - await wait(1000) await waitJobs([ servers[1] ]) + await wait(1500) } for (const id of videoIdsServer1) { diff --git a/server/tests/helpers/request.ts b/server/tests/helpers/request.ts index f8b2d599b..5e77f129e 100644 --- a/server/tests/helpers/request.ts +++ b/server/tests/helpers/request.ts @@ -1,11 +1,11 @@ /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ import 'mocha' -import { doRequest, doRequestAndSaveToFile } from '../../helpers/requests' -import { get4KFileUrl, root, wait } from '../../../shared/extra-utils' -import { join } from 'path' -import { pathExists, remove } from 'fs-extra' import { expect } from 'chai' +import { pathExists, remove } from 'fs-extra' +import { join } from 'path' +import { get4KFileUrl, root, wait } from '../../../shared/extra-utils' +import { doRequest, doRequestAndSaveToFile } from '../../helpers/requests' describe('Request helpers', function () { const destPath1 = join(root(), 'test-output-1.txt') @@ -13,7 +13,7 @@ describe('Request helpers', function () { it('Should throw an error when the bytes limit is exceeded for request', async function () { try { - await doRequest({ uri: get4KFileUrl() }, 3) + await doRequest(get4KFileUrl(), { bodyKBLimit: 3 }) } catch { return } @@ -23,7 +23,7 @@ describe('Request helpers', function () { it('Should throw an error when the bytes limit is exceeded for request and save file', async function () { try { - await doRequestAndSaveToFile({ uri: get4KFileUrl() }, destPath1, 3) + await doRequestAndSaveToFile(get4KFileUrl(), destPath1, { bodyKBLimit: 3 }) } catch { await wait(500) @@ -35,8 +35,8 @@ describe('Request helpers', function () { }) it('Should succeed if the file is below the limit', async function () { - await doRequest({ uri: get4KFileUrl() }, 5) - await doRequestAndSaveToFile({ uri: get4KFileUrl() }, destPath2, 5) + await doRequest(get4KFileUrl(), { bodyKBLimit: 5 }) + await doRequestAndSaveToFile(get4KFileUrl(), destPath2, { bodyKBLimit: 5 }) expect(await pathExists(destPath2)).to.be.true }) diff --git a/shared/extra-utils/requests/activitypub.ts b/shared/extra-utils/requests/activitypub.ts index 4762a8665..2a7f20289 100644 --- a/shared/extra-utils/requests/activitypub.ts +++ b/shared/extra-utils/requests/activitypub.ts @@ -5,14 +5,13 @@ import { activityPubContextify } from '../../../server/helpers/activitypub' function makePOSTAPRequest (url: string, body: any, httpSignature: any, headers: any) { const options = { - method: 'POST', - uri: url, + method: 'POST' as 'POST', json: body, httpSignature, headers } - return doRequest(options) + return doRequest(url, options) } async function makeFollowRequest (to: { url: string }, by: { url: string, privateKey }) { diff --git a/shared/models/server/job.model.ts b/shared/models/server/job.model.ts index 83ef84457..e4acfee8d 100644 --- a/shared/models/server/job.model.ts +++ b/shared/models/server/job.model.ts @@ -59,7 +59,7 @@ export type ActivitypubHttpFetcherPayload = { export type ActivitypubHttpUnicastPayload = { uri: string signatureActorId?: number - body: any + body: object contextType?: ContextType } diff --git a/yarn.lock b/yarn.lock index b2d5a594c..5546830be 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3925,6 +3925,23 @@ globby@^11.0.1: merge2 "^1.3.0" slash "^3.0.0" +got@^11.8.2, got@~11.8.1: + version "11.8.2" + resolved "https://registry.yarnpkg.com/got/-/got-11.8.2.tgz#7abb3959ea28c31f3576f1576c1effce23f33599" + integrity sha512-D0QywKgIe30ODs+fm8wMZiAcZjypcCodPNuMz5H9Mny7RJ+IjJ10BdmGW7OM7fHXP+O7r6ZwapQ/YQmMSvB0UQ== + dependencies: + "@sindresorhus/is" "^4.0.0" + "@szmarczak/http-timer" "^4.0.5" + "@types/cacheable-request" "^6.0.1" + "@types/responselike" "^1.0.0" + cacheable-lookup "^5.0.3" + cacheable-request "^7.0.1" + decompress-response "^6.0.0" + http2-wrapper "^1.0.0-beta.5.2" + lowercase-keys "^2.0.0" + p-cancelable "^2.0.0" + responselike "^2.0.0" + got@^9.6.0: version "9.6.0" resolved "https://registry.yarnpkg.com/got/-/got-9.6.0.tgz#edf45e7d67f99545705de1f7bbeeeb121765ed85" @@ -3942,23 +3959,6 @@ got@^9.6.0: to-readable-stream "^1.0.0" url-parse-lax "^3.0.0" -got@~11.8.1: - version "11.8.2" - resolved "https://registry.yarnpkg.com/got/-/got-11.8.2.tgz#7abb3959ea28c31f3576f1576c1effce23f33599" - integrity sha512-D0QywKgIe30ODs+fm8wMZiAcZjypcCodPNuMz5H9Mny7RJ+IjJ10BdmGW7OM7fHXP+O7r6ZwapQ/YQmMSvB0UQ== - dependencies: - "@sindresorhus/is" "^4.0.0" - "@szmarczak/http-timer" "^4.0.5" - "@types/cacheable-request" "^6.0.1" - "@types/responselike" "^1.0.0" - cacheable-lookup "^5.0.3" - cacheable-request "^7.0.1" - decompress-response "^6.0.0" - http2-wrapper "^1.0.0-beta.5.2" - lowercase-keys "^2.0.0" - p-cancelable "^2.0.0" - responselike "^2.0.0" - graceful-fs@^4.1.2, graceful-fs@^4.1.6, graceful-fs@^4.2.0: version "4.2.6" resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.6.tgz#ff040b2b0853b23c3d31027523706f1885d76bee" @@ -4167,7 +4167,7 @@ http-proxy-agent@^4.0.1: agent-base "6" debug "4" -http-signature@1.3.5, http-signature@~1.2.0: +http-signature@1.3.5: version "1.3.5" resolved "https://registry.yarnpkg.com/http-signature/-/http-signature-1.3.5.tgz#9f19496ffbf3227298d7b5f156e0e1a948678683" integrity sha512-NwoTQYSJoFt34jSBbwzDHDofoA61NGXzu6wXh95o1Ry62EnmKjXb/nR/RknLeZ3G/uGwrlKNY2z7uPt+Cdl7Tw== @@ -4176,6 +4176,15 @@ http-signature@1.3.5, http-signature@~1.2.0: jsprim "^1.2.2" sshpk "^1.14.1" +http-signature@~1.2.0: + version "1.2.0" + resolved "https://registry.yarnpkg.com/http-signature/-/http-signature-1.2.0.tgz#9aecd925114772f3d95b65a60abb8f7c18fbace1" + integrity sha1-muzZJRFHcvPZW2WmCruPfBj7rOE= + dependencies: + assert-plus "^1.0.0" + jsprim "^1.2.2" + sshpk "^1.7.0" + http2-wrapper@^1.0.0-beta.5.2: version "1.0.3" resolved "https://registry.yarnpkg.com/http2-wrapper/-/http2-wrapper-1.0.3.tgz#b8f55e0c1f25d4ebd08b3b0c2c079f9590800b3d" @@ -7060,7 +7069,7 @@ render-media@^4.1.0: stream-to-blob-url "^3.0.2" videostream "^3.2.2" -request@^2.81.0, request@^2.88.0: +request@^2.88.0: version "2.88.2" resolved "https://registry.yarnpkg.com/request/-/request-2.88.2.tgz#d73c918731cb5a87da047e207234146f664d12b3" integrity sha512-MsvtOrfG9ZcrOwAW+Qi+F6HbD0CWXEh9ou77uOb7FM2WPhwT7smM833PzanhJLsgXjN89Ir6V2PczXNnMpwKhw== @@ -7700,7 +7709,7 @@ srt-to-vtt@^1.1.2: through2 "^0.6.3" to-utf-8 "^1.2.0" -sshpk@^1.14.1: +sshpk@^1.14.1, sshpk@^1.7.0: version "1.16.1" resolved "https://registry.yarnpkg.com/sshpk/-/sshpk-1.16.1.tgz#fb661c0bef29b39db40769ee39fa70093d6f6877" integrity sha512-HXXqVUq7+pcKeLqqZj6mHFUMvXtOJt1uoUx09pFW6011inTMxqI8BA8PM95myrIyyKwdnzjdFjLiE6KBPVtJIg== -- 2.41.0