From db4b15f21fbf4e33434e930ffc7fb768cdcf9d42 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Mon, 8 Mar 2021 14:24:11 +0100 Subject: Use got instead of request --- server/lib/activitypub/actor.ts | 41 +++++++++--------------- server/lib/activitypub/crawl.ts | 25 +++++++-------- server/lib/activitypub/playlist.ts | 54 +++++++++++--------------------- server/lib/activitypub/share.ts | 9 ++---- server/lib/activitypub/video-comments.ts | 14 +++------ server/lib/activitypub/video-rates.ts | 22 ++++++------- server/lib/activitypub/videos.ts | 31 ++++++------------ 7 files changed, 71 insertions(+), 125 deletions(-) (limited to 'server/lib/activitypub') diff --git a/server/lib/activitypub/actor.ts b/server/lib/activitypub/actor.ts index a726f9e20..52b6c1f56 100644 --- a/server/lib/activitypub/actor.ts +++ b/server/lib/activitypub/actor.ts @@ -1,26 +1,28 @@ import * as Bluebird from 'bluebird' +import { extname } from 'path' import { Op, Transaction } from 'sequelize' import { URL } from 'url' import { v4 as uuidv4 } from 'uuid' +import { getServerActor } from '@server/models/application/application' +import { HttpStatusCode } from '../../../shared/core-utils/miscs/http-error-codes' import { ActivityPubActor, ActivityPubActorType, ActivityPubOrderedCollection } from '../../../shared/models/activitypub' import { ActivityPubAttributedTo } from '../../../shared/models/activitypub/objects' import { checkUrlsSameHost, getAPId } from '../../helpers/activitypub' +import { ActorFetchByUrlType, fetchActorByUrl } from '../../helpers/actor' import { sanitizeAndCheckActorObject } from '../../helpers/custom-validators/activitypub/actor' import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc' import { retryTransactionWrapper, updateInstanceWithAnother } from '../../helpers/database-utils' import { logger } from '../../helpers/logger' import { createPrivateAndPublicKeys } from '../../helpers/peertube-crypto' -import { doRequest } from '../../helpers/requests' +import { doJSONRequest } from '../../helpers/requests' import { getUrlFromWebfinger } from '../../helpers/webfinger' import { MIMETYPES, WEBSERVER } from '../../initializers/constants' +import { sequelizeTypescript } from '../../initializers/database' import { AccountModel } from '../../models/account/account' import { ActorModel } from '../../models/activitypub/actor' import { AvatarModel } from '../../models/avatar/avatar' import { ServerModel } from '../../models/server/server' import { VideoChannelModel } from '../../models/video/video-channel' -import { JobQueue } from '../job-queue' -import { ActorFetchByUrlType, fetchActorByUrl } from '../../helpers/actor' -import { sequelizeTypescript } from '../../initializers/database' import { MAccount, MAccountDefault, @@ -34,9 +36,7 @@ import { MActorId, MChannel } from '../../types/models' -import { extname } from 'path' -import { getServerActor } from '@server/models/application/application' -import { HttpStatusCode } from '../../../shared/core-utils/miscs/http-error-codes' +import { JobQueue } from '../job-queue' // Set account keys, this could be long so process after the account creation and do not block the client async function generateAndSaveActorKeys (actor: T) { @@ -209,16 +209,10 @@ async function deleteActorAvatarInstance (actor: MActorDefault, t: Transaction) } async function fetchActorTotalItems (url: string) { - const options = { - uri: url, - method: 'GET', - json: true, - activityPub: true - } - try { - const { body } = await doRequest>(options) - return body.totalItems ? body.totalItems : 0 + const { body } = await doJSONRequest>(url, { activityPub: true }) + + return body.totalItems || 0 } catch (err) { logger.warn('Cannot fetch remote actor count %s.', url, { err }) return 0 @@ -449,26 +443,19 @@ type FetchRemoteActorResult = { attributedTo: ActivityPubAttributedTo[] } async function fetchRemoteActor (actorUrl: string): Promise<{ statusCode?: number, result: FetchRemoteActorResult }> { - const options = { - uri: actorUrl, - method: 'GET', - json: true, - activityPub: true - } - logger.info('Fetching remote actor %s.', actorUrl) - const requestResult = await doRequest(options) + const requestResult = await doJSONRequest(actorUrl, { activityPub: true }) const actorJSON = requestResult.body if (sanitizeAndCheckActorObject(actorJSON) === false) { logger.debug('Remote actor JSON is not valid.', { actorJSON }) - return { result: undefined, statusCode: requestResult.response.statusCode } + return { result: undefined, statusCode: requestResult.statusCode } } if (checkUrlsSameHost(actorJSON.id, actorUrl) !== true) { logger.warn('Actor url %s has not the same host than its AP id %s', actorUrl, actorJSON.id) - return { result: undefined, statusCode: requestResult.response.statusCode } + return { result: undefined, statusCode: requestResult.statusCode } } const followersCount = await fetchActorTotalItems(actorJSON.followers) @@ -496,7 +483,7 @@ async function fetchRemoteActor (actorUrl: string): Promise<{ statusCode?: numbe const name = actorJSON.name || actorJSON.preferredUsername return { - statusCode: requestResult.response.statusCode, + statusCode: requestResult.statusCode, result: { actor, name, diff --git a/server/lib/activitypub/crawl.ts b/server/lib/activitypub/crawl.ts index 1ed105bbe..278abf7de 100644 --- a/server/lib/activitypub/crawl.ts +++ b/server/lib/activitypub/crawl.ts @@ -1,27 +1,26 @@ -import { ACTIVITY_PUB, REQUEST_TIMEOUT, WEBSERVER } from '../../initializers/constants' -import { doRequest } from '../../helpers/requests' -import { logger } from '../../helpers/logger' import * as Bluebird from 'bluebird' -import { ActivityPubOrderedCollection } from '../../../shared/models/activitypub' import { URL } from 'url' +import { ActivityPubOrderedCollection } from '../../../shared/models/activitypub' +import { logger } from '../../helpers/logger' +import { doJSONRequest } from '../../helpers/requests' +import { ACTIVITY_PUB, REQUEST_TIMEOUT, WEBSERVER } from '../../initializers/constants' type HandlerFunction = (items: T[]) => (Promise | Bluebird) type CleanerFunction = (startedDate: Date) => (Promise | Bluebird) -async function crawlCollectionPage (uri: string, handler: HandlerFunction, cleaner?: CleanerFunction) { - logger.info('Crawling ActivityPub data on %s.', uri) +async function crawlCollectionPage (argUrl: string, handler: HandlerFunction, cleaner?: CleanerFunction) { + let url = argUrl + + logger.info('Crawling ActivityPub data on %s.', url) const options = { - method: 'GET', - uri, - json: true, activityPub: true, timeout: REQUEST_TIMEOUT } const startDate = new Date() - const response = await doRequest>(options) + const response = await doJSONRequest>(url, options) const firstBody = response.body const limit = ACTIVITY_PUB.FETCH_PAGE_LIMIT @@ -35,9 +34,9 @@ async function crawlCollectionPage (uri: string, handler: HandlerFunction const remoteHost = new URL(nextLink).host if (remoteHost === WEBSERVER.HOST) continue - options.uri = nextLink + url = nextLink - const res = await doRequest>(options) + const res = await doJSONRequest>(url, options) body = res.body } else { // nextLink is already the object we want @@ -49,7 +48,7 @@ async function crawlCollectionPage (uri: string, handler: HandlerFunction if (Array.isArray(body.orderedItems)) { const items = body.orderedItems - logger.info('Processing %i ActivityPub items for %s.', items.length, options.uri) + logger.info('Processing %i ActivityPub items for %s.', items.length, url) await handler(items) } diff --git a/server/lib/activitypub/playlist.ts b/server/lib/activitypub/playlist.ts index d5a3ef7c8..795be60d7 100644 --- a/server/lib/activitypub/playlist.ts +++ b/server/lib/activitypub/playlist.ts @@ -1,24 +1,24 @@ +import * as Bluebird from 'bluebird' +import { HttpStatusCode } from '../../../shared/core-utils/miscs/http-error-codes' +import { PlaylistElementObject } from '../../../shared/models/activitypub/objects/playlist-element-object' import { PlaylistObject } from '../../../shared/models/activitypub/objects/playlist-object' -import { crawlCollectionPage } from './crawl' -import { ACTIVITY_PUB, CRAWL_REQUEST_CONCURRENCY } from '../../initializers/constants' +import { VideoPlaylistPrivacy } from '../../../shared/models/videos/playlist/video-playlist-privacy.model' +import { checkUrlsSameHost } from '../../helpers/activitypub' +import { isPlaylistElementObjectValid, isPlaylistObjectValid } from '../../helpers/custom-validators/activitypub/playlist' import { isArray } from '../../helpers/custom-validators/misc' -import { getOrCreateActorAndServerAndModel } from './actor' import { logger } from '../../helpers/logger' +import { doJSONRequest } from '../../helpers/requests' +import { ACTIVITY_PUB, CRAWL_REQUEST_CONCURRENCY } from '../../initializers/constants' +import { sequelizeTypescript } from '../../initializers/database' import { VideoPlaylistModel } from '../../models/video/video-playlist' -import { doRequest } from '../../helpers/requests' -import { checkUrlsSameHost } from '../../helpers/activitypub' -import * as Bluebird from 'bluebird' -import { PlaylistElementObject } from '../../../shared/models/activitypub/objects/playlist-element-object' -import { getOrCreateVideoAndAccountAndChannel } from './videos' -import { isPlaylistElementObjectValid, isPlaylistObjectValid } from '../../helpers/custom-validators/activitypub/playlist' import { VideoPlaylistElementModel } from '../../models/video/video-playlist-element' -import { VideoPlaylistPrivacy } from '../../../shared/models/videos/playlist/video-playlist-privacy.model' -import { sequelizeTypescript } from '../../initializers/database' -import { createPlaylistMiniatureFromUrl } from '../thumbnail' -import { FilteredModelAttributes } from '../../types/sequelize' import { MAccountDefault, MAccountId, MVideoId } from '../../types/models' import { MVideoPlaylist, MVideoPlaylistId, MVideoPlaylistOwner } from '../../types/models/video/video-playlist' -import { HttpStatusCode } from '../../../shared/core-utils/miscs/http-error-codes' +import { FilteredModelAttributes } from '../../types/sequelize' +import { createPlaylistMiniatureFromUrl } from '../thumbnail' +import { getOrCreateActorAndServerAndModel } from './actor' +import { crawlCollectionPage } from './crawl' +import { getOrCreateVideoAndAccountAndChannel } from './videos' function playlistObjectToDBAttributes (playlistObject: PlaylistObject, byAccount: MAccountId, to: string[]) { const privacy = to.includes(ACTIVITY_PUB.PUBLIC) @@ -56,11 +56,7 @@ async function createAccountPlaylists (playlistUrls: string[], account: MAccount if (exists === true) return // Fetch url - const { body } = await doRequest({ - uri: playlistUrl, - json: true, - activityPub: true - }) + const { body } = await doJSONRequest(playlistUrl, { activityPub: true }) if (!isPlaylistObjectValid(body)) { throw new Error(`Invalid playlist object when fetch account playlists: ${JSON.stringify(body)}`) @@ -164,12 +160,7 @@ async function resetVideoPlaylistElements (elementUrls: string[], playlist: MVid await Bluebird.map(elementUrls, async elementUrl => { try { - // Fetch url - const { body } = await doRequest({ - uri: elementUrl, - json: true, - activityPub: true - }) + const { body } = await doJSONRequest(elementUrl, { activityPub: true }) if (!isPlaylistElementObjectValid(body)) throw new Error(`Invalid body in video get playlist element ${elementUrl}`) @@ -199,21 +190,14 @@ async function resetVideoPlaylistElements (elementUrls: string[], playlist: MVid } async function fetchRemoteVideoPlaylist (playlistUrl: string): Promise<{ statusCode: number, playlistObject: PlaylistObject }> { - const options = { - uri: playlistUrl, - method: 'GET', - json: true, - activityPub: true - } - logger.info('Fetching remote playlist %s.', playlistUrl) - const { response, body } = await doRequest(options) + const { body, statusCode } = await doJSONRequest(playlistUrl, { activityPub: true }) if (isPlaylistObjectValid(body) === false || checkUrlsSameHost(body.id, playlistUrl) !== true) { logger.debug('Remote video playlist JSON is not valid.', { body }) - return { statusCode: response.statusCode, playlistObject: undefined } + return { statusCode, playlistObject: undefined } } - return { statusCode: response.statusCode, playlistObject: body } + return { statusCode, playlistObject: body } } diff --git a/server/lib/activitypub/share.ts b/server/lib/activitypub/share.ts index dde0c628e..c22fa0893 100644 --- a/server/lib/activitypub/share.ts +++ b/server/lib/activitypub/share.ts @@ -3,7 +3,7 @@ import { Transaction } from 'sequelize' import { getServerActor } from '@server/models/application/application' import { checkUrlsSameHost, getAPId } from '../../helpers/activitypub' import { logger, loggerTagsFactory } from '../../helpers/logger' -import { doRequest } from '../../helpers/requests' +import { doJSONRequest } from '../../helpers/requests' import { CRAWL_REQUEST_CONCURRENCY } from '../../initializers/constants' import { VideoShareModel } from '../../models/video/video-share' import { MChannelActorLight, MVideo, MVideoAccountLight, MVideoId } from '../../types/models/video' @@ -40,12 +40,7 @@ async function changeVideoChannelShare ( async function addVideoShares (shareUrls: string[], video: MVideoId) { await Bluebird.map(shareUrls, async shareUrl => { try { - // Fetch url - const { body } = await doRequest({ - uri: shareUrl, - json: true, - activityPub: true - }) + const { body } = await doJSONRequest(shareUrl, { activityPub: true }) if (!body || !body.actor) throw new Error('Body or body actor is invalid') const actorUrl = getAPId(body.actor) diff --git a/server/lib/activitypub/video-comments.ts b/server/lib/activitypub/video-comments.ts index d025ed7f1..f1edfb0ac 100644 --- a/server/lib/activitypub/video-comments.ts +++ b/server/lib/activitypub/video-comments.ts @@ -1,13 +1,13 @@ +import * as Bluebird from 'bluebird' +import { checkUrlsSameHost } from '../../helpers/activitypub' import { sanitizeAndCheckVideoCommentObject } from '../../helpers/custom-validators/activitypub/video-comments' import { logger } from '../../helpers/logger' -import { doRequest } from '../../helpers/requests' +import { doJSONRequest } from '../../helpers/requests' import { ACTIVITY_PUB, CRAWL_REQUEST_CONCURRENCY } from '../../initializers/constants' import { VideoCommentModel } from '../../models/video/video-comment' +import { MCommentOwner, MCommentOwnerVideo, MVideoAccountLightBlacklistAllFiles } from '../../types/models/video' import { getOrCreateActorAndServerAndModel } from './actor' import { getOrCreateVideoAndAccountAndChannel } from './videos' -import * as Bluebird from 'bluebird' -import { checkUrlsSameHost } from '../../helpers/activitypub' -import { MCommentOwner, MCommentOwnerVideo, MVideoAccountLightBlacklistAllFiles } from '../../types/models/video' type ResolveThreadParams = { url: string @@ -126,11 +126,7 @@ async function resolveRemoteParentComment (params: ResolveThreadParams) { throw new Error('Recursion limit reached when resolving a thread') } - const { body } = await doRequest({ - uri: url, - json: true, - activityPub: true - }) + const { body } = await doJSONRequest(url, { activityPub: true }) if (sanitizeAndCheckVideoCommentObject(body) === false) { throw new Error(`Remote video comment JSON ${url} is not valid:` + JSON.stringify(body)) diff --git a/server/lib/activitypub/video-rates.ts b/server/lib/activitypub/video-rates.ts index e246b1313..f40c07fea 100644 --- a/server/lib/activitypub/video-rates.ts +++ b/server/lib/activitypub/video-rates.ts @@ -1,26 +1,22 @@ +import * as Bluebird from 'bluebird' import { Transaction } from 'sequelize' -import { sendLike, sendUndoDislike, sendUndoLike } from './send' +import { doJSONRequest } from '@server/helpers/requests' import { VideoRateType } from '../../../shared/models/videos' -import * as Bluebird from 'bluebird' -import { getOrCreateActorAndServerAndModel } from './actor' -import { AccountVideoRateModel } from '../../models/account/account-video-rate' +import { checkUrlsSameHost, getAPId } from '../../helpers/activitypub' import { logger } from '../../helpers/logger' import { CRAWL_REQUEST_CONCURRENCY } from '../../initializers/constants' -import { doRequest } from '../../helpers/requests' -import { checkUrlsSameHost, getAPId } from '../../helpers/activitypub' -import { getVideoDislikeActivityPubUrlByLocalActor, getVideoLikeActivityPubUrlByLocalActor } from './url' -import { sendDislike } from './send/send-dislike' +import { AccountVideoRateModel } from '../../models/account/account-video-rate' import { MAccountActor, MActorUrl, MVideo, MVideoAccountLight, MVideoId } from '../../types/models' +import { getOrCreateActorAndServerAndModel } from './actor' +import { sendLike, sendUndoDislike, sendUndoLike } from './send' +import { sendDislike } from './send/send-dislike' +import { getVideoDislikeActivityPubUrlByLocalActor, getVideoLikeActivityPubUrlByLocalActor } from './url' async function createRates (ratesUrl: string[], video: MVideo, rate: VideoRateType) { await Bluebird.map(ratesUrl, async rateUrl => { try { // Fetch url - const { body } = await doRequest({ - uri: rateUrl, - json: true, - activityPub: true - }) + const { body } = await doJSONRequest(rateUrl, { activityPub: true }) if (!body || !body.actor) throw new Error('Body or body actor is invalid') const actorUrl = getAPId(body.actor) diff --git a/server/lib/activitypub/videos.ts b/server/lib/activitypub/videos.ts index c02578aad..a5f58dd01 100644 --- a/server/lib/activitypub/videos.ts +++ b/server/lib/activitypub/videos.ts @@ -2,7 +2,6 @@ import * as Bluebird from 'bluebird' import { maxBy, minBy } from 'lodash' import * as magnetUtil from 'magnet-uri' import { basename, join } from 'path' -import * as request from 'request' import { Transaction } from 'sequelize/types' import { TrackerModel } from '@server/models/server/tracker' import { VideoLiveModel } from '@server/models/video/video-live' @@ -31,7 +30,7 @@ import { isArray } from '../../helpers/custom-validators/misc' import { isVideoFileInfoHashValid } from '../../helpers/custom-validators/videos' import { deleteNonExistingModels, resetSequelizeInstance, retryTransactionWrapper } from '../../helpers/database-utils' import { logger } from '../../helpers/logger' -import { doRequest } from '../../helpers/requests' +import { doJSONRequest } from '../../helpers/requests' import { fetchVideoByUrl, getExtFromMimetype, VideoFetchByUrlType } from '../../helpers/video' import { ACTIVITY_PUB, @@ -115,36 +114,26 @@ async function federateVideoIfNeeded (videoArg: MVideoAPWithoutCaption, isNewVid } } -async function fetchRemoteVideo (videoUrl: string): Promise<{ response: request.RequestResponse, videoObject: VideoObject }> { - const options = { - uri: videoUrl, - method: 'GET', - json: true, - activityPub: true - } - +async function fetchRemoteVideo (videoUrl: string): Promise<{ statusCode: number, videoObject: VideoObject }> { logger.info('Fetching remote video %s.', videoUrl) - const { response, body } = await doRequest(options) + const { statusCode, body } = await doJSONRequest(videoUrl, { activityPub: true }) if (sanitizeAndCheckVideoTorrentObject(body) === false || checkUrlsSameHost(body.id, videoUrl) !== true) { logger.debug('Remote video JSON is not valid.', { body }) - return { response, videoObject: undefined } + return { statusCode, videoObject: undefined } } - return { response, videoObject: body } + return { statusCode, videoObject: body } } async function fetchRemoteVideoDescription (video: MVideoAccountLight) { const host = video.VideoChannel.Account.Actor.Server.host const path = video.getDescriptionAPIPath() - const options = { - uri: REMOTE_SCHEME.HTTP + '://' + host + path, - json: true - } + const url = REMOTE_SCHEME.HTTP + '://' + host + path - const { body } = await doRequest(options) - return body.description ? body.description : '' + const { body } = await doJSONRequest(url) + return body.description || '' } function getOrCreateVideoChannelFromVideoObject (videoObject: VideoObject) { @@ -534,8 +523,8 @@ async function refreshVideoIfNeeded (options: { : await VideoModel.loadByUrlAndPopulateAccount(options.video.url) try { - const { response, videoObject } = await fetchRemoteVideo(video.url) - if (response.statusCode === HttpStatusCode.NOT_FOUND_404) { + const { statusCode, videoObject } = await fetchRemoteVideo(video.url) + if (statusCode === HttpStatusCode.NOT_FOUND_404) { logger.info('Cannot refresh remote video %s: video does not exist anymore. Deleting it.', video.url) // Video does not exist anymore -- cgit v1.2.3