From f862be2749b4f2d8dee99128d7e3064a69920e11 Mon Sep 17 00:00:00 2001 From: "mira.bat" <97340105+irafire@users.noreply.github.com> Date: Thu, 27 Jul 2023 17:01:15 +0200 Subject: Add an option to sign federated fetches for mastodon compatibility (#5898) * Fix player error modal Not hidden when we change the video * Correctly dispose player components * Sign cross-server fetch requests for mastodon AUTHORIZED_FETCH compatibilty * Add a remote fetch sign configuration knob * Federated fetches refactoring --------- Co-authored-by: Chocobozzz Co-authored-by: ira --- server/lib/activitypub/activity.ts | 50 ++++++++++++++++------ server/lib/activitypub/actors/get.ts | 4 +- .../lib/activitypub/actors/shared/url-to-object.ts | 6 +-- server/lib/activitypub/crawl.ts | 8 ++-- .../activitypub/playlists/shared/url-to-object.ts | 6 +-- server/lib/activitypub/process/process-create.ts | 4 +- server/lib/activitypub/process/process-undo.ts | 4 +- server/lib/activitypub/process/process-update.ts | 4 +- server/lib/activitypub/send/http.ts | 13 ++++-- server/lib/activitypub/share.ts | 5 +-- server/lib/activitypub/video-comments.ts | 5 ++- .../lib/activitypub/videos/shared/url-to-object.ts | 4 +- .../videos/shared/video-sync-attributes.ts | 12 +++--- .../lib/job-queue/handlers/activitypub-cleaner.ts | 5 ++- .../handlers/activitypub-http-broadcast.ts | 2 +- .../job-queue/handlers/activitypub-http-unicast.ts | 2 +- 16 files changed, 79 insertions(+), 55 deletions(-) (limited to 'server/lib') diff --git a/server/lib/activitypub/activity.ts b/server/lib/activitypub/activity.ts index 0fed3e8fd..391bcd9c6 100644 --- a/server/lib/activitypub/activity.ts +++ b/server/lib/activitypub/activity.ts @@ -1,22 +1,26 @@ -import { doJSONRequest } from '@server/helpers/requests' -import { APObjectId, ActivityObject, ActivityPubActor, ActivityType } from '@shared/models' +import { doJSONRequest, PeerTubeRequestOptions } from '@server/helpers/requests' +import { CONFIG } from '@server/initializers/config' +import { ActivityObject, ActivityPubActor, ActivityType, APObjectId } from '@shared/models' +import { buildSignedRequestOptions } from './send' -function getAPId (object: string | { id: string }) { +export function getAPId (object: string | { id: string }) { if (typeof object === 'string') return object return object.id } -function getActivityStreamDuration (duration: number) { +export function getActivityStreamDuration (duration: number) { // https://www.w3.org/TR/activitystreams-vocabulary/#dfn-duration return 'PT' + duration + 'S' } -function getDurationFromActivityStream (duration: string) { +export function getDurationFromActivityStream (duration: string) { return parseInt(duration.replace(/[^\d]+/, '')) } -function buildAvailableActivities (): ActivityType[] { +// --------------------------------------------------------------------------- + +export function buildAvailableActivities (): ActivityType[] { return [ 'Create', 'Update', @@ -33,9 +37,25 @@ function buildAvailableActivities (): ActivityType[] { ] } -async function fetchAPObject (object: APObjectId) { +// --------------------------------------------------------------------------- + +export async function fetchAP (url: string, moreOptions: PeerTubeRequestOptions = {}) { + const options = { + activityPub: true, + + httpSignature: CONFIG.FEDERATION.SIGN_FEDERATED_FETCHES + ? await buildSignedRequestOptions({ hasPayload: false }) + : undefined, + + ...moreOptions + } + + return doJSONRequest(url, options) +} + +export async function fetchAPObjectIfNeeded (object: APObjectId) { if (typeof object === 'string') { - const { body } = await doJSONRequest>(object, { activityPub: true }) + const { body } = await fetchAP>(object) return body } @@ -43,10 +63,12 @@ async function fetchAPObject (ob return object as Exclude } -export { - getAPId, - fetchAPObject, - getActivityStreamDuration, - buildAvailableActivities, - getDurationFromActivityStream +export async function findLatestAPRedirection (url: string, iteration = 1) { + if (iteration > 10) throw new Error('Too much iterations to find final URL ' + url) + + const { headers } = await fetchAP(url, { followRedirect: false }) + + if (headers.location) return findLatestAPRedirection(headers.location, iteration + 1) + + return url } diff --git a/server/lib/activitypub/actors/get.ts b/server/lib/activitypub/actors/get.ts index b2be3f5fb..dd2bc9f03 100644 --- a/server/lib/activitypub/actors/get.ts +++ b/server/lib/activitypub/actors/get.ts @@ -5,7 +5,7 @@ import { ActorLoadByUrlType, loadActorByUrl } from '@server/lib/model-loaders' import { MActor, MActorAccountChannelId, MActorAccountChannelIdActor, MActorAccountId, MActorFullActor } from '@server/types/models' import { arrayify } from '@shared/core-utils' import { ActivityPubActor, APObjectId } from '@shared/models' -import { fetchAPObject, getAPId } from '../activity' +import { fetchAPObjectIfNeeded, getAPId } from '../activity' import { checkUrlsSameHost } from '../url' import { refreshActorIfNeeded } from './refresh' import { APActorCreator, fetchRemoteActor } from './shared' @@ -87,7 +87,7 @@ async function getOrCreateAPOwner (actorObject: ActivityPubActor, actorUrl: stri async function findOwner (rootUrl: string, attributedTo: APObjectId[] | APObjectId, type: 'Person' | 'Group') { for (const actorToCheck of arrayify(attributedTo)) { - const actorObject = await fetchAPObject(getAPId(actorToCheck)) + const actorObject = await fetchAPObjectIfNeeded(getAPId(actorToCheck)) if (!actorObject) { logger.warn('Unknown attributed to actor %s for owner %s', actorToCheck, rootUrl) diff --git a/server/lib/activitypub/actors/shared/url-to-object.ts b/server/lib/activitypub/actors/shared/url-to-object.ts index 208d108ee..73766bd50 100644 --- a/server/lib/activitypub/actors/shared/url-to-object.ts +++ b/server/lib/activitypub/actors/shared/url-to-object.ts @@ -1,13 +1,13 @@ import { sanitizeAndCheckActorObject } from '@server/helpers/custom-validators/activitypub/actor' import { logger } from '@server/helpers/logger' -import { doJSONRequest } from '@server/helpers/requests' import { ActivityPubActor, ActivityPubOrderedCollection } from '@shared/models' +import { fetchAP } from '../../activity' import { checkUrlsSameHost } from '../../url' async function fetchRemoteActor (actorUrl: string): Promise<{ statusCode: number, actorObject: ActivityPubActor }> { logger.info('Fetching remote actor %s.', actorUrl) - const { body, statusCode } = await doJSONRequest(actorUrl, { activityPub: true }) + const { body, statusCode } = await fetchAP(actorUrl) if (sanitizeAndCheckActorObject(body) === false) { logger.debug('Remote actor JSON is not valid.', { actorJSON: body }) @@ -46,7 +46,7 @@ export { async function fetchActorTotalItems (url: string) { try { - const { body } = await doJSONRequest>(url, { activityPub: true }) + const { body } = await fetchAP>(url) return body.totalItems || 0 } catch (err) { diff --git a/server/lib/activitypub/crawl.ts b/server/lib/activitypub/crawl.ts index 336129b82..b8348e8cf 100644 --- a/server/lib/activitypub/crawl.ts +++ b/server/lib/activitypub/crawl.ts @@ -3,8 +3,8 @@ import { URL } from 'url' import { retryTransactionWrapper } from '@server/helpers/database-utils' import { ActivityPubOrderedCollection } from '../../../shared/models/activitypub' import { logger } from '../../helpers/logger' -import { doJSONRequest } from '../../helpers/requests' import { ACTIVITY_PUB, WEBSERVER } from '../../initializers/constants' +import { fetchAP } from './activity' type HandlerFunction = (items: T[]) => (Promise | Bluebird) type CleanerFunction = (startedDate: Date) => Promise @@ -14,11 +14,9 @@ async function crawlCollectionPage (argUrl: string, handler: HandlerFunction logger.info('Crawling ActivityPub data on %s.', url) - const options = { activityPub: true } - const startDate = new Date() - const response = await doJSONRequest>(url, options) + const response = await fetchAP>(url) const firstBody = response.body const limit = ACTIVITY_PUB.FETCH_PAGE_LIMIT @@ -34,7 +32,7 @@ async function crawlCollectionPage (argUrl: string, handler: HandlerFunction url = nextLink - const res = await doJSONRequest>(url, options) + const res = await fetchAP>(url) body = res.body } else { // nextLink is already the object we want diff --git a/server/lib/activitypub/playlists/shared/url-to-object.ts b/server/lib/activitypub/playlists/shared/url-to-object.ts index 41bee3752..fd9fe5558 100644 --- a/server/lib/activitypub/playlists/shared/url-to-object.ts +++ b/server/lib/activitypub/playlists/shared/url-to-object.ts @@ -1,8 +1,8 @@ import { isPlaylistElementObjectValid, isPlaylistObjectValid } from '@server/helpers/custom-validators/activitypub/playlist' import { isArray } from '@server/helpers/custom-validators/misc' import { logger, loggerTagsFactory } from '@server/helpers/logger' -import { doJSONRequest } from '@server/helpers/requests' import { PlaylistElementObject, PlaylistObject } from '@shared/models' +import { fetchAP } from '../../activity' import { checkUrlsSameHost } from '../../url' async function fetchRemoteVideoPlaylist (playlistUrl: string): Promise<{ statusCode: number, playlistObject: PlaylistObject }> { @@ -10,7 +10,7 @@ async function fetchRemoteVideoPlaylist (playlistUrl: string): Promise<{ statusC logger.info('Fetching remote playlist %s.', playlistUrl, lTags()) - const { body, statusCode } = await doJSONRequest(playlistUrl, { activityPub: true }) + const { body, statusCode } = await fetchAP(playlistUrl) if (isPlaylistObjectValid(body) === false || checkUrlsSameHost(body.id, playlistUrl) !== true) { logger.debug('Remote video playlist JSON is not valid.', { body, ...lTags() }) @@ -30,7 +30,7 @@ async function fetchRemotePlaylistElement (elementUrl: string): Promise<{ status logger.debug('Fetching remote playlist element %s.', elementUrl, lTags()) - const { body, statusCode } = await doJSONRequest(elementUrl, { activityPub: true }) + const { body, statusCode } = await fetchAP(elementUrl) if (!isPlaylistElementObjectValid(body)) throw new Error(`Invalid body in fetch playlist element ${elementUrl}`) diff --git a/server/lib/activitypub/process/process-create.ts b/server/lib/activitypub/process/process-create.ts index 2e64d981e..5f980de65 100644 --- a/server/lib/activitypub/process/process-create.ts +++ b/server/lib/activitypub/process/process-create.ts @@ -18,7 +18,7 @@ import { sequelizeTypescript } from '../../../initializers/database' import { APProcessorOptions } from '../../../types/activitypub-processor.model' import { MActorSignature, MCommentOwnerVideo, MVideoAccountLightBlacklistAllFiles } from '../../../types/models' import { Notifier } from '../../notifier' -import { fetchAPObject } from '../activity' +import { fetchAPObjectIfNeeded } from '../activity' import { createOrUpdateCacheFile } from '../cache-file' import { createOrUpdateLocalVideoViewer } from '../local-video-viewer' import { createOrUpdateVideoPlaylist } from '../playlists' @@ -31,7 +31,7 @@ async function processCreateActivity (options: APProcessorOptions>(activity.object) + const activityObject = await fetchAPObjectIfNeeded>(activity.object) const activityType = activityObject.type if (activityType === 'Video') { diff --git a/server/lib/activitypub/process/process-undo.ts b/server/lib/activitypub/process/process-undo.ts index 25f68724d..a9d8199de 100644 --- a/server/lib/activitypub/process/process-undo.ts +++ b/server/lib/activitypub/process/process-undo.ts @@ -19,7 +19,7 @@ import { VideoRedundancyModel } from '../../../models/redundancy/video-redundanc import { VideoShareModel } from '../../../models/video/video-share' import { APProcessorOptions } from '../../../types/activitypub-processor.model' import { MActorSignature } from '../../../types/models' -import { fetchAPObject } from '../activity' +import { fetchAPObjectIfNeeded } from '../activity' import { forwardVideoRelatedActivity } from '../send/shared/send-utils' import { federateVideoIfNeeded, getOrCreateAPVideo } from '../videos' @@ -32,7 +32,7 @@ async function processUndoActivity (options: APProcessorOptions(activityToUndo.object) + const objectToUndo = await fetchAPObjectIfNeeded(activityToUndo.object) if (objectToUndo.type === 'CacheFile') { return retryTransactionWrapper(processUndoCacheFile, byActor, activity, objectToUndo) diff --git a/server/lib/activitypub/process/process-update.ts b/server/lib/activitypub/process/process-update.ts index 9caa74e04..304ed9de6 100644 --- a/server/lib/activitypub/process/process-update.ts +++ b/server/lib/activitypub/process/process-update.ts @@ -10,7 +10,7 @@ import { sequelizeTypescript } from '../../../initializers/database' import { ActorModel } from '../../../models/actor/actor' import { APProcessorOptions } from '../../../types/activitypub-processor.model' import { MActorFull, MActorSignature } from '../../../types/models' -import { fetchAPObject } from '../activity' +import { fetchAPObjectIfNeeded } from '../activity' import { APActorUpdater } from '../actors/updater' import { createOrUpdateCacheFile } from '../cache-file' import { createOrUpdateVideoPlaylist } from '../playlists' @@ -20,7 +20,7 @@ import { APVideoUpdater, getOrCreateAPVideo } from '../videos' async function processUpdateActivity (options: APProcessorOptions>) { const { activity, byActor } = options - const object = await fetchAPObject(activity.object) + const object = await fetchAPObjectIfNeeded(activity.object) const objectType = object.type if (objectType === 'Video') { diff --git a/server/lib/activitypub/send/http.ts b/server/lib/activitypub/send/http.ts index ad7869853..b461aa55d 100644 --- a/server/lib/activitypub/send/http.ts +++ b/server/lib/activitypub/send/http.ts @@ -23,11 +23,14 @@ async function computeBody ( return body } -async function buildSignedRequestOptions (payload: Payload) { +async function buildSignedRequestOptions (options: { + signatureActorId?: number + hasPayload: boolean +}) { let actor: MActor | null - if (payload.signatureActorId) { - actor = await ActorModel.load(payload.signatureActorId) + if (options.signatureActorId) { + actor = await ActorModel.load(options.signatureActorId) if (!actor) throw new Error('Unknown signature actor id.') } else { // We need to sign the request, so use the server @@ -40,7 +43,9 @@ async function buildSignedRequestOptions (payload: Payload) { authorizationHeaderName: HTTP_SIGNATURE.HEADER_NAME, keyId, key: actor.privateKey, - headers: HTTP_SIGNATURE.HEADERS_TO_SIGN + headers: options.hasPayload + ? HTTP_SIGNATURE.HEADERS_TO_SIGN_WITH_PAYLOAD + : HTTP_SIGNATURE.HEADERS_TO_SIGN_WITHOUT_PAYLOAD } } diff --git a/server/lib/activitypub/share.ts b/server/lib/activitypub/share.ts index af0dd510a..792a73f2a 100644 --- a/server/lib/activitypub/share.ts +++ b/server/lib/activitypub/share.ts @@ -2,11 +2,10 @@ import { map } from 'bluebird' import { Transaction } from 'sequelize' import { getServerActor } from '@server/models/application/application' import { logger, loggerTagsFactory } from '../../helpers/logger' -import { doJSONRequest } from '../../helpers/requests' import { CRAWL_REQUEST_CONCURRENCY } from '../../initializers/constants' import { VideoShareModel } from '../../models/video/video-share' import { MChannelActorLight, MVideo, MVideoAccountLight, MVideoId } from '../../types/models/video' -import { getAPId } from './activity' +import { fetchAP, getAPId } from './activity' import { getOrCreateAPActor } from './actors' import { sendUndoAnnounce, sendVideoAnnounce } from './send' import { checkUrlsSameHost, getLocalVideoAnnounceActivityPubUrl } from './url' @@ -56,7 +55,7 @@ export { // --------------------------------------------------------------------------- async function addVideoShare (shareUrl: string, video: MVideoId) { - const { body } = await doJSONRequest(shareUrl, { activityPub: true }) + const { body } = await fetchAP(shareUrl) if (!body?.actor) throw new Error('Body or body actor is invalid') const actorUrl = getAPId(body.actor) diff --git a/server/lib/activitypub/video-comments.ts b/server/lib/activitypub/video-comments.ts index 4fdb4e0b7..b861be5bd 100644 --- a/server/lib/activitypub/video-comments.ts +++ b/server/lib/activitypub/video-comments.ts @@ -1,12 +1,13 @@ import { map } from 'bluebird' + import { sanitizeAndCheckVideoCommentObject } from '../../helpers/custom-validators/activitypub/video-comments' import { logger } from '../../helpers/logger' -import { doJSONRequest } from '../../helpers/requests' import { ACTIVITY_PUB, CRAWL_REQUEST_CONCURRENCY } from '../../initializers/constants' import { VideoCommentModel } from '../../models/video/video-comment' import { MComment, MCommentOwner, MCommentOwnerVideo, MVideoAccountLightBlacklistAllFiles } from '../../types/models/video' import { isRemoteVideoCommentAccepted } from '../moderation' import { Hooks } from '../plugins/hooks' +import { fetchAP } from './activity' import { getOrCreateAPActor } from './actors' import { checkUrlsSameHost } from './url' import { getOrCreateAPVideo } from './videos' @@ -139,7 +140,7 @@ async function resolveRemoteParentComment (params: ResolveThreadParams) { throw new Error('Recursion limit reached when resolving a thread') } - const { body } = await doJSONRequest(url, { activityPub: true }) + const { body } = await fetchAP(url) if (sanitizeAndCheckVideoCommentObject(body) === false) { throw new Error(`Remote video comment JSON ${url} is not valid:` + JSON.stringify(body)) diff --git a/server/lib/activitypub/videos/shared/url-to-object.ts b/server/lib/activitypub/videos/shared/url-to-object.ts index 5b7007530..7fe008419 100644 --- a/server/lib/activitypub/videos/shared/url-to-object.ts +++ b/server/lib/activitypub/videos/shared/url-to-object.ts @@ -1,7 +1,7 @@ import { sanitizeAndCheckVideoTorrentObject } from '@server/helpers/custom-validators/activitypub/videos' import { logger, loggerTagsFactory } from '@server/helpers/logger' -import { doJSONRequest } from '@server/helpers/requests' import { VideoObject } from '@shared/models' +import { fetchAP } from '../../activity' import { checkUrlsSameHost } from '../../url' const lTags = loggerTagsFactory('ap', 'video') @@ -9,7 +9,7 @@ const lTags = loggerTagsFactory('ap', 'video') async function fetchRemoteVideo (videoUrl: string): Promise<{ statusCode: number, videoObject: VideoObject }> { logger.info('Fetching remote video %s.', videoUrl, lTags(videoUrl)) - const { statusCode, body } = await doJSONRequest(videoUrl, { activityPub: true }) + const { statusCode, body } = await fetchAP(videoUrl) if (sanitizeAndCheckVideoTorrentObject(body) === false || checkUrlsSameHost(body.id, videoUrl) !== true) { logger.debug('Remote video JSON is not valid.', { body, ...lTags(videoUrl) }) diff --git a/server/lib/activitypub/videos/shared/video-sync-attributes.ts b/server/lib/activitypub/videos/shared/video-sync-attributes.ts index aa37f3d34..7fb933559 100644 --- a/server/lib/activitypub/videos/shared/video-sync-attributes.ts +++ b/server/lib/activitypub/videos/shared/video-sync-attributes.ts @@ -1,12 +1,12 @@ import { runInReadCommittedTransaction } from '@server/helpers/database-utils' import { logger, loggerTagsFactory } from '@server/helpers/logger' -import { doJSONRequest } from '@server/helpers/requests' import { JobQueue } from '@server/lib/job-queue' import { VideoModel } from '@server/models/video/video' import { VideoCommentModel } from '@server/models/video/video-comment' import { VideoShareModel } from '@server/models/video/video-share' import { MVideo } from '@server/types/models' import { ActivitypubHttpFetcherPayload, ActivityPubOrderedCollection, VideoObject } from '@shared/models' +import { fetchAP } from '../../activity' import { crawlCollectionPage } from '../../crawl' import { addVideoShares } from '../../share' import { addVideoComments } from '../../video-comments' @@ -63,17 +63,15 @@ async function getRatesCount (type: 'like' | 'dislike', video: MVideo, fetchedVi : fetchedVideo.dislikes logger.info('Sync %s of video %s', type, video.url) - const options = { activityPub: true } - const response = await doJSONRequest>(uri, options) - const totalItems = response.body.totalItems + const { body } = await fetchAP>(uri) - if (isNaN(totalItems)) { - logger.error('Cannot sync %s of video %s, totalItems is not a number', type, video.url, { body: response.body }) + if (isNaN(body.totalItems)) { + logger.error('Cannot sync %s of video %s, totalItems is not a number', type, video.url, { body }) return } - return totalItems + return body.totalItems } function syncShares (video: MVideo, fetchedVideo: VideoObject, isSync: boolean) { diff --git a/server/lib/job-queue/handlers/activitypub-cleaner.ts b/server/lib/job-queue/handlers/activitypub-cleaner.ts index a25f00b0a..6ee9e2429 100644 --- a/server/lib/job-queue/handlers/activitypub-cleaner.ts +++ b/server/lib/job-queue/handlers/activitypub-cleaner.ts @@ -6,8 +6,9 @@ import { isLikeActivityValid } from '@server/helpers/custom-validators/activitypub/activity' import { sanitizeAndCheckVideoCommentObject } from '@server/helpers/custom-validators/activitypub/video-comments' -import { doJSONRequest, PeerTubeRequestError } from '@server/helpers/requests' +import { PeerTubeRequestError } from '@server/helpers/requests' import { AP_CLEANER } from '@server/initializers/constants' +import { fetchAP } from '@server/lib/activitypub/activity' import { checkUrlsSameHost } from '@server/lib/activitypub/url' import { Redis } from '@server/lib/redis' import { VideoModel } from '@server/models/video/video' @@ -85,7 +86,7 @@ async function updateObjectIfNeeded (options: { } try { - const { body } = await doJSONRequest(url, { activityPub: true }) + const { body } = await fetchAP(url) // If not same id, check same host and update if (!body?.id || !bodyValidator(body)) throw new Error(`Body or body id of ${url} is invalid`) diff --git a/server/lib/job-queue/handlers/activitypub-http-broadcast.ts b/server/lib/job-queue/handlers/activitypub-http-broadcast.ts index 57ecf0acc..8904d086f 100644 --- a/server/lib/job-queue/handlers/activitypub-http-broadcast.ts +++ b/server/lib/job-queue/handlers/activitypub-http-broadcast.ts @@ -38,7 +38,7 @@ export { async function buildRequestOptions (payload: ActivitypubHttpBroadcastPayload) { const body = await computeBody(payload) - const httpSignatureOptions = await buildSignedRequestOptions(payload) + const httpSignatureOptions = await buildSignedRequestOptions({ signatureActorId: payload.signatureActorId, hasPayload: true }) return { method: 'POST' as 'POST', diff --git a/server/lib/job-queue/handlers/activitypub-http-unicast.ts b/server/lib/job-queue/handlers/activitypub-http-unicast.ts index 9e4e84002..50fca3f94 100644 --- a/server/lib/job-queue/handlers/activitypub-http-unicast.ts +++ b/server/lib/job-queue/handlers/activitypub-http-unicast.ts @@ -12,7 +12,7 @@ async function processActivityPubHttpUnicast (job: Job) { const uri = payload.uri const body = await computeBody(payload) - const httpSignatureOptions = await buildSignedRequestOptions(payload) + const httpSignatureOptions = await buildSignedRequestOptions({ signatureActorId: payload.signatureActorId, hasPayload: true }) const options = { method: 'POST' as 'POST', -- cgit v1.2.3