X-Git-Url: https://git.immae.eu/?a=blobdiff_plain;f=server%2Flib%2Factivitypub%2Factor.ts;h=5fe7381c9753784aa507c000549cfd701aea61e7;hb=e024fd6a7494b37251da1d59470324305cdb4129;hp=04296864b96c88038ed4726ca166679a10558550;hpb=5224c394b3bbac6ec1543e41fa0ec6db436e84fa;p=github%2FChocobozzz%2FPeerTube.git diff --git a/server/lib/activitypub/actor.ts b/server/lib/activitypub/actor.ts index 04296864b..5fe7381c9 100644 --- a/server/lib/activitypub/actor.ts +++ b/server/lib/activitypub/actor.ts @@ -1,49 +1,75 @@ import * as Bluebird from 'bluebird' -import { Transaction } from 'sequelize' -import * as url from 'url' -import * as uuidv4 from 'uuid/v4' -import { ActivityPubActor, ActivityPubActorType } from '../../../shared/models/activitypub' +import { extname } from 'path' +import { Op, Transaction } from 'sequelize' +import { URL } from 'url' +import { v4 as uuidv4 } from 'uuid' +import { getServerActor } from '@server/models/application/application' +import { ActorImageType } from '@shared/models' +import { HttpStatusCode } from '../../../shared/core-utils/miscs/http-error-codes' +import { ActivityPubActor, ActivityPubActorType, ActivityPubOrderedCollection } from '../../../shared/models/activitypub' import { ActivityPubAttributedTo } from '../../../shared/models/activitypub/objects' import { checkUrlsSameHost, getAPId } from '../../helpers/activitypub' +import { ActorFetchByUrlType, fetchActorByUrl } from '../../helpers/actor' import { sanitizeAndCheckActorObject } from '../../helpers/custom-validators/activitypub/actor' import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc' import { retryTransactionWrapper, updateInstanceWithAnother } from '../../helpers/database-utils' import { logger } from '../../helpers/logger' import { createPrivateAndPublicKeys } from '../../helpers/peertube-crypto' -import { doRequest, downloadImage } from '../../helpers/requests' +import { doJSONRequest, PeerTubeRequestError } from '../../helpers/requests' import { getUrlFromWebfinger } from '../../helpers/webfinger' -import { AVATARS_SIZE, MIMETYPES, WEBSERVER } from '../../initializers/constants' +import { MIMETYPES, WEBSERVER } from '../../initializers/constants' +import { sequelizeTypescript } from '../../initializers/database' import { AccountModel } from '../../models/account/account' +import { ActorImageModel } from '../../models/account/actor-image' import { ActorModel } from '../../models/activitypub/actor' -import { AvatarModel } from '../../models/avatar/avatar' import { ServerModel } from '../../models/server/server' import { VideoChannelModel } from '../../models/video/video-channel' +import { + MAccount, + MAccountDefault, + MActor, + MActorAccountChannelId, + MActorAccountChannelIdActor, + MActorAccountId, + MActorFull, + MActorFullActor, + MActorId, + MActorImage, + MActorImages, + MChannel +} from '../../types/models' import { JobQueue } from '../job-queue' -import { getServerActor } from '../../helpers/utils' -import { ActorFetchByUrlType, fetchActorByUrl } from '../../helpers/actor' -import { CONFIG } from '../../initializers/config' -import { sequelizeTypescript } from '../../initializers/database' // Set account keys, this could be long so process after the account creation and do not block the client -function setAsyncActorKeys (actor: ActorModel) { - return createPrivateAndPublicKeys() - .then(({ publicKey, privateKey }) => { - actor.set('publicKey', publicKey) - actor.set('privateKey', privateKey) - return actor.save() - }) - .catch(err => { - logger.error('Cannot set public/private keys of actor %d.', actor.url, { err }) - return actor - }) +async function generateAndSaveActorKeys (actor: T) { + const { publicKey, privateKey } = await createPrivateAndPublicKeys() + + actor.publicKey = publicKey + actor.privateKey = privateKey + + return actor.save() } +function getOrCreateActorAndServerAndModel ( + activityActor: string | ActivityPubActor, + fetchType: 'all', + recurseIfNeeded?: boolean, + updateCollections?: boolean +): Promise + +function getOrCreateActorAndServerAndModel ( + activityActor: string | ActivityPubActor, + fetchType?: 'association-ids', + recurseIfNeeded?: boolean, + updateCollections?: boolean +): Promise + async function getOrCreateActorAndServerAndModel ( activityActor: string | ActivityPubActor, - fetchType: ActorFetchByUrlType = 'actor-and-association-ids', + fetchType: ActorFetchByUrlType = 'association-ids', recurseIfNeeded = true, updateCollections = false -) { +): Promise { const actorUrl = getAPId(activityActor) let created = false let accountPlaylistsUrl: string @@ -62,7 +88,7 @@ async function getOrCreateActorAndServerAndModel ( // Create the attributed to actor // In PeerTube a video channel is owned by an account - let ownerActor: ActorModel = undefined + let ownerActor: MActorFullActor if (recurseIfNeeded === true && result.actor.type === 'Group') { const accountAttributedTo = result.attributedTo.find(a => a.type === 'Person') if (!accountAttributedTo) throw new Error('Cannot find account attributed to video channel ' + actor.url) @@ -76,7 +102,7 @@ async function getOrCreateActorAndServerAndModel ( const recurseIfNeeded = false ownerActor = await getOrCreateActorAndServerAndModel(accountAttributedTo.id, 'all', recurseIfNeeded) } catch (err) { - logger.error('Cannot get or create account attributed to video channel ' + actor.url) + logger.error('Cannot get or create account attributed to video channel ' + actorUrl) throw new Error(err) } } @@ -86,21 +112,21 @@ async function getOrCreateActorAndServerAndModel ( accountPlaylistsUrl = result.playlists } - if (actor.Account) actor.Account.Actor = actor - if (actor.VideoChannel) actor.VideoChannel.Actor = actor + if (actor.Account) (actor as MActorAccountChannelIdActor).Account.Actor = actor + if (actor.VideoChannel) (actor as MActorAccountChannelIdActor).VideoChannel.Actor = actor const { actor: actorRefreshed, refreshed } = await retryTransactionWrapper(refreshActorIfNeeded, actor, fetchType) - if (!actorRefreshed) throw new Error('Actor ' + actorRefreshed.url + ' does not exist anymore.') + if (!actorRefreshed) throw new Error('Actor ' + actor.url + ' does not exist anymore.') if ((created === true || refreshed === true) && updateCollections === true) { const payload = { uri: actor.outboxUrl, type: 'activity' as 'activity' } - await JobQueue.Instance.createJob({ type: 'activitypub-http-fetcher', payload }) + await JobQueue.Instance.createJobWithPromise({ type: 'activitypub-http-fetcher', payload }) } // We created a new account: fetch the playlists if (created === true && actor.Account && accountPlaylistsUrl) { const payload = { uri: accountPlaylistsUrl, accountId: actor.Account.id, type: 'account-playlists' as 'account-playlists' } - await JobQueue.Instance.createJob({ type: 'activitypub-http-fetcher', payload }) + await JobQueue.Instance.createJobWithPromise({ type: 'activitypub-http-fetcher', payload }) } return actorRefreshed @@ -121,7 +147,7 @@ function buildActorInstance (type: ActivityPubActorType, url: string, preferredU sharedInboxUrl: WEBSERVER.URL + '/inbox', followersUrl: url + '/followers', followingUrl: url + '/following' - }) + }) as MActor } async function updateActorInstance (actorInstance: ActorModel, attributes: ActivityPubActor) { @@ -136,63 +162,115 @@ async function updateActorInstance (actorInstance: ActorModel, attributes: Activ actorInstance.followingCount = followingCount actorInstance.inboxUrl = attributes.inbox actorInstance.outboxUrl = attributes.outbox - actorInstance.sharedInboxUrl = attributes.endpoints.sharedInbox actorInstance.followersUrl = attributes.followers actorInstance.followingUrl = attributes.following + + if (attributes.published) actorInstance.remoteCreatedAt = new Date(attributes.published) + + if (attributes.endpoints?.sharedInbox) { + actorInstance.sharedInboxUrl = attributes.endpoints.sharedInbox + } } -async function updateActorAvatarInstance (actorInstance: ActorModel, avatarName: string, t: Transaction) { - if (avatarName !== undefined) { - if (actorInstance.avatarId) { - try { - await actorInstance.Avatar.destroy({ transaction: t }) - } catch (err) { - logger.error('Cannot remove old avatar of actor %s.', actorInstance.url, { err }) - } +type ImageInfo = { + name: string + fileUrl: string + height: number + width: number + onDisk?: boolean +} +async function updateActorImageInstance (actor: MActorImages, type: ActorImageType, imageInfo: ImageInfo | null, t: Transaction) { + const oldImageModel = type === ActorImageType.AVATAR + ? actor.Avatar + : actor.Banner + + if (oldImageModel) { + // Don't update the avatar if the file URL did not change + if (imageInfo?.fileUrl && oldImageModel.fileUrl === imageInfo.fileUrl) return actor + + try { + await oldImageModel.destroy({ transaction: t }) + + setActorImage(actor, type, null) + } catch (err) { + logger.error('Cannot remove old actor image of actor %s.', actor.url, { err }) } + } - const avatar = await AvatarModel.create({ - filename: avatarName + if (imageInfo) { + const imageModel = await ActorImageModel.create({ + filename: imageInfo.name, + onDisk: imageInfo.onDisk ?? false, + fileUrl: imageInfo.fileUrl, + height: imageInfo.height, + width: imageInfo.width, + type }, { transaction: t }) - actorInstance.set('avatarId', avatar.id) - actorInstance.Avatar = avatar + setActorImage(actor, type, imageModel) } - return actorInstance + return actor } -async function fetchActorTotalItems (url: string) { - const options = { - uri: url, - method: 'GET', - json: true, - activityPub: true +async function deleteActorImageInstance (actor: MActorImages, type: ActorImageType, t: Transaction) { + try { + if (type === ActorImageType.AVATAR) { + await actor.Avatar.destroy({ transaction: t }) + + actor.avatarId = null + actor.Avatar = null + } else { + await actor.Banner.destroy({ transaction: t }) + + actor.bannerId = null + actor.Banner = null + } + } catch (err) { + logger.error('Cannot remove old image of actor %s.', actor.url, { err }) } + return actor +} + +async function fetchActorTotalItems (url: string) { try { - const { body } = await doRequest(options) - return body.totalItems ? body.totalItems : 0 + const { body } = await doJSONRequest>(url, { activityPub: true }) + + return body.totalItems || 0 } catch (err) { logger.warn('Cannot fetch remote actor count %s.', url, { err }) return 0 } } -async function fetchAvatarIfExists (actorJSON: ActivityPubActor) { - if ( - actorJSON.icon && actorJSON.icon.type === 'Image' && MIMETYPES.IMAGE.MIMETYPE_EXT[actorJSON.icon.mediaType] !== undefined && - isActivityPubUrlValid(actorJSON.icon.url) - ) { - const extension = MIMETYPES.IMAGE.MIMETYPE_EXT[actorJSON.icon.mediaType] +function getImageInfoIfExists (actorJSON: ActivityPubActor, type: ActorImageType) { + const mimetypes = MIMETYPES.IMAGE + const icon = type === ActorImageType.AVATAR + ? actorJSON.icon + : actorJSON.image + + if (!icon || icon.type !== 'Image' || !isActivityPubUrlValid(icon.url)) return undefined + + let extension: string - const avatarName = uuidv4() + extension - await downloadImage(actorJSON.icon.url, CONFIG.STORAGE.AVATARS_DIR, avatarName, AVATARS_SIZE) + if (icon.mediaType) { + extension = mimetypes.MIMETYPE_EXT[icon.mediaType] + } else { + const tmp = extname(icon.url) - return avatarName + if (mimetypes.EXT_MIMETYPE[tmp] !== undefined) extension = tmp } - return undefined + if (!extension) return undefined + + return { + name: uuidv4() + extension, + fileUrl: icon.url, + height: icon.height, + width: icon.width, + type + } } async function addFetchOutboxJob (actor: Pick) { @@ -211,14 +289,16 @@ async function addFetchOutboxJob (actor: Pick) { return JobQueue.Instance.createJob({ type: 'activitypub-http-fetcher', payload }) } -async function refreshActorIfNeeded ( - actorArg: ActorModel, +async function refreshActorIfNeeded ( + actorArg: T, fetchedType: ActorFetchByUrlType -): Promise<{ actor: ActorModel, refreshed: boolean }> { +): Promise<{ actor: T | MActorFull, refreshed: boolean }> { if (!actorArg.isOutdated()) return { actor: actorArg, refreshed: false } // We need more attributes - const actor = fetchedType === 'all' ? actorArg : await ActorModel.loadByUrlAndPopulateAccountAndChannel(actorArg.url) + const actor = fetchedType === 'all' + ? actorArg as MActorFull + : await ActorModel.loadByUrlAndPopulateAccountAndChannel(actorArg.url) try { let actorUrl: string @@ -229,13 +309,7 @@ async function refreshActorIfNeeded ( actorUrl = actor.url } - const { result, statusCode } = await fetchRemoteActor(actorUrl) - - if (statusCode === 404) { - logger.info('Deleting actor %s because there is a 404 in refresh actor.', actor.url) - actor.Account ? actor.Account.destroy() : actor.VideoChannel.destroy() - return { actor: undefined, refreshed: false } - } + const { result } = await fetchRemoteActor(actorUrl) if (result === undefined) { logger.warn('Cannot fetch remote actor in refresh actor.') @@ -245,9 +319,8 @@ async function refreshActorIfNeeded ( return sequelizeTypescript.transaction(async t => { updateInstanceWithAnother(actor, result.actor) - if (result.avatarName !== undefined) { - await updateActorAvatarInstance(actor, result.avatarName, t) - } + await updateActorImageInstance(actor, ActorImageType.AVATAR, result.avatar, t) + await updateActorImageInstance(actor, ActorImageType.BANNER, result.banner, t) // Force update actor.setDataValue('updatedAt', new Date()) @@ -269,6 +342,15 @@ async function refreshActorIfNeeded ( return { refreshed: true, actor } }) } catch (err) { + if ((err as PeerTubeRequestError).statusCode === HttpStatusCode.NOT_FOUND_404) { + logger.info('Deleting actor %s because there is a 404 in refresh actor.', actor.url) + actor.Account + ? await actor.Account.destroy() + : await actor.VideoChannel.destroy() + + return { actor: undefined, refreshed: false } + } + logger.warn('Cannot refresh actor %s.', actor.url, { err }) return { actor, refreshed: false } } @@ -277,30 +359,47 @@ async function refreshActorIfNeeded ( export { getOrCreateActorAndServerAndModel, buildActorInstance, - setAsyncActorKeys, + generateAndSaveActorKeys, fetchActorTotalItems, - fetchAvatarIfExists, + getImageInfoIfExists, updateActorInstance, + deleteActorImageInstance, refreshActorIfNeeded, - updateActorAvatarInstance, + updateActorImageInstance, addFetchOutboxJob } // --------------------------------------------------------------------------- +function setActorImage (actorModel: MActorImages, type: ActorImageType, imageModel: MActorImage) { + const id = imageModel + ? imageModel.id + : null + + if (type === ActorImageType.AVATAR) { + actorModel.avatarId = id + actorModel.Avatar = imageModel + } else { + actorModel.bannerId = id + actorModel.Banner = imageModel + } + + return actorModel +} + function saveActorAndServerAndModelIfNotExist ( result: FetchRemoteActorResult, - ownerActor?: ActorModel, + ownerActor?: MActorFullActor, t?: Transaction -): Bluebird | Promise { - let actor = result.actor +): Bluebird | Promise { + const actor = result.actor if (t !== undefined) return save(t) return sequelizeTypescript.transaction(t => save(t)) async function save (t: Transaction) { - const actorHost = url.parse(actor.url).host + const actorHost = new URL(actor.url).host const serverOptions = { where: { @@ -314,33 +413,71 @@ function saveActorAndServerAndModelIfNotExist ( const [ server ] = await ServerModel.findOrCreate(serverOptions) // Save our new account in database - actor.set('serverId', server.id) + actor.serverId = server.id // Avatar? - if (result.avatarName) { - const avatar = await AvatarModel.create({ - filename: result.avatarName + if (result.avatar) { + const avatar = await ActorImageModel.create({ + filename: result.avatar.name, + fileUrl: result.avatar.fileUrl, + width: result.avatar.width, + height: result.avatar.height, + onDisk: false, + type: ActorImageType.AVATAR }, { transaction: t }) - actor.set('avatarId', avatar.id) + + actor.avatarId = avatar.id + } + + // Banner? + if (result.banner) { + const banner = await ActorImageModel.create({ + filename: result.banner.name, + fileUrl: result.banner.fileUrl, + width: result.banner.width, + height: result.banner.height, + onDisk: false, + type: ActorImageType.BANNER + }, { transaction: t }) + + actor.bannerId = banner.id } // Force the actor creation, sometimes Sequelize skips the save() when it thinks the instance already exists // (which could be false in a retried query) - const [ actorCreated ] = await ActorModel.findOrCreate({ + const [ actorCreated, created ] = await ActorModel.findOrCreate({ defaults: actor.toJSON(), where: { - url: actor.url + [Op.or]: [ + { + url: actor.url + }, + { + serverId: actor.serverId, + preferredUsername: actor.preferredUsername + } + ] }, transaction: t }) + // Try to fix non HTTPS accounts of remote instances that fixed their URL afterwards + if (created !== true && actorCreated.url !== actor.url) { + // Only fix http://example.com/account/djidane to https://example.com/account/djidane + if (actorCreated.url.replace(/^http:\/\//, '') !== actor.url.replace(/^https:\/\//, '')) { + throw new Error(`Actor from DB with URL ${actorCreated.url} does not correspond to actor ${actor.url}`) + } + + actorCreated.url = actor.url + await actorCreated.save({ transaction: t }) + } + if (actorCreated.type === 'Person' || actorCreated.type === 'Application') { - actorCreated.Account = await saveAccount(actorCreated, result, t) + actorCreated.Account = await saveAccount(actorCreated, result, t) as MAccountDefault actorCreated.Account.Actor = actorCreated } else if (actorCreated.type === 'Group') { // Video channel - actorCreated.VideoChannel = await saveVideoChannel(actorCreated, result, ownerActor, t) - actorCreated.VideoChannel.Actor = actorCreated - actorCreated.VideoChannel.Account = ownerActor.Account + const channel = await saveVideoChannel(actorCreated, result, ownerActor, t) + actorCreated.VideoChannel = Object.assign(channel, { Actor: actorCreated, Account: ownerActor.Account }) } actorCreated.Server = server @@ -349,36 +486,37 @@ function saveActorAndServerAndModelIfNotExist ( } } +type ImageResult = { + name: string + fileUrl: string + height: number + width: number +} + type FetchRemoteActorResult = { - actor: ActorModel + actor: MActor name: string summary: string support?: string playlists?: string - avatarName?: string + avatar?: ImageResult + banner?: ImageResult attributedTo: ActivityPubAttributedTo[] } async function fetchRemoteActor (actorUrl: string): Promise<{ statusCode?: number, result: FetchRemoteActorResult }> { - const options = { - uri: actorUrl, - method: 'GET', - json: true, - activityPub: true - } - logger.info('Fetching remote actor %s.', actorUrl) - const requestResult = await doRequest(options) + const requestResult = await doJSONRequest(actorUrl, { activityPub: true }) const actorJSON = requestResult.body if (sanitizeAndCheckActorObject(actorJSON) === false) { logger.debug('Remote actor JSON is not valid.', { actorJSON }) - return { result: undefined, statusCode: requestResult.response.statusCode } + return { result: undefined, statusCode: requestResult.statusCode } } if (checkUrlsSameHost(actorJSON.id, actorUrl) !== true) { logger.warn('Actor url %s has not the same host than its AP id %s', actorUrl, actorJSON.id) - return { result: undefined, statusCode: requestResult.response.statusCode } + return { result: undefined, statusCode: requestResult.statusCode } } const followersCount = await fetchActorTotalItems(actorJSON.followers) @@ -394,20 +532,25 @@ async function fetchRemoteActor (actorUrl: string): Promise<{ statusCode?: numbe followingCount: followingCount, inboxUrl: actorJSON.inbox, outboxUrl: actorJSON.outbox, - sharedInboxUrl: actorJSON.endpoints.sharedInbox, followersUrl: actorJSON.followers, - followingUrl: actorJSON.following + followingUrl: actorJSON.following, + + sharedInboxUrl: actorJSON.endpoints?.sharedInbox + ? actorJSON.endpoints.sharedInbox + : null }) - const avatarName = await fetchAvatarIfExists(actorJSON) + const avatarInfo = getImageInfoIfExists(actorJSON, ActorImageType.AVATAR) + const bannerInfo = getImageInfoIfExists(actorJSON, ActorImageType.BANNER) const name = actorJSON.name || actorJSON.preferredUsername return { - statusCode: requestResult.response.statusCode, + statusCode: requestResult.statusCode, result: { actor, name, - avatarName, + avatar: avatarInfo, + banner: bannerInfo, summary: actorJSON.summary, support: actorJSON.support, playlists: actorJSON.playlists, @@ -416,7 +559,7 @@ async function fetchRemoteActor (actorUrl: string): Promise<{ statusCode?: numbe } } -async function saveAccount (actor: ActorModel, result: FetchRemoteActorResult, t: Transaction) { +async function saveAccount (actor: MActorId, result: FetchRemoteActorResult, t: Transaction) { const [ accountCreated ] = await AccountModel.findOrCreate({ defaults: { name: result.name, @@ -429,10 +572,10 @@ async function saveAccount (actor: ActorModel, result: FetchRemoteActorResult, t transaction: t }) - return accountCreated + return accountCreated as MAccount } -async function saveVideoChannel (actor: ActorModel, result: FetchRemoteActorResult, ownerActor: ActorModel, t: Transaction) { +async function saveVideoChannel (actor: MActorId, result: FetchRemoteActorResult, ownerActor: MActorAccountId, t: Transaction) { const [ videoChannelCreated ] = await VideoChannelModel.findOrCreate({ defaults: { name: result.name, @@ -447,5 +590,5 @@ async function saveVideoChannel (actor: ActorModel, result: FetchRemoteActorResu transaction: t }) - return videoChannelCreated + return videoChannelCreated as MChannel }