From a5625b416797fa20f7cccead6201f3faf9427080 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Thu, 4 Jan 2018 14:04:02 +0100 Subject: Refresh remote accounts --- server/controllers/api/users.ts | 24 ++----- server/helpers/database-utils.ts | 12 +++- server/helpers/webfinger.ts | 5 ++ server/initializers/constants.ts | 4 +- server/lib/activitypub/actor.ts | 80 +++++++++++++++++++++++- server/lib/activitypub/process/process-update.ts | 45 +++---------- server/models/activitypub/actor.ts | 13 +++- 7 files changed, 122 insertions(+), 61 deletions(-) diff --git a/server/controllers/api/users.ts b/server/controllers/api/users.ts index ef2b63f51..d8ecbdbe2 100644 --- a/server/controllers/api/users.ts +++ b/server/controllers/api/users.ts @@ -8,6 +8,7 @@ import { retryTransactionWrapper } from '../../helpers/database-utils' import { logger } from '../../helpers/logger' import { createReqFiles, getFormattedObjects } from '../../helpers/utils' import { AVATAR_MIMETYPE_EXT, AVATARS_SIZE, CONFIG, sequelizeTypescript } from '../../initializers' +import { updateActorAvatarInstance } from '../../lib/activitypub' import { sendUpdateUser } from '../../lib/activitypub/send' import { createUserAccountAndChannel } from '../../lib/user' import { @@ -18,7 +19,6 @@ import { import { usersUpdateMyAvatarValidator, videosSortValidator } from '../../middlewares/validators' import { AccountVideoRateModel } from '../../models/account/account-video-rate' import { UserModel } from '../../models/account/user' -import { AvatarModel } from '../../models/avatar/avatar' import { VideoModel } from '../../models/video/video' const reqAvatarFile = createReqFiles('avatarfile', CONFIG.STORAGE.AVATARS_DIR, AVATAR_MIMETYPE_EXT) @@ -248,26 +248,12 @@ async function updateMyAvatar (req: express.Request, res: express.Response, next await unlinkPromise(source) - const { avatar } = await sequelizeTypescript.transaction(async t => { - const avatar = await AvatarModel.create({ - filename: avatarName - }, { transaction: t }) + const avatar = await sequelizeTypescript.transaction(async t => { + await updateActorAvatarInstance(actor, avatarName, t) - if (actor.Avatar) { - try { - await actor.Avatar.destroy({ transaction: t }) - } catch (err) { - logger.error('Cannot remove old avatar of user %s.', user.username, err) - } - } + await sendUpdateUser(user, t) - actor.set('avatarId', avatar.id) - actor.Avatar = avatar - await actor.save({ transaction: t }) - - await sendUpdateUser(user, undefined) - - return { actor, avatar } + return avatar }) return res diff --git a/server/helpers/database-utils.ts b/server/helpers/database-utils.ts index fb8ad22b0..78ca768b9 100644 --- a/server/helpers/database-utils.ts +++ b/server/helpers/database-utils.ts @@ -1,5 +1,6 @@ import * as retry from 'async/retry' import * as Bluebird from 'bluebird' +import { Model } from 'sequelize-typescript' import { logger } from './logger' type RetryTransactionWrapperOptions = { errorMessage: string, arguments?: any[] } @@ -34,9 +35,18 @@ function transactionRetryer (func: (err: any, data: T) => any) { }) } +function updateInstanceWithAnother (instanceToUpdate: Model, baseInstance: Model) { + const obj = baseInstance.toJSON() + + for (const key of Object.keys(obj)) { + instanceToUpdate.set(key, obj[key]) + } +} + // --------------------------------------------------------------------------- export { retryTransactionWrapper, - transactionRetryer + transactionRetryer, + updateInstanceWithAnother } diff --git a/server/helpers/webfinger.ts b/server/helpers/webfinger.ts index de8d52c9b..688bf2bab 100644 --- a/server/helpers/webfinger.ts +++ b/server/helpers/webfinger.ts @@ -15,6 +15,10 @@ async function loadActorUrlOrGetFromWebfinger (name: string, host: string) { const actor = await ActorModel.loadByNameAndHost(name, host) if (actor) return actor.url + return getUrlFromWebfinger(name, host) +} + +async function getUrlFromWebfinger (name: string, host: string) { const webfingerData: WebFingerData = await webfingerLookup(name + '@' + host) return getLinkOrThrow(webfingerData) } @@ -22,6 +26,7 @@ async function loadActorUrlOrGetFromWebfinger (name: string, host: string) { // --------------------------------------------------------------------------- export { + getUrlFromWebfinger, loadActorUrlOrGetFromWebfinger } diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index d2bcea443..1f18b4401 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts @@ -278,7 +278,8 @@ const ACTIVITY_PUB = { VIDEO: [ 'video/mp4', 'video/webm', 'video/ogg' ], // TODO: Merge with VIDEO_MIMETYPE_EXT TORRENT: [ 'application/x-bittorrent' ], MAGNET: [ 'application/x-bittorrent;x-scheme-handler/magnet' ] - } + }, + ACTOR_REFRESH_INTERVAL: 3600 * 24 // 1 day } const ACTIVITY_PUB_ACTOR_TYPES: { [ id: string ]: ActivityPubActorType } = { @@ -350,6 +351,7 @@ if (isTestInstance() === true) { REMOTE_SCHEME.WS = 'ws' STATIC_MAX_AGE = '0' ACTIVITY_PUB.COLLECTION_ITEMS_PER_PAGE = 2 + ACTIVITY_PUB.ACTOR_REFRESH_INTERVAL = 60 // 1 minute CONSTRAINTS_FIELDS.ACTORS.AVATAR.FILE_SIZE.max = 100 * 1024 // 100KB } diff --git a/server/lib/activitypub/actor.ts b/server/lib/activitypub/actor.ts index b6ba2cc22..0882ab843 100644 --- a/server/lib/activitypub/actor.ts +++ b/server/lib/activitypub/actor.ts @@ -7,10 +7,11 @@ import { ActivityPubActor, ActivityPubActorType } from '../../../shared/models/a import { ActivityPubAttributedTo } from '../../../shared/models/activitypub/objects' import { isActorObjectValid } from '../../helpers/custom-validators/activitypub/actor' import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc' -import { retryTransactionWrapper } from '../../helpers/database-utils' +import { retryTransactionWrapper, updateInstanceWithAnother } from '../../helpers/database-utils' import { logger } from '../../helpers/logger' import { createPrivateAndPublicKeys } from '../../helpers/peertube-crypto' import { doRequest, doRequestAndSaveToFile } from '../../helpers/requests' +import { getUrlFromWebfinger } from '../../helpers/webfinger' import { AVATAR_MIMETYPE_EXT, CONFIG, sequelizeTypescript } from '../../initializers' import { AccountModel } from '../../models/account/account' import { ActorModel } from '../../models/activitypub/actor' @@ -63,7 +64,7 @@ async function getOrCreateActorAndServerAndModel (actorUrl: string, recurseIfNee actor = await retryTransactionWrapper(saveActorAndServerAndModelIfNotExist, options) } - return actor + return refreshActorIfNeeded(actor) } function buildActorInstance (type: ActivityPubActorType, url: string, preferredUsername: string, uuid?: string) { @@ -84,6 +85,45 @@ function buildActorInstance (type: ActivityPubActorType, url: string, preferredU }) } +async function updateActorInstance (actorInstance: ActorModel, attributes: ActivityPubActor) { + const followersCount = await fetchActorTotalItems(attributes.followers) + const followingCount = await fetchActorTotalItems(attributes.following) + + actorInstance.set('type', attributes.type) + actorInstance.set('uuid', attributes.uuid) + actorInstance.set('preferredUsername', attributes.preferredUsername) + actorInstance.set('url', attributes.id) + actorInstance.set('publicKey', attributes.publicKey.publicKeyPem) + actorInstance.set('followersCount', followersCount) + actorInstance.set('followingCount', followingCount) + actorInstance.set('inboxUrl', attributes.inbox) + actorInstance.set('outboxUrl', attributes.outbox) + actorInstance.set('sharedInboxUrl', attributes.endpoints.sharedInbox) + actorInstance.set('followersUrl', attributes.followers) + actorInstance.set('followingUrl', attributes.following) +} + +async function updateActorAvatarInstance (actorInstance: ActorModel, avatarName: string, t: Transaction) { + if (avatarName !== undefined) { + if (actorInstance.avatarId) { + try { + await actorInstance.Avatar.destroy({ transaction: t }) + } catch (err) { + logger.error('Cannot remove old avatar of actor %s.', actorInstance.url, err) + } + } + + const avatar = await AvatarModel.create({ + filename: avatarName + }, { transaction: t }) + + actorInstance.set('avatarId', avatar.id) + actorInstance.Avatar = avatar + } + + return actorInstance +} + async function fetchActorTotalItems (url: string) { const options = { uri: url, @@ -129,7 +169,9 @@ export { buildActorInstance, setAsyncActorKeys, fetchActorTotalItems, - fetchAvatarIfExists + fetchAvatarIfExists, + updateActorInstance, + updateActorAvatarInstance } // --------------------------------------------------------------------------- @@ -263,3 +305,35 @@ async function saveVideoChannel (actor: ActorModel, result: FetchRemoteActorResu return videoChannel.save({ transaction: t }) } + +async function refreshActorIfNeeded (actor: ActorModel) { + if (!actor.isOutdated()) return actor + + const actorUrl = await getUrlFromWebfinger(actor.preferredUsername, actor.getHost()) + const result = await fetchRemoteActor(actorUrl) + if (result === undefined) throw new Error('Cannot fetch remote actor in refresh actor.') + + return sequelizeTypescript.transaction(async t => { + updateInstanceWithAnother(actor, result.actor) + + if (result.avatarName !== undefined) { + await updateActorAvatarInstance(actor, result.avatarName, t) + } + + await actor.save({ transaction: t }) + + if (actor.Account) { + await actor.save({ transaction: t }) + + actor.Account.set('name', result.name) + await actor.Account.save({ transaction: t }) + } else if (actor.VideoChannel) { + await actor.save({ transaction: t }) + + actor.VideoChannel.set('name', result.name) + await actor.VideoChannel.save({ transaction: t }) + } + + return actor + }) +} diff --git a/server/lib/activitypub/process/process-update.ts b/server/lib/activitypub/process/process-update.ts index 05ea7d272..2c094f7ca 100644 --- a/server/lib/activitypub/process/process-update.ts +++ b/server/lib/activitypub/process/process-update.ts @@ -8,11 +8,10 @@ import { resetSequelizeInstance } from '../../../helpers/utils' import { sequelizeTypescript } from '../../../initializers' import { AccountModel } from '../../../models/account/account' import { ActorModel } from '../../../models/activitypub/actor' -import { AvatarModel } from '../../../models/avatar/avatar' import { TagModel } from '../../../models/video/tag' import { VideoModel } from '../../../models/video/video' import { VideoFileModel } from '../../../models/video/video-file' -import { fetchActorTotalItems, fetchAvatarIfExists, getOrCreateActorAndServerAndModel } from '../actor' +import { fetchAvatarIfExists, getOrCreateActorAndServerAndModel, updateActorAvatarInstance, updateActorInstance } from '../actor' import { videoActivityObjectToDBAttributes, videoFileActivityUrlToDBAttributes } from './misc' async function processUpdateActivity (activity: ActivityUpdate) { @@ -124,7 +123,6 @@ async function updateRemoteAccount (actor: ActorModel, activity: ActivityUpdate) const accountAttributesToUpdate = activity.object as ActivityPubActor logger.debug('Updating remote account "%s".', accountAttributesToUpdate.uuid) - let actorInstance: ActorModel let accountInstance: AccountModel let actorFieldsSave: object let accountFieldsSave: object @@ -134,39 +132,14 @@ async function updateRemoteAccount (actor: ActorModel, activity: ActivityUpdate) try { await sequelizeTypescript.transaction(async t => { - actorInstance = await ActorModel.loadByUrl(accountAttributesToUpdate.id, t) - if (!actorInstance) throw new Error('Actor ' + accountAttributesToUpdate.id + ' not found.') - - actorFieldsSave = actorInstance.toJSON() - accountInstance = actorInstance.Account - accountFieldsSave = actorInstance.Account.toJSON() - - const followersCount = await fetchActorTotalItems(accountAttributesToUpdate.followers) - const followingCount = await fetchActorTotalItems(accountAttributesToUpdate.following) - - actorInstance.set('type', accountAttributesToUpdate.type) - actorInstance.set('uuid', accountAttributesToUpdate.uuid) - actorInstance.set('preferredUsername', accountAttributesToUpdate.preferredUsername) - actorInstance.set('url', accountAttributesToUpdate.id) - actorInstance.set('publicKey', accountAttributesToUpdate.publicKey.publicKeyPem) - actorInstance.set('followersCount', followersCount) - actorInstance.set('followingCount', followingCount) - actorInstance.set('inboxUrl', accountAttributesToUpdate.inbox) - actorInstance.set('outboxUrl', accountAttributesToUpdate.outbox) - actorInstance.set('sharedInboxUrl', accountAttributesToUpdate.endpoints.sharedInbox) - actorInstance.set('followersUrl', accountAttributesToUpdate.followers) - actorInstance.set('followingUrl', accountAttributesToUpdate.following) + actorFieldsSave = actor.toJSON() + accountInstance = actor.Account + accountFieldsSave = actor.Account.toJSON() - if (avatarName !== undefined) { - if (actorInstance.avatarId) { - await actorInstance.Avatar.destroy({ transaction: t }) - } - - const avatar = await AvatarModel.create({ - filename: avatarName - }, { transaction: t }) + await updateActorInstance(actor, accountAttributesToUpdate) - actor.set('avatarId', avatar.id) + if (avatarName !== undefined) { + await updateActorAvatarInstance(actor, avatarName, t) } await actor.save({ transaction: t }) @@ -177,8 +150,8 @@ async function updateRemoteAccount (actor: ActorModel, activity: ActivityUpdate) logger.info('Remote account with uuid %s updated', accountAttributesToUpdate.uuid) } catch (err) { - if (actorInstance !== undefined && actorFieldsSave !== undefined) { - resetSequelizeInstance(actorInstance, actorFieldsSave) + if (actor !== undefined && actorFieldsSave !== undefined) { + resetSequelizeInstance(actor, actorFieldsSave) } if (accountInstance !== undefined && accountFieldsSave !== undefined) { diff --git a/server/models/activitypub/actor.ts b/server/models/activitypub/actor.ts index ed7fcfe27..707f140af 100644 --- a/server/models/activitypub/actor.ts +++ b/server/models/activitypub/actor.ts @@ -13,7 +13,7 @@ import { isActorPublicKeyValid } from '../../helpers/custom-validators/activitypub/actor' import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc' -import { ACTIVITY_PUB_ACTOR_TYPES, CONFIG, CONSTRAINTS_FIELDS } from '../../initializers' +import { ACTIVITY_PUB, ACTIVITY_PUB_ACTOR_TYPES, CONFIG, CONSTRAINTS_FIELDS } from '../../initializers' import { AccountModel } from '../account/account' import { AvatarModel } from '../avatar/avatar' import { ServerModel } from '../server/server' @@ -375,4 +375,15 @@ export class ActorModel extends Model { return CONFIG.WEBSERVER.URL + this.Avatar.getWebserverPath() } + + isOutdated () { + if (this.isOwned()) return false + + const now = Date.now() + const createdAtTime = this.createdAt.getTime() + const updatedAtTime = this.updatedAt.getTime() + + return (now - createdAtTime) > ACTIVITY_PUB.ACTOR_REFRESH_INTERVAL && + (now - updatedAtTime) > ACTIVITY_PUB.ACTOR_REFRESH_INTERVAL + } } -- cgit v1.2.3