From 744d0eca195bce7dafeb4a958d0eb3c0046be32d Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Mon, 14 Jan 2019 11:30:15 +0100 Subject: Refresh remote actors on GET enpoints --- server/controllers/api/accounts.ts | 7 ++ server/controllers/api/video-channel.ts | 6 ++ server/controllers/api/videos/index.ts | 2 +- server/lib/activitypub/actor.ts | 111 +++++++++++---------- server/lib/activitypub/videos.ts | 2 +- .../job-queue/handlers/activitypub-refresher.ts | 25 +++-- server/models/account/account.ts | 4 + server/models/video/video-channel.ts | 4 + 8 files changed, 99 insertions(+), 62 deletions(-) diff --git a/server/controllers/api/accounts.ts b/server/controllers/api/accounts.ts index a69a83acf..8c0237203 100644 --- a/server/controllers/api/accounts.ts +++ b/server/controllers/api/accounts.ts @@ -14,6 +14,8 @@ import { AccountModel } from '../../models/account/account' import { VideoModel } from '../../models/video/video' import { buildNSFWFilter, isUserAbleToSearchRemoteURI } from '../../helpers/express-utils' import { VideoChannelModel } from '../../models/video/video-channel' +import { JobQueue } from '../../lib/job-queue' +import { logger } from '../../helpers/logger' const accountsRouter = express.Router() @@ -57,6 +59,11 @@ export { function getAccount (req: express.Request, res: express.Response, next: express.NextFunction) { const account: AccountModel = res.locals.account + if (account.isOutdated()) { + JobQueue.Instance.createJob({ type: 'activitypub-refresher', payload: { type: 'actor', url: account.Actor.url } }) + .catch(err => logger.error('Cannot create AP refresher job for actor %s.', account.Actor.url, { err })) + } + return res.json(account.toFormattedJSON()) } diff --git a/server/controllers/api/video-channel.ts b/server/controllers/api/video-channel.ts index 3d6a6af7f..db7602139 100644 --- a/server/controllers/api/video-channel.ts +++ b/server/controllers/api/video-channel.ts @@ -30,6 +30,7 @@ import { updateActorAvatarFile } from '../../lib/avatar' import { auditLoggerFactory, getAuditIdFromRes, VideoChannelAuditView } from '../../helpers/audit-logger' import { resetSequelizeInstance } from '../../helpers/database-utils' import { UserModel } from '../../models/account/user' +import { JobQueue } from '../../lib/job-queue' const auditLogger = auditLoggerFactory('channels') const reqAvatarFile = createReqFiles([ 'avatarfile' ], MIMETYPES.IMAGE.MIMETYPE_EXT, { avatarfile: CONFIG.STORAGE.TMP_DIR }) @@ -197,6 +198,11 @@ async function removeVideoChannel (req: express.Request, res: express.Response) async function getVideoChannel (req: express.Request, res: express.Response, next: express.NextFunction) { const videoChannelWithVideos = await VideoChannelModel.loadAndPopulateAccountAndVideos(res.locals.videoChannel.id) + if (videoChannelWithVideos.isOutdated()) { + JobQueue.Instance.createJob({ type: 'activitypub-refresher', payload: { type: 'actor', url: videoChannelWithVideos.Actor.url } }) + .catch(err => logger.error('Cannot create AP refresher job for actor %s.', videoChannelWithVideos.Actor.url, { err })) + } + return res.json(videoChannelWithVideos.toFormattedJSON()) } diff --git a/server/controllers/api/videos/index.ts b/server/controllers/api/videos/index.ts index 28ac26598..2b2dfa7ca 100644 --- a/server/controllers/api/videos/index.ts +++ b/server/controllers/api/videos/index.ts @@ -399,7 +399,7 @@ function getVideo (req: express.Request, res: express.Response) { const videoInstance = res.locals.video if (videoInstance.isOutdated()) { - JobQueue.Instance.createJob({ type: 'activitypub-refresher', payload: { type: 'video', videoUrl: videoInstance.url } }) + JobQueue.Instance.createJob({ type: 'activitypub-refresher', payload: { type: 'video', url: videoInstance.url } }) .catch(err => logger.error('Cannot create AP refresher job for video %s.', videoInstance.url, { err })) } diff --git a/server/lib/activitypub/actor.ts b/server/lib/activitypub/actor.ts index f80296725..d728c81d1 100644 --- a/server/lib/activitypub/actor.ts +++ b/server/lib/activitypub/actor.ts @@ -201,6 +201,62 @@ async function addFetchOutboxJob (actor: ActorModel) { return JobQueue.Instance.createJob({ type: 'activitypub-http-fetcher', payload }) } +async function refreshActorIfNeeded ( + actorArg: ActorModel, + fetchedType: ActorFetchByUrlType +): Promise<{ actor: ActorModel, refreshed: boolean }> { + if (!actorArg.isOutdated()) return { actor: actorArg, refreshed: false } + + // We need more attributes + const actor = fetchedType === 'all' ? actorArg : await ActorModel.loadByUrlAndPopulateAccountAndChannel(actorArg.url) + + try { + const actorUrl = await getUrlFromWebfinger(actor.preferredUsername + '@' + actor.getHost()) + const { result, statusCode } = await fetchRemoteActor(actorUrl) + + if (statusCode === 404) { + logger.info('Deleting actor %s because there is a 404 in refresh actor.', actor.url) + actor.Account ? actor.Account.destroy() : actor.VideoChannel.destroy() + return { actor: undefined, refreshed: false } + } + + if (result === undefined) { + logger.warn('Cannot fetch remote actor in refresh actor.') + return { actor, refreshed: false } + } + + return sequelizeTypescript.transaction(async t => { + updateInstanceWithAnother(actor, result.actor) + + if (result.avatarName !== undefined) { + await updateActorAvatarInstance(actor, result.avatarName, t) + } + + // Force update + actor.setDataValue('updatedAt', new Date()) + await actor.save({ transaction: t }) + + if (actor.Account) { + actor.Account.set('name', result.name) + actor.Account.set('description', result.summary) + + await actor.Account.save({ transaction: t }) + } else if (actor.VideoChannel) { + actor.VideoChannel.set('name', result.name) + actor.VideoChannel.set('description', result.summary) + actor.VideoChannel.set('support', result.support) + + await actor.VideoChannel.save({ transaction: t }) + } + + return { refreshed: true, actor } + }) + } catch (err) { + logger.warn('Cannot refresh actor.', { err }) + return { actor, refreshed: false } + } +} + export { getOrCreateActorAndServerAndModel, buildActorInstance, @@ -208,6 +264,7 @@ export { fetchActorTotalItems, fetchAvatarIfExists, updateActorInstance, + refreshActorIfNeeded, updateActorAvatarInstance, addFetchOutboxJob } @@ -373,58 +430,4 @@ async function saveVideoChannel (actor: ActorModel, result: FetchRemoteActorResu return videoChannelCreated } -async function refreshActorIfNeeded ( - actorArg: ActorModel, - fetchedType: ActorFetchByUrlType -): Promise<{ actor: ActorModel, refreshed: boolean }> { - if (!actorArg.isOutdated()) return { actor: actorArg, refreshed: false } - - // We need more attributes - const actor = fetchedType === 'all' ? actorArg : await ActorModel.loadByUrlAndPopulateAccountAndChannel(actorArg.url) - - try { - const actorUrl = await getUrlFromWebfinger(actor.preferredUsername + '@' + actor.getHost()) - const { result, statusCode } = await fetchRemoteActor(actorUrl) - - if (statusCode === 404) { - logger.info('Deleting actor %s because there is a 404 in refresh actor.', actor.url) - actor.Account ? actor.Account.destroy() : actor.VideoChannel.destroy() - return { actor: undefined, refreshed: false } - } - - if (result === undefined) { - logger.warn('Cannot fetch remote actor in refresh actor.') - return { actor, refreshed: false } - } - return sequelizeTypescript.transaction(async t => { - updateInstanceWithAnother(actor, result.actor) - - if (result.avatarName !== undefined) { - await updateActorAvatarInstance(actor, result.avatarName, t) - } - - // Force update - actor.setDataValue('updatedAt', new Date()) - await actor.save({ transaction: t }) - - if (actor.Account) { - actor.Account.set('name', result.name) - actor.Account.set('description', result.summary) - - await actor.Account.save({ transaction: t }) - } else if (actor.VideoChannel) { - actor.VideoChannel.set('name', result.name) - actor.VideoChannel.set('description', result.summary) - actor.VideoChannel.set('support', result.support) - - await actor.VideoChannel.save({ transaction: t }) - } - - return { refreshed: true, actor } - }) - } catch (err) { - logger.warn('Cannot refresh actor.', { err }) - return { actor, refreshed: false } - } -} diff --git a/server/lib/activitypub/videos.ts b/server/lib/activitypub/videos.ts index 893768769..cbdd981c5 100644 --- a/server/lib/activitypub/videos.ts +++ b/server/lib/activitypub/videos.ts @@ -179,7 +179,7 @@ async function getOrCreateVideoAndAccountAndChannel (options: { } if (syncParam.refreshVideo === true) videoFromDatabase = await refreshVideoIfNeeded(refreshOptions) - else await JobQueue.Instance.createJob({ type: 'activitypub-refresher', payload: { type: 'video', videoUrl: videoFromDatabase.url } }) + else await JobQueue.Instance.createJob({ type: 'activitypub-refresher', payload: { type: 'video', url: videoFromDatabase.url } }) } return { video: videoFromDatabase, created: false } diff --git a/server/lib/job-queue/handlers/activitypub-refresher.ts b/server/lib/job-queue/handlers/activitypub-refresher.ts index 671b0f487..454b975fe 100644 --- a/server/lib/job-queue/handlers/activitypub-refresher.ts +++ b/server/lib/job-queue/handlers/activitypub-refresher.ts @@ -1,30 +1,33 @@ import * as Bull from 'bull' import { logger } from '../../../helpers/logger' import { fetchVideoByUrl } from '../../../helpers/video' -import { refreshVideoIfNeeded } from '../../activitypub' +import { refreshVideoIfNeeded, refreshActorIfNeeded } from '../../activitypub' +import { ActorModel } from '../../../models/activitypub/actor' export type RefreshPayload = { - videoUrl: string - type: 'video' + type: 'video' | 'actor' + url: string } async function refreshAPObject (job: Bull.Job) { const payload = job.data as RefreshPayload - logger.info('Processing AP refresher in job %d for video %s.', job.id, payload.videoUrl) + logger.info('Processing AP refresher in job %d for %s.', job.id, payload.url) - if (payload.type === 'video') return refreshAPVideo(payload.videoUrl) + if (payload.type === 'video') return refreshVideo(payload.url) + if (payload.type === 'actor') return refreshActor(payload.url) } // --------------------------------------------------------------------------- export { + refreshActor, refreshAPObject } // --------------------------------------------------------------------------- -async function refreshAPVideo (videoUrl: string) { +async function refreshVideo (videoUrl: string) { const fetchType = 'all' as 'all' const syncParam = { likes: true, dislikes: true, shares: true, comments: true, thumbnail: true } @@ -39,3 +42,13 @@ async function refreshAPVideo (videoUrl: string) { await refreshVideoIfNeeded(refreshOptions) } } + +async function refreshActor (actorUrl: string) { + const fetchType = 'all' as 'all' + const actor = await ActorModel.loadByUrlAndPopulateAccountAndChannel(actorUrl) + + if (actor) { + await refreshActorIfNeeded(actor, fetchType) + } + +} diff --git a/server/models/account/account.ts b/server/models/account/account.ts index a99e9b1ad..84ef0b30d 100644 --- a/server/models/account/account.ts +++ b/server/models/account/account.ts @@ -288,6 +288,10 @@ export class AccountModel extends Model { return this.Actor.isOwned() } + isOutdated () { + return this.Actor.isOutdated() + } + getDisplayName () { return this.name } diff --git a/server/models/video/video-channel.ts b/server/models/video/video-channel.ts index 86bf0461a..5598d80f6 100644 --- a/server/models/video/video-channel.ts +++ b/server/models/video/video-channel.ts @@ -470,4 +470,8 @@ export class VideoChannelModel extends Model { getDisplayName () { return this.name } + + isOutdated () { + return this.Actor.isOutdated() + } } -- cgit v1.2.3