diff options
-rw-r--r-- | server/controllers/api/accounts.ts | 7 | ||||
-rw-r--r-- | server/controllers/api/video-channel.ts | 6 | ||||
-rw-r--r-- | server/controllers/api/videos/index.ts | 2 | ||||
-rw-r--r-- | server/lib/activitypub/actor.ts | 111 | ||||
-rw-r--r-- | server/lib/activitypub/videos.ts | 2 | ||||
-rw-r--r-- | server/lib/job-queue/handlers/activitypub-refresher.ts | 25 | ||||
-rw-r--r-- | server/models/account/account.ts | 4 | ||||
-rw-r--r-- | server/models/video/video-channel.ts | 4 |
8 files changed, 99 insertions, 62 deletions
diff --git a/server/controllers/api/accounts.ts b/server/controllers/api/accounts.ts index a69a83acf..8c0237203 100644 --- a/server/controllers/api/accounts.ts +++ b/server/controllers/api/accounts.ts | |||
@@ -14,6 +14,8 @@ import { AccountModel } from '../../models/account/account' | |||
14 | import { VideoModel } from '../../models/video/video' | 14 | import { VideoModel } from '../../models/video/video' |
15 | import { buildNSFWFilter, isUserAbleToSearchRemoteURI } from '../../helpers/express-utils' | 15 | import { buildNSFWFilter, isUserAbleToSearchRemoteURI } from '../../helpers/express-utils' |
16 | import { VideoChannelModel } from '../../models/video/video-channel' | 16 | import { VideoChannelModel } from '../../models/video/video-channel' |
17 | import { JobQueue } from '../../lib/job-queue' | ||
18 | import { logger } from '../../helpers/logger' | ||
17 | 19 | ||
18 | const accountsRouter = express.Router() | 20 | const accountsRouter = express.Router() |
19 | 21 | ||
@@ -57,6 +59,11 @@ export { | |||
57 | function getAccount (req: express.Request, res: express.Response, next: express.NextFunction) { | 59 | function getAccount (req: express.Request, res: express.Response, next: express.NextFunction) { |
58 | const account: AccountModel = res.locals.account | 60 | const account: AccountModel = res.locals.account |
59 | 61 | ||
62 | if (account.isOutdated()) { | ||
63 | JobQueue.Instance.createJob({ type: 'activitypub-refresher', payload: { type: 'actor', url: account.Actor.url } }) | ||
64 | .catch(err => logger.error('Cannot create AP refresher job for actor %s.', account.Actor.url, { err })) | ||
65 | } | ||
66 | |||
60 | return res.json(account.toFormattedJSON()) | 67 | return res.json(account.toFormattedJSON()) |
61 | } | 68 | } |
62 | 69 | ||
diff --git a/server/controllers/api/video-channel.ts b/server/controllers/api/video-channel.ts index 3d6a6af7f..db7602139 100644 --- a/server/controllers/api/video-channel.ts +++ b/server/controllers/api/video-channel.ts | |||
@@ -30,6 +30,7 @@ import { updateActorAvatarFile } from '../../lib/avatar' | |||
30 | import { auditLoggerFactory, getAuditIdFromRes, VideoChannelAuditView } from '../../helpers/audit-logger' | 30 | import { auditLoggerFactory, getAuditIdFromRes, VideoChannelAuditView } from '../../helpers/audit-logger' |
31 | import { resetSequelizeInstance } from '../../helpers/database-utils' | 31 | import { resetSequelizeInstance } from '../../helpers/database-utils' |
32 | import { UserModel } from '../../models/account/user' | 32 | import { UserModel } from '../../models/account/user' |
33 | import { JobQueue } from '../../lib/job-queue' | ||
33 | 34 | ||
34 | const auditLogger = auditLoggerFactory('channels') | 35 | const auditLogger = auditLoggerFactory('channels') |
35 | const reqAvatarFile = createReqFiles([ 'avatarfile' ], MIMETYPES.IMAGE.MIMETYPE_EXT, { avatarfile: CONFIG.STORAGE.TMP_DIR }) | 36 | const reqAvatarFile = createReqFiles([ 'avatarfile' ], MIMETYPES.IMAGE.MIMETYPE_EXT, { avatarfile: CONFIG.STORAGE.TMP_DIR }) |
@@ -197,6 +198,11 @@ async function removeVideoChannel (req: express.Request, res: express.Response) | |||
197 | async function getVideoChannel (req: express.Request, res: express.Response, next: express.NextFunction) { | 198 | async function getVideoChannel (req: express.Request, res: express.Response, next: express.NextFunction) { |
198 | const videoChannelWithVideos = await VideoChannelModel.loadAndPopulateAccountAndVideos(res.locals.videoChannel.id) | 199 | const videoChannelWithVideos = await VideoChannelModel.loadAndPopulateAccountAndVideos(res.locals.videoChannel.id) |
199 | 200 | ||
201 | if (videoChannelWithVideos.isOutdated()) { | ||
202 | JobQueue.Instance.createJob({ type: 'activitypub-refresher', payload: { type: 'actor', url: videoChannelWithVideos.Actor.url } }) | ||
203 | .catch(err => logger.error('Cannot create AP refresher job for actor %s.', videoChannelWithVideos.Actor.url, { err })) | ||
204 | } | ||
205 | |||
200 | return res.json(videoChannelWithVideos.toFormattedJSON()) | 206 | return res.json(videoChannelWithVideos.toFormattedJSON()) |
201 | } | 207 | } |
202 | 208 | ||
diff --git a/server/controllers/api/videos/index.ts b/server/controllers/api/videos/index.ts index 28ac26598..2b2dfa7ca 100644 --- a/server/controllers/api/videos/index.ts +++ b/server/controllers/api/videos/index.ts | |||
@@ -399,7 +399,7 @@ function getVideo (req: express.Request, res: express.Response) { | |||
399 | const videoInstance = res.locals.video | 399 | const videoInstance = res.locals.video |
400 | 400 | ||
401 | if (videoInstance.isOutdated()) { | 401 | if (videoInstance.isOutdated()) { |
402 | JobQueue.Instance.createJob({ type: 'activitypub-refresher', payload: { type: 'video', videoUrl: videoInstance.url } }) | 402 | JobQueue.Instance.createJob({ type: 'activitypub-refresher', payload: { type: 'video', url: videoInstance.url } }) |
403 | .catch(err => logger.error('Cannot create AP refresher job for video %s.', videoInstance.url, { err })) | 403 | .catch(err => logger.error('Cannot create AP refresher job for video %s.', videoInstance.url, { err })) |
404 | } | 404 | } |
405 | 405 | ||
diff --git a/server/lib/activitypub/actor.ts b/server/lib/activitypub/actor.ts index f80296725..d728c81d1 100644 --- a/server/lib/activitypub/actor.ts +++ b/server/lib/activitypub/actor.ts | |||
@@ -201,6 +201,62 @@ async function addFetchOutboxJob (actor: ActorModel) { | |||
201 | return JobQueue.Instance.createJob({ type: 'activitypub-http-fetcher', payload }) | 201 | return JobQueue.Instance.createJob({ type: 'activitypub-http-fetcher', payload }) |
202 | } | 202 | } |
203 | 203 | ||
204 | async function refreshActorIfNeeded ( | ||
205 | actorArg: ActorModel, | ||
206 | fetchedType: ActorFetchByUrlType | ||
207 | ): Promise<{ actor: ActorModel, refreshed: boolean }> { | ||
208 | if (!actorArg.isOutdated()) return { actor: actorArg, refreshed: false } | ||
209 | |||
210 | // We need more attributes | ||
211 | const actor = fetchedType === 'all' ? actorArg : await ActorModel.loadByUrlAndPopulateAccountAndChannel(actorArg.url) | ||
212 | |||
213 | try { | ||
214 | const actorUrl = await getUrlFromWebfinger(actor.preferredUsername + '@' + actor.getHost()) | ||
215 | const { result, statusCode } = await fetchRemoteActor(actorUrl) | ||
216 | |||
217 | if (statusCode === 404) { | ||
218 | logger.info('Deleting actor %s because there is a 404 in refresh actor.', actor.url) | ||
219 | actor.Account ? actor.Account.destroy() : actor.VideoChannel.destroy() | ||
220 | return { actor: undefined, refreshed: false } | ||
221 | } | ||
222 | |||
223 | if (result === undefined) { | ||
224 | logger.warn('Cannot fetch remote actor in refresh actor.') | ||
225 | return { actor, refreshed: false } | ||
226 | } | ||
227 | |||
228 | return sequelizeTypescript.transaction(async t => { | ||
229 | updateInstanceWithAnother(actor, result.actor) | ||
230 | |||
231 | if (result.avatarName !== undefined) { | ||
232 | await updateActorAvatarInstance(actor, result.avatarName, t) | ||
233 | } | ||
234 | |||
235 | // Force update | ||
236 | actor.setDataValue('updatedAt', new Date()) | ||
237 | await actor.save({ transaction: t }) | ||
238 | |||
239 | if (actor.Account) { | ||
240 | actor.Account.set('name', result.name) | ||
241 | actor.Account.set('description', result.summary) | ||
242 | |||
243 | await actor.Account.save({ transaction: t }) | ||
244 | } else if (actor.VideoChannel) { | ||
245 | actor.VideoChannel.set('name', result.name) | ||
246 | actor.VideoChannel.set('description', result.summary) | ||
247 | actor.VideoChannel.set('support', result.support) | ||
248 | |||
249 | await actor.VideoChannel.save({ transaction: t }) | ||
250 | } | ||
251 | |||
252 | return { refreshed: true, actor } | ||
253 | }) | ||
254 | } catch (err) { | ||
255 | logger.warn('Cannot refresh actor.', { err }) | ||
256 | return { actor, refreshed: false } | ||
257 | } | ||
258 | } | ||
259 | |||
204 | export { | 260 | export { |
205 | getOrCreateActorAndServerAndModel, | 261 | getOrCreateActorAndServerAndModel, |
206 | buildActorInstance, | 262 | buildActorInstance, |
@@ -208,6 +264,7 @@ export { | |||
208 | fetchActorTotalItems, | 264 | fetchActorTotalItems, |
209 | fetchAvatarIfExists, | 265 | fetchAvatarIfExists, |
210 | updateActorInstance, | 266 | updateActorInstance, |
267 | refreshActorIfNeeded, | ||
211 | updateActorAvatarInstance, | 268 | updateActorAvatarInstance, |
212 | addFetchOutboxJob | 269 | addFetchOutboxJob |
213 | } | 270 | } |
@@ -373,58 +430,4 @@ async function saveVideoChannel (actor: ActorModel, result: FetchRemoteActorResu | |||
373 | return videoChannelCreated | 430 | return videoChannelCreated |
374 | } | 431 | } |
375 | 432 | ||
376 | async function refreshActorIfNeeded ( | ||
377 | actorArg: ActorModel, | ||
378 | fetchedType: ActorFetchByUrlType | ||
379 | ): Promise<{ actor: ActorModel, refreshed: boolean }> { | ||
380 | if (!actorArg.isOutdated()) return { actor: actorArg, refreshed: false } | ||
381 | |||
382 | // We need more attributes | ||
383 | const actor = fetchedType === 'all' ? actorArg : await ActorModel.loadByUrlAndPopulateAccountAndChannel(actorArg.url) | ||
384 | |||
385 | try { | ||
386 | const actorUrl = await getUrlFromWebfinger(actor.preferredUsername + '@' + actor.getHost()) | ||
387 | const { result, statusCode } = await fetchRemoteActor(actorUrl) | ||
388 | |||
389 | if (statusCode === 404) { | ||
390 | logger.info('Deleting actor %s because there is a 404 in refresh actor.', actor.url) | ||
391 | actor.Account ? actor.Account.destroy() : actor.VideoChannel.destroy() | ||
392 | return { actor: undefined, refreshed: false } | ||
393 | } | ||
394 | |||
395 | if (result === undefined) { | ||
396 | logger.warn('Cannot fetch remote actor in refresh actor.') | ||
397 | return { actor, refreshed: false } | ||
398 | } | ||
399 | 433 | ||
400 | return sequelizeTypescript.transaction(async t => { | ||
401 | updateInstanceWithAnother(actor, result.actor) | ||
402 | |||
403 | if (result.avatarName !== undefined) { | ||
404 | await updateActorAvatarInstance(actor, result.avatarName, t) | ||
405 | } | ||
406 | |||
407 | // Force update | ||
408 | actor.setDataValue('updatedAt', new Date()) | ||
409 | await actor.save({ transaction: t }) | ||
410 | |||
411 | if (actor.Account) { | ||
412 | actor.Account.set('name', result.name) | ||
413 | actor.Account.set('description', result.summary) | ||
414 | |||
415 | await actor.Account.save({ transaction: t }) | ||
416 | } else if (actor.VideoChannel) { | ||
417 | actor.VideoChannel.set('name', result.name) | ||
418 | actor.VideoChannel.set('description', result.summary) | ||
419 | actor.VideoChannel.set('support', result.support) | ||
420 | |||
421 | await actor.VideoChannel.save({ transaction: t }) | ||
422 | } | ||
423 | |||
424 | return { refreshed: true, actor } | ||
425 | }) | ||
426 | } catch (err) { | ||
427 | logger.warn('Cannot refresh actor.', { err }) | ||
428 | return { actor, refreshed: false } | ||
429 | } | ||
430 | } | ||
diff --git a/server/lib/activitypub/videos.ts b/server/lib/activitypub/videos.ts index 893768769..cbdd981c5 100644 --- a/server/lib/activitypub/videos.ts +++ b/server/lib/activitypub/videos.ts | |||
@@ -179,7 +179,7 @@ async function getOrCreateVideoAndAccountAndChannel (options: { | |||
179 | } | 179 | } |
180 | 180 | ||
181 | if (syncParam.refreshVideo === true) videoFromDatabase = await refreshVideoIfNeeded(refreshOptions) | 181 | if (syncParam.refreshVideo === true) videoFromDatabase = await refreshVideoIfNeeded(refreshOptions) |
182 | else await JobQueue.Instance.createJob({ type: 'activitypub-refresher', payload: { type: 'video', videoUrl: videoFromDatabase.url } }) | 182 | else await JobQueue.Instance.createJob({ type: 'activitypub-refresher', payload: { type: 'video', url: videoFromDatabase.url } }) |
183 | } | 183 | } |
184 | 184 | ||
185 | return { video: videoFromDatabase, created: false } | 185 | return { video: videoFromDatabase, created: false } |
diff --git a/server/lib/job-queue/handlers/activitypub-refresher.ts b/server/lib/job-queue/handlers/activitypub-refresher.ts index 671b0f487..454b975fe 100644 --- a/server/lib/job-queue/handlers/activitypub-refresher.ts +++ b/server/lib/job-queue/handlers/activitypub-refresher.ts | |||
@@ -1,30 +1,33 @@ | |||
1 | import * as Bull from 'bull' | 1 | import * as Bull from 'bull' |
2 | import { logger } from '../../../helpers/logger' | 2 | import { logger } from '../../../helpers/logger' |
3 | import { fetchVideoByUrl } from '../../../helpers/video' | 3 | import { fetchVideoByUrl } from '../../../helpers/video' |
4 | import { refreshVideoIfNeeded } from '../../activitypub' | 4 | import { refreshVideoIfNeeded, refreshActorIfNeeded } from '../../activitypub' |
5 | import { ActorModel } from '../../../models/activitypub/actor' | ||
5 | 6 | ||
6 | export type RefreshPayload = { | 7 | export type RefreshPayload = { |
7 | videoUrl: string | 8 | type: 'video' | 'actor' |
8 | type: 'video' | 9 | url: string |
9 | } | 10 | } |
10 | 11 | ||
11 | async function refreshAPObject (job: Bull.Job) { | 12 | async function refreshAPObject (job: Bull.Job) { |
12 | const payload = job.data as RefreshPayload | 13 | const payload = job.data as RefreshPayload |
13 | 14 | ||
14 | logger.info('Processing AP refresher in job %d for video %s.', job.id, payload.videoUrl) | 15 | logger.info('Processing AP refresher in job %d for %s.', job.id, payload.url) |
15 | 16 | ||
16 | if (payload.type === 'video') return refreshAPVideo(payload.videoUrl) | 17 | if (payload.type === 'video') return refreshVideo(payload.url) |
18 | if (payload.type === 'actor') return refreshActor(payload.url) | ||
17 | } | 19 | } |
18 | 20 | ||
19 | // --------------------------------------------------------------------------- | 21 | // --------------------------------------------------------------------------- |
20 | 22 | ||
21 | export { | 23 | export { |
24 | refreshActor, | ||
22 | refreshAPObject | 25 | refreshAPObject |
23 | } | 26 | } |
24 | 27 | ||
25 | // --------------------------------------------------------------------------- | 28 | // --------------------------------------------------------------------------- |
26 | 29 | ||
27 | async function refreshAPVideo (videoUrl: string) { | 30 | async function refreshVideo (videoUrl: string) { |
28 | const fetchType = 'all' as 'all' | 31 | const fetchType = 'all' as 'all' |
29 | const syncParam = { likes: true, dislikes: true, shares: true, comments: true, thumbnail: true } | 32 | const syncParam = { likes: true, dislikes: true, shares: true, comments: true, thumbnail: true } |
30 | 33 | ||
@@ -39,3 +42,13 @@ async function refreshAPVideo (videoUrl: string) { | |||
39 | await refreshVideoIfNeeded(refreshOptions) | 42 | await refreshVideoIfNeeded(refreshOptions) |
40 | } | 43 | } |
41 | } | 44 | } |
45 | |||
46 | async function refreshActor (actorUrl: string) { | ||
47 | const fetchType = 'all' as 'all' | ||
48 | const actor = await ActorModel.loadByUrlAndPopulateAccountAndChannel(actorUrl) | ||
49 | |||
50 | if (actor) { | ||
51 | await refreshActorIfNeeded(actor, fetchType) | ||
52 | } | ||
53 | |||
54 | } | ||
diff --git a/server/models/account/account.ts b/server/models/account/account.ts index a99e9b1ad..84ef0b30d 100644 --- a/server/models/account/account.ts +++ b/server/models/account/account.ts | |||
@@ -288,6 +288,10 @@ export class AccountModel extends Model<AccountModel> { | |||
288 | return this.Actor.isOwned() | 288 | return this.Actor.isOwned() |
289 | } | 289 | } |
290 | 290 | ||
291 | isOutdated () { | ||
292 | return this.Actor.isOutdated() | ||
293 | } | ||
294 | |||
291 | getDisplayName () { | 295 | getDisplayName () { |
292 | return this.name | 296 | return this.name |
293 | } | 297 | } |
diff --git a/server/models/video/video-channel.ts b/server/models/video/video-channel.ts index 86bf0461a..5598d80f6 100644 --- a/server/models/video/video-channel.ts +++ b/server/models/video/video-channel.ts | |||
@@ -470,4 +470,8 @@ export class VideoChannelModel extends Model<VideoChannelModel> { | |||
470 | getDisplayName () { | 470 | getDisplayName () { |
471 | return this.name | 471 | return this.name |
472 | } | 472 | } |
473 | |||
474 | isOutdated () { | ||
475 | return this.Actor.isOutdated() | ||
476 | } | ||
473 | } | 477 | } |