1 import { map } from 'bluebird'
2 import { Job } from 'bull'
3 import { checkUrlsSameHost } from '@server/helpers/activitypub'
5 isAnnounceActivityValid,
6 isDislikeActivityValid,
8 } from '@server/helpers/custom-validators/activitypub/activity'
9 import { sanitizeAndCheckVideoCommentObject } from '@server/helpers/custom-validators/activitypub/video-comments'
10 import { doJSONRequest, PeerTubeRequestError } from '@server/helpers/requests'
11 import { AP_CLEANER } from '@server/initializers/constants'
12 import { Redis } from '@server/lib/redis'
13 import { VideoModel } from '@server/models/video/video'
14 import { VideoCommentModel } from '@server/models/video/video-comment'
15 import { VideoShareModel } from '@server/models/video/video-share'
16 import { HttpStatusCode } from '@shared/models'
17 import { logger, loggerTagsFactory } from '../../../helpers/logger'
18 import { AccountVideoRateModel } from '../../../models/account/account-video-rate'
20 const lTags = loggerTagsFactory('ap-cleaner')
22 // Job to clean remote interactions off local videos
24 async function processActivityPubCleaner (_job: Job) {
25 logger.info('Processing ActivityPub cleaner.', lTags())
28 const rateUrls = await AccountVideoRateModel.listRemoteRateUrlsOfLocalVideos()
29 const { bodyValidator, deleter, updater } = rateOptionsFactory()
31 await map(rateUrls, async rateUrl => {
32 const result = await updateObjectIfNeeded({ url: rateUrl, bodyValidator, updater, deleter })
34 if (result?.status === 'deleted') {
35 const { videoId, type } = result.data
37 await VideoModel.updateRatesOf(videoId, type, undefined)
39 }, { concurrency: AP_CLEANER.CONCURRENCY })
43 const shareUrls = await VideoShareModel.listRemoteShareUrlsOfLocalVideos()
44 const { bodyValidator, deleter, updater } = shareOptionsFactory()
46 await map(shareUrls, async shareUrl => {
47 await updateObjectIfNeeded({ url: shareUrl, bodyValidator, updater, deleter })
48 }, { concurrency: AP_CLEANER.CONCURRENCY })
52 const commentUrls = await VideoCommentModel.listRemoteCommentUrlsOfLocalVideos()
53 const { bodyValidator, deleter, updater } = commentOptionsFactory()
55 await map(commentUrls, async commentUrl => {
56 await updateObjectIfNeeded({ url: commentUrl, bodyValidator, updater, deleter })
57 }, { concurrency: AP_CLEANER.CONCURRENCY })
61 // ---------------------------------------------------------------------------
64 processActivityPubCleaner
67 // ---------------------------------------------------------------------------
69 async function updateObjectIfNeeded <T> (options: {
71 bodyValidator: (body: any) => boolean
72 updater: (url: string, newUrl: string) => Promise<T>
73 deleter: (url: string) => Promise<T> }
74 ): Promise<{ data: T, status: 'deleted' | 'updated' } | null> {
75 const { url, bodyValidator, updater, deleter } = options
77 const on404OrTombstone = async () => {
78 logger.info('Removing remote AP object %s.', url, lTags(url))
79 const data = await deleter(url)
81 return { status: 'deleted' as 'deleted', data }
85 const { body } = await doJSONRequest<any>(url, { activityPub: true })
87 // If not same id, check same host and update
88 if (!body || !body.id || !bodyValidator(body)) throw new Error(`Body or body id of ${url} is invalid`)
90 if (body.type === 'Tombstone') {
91 return on404OrTombstone()
94 const newUrl = body.id
96 if (checkUrlsSameHost(newUrl, url) !== true) {
97 throw new Error(`New url ${newUrl} has not the same host than old url ${url}`)
100 logger.info('Updating remote AP object %s.', url, lTags(url))
101 const data = await updater(url, newUrl)
103 return { status: 'updated', data }
108 // Does not exist anymore, remove entry
109 if ((err as PeerTubeRequestError).statusCode === HttpStatusCode.NOT_FOUND_404) {
110 return on404OrTombstone()
113 logger.debug('Remote AP object %s is unavailable.', url, lTags(url))
115 const unavailability = await Redis.Instance.addAPUnavailability(url)
116 if (unavailability >= AP_CLEANER.UNAVAILABLE_TRESHOLD) {
117 logger.info('Removing unavailable AP resource %s.', url, lTags(url))
118 return on404OrTombstone()
125 function rateOptionsFactory () {
127 bodyValidator: (body: any) => isLikeActivityValid(body) || isDislikeActivityValid(body),
129 updater: async (url: string, newUrl: string) => {
130 const rate = await AccountVideoRateModel.loadByUrl(url, undefined)
133 const videoId = rate.videoId
134 const type = rate.type
138 return { videoId, type }
141 deleter: async (url) => {
142 const rate = await AccountVideoRateModel.loadByUrl(url, undefined)
144 const videoId = rate.videoId
145 const type = rate.type
149 return { videoId, type }
154 function shareOptionsFactory () {
156 bodyValidator: (body: any) => isAnnounceActivityValid(body),
158 updater: async (url: string, newUrl: string) => {
159 const share = await VideoShareModel.loadByUrl(url, undefined)
167 deleter: async (url) => {
168 const share = await VideoShareModel.loadByUrl(url, undefined)
170 await share.destroy()
177 function commentOptionsFactory () {
179 bodyValidator: (body: any) => sanitizeAndCheckVideoCommentObject(body),
181 updater: async (url: string, newUrl: string) => {
182 const comment = await VideoCommentModel.loadByUrlAndPopulateAccountAndVideo(url)
190 deleter: async (url) => {
191 const comment = await VideoCommentModel.loadByUrlAndPopulateAccountAndVideo(url)
193 await comment.destroy()