aboutsummaryrefslogtreecommitdiffhomepage
diff options
context:
space:
mode:
-rw-r--r--server/lib/job-queue/handlers/activitypub-cleaner.ts38
-rw-r--r--server/tests/api/activitypub/cleaner.ts15
-rw-r--r--server/tests/api/server/handle-down.ts2
3 files changed, 26 insertions, 29 deletions
diff --git a/server/lib/job-queue/handlers/activitypub-cleaner.ts b/server/lib/job-queue/handlers/activitypub-cleaner.ts
index 1540bf23a..509dd1cb5 100644
--- a/server/lib/job-queue/handlers/activitypub-cleaner.ts
+++ b/server/lib/job-queue/handlers/activitypub-cleaner.ts
@@ -14,29 +14,27 @@ import { VideoModel } from '@server/models/video/video'
14import { VideoCommentModel } from '@server/models/video/video-comment' 14import { VideoCommentModel } from '@server/models/video/video-comment'
15import { VideoShareModel } from '@server/models/video/video-share' 15import { VideoShareModel } from '@server/models/video/video-share'
16import { HttpStatusCode } from '@shared/models' 16import { HttpStatusCode } from '@shared/models'
17import { logger } from '../../../helpers/logger' 17import { logger, loggerTagsFactory } from '../../../helpers/logger'
18import { AccountVideoRateModel } from '../../../models/account/account-video-rate' 18import { AccountVideoRateModel } from '../../../models/account/account-video-rate'
19 19
20const lTags = loggerTagsFactory('ap-cleaner')
21
20// Job to clean remote interactions off local videos 22// Job to clean remote interactions off local videos
21 23
22async function processActivityPubCleaner (_job: Job) { 24async function processActivityPubCleaner (_job: Job) {
23 logger.info('Processing ActivityPub cleaner.') 25 logger.info('Processing ActivityPub cleaner.', lTags())
24 26
25 { 27 {
26 const rateUrls = await AccountVideoRateModel.listRemoteRateUrlsOfLocalVideos() 28 const rateUrls = await AccountVideoRateModel.listRemoteRateUrlsOfLocalVideos()
27 const { bodyValidator, deleter, updater } = rateOptionsFactory() 29 const { bodyValidator, deleter, updater } = rateOptionsFactory()
28 30
29 await map(rateUrls, async rateUrl => { 31 await map(rateUrls, async rateUrl => {
30 try { 32 const result = await updateObjectIfNeeded({ url: rateUrl, bodyValidator, updater, deleter })
31 const result = await updateObjectIfNeeded({ url: rateUrl, bodyValidator, updater, deleter })
32 33
33 if (result?.status === 'deleted') { 34 if (result?.status === 'deleted') {
34 const { videoId, type } = result.data 35 const { videoId, type } = result.data
35 36
36 await VideoModel.updateRatesOf(videoId, type, undefined) 37 await VideoModel.updateRatesOf(videoId, type, undefined)
37 }
38 } catch (err) {
39 logger.warn('Cannot update/delete remote AP rate %s.', rateUrl, { err })
40 } 38 }
41 }, { concurrency: AP_CLEANER.CONCURRENCY }) 39 }, { concurrency: AP_CLEANER.CONCURRENCY })
42 } 40 }
@@ -46,11 +44,7 @@ async function processActivityPubCleaner (_job: Job) {
46 const { bodyValidator, deleter, updater } = shareOptionsFactory() 44 const { bodyValidator, deleter, updater } = shareOptionsFactory()
47 45
48 await map(shareUrls, async shareUrl => { 46 await map(shareUrls, async shareUrl => {
49 try { 47 await updateObjectIfNeeded({ url: shareUrl, bodyValidator, updater, deleter })
50 await updateObjectIfNeeded({ url: shareUrl, bodyValidator, updater, deleter })
51 } catch (err) {
52 logger.warn('Cannot update/delete remote AP share %s.', shareUrl, { err })
53 }
54 }, { concurrency: AP_CLEANER.CONCURRENCY }) 48 }, { concurrency: AP_CLEANER.CONCURRENCY })
55 } 49 }
56 50
@@ -59,11 +53,7 @@ async function processActivityPubCleaner (_job: Job) {
59 const { bodyValidator, deleter, updater } = commentOptionsFactory() 53 const { bodyValidator, deleter, updater } = commentOptionsFactory()
60 54
61 await map(commentUrls, async commentUrl => { 55 await map(commentUrls, async commentUrl => {
62 try { 56 await updateObjectIfNeeded({ url: commentUrl, bodyValidator, updater, deleter })
63 await updateObjectIfNeeded({ url: commentUrl, bodyValidator, updater, deleter })
64 } catch (err) {
65 logger.warn('Cannot update/delete remote AP comment %s.', commentUrl, { err })
66 }
67 }, { concurrency: AP_CLEANER.CONCURRENCY }) 57 }, { concurrency: AP_CLEANER.CONCURRENCY })
68 } 58 }
69} 59}
@@ -85,7 +75,7 @@ async function updateObjectIfNeeded <T> (options: {
85 const { url, bodyValidator, updater, deleter } = options 75 const { url, bodyValidator, updater, deleter } = options
86 76
87 const on404OrTombstone = async () => { 77 const on404OrTombstone = async () => {
88 logger.info('Removing remote AP object %s.', url) 78 logger.info('Removing remote AP object %s.', url, lTags(url))
89 const data = await deleter(url) 79 const data = await deleter(url)
90 80
91 return { status: 'deleted' as 'deleted', data } 81 return { status: 'deleted' as 'deleted', data }
@@ -107,7 +97,7 @@ async function updateObjectIfNeeded <T> (options: {
107 throw new Error(`New url ${newUrl} has not the same host than old url ${url}`) 97 throw new Error(`New url ${newUrl} has not the same host than old url ${url}`)
108 } 98 }
109 99
110 logger.info('Updating remote AP object %s.', url) 100 logger.info('Updating remote AP object %s.', url, lTags(url))
111 const data = await updater(url, newUrl) 101 const data = await updater(url, newUrl)
112 102
113 return { status: 'updated', data } 103 return { status: 'updated', data }
@@ -120,11 +110,11 @@ async function updateObjectIfNeeded <T> (options: {
120 return on404OrTombstone() 110 return on404OrTombstone()
121 } 111 }
122 112
123 logger.debug('Remote AP object %s is unavailable.', url) 113 logger.debug('Remote AP object %s is unavailable.', url, lTags(url))
124 114
125 const unavailability = await Redis.Instance.addAPUnavailability(url) 115 const unavailability = await Redis.Instance.addAPUnavailability(url)
126 if (unavailability >= AP_CLEANER.UNAVAILABLE_TRESHOLD) { 116 if (unavailability >= AP_CLEANER.UNAVAILABLE_TRESHOLD) {
127 logger.info('Removing unavailable AP resource %s.', url) 117 logger.info('Removing unavailable AP resource %s.', url, lTags(url))
128 return on404OrTombstone() 118 return on404OrTombstone()
129 } 119 }
130 120
diff --git a/server/tests/api/activitypub/cleaner.ts b/server/tests/api/activitypub/cleaner.ts
index dc885023a..d0a151f5c 100644
--- a/server/tests/api/activitypub/cleaner.ts
+++ b/server/tests/api/activitypub/cleaner.ts
@@ -291,12 +291,12 @@ describe('Test AP cleaner', function () {
291 { 291 {
292 const video = await servers[0].videos.get({ id: uuid }) 292 const video = await servers[0].videos.get({ id: uuid })
293 293
294 expect(video.likes).to.equal(3) 294 expect(video.likes).to.equal(2)
295 expect(video.dislikes).to.equal(0) 295 expect(video.dislikes).to.equal(0)
296 } 296 }
297 297
298 { 298 {
299 const { total } = await servers[0].comments.listThreads({ videoId: videoUUID1 }) 299 const { total } = await servers[0].comments.listThreads({ videoId: uuid })
300 expect(total).to.equal(2) 300 expect(total).to.equal(2)
301 } 301 }
302 } 302 }
@@ -319,8 +319,15 @@ describe('Test AP cleaner', function () {
319 await wait(5000) 319 await wait(5000)
320 await expectNotDeleted() 320 await expectNotDeleted()
321 321
322 await wait(15000) 322 let continueWhile = true
323 await expectDeleted() 323
324 do {
325 try {
326 await expectDeleted()
327 continueWhile = false
328 } catch {
329 }
330 } while (continueWhile)
324 }) 331 })
325 332
326 after(async function () { 333 after(async function () {
diff --git a/server/tests/api/server/handle-down.ts b/server/tests/api/server/handle-down.ts
index 7c3836681..3dcd076f5 100644
--- a/server/tests/api/server/handle-down.ts
+++ b/server/tests/api/server/handle-down.ts
@@ -50,7 +50,7 @@ describe('Test handle downs', function () {
50 let commentCommands: CommentsCommand[] 50 let commentCommands: CommentsCommand[]
51 51
52 before(async function () { 52 before(async function () {
53 this.timeout(30000) 53 this.timeout(120000)
54 54
55 servers = await createMultipleServers(3) 55 servers = await createMultipleServers(3)
56 commentCommands = servers.map(s => s.comments) 56 commentCommands = servers.map(s => s.comments)