diff options
-rw-r--r-- | server/lib/job-queue/handlers/activitypub-cleaner.ts | 38 | ||||
-rw-r--r-- | server/tests/api/activitypub/cleaner.ts | 15 | ||||
-rw-r--r-- | server/tests/api/server/handle-down.ts | 2 |
3 files changed, 26 insertions, 29 deletions
diff --git a/server/lib/job-queue/handlers/activitypub-cleaner.ts b/server/lib/job-queue/handlers/activitypub-cleaner.ts index 1540bf23a..509dd1cb5 100644 --- a/server/lib/job-queue/handlers/activitypub-cleaner.ts +++ b/server/lib/job-queue/handlers/activitypub-cleaner.ts | |||
@@ -14,29 +14,27 @@ import { VideoModel } from '@server/models/video/video' | |||
14 | import { VideoCommentModel } from '@server/models/video/video-comment' | 14 | import { VideoCommentModel } from '@server/models/video/video-comment' |
15 | import { VideoShareModel } from '@server/models/video/video-share' | 15 | import { VideoShareModel } from '@server/models/video/video-share' |
16 | import { HttpStatusCode } from '@shared/models' | 16 | import { HttpStatusCode } from '@shared/models' |
17 | import { logger } from '../../../helpers/logger' | 17 | import { logger, loggerTagsFactory } from '../../../helpers/logger' |
18 | import { AccountVideoRateModel } from '../../../models/account/account-video-rate' | 18 | import { AccountVideoRateModel } from '../../../models/account/account-video-rate' |
19 | 19 | ||
20 | const lTags = loggerTagsFactory('ap-cleaner') | ||
21 | |||
20 | // Job to clean remote interactions off local videos | 22 | // Job to clean remote interactions off local videos |
21 | 23 | ||
22 | async function processActivityPubCleaner (_job: Job) { | 24 | async function processActivityPubCleaner (_job: Job) { |
23 | logger.info('Processing ActivityPub cleaner.') | 25 | logger.info('Processing ActivityPub cleaner.', lTags()) |
24 | 26 | ||
25 | { | 27 | { |
26 | const rateUrls = await AccountVideoRateModel.listRemoteRateUrlsOfLocalVideos() | 28 | const rateUrls = await AccountVideoRateModel.listRemoteRateUrlsOfLocalVideos() |
27 | const { bodyValidator, deleter, updater } = rateOptionsFactory() | 29 | const { bodyValidator, deleter, updater } = rateOptionsFactory() |
28 | 30 | ||
29 | await map(rateUrls, async rateUrl => { | 31 | await map(rateUrls, async rateUrl => { |
30 | try { | 32 | const result = await updateObjectIfNeeded({ url: rateUrl, bodyValidator, updater, deleter }) |
31 | const result = await updateObjectIfNeeded({ url: rateUrl, bodyValidator, updater, deleter }) | ||
32 | 33 | ||
33 | if (result?.status === 'deleted') { | 34 | if (result?.status === 'deleted') { |
34 | const { videoId, type } = result.data | 35 | const { videoId, type } = result.data |
35 | 36 | ||
36 | await VideoModel.updateRatesOf(videoId, type, undefined) | 37 | await VideoModel.updateRatesOf(videoId, type, undefined) |
37 | } | ||
38 | } catch (err) { | ||
39 | logger.warn('Cannot update/delete remote AP rate %s.', rateUrl, { err }) | ||
40 | } | 38 | } |
41 | }, { concurrency: AP_CLEANER.CONCURRENCY }) | 39 | }, { concurrency: AP_CLEANER.CONCURRENCY }) |
42 | } | 40 | } |
@@ -46,11 +44,7 @@ async function processActivityPubCleaner (_job: Job) { | |||
46 | const { bodyValidator, deleter, updater } = shareOptionsFactory() | 44 | const { bodyValidator, deleter, updater } = shareOptionsFactory() |
47 | 45 | ||
48 | await map(shareUrls, async shareUrl => { | 46 | await map(shareUrls, async shareUrl => { |
49 | try { | 47 | await updateObjectIfNeeded({ url: shareUrl, bodyValidator, updater, deleter }) |
50 | await updateObjectIfNeeded({ url: shareUrl, bodyValidator, updater, deleter }) | ||
51 | } catch (err) { | ||
52 | logger.warn('Cannot update/delete remote AP share %s.', shareUrl, { err }) | ||
53 | } | ||
54 | }, { concurrency: AP_CLEANER.CONCURRENCY }) | 48 | }, { concurrency: AP_CLEANER.CONCURRENCY }) |
55 | } | 49 | } |
56 | 50 | ||
@@ -59,11 +53,7 @@ async function processActivityPubCleaner (_job: Job) { | |||
59 | const { bodyValidator, deleter, updater } = commentOptionsFactory() | 53 | const { bodyValidator, deleter, updater } = commentOptionsFactory() |
60 | 54 | ||
61 | await map(commentUrls, async commentUrl => { | 55 | await map(commentUrls, async commentUrl => { |
62 | try { | 56 | await updateObjectIfNeeded({ url: commentUrl, bodyValidator, updater, deleter }) |
63 | await updateObjectIfNeeded({ url: commentUrl, bodyValidator, updater, deleter }) | ||
64 | } catch (err) { | ||
65 | logger.warn('Cannot update/delete remote AP comment %s.', commentUrl, { err }) | ||
66 | } | ||
67 | }, { concurrency: AP_CLEANER.CONCURRENCY }) | 57 | }, { concurrency: AP_CLEANER.CONCURRENCY }) |
68 | } | 58 | } |
69 | } | 59 | } |
@@ -85,7 +75,7 @@ async function updateObjectIfNeeded <T> (options: { | |||
85 | const { url, bodyValidator, updater, deleter } = options | 75 | const { url, bodyValidator, updater, deleter } = options |
86 | 76 | ||
87 | const on404OrTombstone = async () => { | 77 | const on404OrTombstone = async () => { |
88 | logger.info('Removing remote AP object %s.', url) | 78 | logger.info('Removing remote AP object %s.', url, lTags(url)) |
89 | const data = await deleter(url) | 79 | const data = await deleter(url) |
90 | 80 | ||
91 | return { status: 'deleted' as 'deleted', data } | 81 | return { status: 'deleted' as 'deleted', data } |
@@ -107,7 +97,7 @@ async function updateObjectIfNeeded <T> (options: { | |||
107 | throw new Error(`New url ${newUrl} has not the same host than old url ${url}`) | 97 | throw new Error(`New url ${newUrl} has not the same host than old url ${url}`) |
108 | } | 98 | } |
109 | 99 | ||
110 | logger.info('Updating remote AP object %s.', url) | 100 | logger.info('Updating remote AP object %s.', url, lTags(url)) |
111 | const data = await updater(url, newUrl) | 101 | const data = await updater(url, newUrl) |
112 | 102 | ||
113 | return { status: 'updated', data } | 103 | return { status: 'updated', data } |
@@ -120,11 +110,11 @@ async function updateObjectIfNeeded <T> (options: { | |||
120 | return on404OrTombstone() | 110 | return on404OrTombstone() |
121 | } | 111 | } |
122 | 112 | ||
123 | logger.debug('Remote AP object %s is unavailable.', url) | 113 | logger.debug('Remote AP object %s is unavailable.', url, lTags(url)) |
124 | 114 | ||
125 | const unavailability = await Redis.Instance.addAPUnavailability(url) | 115 | const unavailability = await Redis.Instance.addAPUnavailability(url) |
126 | if (unavailability >= AP_CLEANER.UNAVAILABLE_TRESHOLD) { | 116 | if (unavailability >= AP_CLEANER.UNAVAILABLE_TRESHOLD) { |
127 | logger.info('Removing unavailable AP resource %s.', url) | 117 | logger.info('Removing unavailable AP resource %s.', url, lTags(url)) |
128 | return on404OrTombstone() | 118 | return on404OrTombstone() |
129 | } | 119 | } |
130 | 120 | ||
diff --git a/server/tests/api/activitypub/cleaner.ts b/server/tests/api/activitypub/cleaner.ts index dc885023a..d0a151f5c 100644 --- a/server/tests/api/activitypub/cleaner.ts +++ b/server/tests/api/activitypub/cleaner.ts | |||
@@ -291,12 +291,12 @@ describe('Test AP cleaner', function () { | |||
291 | { | 291 | { |
292 | const video = await servers[0].videos.get({ id: uuid }) | 292 | const video = await servers[0].videos.get({ id: uuid }) |
293 | 293 | ||
294 | expect(video.likes).to.equal(3) | 294 | expect(video.likes).to.equal(2) |
295 | expect(video.dislikes).to.equal(0) | 295 | expect(video.dislikes).to.equal(0) |
296 | } | 296 | } |
297 | 297 | ||
298 | { | 298 | { |
299 | const { total } = await servers[0].comments.listThreads({ videoId: videoUUID1 }) | 299 | const { total } = await servers[0].comments.listThreads({ videoId: uuid }) |
300 | expect(total).to.equal(2) | 300 | expect(total).to.equal(2) |
301 | } | 301 | } |
302 | } | 302 | } |
@@ -319,8 +319,15 @@ describe('Test AP cleaner', function () { | |||
319 | await wait(5000) | 319 | await wait(5000) |
320 | await expectNotDeleted() | 320 | await expectNotDeleted() |
321 | 321 | ||
322 | await wait(15000) | 322 | let continueWhile = true |
323 | await expectDeleted() | 323 | |
324 | do { | ||
325 | try { | ||
326 | await expectDeleted() | ||
327 | continueWhile = false | ||
328 | } catch { | ||
329 | } | ||
330 | } while (continueWhile) | ||
324 | }) | 331 | }) |
325 | 332 | ||
326 | after(async function () { | 333 | after(async function () { |
diff --git a/server/tests/api/server/handle-down.ts b/server/tests/api/server/handle-down.ts index 7c3836681..3dcd076f5 100644 --- a/server/tests/api/server/handle-down.ts +++ b/server/tests/api/server/handle-down.ts | |||
@@ -50,7 +50,7 @@ describe('Test handle downs', function () { | |||
50 | let commentCommands: CommentsCommand[] | 50 | let commentCommands: CommentsCommand[] |
51 | 51 | ||
52 | before(async function () { | 52 | before(async function () { |
53 | this.timeout(30000) | 53 | this.timeout(120000) |
54 | 54 | ||
55 | servers = await createMultipleServers(3) | 55 | servers = await createMultipleServers(3) |
56 | commentCommands = servers.map(s => s.comments) | 56 | commentCommands = servers.map(s => s.comments) |