From 74d249bc1346c7cfaac7ee49bebbebcf2a01f82a Mon Sep 17 00:00:00 2001
From: Chocobozzz <me@florianbigard.com>
Date: Fri, 26 Feb 2021 16:26:27 +0100
Subject: Add ability to cleanup remote AP interactions

---
 server/initializers/checker-before-init.ts |  2 +-
 server/initializers/config.ts              |  3 ++-
 server/initializers/constants.ts           | 12 +++++++++++-
 3 files changed, 14 insertions(+), 3 deletions(-)

(limited to 'server/initializers')

diff --git a/server/initializers/checker-before-init.ts b/server/initializers/checker-before-init.ts
index 2578de5ed..565e0d1fa 100644
--- a/server/initializers/checker-before-init.ts
+++ b/server/initializers/checker-before-init.ts
@@ -36,7 +36,7 @@ function checkMissedConfig () {
     'rates_limit.login.window', 'rates_limit.login.max', 'rates_limit.ask_send_email.window', 'rates_limit.ask_send_email.max',
     'theme.default',
     'remote_redundancy.videos.accept_from',
-    'federation.videos.federate_unlisted',
+    'federation.videos.federate_unlisted', 'federation.videos.cleanup_remote_interactions',
     'search.remote_uri.users', 'search.remote_uri.anonymous', 'search.search_index.enabled', 'search.search_index.url',
     'search.search_index.disable_local_search', 'search.search_index.is_default_search',
     'live.enabled', 'live.allow_replay', 'live.max_duration', 'live.max_user_lives', 'live.max_instance_lives',
diff --git a/server/initializers/config.ts b/server/initializers/config.ts
index 21ca78584..c16b63c33 100644
--- a/server/initializers/config.ts
+++ b/server/initializers/config.ts
@@ -159,7 +159,8 @@ const CONFIG = {
   },
   FEDERATION: {
     VIDEOS: {
-      FEDERATE_UNLISTED: config.get<boolean>('federation.videos.federate_unlisted')
+      FEDERATE_UNLISTED: config.get<boolean>('federation.videos.federate_unlisted'),
+      CLEANUP_REMOTE_INTERACTIONS: config.get<boolean>('federation.videos.cleanup_remote_interactions')
     }
   },
   ADMIN: {
diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts
index 74192d590..083a29889 100644
--- a/server/initializers/constants.ts
+++ b/server/initializers/constants.ts
@@ -137,6 +137,7 @@ const JOB_ATTEMPTS: { [id in JobType]: number } = {
   'activitypub-http-unicast': 5,
   'activitypub-http-fetcher': 5,
   'activitypub-follow': 5,
+  'activitypub-cleaner': 1,
   'video-file-import': 1,
   'video-transcoding': 1,
   'video-import': 1,
@@ -147,10 +148,12 @@ const JOB_ATTEMPTS: { [id in JobType]: number } = {
   'video-redundancy': 1,
   'video-live-ending': 1
 }
-const JOB_CONCURRENCY: { [id in JobType]?: number } = {
+// Excluded keys are jobs that can be configured by admins
+const JOB_CONCURRENCY: { [id in Exclude<JobType, 'video-transcoding' | 'video-import'>]: number } = {
   'activitypub-http-broadcast': 1,
   'activitypub-http-unicast': 5,
   'activitypub-http-fetcher': 1,
+  'activitypub-cleaner': 1,
   'activitypub-follow': 1,
   'video-file-import': 1,
   'email': 5,
@@ -165,6 +168,7 @@ const JOB_TTL: { [id in JobType]: number } = {
   'activitypub-http-unicast': 60000 * 10, // 10 minutes
   'activitypub-http-fetcher': 1000 * 3600 * 10, // 10 hours
   'activitypub-follow': 60000 * 10, // 10 minutes
+  'activitypub-cleaner': 1000 * 3600, // 1 hour
   'video-file-import': 1000 * 3600, // 1 hour
   'video-transcoding': 1000 * 3600 * 48, // 2 days, transcoding could be long
   'video-import': 1000 * 3600 * 2, // 2 hours
@@ -178,6 +182,9 @@ const JOB_TTL: { [id in JobType]: number } = {
 const REPEAT_JOBS: { [ id: string ]: EveryRepeatOptions | CronRepeatOptions } = {
   'videos-views': {
     cron: randomInt(1, 20) + ' * * * *' // Between 1-20 minutes past the hour
+  },
+  'activitypub-cleaner': {
+    cron: '30 5 * * ' + randomInt(0, 7) // 1 time per week (random day) at 5:30 AM
   }
 }
 const JOB_PRIORITY = {
@@ -188,6 +195,7 @@ const JOB_PRIORITY = {
 }
 
 const BROADCAST_CONCURRENCY = 10 // How many requests in parallel we do in activitypub-http-broadcast job
+const AP_CLEANER_CONCURRENCY = 10 // How many requests in parallel we do in activitypub-cleaner job
 const CRAWL_REQUEST_CONCURRENCY = 1 // How many requests in parallel to fetch remote data (likes, shares...)
 const JOB_REQUEST_TIMEOUT = 7000 // 7 seconds
 const JOB_COMPLETED_LIFETIME = 60000 * 60 * 24 * 2 // 2 days
@@ -756,6 +764,7 @@ if (isTestInstance() === true) {
   SCHEDULER_INTERVALS_MS.autoFollowIndexInstances = 5000
   SCHEDULER_INTERVALS_MS.updateInboxStats = 5000
   REPEAT_JOBS['videos-views'] = { every: 5000 }
+  REPEAT_JOBS['activitypub-cleaner'] = { every: 5000 }
 
   REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR = 1
 
@@ -815,6 +824,7 @@ export {
   REDUNDANCY,
   JOB_CONCURRENCY,
   JOB_ATTEMPTS,
+  AP_CLEANER_CONCURRENCY,
   LAST_MIGRATION_VERSION,
   OAUTH_LIFETIME,
   CUSTOM_HTML_TAG_COMMENTS,
-- 
cgit v1.2.3