Fetch outbox to grab old activities

2 files changed, 74 insertions, 3 deletions

diff --git a/server/lib/jobs/activitypub-http-job-scheduler/activitypub-http-fetcher-handler.ts b/server/lib/jobs/activitypub-http-job-scheduler/activitypub-http-fetcher-handler.ts
new file mode 100644
index 000000000..b8ead32a4
--- /dev/null
+++ b/server/lib/jobs/activitypub-http-job-scheduler/activitypub-http-fetcher-handler.ts
@@ -0,0 +1,69 @@
+import { logger } from '../../../helpers'
+import { buildSignedActivity } from '../../../helpers/activitypub'
+import { doRequest } from '../../../helpers/requests'
+import { database as db } from '../../../initializers'
+import { ActivityPubHttpPayload } from './activitypub-http-job-scheduler'
+import { processActivities } from '../../activitypub/process/process'
+import { ACTIVITY_PUB } from '../../../initializers/constants'
+
+async function process (payload: ActivityPubHttpPayload, jobId: number) {
+  logger.info('Processing ActivityPub fetcher in job %d.', jobId)
+
+  const options = {
+    method: 'GET',
+    uri: '',
+    json: true
+  }
+
+  for (const uri of payload.uris) {
+    options.uri = uri
+    logger.info('Fetching ActivityPub data on %s.', uri)
+
+    const response = await doRequest(options)
+    const firstBody = response.body
+
+    if (firstBody.first && Array.isArray(firstBody.first.orderedItems)) {
+      const activities = firstBody.first.orderedItems
+
+      logger.info('Processing %i items ActivityPub fetcher for %s.', activities.length, uri)
+
+      await processActivities(activities)
+    }
+
+    let limit = ACTIVITY_PUB.FETCH_PAGE_LIMIT
+    let i = 0
+    let nextLink = firstBody.first.next
+    while (nextLink && i < limit) {
+      options.uri = nextLink
+
+      const { body } = await doRequest(options)
+      nextLink = body.nextLink
+      i++
+
+      if (Array.isArray(body.orderedItems)) {
+        const activities = body.orderedItems
+        logger.info('Processing %i items ActivityPub fetcher for %s.', activities.length, uri)
+
+        await processActivities(activities)
+      }
+    }
+  }
+}
+
+function onError (err: Error, jobId: number) {
+  logger.error('Error when broadcasting ActivityPub request in job %d.', jobId, err)
+  return Promise.resolve()
+}
+
+function onSuccess (jobId: number) {
+  logger.info('Job %d is a success.', jobId)
+  return Promise.resolve()
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+  process,
+  onError,
+  onSuccess
+}
diff --git a/server/lib/jobs/activitypub-http-job-scheduler/activitypub-http-job-scheduler.ts b/server/lib/jobs/activitypub-http-job-scheduler/activitypub-http-job-scheduler.ts
index e4f6c94a5..aef217ce7 100644
--- a/server/lib/jobs/activitypub-http-job-scheduler/activitypub-http-job-scheduler.ts
+++ b/server/lib/jobs/activitypub-http-job-scheduler/activitypub-http-job-scheduler.ts
@@ -2,16 +2,18 @@ import { JobScheduler, JobHandler } from '../job-scheduler'
 
 import * as activitypubHttpBroadcastHandler from './activitypub-http-broadcast-handler'
 import * as activitypubHttpUnicastHandler from './activitypub-http-unicast-handler'
+import * as activitypubHttpFetcherHandler from './activitypub-http-fetcher-handler'
 import { JobCategory } from '../../../../shared'
 
 type ActivityPubHttpPayload = {
   uris: string[]
-  signatureAccountId: number
-  body: any
+  signatureAccountId?: number
+  body?: any
 }
 const jobHandlers: { [ handlerName: string ]: JobHandler<ActivityPubHttpPayload, void> } = {
   activitypubHttpBroadcastHandler,
-  activitypubHttpUnicastHandler
+  activitypubHttpUnicastHandler,
+  activitypubHttpFetcherHandler
 }
 const jobCategory: JobCategory = 'activitypub-http'