Refractor and optimize AP collections

Only display urls in general object, and paginate video comments, shares, likes and
dislikes

1 files changed, 3 insertions, 41 deletions

diff --git a/server/lib/job-queue/handlers/activitypub-http-fetcher.ts b/server/lib/job-queue/handlers/activitypub-http-fetcher.ts
index 4683beb2f..10c0e606f 100644
--- a/server/lib/job-queue/handlers/activitypub-http-fetcher.ts
+++ b/server/lib/job-queue/handlers/activitypub-http-fetcher.ts
@@ -1,9 +1,9 @@
 import * as kue from 'kue'
 import { logger } from '../../../helpers/logger'
-import { doRequest } from '../../../helpers/requests'
-import { ACTIVITY_PUB, JOB_REQUEST_TIMEOUT } from '../../../initializers'
 import { processActivities } from '../../activitypub/process'
 import { ActivitypubHttpBroadcastPayload } from './activitypub-http-broadcast'
+import { crawlCollectionPage } from '../../activitypub/crawl'
+import { Activity } from '../../../../shared/models/activitypub'
 
 export type ActivitypubHttpFetcherPayload = {
   uris: string[]
@@ -14,46 +14,8 @@ async function processActivityPubHttpFetcher (job: kue.Job) {
 
   const payload = job.data as ActivitypubHttpBroadcastPayload
 
-  const options = {
-    method: 'GET',
-    uri: '',
-    json: true,
-    activityPub: true,
-    timeout: JOB_REQUEST_TIMEOUT
-  }
-
   for (const uri of payload.uris) {
-    options.uri = uri
-    logger.info('Fetching ActivityPub data on %s.', uri)
-
-    const response = await doRequest(options)
-    const firstBody = response.body
-
-    if (firstBody.first && Array.isArray(firstBody.first.orderedItems)) {
-      const activities = firstBody.first.orderedItems
-
-      logger.info('Processing %i items ActivityPub fetcher for %s.', activities.length, options.uri)
-
-      await processActivities(activities)
-    }
-
-    let limit = ACTIVITY_PUB.FETCH_PAGE_LIMIT
-    let i = 0
-    let nextLink = firstBody.first.next
-    while (nextLink && i < limit) {
-      options.uri = nextLink
-
-      const { body } = await doRequest(options)
-      nextLink = body.next
-      i++
-
-      if (Array.isArray(body.orderedItems)) {
-        const activities = body.orderedItems
-        logger.info('Processing %i items ActivityPub fetcher for %s.', activities.length, options.uri)
-
-        await processActivities(activities)
-      }
-    }
+    await crawlCollectionPage<Activity>(uri, (items) => processActivities(items))
   }
 }