server/lib/activitypub/crawl.ts

   1 import { ACTIVITY_PUB, JOB_REQUEST_TIMEOUT } from '../../initializers'
   2 import { doRequest } from '../../helpers/requests'
   3 import { logger } from '../../helpers/logger'
   4
   5 async function crawlCollectionPage <T> (uri: string, handler: (items: T[]) => Promise<any>) {
   6   logger.info('Crawling ActivityPub data on %s.', uri)
   7
   8   const options = {
   9     method: 'GET',
  10     uri,
  11     json: true,
  12     activityPub: true,
  13     timeout: JOB_REQUEST_TIMEOUT
  14   }
  15
  16   const response = await doRequest(options)
  17   const firstBody = response.body
  18
  19   let limit = ACTIVITY_PUB.FETCH_PAGE_LIMIT
  20   let i = 0
  21   let nextLink = firstBody.first
  22   while (nextLink && i < limit) {
  23     options.uri = nextLink
  24
  25     const { body } = await doRequest(options)
  26     nextLink = body.next
  27     i++
  28
  29     if (Array.isArray(body.orderedItems)) {
  30       const items = body.orderedItems
  31       logger.info('Processing %i ActivityPub items for %s.', items.length, options.uri)
  32
  33       await handler(items)
  34     }
  35   }
  36 }
  37
  38 export {
  39   crawlCollectionPage
  40 }