import { ACTIVITY_PUB, JOB_REQUEST_TIMEOUT } from '../../initializers/constants' import { doRequest } from '../../helpers/requests' import { logger } from '../../helpers/logger' import * as Bluebird from 'bluebird' import { ActivityPubOrderedCollection } from '../../../shared/models/activitypub' type HandlerFunction = (items: T[]) => (Promise | Bluebird) type CleanerFunction = (startedDate: Date) => (Promise | Bluebird) async function crawlCollectionPage (uri: string, handler: HandlerFunction, cleaner?: CleanerFunction) { logger.info('Crawling ActivityPub data on %s.', uri) const options = { method: 'GET', uri, json: true, activityPub: true, timeout: JOB_REQUEST_TIMEOUT } const startDate = new Date() const response = await doRequest>(options) const firstBody = response.body let limit = ACTIVITY_PUB.FETCH_PAGE_LIMIT let i = 0 let nextLink = firstBody.first while (nextLink && i < limit) { options.uri = nextLink const { body } = await doRequest>(options) nextLink = body.next i++ if (Array.isArray(body.orderedItems)) { const items = body.orderedItems logger.info('Processing %i ActivityPub items for %s.', items.length, options.uri) await handler(items) } } if (cleaner) await cleaner(startDate) } export { crawlCollectionPage }