import { ACTIVITY_PUB, JOB_REQUEST_TIMEOUT, WEBSERVER } from '../../initializers/constants' import { doRequest } from '../../helpers/requests' import { logger } from '../../helpers/logger' import * as Bluebird from 'bluebird' import { ActivityPubOrderedCollection } from '../../../shared/models/activitypub' import { parse } from 'url' type HandlerFunction = (items: T[]) => (Promise | Bluebird) type CleanerFunction = (startedDate: Date) => (Promise | Bluebird) async function crawlCollectionPage (uri: string, handler: HandlerFunction, cleaner?: CleanerFunction) { logger.info('Crawling ActivityPub data on %s.', uri) const options = { method: 'GET', uri, json: true, activityPub: true, timeout: JOB_REQUEST_TIMEOUT } const startDate = new Date() const response = await doRequest>(options) const firstBody = response.body let limit = ACTIVITY_PUB.FETCH_PAGE_LIMIT let i = 0 let nextLink = firstBody.first while (nextLink && i < limit) { let body: any if (typeof nextLink === 'string') { // Don't crawl ourselves const remoteHost = parse(nextLink).host if (remoteHost === WEBSERVER.HOST) continue options.uri = nextLink const res = await doRequest>(options) body = res.body } else { // nextLink is already the object we want body = nextLink } nextLink = body.next i++ if (Array.isArray(body.orderedItems)) { const items = body.orderedItems logger.info('Processing %i ActivityPub items for %s.', items.length, options.uri) await handler(items) } } if (cleaner) await cleaner(startDate) } export { crawlCollectionPage }