]>
Commit | Line | Data |
---|---|---|
8fffe21a C |
1 | import { ACTIVITY_PUB, JOB_REQUEST_TIMEOUT } from '../../initializers' |
2 | import { doRequest } from '../../helpers/requests' | |
3 | import { logger } from '../../helpers/logger' | |
f6eebcb3 | 4 | import Bluebird = require('bluebird') |
8fffe21a | 5 | |
f6eebcb3 | 6 | async function crawlCollectionPage <T> (uri: string, handler: (items: T[]) => Promise<any> | Bluebird<any>) { |
8fffe21a C |
7 | logger.info('Crawling ActivityPub data on %s.', uri) |
8 | ||
9 | const options = { | |
10 | method: 'GET', | |
11 | uri, | |
12 | json: true, | |
13 | activityPub: true, | |
14 | timeout: JOB_REQUEST_TIMEOUT | |
15 | } | |
16 | ||
17 | const response = await doRequest(options) | |
18 | const firstBody = response.body | |
19 | ||
20 | let limit = ACTIVITY_PUB.FETCH_PAGE_LIMIT | |
21 | let i = 0 | |
22 | let nextLink = firstBody.first | |
23 | while (nextLink && i < limit) { | |
24 | options.uri = nextLink | |
25 | ||
26 | const { body } = await doRequest(options) | |
27 | nextLink = body.next | |
28 | i++ | |
29 | ||
30 | if (Array.isArray(body.orderedItems)) { | |
31 | const items = body.orderedItems | |
2186386c | 32 | logger.info('Processing %i ActivityPub items for %s.', items.length, options.uri) |
8fffe21a C |
33 | |
34 | await handler(items) | |
35 | } | |
36 | } | |
37 | } | |
38 | ||
39 | export { | |
40 | crawlCollectionPage | |
41 | } |