diff options
author | Chocobozzz <me@florianbigard.com> | 2018-05-25 16:21:16 +0200 |
---|---|---|
committer | Chocobozzz <me@florianbigard.com> | 2018-05-25 16:21:16 +0200 |
commit | 8fffe21a7bc96d08b229293d66ddba576e609790 (patch) | |
tree | 5ebd5f5198a59084c5338ce197d7e836b39200a4 /server/lib/activitypub/crawl.ts | |
parent | e251f170b00b2014ac4e823113c6ff40e3fb1471 (diff) | |
download | PeerTube-8fffe21a7bc96d08b229293d66ddba576e609790.tar.gz PeerTube-8fffe21a7bc96d08b229293d66ddba576e609790.tar.zst PeerTube-8fffe21a7bc96d08b229293d66ddba576e609790.zip |
Refractor and optimize AP collections
Only display urls in general object, and paginate video comments, shares, likes and
dislikes
Diffstat (limited to 'server/lib/activitypub/crawl.ts')
-rw-r--r-- | server/lib/activitypub/crawl.ts | 40 |
1 files changed, 40 insertions, 0 deletions
diff --git a/server/lib/activitypub/crawl.ts b/server/lib/activitypub/crawl.ts new file mode 100644 index 000000000..7305b3969 --- /dev/null +++ b/server/lib/activitypub/crawl.ts | |||
@@ -0,0 +1,40 @@ | |||
1 | import { ACTIVITY_PUB, JOB_REQUEST_TIMEOUT } from '../../initializers' | ||
2 | import { doRequest } from '../../helpers/requests' | ||
3 | import { logger } from '../../helpers/logger' | ||
4 | |||
5 | async function crawlCollectionPage <T> (uri: string, handler: (items: T[]) => Promise<any>) { | ||
6 | logger.info('Crawling ActivityPub data on %s.', uri) | ||
7 | |||
8 | const options = { | ||
9 | method: 'GET', | ||
10 | uri, | ||
11 | json: true, | ||
12 | activityPub: true, | ||
13 | timeout: JOB_REQUEST_TIMEOUT | ||
14 | } | ||
15 | |||
16 | const response = await doRequest(options) | ||
17 | const firstBody = response.body | ||
18 | |||
19 | let limit = ACTIVITY_PUB.FETCH_PAGE_LIMIT | ||
20 | let i = 0 | ||
21 | let nextLink = firstBody.first | ||
22 | while (nextLink && i < limit) { | ||
23 | options.uri = nextLink | ||
24 | |||
25 | const { body } = await doRequest(options) | ||
26 | nextLink = body.next | ||
27 | i++ | ||
28 | |||
29 | if (Array.isArray(body.orderedItems)) { | ||
30 | const items = body.orderedItems | ||
31 | logger.info('Processing %i ActivityPub items for %s.', items.length, nextLink) | ||
32 | |||
33 | await handler(items) | ||
34 | } | ||
35 | } | ||
36 | } | ||
37 | |||
38 | export { | ||
39 | crawlCollectionPage | ||
40 | } | ||