diff options
Diffstat (limited to 'server/lib/activitypub/crawl.ts')
-rw-r--r-- | server/lib/activitypub/crawl.ts | 9 |
1 files changed, 8 insertions, 1 deletions
diff --git a/server/lib/activitypub/crawl.ts b/server/lib/activitypub/crawl.ts index 2675524c6..9f4ca98ba 100644 --- a/server/lib/activitypub/crawl.ts +++ b/server/lib/activitypub/crawl.ts | |||
@@ -4,7 +4,10 @@ import { logger } from '../../helpers/logger' | |||
4 | import * as Bluebird from 'bluebird' | 4 | import * as Bluebird from 'bluebird' |
5 | import { ActivityPubOrderedCollection } from '../../../shared/models/activitypub' | 5 | import { ActivityPubOrderedCollection } from '../../../shared/models/activitypub' |
6 | 6 | ||
7 | async function crawlCollectionPage <T> (uri: string, handler: (items: T[]) => (Promise<any> | Bluebird<any>)) { | 7 | type HandlerFunction<T> = (items: T[]) => (Promise<any> | Bluebird<any>) |
8 | type CleanerFunction = (startedDate: Date) => (Promise<any> | Bluebird<any>) | ||
9 | |||
10 | async function crawlCollectionPage <T> (uri: string, handler: HandlerFunction<T>, cleaner?: CleanerFunction) { | ||
8 | logger.info('Crawling ActivityPub data on %s.', uri) | 11 | logger.info('Crawling ActivityPub data on %s.', uri) |
9 | 12 | ||
10 | const options = { | 13 | const options = { |
@@ -15,6 +18,8 @@ async function crawlCollectionPage <T> (uri: string, handler: (items: T[]) => (P | |||
15 | timeout: JOB_REQUEST_TIMEOUT | 18 | timeout: JOB_REQUEST_TIMEOUT |
16 | } | 19 | } |
17 | 20 | ||
21 | const startDate = new Date() | ||
22 | |||
18 | const response = await doRequest<ActivityPubOrderedCollection<T>>(options) | 23 | const response = await doRequest<ActivityPubOrderedCollection<T>>(options) |
19 | const firstBody = response.body | 24 | const firstBody = response.body |
20 | 25 | ||
@@ -35,6 +40,8 @@ async function crawlCollectionPage <T> (uri: string, handler: (items: T[]) => (P | |||
35 | await handler(items) | 40 | await handler(items) |
36 | } | 41 | } |
37 | } | 42 | } |
43 | |||
44 | if (cleaner) await cleaner(startDate) | ||
38 | } | 45 | } |
39 | 46 | ||
40 | export { | 47 | export { |