diff options
Diffstat (limited to 'server/lib/activitypub/crawl.ts')
| -rw-r--r-- | server/lib/activitypub/crawl.ts | 58 |
1 files changed, 0 insertions, 58 deletions
diff --git a/server/lib/activitypub/crawl.ts b/server/lib/activitypub/crawl.ts deleted file mode 100644 index b8348e8cf..000000000 --- a/server/lib/activitypub/crawl.ts +++ /dev/null | |||
| @@ -1,58 +0,0 @@ | |||
| 1 | import Bluebird from 'bluebird' | ||
| 2 | import { URL } from 'url' | ||
| 3 | import { retryTransactionWrapper } from '@server/helpers/database-utils' | ||
| 4 | import { ActivityPubOrderedCollection } from '../../../shared/models/activitypub' | ||
| 5 | import { logger } from '../../helpers/logger' | ||
| 6 | import { ACTIVITY_PUB, WEBSERVER } from '../../initializers/constants' | ||
| 7 | import { fetchAP } from './activity' | ||
| 8 | |||
| 9 | type HandlerFunction<T> = (items: T[]) => (Promise<any> | Bluebird<any>) | ||
| 10 | type CleanerFunction = (startedDate: Date) => Promise<any> | ||
| 11 | |||
| 12 | async function crawlCollectionPage <T> (argUrl: string, handler: HandlerFunction<T>, cleaner?: CleanerFunction) { | ||
| 13 | let url = argUrl | ||
| 14 | |||
| 15 | logger.info('Crawling ActivityPub data on %s.', url) | ||
| 16 | |||
| 17 | const startDate = new Date() | ||
| 18 | |||
| 19 | const response = await fetchAP<ActivityPubOrderedCollection<T>>(url) | ||
| 20 | const firstBody = response.body | ||
| 21 | |||
| 22 | const limit = ACTIVITY_PUB.FETCH_PAGE_LIMIT | ||
| 23 | let i = 0 | ||
| 24 | let nextLink = firstBody.first | ||
| 25 | while (nextLink && i < limit) { | ||
| 26 | let body: any | ||
| 27 | |||
| 28 | if (typeof nextLink === 'string') { | ||
| 29 | // Don't crawl ourselves | ||
| 30 | const remoteHost = new URL(nextLink).host | ||
| 31 | if (remoteHost === WEBSERVER.HOST) continue | ||
| 32 | |||
| 33 | url = nextLink | ||
| 34 | |||
| 35 | const res = await fetchAP<ActivityPubOrderedCollection<T>>(url) | ||
| 36 | body = res.body | ||
| 37 | } else { | ||
| 38 | // nextLink is already the object we want | ||
| 39 | body = nextLink | ||
| 40 | } | ||
| 41 | |||
| 42 | nextLink = body.next | ||
| 43 | i++ | ||
| 44 | |||
| 45 | if (Array.isArray(body.orderedItems)) { | ||
| 46 | const items = body.orderedItems | ||
| 47 | logger.info('Processing %i ActivityPub items for %s.', items.length, url) | ||
| 48 | |||
| 49 | await handler(items) | ||
| 50 | } | ||
| 51 | } | ||
| 52 | |||
| 53 | if (cleaner) await retryTransactionWrapper(cleaner, startDate) | ||
| 54 | } | ||
| 55 | |||
| 56 | export { | ||
| 57 | crawlCollectionPage | ||
| 58 | } | ||