aboutsummaryrefslogtreecommitdiffhomepage
path: root/server/lib
diff options
context:
space:
mode:
Diffstat (limited to 'server/lib')
-rw-r--r--server/lib/activitypub/crawl.ts8
1 files changed, 7 insertions, 1 deletions
diff --git a/server/lib/activitypub/crawl.ts b/server/lib/activitypub/crawl.ts
index ba5b67bee..0ba59b47d 100644
--- a/server/lib/activitypub/crawl.ts
+++ b/server/lib/activitypub/crawl.ts
@@ -1,8 +1,10 @@
1import { ACTIVITY_PUB, JOB_REQUEST_TIMEOUT } from '../../initializers/constants' 1import { ACTIVITY_PUB, JOB_REQUEST_TIMEOUT, WEBSERVER } from '../../initializers/constants'
2import { doRequest } from '../../helpers/requests' 2import { doRequest } from '../../helpers/requests'
3import { logger } from '../../helpers/logger' 3import { logger } from '../../helpers/logger'
4import * as Bluebird from 'bluebird' 4import * as Bluebird from 'bluebird'
5import { ActivityPubOrderedCollection } from '../../../shared/models/activitypub' 5import { ActivityPubOrderedCollection } from '../../../shared/models/activitypub'
6import { checkUrlsSameHost } from '../../helpers/activitypub'
7import { parse } from "url"
6 8
7type HandlerFunction<T> = (items: T[]) => (Promise<any> | Bluebird<any>) 9type HandlerFunction<T> = (items: T[]) => (Promise<any> | Bluebird<any>)
8type CleanerFunction = (startedDate: Date) => (Promise<any> | Bluebird<any>) 10type CleanerFunction = (startedDate: Date) => (Promise<any> | Bluebird<any>)
@@ -27,6 +29,10 @@ async function crawlCollectionPage <T> (uri: string, handler: HandlerFunction<T>
27 let i = 0 29 let i = 0
28 let nextLink = firstBody.first 30 let nextLink = firstBody.first
29 while (nextLink && i < limit) { 31 while (nextLink && i < limit) {
32 // Don't crawl ourselves
33 const remoteHost = parse(nextLink).host
34 if (remoteHost === WEBSERVER.HOST) continue
35
30 options.uri = nextLink 36 options.uri = nextLink
31 37
32 const { body } = await doRequest<ActivityPubOrderedCollection<T>>(options) 38 const { body } = await doRequest<ActivityPubOrderedCollection<T>>(options)