Refractor and optimize AP collections

Only display urls in general object, and paginate video comments, shares, likes and
dislikes

3 files changed, 51 insertions, 18 deletions

diff --git a/server/lib/activitypub/crawl.ts b/server/lib/activitypub/crawl.ts
new file mode 100644
index 000000000..7305b3969
--- /dev/null
+++ b/server/lib/activitypub/crawl.ts
@@ -0,0 +1,40 @@
+import { ACTIVITY_PUB, JOB_REQUEST_TIMEOUT } from '../../initializers'
+import { doRequest } from '../../helpers/requests'
+import { logger } from '../../helpers/logger'
+
+async function crawlCollectionPage <T> (uri: string, handler: (items: T[]) => Promise<any>) {
+  logger.info('Crawling ActivityPub data on %s.', uri)
+
+  const options = {
+    method: 'GET',
+    uri,
+    json: true,
+    activityPub: true,
+    timeout: JOB_REQUEST_TIMEOUT
+  }
+
+  const response = await doRequest(options)
+  const firstBody = response.body
+
+  let limit = ACTIVITY_PUB.FETCH_PAGE_LIMIT
+  let i = 0
+  let nextLink = firstBody.first
+  while (nextLink && i < limit) {
+    options.uri = nextLink
+
+    const { body } = await doRequest(options)
+    nextLink = body.next
+    i++
+
+    if (Array.isArray(body.orderedItems)) {
+      const items = body.orderedItems
+      logger.info('Processing %i ActivityPub items for %s.', items.length, nextLink)
+
+      await handler(items)
+    }
+  }
+}
+
+export {
+  crawlCollectionPage
+}
diff --git a/server/lib/activitypub/video-comments.ts b/server/lib/activitypub/video-comments.ts
index 60c9179a6..fd03710c2 100644
--- a/server/lib/activitypub/video-comments.ts
+++ b/server/lib/activitypub/video-comments.ts
@@ -37,7 +37,7 @@ async function videoCommentActivityObjectToDBAttributes (video: VideoModel, acto
   }
 }
 
-async function addVideoComments (instance: VideoModel, commentUrls: string[]) {
+async function addVideoComments (commentUrls: string[], instance: VideoModel) {
   for (const commentUrl of commentUrls) {
     await addVideoComment(instance, commentUrl)
   }
diff --git a/server/lib/activitypub/videos.ts b/server/lib/activitypub/videos.ts
index dbd7385a4..be6794cef 100644
--- a/server/lib/activitypub/videos.ts
+++ b/server/lib/activitypub/videos.ts
@@ -20,6 +20,7 @@ import { VideoFileModel } from '../../models/video/video-file'
 import { VideoShareModel } from '../../models/video/video-share'
 import { getOrCreateActorAndServerAndModel } from './actor'
 import { addVideoComments } from './video-comments'
+import { crawlCollectionPage } from './crawl'
 
 function fetchRemoteVideoPreview (video: VideoModel, reject: Function) {
   const host = video.VideoChannel.Account.Actor.Server.host
@@ -216,25 +217,17 @@ async function getOrCreateAccountAndVideoAndChannel (videoObject: VideoTorrentOb
   const video = await retryTransactionWrapper(getOrCreateVideo, options)
 
   // Process outside the transaction because we could fetch remote data
-  if (videoObject.likes && Array.isArray(videoObject.likes.orderedItems)) {
-    logger.info('Adding likes of video %s.', video.uuid)
-    await createRates(videoObject.likes.orderedItems, video, 'like')
-  }
+  logger.info('Adding likes of video %s.', video.uuid)
+  await crawlCollectionPage<string>(videoObject.likes, (items) => createRates(items, video, 'like'))
 
-  if (videoObject.dislikes && Array.isArray(videoObject.dislikes.orderedItems)) {
-    logger.info('Adding dislikes of video %s.', video.uuid)
-    await createRates(videoObject.dislikes.orderedItems, video, 'dislike')
-  }
+  logger.info('Adding dislikes of video %s.', video.uuid)
+  await crawlCollectionPage<string>(videoObject.dislikes, (items) => createRates(items, video, 'dislike'))
 
-  if (videoObject.shares && Array.isArray(videoObject.shares.orderedItems)) {
-    logger.info('Adding shares of video %s.', video.uuid)
-    await addVideoShares(video, videoObject.shares.orderedItems)
-  }
+  logger.info('Adding shares of video %s.', video.uuid)
+  await crawlCollectionPage<string>(videoObject.shares, (items) => addVideoShares(items, video))
 
-  if (videoObject.comments && Array.isArray(videoObject.comments.orderedItems)) {
-    logger.info('Adding comments of video %s.', video.uuid)
-    await addVideoComments(video, videoObject.comments.orderedItems)
-  }
+  logger.info('Adding comments of video %s.', video.uuid)
+  await crawlCollectionPage<string>(videoObject.comments, (items) => addVideoComments(items, video))
 
   return { actor, channelActor, video }
 }
@@ -266,7 +259,7 @@ async function createRates (actorUrls: string[], video: VideoModel, rate: VideoR
   return
 }
 
-async function addVideoShares (instance: VideoModel, shareUrls: string[]) {
+async function addVideoShares (shareUrls: string[], instance: VideoModel) {
   for (const shareUrl of shareUrls) {
     // Fetch url
     const { body } = await doRequest({