Use a global request timeout

[github/Chocobozzz/PeerTube.git] / server / lib / job-queue / handlers / activitypub-http-fetcher.ts

diff --git a/server/lib/job-queue/handlers/activitypub-http-fetcher.ts b/server/lib/job-queue/handlers/activitypub-http-fetcher.ts
index 4683beb2f58f40e88425f5b06c4e30a37abc55fb..04b25f955958a0c4cdd5423b9e5249c51786576f 100644 (file)
--- a/server/lib/job-queue/handlers/activitypub-http-fetcher.ts
+++ b/server/lib/job-queue/handlers/activitypub-http-fetcher.ts
@@ -1,60 +1,47 @@
-import * as kue from 'kue'
+import * as Bull from 'bull'
+import { ActivitypubHttpFetcherPayload, FetchType } from '@shared/models'
 import { logger } from '../../../helpers/logger'
-import { doRequest } from '../../../helpers/requests'
-import { ACTIVITY_PUB, JOB_REQUEST_TIMEOUT } from '../../../initializers'
+import { AccountModel } from '../../../models/account/account'
+import { AccountVideoRateModel } from '../../../models/account/account-video-rate'
+import { VideoModel } from '../../../models/video/video'
+import { VideoCommentModel } from '../../../models/video/video-comment'
+import { VideoShareModel } from '../../../models/video/video-share'
+import { MAccountDefault, MVideoFullLight } from '../../../types/models'
+import { crawlCollectionPage } from '../../activitypub/crawl'
+import { createAccountPlaylists } from '../../activitypub/playlists'
 import { processActivities } from '../../activitypub/process'
-import { ActivitypubHttpBroadcastPayload } from './activitypub-http-broadcast'
+import { addVideoShares } from '../../activitypub/share'
+import { addVideoComments } from '../../activitypub/video-comments'
+import { createRates } from '../../activitypub/video-rates'
 
-export type ActivitypubHttpFetcherPayload = {
-  uris: string[]
-}
-
-async function processActivityPubHttpFetcher (job: kue.Job) {
+async function processActivityPubHttpFetcher (job: Bull.Job) {
   logger.info('Processing ActivityPub fetcher in job %d.', job.id)
 
-  const payload = job.data as ActivitypubHttpBroadcastPayload
-
-  const options = {
-    method: 'GET',
-    uri: '',
-    json: true,
-    activityPub: true,
-    timeout: JOB_REQUEST_TIMEOUT
-  }
-
-  for (const uri of payload.uris) {
-    options.uri = uri
-    logger.info('Fetching ActivityPub data on %s.', uri)
-
-    const response = await doRequest(options)
-    const firstBody = response.body
+  const payload = job.data as ActivitypubHttpFetcherPayload
 
-    if (firstBody.first && Array.isArray(firstBody.first.orderedItems)) {
-      const activities = firstBody.first.orderedItems
+  let video: MVideoFullLight
+  if (payload.videoId) video = await VideoModel.loadAndPopulateAccountAndServerAndTags(payload.videoId)
 
-      logger.info('Processing %i items ActivityPub fetcher for %s.', activities.length, options.uri)
+  let account: MAccountDefault
+  if (payload.accountId) account = await AccountModel.load(payload.accountId)
 
-      await processActivities(activities)
-    }
-
-    let limit = ACTIVITY_PUB.FETCH_PAGE_LIMIT
-    let i = 0
-    let nextLink = firstBody.first.next
-    while (nextLink && i < limit) {
-      options.uri = nextLink
-
-      const { body } = await doRequest(options)
-      nextLink = body.next
-      i++
-
-      if (Array.isArray(body.orderedItems)) {
-        const activities = body.orderedItems
-        logger.info('Processing %i items ActivityPub fetcher for %s.', activities.length, options.uri)
+  const fetcherType: { [ id in FetchType ]: (items: any[]) => Promise<any> } = {
+    'activity': items => processActivities(items, { outboxUrl: payload.uri, fromFetch: true }),
+    'video-likes': items => createRates(items, video, 'like'),
+    'video-dislikes': items => createRates(items, video, 'dislike'),
+    'video-shares': items => addVideoShares(items, video),
+    'video-comments': items => addVideoComments(items),
+    'account-playlists': items => createAccountPlaylists(items, account)
+  }
 
-        await processActivities(activities)
-      }
-    }
+  const cleanerType: { [ id in FetchType ]?: (crawlStartDate: Date) => Promise<any> } = {
+    'video-likes': crawlStartDate => AccountVideoRateModel.cleanOldRatesOf(video.id, 'like' as 'like', crawlStartDate),
+    'video-dislikes': crawlStartDate => AccountVideoRateModel.cleanOldRatesOf(video.id, 'dislike' as 'dislike', crawlStartDate),
+    'video-shares': crawlStartDate => VideoShareModel.cleanOldSharesOf(video.id, crawlStartDate),
+    'video-comments': crawlStartDate => VideoCommentModel.cleanOldCommentsOf(video.id, crawlStartDate)
   }
+
+  return crawlCollectionPage(payload.uri, fetcherType[payload.type], cleanerType[payload.type])
 }
 
 // ---------------------------------------------------------------------------