Cleanup invalid rates/comments/shares

6 files changed, 51 insertions, 34 deletions

diff --git a/server/lib/activitypub/crawl.ts b/server/lib/activitypub/crawl.ts
index 2675524c6..9f4ca98ba 100644
--- a/server/lib/activitypub/crawl.ts
+++ b/server/lib/activitypub/crawl.ts
@@ -4,7 +4,10 @@ import { logger } from '../../helpers/logger'
 import * as Bluebird from 'bluebird'
 import { ActivityPubOrderedCollection } from '../../../shared/models/activitypub'
 
-async function crawlCollectionPage <T> (uri: string, handler: (items: T[]) => (Promise<any> | Bluebird<any>)) {
+type HandlerFunction<T> = (items: T[]) => (Promise<any> | Bluebird<any>)
+type CleanerFunction = (startedDate: Date) => (Promise<any> | Bluebird<any>)
+
+async function crawlCollectionPage <T> (uri: string, handler: HandlerFunction<T>, cleaner?: CleanerFunction) {
   logger.info('Crawling ActivityPub data on %s.', uri)
 
   const options = {
@@ -15,6 +18,8 @@ async function crawlCollectionPage <T> (uri: string, handler: (items: T[]) => (P
     timeout: JOB_REQUEST_TIMEOUT
   }
 
+  const startDate = new Date()
+
   const response = await doRequest<ActivityPubOrderedCollection<T>>(options)
   const firstBody = response.body
 
@@ -35,6 +40,8 @@ async function crawlCollectionPage <T> (uri: string, handler: (items: T[]) => (P
       await handler(items)
     }
   }
+
+  if (cleaner) await cleaner(startDate)
 }
 
 export {
diff --git a/server/lib/activitypub/share.ts b/server/lib/activitypub/share.ts
index 1767df0ae..3bece0ff7 100644
--- a/server/lib/activitypub/share.ts
+++ b/server/lib/activitypub/share.ts
@@ -54,12 +54,7 @@ async function addVideoShares (shareUrls: string[], instance: VideoModel) {
         url: shareUrl
       }
 
-      await VideoShareModel.findOrCreate({
-        where: {
-          url: shareUrl
-        },
-        defaults: entry
-      })
+      await VideoShareModel.upsert(entry)
     } catch (err) {
       logger.warn('Cannot add share %s.', shareUrl, { err })
     }
diff --git a/server/lib/activitypub/video-comments.ts b/server/lib/activitypub/video-comments.ts
index e87301fe7..3f9d8f0fc 100644
--- a/server/lib/activitypub/video-comments.ts
+++ b/server/lib/activitypub/video-comments.ts
@@ -34,8 +34,7 @@ async function videoCommentActivityObjectToDBAttributes (video: VideoModel, acto
     accountId: actor.Account.id,
     inReplyToCommentId,
     originCommentId,
-    createdAt: new Date(comment.published),
-    updatedAt: new Date(comment.updated)
+    createdAt: new Date(comment.published)
   }
 }
 
@@ -74,12 +73,7 @@ async function addVideoComment (videoInstance: VideoModel, commentUrl: string) {
   const entry = await videoCommentActivityObjectToDBAttributes(videoInstance, actor, body)
   if (!entry) return { created: false }
 
-  const [ comment, created ] = await VideoCommentModel.findOrCreate({
-    where: {
-      url: body.id
-    },
-    defaults: entry
-  })
+  const [ comment, created ] = await VideoCommentModel.upsert<VideoCommentModel>(entry, { returning: true })
   comment.Account = actor.Account
   comment.Video = videoInstance
 
diff --git a/server/lib/activitypub/video-rates.ts b/server/lib/activitypub/video-rates.ts
index 7aac79118..ad7d81df6 100644
--- a/server/lib/activitypub/video-rates.ts
+++ b/server/lib/activitypub/video-rates.ts
@@ -38,19 +38,14 @@ async function createRates (ratesUrl: string[], video: VideoModel, rate: VideoRa
 
       const actor = await getOrCreateActorAndServerAndModel(actorUrl)
 
-      const [ , created ] = await AccountVideoRateModel
-        .findOrCreate({
-          where: {
-            videoId: video.id,
-            accountId: actor.Account.id
-          },
-          defaults: {
-            videoId: video.id,
-            accountId: actor.Account.id,
-            type: rate,
-            url: body.id
-          }
-        })
+      const entry = {
+        videoId: video.id,
+        accountId: actor.Account.id,
+        type: rate,
+        url: body.id
+      }
+
+      const created = await AccountVideoRateModel.upsert(entry)
 
       if (created) rateCounts += 1
     } catch (err) {
diff --git a/server/lib/activitypub/videos.ts b/server/lib/activitypub/videos.ts
index 66d0abf35..2c932371b 100644
--- a/server/lib/activitypub/videos.ts
+++ b/server/lib/activitypub/videos.ts
@@ -40,6 +40,9 @@ import { Notifier } from '../notifier'
 import { VideoStreamingPlaylistModel } from '../../models/video/video-streaming-playlist'
 import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type'
 import { FilteredModelAttributes } from 'sequelize-typescript/lib/models/Model'
+import { AccountVideoRateModel } from '../../models/account/account-video-rate'
+import { VideoShareModel } from '../../models/video/video-share'
+import { VideoCommentModel } from '../../models/video/video-comment'
 
 async function federateVideoIfNeeded (video: VideoModel, isNewVideo: boolean, transaction?: sequelize.Transaction) {
   // If the video is not private and published, we federate it
@@ -134,31 +137,43 @@ async function syncVideoExternalAttributes (video: VideoModel, fetchedVideo: Vid
   const jobPayloads: ActivitypubHttpFetcherPayload[] = []
 
   if (syncParam.likes === true) {
-    await crawlCollectionPage<string>(fetchedVideo.likes, items => createRates(items, video, 'like'))
+    const handler = items => createRates(items, video, 'like')
+    const cleaner = crawlStartDate => AccountVideoRateModel.cleanOldRatesOf(video.id, 'like' as 'like', crawlStartDate)
+
+    await crawlCollectionPage<string>(fetchedVideo.likes, handler, cleaner)
       .catch(err => logger.error('Cannot add likes of video %s.', video.uuid, { err }))
   } else {
     jobPayloads.push({ uri: fetchedVideo.likes, videoId: video.id, type: 'video-likes' as 'video-likes' })
   }
 
   if (syncParam.dislikes === true) {
-    await crawlCollectionPage<string>(fetchedVideo.dislikes, items => createRates(items, video, 'dislike'))
+    const handler = items => createRates(items, video, 'dislike')
+    const cleaner = crawlStartDate => AccountVideoRateModel.cleanOldRatesOf(video.id, 'dislike' as 'dislike', crawlStartDate)
+
+    await crawlCollectionPage<string>(fetchedVideo.dislikes, handler, cleaner)
       .catch(err => logger.error('Cannot add dislikes of video %s.', video.uuid, { err }))
   } else {
     jobPayloads.push({ uri: fetchedVideo.dislikes, videoId: video.id, type: 'video-dislikes' as 'video-dislikes' })
   }
 
   if (syncParam.shares === true) {
-    await crawlCollectionPage<string>(fetchedVideo.shares, items => addVideoShares(items, video))
+    const handler = items => addVideoShares(items, video)
+    const cleaner = crawlStartDate => VideoShareModel.cleanOldSharesOf(video.id, crawlStartDate)
+
+    await crawlCollectionPage<string>(fetchedVideo.shares, handler, cleaner)
       .catch(err => logger.error('Cannot add shares of video %s.', video.uuid, { err }))
   } else {
     jobPayloads.push({ uri: fetchedVideo.shares, videoId: video.id, type: 'video-shares' as 'video-shares' })
   }
 
   if (syncParam.comments === true) {
-    await crawlCollectionPage<string>(fetchedVideo.comments, items => addVideoComments(items, video))
+    const handler = items => addVideoComments(items, video)
+    const cleaner = crawlStartDate => VideoCommentModel.cleanOldCommentsOf(video.id, crawlStartDate)
+
+    await crawlCollectionPage<string>(fetchedVideo.comments, handler, cleaner)
       .catch(err => logger.error('Cannot add comments of video %s.', video.uuid, { err }))
   } else {
-    jobPayloads.push({ uri: fetchedVideo.shares, videoId: video.id, type: 'video-shares' as 'video-shares' })
+    jobPayloads.push({ uri: fetchedVideo.comments, videoId: video.id, type: 'video-comments' as 'video-comments' })
   }
 
   await Bluebird.map(jobPayloads, payload => JobQueue.Instance.createJob({ type: 'activitypub-http-fetcher', payload }))
diff --git a/server/lib/job-queue/handlers/activitypub-http-fetcher.ts b/server/lib/job-queue/handlers/activitypub-http-fetcher.ts
index 52225f64f..23d33c26f 100644
--- a/server/lib/job-queue/handlers/activitypub-http-fetcher.ts
+++ b/server/lib/job-queue/handlers/activitypub-http-fetcher.ts
@@ -1,4 +1,5 @@
 import * as Bull from 'bull'
+import * as Bluebird from 'bluebird'
 import { logger } from '../../../helpers/logger'
 import { processActivities } from '../../activitypub/process'
 import { addVideoComments } from '../../activitypub/video-comments'
@@ -7,6 +8,9 @@ import { VideoModel } from '../../../models/video/video'
 import { addVideoShares, createRates } from '../../activitypub'
 import { createAccountPlaylists } from '../../activitypub/playlist'
 import { AccountModel } from '../../../models/account/account'
+import { AccountVideoRateModel } from '../../../models/account/account-video-rate'
+import { VideoShareModel } from '../../../models/video/video-share'
+import { VideoCommentModel } from '../../../models/video/video-comment'
 
 type FetchType = 'activity' | 'video-likes' | 'video-dislikes' | 'video-shares' | 'video-comments' | 'account-playlists'
 
@@ -37,7 +41,14 @@ async function processActivityPubHttpFetcher (job: Bull.Job) {
     'account-playlists': items => createAccountPlaylists(items, account)
   }
 
-  return crawlCollectionPage(payload.uri, fetcherType[payload.type])
+  const cleanerType: { [ id in FetchType ]?: (crawlStartDate: Date) => Bluebird<any> } = {
+    'video-likes': crawlStartDate => AccountVideoRateModel.cleanOldRatesOf(video.id, 'like' as 'like', crawlStartDate),
+    'video-dislikes': crawlStartDate => AccountVideoRateModel.cleanOldRatesOf(video.id, 'dislike' as 'dislike', crawlStartDate),
+    'video-shares': crawlStartDate => VideoShareModel.cleanOldSharesOf(video.id, crawlStartDate),
+    'video-comments': crawlStartDate => VideoCommentModel.cleanOldCommentsOf(video.id, crawlStartDate)
+  }
+
+  return crawlCollectionPage(payload.uri, fetcherType[payload.type], cleanerType[payload.type])
 }
 
 // ---------------------------------------------------------------------------