From 2ba92871319d7af63472c1380664a9f9eeb1c690 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Tue, 19 Mar 2019 16:23:02 +0100 Subject: Cleanup invalid rates/comments/shares --- server/lib/activitypub/crawl.ts | 9 ++++++++- server/lib/activitypub/share.ts | 7 +------ server/lib/activitypub/video-comments.ts | 10 ++-------- server/lib/activitypub/video-rates.ts | 21 ++++++++------------- server/lib/activitypub/videos.ts | 25 ++++++++++++++++++++----- 5 files changed, 39 insertions(+), 33 deletions(-) (limited to 'server/lib/activitypub') diff --git a/server/lib/activitypub/crawl.ts b/server/lib/activitypub/crawl.ts index 2675524c6..9f4ca98ba 100644 --- a/server/lib/activitypub/crawl.ts +++ b/server/lib/activitypub/crawl.ts @@ -4,7 +4,10 @@ import { logger } from '../../helpers/logger' import * as Bluebird from 'bluebird' import { ActivityPubOrderedCollection } from '../../../shared/models/activitypub' -async function crawlCollectionPage (uri: string, handler: (items: T[]) => (Promise | Bluebird)) { +type HandlerFunction = (items: T[]) => (Promise | Bluebird) +type CleanerFunction = (startedDate: Date) => (Promise | Bluebird) + +async function crawlCollectionPage (uri: string, handler: HandlerFunction, cleaner?: CleanerFunction) { logger.info('Crawling ActivityPub data on %s.', uri) const options = { @@ -15,6 +18,8 @@ async function crawlCollectionPage (uri: string, handler: (items: T[]) => (P timeout: JOB_REQUEST_TIMEOUT } + const startDate = new Date() + const response = await doRequest>(options) const firstBody = response.body @@ -35,6 +40,8 @@ async function crawlCollectionPage (uri: string, handler: (items: T[]) => (P await handler(items) } } + + if (cleaner) await cleaner(startDate) } export { diff --git a/server/lib/activitypub/share.ts b/server/lib/activitypub/share.ts index 1767df0ae..3bece0ff7 100644 --- a/server/lib/activitypub/share.ts +++ b/server/lib/activitypub/share.ts @@ -54,12 +54,7 @@ async function addVideoShares (shareUrls: string[], instance: VideoModel) { url: shareUrl } - await VideoShareModel.findOrCreate({ - where: { - url: shareUrl - }, - defaults: entry - }) + await VideoShareModel.upsert(entry) } catch (err) { logger.warn('Cannot add share %s.', shareUrl, { err }) } diff --git a/server/lib/activitypub/video-comments.ts b/server/lib/activitypub/video-comments.ts index e87301fe7..3f9d8f0fc 100644 --- a/server/lib/activitypub/video-comments.ts +++ b/server/lib/activitypub/video-comments.ts @@ -34,8 +34,7 @@ async function videoCommentActivityObjectToDBAttributes (video: VideoModel, acto accountId: actor.Account.id, inReplyToCommentId, originCommentId, - createdAt: new Date(comment.published), - updatedAt: new Date(comment.updated) + createdAt: new Date(comment.published) } } @@ -74,12 +73,7 @@ async function addVideoComment (videoInstance: VideoModel, commentUrl: string) { const entry = await videoCommentActivityObjectToDBAttributes(videoInstance, actor, body) if (!entry) return { created: false } - const [ comment, created ] = await VideoCommentModel.findOrCreate({ - where: { - url: body.id - }, - defaults: entry - }) + const [ comment, created ] = await VideoCommentModel.upsert(entry, { returning: true }) comment.Account = actor.Account comment.Video = videoInstance diff --git a/server/lib/activitypub/video-rates.ts b/server/lib/activitypub/video-rates.ts index 7aac79118..ad7d81df6 100644 --- a/server/lib/activitypub/video-rates.ts +++ b/server/lib/activitypub/video-rates.ts @@ -38,19 +38,14 @@ async function createRates (ratesUrl: string[], video: VideoModel, rate: VideoRa const actor = await getOrCreateActorAndServerAndModel(actorUrl) - const [ , created ] = await AccountVideoRateModel - .findOrCreate({ - where: { - videoId: video.id, - accountId: actor.Account.id - }, - defaults: { - videoId: video.id, - accountId: actor.Account.id, - type: rate, - url: body.id - } - }) + const entry = { + videoId: video.id, + accountId: actor.Account.id, + type: rate, + url: body.id + } + + const created = await AccountVideoRateModel.upsert(entry) if (created) rateCounts += 1 } catch (err) { diff --git a/server/lib/activitypub/videos.ts b/server/lib/activitypub/videos.ts index 66d0abf35..2c932371b 100644 --- a/server/lib/activitypub/videos.ts +++ b/server/lib/activitypub/videos.ts @@ -40,6 +40,9 @@ import { Notifier } from '../notifier' import { VideoStreamingPlaylistModel } from '../../models/video/video-streaming-playlist' import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type' import { FilteredModelAttributes } from 'sequelize-typescript/lib/models/Model' +import { AccountVideoRateModel } from '../../models/account/account-video-rate' +import { VideoShareModel } from '../../models/video/video-share' +import { VideoCommentModel } from '../../models/video/video-comment' async function federateVideoIfNeeded (video: VideoModel, isNewVideo: boolean, transaction?: sequelize.Transaction) { // If the video is not private and published, we federate it @@ -134,31 +137,43 @@ async function syncVideoExternalAttributes (video: VideoModel, fetchedVideo: Vid const jobPayloads: ActivitypubHttpFetcherPayload[] = [] if (syncParam.likes === true) { - await crawlCollectionPage(fetchedVideo.likes, items => createRates(items, video, 'like')) + const handler = items => createRates(items, video, 'like') + const cleaner = crawlStartDate => AccountVideoRateModel.cleanOldRatesOf(video.id, 'like' as 'like', crawlStartDate) + + await crawlCollectionPage(fetchedVideo.likes, handler, cleaner) .catch(err => logger.error('Cannot add likes of video %s.', video.uuid, { err })) } else { jobPayloads.push({ uri: fetchedVideo.likes, videoId: video.id, type: 'video-likes' as 'video-likes' }) } if (syncParam.dislikes === true) { - await crawlCollectionPage(fetchedVideo.dislikes, items => createRates(items, video, 'dislike')) + const handler = items => createRates(items, video, 'dislike') + const cleaner = crawlStartDate => AccountVideoRateModel.cleanOldRatesOf(video.id, 'dislike' as 'dislike', crawlStartDate) + + await crawlCollectionPage(fetchedVideo.dislikes, handler, cleaner) .catch(err => logger.error('Cannot add dislikes of video %s.', video.uuid, { err })) } else { jobPayloads.push({ uri: fetchedVideo.dislikes, videoId: video.id, type: 'video-dislikes' as 'video-dislikes' }) } if (syncParam.shares === true) { - await crawlCollectionPage(fetchedVideo.shares, items => addVideoShares(items, video)) + const handler = items => addVideoShares(items, video) + const cleaner = crawlStartDate => VideoShareModel.cleanOldSharesOf(video.id, crawlStartDate) + + await crawlCollectionPage(fetchedVideo.shares, handler, cleaner) .catch(err => logger.error('Cannot add shares of video %s.', video.uuid, { err })) } else { jobPayloads.push({ uri: fetchedVideo.shares, videoId: video.id, type: 'video-shares' as 'video-shares' }) } if (syncParam.comments === true) { - await crawlCollectionPage(fetchedVideo.comments, items => addVideoComments(items, video)) + const handler = items => addVideoComments(items, video) + const cleaner = crawlStartDate => VideoCommentModel.cleanOldCommentsOf(video.id, crawlStartDate) + + await crawlCollectionPage(fetchedVideo.comments, handler, cleaner) .catch(err => logger.error('Cannot add comments of video %s.', video.uuid, { err })) } else { - jobPayloads.push({ uri: fetchedVideo.shares, videoId: video.id, type: 'video-shares' as 'video-shares' }) + jobPayloads.push({ uri: fetchedVideo.comments, videoId: video.id, type: 'video-comments' as 'video-comments' }) } await Bluebird.map(jobPayloads, payload => JobQueue.Instance.createJob({ type: 'activitypub-http-fetcher', payload })) -- cgit v1.2.3