From 2ba92871319d7af63472c1380664a9f9eeb1c690 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Tue, 19 Mar 2019 16:23:02 +0100 Subject: [PATCH] Cleanup invalid rates/comments/shares --- server/controllers/api/users/my-blocklist.ts | 4 +-- server/controllers/api/videos/watching.ts | 1 - server/helpers/audit-logger.ts | 5 ++-- server/lib/activitypub/crawl.ts | 9 +++++- server/lib/activitypub/share.ts | 7 +---- server/lib/activitypub/video-comments.ts | 10 ++----- server/lib/activitypub/video-rates.ts | 21 +++++--------- server/lib/activitypub/videos.ts | 25 ++++++++++++---- .../handlers/activitypub-http-fetcher.ts | 13 ++++++++- server/middlewares/validators/users.ts | 2 +- server/models/account/account-video-rate.ts | 29 ++++++++++++++++++- server/models/video/video-comment.ts | 14 +++++++++ server/models/video/video-share.ts | 14 +++++++++ server/models/video/video.ts | 2 +- server/tests/api/activitypub/refresher.ts | 2 +- server/tests/api/videos/multiple-servers.ts | 8 ++--- 16 files changed, 117 insertions(+), 49 deletions(-) diff --git a/server/controllers/api/users/my-blocklist.ts b/server/controllers/api/users/my-blocklist.ts index 481e75139..713c16022 100644 --- a/server/controllers/api/users/my-blocklist.ts +++ b/server/controllers/api/users/my-blocklist.ts @@ -17,11 +17,9 @@ import { serversBlocklistSortValidator, unblockServerByAccountValidator } from '../../../middlewares/validators' -import { AccountModel } from '../../../models/account/account' import { AccountBlocklistModel } from '../../../models/account/account-blocklist' import { addAccountInBlocklist, addServerInBlocklist, removeAccountFromBlocklist, removeServerFromBlocklist } from '../../../lib/blocklist' import { ServerBlocklistModel } from '../../../models/server/server-blocklist' -import { ServerModel } from '../../../models/server/server' const myBlocklistRouter = express.Router() @@ -83,7 +81,7 @@ async function listBlockedAccounts (req: express.Request, res: express.Response) async function blockAccount (req: express.Request, res: express.Response) { const user = res.locals.oauth.token.User - const accountToBlock = res.locals.account + const accountToBlock = res.locals.account await addAccountInBlocklist(user.Account.id, accountToBlock.id) diff --git a/server/controllers/api/videos/watching.ts b/server/controllers/api/videos/watching.ts index 6bc60e045..dcd1f070d 100644 --- a/server/controllers/api/videos/watching.ts +++ b/server/controllers/api/videos/watching.ts @@ -2,7 +2,6 @@ import * as express from 'express' import { UserWatchingVideo } from '../../../../shared' import { asyncMiddleware, asyncRetryTransactionMiddleware, authenticate, videoWatchingValidator } from '../../../middlewares' import { UserVideoHistoryModel } from '../../../models/account/user-video-history' -import { UserModel } from '../../../models/account/user' const watchingRouter = express.Router() diff --git a/server/helpers/audit-logger.ts b/server/helpers/audit-logger.ts index a121f0b8a..af37bce16 100644 --- a/server/helpers/audit-logger.ts +++ b/server/helpers/audit-logger.ts @@ -6,13 +6,12 @@ import * as flatten from 'flat' import * as winston from 'winston' import { CONFIG } from '../initializers' import { jsonLoggerFormat, labelFormatter } from './logger' -import { VideoDetails, User, VideoChannel, VideoAbuse, VideoImport } from '../../shared' +import { User, VideoAbuse, VideoChannel, VideoDetails, VideoImport } from '../../shared' import { VideoComment } from '../../shared/models/videos/video-comment.model' import { CustomConfig } from '../../shared/models/server/custom-config.model' -import { UserModel } from '../models/account/user' function getAuditIdFromRes (res: express.Response) { - return (res.locals.oauth.token.User as UserModel).username + return res.locals.oauth.token.User.username } enum AUDIT_TYPE { diff --git a/server/lib/activitypub/crawl.ts b/server/lib/activitypub/crawl.ts index 2675524c6..9f4ca98ba 100644 --- a/server/lib/activitypub/crawl.ts +++ b/server/lib/activitypub/crawl.ts @@ -4,7 +4,10 @@ import { logger } from '../../helpers/logger' import * as Bluebird from 'bluebird' import { ActivityPubOrderedCollection } from '../../../shared/models/activitypub' -async function crawlCollectionPage (uri: string, handler: (items: T[]) => (Promise | Bluebird)) { +type HandlerFunction = (items: T[]) => (Promise | Bluebird) +type CleanerFunction = (startedDate: Date) => (Promise | Bluebird) + +async function crawlCollectionPage (uri: string, handler: HandlerFunction, cleaner?: CleanerFunction) { logger.info('Crawling ActivityPub data on %s.', uri) const options = { @@ -15,6 +18,8 @@ async function crawlCollectionPage (uri: string, handler: (items: T[]) => (P timeout: JOB_REQUEST_TIMEOUT } + const startDate = new Date() + const response = await doRequest>(options) const firstBody = response.body @@ -35,6 +40,8 @@ async function crawlCollectionPage (uri: string, handler: (items: T[]) => (P await handler(items) } } + + if (cleaner) await cleaner(startDate) } export { diff --git a/server/lib/activitypub/share.ts b/server/lib/activitypub/share.ts index 1767df0ae..3bece0ff7 100644 --- a/server/lib/activitypub/share.ts +++ b/server/lib/activitypub/share.ts @@ -54,12 +54,7 @@ async function addVideoShares (shareUrls: string[], instance: VideoModel) { url: shareUrl } - await VideoShareModel.findOrCreate({ - where: { - url: shareUrl - }, - defaults: entry - }) + await VideoShareModel.upsert(entry) } catch (err) { logger.warn('Cannot add share %s.', shareUrl, { err }) } diff --git a/server/lib/activitypub/video-comments.ts b/server/lib/activitypub/video-comments.ts index e87301fe7..3f9d8f0fc 100644 --- a/server/lib/activitypub/video-comments.ts +++ b/server/lib/activitypub/video-comments.ts @@ -34,8 +34,7 @@ async function videoCommentActivityObjectToDBAttributes (video: VideoModel, acto accountId: actor.Account.id, inReplyToCommentId, originCommentId, - createdAt: new Date(comment.published), - updatedAt: new Date(comment.updated) + createdAt: new Date(comment.published) } } @@ -74,12 +73,7 @@ async function addVideoComment (videoInstance: VideoModel, commentUrl: string) { const entry = await videoCommentActivityObjectToDBAttributes(videoInstance, actor, body) if (!entry) return { created: false } - const [ comment, created ] = await VideoCommentModel.findOrCreate({ - where: { - url: body.id - }, - defaults: entry - }) + const [ comment, created ] = await VideoCommentModel.upsert(entry, { returning: true }) comment.Account = actor.Account comment.Video = videoInstance diff --git a/server/lib/activitypub/video-rates.ts b/server/lib/activitypub/video-rates.ts index 7aac79118..ad7d81df6 100644 --- a/server/lib/activitypub/video-rates.ts +++ b/server/lib/activitypub/video-rates.ts @@ -38,19 +38,14 @@ async function createRates (ratesUrl: string[], video: VideoModel, rate: VideoRa const actor = await getOrCreateActorAndServerAndModel(actorUrl) - const [ , created ] = await AccountVideoRateModel - .findOrCreate({ - where: { - videoId: video.id, - accountId: actor.Account.id - }, - defaults: { - videoId: video.id, - accountId: actor.Account.id, - type: rate, - url: body.id - } - }) + const entry = { + videoId: video.id, + accountId: actor.Account.id, + type: rate, + url: body.id + } + + const created = await AccountVideoRateModel.upsert(entry) if (created) rateCounts += 1 } catch (err) { diff --git a/server/lib/activitypub/videos.ts b/server/lib/activitypub/videos.ts index 66d0abf35..2c932371b 100644 --- a/server/lib/activitypub/videos.ts +++ b/server/lib/activitypub/videos.ts @@ -40,6 +40,9 @@ import { Notifier } from '../notifier' import { VideoStreamingPlaylistModel } from '../../models/video/video-streaming-playlist' import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type' import { FilteredModelAttributes } from 'sequelize-typescript/lib/models/Model' +import { AccountVideoRateModel } from '../../models/account/account-video-rate' +import { VideoShareModel } from '../../models/video/video-share' +import { VideoCommentModel } from '../../models/video/video-comment' async function federateVideoIfNeeded (video: VideoModel, isNewVideo: boolean, transaction?: sequelize.Transaction) { // If the video is not private and published, we federate it @@ -134,31 +137,43 @@ async function syncVideoExternalAttributes (video: VideoModel, fetchedVideo: Vid const jobPayloads: ActivitypubHttpFetcherPayload[] = [] if (syncParam.likes === true) { - await crawlCollectionPage(fetchedVideo.likes, items => createRates(items, video, 'like')) + const handler = items => createRates(items, video, 'like') + const cleaner = crawlStartDate => AccountVideoRateModel.cleanOldRatesOf(video.id, 'like' as 'like', crawlStartDate) + + await crawlCollectionPage(fetchedVideo.likes, handler, cleaner) .catch(err => logger.error('Cannot add likes of video %s.', video.uuid, { err })) } else { jobPayloads.push({ uri: fetchedVideo.likes, videoId: video.id, type: 'video-likes' as 'video-likes' }) } if (syncParam.dislikes === true) { - await crawlCollectionPage(fetchedVideo.dislikes, items => createRates(items, video, 'dislike')) + const handler = items => createRates(items, video, 'dislike') + const cleaner = crawlStartDate => AccountVideoRateModel.cleanOldRatesOf(video.id, 'dislike' as 'dislike', crawlStartDate) + + await crawlCollectionPage(fetchedVideo.dislikes, handler, cleaner) .catch(err => logger.error('Cannot add dislikes of video %s.', video.uuid, { err })) } else { jobPayloads.push({ uri: fetchedVideo.dislikes, videoId: video.id, type: 'video-dislikes' as 'video-dislikes' }) } if (syncParam.shares === true) { - await crawlCollectionPage(fetchedVideo.shares, items => addVideoShares(items, video)) + const handler = items => addVideoShares(items, video) + const cleaner = crawlStartDate => VideoShareModel.cleanOldSharesOf(video.id, crawlStartDate) + + await crawlCollectionPage(fetchedVideo.shares, handler, cleaner) .catch(err => logger.error('Cannot add shares of video %s.', video.uuid, { err })) } else { jobPayloads.push({ uri: fetchedVideo.shares, videoId: video.id, type: 'video-shares' as 'video-shares' }) } if (syncParam.comments === true) { - await crawlCollectionPage(fetchedVideo.comments, items => addVideoComments(items, video)) + const handler = items => addVideoComments(items, video) + const cleaner = crawlStartDate => VideoCommentModel.cleanOldCommentsOf(video.id, crawlStartDate) + + await crawlCollectionPage(fetchedVideo.comments, handler, cleaner) .catch(err => logger.error('Cannot add comments of video %s.', video.uuid, { err })) } else { - jobPayloads.push({ uri: fetchedVideo.shares, videoId: video.id, type: 'video-shares' as 'video-shares' }) + jobPayloads.push({ uri: fetchedVideo.comments, videoId: video.id, type: 'video-comments' as 'video-comments' }) } await Bluebird.map(jobPayloads, payload => JobQueue.Instance.createJob({ type: 'activitypub-http-fetcher', payload })) diff --git a/server/lib/job-queue/handlers/activitypub-http-fetcher.ts b/server/lib/job-queue/handlers/activitypub-http-fetcher.ts index 52225f64f..23d33c26f 100644 --- a/server/lib/job-queue/handlers/activitypub-http-fetcher.ts +++ b/server/lib/job-queue/handlers/activitypub-http-fetcher.ts @@ -1,4 +1,5 @@ import * as Bull from 'bull' +import * as Bluebird from 'bluebird' import { logger } from '../../../helpers/logger' import { processActivities } from '../../activitypub/process' import { addVideoComments } from '../../activitypub/video-comments' @@ -7,6 +8,9 @@ import { VideoModel } from '../../../models/video/video' import { addVideoShares, createRates } from '../../activitypub' import { createAccountPlaylists } from '../../activitypub/playlist' import { AccountModel } from '../../../models/account/account' +import { AccountVideoRateModel } from '../../../models/account/account-video-rate' +import { VideoShareModel } from '../../../models/video/video-share' +import { VideoCommentModel } from '../../../models/video/video-comment' type FetchType = 'activity' | 'video-likes' | 'video-dislikes' | 'video-shares' | 'video-comments' | 'account-playlists' @@ -37,7 +41,14 @@ async function processActivityPubHttpFetcher (job: Bull.Job) { 'account-playlists': items => createAccountPlaylists(items, account) } - return crawlCollectionPage(payload.uri, fetcherType[payload.type]) + const cleanerType: { [ id in FetchType ]?: (crawlStartDate: Date) => Bluebird } = { + 'video-likes': crawlStartDate => AccountVideoRateModel.cleanOldRatesOf(video.id, 'like' as 'like', crawlStartDate), + 'video-dislikes': crawlStartDate => AccountVideoRateModel.cleanOldRatesOf(video.id, 'dislike' as 'dislike', crawlStartDate), + 'video-shares': crawlStartDate => VideoShareModel.cleanOldSharesOf(video.id, crawlStartDate), + 'video-comments': crawlStartDate => VideoCommentModel.cleanOldCommentsOf(video.id, crawlStartDate) + } + + return crawlCollectionPage(payload.uri, fetcherType[payload.type], cleanerType[payload.type]) } // --------------------------------------------------------------------------- diff --git a/server/middlewares/validators/users.ts b/server/middlewares/validators/users.ts index e8ade0f97..4be446732 100644 --- a/server/middlewares/validators/users.ts +++ b/server/middlewares/validators/users.ts @@ -160,7 +160,7 @@ const usersUpdateMeValidator = [ .end() } - const user= res.locals.oauth.token.User + const user = res.locals.oauth.token.User if (await user.isPasswordMatch(req.body.currentPassword) !== true) { return res.status(401) .send({ error: 'currentPassword is invalid.' }) diff --git a/server/models/account/account-video-rate.ts b/server/models/account/account-video-rate.ts index 18762f0c5..e5d39582b 100644 --- a/server/models/account/account-video-rate.ts +++ b/server/models/account/account-video-rate.ts @@ -1,5 +1,5 @@ import { values } from 'lodash' -import { Transaction } from 'sequelize' +import { Transaction, Op } from 'sequelize' import { AllowNull, BelongsTo, Column, CreatedAt, DataType, ForeignKey, Is, Model, Table, UpdatedAt } from 'sequelize-typescript' import { IFindOptions } from 'sequelize-typescript/lib/interfaces/IFindOptions' import { VideoRateType } from '../../../shared/models/videos' @@ -158,4 +158,31 @@ export class AccountVideoRateModel extends Model { return AccountVideoRateModel.findAndCountAll(query) } + + static cleanOldRatesOf (videoId: number, type: VideoRateType, beforeUpdatedAt: Date) { + return AccountVideoRateModel.sequelize.transaction(async t => { + const query = { + where: { + updatedAt: { + [Op.lt]: beforeUpdatedAt + }, + videoId, + type + }, + transaction: t + } + + const deleted = await AccountVideoRateModel.destroy(query) + + const options = { + transaction: t, + where: { + id: videoId + } + } + + if (type === 'like') await VideoModel.increment({ likes: -deleted }, options) + else if (type === 'dislike') await VideoModel.increment({ dislikes: -deleted }, options) + }) + } } diff --git a/server/models/video/video-comment.ts b/server/models/video/video-comment.ts index 1163f9a0e..e733138c1 100644 --- a/server/models/video/video-comment.ts +++ b/server/models/video/video-comment.ts @@ -1,4 +1,5 @@ import * as Sequelize from 'sequelize' +import { Op } from 'sequelize' import { AllowNull, BeforeDestroy, @@ -453,6 +454,19 @@ export class VideoCommentModel extends Model { } } + static cleanOldCommentsOf (videoId: number, beforeUpdatedAt: Date) { + const query = { + where: { + updatedAt: { + [Op.lt]: beforeUpdatedAt + }, + videoId + } + } + + return VideoCommentModel.destroy(query) + } + getCommentStaticPath () { return this.Video.getWatchStaticPath() + ';threadId=' + this.getThreadId() } diff --git a/server/models/video/video-share.ts b/server/models/video/video-share.ts index 7df0ed18d..fb52b35d9 100644 --- a/server/models/video/video-share.ts +++ b/server/models/video/video-share.ts @@ -1,4 +1,5 @@ import * as Sequelize from 'sequelize' +import { Op } from 'sequelize' import * as Bluebird from 'bluebird' import { AllowNull, BelongsTo, Column, CreatedAt, DataType, ForeignKey, Is, Model, Scopes, Table, UpdatedAt } from 'sequelize-typescript' import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc' @@ -200,4 +201,17 @@ export class VideoShareModel extends Model { return VideoShareModel.findAndCountAll(query) } + + static cleanOldSharesOf (videoId: number, beforeUpdatedAt: Date) { + const query = { + where: { + updatedAt: { + [Op.lt]: beforeUpdatedAt + }, + videoId + } + } + + return VideoShareModel.destroy(query) + } } diff --git a/server/models/video/video.ts b/server/models/video/video.ts index fb037c21a..b0d92b674 100644 --- a/server/models/video/video.ts +++ b/server/models/video/video.ts @@ -1547,7 +1547,7 @@ export class VideoModel extends Model { attributes: query.attributes, order: [ // Keep original order Sequelize.literal( - ids.map(id => `"VideoModel".id = ${id}`).join(', ') + ids.map(id => `"VideoModel".id = ${id} DESC`).join(', ') ) ] } diff --git a/server/tests/api/activitypub/refresher.ts b/server/tests/api/activitypub/refresher.ts index ae4859076..665a9f9f0 100644 --- a/server/tests/api/activitypub/refresher.ts +++ b/server/tests/api/activitypub/refresher.ts @@ -8,7 +8,7 @@ import { generateUserAccessToken, getVideo, getVideoPlaylist, - killallServers, + killallServers, rateVideo, reRunServer, ServerInfo, setAccessTokensToServers, diff --git a/server/tests/api/videos/multiple-servers.ts b/server/tests/api/videos/multiple-servers.ts index 7e2fcb630..f91678140 100644 --- a/server/tests/api/videos/multiple-servers.ts +++ b/server/tests/api/videos/multiple-servers.ts @@ -579,15 +579,15 @@ describe('Test multiple servers', function () { this.timeout(20000) await rateVideo(servers[0].url, servers[0].accessToken, remoteVideosServer1[0], 'like') - await wait(200) + await wait(500) await rateVideo(servers[0].url, servers[0].accessToken, remoteVideosServer1[0], 'dislike') - await wait(200) + await wait(500) await rateVideo(servers[0].url, servers[0].accessToken, remoteVideosServer1[0], 'like') await rateVideo(servers[2].url, servers[2].accessToken, localVideosServer3[1], 'like') - await wait(200) + await wait(500) await rateVideo(servers[2].url, servers[2].accessToken, localVideosServer3[1], 'dislike') await rateVideo(servers[2].url, servers[2].accessToken, remoteVideosServer3[1], 'dislike') - await wait(200) + await wait(500) await rateVideo(servers[2].url, servers[2].accessToken, remoteVideosServer3[0], 'like') await waitJobs(servers) -- 2.41.0