aboutsummaryrefslogtreecommitdiffhomepage
path: root/server/lib/activitypub
diff options
context:
space:
mode:
authorChocobozzz <me@florianbigard.com>2019-03-19 16:23:02 +0100
committerChocobozzz <me@florianbigard.com>2019-03-19 16:23:02 +0100
commit2ba92871319d7af63472c1380664a9f9eeb1c690 (patch)
treed593b2dfea29a8171b9f6afaaef076321f5edf71 /server/lib/activitypub
parentd74d29ad9e35929491cf37223398d2535ab23de0 (diff)
downloadPeerTube-2ba92871319d7af63472c1380664a9f9eeb1c690.tar.gz
PeerTube-2ba92871319d7af63472c1380664a9f9eeb1c690.tar.zst
PeerTube-2ba92871319d7af63472c1380664a9f9eeb1c690.zip
Cleanup invalid rates/comments/shares
Diffstat (limited to 'server/lib/activitypub')
-rw-r--r--server/lib/activitypub/crawl.ts9
-rw-r--r--server/lib/activitypub/share.ts7
-rw-r--r--server/lib/activitypub/video-comments.ts10
-rw-r--r--server/lib/activitypub/video-rates.ts21
-rw-r--r--server/lib/activitypub/videos.ts25
5 files changed, 39 insertions, 33 deletions
diff --git a/server/lib/activitypub/crawl.ts b/server/lib/activitypub/crawl.ts
index 2675524c6..9f4ca98ba 100644
--- a/server/lib/activitypub/crawl.ts
+++ b/server/lib/activitypub/crawl.ts
@@ -4,7 +4,10 @@ import { logger } from '../../helpers/logger'
4import * as Bluebird from 'bluebird' 4import * as Bluebird from 'bluebird'
5import { ActivityPubOrderedCollection } from '../../../shared/models/activitypub' 5import { ActivityPubOrderedCollection } from '../../../shared/models/activitypub'
6 6
7async function crawlCollectionPage <T> (uri: string, handler: (items: T[]) => (Promise<any> | Bluebird<any>)) { 7type HandlerFunction<T> = (items: T[]) => (Promise<any> | Bluebird<any>)
8type CleanerFunction = (startedDate: Date) => (Promise<any> | Bluebird<any>)
9
10async function crawlCollectionPage <T> (uri: string, handler: HandlerFunction<T>, cleaner?: CleanerFunction) {
8 logger.info('Crawling ActivityPub data on %s.', uri) 11 logger.info('Crawling ActivityPub data on %s.', uri)
9 12
10 const options = { 13 const options = {
@@ -15,6 +18,8 @@ async function crawlCollectionPage <T> (uri: string, handler: (items: T[]) => (P
15 timeout: JOB_REQUEST_TIMEOUT 18 timeout: JOB_REQUEST_TIMEOUT
16 } 19 }
17 20
21 const startDate = new Date()
22
18 const response = await doRequest<ActivityPubOrderedCollection<T>>(options) 23 const response = await doRequest<ActivityPubOrderedCollection<T>>(options)
19 const firstBody = response.body 24 const firstBody = response.body
20 25
@@ -35,6 +40,8 @@ async function crawlCollectionPage <T> (uri: string, handler: (items: T[]) => (P
35 await handler(items) 40 await handler(items)
36 } 41 }
37 } 42 }
43
44 if (cleaner) await cleaner(startDate)
38} 45}
39 46
40export { 47export {
diff --git a/server/lib/activitypub/share.ts b/server/lib/activitypub/share.ts
index 1767df0ae..3bece0ff7 100644
--- a/server/lib/activitypub/share.ts
+++ b/server/lib/activitypub/share.ts
@@ -54,12 +54,7 @@ async function addVideoShares (shareUrls: string[], instance: VideoModel) {
54 url: shareUrl 54 url: shareUrl
55 } 55 }
56 56
57 await VideoShareModel.findOrCreate({ 57 await VideoShareModel.upsert(entry)
58 where: {
59 url: shareUrl
60 },
61 defaults: entry
62 })
63 } catch (err) { 58 } catch (err) {
64 logger.warn('Cannot add share %s.', shareUrl, { err }) 59 logger.warn('Cannot add share %s.', shareUrl, { err })
65 } 60 }
diff --git a/server/lib/activitypub/video-comments.ts b/server/lib/activitypub/video-comments.ts
index e87301fe7..3f9d8f0fc 100644
--- a/server/lib/activitypub/video-comments.ts
+++ b/server/lib/activitypub/video-comments.ts
@@ -34,8 +34,7 @@ async function videoCommentActivityObjectToDBAttributes (video: VideoModel, acto
34 accountId: actor.Account.id, 34 accountId: actor.Account.id,
35 inReplyToCommentId, 35 inReplyToCommentId,
36 originCommentId, 36 originCommentId,
37 createdAt: new Date(comment.published), 37 createdAt: new Date(comment.published)
38 updatedAt: new Date(comment.updated)
39 } 38 }
40} 39}
41 40
@@ -74,12 +73,7 @@ async function addVideoComment (videoInstance: VideoModel, commentUrl: string) {
74 const entry = await videoCommentActivityObjectToDBAttributes(videoInstance, actor, body) 73 const entry = await videoCommentActivityObjectToDBAttributes(videoInstance, actor, body)
75 if (!entry) return { created: false } 74 if (!entry) return { created: false }
76 75
77 const [ comment, created ] = await VideoCommentModel.findOrCreate({ 76 const [ comment, created ] = await VideoCommentModel.upsert<VideoCommentModel>(entry, { returning: true })
78 where: {
79 url: body.id
80 },
81 defaults: entry
82 })
83 comment.Account = actor.Account 77 comment.Account = actor.Account
84 comment.Video = videoInstance 78 comment.Video = videoInstance
85 79
diff --git a/server/lib/activitypub/video-rates.ts b/server/lib/activitypub/video-rates.ts
index 7aac79118..ad7d81df6 100644
--- a/server/lib/activitypub/video-rates.ts
+++ b/server/lib/activitypub/video-rates.ts
@@ -38,19 +38,14 @@ async function createRates (ratesUrl: string[], video: VideoModel, rate: VideoRa
38 38
39 const actor = await getOrCreateActorAndServerAndModel(actorUrl) 39 const actor = await getOrCreateActorAndServerAndModel(actorUrl)
40 40
41 const [ , created ] = await AccountVideoRateModel 41 const entry = {
42 .findOrCreate({ 42 videoId: video.id,
43 where: { 43 accountId: actor.Account.id,
44 videoId: video.id, 44 type: rate,
45 accountId: actor.Account.id 45 url: body.id
46 }, 46 }
47 defaults: { 47
48 videoId: video.id, 48 const created = await AccountVideoRateModel.upsert(entry)
49 accountId: actor.Account.id,
50 type: rate,
51 url: body.id
52 }
53 })
54 49
55 if (created) rateCounts += 1 50 if (created) rateCounts += 1
56 } catch (err) { 51 } catch (err) {
diff --git a/server/lib/activitypub/videos.ts b/server/lib/activitypub/videos.ts
index 66d0abf35..2c932371b 100644
--- a/server/lib/activitypub/videos.ts
+++ b/server/lib/activitypub/videos.ts
@@ -40,6 +40,9 @@ import { Notifier } from '../notifier'
40import { VideoStreamingPlaylistModel } from '../../models/video/video-streaming-playlist' 40import { VideoStreamingPlaylistModel } from '../../models/video/video-streaming-playlist'
41import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type' 41import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type'
42import { FilteredModelAttributes } from 'sequelize-typescript/lib/models/Model' 42import { FilteredModelAttributes } from 'sequelize-typescript/lib/models/Model'
43import { AccountVideoRateModel } from '../../models/account/account-video-rate'
44import { VideoShareModel } from '../../models/video/video-share'
45import { VideoCommentModel } from '../../models/video/video-comment'
43 46
44async function federateVideoIfNeeded (video: VideoModel, isNewVideo: boolean, transaction?: sequelize.Transaction) { 47async function federateVideoIfNeeded (video: VideoModel, isNewVideo: boolean, transaction?: sequelize.Transaction) {
45 // If the video is not private and published, we federate it 48 // If the video is not private and published, we federate it
@@ -134,31 +137,43 @@ async function syncVideoExternalAttributes (video: VideoModel, fetchedVideo: Vid
134 const jobPayloads: ActivitypubHttpFetcherPayload[] = [] 137 const jobPayloads: ActivitypubHttpFetcherPayload[] = []
135 138
136 if (syncParam.likes === true) { 139 if (syncParam.likes === true) {
137 await crawlCollectionPage<string>(fetchedVideo.likes, items => createRates(items, video, 'like')) 140 const handler = items => createRates(items, video, 'like')
141 const cleaner = crawlStartDate => AccountVideoRateModel.cleanOldRatesOf(video.id, 'like' as 'like', crawlStartDate)
142
143 await crawlCollectionPage<string>(fetchedVideo.likes, handler, cleaner)
138 .catch(err => logger.error('Cannot add likes of video %s.', video.uuid, { err })) 144 .catch(err => logger.error('Cannot add likes of video %s.', video.uuid, { err }))
139 } else { 145 } else {
140 jobPayloads.push({ uri: fetchedVideo.likes, videoId: video.id, type: 'video-likes' as 'video-likes' }) 146 jobPayloads.push({ uri: fetchedVideo.likes, videoId: video.id, type: 'video-likes' as 'video-likes' })
141 } 147 }
142 148
143 if (syncParam.dislikes === true) { 149 if (syncParam.dislikes === true) {
144 await crawlCollectionPage<string>(fetchedVideo.dislikes, items => createRates(items, video, 'dislike')) 150 const handler = items => createRates(items, video, 'dislike')
151 const cleaner = crawlStartDate => AccountVideoRateModel.cleanOldRatesOf(video.id, 'dislike' as 'dislike', crawlStartDate)
152
153 await crawlCollectionPage<string>(fetchedVideo.dislikes, handler, cleaner)
145 .catch(err => logger.error('Cannot add dislikes of video %s.', video.uuid, { err })) 154 .catch(err => logger.error('Cannot add dislikes of video %s.', video.uuid, { err }))
146 } else { 155 } else {
147 jobPayloads.push({ uri: fetchedVideo.dislikes, videoId: video.id, type: 'video-dislikes' as 'video-dislikes' }) 156 jobPayloads.push({ uri: fetchedVideo.dislikes, videoId: video.id, type: 'video-dislikes' as 'video-dislikes' })
148 } 157 }
149 158
150 if (syncParam.shares === true) { 159 if (syncParam.shares === true) {
151 await crawlCollectionPage<string>(fetchedVideo.shares, items => addVideoShares(items, video)) 160 const handler = items => addVideoShares(items, video)
161 const cleaner = crawlStartDate => VideoShareModel.cleanOldSharesOf(video.id, crawlStartDate)
162
163 await crawlCollectionPage<string>(fetchedVideo.shares, handler, cleaner)
152 .catch(err => logger.error('Cannot add shares of video %s.', video.uuid, { err })) 164 .catch(err => logger.error('Cannot add shares of video %s.', video.uuid, { err }))
153 } else { 165 } else {
154 jobPayloads.push({ uri: fetchedVideo.shares, videoId: video.id, type: 'video-shares' as 'video-shares' }) 166 jobPayloads.push({ uri: fetchedVideo.shares, videoId: video.id, type: 'video-shares' as 'video-shares' })
155 } 167 }
156 168
157 if (syncParam.comments === true) { 169 if (syncParam.comments === true) {
158 await crawlCollectionPage<string>(fetchedVideo.comments, items => addVideoComments(items, video)) 170 const handler = items => addVideoComments(items, video)
171 const cleaner = crawlStartDate => VideoCommentModel.cleanOldCommentsOf(video.id, crawlStartDate)
172
173 await crawlCollectionPage<string>(fetchedVideo.comments, handler, cleaner)
159 .catch(err => logger.error('Cannot add comments of video %s.', video.uuid, { err })) 174 .catch(err => logger.error('Cannot add comments of video %s.', video.uuid, { err }))
160 } else { 175 } else {
161 jobPayloads.push({ uri: fetchedVideo.shares, videoId: video.id, type: 'video-shares' as 'video-shares' }) 176 jobPayloads.push({ uri: fetchedVideo.comments, videoId: video.id, type: 'video-comments' as 'video-comments' })
162 } 177 }
163 178
164 await Bluebird.map(jobPayloads, payload => JobQueue.Instance.createJob({ type: 'activitypub-http-fetcher', payload })) 179 await Bluebird.map(jobPayloads, payload => JobQueue.Instance.createJob({ type: 'activitypub-http-fetcher', payload }))