aboutsummaryrefslogtreecommitdiffhomepage
path: root/server/lib/activitypub
diff options
context:
space:
mode:
authorChocobozzz <me@florianbigard.com>2019-08-06 17:19:53 +0200
committerChocobozzz <me@florianbigard.com>2019-08-06 17:26:51 +0200
commit6b9c966f6428c9e47bead3410a0401e8ebd744bf (patch)
tree282218ec56725b0e2e878b0471cd08a54fd91998 /server/lib/activitypub
parent466e3f20a537f1eff4b4fd03297df11ba371d049 (diff)
downloadPeerTube-6b9c966f6428c9e47bead3410a0401e8ebd744bf.tar.gz
PeerTube-6b9c966f6428c9e47bead3410a0401e8ebd744bf.tar.zst
PeerTube-6b9c966f6428c9e47bead3410a0401e8ebd744bf.zip
Automatically remove bad followings
Diffstat (limited to 'server/lib/activitypub')
-rw-r--r--server/lib/activitypub/actor.ts10
-rw-r--r--server/lib/activitypub/process/process-create.ts11
-rw-r--r--server/lib/activitypub/video-comments.ts230
-rw-r--r--server/lib/activitypub/videos.ts16
4 files changed, 135 insertions, 132 deletions
diff --git a/server/lib/activitypub/actor.ts b/server/lib/activitypub/actor.ts
index 38eb87d1e..0e6596f10 100644
--- a/server/lib/activitypub/actor.ts
+++ b/server/lib/activitypub/actor.ts
@@ -254,14 +254,14 @@ async function refreshActorIfNeeded (
254 await actor.save({ transaction: t }) 254 await actor.save({ transaction: t })
255 255
256 if (actor.Account) { 256 if (actor.Account) {
257 actor.Account.set('name', result.name) 257 actor.Account.name = result.name
258 actor.Account.set('description', result.summary) 258 actor.Account.description = result.summary
259 259
260 await actor.Account.save({ transaction: t }) 260 await actor.Account.save({ transaction: t })
261 } else if (actor.VideoChannel) { 261 } else if (actor.VideoChannel) {
262 actor.VideoChannel.set('name', result.name) 262 actor.VideoChannel.name = result.name
263 actor.VideoChannel.set('description', result.summary) 263 actor.VideoChannel.description = result.summary
264 actor.VideoChannel.set('support', result.support) 264 actor.VideoChannel.support = result.support
265 265
266 await actor.VideoChannel.save({ transaction: t }) 266 await actor.VideoChannel.save({ transaction: t })
267 } 267 }
diff --git a/server/lib/activitypub/process/process-create.ts b/server/lib/activitypub/process/process-create.ts
index a979771b6..b81021163 100644
--- a/server/lib/activitypub/process/process-create.ts
+++ b/server/lib/activitypub/process/process-create.ts
@@ -4,7 +4,7 @@ import { retryTransactionWrapper } from '../../../helpers/database-utils'
4import { logger } from '../../../helpers/logger' 4import { logger } from '../../../helpers/logger'
5import { sequelizeTypescript } from '../../../initializers' 5import { sequelizeTypescript } from '../../../initializers'
6import { ActorModel } from '../../../models/activitypub/actor' 6import { ActorModel } from '../../../models/activitypub/actor'
7import { addVideoComment, resolveThread } from '../video-comments' 7import { resolveThread } from '../video-comments'
8import { getOrCreateVideoAndAccountAndChannel } from '../videos' 8import { getOrCreateVideoAndAccountAndChannel } from '../videos'
9import { forwardVideoRelatedActivity } from '../send/utils' 9import { forwardVideoRelatedActivity } from '../send/utils'
10import { createOrUpdateCacheFile } from '../cache-file' 10import { createOrUpdateCacheFile } from '../cache-file'
@@ -13,6 +13,7 @@ import { PlaylistObject } from '../../../../shared/models/activitypub/objects/pl
13import { createOrUpdateVideoPlaylist } from '../playlist' 13import { createOrUpdateVideoPlaylist } from '../playlist'
14import { VideoModel } from '../../../models/video/video' 14import { VideoModel } from '../../../models/video/video'
15import { APProcessorOptions } from '../../../typings/activitypub-processor.model' 15import { APProcessorOptions } from '../../../typings/activitypub-processor.model'
16import { VideoCommentModel } from '../../../models/video/video-comment'
16 17
17async function processCreateActivity (options: APProcessorOptions<ActivityCreate>) { 18async function processCreateActivity (options: APProcessorOptions<ActivityCreate>) {
18 const { activity, byActor } = options 19 const { activity, byActor } = options
@@ -83,9 +84,13 @@ async function processCreateVideoComment (activity: ActivityCreate, byActor: Act
83 if (!byAccount) throw new Error('Cannot create video comment with the non account actor ' + byActor.url) 84 if (!byAccount) throw new Error('Cannot create video comment with the non account actor ' + byActor.url)
84 85
85 let video: VideoModel 86 let video: VideoModel
87 let created: boolean
88 let comment: VideoCommentModel
86 try { 89 try {
87 const resolveThreadResult = await resolveThread(commentObject.inReplyTo) 90 const resolveThreadResult = await resolveThread({ url: commentObject.id, isVideo: false })
88 video = resolveThreadResult.video 91 video = resolveThreadResult.video
92 created = resolveThreadResult.commentCreated
93 comment = resolveThreadResult.comment
89 } catch (err) { 94 } catch (err) {
90 logger.debug( 95 logger.debug(
91 'Cannot process video comment because we could not resolve thread %s. Maybe it was not a video thread, so skip it.', 96 'Cannot process video comment because we could not resolve thread %s. Maybe it was not a video thread, so skip it.',
@@ -95,8 +100,6 @@ async function processCreateVideoComment (activity: ActivityCreate, byActor: Act
95 return 100 return
96 } 101 }
97 102
98 const { comment, created } = await addVideoComment(video, commentObject.id)
99
100 if (video.isOwned() && created === true) { 103 if (video.isOwned() && created === true) {
101 // Don't resend the activity to the sender 104 // Don't resend the activity to the sender
102 const exceptions = [ byActor ] 105 const exceptions = [ byActor ]
diff --git a/server/lib/activitypub/video-comments.ts b/server/lib/activitypub/video-comments.ts
index 921abdb8d..92e1a9020 100644
--- a/server/lib/activitypub/video-comments.ts
+++ b/server/lib/activitypub/video-comments.ts
@@ -1,9 +1,7 @@
1import { VideoCommentObject } from '../../../shared/models/activitypub/objects/video-comment-object'
2import { sanitizeAndCheckVideoCommentObject } from '../../helpers/custom-validators/activitypub/video-comments' 1import { sanitizeAndCheckVideoCommentObject } from '../../helpers/custom-validators/activitypub/video-comments'
3import { logger } from '../../helpers/logger' 2import { logger } from '../../helpers/logger'
4import { doRequest } from '../../helpers/requests' 3import { doRequest } from '../../helpers/requests'
5import { ACTIVITY_PUB, CRAWL_REQUEST_CONCURRENCY } from '../../initializers/constants' 4import { ACTIVITY_PUB, CRAWL_REQUEST_CONCURRENCY } from '../../initializers/constants'
6import { ActorModel } from '../../models/activitypub/actor'
7import { VideoModel } from '../../models/video/video' 5import { VideoModel } from '../../models/video/video'
8import { VideoCommentModel } from '../../models/video/video-comment' 6import { VideoCommentModel } from '../../models/video/video-comment'
9import { getOrCreateActorAndServerAndModel } from './actor' 7import { getOrCreateActorAndServerAndModel } from './actor'
@@ -11,79 +9,53 @@ import { getOrCreateVideoAndAccountAndChannel } from './videos'
11import * as Bluebird from 'bluebird' 9import * as Bluebird from 'bluebird'
12import { checkUrlsSameHost } from '../../helpers/activitypub' 10import { checkUrlsSameHost } from '../../helpers/activitypub'
13 11
14async function videoCommentActivityObjectToDBAttributes (video: VideoModel, actor: ActorModel, comment: VideoCommentObject) { 12type ResolveThreadParams = {
15 let originCommentId: number = null 13 url: string,
16 let inReplyToCommentId: number = null 14 comments?: VideoCommentModel[],
17 15 isVideo?: boolean,
18 // If this is not a reply to the video (thread), create or get the parent comment 16 commentCreated?: boolean
19 if (video.url !== comment.inReplyTo) {
20 const { comment: parent } = await addVideoComment(video, comment.inReplyTo)
21 if (!parent) {
22 logger.warn('Cannot fetch or get parent comment %s of comment %s.', comment.inReplyTo, comment.id)
23 return undefined
24 }
25
26 originCommentId = parent.originCommentId || parent.id
27 inReplyToCommentId = parent.id
28 }
29
30 return {
31 url: comment.id,
32 text: comment.content,
33 videoId: video.id,
34 accountId: actor.Account.id,
35 inReplyToCommentId,
36 originCommentId,
37 createdAt: new Date(comment.published)
38 }
39} 17}
18type ResolveThreadResult = Promise<{ video: VideoModel, comment: VideoCommentModel, commentCreated: boolean }>
40 19
41async function addVideoComments (commentUrls: string[], instance: VideoModel) { 20async function addVideoComments (commentUrls: string[]) {
42 return Bluebird.map(commentUrls, commentUrl => { 21 return Bluebird.map(commentUrls, commentUrl => {
43 return addVideoComment(instance, commentUrl) 22 return resolveThread({ url: commentUrl, isVideo: false })
44 }, { concurrency: CRAWL_REQUEST_CONCURRENCY }) 23 }, { concurrency: CRAWL_REQUEST_CONCURRENCY })
45} 24}
46 25
47async function addVideoComment (videoInstance: VideoModel, commentUrl: string) { 26async function resolveThread (params: ResolveThreadParams): ResolveThreadResult {
48 logger.info('Fetching remote video comment %s.', commentUrl) 27 const { url, isVideo } = params
28 if (params.commentCreated === undefined) params.commentCreated = false
29 if (params.comments === undefined) params.comments = []
49 30
50 const { body } = await doRequest({ 31 // Already have this comment?
51 uri: commentUrl, 32 if (isVideo !== true) {
52 json: true, 33 const result = await resolveCommentFromDB(params)
53 activityPub: true 34 if (result) return result
54 })
55
56 if (sanitizeAndCheckVideoCommentObject(body) === false) {
57 logger.debug('Remote video comment JSON %s is not valid.', commentUrl, { body })
58 return { created: false }
59 } 35 }
60 36
61 const actorUrl = body.attributedTo 37 try {
62 if (!actorUrl) return { created: false } 38 if (isVideo !== false) return await tryResolveThreadFromVideo(params)
63 39
64 if (checkUrlsSameHost(commentUrl, actorUrl) !== true) { 40 return resolveParentComment(params)
65 throw new Error(`Actor url ${actorUrl} has not the same host than the comment url ${commentUrl}`) 41 } catch (err) {
66 } 42 logger.debug('Cannot get or create account and video and channel for reply %s, fetch comment', url, { err })
67 43
68 if (checkUrlsSameHost(body.id, commentUrl) !== true) { 44 return resolveParentComment(params)
69 throw new Error(`Comment url ${commentUrl} host is different from the AP object id ${body.id}`)
70 } 45 }
46}
71 47
72 const actor = await getOrCreateActorAndServerAndModel(actorUrl, 'all') 48export {
73 const entry = await videoCommentActivityObjectToDBAttributes(videoInstance, actor, body) 49 addVideoComments,
74 if (!entry) return { created: false } 50 resolveThread
51}
75 52
76 const [ comment, created ] = await VideoCommentModel.upsert<VideoCommentModel>(entry, { returning: true }) 53// ---------------------------------------------------------------------------
77 comment.Account = actor.Account
78 comment.Video = videoInstance
79 54
80 return { comment, created } 55async function resolveCommentFromDB (params: ResolveThreadParams) {
81} 56 const { url, comments, commentCreated } = params
82 57
83type ResolveThreadResult = Promise<{ video: VideoModel, parents: VideoCommentModel[] }> 58 const commentFromDatabase = await VideoCommentModel.loadByUrlAndPopulateReplyAndVideoUrlAndAccount(url)
84async function resolveThread (url: string, comments: VideoCommentModel[] = []): ResolveThreadResult {
85 // Already have this comment?
86 const commentFromDatabase = await VideoCommentModel.loadByUrlAndPopulateReplyAndVideo(url)
87 if (commentFromDatabase) { 59 if (commentFromDatabase) {
88 let parentComments = comments.concat([ commentFromDatabase ]) 60 let parentComments = comments.concat([ commentFromDatabase ])
89 61
@@ -94,79 +66,97 @@ async function resolveThread (url: string, comments: VideoCommentModel[] = []):
94 parentComments = parentComments.concat(data) 66 parentComments = parentComments.concat(data)
95 } 67 }
96 68
97 return resolveThread(commentFromDatabase.Video.url, parentComments) 69 return resolveThread({
70 url: commentFromDatabase.Video.url,
71 comments: parentComments,
72 isVideo: true,
73 commentCreated
74 })
98 } 75 }
99 76
100 try { 77 return undefined
101 // Maybe it's a reply to a video? 78}
102 // If yes, it's done: we resolved all the thread 79
103 const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: url }) 80async function tryResolveThreadFromVideo (params: ResolveThreadParams) {
104 81 const { url, comments, commentCreated } = params
105 if (comments.length !== 0) { 82
106 const firstReply = comments[ comments.length - 1 ] 83 // Maybe it's a reply to a video?
107 firstReply.inReplyToCommentId = null 84 // If yes, it's done: we resolved all the thread
108 firstReply.originCommentId = null 85 const syncParam = { likes: true, dislikes: true, shares: true, comments: false, thumbnail: true, refreshVideo: false }
109 firstReply.videoId = video.id 86 const { video } = await getOrCreateVideoAndAccountAndChannel({ videoObject: url, syncParam })
110 comments[comments.length - 1] = await firstReply.save() 87
111 88 let resultComment: VideoCommentModel
112 for (let i = comments.length - 2; i >= 0; i--) { 89 if (comments.length !== 0) {
113 const comment = comments[ i ] 90 const firstReply = comments[ comments.length - 1 ]
114 comment.originCommentId = firstReply.id 91 firstReply.inReplyToCommentId = null
115 comment.inReplyToCommentId = comments[ i + 1 ].id 92 firstReply.originCommentId = null
116 comment.videoId = video.id 93 firstReply.videoId = video.id
117 94 firstReply.changed('updatedAt', true)
118 comments[i] = await comment.save() 95 firstReply.Video = video
119 } 96
97 comments[comments.length - 1] = await firstReply.save()
98
99 for (let i = comments.length - 2; i >= 0; i--) {
100 const comment = comments[ i ]
101 comment.originCommentId = firstReply.id
102 comment.inReplyToCommentId = comments[ i + 1 ].id
103 comment.videoId = video.id
104 comment.changed('updatedAt', true)
105 comment.Video = video
106
107 comments[i] = await comment.save()
120 } 108 }
121 109
122 return { video, parents: comments } 110 resultComment = comments[0]
123 } catch (err) { 111 }
124 logger.debug('Cannot get or create account and video and channel for reply %s, fetch comment', url, { err })
125 112
126 if (comments.length > ACTIVITY_PUB.MAX_RECURSION_COMMENTS) { 113 return { video, comment: resultComment, commentCreated }
127 throw new Error('Recursion limit reached when resolving a thread') 114}
128 }
129 115
130 const { body } = await doRequest({ 116async function resolveParentComment (params: ResolveThreadParams) {
131 uri: url, 117 const { url, comments } = params
132 json: true,
133 activityPub: true
134 })
135 118
136 if (sanitizeAndCheckVideoCommentObject(body) === false) { 119 if (comments.length > ACTIVITY_PUB.MAX_RECURSION_COMMENTS) {
137 throw new Error('Remote video comment JSON is not valid:' + JSON.stringify(body)) 120 throw new Error('Recursion limit reached when resolving a thread')
138 } 121 }
139 122
140 const actorUrl = body.attributedTo 123 const { body } = await doRequest({
141 if (!actorUrl) throw new Error('Miss attributed to in comment') 124 uri: url,
125 json: true,
126 activityPub: true
127 })
142 128
143 if (checkUrlsSameHost(url, actorUrl) !== true) { 129 if (sanitizeAndCheckVideoCommentObject(body) === false) {
144 throw new Error(`Actor url ${actorUrl} has not the same host than the comment url ${url}`) 130 throw new Error('Remote video comment JSON is not valid:' + JSON.stringify(body))
145 } 131 }
146 132
147 if (checkUrlsSameHost(body.id, url) !== true) { 133 const actorUrl = body.attributedTo
148 throw new Error(`Comment url ${url} host is different from the AP object id ${body.id}`) 134 if (!actorUrl) throw new Error('Miss attributed to in comment')
149 }
150 135
151 const actor = await getOrCreateActorAndServerAndModel(actorUrl) 136 if (checkUrlsSameHost(url, actorUrl) !== true) {
152 const comment = new VideoCommentModel({ 137 throw new Error(`Actor url ${actorUrl} has not the same host than the comment url ${url}`)
153 url: body.id, 138 }
154 text: body.content,
155 videoId: null,
156 accountId: actor.Account.id,
157 inReplyToCommentId: null,
158 originCommentId: null,
159 createdAt: new Date(body.published),
160 updatedAt: new Date(body.updated)
161 })
162 139
163 return resolveThread(body.inReplyTo, comments.concat([ comment ])) 140 if (checkUrlsSameHost(body.id, url) !== true) {
141 throw new Error(`Comment url ${url} host is different from the AP object id ${body.id}`)
164 } 142 }
165}
166 143
167export { 144 const actor = await getOrCreateActorAndServerAndModel(actorUrl)
168 videoCommentActivityObjectToDBAttributes, 145 const comment = new VideoCommentModel({
169 addVideoComments, 146 url: body.id,
170 addVideoComment, 147 text: body.content,
171 resolveThread 148 videoId: null,
149 accountId: actor.Account.id,
150 inReplyToCommentId: null,
151 originCommentId: null,
152 createdAt: new Date(body.published),
153 updatedAt: new Date(body.updated)
154 })
155 comment.Account = actor.Account
156
157 return resolveThread({
158 url: body.inReplyTo,
159 comments: comments.concat([ comment ]),
160 commentCreated: true
161 })
172} 162}
diff --git a/server/lib/activitypub/videos.ts b/server/lib/activitypub/videos.ts
index d7bc3d650..2102702e1 100644
--- a/server/lib/activitypub/videos.ts
+++ b/server/lib/activitypub/videos.ts
@@ -56,6 +56,7 @@ import { join } from 'path'
56import { FilteredModelAttributes } from '../../typings/sequelize' 56import { FilteredModelAttributes } from '../../typings/sequelize'
57import { Hooks } from '../plugins/hooks' 57import { Hooks } from '../plugins/hooks'
58import { autoBlacklistVideoIfNeeded } from '../video-blacklist' 58import { autoBlacklistVideoIfNeeded } from '../video-blacklist'
59import { ActorFollowScoreCache } from '../files-cache'
59 60
60async function federateVideoIfNeeded (video: VideoModel, isNewVideo: boolean, transaction?: sequelize.Transaction) { 61async function federateVideoIfNeeded (video: VideoModel, isNewVideo: boolean, transaction?: sequelize.Transaction) {
61 if ( 62 if (
@@ -182,7 +183,7 @@ async function syncVideoExternalAttributes (video: VideoModel, fetchedVideo: Vid
182 } 183 }
183 184
184 if (syncParam.comments === true) { 185 if (syncParam.comments === true) {
185 const handler = items => addVideoComments(items, video) 186 const handler = items => addVideoComments(items)
186 const cleaner = crawlStartDate => VideoCommentModel.cleanOldCommentsOf(video.id, crawlStartDate) 187 const cleaner = crawlStartDate => VideoCommentModel.cleanOldCommentsOf(video.id, crawlStartDate)
187 188
188 await crawlCollectionPage<string>(fetchedVideo.comments, handler, cleaner) 189 await crawlCollectionPage<string>(fetchedVideo.comments, handler, cleaner)
@@ -421,10 +422,14 @@ async function refreshVideoIfNeeded (options: {
421 await retryTransactionWrapper(updateVideoFromAP, updateOptions) 422 await retryTransactionWrapper(updateVideoFromAP, updateOptions)
422 await syncVideoExternalAttributes(video, videoObject, options.syncParam) 423 await syncVideoExternalAttributes(video, videoObject, options.syncParam)
423 424
425 ActorFollowScoreCache.Instance.addGoodServerId(video.VideoChannel.Actor.serverId)
426
424 return video 427 return video
425 } catch (err) { 428 } catch (err) {
426 logger.warn('Cannot refresh video %s.', options.video.url, { err }) 429 logger.warn('Cannot refresh video %s.', options.video.url, { err })
427 430
431 ActorFollowScoreCache.Instance.addBadServerId(video.VideoChannel.Actor.serverId)
432
428 // Don't refresh in loop 433 // Don't refresh in loop
429 await video.setAsRefreshed() 434 await video.setAsRefreshed()
430 return video 435 return video
@@ -500,7 +505,7 @@ async function createVideo (videoObject: VideoTorrentObject, channelActor: Actor
500 505
501 const videoStreamingPlaylists = streamingPlaylistActivityUrlToDBAttributes(videoCreated, videoObject, videoFiles) 506 const videoStreamingPlaylists = streamingPlaylistActivityUrlToDBAttributes(videoCreated, videoObject, videoFiles)
502 const playlistPromises = videoStreamingPlaylists.map(p => VideoStreamingPlaylistModel.create(p, { transaction: t })) 507 const playlistPromises = videoStreamingPlaylists.map(p => VideoStreamingPlaylistModel.create(p, { transaction: t }))
503 await Promise.all(playlistPromises) 508 const streamingPlaylists = await Promise.all(playlistPromises)
504 509
505 // Process tags 510 // Process tags
506 const tags = videoObject.tag 511 const tags = videoObject.tag
@@ -513,7 +518,12 @@ async function createVideo (videoObject: VideoTorrentObject, channelActor: Actor
513 const videoCaptionsPromises = videoObject.subtitleLanguage.map(c => { 518 const videoCaptionsPromises = videoObject.subtitleLanguage.map(c => {
514 return VideoCaptionModel.insertOrReplaceLanguage(videoCreated.id, c.identifier, t) 519 return VideoCaptionModel.insertOrReplaceLanguage(videoCreated.id, c.identifier, t)
515 }) 520 })
516 await Promise.all(videoCaptionsPromises) 521 const captions = await Promise.all(videoCaptionsPromises)
522
523 video.VideoFiles = videoFiles
524 video.VideoStreamingPlaylists = streamingPlaylists
525 video.Tags = tagInstances
526 video.VideoCaptions = captions
517 527
518 const autoBlacklisted = await autoBlacklistVideoIfNeeded({ 528 const autoBlacklisted = await autoBlacklistVideoIfNeeded({
519 video, 529 video,