aboutsummaryrefslogtreecommitdiffhomepage
diff options
context:
space:
mode:
-rw-r--r--client/src/app/+admin/system/jobs/jobs.component.ts1
-rw-r--r--config/default.yaml6
-rw-r--r--config/production.yaml.example6
-rw-r--r--server/helpers/custom-validators/activitypub/activity.ts26
-rw-r--r--server/helpers/custom-validators/activitypub/rate.ts13
-rw-r--r--server/helpers/custom-validators/activitypub/share.ts11
-rw-r--r--server/initializers/checker-before-init.ts2
-rw-r--r--server/initializers/config.ts3
-rw-r--r--server/initializers/constants.ts12
-rw-r--r--server/lib/activitypub/video-comments.ts8
-rw-r--r--server/lib/job-queue/handlers/activitypub-cleaner.ts194
-rw-r--r--server/lib/job-queue/handlers/actor-keys.ts2
-rw-r--r--server/lib/job-queue/job-queue.ts10
-rw-r--r--server/middlewares/validators/videos/video-rates.ts6
-rw-r--r--server/models/account/account-video-rate.ts30
-rw-r--r--server/models/video/video-comment.ts14
-rw-r--r--server/models/video/video-share.ts13
-rw-r--r--server/models/video/video.ts20
-rw-r--r--server/tests/api/activitypub/cleaner.ts283
-rw-r--r--server/tests/api/activitypub/index.ts1
-rw-r--r--shared/extra-utils/miscs/sql.ts39
-rw-r--r--shared/extra-utils/server/jobs.ts3
-rw-r--r--shared/extra-utils/videos/videos.ts2
-rw-r--r--shared/models/server/job.model.ts1
24 files changed, 655 insertions, 51 deletions
diff --git a/client/src/app/+admin/system/jobs/jobs.component.ts b/client/src/app/+admin/system/jobs/jobs.component.ts
index 9f5c04406..43578eedd 100644
--- a/client/src/app/+admin/system/jobs/jobs.component.ts
+++ b/client/src/app/+admin/system/jobs/jobs.component.ts
@@ -28,6 +28,7 @@ export class JobsComponent extends RestTable implements OnInit {
28 'activitypub-http-fetcher', 28 'activitypub-http-fetcher',
29 'activitypub-http-unicast', 29 'activitypub-http-unicast',
30 'activitypub-refresher', 30 'activitypub-refresher',
31 'activitypub-cleaner',
31 'actor-keys', 32 'actor-keys',
32 'email', 33 'email',
33 'video-file-import', 34 'video-file-import',
diff --git a/config/default.yaml b/config/default.yaml
index 2d8afe1c3..a09d20b9d 100644
--- a/config/default.yaml
+++ b/config/default.yaml
@@ -192,6 +192,12 @@ federation:
192 videos: 192 videos:
193 federate_unlisted: false 193 federate_unlisted: false
194 194
195 # Add a weekly job that cleans up remote AP interactions on local videos (shares, rates and comments)
196 # It removes objects that do not exist anymore, and potentially fix their URLs
197 # This setting is opt-in because due to an old bug in PeerTube, remote rates sent by instance before PeerTube 3.0 will be deleted
198 # We still suggest you to enable this setting even if your users will loose most of their video's likes/dislikes
199 cleanup_remote_interactions: false
200
195cache: 201cache:
196 previews: 202 previews:
197 size: 500 # Max number of previews you want to cache 203 size: 500 # Max number of previews you want to cache
diff --git a/config/production.yaml.example b/config/production.yaml.example
index 2794c543c..31c0e6b96 100644
--- a/config/production.yaml.example
+++ b/config/production.yaml.example
@@ -190,6 +190,12 @@ federation:
190 videos: 190 videos:
191 federate_unlisted: false 191 federate_unlisted: false
192 192
193 # Add a weekly job that cleans up remote AP interactions on local videos (shares, rates and comments)
194 # It removes objects that do not exist anymore, and potentially fix their URLs
195 # This setting is opt-in because due to an old bug in PeerTube, remote rates sent by instance before PeerTube 3.0 will be deleted
196 # We still suggest you to enable this setting even if your users will loose most of their video's likes/dislikes
197 cleanup_remote_interactions: false
198
193 199
194############################################################################### 200###############################################################################
195# 201#
diff --git a/server/helpers/custom-validators/activitypub/activity.ts b/server/helpers/custom-validators/activitypub/activity.ts
index 8b8c0685f..da79b2782 100644
--- a/server/helpers/custom-validators/activitypub/activity.ts
+++ b/server/helpers/custom-validators/activitypub/activity.ts
@@ -1,15 +1,16 @@
1import validator from 'validator' 1import validator from 'validator'
2import { Activity, ActivityType } from '../../../../shared/models/activitypub' 2import { Activity, ActivityType } from '../../../../shared/models/activitypub'
3import { exists } from '../misc'
3import { sanitizeAndCheckActorObject } from './actor' 4import { sanitizeAndCheckActorObject } from './actor'
5import { isCacheFileObjectValid } from './cache-file'
6import { isFlagActivityValid } from './flag'
4import { isActivityPubUrlValid, isBaseActivityValid, isObjectValid } from './misc' 7import { isActivityPubUrlValid, isBaseActivityValid, isObjectValid } from './misc'
5import { isDislikeActivityValid } from './rate' 8import { isPlaylistObjectValid } from './playlist'
9import { isDislikeActivityValid, isLikeActivityValid } from './rate'
10import { isShareActivityValid } from './share'
6import { sanitizeAndCheckVideoCommentObject } from './video-comments' 11import { sanitizeAndCheckVideoCommentObject } from './video-comments'
7import { sanitizeAndCheckVideoTorrentObject } from './videos' 12import { sanitizeAndCheckVideoTorrentObject } from './videos'
8import { isViewActivityValid } from './view' 13import { isViewActivityValid } from './view'
9import { exists } from '../misc'
10import { isCacheFileObjectValid } from './cache-file'
11import { isFlagActivityValid } from './flag'
12import { isPlaylistObjectValid } from './playlist'
13 14
14function isRootActivityValid (activity: any) { 15function isRootActivityValid (activity: any) {
15 return isCollection(activity) || isActivity(activity) 16 return isCollection(activity) || isActivity(activity)
@@ -70,8 +71,11 @@ function checkFlagActivity (activity: any) {
70} 71}
71 72
72function checkDislikeActivity (activity: any) { 73function checkDislikeActivity (activity: any) {
73 return isBaseActivityValid(activity, 'Dislike') && 74 return isDislikeActivityValid(activity)
74 isDislikeActivityValid(activity) 75}
76
77function checkLikeActivity (activity: any) {
78 return isLikeActivityValid(activity)
75} 79}
76 80
77function checkCreateActivity (activity: any) { 81function checkCreateActivity (activity: any) {
@@ -118,8 +122,7 @@ function checkRejectActivity (activity: any) {
118} 122}
119 123
120function checkAnnounceActivity (activity: any) { 124function checkAnnounceActivity (activity: any) {
121 return isBaseActivityValid(activity, 'Announce') && 125 return isShareActivityValid(activity)
122 isObjectValid(activity.object)
123} 126}
124 127
125function checkUndoActivity (activity: any) { 128function checkUndoActivity (activity: any) {
@@ -132,8 +135,3 @@ function checkUndoActivity (activity: any) {
132 checkCreateActivity(activity.object) 135 checkCreateActivity(activity.object)
133 ) 136 )
134} 137}
135
136function checkLikeActivity (activity: any) {
137 return isBaseActivityValid(activity, 'Like') &&
138 isObjectValid(activity.object)
139}
diff --git a/server/helpers/custom-validators/activitypub/rate.ts b/server/helpers/custom-validators/activitypub/rate.ts
index ba68e8074..aafdda443 100644
--- a/server/helpers/custom-validators/activitypub/rate.ts
+++ b/server/helpers/custom-validators/activitypub/rate.ts
@@ -1,13 +1,18 @@
1import { isActivityPubUrlValid, isObjectValid } from './misc' 1import { isBaseActivityValid, isObjectValid } from './misc'
2
3function isLikeActivityValid (activity: any) {
4 return isBaseActivityValid(activity, 'Like') &&
5 isObjectValid(activity.object)
6}
2 7
3function isDislikeActivityValid (activity: any) { 8function isDislikeActivityValid (activity: any) {
4 return activity.type === 'Dislike' && 9 return isBaseActivityValid(activity, 'Dislike') &&
5 isActivityPubUrlValid(activity.actor) &&
6 isObjectValid(activity.object) 10 isObjectValid(activity.object)
7} 11}
8 12
9// --------------------------------------------------------------------------- 13// ---------------------------------------------------------------------------
10 14
11export { 15export {
12 isDislikeActivityValid 16 isDislikeActivityValid,
17 isLikeActivityValid
13} 18}
diff --git a/server/helpers/custom-validators/activitypub/share.ts b/server/helpers/custom-validators/activitypub/share.ts
new file mode 100644
index 000000000..fb5e4c05e
--- /dev/null
+++ b/server/helpers/custom-validators/activitypub/share.ts
@@ -0,0 +1,11 @@
1import { isBaseActivityValid, isObjectValid } from './misc'
2
3function isShareActivityValid (activity: any) {
4 return isBaseActivityValid(activity, 'Announce') &&
5 isObjectValid(activity.object)
6}
7// ---------------------------------------------------------------------------
8
9export {
10 isShareActivityValid
11}
diff --git a/server/initializers/checker-before-init.ts b/server/initializers/checker-before-init.ts
index 2578de5ed..565e0d1fa 100644
--- a/server/initializers/checker-before-init.ts
+++ b/server/initializers/checker-before-init.ts
@@ -36,7 +36,7 @@ function checkMissedConfig () {
36 'rates_limit.login.window', 'rates_limit.login.max', 'rates_limit.ask_send_email.window', 'rates_limit.ask_send_email.max', 36 'rates_limit.login.window', 'rates_limit.login.max', 'rates_limit.ask_send_email.window', 'rates_limit.ask_send_email.max',
37 'theme.default', 37 'theme.default',
38 'remote_redundancy.videos.accept_from', 38 'remote_redundancy.videos.accept_from',
39 'federation.videos.federate_unlisted', 39 'federation.videos.federate_unlisted', 'federation.videos.cleanup_remote_interactions',
40 'search.remote_uri.users', 'search.remote_uri.anonymous', 'search.search_index.enabled', 'search.search_index.url', 40 'search.remote_uri.users', 'search.remote_uri.anonymous', 'search.search_index.enabled', 'search.search_index.url',
41 'search.search_index.disable_local_search', 'search.search_index.is_default_search', 41 'search.search_index.disable_local_search', 'search.search_index.is_default_search',
42 'live.enabled', 'live.allow_replay', 'live.max_duration', 'live.max_user_lives', 'live.max_instance_lives', 42 'live.enabled', 'live.allow_replay', 'live.max_duration', 'live.max_user_lives', 'live.max_instance_lives',
diff --git a/server/initializers/config.ts b/server/initializers/config.ts
index 21ca78584..c16b63c33 100644
--- a/server/initializers/config.ts
+++ b/server/initializers/config.ts
@@ -159,7 +159,8 @@ const CONFIG = {
159 }, 159 },
160 FEDERATION: { 160 FEDERATION: {
161 VIDEOS: { 161 VIDEOS: {
162 FEDERATE_UNLISTED: config.get<boolean>('federation.videos.federate_unlisted') 162 FEDERATE_UNLISTED: config.get<boolean>('federation.videos.federate_unlisted'),
163 CLEANUP_REMOTE_INTERACTIONS: config.get<boolean>('federation.videos.cleanup_remote_interactions')
163 } 164 }
164 }, 165 },
165 ADMIN: { 166 ADMIN: {
diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts
index 74192d590..083a29889 100644
--- a/server/initializers/constants.ts
+++ b/server/initializers/constants.ts
@@ -137,6 +137,7 @@ const JOB_ATTEMPTS: { [id in JobType]: number } = {
137 'activitypub-http-unicast': 5, 137 'activitypub-http-unicast': 5,
138 'activitypub-http-fetcher': 5, 138 'activitypub-http-fetcher': 5,
139 'activitypub-follow': 5, 139 'activitypub-follow': 5,
140 'activitypub-cleaner': 1,
140 'video-file-import': 1, 141 'video-file-import': 1,
141 'video-transcoding': 1, 142 'video-transcoding': 1,
142 'video-import': 1, 143 'video-import': 1,
@@ -147,10 +148,12 @@ const JOB_ATTEMPTS: { [id in JobType]: number } = {
147 'video-redundancy': 1, 148 'video-redundancy': 1,
148 'video-live-ending': 1 149 'video-live-ending': 1
149} 150}
150const JOB_CONCURRENCY: { [id in JobType]?: number } = { 151// Excluded keys are jobs that can be configured by admins
152const JOB_CONCURRENCY: { [id in Exclude<JobType, 'video-transcoding' | 'video-import'>]: number } = {
151 'activitypub-http-broadcast': 1, 153 'activitypub-http-broadcast': 1,
152 'activitypub-http-unicast': 5, 154 'activitypub-http-unicast': 5,
153 'activitypub-http-fetcher': 1, 155 'activitypub-http-fetcher': 1,
156 'activitypub-cleaner': 1,
154 'activitypub-follow': 1, 157 'activitypub-follow': 1,
155 'video-file-import': 1, 158 'video-file-import': 1,
156 'email': 5, 159 'email': 5,
@@ -165,6 +168,7 @@ const JOB_TTL: { [id in JobType]: number } = {
165 'activitypub-http-unicast': 60000 * 10, // 10 minutes 168 'activitypub-http-unicast': 60000 * 10, // 10 minutes
166 'activitypub-http-fetcher': 1000 * 3600 * 10, // 10 hours 169 'activitypub-http-fetcher': 1000 * 3600 * 10, // 10 hours
167 'activitypub-follow': 60000 * 10, // 10 minutes 170 'activitypub-follow': 60000 * 10, // 10 minutes
171 'activitypub-cleaner': 1000 * 3600, // 1 hour
168 'video-file-import': 1000 * 3600, // 1 hour 172 'video-file-import': 1000 * 3600, // 1 hour
169 'video-transcoding': 1000 * 3600 * 48, // 2 days, transcoding could be long 173 'video-transcoding': 1000 * 3600 * 48, // 2 days, transcoding could be long
170 'video-import': 1000 * 3600 * 2, // 2 hours 174 'video-import': 1000 * 3600 * 2, // 2 hours
@@ -178,6 +182,9 @@ const JOB_TTL: { [id in JobType]: number } = {
178const REPEAT_JOBS: { [ id: string ]: EveryRepeatOptions | CronRepeatOptions } = { 182const REPEAT_JOBS: { [ id: string ]: EveryRepeatOptions | CronRepeatOptions } = {
179 'videos-views': { 183 'videos-views': {
180 cron: randomInt(1, 20) + ' * * * *' // Between 1-20 minutes past the hour 184 cron: randomInt(1, 20) + ' * * * *' // Between 1-20 minutes past the hour
185 },
186 'activitypub-cleaner': {
187 cron: '30 5 * * ' + randomInt(0, 7) // 1 time per week (random day) at 5:30 AM
181 } 188 }
182} 189}
183const JOB_PRIORITY = { 190const JOB_PRIORITY = {
@@ -188,6 +195,7 @@ const JOB_PRIORITY = {
188} 195}
189 196
190const BROADCAST_CONCURRENCY = 10 // How many requests in parallel we do in activitypub-http-broadcast job 197const BROADCAST_CONCURRENCY = 10 // How many requests in parallel we do in activitypub-http-broadcast job
198const AP_CLEANER_CONCURRENCY = 10 // How many requests in parallel we do in activitypub-cleaner job
191const CRAWL_REQUEST_CONCURRENCY = 1 // How many requests in parallel to fetch remote data (likes, shares...) 199const CRAWL_REQUEST_CONCURRENCY = 1 // How many requests in parallel to fetch remote data (likes, shares...)
192const JOB_REQUEST_TIMEOUT = 7000 // 7 seconds 200const JOB_REQUEST_TIMEOUT = 7000 // 7 seconds
193const JOB_COMPLETED_LIFETIME = 60000 * 60 * 24 * 2 // 2 days 201const JOB_COMPLETED_LIFETIME = 60000 * 60 * 24 * 2 // 2 days
@@ -756,6 +764,7 @@ if (isTestInstance() === true) {
756 SCHEDULER_INTERVALS_MS.autoFollowIndexInstances = 5000 764 SCHEDULER_INTERVALS_MS.autoFollowIndexInstances = 5000
757 SCHEDULER_INTERVALS_MS.updateInboxStats = 5000 765 SCHEDULER_INTERVALS_MS.updateInboxStats = 5000
758 REPEAT_JOBS['videos-views'] = { every: 5000 } 766 REPEAT_JOBS['videos-views'] = { every: 5000 }
767 REPEAT_JOBS['activitypub-cleaner'] = { every: 5000 }
759 768
760 REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR = 1 769 REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR = 1
761 770
@@ -815,6 +824,7 @@ export {
815 REDUNDANCY, 824 REDUNDANCY,
816 JOB_CONCURRENCY, 825 JOB_CONCURRENCY,
817 JOB_ATTEMPTS, 826 JOB_ATTEMPTS,
827 AP_CLEANER_CONCURRENCY,
818 LAST_MIGRATION_VERSION, 828 LAST_MIGRATION_VERSION,
819 OAUTH_LIFETIME, 829 OAUTH_LIFETIME,
820 CUSTOM_HTML_TAG_COMMENTS, 830 CUSTOM_HTML_TAG_COMMENTS,
diff --git a/server/lib/activitypub/video-comments.ts b/server/lib/activitypub/video-comments.ts
index 902d877c4..d025ed7f1 100644
--- a/server/lib/activitypub/video-comments.ts
+++ b/server/lib/activitypub/video-comments.ts
@@ -41,10 +41,10 @@ async function resolveThread (params: ResolveThreadParams): ResolveThreadResult
41 return await tryResolveThreadFromVideo(params) 41 return await tryResolveThreadFromVideo(params)
42 } 42 }
43 } catch (err) { 43 } catch (err) {
44 logger.debug('Cannot get or create account and video and channel for reply %s, fetch comment', url, { err }) 44 logger.debug('Cannot resolve thread from video %s, maybe because it was not a video', url, { err })
45 } 45 }
46 46
47 return resolveParentComment(params) 47 return resolveRemoteParentComment(params)
48} 48}
49 49
50export { 50export {
@@ -119,7 +119,7 @@ async function tryResolveThreadFromVideo (params: ResolveThreadParams) {
119 return { video, comment: resultComment, commentCreated } 119 return { video, comment: resultComment, commentCreated }
120} 120}
121 121
122async function resolveParentComment (params: ResolveThreadParams) { 122async function resolveRemoteParentComment (params: ResolveThreadParams) {
123 const { url, comments } = params 123 const { url, comments } = params
124 124
125 if (comments.length > ACTIVITY_PUB.MAX_RECURSION_COMMENTS) { 125 if (comments.length > ACTIVITY_PUB.MAX_RECURSION_COMMENTS) {
@@ -133,7 +133,7 @@ async function resolveParentComment (params: ResolveThreadParams) {
133 }) 133 })
134 134
135 if (sanitizeAndCheckVideoCommentObject(body) === false) { 135 if (sanitizeAndCheckVideoCommentObject(body) === false) {
136 throw new Error('Remote video comment JSON is not valid:' + JSON.stringify(body)) 136 throw new Error(`Remote video comment JSON ${url} is not valid:` + JSON.stringify(body))
137 } 137 }
138 138
139 const actorUrl = body.attributedTo 139 const actorUrl = body.attributedTo
diff --git a/server/lib/job-queue/handlers/activitypub-cleaner.ts b/server/lib/job-queue/handlers/activitypub-cleaner.ts
new file mode 100644
index 000000000..b58bbc983
--- /dev/null
+++ b/server/lib/job-queue/handlers/activitypub-cleaner.ts
@@ -0,0 +1,194 @@
1import * as Bluebird from 'bluebird'
2import * as Bull from 'bull'
3import { checkUrlsSameHost } from '@server/helpers/activitypub'
4import { isDislikeActivityValid, isLikeActivityValid } from '@server/helpers/custom-validators/activitypub/rate'
5import { isShareActivityValid } from '@server/helpers/custom-validators/activitypub/share'
6import { sanitizeAndCheckVideoCommentObject } from '@server/helpers/custom-validators/activitypub/video-comments'
7import { doRequest } from '@server/helpers/requests'
8import { AP_CLEANER_CONCURRENCY } from '@server/initializers/constants'
9import { VideoModel } from '@server/models/video/video'
10import { VideoCommentModel } from '@server/models/video/video-comment'
11import { VideoShareModel } from '@server/models/video/video-share'
12import { HttpStatusCode } from '@shared/core-utils'
13import { logger } from '../../../helpers/logger'
14import { AccountVideoRateModel } from '../../../models/account/account-video-rate'
15
16// Job to clean remote interactions off local videos
17
18async function processActivityPubCleaner (_job: Bull.Job) {
19 logger.info('Processing ActivityPub cleaner.')
20
21 {
22 const rateUrls = await AccountVideoRateModel.listRemoteRateUrlsOfLocalVideos()
23 const { bodyValidator, deleter, updater } = rateOptionsFactory()
24
25 await Bluebird.map(rateUrls, async rateUrl => {
26 try {
27 const result = await updateObjectIfNeeded(rateUrl, bodyValidator, updater, deleter)
28
29 if (result?.status === 'deleted') {
30 const { videoId, type } = result.data
31
32 await VideoModel.updateRatesOf(videoId, type, undefined)
33 }
34 } catch (err) {
35 logger.warn('Cannot update/delete remote AP rate %s.', rateUrl, { err })
36 }
37 }, { concurrency: AP_CLEANER_CONCURRENCY })
38 }
39
40 {
41 const shareUrls = await VideoShareModel.listRemoteShareUrlsOfLocalVideos()
42 const { bodyValidator, deleter, updater } = shareOptionsFactory()
43
44 await Bluebird.map(shareUrls, async shareUrl => {
45 try {
46 await updateObjectIfNeeded(shareUrl, bodyValidator, updater, deleter)
47 } catch (err) {
48 logger.warn('Cannot update/delete remote AP share %s.', shareUrl, { err })
49 }
50 }, { concurrency: AP_CLEANER_CONCURRENCY })
51 }
52
53 {
54 const commentUrls = await VideoCommentModel.listRemoteCommentUrlsOfLocalVideos()
55 const { bodyValidator, deleter, updater } = commentOptionsFactory()
56
57 await Bluebird.map(commentUrls, async commentUrl => {
58 try {
59 await updateObjectIfNeeded(commentUrl, bodyValidator, updater, deleter)
60 } catch (err) {
61 logger.warn('Cannot update/delete remote AP comment %s.', commentUrl, { err })
62 }
63 }, { concurrency: AP_CLEANER_CONCURRENCY })
64 }
65}
66
67// ---------------------------------------------------------------------------
68
69export {
70 processActivityPubCleaner
71}
72
73// ---------------------------------------------------------------------------
74
75async function updateObjectIfNeeded <T> (
76 url: string,
77 bodyValidator: (body: any) => boolean,
78 updater: (url: string, newUrl: string) => Promise<T>,
79 deleter: (url: string) => Promise<T>
80): Promise<{ data: T, status: 'deleted' | 'updated' } | null> {
81 // Fetch url
82 const { response, body } = await doRequest<any>({
83 uri: url,
84 json: true,
85 activityPub: true
86 })
87
88 // Does not exist anymore, remove entry
89 if (response.statusCode === HttpStatusCode.NOT_FOUND_404) {
90 logger.info('Removing remote AP object %s.', url)
91 const data = await deleter(url)
92
93 return { status: 'deleted', data }
94 }
95
96 // If not same id, check same host and update
97 if (!body || !body.id || !bodyValidator(body)) throw new Error(`Body or body id of ${url} is invalid`)
98
99 if (body.type === 'Tombstone') {
100 logger.info('Removing remote AP object %s.', url)
101 const data = await deleter(url)
102
103 return { status: 'deleted', data }
104 }
105
106 const newUrl = body.id
107 if (newUrl !== url) {
108 if (checkUrlsSameHost(newUrl, url) !== true) {
109 throw new Error(`New url ${newUrl} has not the same host than old url ${url}`)
110 }
111
112 logger.info('Updating remote AP object %s.', url)
113 const data = await updater(url, newUrl)
114
115 return { status: 'updated', data }
116 }
117
118 return null
119}
120
121function rateOptionsFactory () {
122 return {
123 bodyValidator: (body: any) => isLikeActivityValid(body) || isDislikeActivityValid(body),
124
125 updater: async (url: string, newUrl: string) => {
126 const rate = await AccountVideoRateModel.loadByUrl(url, undefined)
127 rate.url = newUrl
128
129 const videoId = rate.videoId
130 const type = rate.type
131
132 await rate.save()
133
134 return { videoId, type }
135 },
136
137 deleter: async (url) => {
138 const rate = await AccountVideoRateModel.loadByUrl(url, undefined)
139
140 const videoId = rate.videoId
141 const type = rate.type
142
143 await rate.destroy()
144
145 return { videoId, type }
146 }
147 }
148}
149
150function shareOptionsFactory () {
151 return {
152 bodyValidator: (body: any) => isShareActivityValid(body),
153
154 updater: async (url: string, newUrl: string) => {
155 const share = await VideoShareModel.loadByUrl(url, undefined)
156 share.url = newUrl
157
158 await share.save()
159
160 return undefined
161 },
162
163 deleter: async (url) => {
164 const share = await VideoShareModel.loadByUrl(url, undefined)
165
166 await share.destroy()
167
168 return undefined
169 }
170 }
171}
172
173function commentOptionsFactory () {
174 return {
175 bodyValidator: (body: any) => sanitizeAndCheckVideoCommentObject(body),
176
177 updater: async (url: string, newUrl: string) => {
178 const comment = await VideoCommentModel.loadByUrlAndPopulateAccountAndVideo(url)
179 comment.url = newUrl
180
181 await comment.save()
182
183 return undefined
184 },
185
186 deleter: async (url) => {
187 const comment = await VideoCommentModel.loadByUrlAndPopulateAccountAndVideo(url)
188
189 await comment.destroy()
190
191 return undefined
192 }
193 }
194}
diff --git a/server/lib/job-queue/handlers/actor-keys.ts b/server/lib/job-queue/handlers/actor-keys.ts
index 8da549640..125307843 100644
--- a/server/lib/job-queue/handlers/actor-keys.ts
+++ b/server/lib/job-queue/handlers/actor-keys.ts
@@ -6,7 +6,7 @@ import { logger } from '../../../helpers/logger'
6 6
7async function processActorKeys (job: Bull.Job) { 7async function processActorKeys (job: Bull.Job) {
8 const payload = job.data as ActorKeysPayload 8 const payload = job.data as ActorKeysPayload
9 logger.info('Processing email in job %d.', job.id) 9 logger.info('Processing actor keys in job %d.', job.id)
10 10
11 const actor = await ActorModel.load(payload.actorId) 11 const actor = await ActorModel.load(payload.actorId)
12 12
diff --git a/server/lib/job-queue/job-queue.ts b/server/lib/job-queue/job-queue.ts
index efda2e038..42e8347b1 100644
--- a/server/lib/job-queue/job-queue.ts
+++ b/server/lib/job-queue/job-queue.ts
@@ -21,6 +21,7 @@ import {
21import { logger } from '../../helpers/logger' 21import { logger } from '../../helpers/logger'
22import { JOB_ATTEMPTS, JOB_COMPLETED_LIFETIME, JOB_CONCURRENCY, JOB_TTL, REPEAT_JOBS, WEBSERVER } from '../../initializers/constants' 22import { JOB_ATTEMPTS, JOB_COMPLETED_LIFETIME, JOB_CONCURRENCY, JOB_TTL, REPEAT_JOBS, WEBSERVER } from '../../initializers/constants'
23import { Redis } from '../redis' 23import { Redis } from '../redis'
24import { processActivityPubCleaner } from './handlers/activitypub-cleaner'
24import { processActivityPubFollow } from './handlers/activitypub-follow' 25import { processActivityPubFollow } from './handlers/activitypub-follow'
25import { processActivityPubHttpBroadcast } from './handlers/activitypub-http-broadcast' 26import { processActivityPubHttpBroadcast } from './handlers/activitypub-http-broadcast'
26import { processActivityPubHttpFetcher } from './handlers/activitypub-http-fetcher' 27import { processActivityPubHttpFetcher } from './handlers/activitypub-http-fetcher'
@@ -38,6 +39,7 @@ type CreateJobArgument =
38 { type: 'activitypub-http-broadcast', payload: ActivitypubHttpBroadcastPayload } | 39 { type: 'activitypub-http-broadcast', payload: ActivitypubHttpBroadcastPayload } |
39 { type: 'activitypub-http-unicast', payload: ActivitypubHttpUnicastPayload } | 40 { type: 'activitypub-http-unicast', payload: ActivitypubHttpUnicastPayload } |
40 { type: 'activitypub-http-fetcher', payload: ActivitypubHttpFetcherPayload } | 41 { type: 'activitypub-http-fetcher', payload: ActivitypubHttpFetcherPayload } |
42 { type: 'activitypub-http-cleaner', payload: {} } |
41 { type: 'activitypub-follow', payload: ActivitypubFollowPayload } | 43 { type: 'activitypub-follow', payload: ActivitypubFollowPayload } |
42 { type: 'video-file-import', payload: VideoFileImportPayload } | 44 { type: 'video-file-import', payload: VideoFileImportPayload } |
43 { type: 'video-transcoding', payload: VideoTranscodingPayload } | 45 { type: 'video-transcoding', payload: VideoTranscodingPayload } |
@@ -58,6 +60,7 @@ const handlers: { [id in JobType]: (job: Bull.Job) => Promise<any> } = {
58 'activitypub-http-broadcast': processActivityPubHttpBroadcast, 60 'activitypub-http-broadcast': processActivityPubHttpBroadcast,
59 'activitypub-http-unicast': processActivityPubHttpUnicast, 61 'activitypub-http-unicast': processActivityPubHttpUnicast,
60 'activitypub-http-fetcher': processActivityPubHttpFetcher, 62 'activitypub-http-fetcher': processActivityPubHttpFetcher,
63 'activitypub-cleaner': processActivityPubCleaner,
61 'activitypub-follow': processActivityPubFollow, 64 'activitypub-follow': processActivityPubFollow,
62 'video-file-import': processVideoFileImport, 65 'video-file-import': processVideoFileImport,
63 'video-transcoding': processVideoTranscoding, 66 'video-transcoding': processVideoTranscoding,
@@ -75,6 +78,7 @@ const jobTypes: JobType[] = [
75 'activitypub-http-broadcast', 78 'activitypub-http-broadcast',
76 'activitypub-http-fetcher', 79 'activitypub-http-fetcher',
77 'activitypub-http-unicast', 80 'activitypub-http-unicast',
81 'activitypub-cleaner',
78 'email', 82 'email',
79 'video-transcoding', 83 'video-transcoding',
80 'video-file-import', 84 'video-file-import',
@@ -233,6 +237,12 @@ class JobQueue {
233 this.queues['videos-views'].add({}, { 237 this.queues['videos-views'].add({}, {
234 repeat: REPEAT_JOBS['videos-views'] 238 repeat: REPEAT_JOBS['videos-views']
235 }).catch(err => logger.error('Cannot add repeatable job.', { err })) 239 }).catch(err => logger.error('Cannot add repeatable job.', { err }))
240
241 if (CONFIG.FEDERATION.VIDEOS.CLEANUP_REMOTE_INTERACTIONS) {
242 this.queues['activitypub-cleaner'].add({}, {
243 repeat: REPEAT_JOBS['activitypub-cleaner']
244 }).catch(err => logger.error('Cannot add repeatable job.', { err }))
245 }
236 } 246 }
237 247
238 private filterJobTypes (jobType?: JobType) { 248 private filterJobTypes (jobType?: JobType) {
diff --git a/server/middlewares/validators/videos/video-rates.ts b/server/middlewares/validators/videos/video-rates.ts
index 7dcba15f1..01bdef25f 100644
--- a/server/middlewares/validators/videos/video-rates.ts
+++ b/server/middlewares/validators/videos/video-rates.ts
@@ -1,6 +1,6 @@
1import * as express from 'express' 1import * as express from 'express'
2import { body, param, query } from 'express-validator' 2import { body, param, query } from 'express-validator'
3import { isIdOrUUIDValid } from '../../../helpers/custom-validators/misc' 3import { isIdOrUUIDValid, isIdValid } from '../../../helpers/custom-validators/misc'
4import { isRatingValid } from '../../../helpers/custom-validators/video-rates' 4import { isRatingValid } from '../../../helpers/custom-validators/video-rates'
5import { isVideoRatingTypeValid } from '../../../helpers/custom-validators/videos' 5import { isVideoRatingTypeValid } from '../../../helpers/custom-validators/videos'
6import { logger } from '../../../helpers/logger' 6import { logger } from '../../../helpers/logger'
@@ -28,14 +28,14 @@ const videoUpdateRateValidator = [
28const getAccountVideoRateValidatorFactory = function (rateType: VideoRateType) { 28const getAccountVideoRateValidatorFactory = function (rateType: VideoRateType) {
29 return [ 29 return [
30 param('name').custom(isAccountNameValid).withMessage('Should have a valid account name'), 30 param('name').custom(isAccountNameValid).withMessage('Should have a valid account name'),
31 param('videoId').custom(isIdOrUUIDValid).not().isEmpty().withMessage('Should have a valid videoId'), 31 param('videoId').custom(isIdValid).not().isEmpty().withMessage('Should have a valid videoId'),
32 32
33 async (req: express.Request, res: express.Response, next: express.NextFunction) => { 33 async (req: express.Request, res: express.Response, next: express.NextFunction) => {
34 logger.debug('Checking videoCommentGetValidator parameters.', { parameters: req.params }) 34 logger.debug('Checking videoCommentGetValidator parameters.', { parameters: req.params })
35 35
36 if (areValidationErrors(req, res)) return 36 if (areValidationErrors(req, res)) return
37 37
38 const rate = await AccountVideoRateModel.loadLocalAndPopulateVideo(rateType, req.params.name, req.params.videoId) 38 const rate = await AccountVideoRateModel.loadLocalAndPopulateVideo(rateType, req.params.name, +req.params.videoId)
39 if (!rate) { 39 if (!rate) {
40 return res.status(HttpStatusCode.NOT_FOUND_404) 40 return res.status(HttpStatusCode.NOT_FOUND_404)
41 .json({ error: 'Video rate not found' }) 41 .json({ error: 'Video rate not found' })
diff --git a/server/models/account/account-video-rate.ts b/server/models/account/account-video-rate.ts
index d9c529491..801f76bba 100644
--- a/server/models/account/account-video-rate.ts
+++ b/server/models/account/account-video-rate.ts
@@ -146,10 +146,22 @@ export class AccountVideoRateModel extends Model {
146 return AccountVideoRateModel.findAndCountAll(query) 146 return AccountVideoRateModel.findAndCountAll(query)
147 } 147 }
148 148
149 static listRemoteRateUrlsOfLocalVideos () {
150 const query = `SELECT "accountVideoRate".url FROM "accountVideoRate" ` +
151 `INNER JOIN account ON account.id = "accountVideoRate"."accountId" ` +
152 `INNER JOIN actor ON actor.id = account."actorId" AND actor."serverId" IS NOT NULL ` +
153 `INNER JOIN video ON video.id = "accountVideoRate"."videoId" AND video.remote IS FALSE`
154
155 return AccountVideoRateModel.sequelize.query<{ url: string }>(query, {
156 type: QueryTypes.SELECT,
157 raw: true
158 }).then(rows => rows.map(r => r.url))
159 }
160
149 static loadLocalAndPopulateVideo ( 161 static loadLocalAndPopulateVideo (
150 rateType: VideoRateType, 162 rateType: VideoRateType,
151 accountName: string, 163 accountName: string,
152 videoId: number | string, 164 videoId: number,
153 t?: Transaction 165 t?: Transaction
154 ): Promise<MAccountVideoRateAccountVideo> { 166 ): Promise<MAccountVideoRateAccountVideo> {
155 const options: FindOptions = { 167 const options: FindOptions = {
@@ -241,21 +253,7 @@ export class AccountVideoRateModel extends Model {
241 253
242 await AccountVideoRateModel.destroy(query) 254 await AccountVideoRateModel.destroy(query)
243 255
244 const field = type === 'like' 256 return VideoModel.updateRatesOf(videoId, type, t)
245 ? 'likes'
246 : 'dislikes'
247
248 const rawQuery = `UPDATE "video" SET "${field}" = ` +
249 '(' +
250 'SELECT COUNT(id) FROM "accountVideoRate" WHERE "accountVideoRate"."videoId" = "video"."id" AND type = :rateType' +
251 ') ' +
252 'WHERE "video"."id" = :videoId'
253
254 return AccountVideoRateModel.sequelize.query(rawQuery, {
255 transaction: t,
256 replacements: { videoId, rateType: type },
257 type: QueryTypes.UPDATE
258 })
259 }) 257 })
260 } 258 }
261 259
diff --git a/server/models/video/video-comment.ts b/server/models/video/video-comment.ts
index dc7556d44..151c2bc81 100644
--- a/server/models/video/video-comment.ts
+++ b/server/models/video/video-comment.ts
@@ -1,5 +1,5 @@
1import { uniq } from 'lodash' 1import { uniq } from 'lodash'
2import { FindAndCountOptions, FindOptions, Op, Order, ScopeOptions, Sequelize, Transaction, WhereOptions } from 'sequelize' 2import { FindAndCountOptions, FindOptions, Op, Order, QueryTypes, ScopeOptions, Sequelize, Transaction, WhereOptions } from 'sequelize'
3import { 3import {
4 AllowNull, 4 AllowNull,
5 BelongsTo, 5 BelongsTo,
@@ -696,6 +696,18 @@ export class VideoCommentModel extends Model {
696 } 696 }
697 } 697 }
698 698
699 static listRemoteCommentUrlsOfLocalVideos () {
700 const query = `SELECT "videoComment".url FROM "videoComment" ` +
701 `INNER JOIN account ON account.id = "videoComment"."accountId" ` +
702 `INNER JOIN actor ON actor.id = "account"."actorId" AND actor."serverId" IS NOT NULL ` +
703 `INNER JOIN video ON video.id = "videoComment"."videoId" AND video.remote IS FALSE`
704
705 return VideoCommentModel.sequelize.query<{ url: string }>(query, {
706 type: QueryTypes.SELECT,
707 raw: true
708 }).then(rows => rows.map(r => r.url))
709 }
710
699 static cleanOldCommentsOf (videoId: number, beforeUpdatedAt: Date) { 711 static cleanOldCommentsOf (videoId: number, beforeUpdatedAt: Date) {
700 const query = { 712 const query = {
701 where: { 713 where: {
diff --git a/server/models/video/video-share.ts b/server/models/video/video-share.ts
index b7f5f3fa3..5059c1fa6 100644
--- a/server/models/video/video-share.ts
+++ b/server/models/video/video-share.ts
@@ -1,4 +1,4 @@
1import { literal, Op, Transaction } from 'sequelize' 1import { literal, Op, QueryTypes, Transaction } from 'sequelize'
2import { AllowNull, BelongsTo, Column, CreatedAt, DataType, ForeignKey, Is, Model, Scopes, Table, UpdatedAt } from 'sequelize-typescript' 2import { AllowNull, BelongsTo, Column, CreatedAt, DataType, ForeignKey, Is, Model, Scopes, Table, UpdatedAt } from 'sequelize-typescript'
3import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc' 3import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc'
4import { CONSTRAINTS_FIELDS } from '../../initializers/constants' 4import { CONSTRAINTS_FIELDS } from '../../initializers/constants'
@@ -185,6 +185,17 @@ export class VideoShareModel extends Model {
185 return VideoShareModel.findAndCountAll(query) 185 return VideoShareModel.findAndCountAll(query)
186 } 186 }
187 187
188 static listRemoteShareUrlsOfLocalVideos () {
189 const query = `SELECT "videoShare".url FROM "videoShare" ` +
190 `INNER JOIN actor ON actor.id = "videoShare"."actorId" AND actor."serverId" IS NOT NULL ` +
191 `INNER JOIN video ON video.id = "videoShare"."videoId" AND video.remote IS FALSE`
192
193 return VideoShareModel.sequelize.query<{ url: string }>(query, {
194 type: QueryTypes.SELECT,
195 raw: true
196 }).then(rows => rows.map(r => r.url))
197 }
198
188 static cleanOldSharesOf (videoId: number, beforeUpdatedAt: Date) { 199 static cleanOldSharesOf (videoId: number, beforeUpdatedAt: Date) {
189 const query = { 200 const query = {
190 where: { 201 where: {
diff --git a/server/models/video/video.ts b/server/models/video/video.ts
index 8894843e0..b4c7da655 100644
--- a/server/models/video/video.ts
+++ b/server/models/video/video.ts
@@ -34,7 +34,7 @@ import { ModelCache } from '@server/models/model-cache'
34import { VideoFile } from '@shared/models/videos/video-file.model' 34import { VideoFile } from '@shared/models/videos/video-file.model'
35import { ResultList, UserRight, VideoPrivacy, VideoState } from '../../../shared' 35import { ResultList, UserRight, VideoPrivacy, VideoState } from '../../../shared'
36import { VideoObject } from '../../../shared/models/activitypub/objects' 36import { VideoObject } from '../../../shared/models/activitypub/objects'
37import { Video, VideoDetails } from '../../../shared/models/videos' 37import { Video, VideoDetails, VideoRateType } from '../../../shared/models/videos'
38import { ThumbnailType } from '../../../shared/models/videos/thumbnail.type' 38import { ThumbnailType } from '../../../shared/models/videos/thumbnail.type'
39import { VideoFilter } from '../../../shared/models/videos/video-query.type' 39import { VideoFilter } from '../../../shared/models/videos/video-query.type'
40import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type' 40import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type'
@@ -1509,6 +1509,24 @@ export class VideoModel extends Model {
1509 }) 1509 })
1510 } 1510 }
1511 1511
1512 static updateRatesOf (videoId: number, type: VideoRateType, t: Transaction) {
1513 const field = type === 'like'
1514 ? 'likes'
1515 : 'dislikes'
1516
1517 const rawQuery = `UPDATE "video" SET "${field}" = ` +
1518 '(' +
1519 'SELECT COUNT(id) FROM "accountVideoRate" WHERE "accountVideoRate"."videoId" = "video"."id" AND type = :rateType' +
1520 ') ' +
1521 'WHERE "video"."id" = :videoId'
1522
1523 return AccountVideoRateModel.sequelize.query(rawQuery, {
1524 transaction: t,
1525 replacements: { videoId, rateType: type },
1526 type: QueryTypes.UPDATE
1527 })
1528 }
1529
1512 static checkVideoHasInstanceFollow (videoId: number, followerActorId: number) { 1530 static checkVideoHasInstanceFollow (videoId: number, followerActorId: number) {
1513 // Instances only share videos 1531 // Instances only share videos
1514 const query = 'SELECT 1 FROM "videoShare" ' + 1532 const query = 'SELECT 1 FROM "videoShare" ' +
diff --git a/server/tests/api/activitypub/cleaner.ts b/server/tests/api/activitypub/cleaner.ts
new file mode 100644
index 000000000..75ef56ce3
--- /dev/null
+++ b/server/tests/api/activitypub/cleaner.ts
@@ -0,0 +1,283 @@
1/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
2
3import 'mocha'
4import * as chai from 'chai'
5import {
6 cleanupTests,
7 closeAllSequelize,
8 deleteAll,
9 doubleFollow,
10 getCount,
11 selectQuery,
12 setVideoField,
13 updateQuery,
14 wait
15} from '../../../../shared/extra-utils'
16import { flushAndRunMultipleServers, ServerInfo, setAccessTokensToServers } from '../../../../shared/extra-utils/index'
17import { waitJobs } from '../../../../shared/extra-utils/server/jobs'
18import { addVideoCommentThread, getVideoCommentThreads } from '../../../../shared/extra-utils/videos/video-comments'
19import { getVideo, rateVideo, uploadVideoAndGetId } from '../../../../shared/extra-utils/videos/videos'
20
21const expect = chai.expect
22
23describe('Test AP cleaner', function () {
24 let servers: ServerInfo[] = []
25 let videoUUID1: string
26 let videoUUID2: string
27 let videoUUID3: string
28
29 let videoUUIDs: string[]
30
31 before(async function () {
32 this.timeout(120000)
33
34 const config = {
35 federation: {
36 videos: { cleanup_remote_interactions: true }
37 }
38 }
39 servers = await flushAndRunMultipleServers(3, config)
40
41 // Get the access tokens
42 await setAccessTokensToServers(servers)
43
44 await Promise.all([
45 doubleFollow(servers[0], servers[1]),
46 doubleFollow(servers[1], servers[2]),
47 doubleFollow(servers[0], servers[2])
48 ])
49
50 // Update 1 local share, check 6 shares
51
52 // Create 1 comment per video
53 // Update 1 remote URL and 1 local URL on
54
55 videoUUID1 = (await uploadVideoAndGetId({ server: servers[0], videoName: 'server 1' })).uuid
56 videoUUID2 = (await uploadVideoAndGetId({ server: servers[1], videoName: 'server 2' })).uuid
57 videoUUID3 = (await uploadVideoAndGetId({ server: servers[2], videoName: 'server 3' })).uuid
58
59 videoUUIDs = [ videoUUID1, videoUUID2, videoUUID3 ]
60
61 await waitJobs(servers)
62
63 for (const server of servers) {
64 for (const uuid of videoUUIDs) {
65 await rateVideo(server.url, server.accessToken, uuid, 'like')
66 await addVideoCommentThread(server.url, server.accessToken, uuid, 'comment')
67 }
68 }
69
70 await waitJobs(servers)
71 })
72
73 it('Should have the correct likes', async function () {
74 for (const server of servers) {
75 for (const uuid of videoUUIDs) {
76 const res = await getVideo(server.url, uuid)
77 expect(res.body.likes).to.equal(3)
78 expect(res.body.dislikes).to.equal(0)
79 }
80 }
81 })
82
83 it('Should destroy server 3 internal likes and correctly clean them', async function () {
84 this.timeout(20000)
85
86 await deleteAll(servers[2].internalServerNumber, 'accountVideoRate')
87 for (const uuid of videoUUIDs) {
88 await setVideoField(servers[2].internalServerNumber, uuid, 'likes', '0')
89 }
90
91 await wait(5000)
92 await waitJobs(servers)
93
94 // Updated rates of my video
95 {
96 const res = await getVideo(servers[0].url, videoUUID1)
97 expect(res.body.likes).to.equal(2)
98 expect(res.body.dislikes).to.equal(0)
99 }
100
101 // Did not update rates of a remote video
102 {
103 const res = await getVideo(servers[0].url, videoUUID2)
104 expect(res.body.likes).to.equal(3)
105 expect(res.body.dislikes).to.equal(0)
106 }
107 })
108
109 it('Should update rates to dislikes', async function () {
110 this.timeout(20000)
111
112 for (const server of servers) {
113 for (const uuid of videoUUIDs) {
114 await rateVideo(server.url, server.accessToken, uuid, 'dislike')
115 }
116 }
117
118 await waitJobs(servers)
119
120 for (const server of servers) {
121 for (const uuid of videoUUIDs) {
122 const res = await getVideo(server.url, uuid)
123 expect(res.body.likes).to.equal(0)
124 expect(res.body.dislikes).to.equal(3)
125 }
126 }
127 })
128
129 it('Should destroy server 3 internal dislikes and correctly clean them', async function () {
130 this.timeout(20000)
131
132 await deleteAll(servers[2].internalServerNumber, 'accountVideoRate')
133
134 for (const uuid of videoUUIDs) {
135 await setVideoField(servers[2].internalServerNumber, uuid, 'dislikes', '0')
136 }
137
138 await wait(5000)
139 await waitJobs(servers)
140
141 // Updated rates of my video
142 {
143 const res = await getVideo(servers[0].url, videoUUID1)
144 expect(res.body.likes).to.equal(0)
145 expect(res.body.dislikes).to.equal(2)
146 }
147
148 // Did not update rates of a remote video
149 {
150 const res = await getVideo(servers[0].url, videoUUID2)
151 expect(res.body.likes).to.equal(0)
152 expect(res.body.dislikes).to.equal(3)
153 }
154 })
155
156 it('Should destroy server 3 internal shares and correctly clean them', async function () {
157 this.timeout(20000)
158
159 const preCount = await getCount(servers[0].internalServerNumber, 'videoShare')
160 expect(preCount).to.equal(6)
161
162 await deleteAll(servers[2].internalServerNumber, 'videoShare')
163 await wait(5000)
164 await waitJobs(servers)
165
166 // Still 6 because we don't have remote shares on local videos
167 const postCount = await getCount(servers[0].internalServerNumber, 'videoShare')
168 expect(postCount).to.equal(6)
169 })
170
171 it('Should destroy server 3 internal comments and correctly clean them', async function () {
172 this.timeout(20000)
173
174 {
175 const res = await getVideoCommentThreads(servers[0].url, videoUUID1, 0, 5)
176 expect(res.body.total).to.equal(3)
177 }
178
179 await deleteAll(servers[2].internalServerNumber, 'videoComment')
180
181 await wait(5000)
182 await waitJobs(servers)
183
184 {
185 const res = await getVideoCommentThreads(servers[0].url, videoUUID1, 0, 5)
186 expect(res.body.total).to.equal(2)
187 }
188 })
189
190 it('Should correctly update rate URLs', async function () {
191 this.timeout(30000)
192
193 async function check (like: string, ofServerUrl: string, urlSuffix: string, remote: 'true' | 'false') {
194 const query = `SELECT "videoId", "accountVideoRate".url FROM "accountVideoRate" ` +
195 `INNER JOIN video ON "accountVideoRate"."videoId" = video.id AND remote IS ${remote} WHERE "accountVideoRate"."url" LIKE '${like}'`
196 const res = await selectQuery(servers[0].internalServerNumber, query)
197
198 for (const rate of res) {
199 const matcher = new RegExp(`^${ofServerUrl}/accounts/root/dislikes/\\d+${urlSuffix}$`)
200 expect(rate.url).to.match(matcher)
201 }
202 }
203
204 async function checkLocal () {
205 const startsWith = 'http://' + servers[0].host + '%'
206 // On local videos
207 await check(startsWith, servers[0].url, '', 'false')
208 // On remote videos
209 await check(startsWith, servers[0].url, '', 'true')
210 }
211
212 async function checkRemote (suffix: string) {
213 const startsWith = 'http://' + servers[1].host + '%'
214 // On local videos
215 await check(startsWith, servers[1].url, suffix, 'false')
216 // On remote videos, we should not update URLs so no suffix
217 await check(startsWith, servers[1].url, '', 'true')
218 }
219
220 await checkLocal()
221 await checkRemote('')
222
223 {
224 const query = `UPDATE "accountVideoRate" SET url = url || 'stan'`
225 await updateQuery(servers[1].internalServerNumber, query)
226
227 await wait(5000)
228 await waitJobs(servers)
229 }
230
231 await checkLocal()
232 await checkRemote('stan')
233 })
234
235 it('Should correctly update comment URLs', async function () {
236 this.timeout(30000)
237
238 async function check (like: string, ofServerUrl: string, urlSuffix: string, remote: 'true' | 'false') {
239 const query = `SELECT "videoId", "videoComment".url, uuid as "videoUUID" FROM "videoComment" ` +
240 `INNER JOIN video ON "videoComment"."videoId" = video.id AND remote IS ${remote} WHERE "videoComment"."url" LIKE '${like}'`
241
242 const res = await selectQuery(servers[0].internalServerNumber, query)
243
244 for (const comment of res) {
245 const matcher = new RegExp(`${ofServerUrl}/videos/watch/${comment.videoUUID}/comments/\\d+${urlSuffix}`)
246 expect(comment.url).to.match(matcher)
247 }
248 }
249
250 async function checkLocal () {
251 const startsWith = 'http://' + servers[0].host + '%'
252 // On local videos
253 await check(startsWith, servers[0].url, '', 'false')
254 // On remote videos
255 await check(startsWith, servers[0].url, '', 'true')
256 }
257
258 async function checkRemote (suffix: string) {
259 const startsWith = 'http://' + servers[1].host + '%'
260 // On local videos
261 await check(startsWith, servers[1].url, suffix, 'false')
262 // On remote videos, we should not update URLs so no suffix
263 await check(startsWith, servers[1].url, '', 'true')
264 }
265
266 {
267 const query = `UPDATE "videoComment" SET url = url || 'kyle'`
268 await updateQuery(servers[1].internalServerNumber, query)
269
270 await wait(5000)
271 await waitJobs(servers)
272 }
273
274 await checkLocal()
275 await checkRemote('kyle')
276 })
277
278 after(async function () {
279 await cleanupTests(servers)
280
281 await closeAllSequelize(servers)
282 })
283})
diff --git a/server/tests/api/activitypub/index.ts b/server/tests/api/activitypub/index.ts
index 92bd6f660..324b444e4 100644
--- a/server/tests/api/activitypub/index.ts
+++ b/server/tests/api/activitypub/index.ts
@@ -1,3 +1,4 @@
1import './cleaner'
1import './client' 2import './client'
2import './fetch' 3import './fetch'
3import './refresher' 4import './refresher'
diff --git a/shared/extra-utils/miscs/sql.ts b/shared/extra-utils/miscs/sql.ts
index e68812e1b..740f0c2d6 100644
--- a/shared/extra-utils/miscs/sql.ts
+++ b/shared/extra-utils/miscs/sql.ts
@@ -24,6 +24,25 @@ function getSequelize (internalServerNumber: number) {
24 return seq 24 return seq
25} 25}
26 26
27function deleteAll (internalServerNumber: number, table: string) {
28 const seq = getSequelize(internalServerNumber)
29
30 const options = { type: QueryTypes.DELETE }
31
32 return seq.query(`DELETE FROM "${table}"`, options)
33}
34
35async function getCount (internalServerNumber: number, table: string) {
36 const seq = getSequelize(internalServerNumber)
37
38 const options = { type: QueryTypes.SELECT as QueryTypes.SELECT }
39
40 const [ { total } ] = await seq.query<{ total: string }>(`SELECT COUNT(*) as total FROM "${table}"`, options)
41 if (total === null) return 0
42
43 return parseInt(total, 10)
44}
45
27function setActorField (internalServerNumber: number, to: string, field: string, value: string) { 46function setActorField (internalServerNumber: number, to: string, field: string, value: string) {
28 const seq = getSequelize(internalServerNumber) 47 const seq = getSequelize(internalServerNumber)
29 48
@@ -63,6 +82,20 @@ async function countVideoViewsOf (internalServerNumber: number, uuid: string) {
63 return parseInt(total + '', 10) 82 return parseInt(total + '', 10)
64} 83}
65 84
85function selectQuery (internalServerNumber: number, query: string) {
86 const seq = getSequelize(internalServerNumber)
87 const options = { type: QueryTypes.SELECT as QueryTypes.SELECT }
88
89 return seq.query<any>(query, options)
90}
91
92function updateQuery (internalServerNumber: number, query: string) {
93 const seq = getSequelize(internalServerNumber)
94 const options = { type: QueryTypes.UPDATE as QueryTypes.UPDATE }
95
96 return seq.query(query, options)
97}
98
66async function closeAllSequelize (servers: ServerInfo[]) { 99async function closeAllSequelize (servers: ServerInfo[]) {
67 for (const server of servers) { 100 for (const server of servers) {
68 if (sequelizes[server.internalServerNumber]) { 101 if (sequelizes[server.internalServerNumber]) {
@@ -95,6 +128,10 @@ export {
95 setActorField, 128 setActorField,
96 countVideoViewsOf, 129 countVideoViewsOf,
97 setPluginVersion, 130 setPluginVersion,
131 selectQuery,
132 deleteAll,
133 updateQuery,
98 setActorFollowScores, 134 setActorFollowScores,
99 closeAllSequelize 135 closeAllSequelize,
136 getCount
100} 137}
diff --git a/shared/extra-utils/server/jobs.ts b/shared/extra-utils/server/jobs.ts
index 97971f960..704929bd4 100644
--- a/shared/extra-utils/server/jobs.ts
+++ b/shared/extra-utils/server/jobs.ts
@@ -63,6 +63,7 @@ async function waitJobs (serversArg: ServerInfo[] | ServerInfo) {
63 else servers = serversArg as ServerInfo[] 63 else servers = serversArg as ServerInfo[]
64 64
65 const states: JobState[] = [ 'waiting', 'active', 'delayed' ] 65 const states: JobState[] = [ 'waiting', 'active', 'delayed' ]
66 const repeatableJobs = [ 'videos-views', 'activitypub-cleaner' ]
66 let pendingRequests: boolean 67 let pendingRequests: boolean
67 68
68 function tasksBuilder () { 69 function tasksBuilder () {
@@ -79,7 +80,7 @@ async function waitJobs (serversArg: ServerInfo[] | ServerInfo) {
79 count: 10, 80 count: 10,
80 sort: '-createdAt' 81 sort: '-createdAt'
81 }).then(res => res.body.data) 82 }).then(res => res.body.data)
82 .then((jobs: Job[]) => jobs.filter(j => j.type !== 'videos-views')) 83 .then((jobs: Job[]) => jobs.filter(j => !repeatableJobs.includes(j.type)))
83 .then(jobs => { 84 .then(jobs => {
84 if (jobs.length !== 0) { 85 if (jobs.length !== 0) {
85 pendingRequests = true 86 pendingRequests = true
diff --git a/shared/extra-utils/videos/videos.ts b/shared/extra-utils/videos/videos.ts
index 0b6a54046..67fe82d41 100644
--- a/shared/extra-utils/videos/videos.ts
+++ b/shared/extra-utils/videos/videos.ts
@@ -498,7 +498,7 @@ function updateVideo (
498 }) 498 })
499} 499}
500 500
501function rateVideo (url: string, accessToken: string, id: number, rating: string, specialStatus = HttpStatusCode.NO_CONTENT_204) { 501function rateVideo (url: string, accessToken: string, id: number | string, rating: string, specialStatus = HttpStatusCode.NO_CONTENT_204) {
502 const path = '/api/v1/videos/' + id + '/rate' 502 const path = '/api/v1/videos/' + id + '/rate'
503 503
504 return request(url) 504 return request(url)
diff --git a/shared/models/server/job.model.ts b/shared/models/server/job.model.ts
index c693827b0..83ef84457 100644
--- a/shared/models/server/job.model.ts
+++ b/shared/models/server/job.model.ts
@@ -8,6 +8,7 @@ export type JobType =
8 | 'activitypub-http-unicast' 8 | 'activitypub-http-unicast'
9 | 'activitypub-http-broadcast' 9 | 'activitypub-http-broadcast'
10 | 'activitypub-http-fetcher' 10 | 'activitypub-http-fetcher'
11 | 'activitypub-cleaner'
11 | 'activitypub-follow' 12 | 'activitypub-follow'
12 | 'video-file-import' 13 | 'video-file-import'
13 | 'video-transcoding' 14 | 'video-transcoding'