diff options
24 files changed, 655 insertions, 51 deletions
diff --git a/client/src/app/+admin/system/jobs/jobs.component.ts b/client/src/app/+admin/system/jobs/jobs.component.ts index 9f5c04406..43578eedd 100644 --- a/client/src/app/+admin/system/jobs/jobs.component.ts +++ b/client/src/app/+admin/system/jobs/jobs.component.ts | |||
@@ -28,6 +28,7 @@ export class JobsComponent extends RestTable implements OnInit { | |||
28 | 'activitypub-http-fetcher', | 28 | 'activitypub-http-fetcher', |
29 | 'activitypub-http-unicast', | 29 | 'activitypub-http-unicast', |
30 | 'activitypub-refresher', | 30 | 'activitypub-refresher', |
31 | 'activitypub-cleaner', | ||
31 | 'actor-keys', | 32 | 'actor-keys', |
32 | 'email', | 33 | 'email', |
33 | 'video-file-import', | 34 | 'video-file-import', |
diff --git a/config/default.yaml b/config/default.yaml index 2d8afe1c3..a09d20b9d 100644 --- a/config/default.yaml +++ b/config/default.yaml | |||
@@ -192,6 +192,12 @@ federation: | |||
192 | videos: | 192 | videos: |
193 | federate_unlisted: false | 193 | federate_unlisted: false |
194 | 194 | ||
195 | # Add a weekly job that cleans up remote AP interactions on local videos (shares, rates and comments) | ||
196 | # It removes objects that do not exist anymore, and potentially fix their URLs | ||
197 | # This setting is opt-in because due to an old bug in PeerTube, remote rates sent by instance before PeerTube 3.0 will be deleted | ||
198 | # We still suggest you to enable this setting even if your users will loose most of their video's likes/dislikes | ||
199 | cleanup_remote_interactions: false | ||
200 | |||
195 | cache: | 201 | cache: |
196 | previews: | 202 | previews: |
197 | size: 500 # Max number of previews you want to cache | 203 | size: 500 # Max number of previews you want to cache |
diff --git a/config/production.yaml.example b/config/production.yaml.example index 2794c543c..31c0e6b96 100644 --- a/config/production.yaml.example +++ b/config/production.yaml.example | |||
@@ -190,6 +190,12 @@ federation: | |||
190 | videos: | 190 | videos: |
191 | federate_unlisted: false | 191 | federate_unlisted: false |
192 | 192 | ||
193 | # Add a weekly job that cleans up remote AP interactions on local videos (shares, rates and comments) | ||
194 | # It removes objects that do not exist anymore, and potentially fix their URLs | ||
195 | # This setting is opt-in because due to an old bug in PeerTube, remote rates sent by instance before PeerTube 3.0 will be deleted | ||
196 | # We still suggest you to enable this setting even if your users will loose most of their video's likes/dislikes | ||
197 | cleanup_remote_interactions: false | ||
198 | |||
193 | 199 | ||
194 | ############################################################################### | 200 | ############################################################################### |
195 | # | 201 | # |
diff --git a/server/helpers/custom-validators/activitypub/activity.ts b/server/helpers/custom-validators/activitypub/activity.ts index 8b8c0685f..da79b2782 100644 --- a/server/helpers/custom-validators/activitypub/activity.ts +++ b/server/helpers/custom-validators/activitypub/activity.ts | |||
@@ -1,15 +1,16 @@ | |||
1 | import validator from 'validator' | 1 | import validator from 'validator' |
2 | import { Activity, ActivityType } from '../../../../shared/models/activitypub' | 2 | import { Activity, ActivityType } from '../../../../shared/models/activitypub' |
3 | import { exists } from '../misc' | ||
3 | import { sanitizeAndCheckActorObject } from './actor' | 4 | import { sanitizeAndCheckActorObject } from './actor' |
5 | import { isCacheFileObjectValid } from './cache-file' | ||
6 | import { isFlagActivityValid } from './flag' | ||
4 | import { isActivityPubUrlValid, isBaseActivityValid, isObjectValid } from './misc' | 7 | import { isActivityPubUrlValid, isBaseActivityValid, isObjectValid } from './misc' |
5 | import { isDislikeActivityValid } from './rate' | 8 | import { isPlaylistObjectValid } from './playlist' |
9 | import { isDislikeActivityValid, isLikeActivityValid } from './rate' | ||
10 | import { isShareActivityValid } from './share' | ||
6 | import { sanitizeAndCheckVideoCommentObject } from './video-comments' | 11 | import { sanitizeAndCheckVideoCommentObject } from './video-comments' |
7 | import { sanitizeAndCheckVideoTorrentObject } from './videos' | 12 | import { sanitizeAndCheckVideoTorrentObject } from './videos' |
8 | import { isViewActivityValid } from './view' | 13 | import { isViewActivityValid } from './view' |
9 | import { exists } from '../misc' | ||
10 | import { isCacheFileObjectValid } from './cache-file' | ||
11 | import { isFlagActivityValid } from './flag' | ||
12 | import { isPlaylistObjectValid } from './playlist' | ||
13 | 14 | ||
14 | function isRootActivityValid (activity: any) { | 15 | function isRootActivityValid (activity: any) { |
15 | return isCollection(activity) || isActivity(activity) | 16 | return isCollection(activity) || isActivity(activity) |
@@ -70,8 +71,11 @@ function checkFlagActivity (activity: any) { | |||
70 | } | 71 | } |
71 | 72 | ||
72 | function checkDislikeActivity (activity: any) { | 73 | function checkDislikeActivity (activity: any) { |
73 | return isBaseActivityValid(activity, 'Dislike') && | 74 | return isDislikeActivityValid(activity) |
74 | isDislikeActivityValid(activity) | 75 | } |
76 | |||
77 | function checkLikeActivity (activity: any) { | ||
78 | return isLikeActivityValid(activity) | ||
75 | } | 79 | } |
76 | 80 | ||
77 | function checkCreateActivity (activity: any) { | 81 | function checkCreateActivity (activity: any) { |
@@ -118,8 +122,7 @@ function checkRejectActivity (activity: any) { | |||
118 | } | 122 | } |
119 | 123 | ||
120 | function checkAnnounceActivity (activity: any) { | 124 | function checkAnnounceActivity (activity: any) { |
121 | return isBaseActivityValid(activity, 'Announce') && | 125 | return isShareActivityValid(activity) |
122 | isObjectValid(activity.object) | ||
123 | } | 126 | } |
124 | 127 | ||
125 | function checkUndoActivity (activity: any) { | 128 | function checkUndoActivity (activity: any) { |
@@ -132,8 +135,3 @@ function checkUndoActivity (activity: any) { | |||
132 | checkCreateActivity(activity.object) | 135 | checkCreateActivity(activity.object) |
133 | ) | 136 | ) |
134 | } | 137 | } |
135 | |||
136 | function checkLikeActivity (activity: any) { | ||
137 | return isBaseActivityValid(activity, 'Like') && | ||
138 | isObjectValid(activity.object) | ||
139 | } | ||
diff --git a/server/helpers/custom-validators/activitypub/rate.ts b/server/helpers/custom-validators/activitypub/rate.ts index ba68e8074..aafdda443 100644 --- a/server/helpers/custom-validators/activitypub/rate.ts +++ b/server/helpers/custom-validators/activitypub/rate.ts | |||
@@ -1,13 +1,18 @@ | |||
1 | import { isActivityPubUrlValid, isObjectValid } from './misc' | 1 | import { isBaseActivityValid, isObjectValid } from './misc' |
2 | |||
3 | function isLikeActivityValid (activity: any) { | ||
4 | return isBaseActivityValid(activity, 'Like') && | ||
5 | isObjectValid(activity.object) | ||
6 | } | ||
2 | 7 | ||
3 | function isDislikeActivityValid (activity: any) { | 8 | function isDislikeActivityValid (activity: any) { |
4 | return activity.type === 'Dislike' && | 9 | return isBaseActivityValid(activity, 'Dislike') && |
5 | isActivityPubUrlValid(activity.actor) && | ||
6 | isObjectValid(activity.object) | 10 | isObjectValid(activity.object) |
7 | } | 11 | } |
8 | 12 | ||
9 | // --------------------------------------------------------------------------- | 13 | // --------------------------------------------------------------------------- |
10 | 14 | ||
11 | export { | 15 | export { |
12 | isDislikeActivityValid | 16 | isDislikeActivityValid, |
17 | isLikeActivityValid | ||
13 | } | 18 | } |
diff --git a/server/helpers/custom-validators/activitypub/share.ts b/server/helpers/custom-validators/activitypub/share.ts new file mode 100644 index 000000000..fb5e4c05e --- /dev/null +++ b/server/helpers/custom-validators/activitypub/share.ts | |||
@@ -0,0 +1,11 @@ | |||
1 | import { isBaseActivityValid, isObjectValid } from './misc' | ||
2 | |||
3 | function isShareActivityValid (activity: any) { | ||
4 | return isBaseActivityValid(activity, 'Announce') && | ||
5 | isObjectValid(activity.object) | ||
6 | } | ||
7 | // --------------------------------------------------------------------------- | ||
8 | |||
9 | export { | ||
10 | isShareActivityValid | ||
11 | } | ||
diff --git a/server/initializers/checker-before-init.ts b/server/initializers/checker-before-init.ts index 2578de5ed..565e0d1fa 100644 --- a/server/initializers/checker-before-init.ts +++ b/server/initializers/checker-before-init.ts | |||
@@ -36,7 +36,7 @@ function checkMissedConfig () { | |||
36 | 'rates_limit.login.window', 'rates_limit.login.max', 'rates_limit.ask_send_email.window', 'rates_limit.ask_send_email.max', | 36 | 'rates_limit.login.window', 'rates_limit.login.max', 'rates_limit.ask_send_email.window', 'rates_limit.ask_send_email.max', |
37 | 'theme.default', | 37 | 'theme.default', |
38 | 'remote_redundancy.videos.accept_from', | 38 | 'remote_redundancy.videos.accept_from', |
39 | 'federation.videos.federate_unlisted', | 39 | 'federation.videos.federate_unlisted', 'federation.videos.cleanup_remote_interactions', |
40 | 'search.remote_uri.users', 'search.remote_uri.anonymous', 'search.search_index.enabled', 'search.search_index.url', | 40 | 'search.remote_uri.users', 'search.remote_uri.anonymous', 'search.search_index.enabled', 'search.search_index.url', |
41 | 'search.search_index.disable_local_search', 'search.search_index.is_default_search', | 41 | 'search.search_index.disable_local_search', 'search.search_index.is_default_search', |
42 | 'live.enabled', 'live.allow_replay', 'live.max_duration', 'live.max_user_lives', 'live.max_instance_lives', | 42 | 'live.enabled', 'live.allow_replay', 'live.max_duration', 'live.max_user_lives', 'live.max_instance_lives', |
diff --git a/server/initializers/config.ts b/server/initializers/config.ts index 21ca78584..c16b63c33 100644 --- a/server/initializers/config.ts +++ b/server/initializers/config.ts | |||
@@ -159,7 +159,8 @@ const CONFIG = { | |||
159 | }, | 159 | }, |
160 | FEDERATION: { | 160 | FEDERATION: { |
161 | VIDEOS: { | 161 | VIDEOS: { |
162 | FEDERATE_UNLISTED: config.get<boolean>('federation.videos.federate_unlisted') | 162 | FEDERATE_UNLISTED: config.get<boolean>('federation.videos.federate_unlisted'), |
163 | CLEANUP_REMOTE_INTERACTIONS: config.get<boolean>('federation.videos.cleanup_remote_interactions') | ||
163 | } | 164 | } |
164 | }, | 165 | }, |
165 | ADMIN: { | 166 | ADMIN: { |
diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index 74192d590..083a29889 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts | |||
@@ -137,6 +137,7 @@ const JOB_ATTEMPTS: { [id in JobType]: number } = { | |||
137 | 'activitypub-http-unicast': 5, | 137 | 'activitypub-http-unicast': 5, |
138 | 'activitypub-http-fetcher': 5, | 138 | 'activitypub-http-fetcher': 5, |
139 | 'activitypub-follow': 5, | 139 | 'activitypub-follow': 5, |
140 | 'activitypub-cleaner': 1, | ||
140 | 'video-file-import': 1, | 141 | 'video-file-import': 1, |
141 | 'video-transcoding': 1, | 142 | 'video-transcoding': 1, |
142 | 'video-import': 1, | 143 | 'video-import': 1, |
@@ -147,10 +148,12 @@ const JOB_ATTEMPTS: { [id in JobType]: number } = { | |||
147 | 'video-redundancy': 1, | 148 | 'video-redundancy': 1, |
148 | 'video-live-ending': 1 | 149 | 'video-live-ending': 1 |
149 | } | 150 | } |
150 | const JOB_CONCURRENCY: { [id in JobType]?: number } = { | 151 | // Excluded keys are jobs that can be configured by admins |
152 | const JOB_CONCURRENCY: { [id in Exclude<JobType, 'video-transcoding' | 'video-import'>]: number } = { | ||
151 | 'activitypub-http-broadcast': 1, | 153 | 'activitypub-http-broadcast': 1, |
152 | 'activitypub-http-unicast': 5, | 154 | 'activitypub-http-unicast': 5, |
153 | 'activitypub-http-fetcher': 1, | 155 | 'activitypub-http-fetcher': 1, |
156 | 'activitypub-cleaner': 1, | ||
154 | 'activitypub-follow': 1, | 157 | 'activitypub-follow': 1, |
155 | 'video-file-import': 1, | 158 | 'video-file-import': 1, |
156 | 'email': 5, | 159 | 'email': 5, |
@@ -165,6 +168,7 @@ const JOB_TTL: { [id in JobType]: number } = { | |||
165 | 'activitypub-http-unicast': 60000 * 10, // 10 minutes | 168 | 'activitypub-http-unicast': 60000 * 10, // 10 minutes |
166 | 'activitypub-http-fetcher': 1000 * 3600 * 10, // 10 hours | 169 | 'activitypub-http-fetcher': 1000 * 3600 * 10, // 10 hours |
167 | 'activitypub-follow': 60000 * 10, // 10 minutes | 170 | 'activitypub-follow': 60000 * 10, // 10 minutes |
171 | 'activitypub-cleaner': 1000 * 3600, // 1 hour | ||
168 | 'video-file-import': 1000 * 3600, // 1 hour | 172 | 'video-file-import': 1000 * 3600, // 1 hour |
169 | 'video-transcoding': 1000 * 3600 * 48, // 2 days, transcoding could be long | 173 | 'video-transcoding': 1000 * 3600 * 48, // 2 days, transcoding could be long |
170 | 'video-import': 1000 * 3600 * 2, // 2 hours | 174 | 'video-import': 1000 * 3600 * 2, // 2 hours |
@@ -178,6 +182,9 @@ const JOB_TTL: { [id in JobType]: number } = { | |||
178 | const REPEAT_JOBS: { [ id: string ]: EveryRepeatOptions | CronRepeatOptions } = { | 182 | const REPEAT_JOBS: { [ id: string ]: EveryRepeatOptions | CronRepeatOptions } = { |
179 | 'videos-views': { | 183 | 'videos-views': { |
180 | cron: randomInt(1, 20) + ' * * * *' // Between 1-20 minutes past the hour | 184 | cron: randomInt(1, 20) + ' * * * *' // Between 1-20 minutes past the hour |
185 | }, | ||
186 | 'activitypub-cleaner': { | ||
187 | cron: '30 5 * * ' + randomInt(0, 7) // 1 time per week (random day) at 5:30 AM | ||
181 | } | 188 | } |
182 | } | 189 | } |
183 | const JOB_PRIORITY = { | 190 | const JOB_PRIORITY = { |
@@ -188,6 +195,7 @@ const JOB_PRIORITY = { | |||
188 | } | 195 | } |
189 | 196 | ||
190 | const BROADCAST_CONCURRENCY = 10 // How many requests in parallel we do in activitypub-http-broadcast job | 197 | const BROADCAST_CONCURRENCY = 10 // How many requests in parallel we do in activitypub-http-broadcast job |
198 | const AP_CLEANER_CONCURRENCY = 10 // How many requests in parallel we do in activitypub-cleaner job | ||
191 | const CRAWL_REQUEST_CONCURRENCY = 1 // How many requests in parallel to fetch remote data (likes, shares...) | 199 | const CRAWL_REQUEST_CONCURRENCY = 1 // How many requests in parallel to fetch remote data (likes, shares...) |
192 | const JOB_REQUEST_TIMEOUT = 7000 // 7 seconds | 200 | const JOB_REQUEST_TIMEOUT = 7000 // 7 seconds |
193 | const JOB_COMPLETED_LIFETIME = 60000 * 60 * 24 * 2 // 2 days | 201 | const JOB_COMPLETED_LIFETIME = 60000 * 60 * 24 * 2 // 2 days |
@@ -756,6 +764,7 @@ if (isTestInstance() === true) { | |||
756 | SCHEDULER_INTERVALS_MS.autoFollowIndexInstances = 5000 | 764 | SCHEDULER_INTERVALS_MS.autoFollowIndexInstances = 5000 |
757 | SCHEDULER_INTERVALS_MS.updateInboxStats = 5000 | 765 | SCHEDULER_INTERVALS_MS.updateInboxStats = 5000 |
758 | REPEAT_JOBS['videos-views'] = { every: 5000 } | 766 | REPEAT_JOBS['videos-views'] = { every: 5000 } |
767 | REPEAT_JOBS['activitypub-cleaner'] = { every: 5000 } | ||
759 | 768 | ||
760 | REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR = 1 | 769 | REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR = 1 |
761 | 770 | ||
@@ -815,6 +824,7 @@ export { | |||
815 | REDUNDANCY, | 824 | REDUNDANCY, |
816 | JOB_CONCURRENCY, | 825 | JOB_CONCURRENCY, |
817 | JOB_ATTEMPTS, | 826 | JOB_ATTEMPTS, |
827 | AP_CLEANER_CONCURRENCY, | ||
818 | LAST_MIGRATION_VERSION, | 828 | LAST_MIGRATION_VERSION, |
819 | OAUTH_LIFETIME, | 829 | OAUTH_LIFETIME, |
820 | CUSTOM_HTML_TAG_COMMENTS, | 830 | CUSTOM_HTML_TAG_COMMENTS, |
diff --git a/server/lib/activitypub/video-comments.ts b/server/lib/activitypub/video-comments.ts index 902d877c4..d025ed7f1 100644 --- a/server/lib/activitypub/video-comments.ts +++ b/server/lib/activitypub/video-comments.ts | |||
@@ -41,10 +41,10 @@ async function resolveThread (params: ResolveThreadParams): ResolveThreadResult | |||
41 | return await tryResolveThreadFromVideo(params) | 41 | return await tryResolveThreadFromVideo(params) |
42 | } | 42 | } |
43 | } catch (err) { | 43 | } catch (err) { |
44 | logger.debug('Cannot get or create account and video and channel for reply %s, fetch comment', url, { err }) | 44 | logger.debug('Cannot resolve thread from video %s, maybe because it was not a video', url, { err }) |
45 | } | 45 | } |
46 | 46 | ||
47 | return resolveParentComment(params) | 47 | return resolveRemoteParentComment(params) |
48 | } | 48 | } |
49 | 49 | ||
50 | export { | 50 | export { |
@@ -119,7 +119,7 @@ async function tryResolveThreadFromVideo (params: ResolveThreadParams) { | |||
119 | return { video, comment: resultComment, commentCreated } | 119 | return { video, comment: resultComment, commentCreated } |
120 | } | 120 | } |
121 | 121 | ||
122 | async function resolveParentComment (params: ResolveThreadParams) { | 122 | async function resolveRemoteParentComment (params: ResolveThreadParams) { |
123 | const { url, comments } = params | 123 | const { url, comments } = params |
124 | 124 | ||
125 | if (comments.length > ACTIVITY_PUB.MAX_RECURSION_COMMENTS) { | 125 | if (comments.length > ACTIVITY_PUB.MAX_RECURSION_COMMENTS) { |
@@ -133,7 +133,7 @@ async function resolveParentComment (params: ResolveThreadParams) { | |||
133 | }) | 133 | }) |
134 | 134 | ||
135 | if (sanitizeAndCheckVideoCommentObject(body) === false) { | 135 | if (sanitizeAndCheckVideoCommentObject(body) === false) { |
136 | throw new Error('Remote video comment JSON is not valid:' + JSON.stringify(body)) | 136 | throw new Error(`Remote video comment JSON ${url} is not valid:` + JSON.stringify(body)) |
137 | } | 137 | } |
138 | 138 | ||
139 | const actorUrl = body.attributedTo | 139 | const actorUrl = body.attributedTo |
diff --git a/server/lib/job-queue/handlers/activitypub-cleaner.ts b/server/lib/job-queue/handlers/activitypub-cleaner.ts new file mode 100644 index 000000000..b58bbc983 --- /dev/null +++ b/server/lib/job-queue/handlers/activitypub-cleaner.ts | |||
@@ -0,0 +1,194 @@ | |||
1 | import * as Bluebird from 'bluebird' | ||
2 | import * as Bull from 'bull' | ||
3 | import { checkUrlsSameHost } from '@server/helpers/activitypub' | ||
4 | import { isDislikeActivityValid, isLikeActivityValid } from '@server/helpers/custom-validators/activitypub/rate' | ||
5 | import { isShareActivityValid } from '@server/helpers/custom-validators/activitypub/share' | ||
6 | import { sanitizeAndCheckVideoCommentObject } from '@server/helpers/custom-validators/activitypub/video-comments' | ||
7 | import { doRequest } from '@server/helpers/requests' | ||
8 | import { AP_CLEANER_CONCURRENCY } from '@server/initializers/constants' | ||
9 | import { VideoModel } from '@server/models/video/video' | ||
10 | import { VideoCommentModel } from '@server/models/video/video-comment' | ||
11 | import { VideoShareModel } from '@server/models/video/video-share' | ||
12 | import { HttpStatusCode } from '@shared/core-utils' | ||
13 | import { logger } from '../../../helpers/logger' | ||
14 | import { AccountVideoRateModel } from '../../../models/account/account-video-rate' | ||
15 | |||
16 | // Job to clean remote interactions off local videos | ||
17 | |||
18 | async function processActivityPubCleaner (_job: Bull.Job) { | ||
19 | logger.info('Processing ActivityPub cleaner.') | ||
20 | |||
21 | { | ||
22 | const rateUrls = await AccountVideoRateModel.listRemoteRateUrlsOfLocalVideos() | ||
23 | const { bodyValidator, deleter, updater } = rateOptionsFactory() | ||
24 | |||
25 | await Bluebird.map(rateUrls, async rateUrl => { | ||
26 | try { | ||
27 | const result = await updateObjectIfNeeded(rateUrl, bodyValidator, updater, deleter) | ||
28 | |||
29 | if (result?.status === 'deleted') { | ||
30 | const { videoId, type } = result.data | ||
31 | |||
32 | await VideoModel.updateRatesOf(videoId, type, undefined) | ||
33 | } | ||
34 | } catch (err) { | ||
35 | logger.warn('Cannot update/delete remote AP rate %s.', rateUrl, { err }) | ||
36 | } | ||
37 | }, { concurrency: AP_CLEANER_CONCURRENCY }) | ||
38 | } | ||
39 | |||
40 | { | ||
41 | const shareUrls = await VideoShareModel.listRemoteShareUrlsOfLocalVideos() | ||
42 | const { bodyValidator, deleter, updater } = shareOptionsFactory() | ||
43 | |||
44 | await Bluebird.map(shareUrls, async shareUrl => { | ||
45 | try { | ||
46 | await updateObjectIfNeeded(shareUrl, bodyValidator, updater, deleter) | ||
47 | } catch (err) { | ||
48 | logger.warn('Cannot update/delete remote AP share %s.', shareUrl, { err }) | ||
49 | } | ||
50 | }, { concurrency: AP_CLEANER_CONCURRENCY }) | ||
51 | } | ||
52 | |||
53 | { | ||
54 | const commentUrls = await VideoCommentModel.listRemoteCommentUrlsOfLocalVideos() | ||
55 | const { bodyValidator, deleter, updater } = commentOptionsFactory() | ||
56 | |||
57 | await Bluebird.map(commentUrls, async commentUrl => { | ||
58 | try { | ||
59 | await updateObjectIfNeeded(commentUrl, bodyValidator, updater, deleter) | ||
60 | } catch (err) { | ||
61 | logger.warn('Cannot update/delete remote AP comment %s.', commentUrl, { err }) | ||
62 | } | ||
63 | }, { concurrency: AP_CLEANER_CONCURRENCY }) | ||
64 | } | ||
65 | } | ||
66 | |||
67 | // --------------------------------------------------------------------------- | ||
68 | |||
69 | export { | ||
70 | processActivityPubCleaner | ||
71 | } | ||
72 | |||
73 | // --------------------------------------------------------------------------- | ||
74 | |||
75 | async function updateObjectIfNeeded <T> ( | ||
76 | url: string, | ||
77 | bodyValidator: (body: any) => boolean, | ||
78 | updater: (url: string, newUrl: string) => Promise<T>, | ||
79 | deleter: (url: string) => Promise<T> | ||
80 | ): Promise<{ data: T, status: 'deleted' | 'updated' } | null> { | ||
81 | // Fetch url | ||
82 | const { response, body } = await doRequest<any>({ | ||
83 | uri: url, | ||
84 | json: true, | ||
85 | activityPub: true | ||
86 | }) | ||
87 | |||
88 | // Does not exist anymore, remove entry | ||
89 | if (response.statusCode === HttpStatusCode.NOT_FOUND_404) { | ||
90 | logger.info('Removing remote AP object %s.', url) | ||
91 | const data = await deleter(url) | ||
92 | |||
93 | return { status: 'deleted', data } | ||
94 | } | ||
95 | |||
96 | // If not same id, check same host and update | ||
97 | if (!body || !body.id || !bodyValidator(body)) throw new Error(`Body or body id of ${url} is invalid`) | ||
98 | |||
99 | if (body.type === 'Tombstone') { | ||
100 | logger.info('Removing remote AP object %s.', url) | ||
101 | const data = await deleter(url) | ||
102 | |||
103 | return { status: 'deleted', data } | ||
104 | } | ||
105 | |||
106 | const newUrl = body.id | ||
107 | if (newUrl !== url) { | ||
108 | if (checkUrlsSameHost(newUrl, url) !== true) { | ||
109 | throw new Error(`New url ${newUrl} has not the same host than old url ${url}`) | ||
110 | } | ||
111 | |||
112 | logger.info('Updating remote AP object %s.', url) | ||
113 | const data = await updater(url, newUrl) | ||
114 | |||
115 | return { status: 'updated', data } | ||
116 | } | ||
117 | |||
118 | return null | ||
119 | } | ||
120 | |||
121 | function rateOptionsFactory () { | ||
122 | return { | ||
123 | bodyValidator: (body: any) => isLikeActivityValid(body) || isDislikeActivityValid(body), | ||
124 | |||
125 | updater: async (url: string, newUrl: string) => { | ||
126 | const rate = await AccountVideoRateModel.loadByUrl(url, undefined) | ||
127 | rate.url = newUrl | ||
128 | |||
129 | const videoId = rate.videoId | ||
130 | const type = rate.type | ||
131 | |||
132 | await rate.save() | ||
133 | |||
134 | return { videoId, type } | ||
135 | }, | ||
136 | |||
137 | deleter: async (url) => { | ||
138 | const rate = await AccountVideoRateModel.loadByUrl(url, undefined) | ||
139 | |||
140 | const videoId = rate.videoId | ||
141 | const type = rate.type | ||
142 | |||
143 | await rate.destroy() | ||
144 | |||
145 | return { videoId, type } | ||
146 | } | ||
147 | } | ||
148 | } | ||
149 | |||
150 | function shareOptionsFactory () { | ||
151 | return { | ||
152 | bodyValidator: (body: any) => isShareActivityValid(body), | ||
153 | |||
154 | updater: async (url: string, newUrl: string) => { | ||
155 | const share = await VideoShareModel.loadByUrl(url, undefined) | ||
156 | share.url = newUrl | ||
157 | |||
158 | await share.save() | ||
159 | |||
160 | return undefined | ||
161 | }, | ||
162 | |||
163 | deleter: async (url) => { | ||
164 | const share = await VideoShareModel.loadByUrl(url, undefined) | ||
165 | |||
166 | await share.destroy() | ||
167 | |||
168 | return undefined | ||
169 | } | ||
170 | } | ||
171 | } | ||
172 | |||
173 | function commentOptionsFactory () { | ||
174 | return { | ||
175 | bodyValidator: (body: any) => sanitizeAndCheckVideoCommentObject(body), | ||
176 | |||
177 | updater: async (url: string, newUrl: string) => { | ||
178 | const comment = await VideoCommentModel.loadByUrlAndPopulateAccountAndVideo(url) | ||
179 | comment.url = newUrl | ||
180 | |||
181 | await comment.save() | ||
182 | |||
183 | return undefined | ||
184 | }, | ||
185 | |||
186 | deleter: async (url) => { | ||
187 | const comment = await VideoCommentModel.loadByUrlAndPopulateAccountAndVideo(url) | ||
188 | |||
189 | await comment.destroy() | ||
190 | |||
191 | return undefined | ||
192 | } | ||
193 | } | ||
194 | } | ||
diff --git a/server/lib/job-queue/handlers/actor-keys.ts b/server/lib/job-queue/handlers/actor-keys.ts index 8da549640..125307843 100644 --- a/server/lib/job-queue/handlers/actor-keys.ts +++ b/server/lib/job-queue/handlers/actor-keys.ts | |||
@@ -6,7 +6,7 @@ import { logger } from '../../../helpers/logger' | |||
6 | 6 | ||
7 | async function processActorKeys (job: Bull.Job) { | 7 | async function processActorKeys (job: Bull.Job) { |
8 | const payload = job.data as ActorKeysPayload | 8 | const payload = job.data as ActorKeysPayload |
9 | logger.info('Processing email in job %d.', job.id) | 9 | logger.info('Processing actor keys in job %d.', job.id) |
10 | 10 | ||
11 | const actor = await ActorModel.load(payload.actorId) | 11 | const actor = await ActorModel.load(payload.actorId) |
12 | 12 | ||
diff --git a/server/lib/job-queue/job-queue.ts b/server/lib/job-queue/job-queue.ts index efda2e038..42e8347b1 100644 --- a/server/lib/job-queue/job-queue.ts +++ b/server/lib/job-queue/job-queue.ts | |||
@@ -21,6 +21,7 @@ import { | |||
21 | import { logger } from '../../helpers/logger' | 21 | import { logger } from '../../helpers/logger' |
22 | import { JOB_ATTEMPTS, JOB_COMPLETED_LIFETIME, JOB_CONCURRENCY, JOB_TTL, REPEAT_JOBS, WEBSERVER } from '../../initializers/constants' | 22 | import { JOB_ATTEMPTS, JOB_COMPLETED_LIFETIME, JOB_CONCURRENCY, JOB_TTL, REPEAT_JOBS, WEBSERVER } from '../../initializers/constants' |
23 | import { Redis } from '../redis' | 23 | import { Redis } from '../redis' |
24 | import { processActivityPubCleaner } from './handlers/activitypub-cleaner' | ||
24 | import { processActivityPubFollow } from './handlers/activitypub-follow' | 25 | import { processActivityPubFollow } from './handlers/activitypub-follow' |
25 | import { processActivityPubHttpBroadcast } from './handlers/activitypub-http-broadcast' | 26 | import { processActivityPubHttpBroadcast } from './handlers/activitypub-http-broadcast' |
26 | import { processActivityPubHttpFetcher } from './handlers/activitypub-http-fetcher' | 27 | import { processActivityPubHttpFetcher } from './handlers/activitypub-http-fetcher' |
@@ -38,6 +39,7 @@ type CreateJobArgument = | |||
38 | { type: 'activitypub-http-broadcast', payload: ActivitypubHttpBroadcastPayload } | | 39 | { type: 'activitypub-http-broadcast', payload: ActivitypubHttpBroadcastPayload } | |
39 | { type: 'activitypub-http-unicast', payload: ActivitypubHttpUnicastPayload } | | 40 | { type: 'activitypub-http-unicast', payload: ActivitypubHttpUnicastPayload } | |
40 | { type: 'activitypub-http-fetcher', payload: ActivitypubHttpFetcherPayload } | | 41 | { type: 'activitypub-http-fetcher', payload: ActivitypubHttpFetcherPayload } | |
42 | { type: 'activitypub-http-cleaner', payload: {} } | | ||
41 | { type: 'activitypub-follow', payload: ActivitypubFollowPayload } | | 43 | { type: 'activitypub-follow', payload: ActivitypubFollowPayload } | |
42 | { type: 'video-file-import', payload: VideoFileImportPayload } | | 44 | { type: 'video-file-import', payload: VideoFileImportPayload } | |
43 | { type: 'video-transcoding', payload: VideoTranscodingPayload } | | 45 | { type: 'video-transcoding', payload: VideoTranscodingPayload } | |
@@ -58,6 +60,7 @@ const handlers: { [id in JobType]: (job: Bull.Job) => Promise<any> } = { | |||
58 | 'activitypub-http-broadcast': processActivityPubHttpBroadcast, | 60 | 'activitypub-http-broadcast': processActivityPubHttpBroadcast, |
59 | 'activitypub-http-unicast': processActivityPubHttpUnicast, | 61 | 'activitypub-http-unicast': processActivityPubHttpUnicast, |
60 | 'activitypub-http-fetcher': processActivityPubHttpFetcher, | 62 | 'activitypub-http-fetcher': processActivityPubHttpFetcher, |
63 | 'activitypub-cleaner': processActivityPubCleaner, | ||
61 | 'activitypub-follow': processActivityPubFollow, | 64 | 'activitypub-follow': processActivityPubFollow, |
62 | 'video-file-import': processVideoFileImport, | 65 | 'video-file-import': processVideoFileImport, |
63 | 'video-transcoding': processVideoTranscoding, | 66 | 'video-transcoding': processVideoTranscoding, |
@@ -75,6 +78,7 @@ const jobTypes: JobType[] = [ | |||
75 | 'activitypub-http-broadcast', | 78 | 'activitypub-http-broadcast', |
76 | 'activitypub-http-fetcher', | 79 | 'activitypub-http-fetcher', |
77 | 'activitypub-http-unicast', | 80 | 'activitypub-http-unicast', |
81 | 'activitypub-cleaner', | ||
78 | 'email', | 82 | 'email', |
79 | 'video-transcoding', | 83 | 'video-transcoding', |
80 | 'video-file-import', | 84 | 'video-file-import', |
@@ -233,6 +237,12 @@ class JobQueue { | |||
233 | this.queues['videos-views'].add({}, { | 237 | this.queues['videos-views'].add({}, { |
234 | repeat: REPEAT_JOBS['videos-views'] | 238 | repeat: REPEAT_JOBS['videos-views'] |
235 | }).catch(err => logger.error('Cannot add repeatable job.', { err })) | 239 | }).catch(err => logger.error('Cannot add repeatable job.', { err })) |
240 | |||
241 | if (CONFIG.FEDERATION.VIDEOS.CLEANUP_REMOTE_INTERACTIONS) { | ||
242 | this.queues['activitypub-cleaner'].add({}, { | ||
243 | repeat: REPEAT_JOBS['activitypub-cleaner'] | ||
244 | }).catch(err => logger.error('Cannot add repeatable job.', { err })) | ||
245 | } | ||
236 | } | 246 | } |
237 | 247 | ||
238 | private filterJobTypes (jobType?: JobType) { | 248 | private filterJobTypes (jobType?: JobType) { |
diff --git a/server/middlewares/validators/videos/video-rates.ts b/server/middlewares/validators/videos/video-rates.ts index 7dcba15f1..01bdef25f 100644 --- a/server/middlewares/validators/videos/video-rates.ts +++ b/server/middlewares/validators/videos/video-rates.ts | |||
@@ -1,6 +1,6 @@ | |||
1 | import * as express from 'express' | 1 | import * as express from 'express' |
2 | import { body, param, query } from 'express-validator' | 2 | import { body, param, query } from 'express-validator' |
3 | import { isIdOrUUIDValid } from '../../../helpers/custom-validators/misc' | 3 | import { isIdOrUUIDValid, isIdValid } from '../../../helpers/custom-validators/misc' |
4 | import { isRatingValid } from '../../../helpers/custom-validators/video-rates' | 4 | import { isRatingValid } from '../../../helpers/custom-validators/video-rates' |
5 | import { isVideoRatingTypeValid } from '../../../helpers/custom-validators/videos' | 5 | import { isVideoRatingTypeValid } from '../../../helpers/custom-validators/videos' |
6 | import { logger } from '../../../helpers/logger' | 6 | import { logger } from '../../../helpers/logger' |
@@ -28,14 +28,14 @@ const videoUpdateRateValidator = [ | |||
28 | const getAccountVideoRateValidatorFactory = function (rateType: VideoRateType) { | 28 | const getAccountVideoRateValidatorFactory = function (rateType: VideoRateType) { |
29 | return [ | 29 | return [ |
30 | param('name').custom(isAccountNameValid).withMessage('Should have a valid account name'), | 30 | param('name').custom(isAccountNameValid).withMessage('Should have a valid account name'), |
31 | param('videoId').custom(isIdOrUUIDValid).not().isEmpty().withMessage('Should have a valid videoId'), | 31 | param('videoId').custom(isIdValid).not().isEmpty().withMessage('Should have a valid videoId'), |
32 | 32 | ||
33 | async (req: express.Request, res: express.Response, next: express.NextFunction) => { | 33 | async (req: express.Request, res: express.Response, next: express.NextFunction) => { |
34 | logger.debug('Checking videoCommentGetValidator parameters.', { parameters: req.params }) | 34 | logger.debug('Checking videoCommentGetValidator parameters.', { parameters: req.params }) |
35 | 35 | ||
36 | if (areValidationErrors(req, res)) return | 36 | if (areValidationErrors(req, res)) return |
37 | 37 | ||
38 | const rate = await AccountVideoRateModel.loadLocalAndPopulateVideo(rateType, req.params.name, req.params.videoId) | 38 | const rate = await AccountVideoRateModel.loadLocalAndPopulateVideo(rateType, req.params.name, +req.params.videoId) |
39 | if (!rate) { | 39 | if (!rate) { |
40 | return res.status(HttpStatusCode.NOT_FOUND_404) | 40 | return res.status(HttpStatusCode.NOT_FOUND_404) |
41 | .json({ error: 'Video rate not found' }) | 41 | .json({ error: 'Video rate not found' }) |
diff --git a/server/models/account/account-video-rate.ts b/server/models/account/account-video-rate.ts index d9c529491..801f76bba 100644 --- a/server/models/account/account-video-rate.ts +++ b/server/models/account/account-video-rate.ts | |||
@@ -146,10 +146,22 @@ export class AccountVideoRateModel extends Model { | |||
146 | return AccountVideoRateModel.findAndCountAll(query) | 146 | return AccountVideoRateModel.findAndCountAll(query) |
147 | } | 147 | } |
148 | 148 | ||
149 | static listRemoteRateUrlsOfLocalVideos () { | ||
150 | const query = `SELECT "accountVideoRate".url FROM "accountVideoRate" ` + | ||
151 | `INNER JOIN account ON account.id = "accountVideoRate"."accountId" ` + | ||
152 | `INNER JOIN actor ON actor.id = account."actorId" AND actor."serverId" IS NOT NULL ` + | ||
153 | `INNER JOIN video ON video.id = "accountVideoRate"."videoId" AND video.remote IS FALSE` | ||
154 | |||
155 | return AccountVideoRateModel.sequelize.query<{ url: string }>(query, { | ||
156 | type: QueryTypes.SELECT, | ||
157 | raw: true | ||
158 | }).then(rows => rows.map(r => r.url)) | ||
159 | } | ||
160 | |||
149 | static loadLocalAndPopulateVideo ( | 161 | static loadLocalAndPopulateVideo ( |
150 | rateType: VideoRateType, | 162 | rateType: VideoRateType, |
151 | accountName: string, | 163 | accountName: string, |
152 | videoId: number | string, | 164 | videoId: number, |
153 | t?: Transaction | 165 | t?: Transaction |
154 | ): Promise<MAccountVideoRateAccountVideo> { | 166 | ): Promise<MAccountVideoRateAccountVideo> { |
155 | const options: FindOptions = { | 167 | const options: FindOptions = { |
@@ -241,21 +253,7 @@ export class AccountVideoRateModel extends Model { | |||
241 | 253 | ||
242 | await AccountVideoRateModel.destroy(query) | 254 | await AccountVideoRateModel.destroy(query) |
243 | 255 | ||
244 | const field = type === 'like' | 256 | return VideoModel.updateRatesOf(videoId, type, t) |
245 | ? 'likes' | ||
246 | : 'dislikes' | ||
247 | |||
248 | const rawQuery = `UPDATE "video" SET "${field}" = ` + | ||
249 | '(' + | ||
250 | 'SELECT COUNT(id) FROM "accountVideoRate" WHERE "accountVideoRate"."videoId" = "video"."id" AND type = :rateType' + | ||
251 | ') ' + | ||
252 | 'WHERE "video"."id" = :videoId' | ||
253 | |||
254 | return AccountVideoRateModel.sequelize.query(rawQuery, { | ||
255 | transaction: t, | ||
256 | replacements: { videoId, rateType: type }, | ||
257 | type: QueryTypes.UPDATE | ||
258 | }) | ||
259 | }) | 257 | }) |
260 | } | 258 | } |
261 | 259 | ||
diff --git a/server/models/video/video-comment.ts b/server/models/video/video-comment.ts index dc7556d44..151c2bc81 100644 --- a/server/models/video/video-comment.ts +++ b/server/models/video/video-comment.ts | |||
@@ -1,5 +1,5 @@ | |||
1 | import { uniq } from 'lodash' | 1 | import { uniq } from 'lodash' |
2 | import { FindAndCountOptions, FindOptions, Op, Order, ScopeOptions, Sequelize, Transaction, WhereOptions } from 'sequelize' | 2 | import { FindAndCountOptions, FindOptions, Op, Order, QueryTypes, ScopeOptions, Sequelize, Transaction, WhereOptions } from 'sequelize' |
3 | import { | 3 | import { |
4 | AllowNull, | 4 | AllowNull, |
5 | BelongsTo, | 5 | BelongsTo, |
@@ -696,6 +696,18 @@ export class VideoCommentModel extends Model { | |||
696 | } | 696 | } |
697 | } | 697 | } |
698 | 698 | ||
699 | static listRemoteCommentUrlsOfLocalVideos () { | ||
700 | const query = `SELECT "videoComment".url FROM "videoComment" ` + | ||
701 | `INNER JOIN account ON account.id = "videoComment"."accountId" ` + | ||
702 | `INNER JOIN actor ON actor.id = "account"."actorId" AND actor."serverId" IS NOT NULL ` + | ||
703 | `INNER JOIN video ON video.id = "videoComment"."videoId" AND video.remote IS FALSE` | ||
704 | |||
705 | return VideoCommentModel.sequelize.query<{ url: string }>(query, { | ||
706 | type: QueryTypes.SELECT, | ||
707 | raw: true | ||
708 | }).then(rows => rows.map(r => r.url)) | ||
709 | } | ||
710 | |||
699 | static cleanOldCommentsOf (videoId: number, beforeUpdatedAt: Date) { | 711 | static cleanOldCommentsOf (videoId: number, beforeUpdatedAt: Date) { |
700 | const query = { | 712 | const query = { |
701 | where: { | 713 | where: { |
diff --git a/server/models/video/video-share.ts b/server/models/video/video-share.ts index b7f5f3fa3..5059c1fa6 100644 --- a/server/models/video/video-share.ts +++ b/server/models/video/video-share.ts | |||
@@ -1,4 +1,4 @@ | |||
1 | import { literal, Op, Transaction } from 'sequelize' | 1 | import { literal, Op, QueryTypes, Transaction } from 'sequelize' |
2 | import { AllowNull, BelongsTo, Column, CreatedAt, DataType, ForeignKey, Is, Model, Scopes, Table, UpdatedAt } from 'sequelize-typescript' | 2 | import { AllowNull, BelongsTo, Column, CreatedAt, DataType, ForeignKey, Is, Model, Scopes, Table, UpdatedAt } from 'sequelize-typescript' |
3 | import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc' | 3 | import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc' |
4 | import { CONSTRAINTS_FIELDS } from '../../initializers/constants' | 4 | import { CONSTRAINTS_FIELDS } from '../../initializers/constants' |
@@ -185,6 +185,17 @@ export class VideoShareModel extends Model { | |||
185 | return VideoShareModel.findAndCountAll(query) | 185 | return VideoShareModel.findAndCountAll(query) |
186 | } | 186 | } |
187 | 187 | ||
188 | static listRemoteShareUrlsOfLocalVideos () { | ||
189 | const query = `SELECT "videoShare".url FROM "videoShare" ` + | ||
190 | `INNER JOIN actor ON actor.id = "videoShare"."actorId" AND actor."serverId" IS NOT NULL ` + | ||
191 | `INNER JOIN video ON video.id = "videoShare"."videoId" AND video.remote IS FALSE` | ||
192 | |||
193 | return VideoShareModel.sequelize.query<{ url: string }>(query, { | ||
194 | type: QueryTypes.SELECT, | ||
195 | raw: true | ||
196 | }).then(rows => rows.map(r => r.url)) | ||
197 | } | ||
198 | |||
188 | static cleanOldSharesOf (videoId: number, beforeUpdatedAt: Date) { | 199 | static cleanOldSharesOf (videoId: number, beforeUpdatedAt: Date) { |
189 | const query = { | 200 | const query = { |
190 | where: { | 201 | where: { |
diff --git a/server/models/video/video.ts b/server/models/video/video.ts index 8894843e0..b4c7da655 100644 --- a/server/models/video/video.ts +++ b/server/models/video/video.ts | |||
@@ -34,7 +34,7 @@ import { ModelCache } from '@server/models/model-cache' | |||
34 | import { VideoFile } from '@shared/models/videos/video-file.model' | 34 | import { VideoFile } from '@shared/models/videos/video-file.model' |
35 | import { ResultList, UserRight, VideoPrivacy, VideoState } from '../../../shared' | 35 | import { ResultList, UserRight, VideoPrivacy, VideoState } from '../../../shared' |
36 | import { VideoObject } from '../../../shared/models/activitypub/objects' | 36 | import { VideoObject } from '../../../shared/models/activitypub/objects' |
37 | import { Video, VideoDetails } from '../../../shared/models/videos' | 37 | import { Video, VideoDetails, VideoRateType } from '../../../shared/models/videos' |
38 | import { ThumbnailType } from '../../../shared/models/videos/thumbnail.type' | 38 | import { ThumbnailType } from '../../../shared/models/videos/thumbnail.type' |
39 | import { VideoFilter } from '../../../shared/models/videos/video-query.type' | 39 | import { VideoFilter } from '../../../shared/models/videos/video-query.type' |
40 | import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type' | 40 | import { VideoStreamingPlaylistType } from '../../../shared/models/videos/video-streaming-playlist.type' |
@@ -1509,6 +1509,24 @@ export class VideoModel extends Model { | |||
1509 | }) | 1509 | }) |
1510 | } | 1510 | } |
1511 | 1511 | ||
1512 | static updateRatesOf (videoId: number, type: VideoRateType, t: Transaction) { | ||
1513 | const field = type === 'like' | ||
1514 | ? 'likes' | ||
1515 | : 'dislikes' | ||
1516 | |||
1517 | const rawQuery = `UPDATE "video" SET "${field}" = ` + | ||
1518 | '(' + | ||
1519 | 'SELECT COUNT(id) FROM "accountVideoRate" WHERE "accountVideoRate"."videoId" = "video"."id" AND type = :rateType' + | ||
1520 | ') ' + | ||
1521 | 'WHERE "video"."id" = :videoId' | ||
1522 | |||
1523 | return AccountVideoRateModel.sequelize.query(rawQuery, { | ||
1524 | transaction: t, | ||
1525 | replacements: { videoId, rateType: type }, | ||
1526 | type: QueryTypes.UPDATE | ||
1527 | }) | ||
1528 | } | ||
1529 | |||
1512 | static checkVideoHasInstanceFollow (videoId: number, followerActorId: number) { | 1530 | static checkVideoHasInstanceFollow (videoId: number, followerActorId: number) { |
1513 | // Instances only share videos | 1531 | // Instances only share videos |
1514 | const query = 'SELECT 1 FROM "videoShare" ' + | 1532 | const query = 'SELECT 1 FROM "videoShare" ' + |
diff --git a/server/tests/api/activitypub/cleaner.ts b/server/tests/api/activitypub/cleaner.ts new file mode 100644 index 000000000..75ef56ce3 --- /dev/null +++ b/server/tests/api/activitypub/cleaner.ts | |||
@@ -0,0 +1,283 @@ | |||
1 | /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */ | ||
2 | |||
3 | import 'mocha' | ||
4 | import * as chai from 'chai' | ||
5 | import { | ||
6 | cleanupTests, | ||
7 | closeAllSequelize, | ||
8 | deleteAll, | ||
9 | doubleFollow, | ||
10 | getCount, | ||
11 | selectQuery, | ||
12 | setVideoField, | ||
13 | updateQuery, | ||
14 | wait | ||
15 | } from '../../../../shared/extra-utils' | ||
16 | import { flushAndRunMultipleServers, ServerInfo, setAccessTokensToServers } from '../../../../shared/extra-utils/index' | ||
17 | import { waitJobs } from '../../../../shared/extra-utils/server/jobs' | ||
18 | import { addVideoCommentThread, getVideoCommentThreads } from '../../../../shared/extra-utils/videos/video-comments' | ||
19 | import { getVideo, rateVideo, uploadVideoAndGetId } from '../../../../shared/extra-utils/videos/videos' | ||
20 | |||
21 | const expect = chai.expect | ||
22 | |||
23 | describe('Test AP cleaner', function () { | ||
24 | let servers: ServerInfo[] = [] | ||
25 | let videoUUID1: string | ||
26 | let videoUUID2: string | ||
27 | let videoUUID3: string | ||
28 | |||
29 | let videoUUIDs: string[] | ||
30 | |||
31 | before(async function () { | ||
32 | this.timeout(120000) | ||
33 | |||
34 | const config = { | ||
35 | federation: { | ||
36 | videos: { cleanup_remote_interactions: true } | ||
37 | } | ||
38 | } | ||
39 | servers = await flushAndRunMultipleServers(3, config) | ||
40 | |||
41 | // Get the access tokens | ||
42 | await setAccessTokensToServers(servers) | ||
43 | |||
44 | await Promise.all([ | ||
45 | doubleFollow(servers[0], servers[1]), | ||
46 | doubleFollow(servers[1], servers[2]), | ||
47 | doubleFollow(servers[0], servers[2]) | ||
48 | ]) | ||
49 | |||
50 | // Update 1 local share, check 6 shares | ||
51 | |||
52 | // Create 1 comment per video | ||
53 | // Update 1 remote URL and 1 local URL on | ||
54 | |||
55 | videoUUID1 = (await uploadVideoAndGetId({ server: servers[0], videoName: 'server 1' })).uuid | ||
56 | videoUUID2 = (await uploadVideoAndGetId({ server: servers[1], videoName: 'server 2' })).uuid | ||
57 | videoUUID3 = (await uploadVideoAndGetId({ server: servers[2], videoName: 'server 3' })).uuid | ||
58 | |||
59 | videoUUIDs = [ videoUUID1, videoUUID2, videoUUID3 ] | ||
60 | |||
61 | await waitJobs(servers) | ||
62 | |||
63 | for (const server of servers) { | ||
64 | for (const uuid of videoUUIDs) { | ||
65 | await rateVideo(server.url, server.accessToken, uuid, 'like') | ||
66 | await addVideoCommentThread(server.url, server.accessToken, uuid, 'comment') | ||
67 | } | ||
68 | } | ||
69 | |||
70 | await waitJobs(servers) | ||
71 | }) | ||
72 | |||
73 | it('Should have the correct likes', async function () { | ||
74 | for (const server of servers) { | ||
75 | for (const uuid of videoUUIDs) { | ||
76 | const res = await getVideo(server.url, uuid) | ||
77 | expect(res.body.likes).to.equal(3) | ||
78 | expect(res.body.dislikes).to.equal(0) | ||
79 | } | ||
80 | } | ||
81 | }) | ||
82 | |||
83 | it('Should destroy server 3 internal likes and correctly clean them', async function () { | ||
84 | this.timeout(20000) | ||
85 | |||
86 | await deleteAll(servers[2].internalServerNumber, 'accountVideoRate') | ||
87 | for (const uuid of videoUUIDs) { | ||
88 | await setVideoField(servers[2].internalServerNumber, uuid, 'likes', '0') | ||
89 | } | ||
90 | |||
91 | await wait(5000) | ||
92 | await waitJobs(servers) | ||
93 | |||
94 | // Updated rates of my video | ||
95 | { | ||
96 | const res = await getVideo(servers[0].url, videoUUID1) | ||
97 | expect(res.body.likes).to.equal(2) | ||
98 | expect(res.body.dislikes).to.equal(0) | ||
99 | } | ||
100 | |||
101 | // Did not update rates of a remote video | ||
102 | { | ||
103 | const res = await getVideo(servers[0].url, videoUUID2) | ||
104 | expect(res.body.likes).to.equal(3) | ||
105 | expect(res.body.dislikes).to.equal(0) | ||
106 | } | ||
107 | }) | ||
108 | |||
109 | it('Should update rates to dislikes', async function () { | ||
110 | this.timeout(20000) | ||
111 | |||
112 | for (const server of servers) { | ||
113 | for (const uuid of videoUUIDs) { | ||
114 | await rateVideo(server.url, server.accessToken, uuid, 'dislike') | ||
115 | } | ||
116 | } | ||
117 | |||
118 | await waitJobs(servers) | ||
119 | |||
120 | for (const server of servers) { | ||
121 | for (const uuid of videoUUIDs) { | ||
122 | const res = await getVideo(server.url, uuid) | ||
123 | expect(res.body.likes).to.equal(0) | ||
124 | expect(res.body.dislikes).to.equal(3) | ||
125 | } | ||
126 | } | ||
127 | }) | ||
128 | |||
129 | it('Should destroy server 3 internal dislikes and correctly clean them', async function () { | ||
130 | this.timeout(20000) | ||
131 | |||
132 | await deleteAll(servers[2].internalServerNumber, 'accountVideoRate') | ||
133 | |||
134 | for (const uuid of videoUUIDs) { | ||
135 | await setVideoField(servers[2].internalServerNumber, uuid, 'dislikes', '0') | ||
136 | } | ||
137 | |||
138 | await wait(5000) | ||
139 | await waitJobs(servers) | ||
140 | |||
141 | // Updated rates of my video | ||
142 | { | ||
143 | const res = await getVideo(servers[0].url, videoUUID1) | ||
144 | expect(res.body.likes).to.equal(0) | ||
145 | expect(res.body.dislikes).to.equal(2) | ||
146 | } | ||
147 | |||
148 | // Did not update rates of a remote video | ||
149 | { | ||
150 | const res = await getVideo(servers[0].url, videoUUID2) | ||
151 | expect(res.body.likes).to.equal(0) | ||
152 | expect(res.body.dislikes).to.equal(3) | ||
153 | } | ||
154 | }) | ||
155 | |||
156 | it('Should destroy server 3 internal shares and correctly clean them', async function () { | ||
157 | this.timeout(20000) | ||
158 | |||
159 | const preCount = await getCount(servers[0].internalServerNumber, 'videoShare') | ||
160 | expect(preCount).to.equal(6) | ||
161 | |||
162 | await deleteAll(servers[2].internalServerNumber, 'videoShare') | ||
163 | await wait(5000) | ||
164 | await waitJobs(servers) | ||
165 | |||
166 | // Still 6 because we don't have remote shares on local videos | ||
167 | const postCount = await getCount(servers[0].internalServerNumber, 'videoShare') | ||
168 | expect(postCount).to.equal(6) | ||
169 | }) | ||
170 | |||
171 | it('Should destroy server 3 internal comments and correctly clean them', async function () { | ||
172 | this.timeout(20000) | ||
173 | |||
174 | { | ||
175 | const res = await getVideoCommentThreads(servers[0].url, videoUUID1, 0, 5) | ||
176 | expect(res.body.total).to.equal(3) | ||
177 | } | ||
178 | |||
179 | await deleteAll(servers[2].internalServerNumber, 'videoComment') | ||
180 | |||
181 | await wait(5000) | ||
182 | await waitJobs(servers) | ||
183 | |||
184 | { | ||
185 | const res = await getVideoCommentThreads(servers[0].url, videoUUID1, 0, 5) | ||
186 | expect(res.body.total).to.equal(2) | ||
187 | } | ||
188 | }) | ||
189 | |||
190 | it('Should correctly update rate URLs', async function () { | ||
191 | this.timeout(30000) | ||
192 | |||
193 | async function check (like: string, ofServerUrl: string, urlSuffix: string, remote: 'true' | 'false') { | ||
194 | const query = `SELECT "videoId", "accountVideoRate".url FROM "accountVideoRate" ` + | ||
195 | `INNER JOIN video ON "accountVideoRate"."videoId" = video.id AND remote IS ${remote} WHERE "accountVideoRate"."url" LIKE '${like}'` | ||
196 | const res = await selectQuery(servers[0].internalServerNumber, query) | ||
197 | |||
198 | for (const rate of res) { | ||
199 | const matcher = new RegExp(`^${ofServerUrl}/accounts/root/dislikes/\\d+${urlSuffix}$`) | ||
200 | expect(rate.url).to.match(matcher) | ||
201 | } | ||
202 | } | ||
203 | |||
204 | async function checkLocal () { | ||
205 | const startsWith = 'http://' + servers[0].host + '%' | ||
206 | // On local videos | ||
207 | await check(startsWith, servers[0].url, '', 'false') | ||
208 | // On remote videos | ||
209 | await check(startsWith, servers[0].url, '', 'true') | ||
210 | } | ||
211 | |||
212 | async function checkRemote (suffix: string) { | ||
213 | const startsWith = 'http://' + servers[1].host + '%' | ||
214 | // On local videos | ||
215 | await check(startsWith, servers[1].url, suffix, 'false') | ||
216 | // On remote videos, we should not update URLs so no suffix | ||
217 | await check(startsWith, servers[1].url, '', 'true') | ||
218 | } | ||
219 | |||
220 | await checkLocal() | ||
221 | await checkRemote('') | ||
222 | |||
223 | { | ||
224 | const query = `UPDATE "accountVideoRate" SET url = url || 'stan'` | ||
225 | await updateQuery(servers[1].internalServerNumber, query) | ||
226 | |||
227 | await wait(5000) | ||
228 | await waitJobs(servers) | ||
229 | } | ||
230 | |||
231 | await checkLocal() | ||
232 | await checkRemote('stan') | ||
233 | }) | ||
234 | |||
235 | it('Should correctly update comment URLs', async function () { | ||
236 | this.timeout(30000) | ||
237 | |||
238 | async function check (like: string, ofServerUrl: string, urlSuffix: string, remote: 'true' | 'false') { | ||
239 | const query = `SELECT "videoId", "videoComment".url, uuid as "videoUUID" FROM "videoComment" ` + | ||
240 | `INNER JOIN video ON "videoComment"."videoId" = video.id AND remote IS ${remote} WHERE "videoComment"."url" LIKE '${like}'` | ||
241 | |||
242 | const res = await selectQuery(servers[0].internalServerNumber, query) | ||
243 | |||
244 | for (const comment of res) { | ||
245 | const matcher = new RegExp(`${ofServerUrl}/videos/watch/${comment.videoUUID}/comments/\\d+${urlSuffix}`) | ||
246 | expect(comment.url).to.match(matcher) | ||
247 | } | ||
248 | } | ||
249 | |||
250 | async function checkLocal () { | ||
251 | const startsWith = 'http://' + servers[0].host + '%' | ||
252 | // On local videos | ||
253 | await check(startsWith, servers[0].url, '', 'false') | ||
254 | // On remote videos | ||
255 | await check(startsWith, servers[0].url, '', 'true') | ||
256 | } | ||
257 | |||
258 | async function checkRemote (suffix: string) { | ||
259 | const startsWith = 'http://' + servers[1].host + '%' | ||
260 | // On local videos | ||
261 | await check(startsWith, servers[1].url, suffix, 'false') | ||
262 | // On remote videos, we should not update URLs so no suffix | ||
263 | await check(startsWith, servers[1].url, '', 'true') | ||
264 | } | ||
265 | |||
266 | { | ||
267 | const query = `UPDATE "videoComment" SET url = url || 'kyle'` | ||
268 | await updateQuery(servers[1].internalServerNumber, query) | ||
269 | |||
270 | await wait(5000) | ||
271 | await waitJobs(servers) | ||
272 | } | ||
273 | |||
274 | await checkLocal() | ||
275 | await checkRemote('kyle') | ||
276 | }) | ||
277 | |||
278 | after(async function () { | ||
279 | await cleanupTests(servers) | ||
280 | |||
281 | await closeAllSequelize(servers) | ||
282 | }) | ||
283 | }) | ||
diff --git a/server/tests/api/activitypub/index.ts b/server/tests/api/activitypub/index.ts index 92bd6f660..324b444e4 100644 --- a/server/tests/api/activitypub/index.ts +++ b/server/tests/api/activitypub/index.ts | |||
@@ -1,3 +1,4 @@ | |||
1 | import './cleaner' | ||
1 | import './client' | 2 | import './client' |
2 | import './fetch' | 3 | import './fetch' |
3 | import './refresher' | 4 | import './refresher' |
diff --git a/shared/extra-utils/miscs/sql.ts b/shared/extra-utils/miscs/sql.ts index e68812e1b..740f0c2d6 100644 --- a/shared/extra-utils/miscs/sql.ts +++ b/shared/extra-utils/miscs/sql.ts | |||
@@ -24,6 +24,25 @@ function getSequelize (internalServerNumber: number) { | |||
24 | return seq | 24 | return seq |
25 | } | 25 | } |
26 | 26 | ||
27 | function deleteAll (internalServerNumber: number, table: string) { | ||
28 | const seq = getSequelize(internalServerNumber) | ||
29 | |||
30 | const options = { type: QueryTypes.DELETE } | ||
31 | |||
32 | return seq.query(`DELETE FROM "${table}"`, options) | ||
33 | } | ||
34 | |||
35 | async function getCount (internalServerNumber: number, table: string) { | ||
36 | const seq = getSequelize(internalServerNumber) | ||
37 | |||
38 | const options = { type: QueryTypes.SELECT as QueryTypes.SELECT } | ||
39 | |||
40 | const [ { total } ] = await seq.query<{ total: string }>(`SELECT COUNT(*) as total FROM "${table}"`, options) | ||
41 | if (total === null) return 0 | ||
42 | |||
43 | return parseInt(total, 10) | ||
44 | } | ||
45 | |||
27 | function setActorField (internalServerNumber: number, to: string, field: string, value: string) { | 46 | function setActorField (internalServerNumber: number, to: string, field: string, value: string) { |
28 | const seq = getSequelize(internalServerNumber) | 47 | const seq = getSequelize(internalServerNumber) |
29 | 48 | ||
@@ -63,6 +82,20 @@ async function countVideoViewsOf (internalServerNumber: number, uuid: string) { | |||
63 | return parseInt(total + '', 10) | 82 | return parseInt(total + '', 10) |
64 | } | 83 | } |
65 | 84 | ||
85 | function selectQuery (internalServerNumber: number, query: string) { | ||
86 | const seq = getSequelize(internalServerNumber) | ||
87 | const options = { type: QueryTypes.SELECT as QueryTypes.SELECT } | ||
88 | |||
89 | return seq.query<any>(query, options) | ||
90 | } | ||
91 | |||
92 | function updateQuery (internalServerNumber: number, query: string) { | ||
93 | const seq = getSequelize(internalServerNumber) | ||
94 | const options = { type: QueryTypes.UPDATE as QueryTypes.UPDATE } | ||
95 | |||
96 | return seq.query(query, options) | ||
97 | } | ||
98 | |||
66 | async function closeAllSequelize (servers: ServerInfo[]) { | 99 | async function closeAllSequelize (servers: ServerInfo[]) { |
67 | for (const server of servers) { | 100 | for (const server of servers) { |
68 | if (sequelizes[server.internalServerNumber]) { | 101 | if (sequelizes[server.internalServerNumber]) { |
@@ -95,6 +128,10 @@ export { | |||
95 | setActorField, | 128 | setActorField, |
96 | countVideoViewsOf, | 129 | countVideoViewsOf, |
97 | setPluginVersion, | 130 | setPluginVersion, |
131 | selectQuery, | ||
132 | deleteAll, | ||
133 | updateQuery, | ||
98 | setActorFollowScores, | 134 | setActorFollowScores, |
99 | closeAllSequelize | 135 | closeAllSequelize, |
136 | getCount | ||
100 | } | 137 | } |
diff --git a/shared/extra-utils/server/jobs.ts b/shared/extra-utils/server/jobs.ts index 97971f960..704929bd4 100644 --- a/shared/extra-utils/server/jobs.ts +++ b/shared/extra-utils/server/jobs.ts | |||
@@ -63,6 +63,7 @@ async function waitJobs (serversArg: ServerInfo[] | ServerInfo) { | |||
63 | else servers = serversArg as ServerInfo[] | 63 | else servers = serversArg as ServerInfo[] |
64 | 64 | ||
65 | const states: JobState[] = [ 'waiting', 'active', 'delayed' ] | 65 | const states: JobState[] = [ 'waiting', 'active', 'delayed' ] |
66 | const repeatableJobs = [ 'videos-views', 'activitypub-cleaner' ] | ||
66 | let pendingRequests: boolean | 67 | let pendingRequests: boolean |
67 | 68 | ||
68 | function tasksBuilder () { | 69 | function tasksBuilder () { |
@@ -79,7 +80,7 @@ async function waitJobs (serversArg: ServerInfo[] | ServerInfo) { | |||
79 | count: 10, | 80 | count: 10, |
80 | sort: '-createdAt' | 81 | sort: '-createdAt' |
81 | }).then(res => res.body.data) | 82 | }).then(res => res.body.data) |
82 | .then((jobs: Job[]) => jobs.filter(j => j.type !== 'videos-views')) | 83 | .then((jobs: Job[]) => jobs.filter(j => !repeatableJobs.includes(j.type))) |
83 | .then(jobs => { | 84 | .then(jobs => { |
84 | if (jobs.length !== 0) { | 85 | if (jobs.length !== 0) { |
85 | pendingRequests = true | 86 | pendingRequests = true |
diff --git a/shared/extra-utils/videos/videos.ts b/shared/extra-utils/videos/videos.ts index 0b6a54046..67fe82d41 100644 --- a/shared/extra-utils/videos/videos.ts +++ b/shared/extra-utils/videos/videos.ts | |||
@@ -498,7 +498,7 @@ function updateVideo ( | |||
498 | }) | 498 | }) |
499 | } | 499 | } |
500 | 500 | ||
501 | function rateVideo (url: string, accessToken: string, id: number, rating: string, specialStatus = HttpStatusCode.NO_CONTENT_204) { | 501 | function rateVideo (url: string, accessToken: string, id: number | string, rating: string, specialStatus = HttpStatusCode.NO_CONTENT_204) { |
502 | const path = '/api/v1/videos/' + id + '/rate' | 502 | const path = '/api/v1/videos/' + id + '/rate' |
503 | 503 | ||
504 | return request(url) | 504 | return request(url) |
diff --git a/shared/models/server/job.model.ts b/shared/models/server/job.model.ts index c693827b0..83ef84457 100644 --- a/shared/models/server/job.model.ts +++ b/shared/models/server/job.model.ts | |||
@@ -8,6 +8,7 @@ export type JobType = | |||
8 | | 'activitypub-http-unicast' | 8 | | 'activitypub-http-unicast' |
9 | | 'activitypub-http-broadcast' | 9 | | 'activitypub-http-broadcast' |
10 | | 'activitypub-http-fetcher' | 10 | | 'activitypub-http-fetcher' |
11 | | 'activitypub-cleaner' | ||
11 | | 'activitypub-follow' | 12 | | 'activitypub-follow' |
12 | | 'video-file-import' | 13 | | 'video-file-import' |
13 | | 'video-transcoding' | 14 | | 'video-transcoding' |