diff options
Diffstat (limited to 'server')
-rw-r--r-- | server/initializers/constants.ts | 7 | ||||
-rw-r--r-- | server/lib/schedulers/auto-follow-index-instances.ts | 72 | ||||
-rw-r--r-- | server/models/activitypub/actor-follow.ts | 43 | ||||
-rw-r--r-- | server/tests/api/server/auto-follows.ts | 83 |
4 files changed, 191 insertions, 14 deletions
diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index 908231a88..7c0c5a87c 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts | |||
@@ -168,10 +168,15 @@ const SCHEDULER_INTERVALS_MS = { | |||
168 | updateVideos: 60000, // 1 minute | 168 | updateVideos: 60000, // 1 minute |
169 | youtubeDLUpdate: 60000 * 60 * 24, // 1 day | 169 | youtubeDLUpdate: 60000 * 60 * 24, // 1 day |
170 | checkPlugins: CONFIG.PLUGINS.INDEX.CHECK_LATEST_VERSIONS_INTERVAL, | 170 | checkPlugins: CONFIG.PLUGINS.INDEX.CHECK_LATEST_VERSIONS_INTERVAL, |
171 | autoFollowIndexInstances: 60000 * 60 * 24, // 1 day | ||
171 | removeOldViews: 60000 * 60 * 24, // 1 day | 172 | removeOldViews: 60000 * 60 * 24, // 1 day |
172 | removeOldHistory: 60000 * 60 * 24 // 1 day | 173 | removeOldHistory: 60000 * 60 * 24 // 1 day |
173 | } | 174 | } |
174 | 175 | ||
176 | const INSTANCES_INDEX = { | ||
177 | HOSTS_PATH: '/api/v1/instances/hosts' | ||
178 | } | ||
179 | |||
175 | // --------------------------------------------------------------------------- | 180 | // --------------------------------------------------------------------------- |
176 | 181 | ||
177 | const CONSTRAINTS_FIELDS = { | 182 | const CONSTRAINTS_FIELDS = { |
@@ -633,6 +638,7 @@ if (isTestInstance() === true) { | |||
633 | SCHEDULER_INTERVALS_MS.removeOldHistory = 5000 | 638 | SCHEDULER_INTERVALS_MS.removeOldHistory = 5000 |
634 | SCHEDULER_INTERVALS_MS.removeOldViews = 5000 | 639 | SCHEDULER_INTERVALS_MS.removeOldViews = 5000 |
635 | SCHEDULER_INTERVALS_MS.updateVideos = 5000 | 640 | SCHEDULER_INTERVALS_MS.updateVideos = 5000 |
641 | SCHEDULER_INTERVALS_MS.autoFollowIndexInstances = 5000 | ||
636 | REPEAT_JOBS[ 'videos-views' ] = { every: 5000 } | 642 | REPEAT_JOBS[ 'videos-views' ] = { every: 5000 } |
637 | 643 | ||
638 | REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR = 1 | 644 | REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR = 1 |
@@ -683,6 +689,7 @@ export { | |||
683 | PREVIEWS_SIZE, | 689 | PREVIEWS_SIZE, |
684 | REMOTE_SCHEME, | 690 | REMOTE_SCHEME, |
685 | FOLLOW_STATES, | 691 | FOLLOW_STATES, |
692 | INSTANCES_INDEX, | ||
686 | DEFAULT_USER_THEME_NAME, | 693 | DEFAULT_USER_THEME_NAME, |
687 | SERVER_ACTOR_NAME, | 694 | SERVER_ACTOR_NAME, |
688 | PLUGIN_GLOBAL_CSS_FILE_NAME, | 695 | PLUGIN_GLOBAL_CSS_FILE_NAME, |
diff --git a/server/lib/schedulers/auto-follow-index-instances.ts b/server/lib/schedulers/auto-follow-index-instances.ts new file mode 100644 index 000000000..ef11fc87f --- /dev/null +++ b/server/lib/schedulers/auto-follow-index-instances.ts | |||
@@ -0,0 +1,72 @@ | |||
1 | import { logger } from '../../helpers/logger' | ||
2 | import { AbstractScheduler } from './abstract-scheduler' | ||
3 | import { INSTANCES_INDEX, SCHEDULER_INTERVALS_MS, SERVER_ACTOR_NAME } from '../../initializers/constants' | ||
4 | import { CONFIG } from '../../initializers/config' | ||
5 | import { chunk } from 'lodash' | ||
6 | import { doRequest } from '@server/helpers/requests' | ||
7 | import { ActorFollowModel } from '@server/models/activitypub/actor-follow' | ||
8 | import { JobQueue } from '@server/lib/job-queue' | ||
9 | import { getServerActor } from '@server/helpers/utils' | ||
10 | |||
11 | export class AutoFollowIndexInstances extends AbstractScheduler { | ||
12 | |||
13 | private static instance: AbstractScheduler | ||
14 | |||
15 | protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.autoFollowIndexInstances | ||
16 | |||
17 | private lastCheck: Date | ||
18 | |||
19 | private constructor () { | ||
20 | super() | ||
21 | } | ||
22 | |||
23 | protected async internalExecute () { | ||
24 | return this.autoFollow() | ||
25 | } | ||
26 | |||
27 | private async autoFollow () { | ||
28 | if (CONFIG.FOLLOWINGS.INSTANCE.AUTO_FOLLOW_INDEX.ENABLED === false) return | ||
29 | |||
30 | const indexUrl = CONFIG.FOLLOWINGS.INSTANCE.AUTO_FOLLOW_INDEX.INDEX_URL | ||
31 | |||
32 | logger.info('Auto follow instances of index %s.', indexUrl) | ||
33 | |||
34 | try { | ||
35 | const serverActor = await getServerActor() | ||
36 | |||
37 | const uri = indexUrl + INSTANCES_INDEX.HOSTS_PATH | ||
38 | |||
39 | const qs = this.lastCheck ? { since: this.lastCheck.toISOString() } : {} | ||
40 | this.lastCheck = new Date() | ||
41 | |||
42 | const { body } = await doRequest({ uri, qs, json: true }) | ||
43 | |||
44 | const hosts: string[] = body.data.map(o => o.host) | ||
45 | const chunks = chunk(hosts, 20) | ||
46 | |||
47 | for (const chunk of chunks) { | ||
48 | const unfollowedHosts = await ActorFollowModel.keepUnfollowedInstance(chunk) | ||
49 | |||
50 | for (const unfollowedHost of unfollowedHosts) { | ||
51 | const payload = { | ||
52 | host: unfollowedHost, | ||
53 | name: SERVER_ACTOR_NAME, | ||
54 | followerActorId: serverActor.id, | ||
55 | isAutoFollow: true | ||
56 | } | ||
57 | |||
58 | await JobQueue.Instance.createJob({ type: 'activitypub-follow', payload }) | ||
59 | .catch(err => logger.error('Cannot create follow job for %s.', unfollowedHost, err)) | ||
60 | } | ||
61 | } | ||
62 | |||
63 | } catch (err) { | ||
64 | logger.error('Cannot auto follow hosts of index %s.', indexUrl, { err }) | ||
65 | } | ||
66 | |||
67 | } | ||
68 | |||
69 | static get Instance () { | ||
70 | return this.instance || (this.instance = new this()) | ||
71 | } | ||
72 | } | ||
diff --git a/server/models/activitypub/actor-follow.ts b/server/models/activitypub/actor-follow.ts index c8b3aae9f..0833b9a93 100644 --- a/server/models/activitypub/actor-follow.ts +++ b/server/models/activitypub/actor-follow.ts | |||
@@ -1,5 +1,5 @@ | |||
1 | import * as Bluebird from 'bluebird' | 1 | import * as Bluebird from 'bluebird' |
2 | import { values } from 'lodash' | 2 | import { values, difference } from 'lodash' |
3 | import { | 3 | import { |
4 | AfterCreate, | 4 | AfterCreate, |
5 | AfterDestroy, | 5 | AfterDestroy, |
@@ -21,7 +21,7 @@ import { FollowState } from '../../../shared/models/actors' | |||
21 | import { ActorFollow } from '../../../shared/models/actors/follow.model' | 21 | import { ActorFollow } from '../../../shared/models/actors/follow.model' |
22 | import { logger } from '../../helpers/logger' | 22 | import { logger } from '../../helpers/logger' |
23 | import { getServerActor } from '../../helpers/utils' | 23 | import { getServerActor } from '../../helpers/utils' |
24 | import { ACTOR_FOLLOW_SCORE, FOLLOW_STATES } from '../../initializers/constants' | 24 | import { ACTOR_FOLLOW_SCORE, FOLLOW_STATES, SERVER_ACTOR_NAME } from '../../initializers/constants' |
25 | import { ServerModel } from '../server/server' | 25 | import { ServerModel } from '../server/server' |
26 | import { createSafeIn, getSort } from '../utils' | 26 | import { createSafeIn, getSort } from '../utils' |
27 | import { ActorModel, unusedActorAttributesForAPI } from './actor' | 27 | import { ActorModel, unusedActorAttributesForAPI } from './actor' |
@@ -435,6 +435,45 @@ export class ActorFollowModel extends Model<ActorFollowModel> { | |||
435 | }) | 435 | }) |
436 | } | 436 | } |
437 | 437 | ||
438 | static async keepUnfollowedInstance (hosts: string[]) { | ||
439 | const followerId = (await getServerActor()).id | ||
440 | |||
441 | const query = { | ||
442 | attributes: [], | ||
443 | where: { | ||
444 | actorId: followerId | ||
445 | }, | ||
446 | include: [ | ||
447 | { | ||
448 | attributes: [ ], | ||
449 | model: ActorModel.unscoped(), | ||
450 | required: true, | ||
451 | as: 'ActorFollowing', | ||
452 | where: { | ||
453 | preferredUsername: SERVER_ACTOR_NAME | ||
454 | }, | ||
455 | include: [ | ||
456 | { | ||
457 | attributes: [ 'host' ], | ||
458 | model: ServerModel.unscoped(), | ||
459 | required: true, | ||
460 | where: { | ||
461 | host: { | ||
462 | [Op.in]: hosts | ||
463 | } | ||
464 | } | ||
465 | } | ||
466 | ] | ||
467 | } | ||
468 | ] | ||
469 | } | ||
470 | |||
471 | const res = await ActorFollowModel.findAll(query) | ||
472 | const followedHosts = res.map(res => res.ActorFollowing.Server.host) | ||
473 | |||
474 | return difference(hosts, followedHosts) | ||
475 | } | ||
476 | |||
438 | static listAcceptedFollowerUrlsForAP (actorIds: number[], t: Transaction, start?: number, count?: number) { | 477 | static listAcceptedFollowerUrlsForAP (actorIds: number[], t: Transaction, start?: number, count?: number) { |
439 | return ActorFollowModel.createListAcceptedFollowForApiQuery('followers', actorIds, t, start, count) | 478 | return ActorFollowModel.createListAcceptedFollowForApiQuery('followers', actorIds, t, start, count) |
440 | } | 479 | } |
diff --git a/server/tests/api/server/auto-follows.ts b/server/tests/api/server/auto-follows.ts index 32ad259c9..dea9191f2 100644 --- a/server/tests/api/server/auto-follows.ts +++ b/server/tests/api/server/auto-follows.ts | |||
@@ -6,10 +6,12 @@ import { | |||
6 | acceptFollower, | 6 | acceptFollower, |
7 | cleanupTests, | 7 | cleanupTests, |
8 | flushAndRunMultipleServers, | 8 | flushAndRunMultipleServers, |
9 | MockInstancesIndex, | ||
9 | ServerInfo, | 10 | ServerInfo, |
10 | setAccessTokensToServers, | 11 | setAccessTokensToServers, |
11 | unfollow, | 12 | unfollow, |
12 | updateCustomSubConfig | 13 | updateCustomSubConfig, |
14 | wait | ||
13 | } from '../../../../shared/extra-utils/index' | 15 | } from '../../../../shared/extra-utils/index' |
14 | import { follow, getFollowersListPaginationAndSort, getFollowingListPaginationAndSort } from '../../../../shared/extra-utils/server/follows' | 16 | import { follow, getFollowersListPaginationAndSort, getFollowingListPaginationAndSort } from '../../../../shared/extra-utils/server/follows' |
15 | import { waitJobs } from '../../../../shared/extra-utils/server/jobs' | 17 | import { waitJobs } from '../../../../shared/extra-utils/server/jobs' |
@@ -22,13 +24,14 @@ async function checkFollow (follower: ServerInfo, following: ServerInfo, exists: | |||
22 | const res = await getFollowersListPaginationAndSort(following.url, 0, 5, '-createdAt') | 24 | const res = await getFollowersListPaginationAndSort(following.url, 0, 5, '-createdAt') |
23 | const follows = res.body.data as ActorFollow[] | 25 | const follows = res.body.data as ActorFollow[] |
24 | 26 | ||
25 | if (exists === true) { | 27 | const follow = follows.find(f => { |
26 | expect(res.body.total).to.equal(1) | 28 | return f.follower.host === follower.host && f.state === 'accepted' |
29 | }) | ||
27 | 30 | ||
28 | expect(follows[ 0 ].follower.host).to.equal(follower.host) | 31 | if (exists === true) { |
29 | expect(follows[ 0 ].state).to.equal('accepted') | 32 | expect(follow).to.exist |
30 | } else { | 33 | } else { |
31 | expect(follows.filter(f => f.state === 'accepted')).to.have.lengthOf(0) | 34 | expect(follow).to.be.undefined |
32 | } | 35 | } |
33 | } | 36 | } |
34 | 37 | ||
@@ -36,13 +39,14 @@ async function checkFollow (follower: ServerInfo, following: ServerInfo, exists: | |||
36 | const res = await getFollowingListPaginationAndSort(follower.url, 0, 5, '-createdAt') | 39 | const res = await getFollowingListPaginationAndSort(follower.url, 0, 5, '-createdAt') |
37 | const follows = res.body.data as ActorFollow[] | 40 | const follows = res.body.data as ActorFollow[] |
38 | 41 | ||
39 | if (exists === true) { | 42 | const follow = follows.find(f => { |
40 | expect(res.body.total).to.equal(1) | 43 | return f.following.host === following.host && f.state === 'accepted' |
44 | }) | ||
41 | 45 | ||
42 | expect(follows[ 0 ].following.host).to.equal(following.host) | 46 | if (exists === true) { |
43 | expect(follows[ 0 ].state).to.equal('accepted') | 47 | expect(follow).to.exist |
44 | } else { | 48 | } else { |
45 | expect(follows.filter(f => f.state === 'accepted')).to.have.lengthOf(0) | 49 | expect(follow).to.be.undefined |
46 | } | 50 | } |
47 | } | 51 | } |
48 | } | 52 | } |
@@ -71,7 +75,7 @@ describe('Test auto follows', function () { | |||
71 | before(async function () { | 75 | before(async function () { |
72 | this.timeout(30000) | 76 | this.timeout(30000) |
73 | 77 | ||
74 | servers = await flushAndRunMultipleServers(2) | 78 | servers = await flushAndRunMultipleServers(3) |
75 | 79 | ||
76 | // Get the access tokens | 80 | // Get the access tokens |
77 | await setAccessTokensToServers(servers) | 81 | await setAccessTokensToServers(servers) |
@@ -142,6 +146,61 @@ describe('Test auto follows', function () { | |||
142 | }) | 146 | }) |
143 | }) | 147 | }) |
144 | 148 | ||
149 | describe('Auto follow index', function () { | ||
150 | const instanceIndexServer = new MockInstancesIndex() | ||
151 | |||
152 | before(async () => { | ||
153 | await instanceIndexServer.initialize() | ||
154 | }) | ||
155 | |||
156 | it('Should not auto follow index if the option is not enabled', async function () { | ||
157 | this.timeout(30000) | ||
158 | |||
159 | await wait(5000) | ||
160 | await waitJobs(servers) | ||
161 | |||
162 | await checkFollow(servers[ 0 ], servers[ 1 ], false) | ||
163 | await checkFollow(servers[ 1 ], servers[ 0 ], false) | ||
164 | }) | ||
165 | |||
166 | it('Should auto follow the index', async function () { | ||
167 | this.timeout(30000) | ||
168 | |||
169 | instanceIndexServer.addInstance(servers[1].host) | ||
170 | |||
171 | const config = { | ||
172 | followings: { | ||
173 | instance: { | ||
174 | autoFollowIndex: { | ||
175 | indexUrl: 'http://localhost:42100', | ||
176 | enabled: true | ||
177 | } | ||
178 | } | ||
179 | } | ||
180 | } | ||
181 | await updateCustomSubConfig(servers[0].url, servers[0].accessToken, config) | ||
182 | |||
183 | await wait(5000) | ||
184 | await waitJobs(servers) | ||
185 | |||
186 | await checkFollow(servers[ 0 ], servers[ 1 ], true) | ||
187 | |||
188 | await resetFollows(servers) | ||
189 | }) | ||
190 | |||
191 | it('Should follow new added instances in the index but not old ones', async function () { | ||
192 | this.timeout(30000) | ||
193 | |||
194 | instanceIndexServer.addInstance(servers[2].host) | ||
195 | |||
196 | await wait(5000) | ||
197 | await waitJobs(servers) | ||
198 | |||
199 | await checkFollow(servers[ 0 ], servers[ 1 ], false) | ||
200 | await checkFollow(servers[ 0 ], servers[ 2 ], true) | ||
201 | }) | ||
202 | }) | ||
203 | |||
145 | after(async function () { | 204 | after(async function () { |
146 | await cleanupTests(servers) | 205 | await cleanupTests(servers) |
147 | }) | 206 | }) |