aboutsummaryrefslogtreecommitdiffhomepage
diff options
context:
space:
mode:
authorChocobozzz <me@florianbigard.com>2019-09-04 11:18:33 +0200
committerChocobozzz <chocobozzz@cpy.re>2019-09-04 16:24:58 +0200
commit6f1b4fa417786c2015f16b435e872aa65378efd7 (patch)
treebcf97760b4819d3204f42592a9e6f052674b1f01
parent8424c4026afd7304880a4ce8138a04ffb3d8c938 (diff)
downloadPeerTube-6f1b4fa417786c2015f16b435e872aa65378efd7.tar.gz
PeerTube-6f1b4fa417786c2015f16b435e872aa65378efd7.tar.zst
PeerTube-6f1b4fa417786c2015f16b435e872aa65378efd7.zip
Add auto follow instances index support
-rw-r--r--server.ts2
-rw-r--r--server/initializers/constants.ts7
-rw-r--r--server/lib/schedulers/auto-follow-index-instances.ts72
-rw-r--r--server/models/activitypub/actor-follow.ts43
-rw-r--r--server/tests/api/server/auto-follows.ts83
-rw-r--r--shared/extra-utils/index.ts1
-rw-r--r--shared/extra-utils/instances-index/mock-instances-index.ts38
-rw-r--r--tsconfig.json3
8 files changed, 234 insertions, 15 deletions
diff --git a/server.ts b/server.ts
index d5f8f0b2b..5cfa09445 100644
--- a/server.ts
+++ b/server.ts
@@ -115,6 +115,7 @@ import { UpdateVideosScheduler } from './server/lib/schedulers/update-videos-sch
115import { YoutubeDlUpdateScheduler } from './server/lib/schedulers/youtube-dl-update-scheduler' 115import { YoutubeDlUpdateScheduler } from './server/lib/schedulers/youtube-dl-update-scheduler'
116import { VideosRedundancyScheduler } from './server/lib/schedulers/videos-redundancy-scheduler' 116import { VideosRedundancyScheduler } from './server/lib/schedulers/videos-redundancy-scheduler'
117import { RemoveOldHistoryScheduler } from './server/lib/schedulers/remove-old-history-scheduler' 117import { RemoveOldHistoryScheduler } from './server/lib/schedulers/remove-old-history-scheduler'
118import { AutoFollowIndexInstances } from './server/lib/schedulers/auto-follow-index-instances'
118import { isHTTPSignatureDigestValid } from './server/helpers/peertube-crypto' 119import { isHTTPSignatureDigestValid } from './server/helpers/peertube-crypto'
119import { PeerTubeSocket } from './server/lib/peertube-socket' 120import { PeerTubeSocket } from './server/lib/peertube-socket'
120import { updateStreamingPlaylistsInfohashesIfNeeded } from './server/lib/hls' 121import { updateStreamingPlaylistsInfohashesIfNeeded } from './server/lib/hls'
@@ -260,6 +261,7 @@ async function startApplication () {
260 RemoveOldHistoryScheduler.Instance.enable() 261 RemoveOldHistoryScheduler.Instance.enable()
261 RemoveOldViewsScheduler.Instance.enable() 262 RemoveOldViewsScheduler.Instance.enable()
262 PluginsCheckScheduler.Instance.enable() 263 PluginsCheckScheduler.Instance.enable()
264 AutoFollowIndexInstances.Instance.enable()
263 265
264 // Redis initialization 266 // Redis initialization
265 Redis.Instance.init() 267 Redis.Instance.init()
diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts
index 908231a88..7c0c5a87c 100644
--- a/server/initializers/constants.ts
+++ b/server/initializers/constants.ts
@@ -168,10 +168,15 @@ const SCHEDULER_INTERVALS_MS = {
168 updateVideos: 60000, // 1 minute 168 updateVideos: 60000, // 1 minute
169 youtubeDLUpdate: 60000 * 60 * 24, // 1 day 169 youtubeDLUpdate: 60000 * 60 * 24, // 1 day
170 checkPlugins: CONFIG.PLUGINS.INDEX.CHECK_LATEST_VERSIONS_INTERVAL, 170 checkPlugins: CONFIG.PLUGINS.INDEX.CHECK_LATEST_VERSIONS_INTERVAL,
171 autoFollowIndexInstances: 60000 * 60 * 24, // 1 day
171 removeOldViews: 60000 * 60 * 24, // 1 day 172 removeOldViews: 60000 * 60 * 24, // 1 day
172 removeOldHistory: 60000 * 60 * 24 // 1 day 173 removeOldHistory: 60000 * 60 * 24 // 1 day
173} 174}
174 175
176const INSTANCES_INDEX = {
177 HOSTS_PATH: '/api/v1/instances/hosts'
178}
179
175// --------------------------------------------------------------------------- 180// ---------------------------------------------------------------------------
176 181
177const CONSTRAINTS_FIELDS = { 182const CONSTRAINTS_FIELDS = {
@@ -633,6 +638,7 @@ if (isTestInstance() === true) {
633 SCHEDULER_INTERVALS_MS.removeOldHistory = 5000 638 SCHEDULER_INTERVALS_MS.removeOldHistory = 5000
634 SCHEDULER_INTERVALS_MS.removeOldViews = 5000 639 SCHEDULER_INTERVALS_MS.removeOldViews = 5000
635 SCHEDULER_INTERVALS_MS.updateVideos = 5000 640 SCHEDULER_INTERVALS_MS.updateVideos = 5000
641 SCHEDULER_INTERVALS_MS.autoFollowIndexInstances = 5000
636 REPEAT_JOBS[ 'videos-views' ] = { every: 5000 } 642 REPEAT_JOBS[ 'videos-views' ] = { every: 5000 }
637 643
638 REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR = 1 644 REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR = 1
@@ -683,6 +689,7 @@ export {
683 PREVIEWS_SIZE, 689 PREVIEWS_SIZE,
684 REMOTE_SCHEME, 690 REMOTE_SCHEME,
685 FOLLOW_STATES, 691 FOLLOW_STATES,
692 INSTANCES_INDEX,
686 DEFAULT_USER_THEME_NAME, 693 DEFAULT_USER_THEME_NAME,
687 SERVER_ACTOR_NAME, 694 SERVER_ACTOR_NAME,
688 PLUGIN_GLOBAL_CSS_FILE_NAME, 695 PLUGIN_GLOBAL_CSS_FILE_NAME,
diff --git a/server/lib/schedulers/auto-follow-index-instances.ts b/server/lib/schedulers/auto-follow-index-instances.ts
new file mode 100644
index 000000000..ef11fc87f
--- /dev/null
+++ b/server/lib/schedulers/auto-follow-index-instances.ts
@@ -0,0 +1,72 @@
1import { logger } from '../../helpers/logger'
2import { AbstractScheduler } from './abstract-scheduler'
3import { INSTANCES_INDEX, SCHEDULER_INTERVALS_MS, SERVER_ACTOR_NAME } from '../../initializers/constants'
4import { CONFIG } from '../../initializers/config'
5import { chunk } from 'lodash'
6import { doRequest } from '@server/helpers/requests'
7import { ActorFollowModel } from '@server/models/activitypub/actor-follow'
8import { JobQueue } from '@server/lib/job-queue'
9import { getServerActor } from '@server/helpers/utils'
10
11export class AutoFollowIndexInstances extends AbstractScheduler {
12
13 private static instance: AbstractScheduler
14
15 protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.autoFollowIndexInstances
16
17 private lastCheck: Date
18
19 private constructor () {
20 super()
21 }
22
23 protected async internalExecute () {
24 return this.autoFollow()
25 }
26
27 private async autoFollow () {
28 if (CONFIG.FOLLOWINGS.INSTANCE.AUTO_FOLLOW_INDEX.ENABLED === false) return
29
30 const indexUrl = CONFIG.FOLLOWINGS.INSTANCE.AUTO_FOLLOW_INDEX.INDEX_URL
31
32 logger.info('Auto follow instances of index %s.', indexUrl)
33
34 try {
35 const serverActor = await getServerActor()
36
37 const uri = indexUrl + INSTANCES_INDEX.HOSTS_PATH
38
39 const qs = this.lastCheck ? { since: this.lastCheck.toISOString() } : {}
40 this.lastCheck = new Date()
41
42 const { body } = await doRequest({ uri, qs, json: true })
43
44 const hosts: string[] = body.data.map(o => o.host)
45 const chunks = chunk(hosts, 20)
46
47 for (const chunk of chunks) {
48 const unfollowedHosts = await ActorFollowModel.keepUnfollowedInstance(chunk)
49
50 for (const unfollowedHost of unfollowedHosts) {
51 const payload = {
52 host: unfollowedHost,
53 name: SERVER_ACTOR_NAME,
54 followerActorId: serverActor.id,
55 isAutoFollow: true
56 }
57
58 await JobQueue.Instance.createJob({ type: 'activitypub-follow', payload })
59 .catch(err => logger.error('Cannot create follow job for %s.', unfollowedHost, err))
60 }
61 }
62
63 } catch (err) {
64 logger.error('Cannot auto follow hosts of index %s.', indexUrl, { err })
65 }
66
67 }
68
69 static get Instance () {
70 return this.instance || (this.instance = new this())
71 }
72}
diff --git a/server/models/activitypub/actor-follow.ts b/server/models/activitypub/actor-follow.ts
index c8b3aae9f..0833b9a93 100644
--- a/server/models/activitypub/actor-follow.ts
+++ b/server/models/activitypub/actor-follow.ts
@@ -1,5 +1,5 @@
1import * as Bluebird from 'bluebird' 1import * as Bluebird from 'bluebird'
2import { values } from 'lodash' 2import { values, difference } from 'lodash'
3import { 3import {
4 AfterCreate, 4 AfterCreate,
5 AfterDestroy, 5 AfterDestroy,
@@ -21,7 +21,7 @@ import { FollowState } from '../../../shared/models/actors'
21import { ActorFollow } from '../../../shared/models/actors/follow.model' 21import { ActorFollow } from '../../../shared/models/actors/follow.model'
22import { logger } from '../../helpers/logger' 22import { logger } from '../../helpers/logger'
23import { getServerActor } from '../../helpers/utils' 23import { getServerActor } from '../../helpers/utils'
24import { ACTOR_FOLLOW_SCORE, FOLLOW_STATES } from '../../initializers/constants' 24import { ACTOR_FOLLOW_SCORE, FOLLOW_STATES, SERVER_ACTOR_NAME } from '../../initializers/constants'
25import { ServerModel } from '../server/server' 25import { ServerModel } from '../server/server'
26import { createSafeIn, getSort } from '../utils' 26import { createSafeIn, getSort } from '../utils'
27import { ActorModel, unusedActorAttributesForAPI } from './actor' 27import { ActorModel, unusedActorAttributesForAPI } from './actor'
@@ -435,6 +435,45 @@ export class ActorFollowModel extends Model<ActorFollowModel> {
435 }) 435 })
436 } 436 }
437 437
438 static async keepUnfollowedInstance (hosts: string[]) {
439 const followerId = (await getServerActor()).id
440
441 const query = {
442 attributes: [],
443 where: {
444 actorId: followerId
445 },
446 include: [
447 {
448 attributes: [ ],
449 model: ActorModel.unscoped(),
450 required: true,
451 as: 'ActorFollowing',
452 where: {
453 preferredUsername: SERVER_ACTOR_NAME
454 },
455 include: [
456 {
457 attributes: [ 'host' ],
458 model: ServerModel.unscoped(),
459 required: true,
460 where: {
461 host: {
462 [Op.in]: hosts
463 }
464 }
465 }
466 ]
467 }
468 ]
469 }
470
471 const res = await ActorFollowModel.findAll(query)
472 const followedHosts = res.map(res => res.ActorFollowing.Server.host)
473
474 return difference(hosts, followedHosts)
475 }
476
438 static listAcceptedFollowerUrlsForAP (actorIds: number[], t: Transaction, start?: number, count?: number) { 477 static listAcceptedFollowerUrlsForAP (actorIds: number[], t: Transaction, start?: number, count?: number) {
439 return ActorFollowModel.createListAcceptedFollowForApiQuery('followers', actorIds, t, start, count) 478 return ActorFollowModel.createListAcceptedFollowForApiQuery('followers', actorIds, t, start, count)
440 } 479 }
diff --git a/server/tests/api/server/auto-follows.ts b/server/tests/api/server/auto-follows.ts
index 32ad259c9..dea9191f2 100644
--- a/server/tests/api/server/auto-follows.ts
+++ b/server/tests/api/server/auto-follows.ts
@@ -6,10 +6,12 @@ import {
6 acceptFollower, 6 acceptFollower,
7 cleanupTests, 7 cleanupTests,
8 flushAndRunMultipleServers, 8 flushAndRunMultipleServers,
9 MockInstancesIndex,
9 ServerInfo, 10 ServerInfo,
10 setAccessTokensToServers, 11 setAccessTokensToServers,
11 unfollow, 12 unfollow,
12 updateCustomSubConfig 13 updateCustomSubConfig,
14 wait
13} from '../../../../shared/extra-utils/index' 15} from '../../../../shared/extra-utils/index'
14import { follow, getFollowersListPaginationAndSort, getFollowingListPaginationAndSort } from '../../../../shared/extra-utils/server/follows' 16import { follow, getFollowersListPaginationAndSort, getFollowingListPaginationAndSort } from '../../../../shared/extra-utils/server/follows'
15import { waitJobs } from '../../../../shared/extra-utils/server/jobs' 17import { waitJobs } from '../../../../shared/extra-utils/server/jobs'
@@ -22,13 +24,14 @@ async function checkFollow (follower: ServerInfo, following: ServerInfo, exists:
22 const res = await getFollowersListPaginationAndSort(following.url, 0, 5, '-createdAt') 24 const res = await getFollowersListPaginationAndSort(following.url, 0, 5, '-createdAt')
23 const follows = res.body.data as ActorFollow[] 25 const follows = res.body.data as ActorFollow[]
24 26
25 if (exists === true) { 27 const follow = follows.find(f => {
26 expect(res.body.total).to.equal(1) 28 return f.follower.host === follower.host && f.state === 'accepted'
29 })
27 30
28 expect(follows[ 0 ].follower.host).to.equal(follower.host) 31 if (exists === true) {
29 expect(follows[ 0 ].state).to.equal('accepted') 32 expect(follow).to.exist
30 } else { 33 } else {
31 expect(follows.filter(f => f.state === 'accepted')).to.have.lengthOf(0) 34 expect(follow).to.be.undefined
32 } 35 }
33 } 36 }
34 37
@@ -36,13 +39,14 @@ async function checkFollow (follower: ServerInfo, following: ServerInfo, exists:
36 const res = await getFollowingListPaginationAndSort(follower.url, 0, 5, '-createdAt') 39 const res = await getFollowingListPaginationAndSort(follower.url, 0, 5, '-createdAt')
37 const follows = res.body.data as ActorFollow[] 40 const follows = res.body.data as ActorFollow[]
38 41
39 if (exists === true) { 42 const follow = follows.find(f => {
40 expect(res.body.total).to.equal(1) 43 return f.following.host === following.host && f.state === 'accepted'
44 })
41 45
42 expect(follows[ 0 ].following.host).to.equal(following.host) 46 if (exists === true) {
43 expect(follows[ 0 ].state).to.equal('accepted') 47 expect(follow).to.exist
44 } else { 48 } else {
45 expect(follows.filter(f => f.state === 'accepted')).to.have.lengthOf(0) 49 expect(follow).to.be.undefined
46 } 50 }
47 } 51 }
48} 52}
@@ -71,7 +75,7 @@ describe('Test auto follows', function () {
71 before(async function () { 75 before(async function () {
72 this.timeout(30000) 76 this.timeout(30000)
73 77
74 servers = await flushAndRunMultipleServers(2) 78 servers = await flushAndRunMultipleServers(3)
75 79
76 // Get the access tokens 80 // Get the access tokens
77 await setAccessTokensToServers(servers) 81 await setAccessTokensToServers(servers)
@@ -142,6 +146,61 @@ describe('Test auto follows', function () {
142 }) 146 })
143 }) 147 })
144 148
149 describe('Auto follow index', function () {
150 const instanceIndexServer = new MockInstancesIndex()
151
152 before(async () => {
153 await instanceIndexServer.initialize()
154 })
155
156 it('Should not auto follow index if the option is not enabled', async function () {
157 this.timeout(30000)
158
159 await wait(5000)
160 await waitJobs(servers)
161
162 await checkFollow(servers[ 0 ], servers[ 1 ], false)
163 await checkFollow(servers[ 1 ], servers[ 0 ], false)
164 })
165
166 it('Should auto follow the index', async function () {
167 this.timeout(30000)
168
169 instanceIndexServer.addInstance(servers[1].host)
170
171 const config = {
172 followings: {
173 instance: {
174 autoFollowIndex: {
175 indexUrl: 'http://localhost:42100',
176 enabled: true
177 }
178 }
179 }
180 }
181 await updateCustomSubConfig(servers[0].url, servers[0].accessToken, config)
182
183 await wait(5000)
184 await waitJobs(servers)
185
186 await checkFollow(servers[ 0 ], servers[ 1 ], true)
187
188 await resetFollows(servers)
189 })
190
191 it('Should follow new added instances in the index but not old ones', async function () {
192 this.timeout(30000)
193
194 instanceIndexServer.addInstance(servers[2].host)
195
196 await wait(5000)
197 await waitJobs(servers)
198
199 await checkFollow(servers[ 0 ], servers[ 1 ], false)
200 await checkFollow(servers[ 0 ], servers[ 2 ], true)
201 })
202 })
203
145 after(async function () { 204 after(async function () {
146 await cleanupTests(servers) 205 await cleanupTests(servers)
147 }) 206 })
diff --git a/shared/extra-utils/index.ts b/shared/extra-utils/index.ts
index 53ddaa681..78acf72aa 100644
--- a/shared/extra-utils/index.ts
+++ b/shared/extra-utils/index.ts
@@ -24,4 +24,5 @@ export * from './videos/video-streaming-playlists'
24export * from './videos/videos' 24export * from './videos/videos'
25export * from './videos/video-change-ownership' 25export * from './videos/video-change-ownership'
26export * from './feeds/feeds' 26export * from './feeds/feeds'
27export * from './instances-index/mock-instances-index'
27export * from './search/videos' 28export * from './search/videos'
diff --git a/shared/extra-utils/instances-index/mock-instances-index.ts b/shared/extra-utils/instances-index/mock-instances-index.ts
new file mode 100644
index 000000000..cfa4523c1
--- /dev/null
+++ b/shared/extra-utils/instances-index/mock-instances-index.ts
@@ -0,0 +1,38 @@
1import * as express from 'express'
2
3export class MockInstancesIndex {
4 private indexInstances: { host: string, createdAt: string }[] = []
5
6 initialize () {
7 return new Promise(res => {
8 const app = express()
9
10 app.use('/', (req: express.Request, res: express.Response, next: express.NextFunction) => {
11 if (process.env.DEBUG) console.log('Receiving request on mocked server %s.', req.url)
12
13 return next()
14 })
15
16 app.get('/api/v1/instances/hosts', (req: express.Request, res: express.Response) => {
17 const since = req.query.since
18
19 const filtered = this.indexInstances.filter(i => {
20 if (!since) return true
21
22 return i.createdAt > since
23 })
24
25 return res.json({
26 total: filtered.length,
27 data: filtered
28 })
29 })
30
31 app.listen(42100, () => res())
32 })
33 }
34
35 addInstance (host: string) {
36 this.indexInstances.push({ host, createdAt: new Date().toISOString() })
37 }
38}
diff --git a/tsconfig.json b/tsconfig.json
index 7e05994fb..f2985f82b 100644
--- a/tsconfig.json
+++ b/tsconfig.json
@@ -17,7 +17,8 @@
17 "typeRoots": [ "node_modules/@types", "server/typings" ], 17 "typeRoots": [ "node_modules/@types", "server/typings" ],
18 "baseUrl": "./", 18 "baseUrl": "./",
19 "paths": { 19 "paths": {
20 "@server/*": [ "server/*" ] 20 "@server/*": [ "server/*" ],
21 "@shared/*": [ "shared/*" ]
21 } 22 }
22 }, 23 },
23 "exclude": [ 24 "exclude": [