aboutsummaryrefslogtreecommitdiffhomepage
path: root/server
diff options
context:
space:
mode:
Diffstat (limited to 'server')
-rw-r--r--server/controllers/api/server/follows.ts78
-rw-r--r--server/initializers/constants.ts4
-rw-r--r--server/lib/job-queue/handlers/activitypub-follow.ts68
-rw-r--r--server/lib/job-queue/handlers/activitypub-http-broadcast.ts17
-rw-r--r--server/lib/job-queue/job-queue.ts5
-rw-r--r--server/tests/utils/server/servers.ts2
6 files changed, 99 insertions, 75 deletions
diff --git a/server/controllers/api/server/follows.ts b/server/controllers/api/server/follows.ts
index bb0063473..e78361c9a 100644
--- a/server/controllers/api/server/follows.ts
+++ b/server/controllers/api/server/follows.ts
@@ -1,20 +1,22 @@
1import * as express from 'express' 1import * as express from 'express'
2import { UserRight } from '../../../../shared/models/users' 2import { UserRight } from '../../../../shared/models/users'
3import { sanitizeHost } from '../../../helpers/core-utils'
4import { retryTransactionWrapper } from '../../../helpers/database-utils'
5import { logger } from '../../../helpers/logger' 3import { logger } from '../../../helpers/logger'
6import { getFormattedObjects, getServerActor } from '../../../helpers/utils' 4import { getFormattedObjects, getServerActor } from '../../../helpers/utils'
7import { loadActorUrlOrGetFromWebfinger } from '../../../helpers/webfinger' 5import { sequelizeTypescript } from '../../../initializers'
8import { REMOTE_SCHEME, sequelizeTypescript, SERVER_ACTOR_NAME } from '../../../initializers' 6import { sendUndoFollow } from '../../../lib/activitypub/send'
9import { getOrCreateActorAndServerAndModel } from '../../../lib/activitypub/actor'
10import { sendFollow, sendUndoFollow } from '../../../lib/activitypub/send'
11import { 7import {
12 asyncMiddleware, authenticate, ensureUserHasRight, paginationValidator, removeFollowingValidator, setBodyHostsPort, setDefaultSort, 8 asyncMiddleware,
13 setDefaultPagination 9 authenticate,
10 ensureUserHasRight,
11 paginationValidator,
12 removeFollowingValidator,
13 setBodyHostsPort,
14 setDefaultPagination,
15 setDefaultSort
14} from '../../../middlewares' 16} from '../../../middlewares'
15import { followersSortValidator, followingSortValidator, followValidator } from '../../../middlewares/validators' 17import { followersSortValidator, followingSortValidator, followValidator } from '../../../middlewares/validators'
16import { ActorModel } from '../../../models/activitypub/actor'
17import { ActorFollowModel } from '../../../models/activitypub/actor-follow' 18import { ActorFollowModel } from '../../../models/activitypub/actor-follow'
19import { JobQueue } from '../../../lib/job-queue'
18 20
19const serverFollowsRouter = express.Router() 21const serverFollowsRouter = express.Router()
20serverFollowsRouter.get('/following', 22serverFollowsRouter.get('/following',
@@ -30,7 +32,7 @@ serverFollowsRouter.post('/following',
30 ensureUserHasRight(UserRight.MANAGE_SERVER_FOLLOW), 32 ensureUserHasRight(UserRight.MANAGE_SERVER_FOLLOW),
31 followValidator, 33 followValidator,
32 setBodyHostsPort, 34 setBodyHostsPort,
33 asyncMiddleware(followRetry) 35 asyncMiddleware(followInstance)
34) 36)
35 37
36serverFollowsRouter.delete('/following/:host', 38serverFollowsRouter.delete('/following/:host',
@@ -70,67 +72,17 @@ async function listFollowers (req: express.Request, res: express.Response, next:
70 return res.json(getFormattedObjects(resultList.data, resultList.total)) 72 return res.json(getFormattedObjects(resultList.data, resultList.total))
71} 73}
72 74
73async function followRetry (req: express.Request, res: express.Response, next: express.NextFunction) { 75async function followInstance (req: express.Request, res: express.Response, next: express.NextFunction) {
74 const hosts = req.body.hosts as string[] 76 const hosts = req.body.hosts as string[]
75 const fromActor = await getServerActor()
76
77 const tasks: Promise<any>[] = []
78 const actorName = SERVER_ACTOR_NAME
79 77
80 for (const host of hosts) { 78 for (const host of hosts) {
81 const sanitizedHost = sanitizeHost(host, REMOTE_SCHEME.HTTP) 79 JobQueue.Instance.createJob({ type: 'activitypub-follow', payload: { host } })
82 80 .catch(err => logger.error('Cannot create follow job for %s.', host, err))
83 // We process each host in a specific transaction
84 // First, we add the follow request in the database
85 // Then we send the follow request to other actor
86 const p = loadActorUrlOrGetFromWebfinger(actorName, sanitizedHost)
87 .then(actorUrl => getOrCreateActorAndServerAndModel(actorUrl))
88 .then(targetActor => {
89 const options = {
90 arguments: [ fromActor, targetActor ],
91 errorMessage: 'Cannot follow with many retries.'
92 }
93
94 return retryTransactionWrapper(follow, options)
95 })
96 .catch(err => logger.warn('Cannot follow server %s.', sanitizedHost, { err }))
97
98 tasks.push(p)
99 } 81 }
100 82
101 // Don't make the client wait the tasks
102 Promise.all(tasks)
103 .catch(err => logger.error('Error in follow.', { err }))
104
105 return res.status(204).end() 83 return res.status(204).end()
106} 84}
107 85
108function follow (fromActor: ActorModel, targetActor: ActorModel) {
109 if (fromActor.id === targetActor.id) {
110 throw new Error('Follower is the same than target actor.')
111 }
112
113 return sequelizeTypescript.transaction(async t => {
114 const [ actorFollow ] = await ActorFollowModel.findOrCreate({
115 where: {
116 actorId: fromActor.id,
117 targetActorId: targetActor.id
118 },
119 defaults: {
120 state: 'pending',
121 actorId: fromActor.id,
122 targetActorId: targetActor.id
123 },
124 transaction: t
125 })
126 actorFollow.ActorFollowing = targetActor
127 actorFollow.ActorFollower = fromActor
128
129 // Send a notification to remote server
130 await sendFollow(actorFollow)
131 })
132}
133
134async function removeFollow (req: express.Request, res: express.Response, next: express.NextFunction) { 86async function removeFollow (req: express.Request, res: express.Response, next: express.NextFunction) {
135 const follow: ActorFollowModel = res.locals.follow 87 const follow: ActorFollowModel = res.locals.follow
136 88
diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts
index 9fde989c5..5ee13389d 100644
--- a/server/initializers/constants.ts
+++ b/server/initializers/constants.ts
@@ -65,6 +65,7 @@ const JOB_ATTEMPTS: { [ id in JobType ]: number } = {
65 'activitypub-http-broadcast': 5, 65 'activitypub-http-broadcast': 5,
66 'activitypub-http-unicast': 5, 66 'activitypub-http-unicast': 5,
67 'activitypub-http-fetcher': 5, 67 'activitypub-http-fetcher': 5,
68 'activitypub-follow': 5,
68 'video-file': 1, 69 'video-file': 1,
69 'email': 5 70 'email': 5
70} 71}
@@ -72,9 +73,11 @@ const JOB_CONCURRENCY: { [ id in JobType ]: number } = {
72 'activitypub-http-broadcast': 1, 73 'activitypub-http-broadcast': 1,
73 'activitypub-http-unicast': 5, 74 'activitypub-http-unicast': 5,
74 'activitypub-http-fetcher': 1, 75 'activitypub-http-fetcher': 1,
76 'activitypub-follow': 3,
75 'video-file': 1, 77 'video-file': 1,
76 'email': 5 78 'email': 5
77} 79}
80const BROADCAST_CONCURRENCY = 5 // How many requests in parallel we do in activitypub-http-broadcast job
78// 2 days 81// 2 days
79const JOB_COMPLETED_LIFETIME = 60000 * 60 * 24 * 2 82const JOB_COMPLETED_LIFETIME = 60000 * 60 * 24 * 2
80 83
@@ -461,6 +464,7 @@ export {
461 LAST_MIGRATION_VERSION, 464 LAST_MIGRATION_VERSION,
462 OAUTH_LIFETIME, 465 OAUTH_LIFETIME,
463 OPENGRAPH_AND_OEMBED_COMMENT, 466 OPENGRAPH_AND_OEMBED_COMMENT,
467 BROADCAST_CONCURRENCY,
464 PAGINATION_COUNT_DEFAULT, 468 PAGINATION_COUNT_DEFAULT,
465 ACTOR_FOLLOW_SCORE, 469 ACTOR_FOLLOW_SCORE,
466 PREVIEWS_SIZE, 470 PREVIEWS_SIZE,
diff --git a/server/lib/job-queue/handlers/activitypub-follow.ts b/server/lib/job-queue/handlers/activitypub-follow.ts
new file mode 100644
index 000000000..6764a4037
--- /dev/null
+++ b/server/lib/job-queue/handlers/activitypub-follow.ts
@@ -0,0 +1,68 @@
1import * as kue from 'kue'
2import { logger } from '../../../helpers/logger'
3import { getServerActor } from '../../../helpers/utils'
4import { REMOTE_SCHEME, sequelizeTypescript, SERVER_ACTOR_NAME } from '../../../initializers'
5import { sendFollow } from '../../activitypub/send'
6import { sanitizeHost } from '../../../helpers/core-utils'
7import { loadActorUrlOrGetFromWebfinger } from '../../../helpers/webfinger'
8import { getOrCreateActorAndServerAndModel } from '../../activitypub/actor'
9import { retryTransactionWrapper } from '../../../helpers/database-utils'
10import { ActorFollowModel } from '../../../models/activitypub/actor-follow'
11import { ActorModel } from '../../../models/activitypub/actor'
12
13export type ActivitypubFollowPayload = {
14 host: string
15}
16
17async function processActivityPubFollow (job: kue.Job) {
18 const payload = job.data as ActivitypubFollowPayload
19 const host = payload.host
20
21 logger.info('Processing ActivityPub follow in job %d.', job.id)
22
23 const sanitizedHost = sanitizeHost(host, REMOTE_SCHEME.HTTP)
24
25 const actorUrl = await loadActorUrlOrGetFromWebfinger(SERVER_ACTOR_NAME, sanitizedHost)
26 const targetActor = await getOrCreateActorAndServerAndModel(actorUrl)
27
28 const fromActor = await getServerActor()
29 const options = {
30 arguments: [ fromActor, targetActor ],
31 errorMessage: 'Cannot follow with many retries.'
32 }
33
34 return retryTransactionWrapper(follow, options)
35}
36// ---------------------------------------------------------------------------
37
38export {
39 processActivityPubFollow
40}
41
42// ---------------------------------------------------------------------------
43
44function follow (fromActor: ActorModel, targetActor: ActorModel) {
45 if (fromActor.id === targetActor.id) {
46 throw new Error('Follower is the same than target actor.')
47 }
48
49 return sequelizeTypescript.transaction(async t => {
50 const [ actorFollow ] = await ActorFollowModel.findOrCreate({
51 where: {
52 actorId: fromActor.id,
53 targetActorId: targetActor.id
54 },
55 defaults: {
56 state: 'pending',
57 actorId: fromActor.id,
58 targetActorId: targetActor.id
59 },
60 transaction: t
61 })
62 actorFollow.ActorFollowing = targetActor
63 actorFollow.ActorFollower = fromActor
64
65 // Send a notification to remote server
66 await sendFollow(actorFollow)
67 })
68}
diff --git a/server/lib/job-queue/handlers/activitypub-http-broadcast.ts b/server/lib/job-queue/handlers/activitypub-http-broadcast.ts
index 78878fc01..38b8393f4 100644
--- a/server/lib/job-queue/handlers/activitypub-http-broadcast.ts
+++ b/server/lib/job-queue/handlers/activitypub-http-broadcast.ts
@@ -1,8 +1,10 @@
1import * as kue from 'kue' 1import * as kue from 'kue'
2import * as Bluebird from 'bluebird'
2import { logger } from '../../../helpers/logger' 3import { logger } from '../../../helpers/logger'
3import { doRequest } from '../../../helpers/requests' 4import { doRequest } from '../../../helpers/requests'
4import { ActorFollowModel } from '../../../models/activitypub/actor-follow' 5import { ActorFollowModel } from '../../../models/activitypub/actor-follow'
5import { buildSignedRequestOptions, computeBody } from './utils/activitypub-http-utils' 6import { buildSignedRequestOptions, computeBody } from './utils/activitypub-http-utils'
7import { BROADCAST_CONCURRENCY } from '../../../initializers'
6 8
7export type ActivitypubHttpBroadcastPayload = { 9export type ActivitypubHttpBroadcastPayload = {
8 uris: string[] 10 uris: string[]
@@ -28,16 +30,11 @@ async function processActivityPubHttpBroadcast (job: kue.Job) {
28 const badUrls: string[] = [] 30 const badUrls: string[] = []
29 const goodUrls: string[] = [] 31 const goodUrls: string[] = []
30 32
31 for (const uri of payload.uris) { 33 await Bluebird.map(payload.uris, uri => {
32 options.uri = uri 34 return doRequest(Object.assign({}, options, { uri }))
33 35 .then(() => goodUrls.push(uri))
34 try { 36 .catch(() => badUrls.push(uri))
35 await doRequest(options) 37 }, { concurrency: BROADCAST_CONCURRENCY })
36 goodUrls.push(uri)
37 } catch (err) {
38 badUrls.push(uri)
39 }
40 }
41 38
42 return ActorFollowModel.updateActorFollowsScore(goodUrls, badUrls, undefined) 39 return ActorFollowModel.updateActorFollowsScore(goodUrls, badUrls, undefined)
43} 40}
diff --git a/server/lib/job-queue/job-queue.ts b/server/lib/job-queue/job-queue.ts
index 1dc28755e..bf40a9206 100644
--- a/server/lib/job-queue/job-queue.ts
+++ b/server/lib/job-queue/job-queue.ts
@@ -8,11 +8,13 @@ import { ActivitypubHttpFetcherPayload, processActivityPubHttpFetcher } from './
8import { ActivitypubHttpUnicastPayload, processActivityPubHttpUnicast } from './handlers/activitypub-http-unicast' 8import { ActivitypubHttpUnicastPayload, processActivityPubHttpUnicast } from './handlers/activitypub-http-unicast'
9import { EmailPayload, processEmail } from './handlers/email' 9import { EmailPayload, processEmail } from './handlers/email'
10import { processVideoFile, VideoFilePayload } from './handlers/video-file' 10import { processVideoFile, VideoFilePayload } from './handlers/video-file'
11import { ActivitypubFollowPayload, processActivityPubFollow } from './handlers/activitypub-follow'
11 12
12type CreateJobArgument = 13type CreateJobArgument =
13 { type: 'activitypub-http-broadcast', payload: ActivitypubHttpBroadcastPayload } | 14 { type: 'activitypub-http-broadcast', payload: ActivitypubHttpBroadcastPayload } |
14 { type: 'activitypub-http-unicast', payload: ActivitypubHttpUnicastPayload } | 15 { type: 'activitypub-http-unicast', payload: ActivitypubHttpUnicastPayload } |
15 { type: 'activitypub-http-fetcher', payload: ActivitypubHttpFetcherPayload } | 16 { type: 'activitypub-http-fetcher', payload: ActivitypubHttpFetcherPayload } |
17 { type: 'activitypub-follow', payload: ActivitypubFollowPayload } |
16 { type: 'video-file', payload: VideoFilePayload } | 18 { type: 'video-file', payload: VideoFilePayload } |
17 { type: 'email', payload: EmailPayload } 19 { type: 'email', payload: EmailPayload }
18 20
@@ -20,6 +22,7 @@ const handlers: { [ id in JobType ]: (job: kue.Job) => Promise<any>} = {
20 'activitypub-http-broadcast': processActivityPubHttpBroadcast, 22 'activitypub-http-broadcast': processActivityPubHttpBroadcast,
21 'activitypub-http-unicast': processActivityPubHttpUnicast, 23 'activitypub-http-unicast': processActivityPubHttpUnicast,
22 'activitypub-http-fetcher': processActivityPubHttpFetcher, 24 'activitypub-http-fetcher': processActivityPubHttpFetcher,
25 'activitypub-follow': processActivityPubFollow,
23 'video-file': processVideoFile, 26 'video-file': processVideoFile,
24 'email': processEmail 27 'email': processEmail
25} 28}
@@ -50,7 +53,7 @@ class JobQueue {
50 } 53 }
51 }) 54 })
52 55
53 this.jobQueue.setMaxListeners(15) 56 this.jobQueue.setMaxListeners(20)
54 57
55 this.jobQueue.on('error', err => { 58 this.jobQueue.on('error', err => {
56 logger.error('Error in job queue.', { err }) 59 logger.error('Error in job queue.', { err })
diff --git a/server/tests/utils/server/servers.ts b/server/tests/utils/server/servers.ts
index 8373c73ab..1372c03c3 100644
--- a/server/tests/utils/server/servers.ts
+++ b/server/tests/utils/server/servers.ts
@@ -88,7 +88,7 @@ function runServer (serverNumber: number, configOverride?: Object) {
88 88
89 // These actions are async so we need to be sure that they have both been done 89 // These actions are async so we need to be sure that they have both been done
90 const serverRunString = { 90 const serverRunString = {
91 'Server listening on port': false 91 'Server listening': false
92 } 92 }
93 const key = 'Database peertube_test' + serverNumber + ' is ready' 93 const key = 'Database peertube_test' + serverNumber + ' is ready'
94 serverRunString[key] = false 94 serverRunString[key] = false