aboutsummaryrefslogtreecommitdiffhomepage
diff options
context:
space:
mode:
authorChocobozzz <florian.bigard@gmail.com>2017-11-22 10:29:55 +0100
committerChocobozzz <florian.bigard@gmail.com>2017-11-27 19:40:53 +0100
commitc986175d68a18e96fbd41537a05c7796a2c64f38 (patch)
treee8a8a02abb57e637451afbf3b1b0e7a0dbd4206c
parente71bcc0f4b31ecfd84a786411febfc6d18a85258 (diff)
downloadPeerTube-c986175d68a18e96fbd41537a05c7796a2c64f38.tar.gz
PeerTube-c986175d68a18e96fbd41537a05c7796a2c64f38.tar.zst
PeerTube-c986175d68a18e96fbd41537a05c7796a2c64f38.zip
Fetch outbox to grab old activities
-rw-r--r--.codeclimate.yml3
-rw-r--r--client/src/app/+admin/follows/followers-list/followers-list.component.html1
-rw-r--r--client/src/app/+admin/follows/following-list/following-list.component.html1
-rw-r--r--server/controllers/activitypub/inbox.ts33
-rw-r--r--server/controllers/activitypub/outbox.ts2
-rw-r--r--server/controllers/api/server/follows.ts3
-rw-r--r--server/helpers/activitypub.ts14
-rw-r--r--server/initializers/constants.ts1
-rw-r--r--server/lib/activitypub/fetch.ts15
-rw-r--r--server/lib/activitypub/index.ts1
-rw-r--r--server/lib/activitypub/process/index.ts1
-rw-r--r--server/lib/activitypub/process/process.ts38
-rw-r--r--server/lib/activitypub/send/send-create.ts2
-rw-r--r--server/lib/jobs/activitypub-http-job-scheduler/activitypub-http-fetcher-handler.ts69
-rw-r--r--server/lib/jobs/activitypub-http-job-scheduler/activitypub-http-job-scheduler.ts8
15 files changed, 146 insertions, 46 deletions
diff --git a/.codeclimate.yml b/.codeclimate.yml
index 2318cfa4b..2b58e82b2 100644
--- a/.codeclimate.yml
+++ b/.codeclimate.yml
@@ -6,7 +6,7 @@ engines:
6 enabled: true 6 enabled: true
7 config: 7 config:
8 languages: 8 languages:
9 - javascript 9 - typescript
10 eslint: 10 eslint:
11 enabled: true 11 enabled: true
12 fixme: 12 fixme:
@@ -18,7 +18,6 @@ ratings:
18exclude_paths: 18exclude_paths:
19- config/ 19- config/
20- node_modules/ 20- node_modules/
21- client
22- scripts/ 21- scripts/
23- server/tests/ 22- server/tests/
24- .tmp/ 23- .tmp/
diff --git a/client/src/app/+admin/follows/followers-list/followers-list.component.html b/client/src/app/+admin/follows/followers-list/followers-list.component.html
index 84c49ae80..473801822 100644
--- a/client/src/app/+admin/follows/followers-list/followers-list.component.html
+++ b/client/src/app/+admin/follows/followers-list/followers-list.component.html
@@ -8,7 +8,6 @@
8 > 8 >
9 <p-column field="id" header="ID"></p-column> 9 <p-column field="id" header="ID"></p-column>
10 <p-column field="follower.host" header="Host"></p-column> 10 <p-column field="follower.host" header="Host"></p-column>
11 <p-column field="email" header="Email"></p-column>
12 <p-column field="follower.score" header="Score"></p-column> 11 <p-column field="follower.score" header="Score"></p-column>
13 <p-column field="state" header="State"></p-column> 12 <p-column field="state" header="State"></p-column>
14 <p-column field="createdAt" header="Created date" [sortable]="true"></p-column> 13 <p-column field="createdAt" header="Created date" [sortable]="true"></p-column>
diff --git a/client/src/app/+admin/follows/following-list/following-list.component.html b/client/src/app/+admin/follows/following-list/following-list.component.html
index dbc9852d0..a73084312 100644
--- a/client/src/app/+admin/follows/following-list/following-list.component.html
+++ b/client/src/app/+admin/follows/following-list/following-list.component.html
@@ -8,7 +8,6 @@
8 > 8 >
9 <p-column field="id" header="ID"></p-column> 9 <p-column field="id" header="ID"></p-column>
10 <p-column field="following.host" header="Host"></p-column> 10 <p-column field="following.host" header="Host"></p-column>
11 <p-column field="email" header="Email"></p-column>
12 <p-column field="state" header="State"></p-column> 11 <p-column field="state" header="State"></p-column>
13 <p-column field="createdAt" header="Created date" [sortable]="true"></p-column> 12 <p-column field="createdAt" header="Created date" [sortable]="true"></p-column>
14 <p-column header="Unfollow" styleClass="action-cell"> 13 <p-column header="Unfollow" styleClass="action-cell">
diff --git a/server/controllers/activitypub/inbox.ts b/server/controllers/activitypub/inbox.ts
index 807d0bdf4..30e7f706b 100644
--- a/server/controllers/activitypub/inbox.ts
+++ b/server/controllers/activitypub/inbox.ts
@@ -1,27 +1,10 @@
1import * as express from 'express' 1import * as express from 'express'
2import { Activity, ActivityPubCollection, ActivityPubOrderedCollection, ActivityType, RootActivity } from '../../../shared' 2import { Activity, ActivityPubCollection, ActivityPubOrderedCollection, RootActivity } from '../../../shared'
3import { logger } from '../../helpers' 3import { logger } from '../../helpers'
4import { isActivityValid } from '../../helpers/custom-validators/activitypub/activity' 4import { isActivityValid } from '../../helpers/custom-validators/activitypub/activity'
5import { processCreateActivity, processUpdateActivity, processUndoActivity } from '../../lib' 5import { processActivities } from '../../lib/activitypub/process/process'
6import { processAcceptActivity } from '../../lib/activitypub/process/process-accept'
7import { processAddActivity } from '../../lib/activitypub/process/process-add'
8import { processAnnounceActivity } from '../../lib/activitypub/process/process-announce'
9import { processDeleteActivity } from '../../lib/activitypub/process/process-delete'
10import { processFollowActivity } from '../../lib/activitypub/process/process-follow'
11import { asyncMiddleware, checkSignature, localAccountValidator, signatureValidator } from '../../middlewares' 6import { asyncMiddleware, checkSignature, localAccountValidator, signatureValidator } from '../../middlewares'
12import { activityPubValidator } from '../../middlewares/validators/activitypub/activity' 7import { activityPubValidator } from '../../middlewares/validators/activitypub/activity'
13import { AccountInstance } from '../../models/account/account-interface'
14
15const processActivity: { [ P in ActivityType ]: (activity: Activity, inboxAccount?: AccountInstance) => Promise<any> } = {
16 Create: processCreateActivity,
17 Add: processAddActivity,
18 Update: processUpdateActivity,
19 Delete: processDeleteActivity,
20 Follow: processFollowActivity,
21 Accept: processAcceptActivity,
22 Announce: processAnnounceActivity,
23 Undo: processUndoActivity
24}
25 8
26const inboxRouter = express.Router() 9const inboxRouter = express.Router()
27 10
@@ -69,15 +52,3 @@ async function inboxController (req: express.Request, res: express.Response, nex
69 52
70 res.status(204).end() 53 res.status(204).end()
71} 54}
72
73async function processActivities (activities: Activity[], inboxAccount?: AccountInstance) {
74 for (const activity of activities) {
75 const activityProcessor = processActivity[activity.type]
76 if (activityProcessor === undefined) {
77 logger.warn('Unknown activity type %s.', activity.type, { activityId: activity.id })
78 continue
79 }
80
81 await activityProcessor(activity, inboxAccount)
82 }
83}
diff --git a/server/controllers/activitypub/outbox.ts b/server/controllers/activitypub/outbox.ts
index 396fa2db5..1a74bde33 100644
--- a/server/controllers/activitypub/outbox.ts
+++ b/server/controllers/activitypub/outbox.ts
@@ -34,8 +34,6 @@ async function outboxController (req: express.Request, res: express.Response, ne
34 const data = await db.Video.listAllAndSharedByAccountForOutbox(account.id, start, count) 34 const data = await db.Video.listAllAndSharedByAccountForOutbox(account.id, start, count)
35 const activities: Activity[] = [] 35 const activities: Activity[] = []
36 36
37 console.log(account.url)
38
39 for (const video of data.data) { 37 for (const video of data.data) {
40 const videoObject = video.toActivityPubObject() 38 const videoObject = video.toActivityPubObject()
41 let addActivity: ActivityAdd = await addActivityData(video.url, account, video, video.VideoChannel.url, videoObject) 39 let addActivity: ActivityAdd = await addActivityData(video.url, account, video, video.VideoChannel.url, videoObject)
diff --git a/server/controllers/api/server/follows.ts b/server/controllers/api/server/follows.ts
index c759824e0..4b54afc8d 100644
--- a/server/controllers/api/server/follows.ts
+++ b/server/controllers/api/server/follows.ts
@@ -19,6 +19,7 @@ import { sendUndoFollow } from '../../../lib/activitypub/send/send-undo'
19import { AccountInstance } from '../../../models/account/account-interface' 19import { AccountInstance } from '../../../models/account/account-interface'
20import { retryTransactionWrapper } from '../../../helpers/database-utils' 20import { retryTransactionWrapper } from '../../../helpers/database-utils'
21import { saveAccountAndServerIfNotExist } from '../../../lib/activitypub/account' 21import { saveAccountAndServerIfNotExist } from '../../../lib/activitypub/account'
22import { addFetchOutboxJob } from '../../../lib/activitypub/fetch'
22 23
23const serverFollowsRouter = express.Router() 24const serverFollowsRouter = express.Router()
24 25
@@ -136,6 +137,8 @@ async function follow (fromAccount: AccountInstance, targetAccount: AccountInsta
136 if (accountFollow.state === 'pending') { 137 if (accountFollow.state === 'pending') {
137 await sendFollow(accountFollow, t) 138 await sendFollow(accountFollow, t)
138 } 139 }
140
141 await addFetchOutboxJob(targetAccount, t)
139 }) 142 })
140 } catch (err) { 143 } catch (err) {
141 // Reset target account 144 // Reset target account
diff --git a/server/helpers/activitypub.ts b/server/helpers/activitypub.ts
index 04d85b8e6..fb4a43a01 100644
--- a/server/helpers/activitypub.ts
+++ b/server/helpers/activitypub.ts
@@ -46,14 +46,16 @@ function activityPubCollectionPagination (url: string, page: number, result: Res
46 orderedItems: result.data 46 orderedItems: result.data
47 } 47 }
48 48
49 const obj = { 49 if (page === 1) {
50 id: url, 50 return activityPubContextify({
51 type: 'OrderedCollection', 51 id: url,
52 totalItems: result.total, 52 type: 'OrderedCollection',
53 orderedItems: orderedCollectionPagination 53 totalItems: result.total,
54 first: orderedCollectionPagination
55 })
54 } 56 }
55 57
56 return activityPubContextify(obj) 58 return orderedCollectionPagination
57} 59}
58 60
59function buildSignedActivity (byAccount: AccountInstance, data: Object) { 61function buildSignedActivity (byAccount: AccountInstance, data: Object) {
diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts
index 7c0640cc0..398691eba 100644
--- a/server/initializers/constants.ts
+++ b/server/initializers/constants.ts
@@ -228,6 +228,7 @@ const ACTIVITY_PUB = {
228 ACCEPT_HEADER: 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"', 228 ACCEPT_HEADER: 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"',
229 PUBLIC: 'https://www.w3.org/ns/activitystreams#Public', 229 PUBLIC: 'https://www.w3.org/ns/activitystreams#Public',
230 COLLECTION_ITEMS_PER_PAGE: 10, 230 COLLECTION_ITEMS_PER_PAGE: 10,
231 FETCH_PAGE_LIMIT: 100,
231 URL_MIME_TYPES: { 232 URL_MIME_TYPES: {
232 VIDEO: [ 'video/mp4', 'video/webm', 'video/ogg' ], // TODO: Merge with VIDEO_MIMETYPE_EXT 233 VIDEO: [ 'video/mp4', 'video/webm', 'video/ogg' ], // TODO: Merge with VIDEO_MIMETYPE_EXT
233 TORRENT: [ 'application/x-bittorrent' ], 234 TORRENT: [ 'application/x-bittorrent' ],
diff --git a/server/lib/activitypub/fetch.ts b/server/lib/activitypub/fetch.ts
new file mode 100644
index 000000000..fd2af0761
--- /dev/null
+++ b/server/lib/activitypub/fetch.ts
@@ -0,0 +1,15 @@
1import { Transaction } from 'sequelize'
2import { AccountInstance } from '../../models/account/account-interface'
3import { activitypubHttpJobScheduler, ActivityPubHttpPayload } from '../jobs/activitypub-http-job-scheduler/activitypub-http-job-scheduler'
4
5async function addFetchOutboxJob (account: AccountInstance, t: Transaction) {
6 const jobPayload: ActivityPubHttpPayload = {
7 uris: [ account.outboxUrl ]
8 }
9
10 return activitypubHttpJobScheduler.createJob(t, 'activitypubHttpFetcherHandler', jobPayload)
11}
12
13export {
14 addFetchOutboxJob
15}
diff --git a/server/lib/activitypub/index.ts b/server/lib/activitypub/index.ts
index b1daa70bb..fcea662a6 100644
--- a/server/lib/activitypub/index.ts
+++ b/server/lib/activitypub/index.ts
@@ -1,6 +1,7 @@
1export * from './process' 1export * from './process'
2export * from './send' 2export * from './send'
3export * from './account' 3export * from './account'
4export * from './fetch'
4export * from './share' 5export * from './share'
5export * from './video-channels' 6export * from './video-channels'
6export * from './videos' 7export * from './videos'
diff --git a/server/lib/activitypub/process/index.ts b/server/lib/activitypub/process/index.ts
index e80b46b6f..c68312053 100644
--- a/server/lib/activitypub/process/index.ts
+++ b/server/lib/activitypub/process/index.ts
@@ -1,3 +1,4 @@
1export * from './process'
1export * from './process-accept' 2export * from './process-accept'
2export * from './process-add' 3export * from './process-add'
3export * from './process-announce' 4export * from './process-announce'
diff --git a/server/lib/activitypub/process/process.ts b/server/lib/activitypub/process/process.ts
new file mode 100644
index 000000000..613597341
--- /dev/null
+++ b/server/lib/activitypub/process/process.ts
@@ -0,0 +1,38 @@
1import { Activity, ActivityType } from '../../../../shared/models/activitypub/activity'
2import { AccountInstance } from '../../../models/account/account-interface'
3import { processAcceptActivity } from './process-accept'
4import { processAddActivity } from './process-add'
5import { processAnnounceActivity } from './process-announce'
6import { processCreateActivity } from './process-create'
7import { processDeleteActivity } from './process-delete'
8import { processFollowActivity } from './process-follow'
9import { processUndoActivity } from './process-undo'
10import { processUpdateActivity } from './process-update'
11import { logger } from '../../../helpers/logger'
12
13const processActivity: { [ P in ActivityType ]: (activity: Activity, inboxAccount?: AccountInstance) => Promise<any> } = {
14 Create: processCreateActivity,
15 Add: processAddActivity,
16 Update: processUpdateActivity,
17 Delete: processDeleteActivity,
18 Follow: processFollowActivity,
19 Accept: processAcceptActivity,
20 Announce: processAnnounceActivity,
21 Undo: processUndoActivity
22}
23
24async function processActivities (activities: Activity[], inboxAccount?: AccountInstance) {
25 for (const activity of activities) {
26 const activityProcessor = processActivity[activity.type]
27 if (activityProcessor === undefined) {
28 logger.warn('Unknown activity type %s.', activity.type, { activityId: activity.id })
29 continue
30 }
31
32 await activityProcessor(activity, inboxAccount)
33 }
34}
35
36export {
37 processActivities
38}
diff --git a/server/lib/activitypub/send/send-create.ts b/server/lib/activitypub/send/send-create.ts
index 14b666fc9..df8e0a642 100644
--- a/server/lib/activitypub/send/send-create.ts
+++ b/server/lib/activitypub/send/send-create.ts
@@ -21,6 +21,8 @@ async function sendVideoAbuse (byAccount: AccountInstance, videoAbuse: VideoAbus
21 return unicastTo(data, byAccount, video.VideoChannel.Account.sharedInboxUrl, t) 21 return unicastTo(data, byAccount, video.VideoChannel.Account.sharedInboxUrl, t)
22} 22}
23 23
24// async function sendCreateView ()
25
24async function createActivityData (url: string, byAccount: AccountInstance, object: any) { 26async function createActivityData (url: string, byAccount: AccountInstance, object: any) {
25 const { to, cc } = await getAudience(byAccount) 27 const { to, cc } = await getAudience(byAccount)
26 const activity: ActivityCreate = { 28 const activity: ActivityCreate = {
diff --git a/server/lib/jobs/activitypub-http-job-scheduler/activitypub-http-fetcher-handler.ts b/server/lib/jobs/activitypub-http-job-scheduler/activitypub-http-fetcher-handler.ts
new file mode 100644
index 000000000..b8ead32a4
--- /dev/null
+++ b/server/lib/jobs/activitypub-http-job-scheduler/activitypub-http-fetcher-handler.ts
@@ -0,0 +1,69 @@
1import { logger } from '../../../helpers'
2import { buildSignedActivity } from '../../../helpers/activitypub'
3import { doRequest } from '../../../helpers/requests'
4import { database as db } from '../../../initializers'
5import { ActivityPubHttpPayload } from './activitypub-http-job-scheduler'
6import { processActivities } from '../../activitypub/process/process'
7import { ACTIVITY_PUB } from '../../../initializers/constants'
8
9async function process (payload: ActivityPubHttpPayload, jobId: number) {
10 logger.info('Processing ActivityPub fetcher in job %d.', jobId)
11
12 const options = {
13 method: 'GET',
14 uri: '',
15 json: true
16 }
17
18 for (const uri of payload.uris) {
19 options.uri = uri
20 logger.info('Fetching ActivityPub data on %s.', uri)
21
22 const response = await doRequest(options)
23 const firstBody = response.body
24
25 if (firstBody.first && Array.isArray(firstBody.first.orderedItems)) {
26 const activities = firstBody.first.orderedItems
27
28 logger.info('Processing %i items ActivityPub fetcher for %s.', activities.length, uri)
29
30 await processActivities(activities)
31 }
32
33 let limit = ACTIVITY_PUB.FETCH_PAGE_LIMIT
34 let i = 0
35 let nextLink = firstBody.first.next
36 while (nextLink && i < limit) {
37 options.uri = nextLink
38
39 const { body } = await doRequest(options)
40 nextLink = body.nextLink
41 i++
42
43 if (Array.isArray(body.orderedItems)) {
44 const activities = body.orderedItems
45 logger.info('Processing %i items ActivityPub fetcher for %s.', activities.length, uri)
46
47 await processActivities(activities)
48 }
49 }
50 }
51}
52
53function onError (err: Error, jobId: number) {
54 logger.error('Error when broadcasting ActivityPub request in job %d.', jobId, err)
55 return Promise.resolve()
56}
57
58function onSuccess (jobId: number) {
59 logger.info('Job %d is a success.', jobId)
60 return Promise.resolve()
61}
62
63// ---------------------------------------------------------------------------
64
65export {
66 process,
67 onError,
68 onSuccess
69}
diff --git a/server/lib/jobs/activitypub-http-job-scheduler/activitypub-http-job-scheduler.ts b/server/lib/jobs/activitypub-http-job-scheduler/activitypub-http-job-scheduler.ts
index e4f6c94a5..aef217ce7 100644
--- a/server/lib/jobs/activitypub-http-job-scheduler/activitypub-http-job-scheduler.ts
+++ b/server/lib/jobs/activitypub-http-job-scheduler/activitypub-http-job-scheduler.ts
@@ -2,16 +2,18 @@ import { JobScheduler, JobHandler } from '../job-scheduler'
2 2
3import * as activitypubHttpBroadcastHandler from './activitypub-http-broadcast-handler' 3import * as activitypubHttpBroadcastHandler from './activitypub-http-broadcast-handler'
4import * as activitypubHttpUnicastHandler from './activitypub-http-unicast-handler' 4import * as activitypubHttpUnicastHandler from './activitypub-http-unicast-handler'
5import * as activitypubHttpFetcherHandler from './activitypub-http-fetcher-handler'
5import { JobCategory } from '../../../../shared' 6import { JobCategory } from '../../../../shared'
6 7
7type ActivityPubHttpPayload = { 8type ActivityPubHttpPayload = {
8 uris: string[] 9 uris: string[]
9 signatureAccountId: number 10 signatureAccountId?: number
10 body: any 11 body?: any
11} 12}
12const jobHandlers: { [ handlerName: string ]: JobHandler<ActivityPubHttpPayload, void> } = { 13const jobHandlers: { [ handlerName: string ]: JobHandler<ActivityPubHttpPayload, void> } = {
13 activitypubHttpBroadcastHandler, 14 activitypubHttpBroadcastHandler,
14 activitypubHttpUnicastHandler 15 activitypubHttpUnicastHandler,
16 activitypubHttpFetcherHandler
15} 17}
16const jobCategory: JobCategory = 'activitypub-http' 18const jobCategory: JobCategory = 'activitypub-http'
17 19