aboutsummaryrefslogtreecommitdiffhomepage
diff options
context:
space:
mode:
authorChocobozzz <florian.bigard@gmail.com>2017-11-22 11:27:40 +0100
committerChocobozzz <florian.bigard@gmail.com>2017-11-27 19:40:53 +0100
commitc46edbc2f6ca310b2f0331f979ac6caf27f6eb92 (patch)
tree073e32adb1bf93a597a269432e33a8f28365deb5
parentc986175d68a18e96fbd41537a05c7796a2c64f38 (diff)
downloadPeerTube-c46edbc2f6ca310b2f0331f979ac6caf27f6eb92.tar.gz
PeerTube-c46edbc2f6ca310b2f0331f979ac6caf27f6eb92.tar.zst
PeerTube-c46edbc2f6ca310b2f0331f979ac6caf27f6eb92.zip
Fetch outbox to grab old activities tests
-rwxr-xr-xscripts/parse-log.ts4
-rw-r--r--server/controllers/activitypub/client.ts4
-rw-r--r--server/controllers/activitypub/outbox.ts2
-rw-r--r--server/controllers/api/server/follows.ts13
-rw-r--r--server/helpers/activitypub.ts9
-rw-r--r--server/initializers/constants.ts1
-rw-r--r--server/lib/activitypub/process/process-accept.ts2
-rw-r--r--server/lib/activitypub/process/process-add.ts2
-rw-r--r--server/lib/jobs/activitypub-http-job-scheduler/activitypub-http-fetcher-handler.ts6
-rw-r--r--server/models/video/video.ts38
-rw-r--r--server/tests/api/follows.ts30
11 files changed, 81 insertions, 30 deletions
diff --git a/scripts/parse-log.ts b/scripts/parse-log.ts
index 24a09c885..e2c42bf4c 100755
--- a/scripts/parse-log.ts
+++ b/scripts/parse-log.ts
@@ -2,7 +2,6 @@ import { createReadStream } from 'fs'
2import { join } from 'path' 2import { join } from 'path'
3import { createInterface } from 'readline' 3import { createInterface } from 'readline'
4import * as winston from 'winston' 4import * as winston from 'winston'
5import { readFileBufferPromise } from '../server/helpers/core-utils'
6import { CONFIG } from '../server/initializers/constants' 5import { CONFIG } from '../server/initializers/constants'
7 6
8const label = CONFIG.WEBSERVER.HOSTNAME + ':' + CONFIG.WEBSERVER.PORT 7const label = CONFIG.WEBSERVER.HOSTNAME + ':' + CONFIG.WEBSERVER.PORT
@@ -16,7 +15,8 @@ const logger = new winston.Logger({
16 humanReadableUnhandledException: true, 15 humanReadableUnhandledException: true,
17 json: false, 16 json: false,
18 colorize: true, 17 colorize: true,
19 prettyPrint: true 18 prettyPrint: true,
19 stderrLevels: []
20 }) 20 })
21 ], 21 ],
22 exitOnError: true 22 exitOnError: true
diff --git a/server/controllers/activitypub/client.ts b/server/controllers/activitypub/client.ts
index 24c8665a5..eee89e2fd 100644
--- a/server/controllers/activitypub/client.ts
+++ b/server/controllers/activitypub/client.ts
@@ -56,7 +56,7 @@ async function accountController (req: express.Request, res: express.Response, n
56async function accountFollowersController (req: express.Request, res: express.Response, next: express.NextFunction) { 56async function accountFollowersController (req: express.Request, res: express.Response, next: express.NextFunction) {
57 const account: AccountInstance = res.locals.account 57 const account: AccountInstance = res.locals.account
58 58
59 const page = req.params.page || 1 59 const page = req.query.page || 1
60 const { start, count } = pageToStartAndCount(page, ACTIVITY_PUB.COLLECTION_ITEMS_PER_PAGE) 60 const { start, count } = pageToStartAndCount(page, ACTIVITY_PUB.COLLECTION_ITEMS_PER_PAGE)
61 61
62 const result = await db.AccountFollow.listAcceptedFollowerUrlsForApi([ account.id ], start, count) 62 const result = await db.AccountFollow.listAcceptedFollowerUrlsForApi([ account.id ], start, count)
@@ -68,7 +68,7 @@ async function accountFollowersController (req: express.Request, res: express.Re
68async function accountFollowingController (req: express.Request, res: express.Response, next: express.NextFunction) { 68async function accountFollowingController (req: express.Request, res: express.Response, next: express.NextFunction) {
69 const account: AccountInstance = res.locals.account 69 const account: AccountInstance = res.locals.account
70 70
71 const page = req.params.page || 1 71 const page = req.query.page || 1
72 const { start, count } = pageToStartAndCount(page, ACTIVITY_PUB.COLLECTION_ITEMS_PER_PAGE) 72 const { start, count } = pageToStartAndCount(page, ACTIVITY_PUB.COLLECTION_ITEMS_PER_PAGE)
73 73
74 const result = await db.AccountFollow.listAcceptedFollowingUrlsForApi([ account.id ], start, count) 74 const result = await db.AccountFollow.listAcceptedFollowingUrlsForApi([ account.id ], start, count)
diff --git a/server/controllers/activitypub/outbox.ts b/server/controllers/activitypub/outbox.ts
index 1a74bde33..74d399763 100644
--- a/server/controllers/activitypub/outbox.ts
+++ b/server/controllers/activitypub/outbox.ts
@@ -28,7 +28,7 @@ export {
28async function outboxController (req: express.Request, res: express.Response, next: express.NextFunction) { 28async function outboxController (req: express.Request, res: express.Response, next: express.NextFunction) {
29 const account: AccountInstance = res.locals.account 29 const account: AccountInstance = res.locals.account
30 30
31 const page = req.params.page || 1 31 const page = req.query.page || 1
32 const { start, count } = pageToStartAndCount(page, ACTIVITY_PUB.COLLECTION_ITEMS_PER_PAGE) 32 const { start, count } = pageToStartAndCount(page, ACTIVITY_PUB.COLLECTION_ITEMS_PER_PAGE)
33 33
34 const data = await db.Video.listAllAndSharedByAccountForOutbox(account.id, start, count) 34 const data = await db.Video.listAllAndSharedByAccountForOutbox(account.id, start, count)
diff --git a/server/controllers/api/server/follows.ts b/server/controllers/api/server/follows.ts
index 4b54afc8d..391f8bdca 100644
--- a/server/controllers/api/server/follows.ts
+++ b/server/controllers/api/server/follows.ts
@@ -1,11 +1,15 @@
1import * as express from 'express' 1import * as express from 'express'
2import { UserRight } from '../../../../shared/models/users/user-right.enum' 2import { UserRight } from '../../../../shared/models/users/user-right.enum'
3import { getFormattedObjects } from '../../../helpers' 3import { getFormattedObjects } from '../../../helpers'
4import { retryTransactionWrapper } from '../../../helpers/database-utils'
4import { logger } from '../../../helpers/logger' 5import { logger } from '../../../helpers/logger'
5import { getServerAccount } from '../../../helpers/utils' 6import { getServerAccount } from '../../../helpers/utils'
6import { getAccountFromWebfinger } from '../../../helpers/webfinger' 7import { getAccountFromWebfinger } from '../../../helpers/webfinger'
7import { SERVER_ACCOUNT_NAME } from '../../../initializers/constants' 8import { SERVER_ACCOUNT_NAME } from '../../../initializers/constants'
8import { database as db } from '../../../initializers/database' 9import { database as db } from '../../../initializers/database'
10import { saveAccountAndServerIfNotExist } from '../../../lib/activitypub/account'
11import { sendUndoFollow } from '../../../lib/activitypub/send/send-undo'
12import { sendFollow } from '../../../lib/index'
9import { asyncMiddleware, paginationValidator, removeFollowingValidator, setFollowersSort, setPagination } from '../../../middlewares' 13import { asyncMiddleware, paginationValidator, removeFollowingValidator, setFollowersSort, setPagination } from '../../../middlewares'
10import { authenticate } from '../../../middlewares/oauth' 14import { authenticate } from '../../../middlewares/oauth'
11import { setBodyHostsPort } from '../../../middlewares/servers' 15import { setBodyHostsPort } from '../../../middlewares/servers'
@@ -13,13 +17,8 @@ import { setFollowingSort } from '../../../middlewares/sort'
13import { ensureUserHasRight } from '../../../middlewares/user-right' 17import { ensureUserHasRight } from '../../../middlewares/user-right'
14import { followValidator } from '../../../middlewares/validators/follows' 18import { followValidator } from '../../../middlewares/validators/follows'
15import { followersSortValidator, followingSortValidator } from '../../../middlewares/validators/sort' 19import { followersSortValidator, followingSortValidator } from '../../../middlewares/validators/sort'
16import { AccountFollowInstance } from '../../../models/index'
17import { sendFollow } from '../../../lib/index'
18import { sendUndoFollow } from '../../../lib/activitypub/send/send-undo'
19import { AccountInstance } from '../../../models/account/account-interface' 20import { AccountInstance } from '../../../models/account/account-interface'
20import { retryTransactionWrapper } from '../../../helpers/database-utils' 21import { AccountFollowInstance } from '../../../models/index'
21import { saveAccountAndServerIfNotExist } from '../../../lib/activitypub/account'
22import { addFetchOutboxJob } from '../../../lib/activitypub/fetch'
23 22
24const serverFollowsRouter = express.Router() 23const serverFollowsRouter = express.Router()
25 24
@@ -137,8 +136,6 @@ async function follow (fromAccount: AccountInstance, targetAccount: AccountInsta
137 if (accountFollow.state === 'pending') { 136 if (accountFollow.state === 'pending') {
138 await sendFollow(accountFollow, t) 137 await sendFollow(accountFollow, t)
139 } 138 }
140
141 await addFetchOutboxJob(targetAccount, t)
142 }) 139 })
143 } catch (err) { 140 } catch (err) {
144 // Reset target account 141 // Reset target account
diff --git a/server/helpers/activitypub.ts b/server/helpers/activitypub.ts
index fb4a43a01..54c460200 100644
--- a/server/helpers/activitypub.ts
+++ b/server/helpers/activitypub.ts
@@ -24,12 +24,15 @@ function activityPubContextify <T> (data: T) {
24 }) 24 })
25} 25}
26 26
27function activityPubCollectionPagination (url: string, page: number, result: ResultList<any>) { 27function activityPubCollectionPagination (url: string, page: any, result: ResultList<any>) {
28 let next: string 28 let next: string
29 let prev: string 29 let prev: string
30 30
31 // Assert page is a number
32 page = parseInt(page, 10)
33
31 // There are more results 34 // There are more results
32 if (result.total > ((page + 1) * ACTIVITY_PUB.COLLECTION_ITEMS_PER_PAGE)) { 35 if (result.total > page * ACTIVITY_PUB.COLLECTION_ITEMS_PER_PAGE) {
33 next = url + '?page=' + (page + 1) 36 next = url + '?page=' + (page + 1)
34 } 37 }
35 38
@@ -53,6 +56,8 @@ function activityPubCollectionPagination (url: string, page: number, result: Res
53 totalItems: result.total, 56 totalItems: result.total,
54 first: orderedCollectionPagination 57 first: orderedCollectionPagination
55 }) 58 })
59 } else {
60 orderedCollectionPagination['totalItems'] = result.total
56 } 61 }
57 62
58 return orderedCollectionPagination 63 return orderedCollectionPagination
diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts
index 398691eba..9e61f01aa 100644
--- a/server/initializers/constants.ts
+++ b/server/initializers/constants.ts
@@ -328,6 +328,7 @@ if (isTestInstance() === true) {
328 REMOTE_SCHEME.HTTP = 'http' 328 REMOTE_SCHEME.HTTP = 'http'
329 REMOTE_SCHEME.WS = 'ws' 329 REMOTE_SCHEME.WS = 'ws'
330 STATIC_MAX_AGE = '0' 330 STATIC_MAX_AGE = '0'
331 ACTIVITY_PUB.COLLECTION_ITEMS_PER_PAGE = 2
331} 332}
332 333
333// --------------------------------------------------------------------------- 334// ---------------------------------------------------------------------------
diff --git a/server/lib/activitypub/process/process-accept.ts b/server/lib/activitypub/process/process-accept.ts
index e159c41b5..73c6cb279 100644
--- a/server/lib/activitypub/process/process-accept.ts
+++ b/server/lib/activitypub/process/process-accept.ts
@@ -1,6 +1,7 @@
1import { ActivityAccept } from '../../../../shared/models/activitypub/activity' 1import { ActivityAccept } from '../../../../shared/models/activitypub/activity'
2import { database as db } from '../../../initializers' 2import { database as db } from '../../../initializers'
3import { AccountInstance } from '../../../models/account/account-interface' 3import { AccountInstance } from '../../../models/account/account-interface'
4import { addFetchOutboxJob } from '../fetch'
4 5
5async function processAcceptActivity (activity: ActivityAccept, inboxAccount?: AccountInstance) { 6async function processAcceptActivity (activity: ActivityAccept, inboxAccount?: AccountInstance) {
6 if (inboxAccount === undefined) throw new Error('Need to accept on explicit inbox.') 7 if (inboxAccount === undefined) throw new Error('Need to accept on explicit inbox.')
@@ -24,4 +25,5 @@ async function processAccept (account: AccountInstance, targetAccount: AccountIn
24 25
25 follow.set('state', 'accepted') 26 follow.set('state', 'accepted')
26 await follow.save() 27 await follow.save()
28 await addFetchOutboxJob(targetAccount, undefined)
27} 29}
diff --git a/server/lib/activitypub/process/process-add.ts b/server/lib/activitypub/process/process-add.ts
index edc90dee5..332c18cc0 100644
--- a/server/lib/activitypub/process/process-add.ts
+++ b/server/lib/activitypub/process/process-add.ts
@@ -48,7 +48,7 @@ function addRemoteVideo (account: AccountInstance,
48 activity: ActivityAdd, 48 activity: ActivityAdd,
49 videoChannel: VideoChannelInstance, 49 videoChannel: VideoChannelInstance,
50 videoToCreateData: VideoTorrentObject) { 50 videoToCreateData: VideoTorrentObject) {
51 logger.debug('Adding remote video %s.', videoToCreateData.url) 51 logger.debug('Adding remote video %s.', videoToCreateData.id)
52 52
53 return db.sequelize.transaction(async t => { 53 return db.sequelize.transaction(async t => {
54 const sequelizeOptions = { 54 const sequelizeOptions = {
diff --git a/server/lib/jobs/activitypub-http-job-scheduler/activitypub-http-fetcher-handler.ts b/server/lib/jobs/activitypub-http-job-scheduler/activitypub-http-fetcher-handler.ts
index b8ead32a4..09efaa622 100644
--- a/server/lib/jobs/activitypub-http-job-scheduler/activitypub-http-fetcher-handler.ts
+++ b/server/lib/jobs/activitypub-http-job-scheduler/activitypub-http-fetcher-handler.ts
@@ -25,7 +25,7 @@ async function process (payload: ActivityPubHttpPayload, jobId: number) {
25 if (firstBody.first && Array.isArray(firstBody.first.orderedItems)) { 25 if (firstBody.first && Array.isArray(firstBody.first.orderedItems)) {
26 const activities = firstBody.first.orderedItems 26 const activities = firstBody.first.orderedItems
27 27
28 logger.info('Processing %i items ActivityPub fetcher for %s.', activities.length, uri) 28 logger.info('Processing %i items ActivityPub fetcher for %s.', activities.length, options.uri)
29 29
30 await processActivities(activities) 30 await processActivities(activities)
31 } 31 }
@@ -37,12 +37,12 @@ async function process (payload: ActivityPubHttpPayload, jobId: number) {
37 options.uri = nextLink 37 options.uri = nextLink
38 38
39 const { body } = await doRequest(options) 39 const { body } = await doRequest(options)
40 nextLink = body.nextLink 40 nextLink = body.next
41 i++ 41 i++
42 42
43 if (Array.isArray(body.orderedItems)) { 43 if (Array.isArray(body.orderedItems)) {
44 const activities = body.orderedItems 44 const activities = body.orderedItems
45 logger.info('Processing %i items ActivityPub fetcher for %s.', activities.length, uri) 45 logger.info('Processing %i items ActivityPub fetcher for %s.', activities.length, options.uri)
46 46
47 await processActivities(activities) 47 await processActivities(activities)
48 } 48 }
diff --git a/server/models/video/video.ts b/server/models/video/video.ts
index 3b7e83779..9b411a92e 100644
--- a/server/models/video/video.ts
+++ b/server/models/video/video.ts
@@ -46,6 +46,7 @@ import { TagInstance } from './tag-interface'
46import { VideoFileInstance, VideoFileModel } from './video-file-interface' 46import { VideoFileInstance, VideoFileModel } from './video-file-interface'
47import { VideoAttributes, VideoInstance, VideoMethods } from './video-interface' 47import { VideoAttributes, VideoInstance, VideoMethods } from './video-interface'
48import { sendDeleteVideo } from '../../lib/index' 48import { sendDeleteVideo } from '../../lib/index'
49import * as Bluebird from 'bluebird'
49 50
50const Buffer = safeBuffer.Buffer 51const Buffer = safeBuffer.Buffer
51 52
@@ -786,14 +787,21 @@ list = function () {
786} 787}
787 788
788listAllAndSharedByAccountForOutbox = function (accountId: number, start: number, count: number) { 789listAllAndSharedByAccountForOutbox = function (accountId: number, start: number, count: number) {
789 const queryVideo = 'SELECT "Video"."id" FROM "Videos" AS "Video" ' + 790 function getRawQuery (select: string) {
790 'INNER JOIN "VideoChannels" AS "VideoChannel" ON "VideoChannel"."id" = "Video"."channelId" ' + 791 const queryVideo = 'SELECT ' + select + ' FROM "Videos" AS "Video" ' +
791 'WHERE "VideoChannel"."accountId" = ' + accountId 792 'INNER JOIN "VideoChannels" AS "VideoChannel" ON "VideoChannel"."id" = "Video"."channelId" ' +
792 const queryVideoShare = 'SELECT "Video"."id" FROM "VideoShares" AS "VideoShare" ' + 793 'WHERE "VideoChannel"."accountId" = ' + accountId
793 'INNER JOIN "Videos" AS "Video" ON "Video"."id" = "VideoShare"."videoId" ' + 794 const queryVideoShare = 'SELECT ' + select + ' FROM "VideoShares" AS "VideoShare" ' +
794 'INNER JOIN "VideoChannels" AS "VideoChannel" ON "VideoChannel"."id" = "Video"."channelId" ' + 795 'INNER JOIN "Videos" AS "Video" ON "Video"."id" = "VideoShare"."videoId" ' +
795 'WHERE "VideoShare"."accountId" = ' + accountId 796 'WHERE "VideoShare"."accountId" = ' + accountId
796 const rawQuery = `(${queryVideo}) UNION (${queryVideoShare}) LIMIT ${count} OFFSET ${start}` 797
798 let rawQuery = `(${queryVideo}) UNION (${queryVideoShare})`
799
800 return rawQuery
801 }
802
803 const rawQuery = getRawQuery('"Video"."id"')
804 const rawCountQuery = getRawQuery('COUNT("Video"."id") as "total"')
797 805
798 const query = { 806 const query = {
799 distinct: true, 807 distinct: true,
@@ -825,10 +833,20 @@ listAllAndSharedByAccountForOutbox = function (accountId: number, start: number,
825 ] 833 ]
826 } 834 }
827 835
828 return Video.findAndCountAll(query).then(({ rows, count }) => { 836 return Bluebird.all([
837 Video.findAll(query),
838 Video['sequelize'].query(rawCountQuery, { type: Sequelize.QueryTypes.SELECT })
839 ]).then(([ rows, totals ]) => {
840 // totals: totalVideos + totalVideoShares
841 let totalVideos = 0
842 let totalVideoShares = 0
843 if (totals[0]) totalVideos = parseInt(totals[0].total, 10)
844 if (totals[1]) totalVideoShares = parseInt(totals[1].total, 10)
845
846 const total = totalVideos + totalVideoShares
829 return { 847 return {
830 data: rows, 848 data: rows,
831 total: count 849 total: total
832 } 850 }
833 }) 851 })
834} 852}
diff --git a/server/tests/api/follows.ts b/server/tests/api/follows.ts
index b2f53d3a7..875d814a7 100644
--- a/server/tests/api/follows.ts
+++ b/server/tests/api/follows.ts
@@ -22,7 +22,7 @@ describe('Test follows', function () {
22 let server3Id: number 22 let server3Id: number
23 23
24 before(async function () { 24 before(async function () {
25 this.timeout(120000) 25 this.timeout(20000)
26 26
27 servers = await flushAndRunMultipleServers(3) 27 servers = await flushAndRunMultipleServers(3)
28 28
@@ -163,6 +163,34 @@ describe('Test follows', function () {
163 expect(res.body.data[0].name).to.equal('server3') 163 expect(res.body.data[0].name).to.equal('server3')
164 }) 164 })
165 165
166 it('Should propagate previous uploaded videos on a new following', async function () {
167 this.timeout(20000)
168
169 await uploadVideo(servers[2].url, servers[2].accessToken, { name: 'server3-2' })
170 await uploadVideo(servers[2].url, servers[2].accessToken, { name: 'server3-3' })
171 await uploadVideo(servers[2].url, servers[2].accessToken, { name: 'server3-4' })
172 await uploadVideo(servers[2].url, servers[2].accessToken, { name: 'server3-5' })
173 await uploadVideo(servers[2].url, servers[2].accessToken, { name: 'server3-6' })
174
175 await wait(5000)
176
177 // Server 1 follows server 3
178 await follow(servers[0].url, [ servers[2].url ], servers[0].accessToken)
179
180 await wait(7000)
181
182 let res = await getVideosList(servers[0].url)
183 expect(res.body.total).to.equal(7)
184
185 const video2 = res.body.data.find(v => v.name === 'server3-2')
186 const video4 = res.body.data.find(v => v.name === 'server3-4')
187 const video6 = res.body.data.find(v => v.name === 'server3-6')
188
189 expect(video2).to.not.be.undefined
190 expect(video4).to.not.be.undefined
191 expect(video6).to.not.be.undefined
192 })
193
166 after(async function () { 194 after(async function () {
167 killallServers(servers) 195 killallServers(servers)
168 196