aboutsummaryrefslogtreecommitdiffhomepage
diff options
context:
space:
mode:
authorChocobozzz <me@florianbigard.com>2018-05-25 16:21:16 +0200
committerChocobozzz <me@florianbigard.com>2018-05-25 16:21:16 +0200
commit8fffe21a7bc96d08b229293d66ddba576e609790 (patch)
tree5ebd5f5198a59084c5338ce197d7e836b39200a4
parente251f170b00b2014ac4e823113c6ff40e3fb1471 (diff)
downloadPeerTube-8fffe21a7bc96d08b229293d66ddba576e609790.tar.gz
PeerTube-8fffe21a7bc96d08b229293d66ddba576e609790.tar.zst
PeerTube-8fffe21a7bc96d08b229293d66ddba576e609790.zip
Refractor and optimize AP collections
Only display urls in general object, and paginate video comments, shares, likes and dislikes
-rw-r--r--server/controllers/activitypub/client.ts86
-rw-r--r--server/controllers/activitypub/outbox.ts24
-rw-r--r--server/controllers/activitypub/utils.ts11
-rw-r--r--server/helpers/activitypub.ts44
-rw-r--r--server/lib/activitypub/crawl.ts40
-rw-r--r--server/lib/activitypub/video-comments.ts2
-rw-r--r--server/lib/activitypub/videos.ts27
-rw-r--r--server/lib/job-queue/handlers/activitypub-http-fetcher.ts44
-rw-r--r--server/models/account/account-video-rate.ts29
-rw-r--r--server/models/activitypub/actor-follow.ts3
-rw-r--r--server/models/video/video-comment.ts14
-rw-r--r--server/models/video/video-share.ts13
-rw-r--r--server/models/video/video.ts128
-rw-r--r--shared/models/activitypub/objects/video-torrent-object.ts8
14 files changed, 214 insertions, 259 deletions
diff --git a/server/controllers/activitypub/client.ts b/server/controllers/activitypub/client.ts
index 767fde5d9..1c780783c 100644
--- a/server/controllers/activitypub/client.ts
+++ b/server/controllers/activitypub/client.ts
@@ -1,9 +1,8 @@
1// Intercept ActivityPub client requests 1// Intercept ActivityPub client requests
2import * as express from 'express' 2import * as express from 'express'
3import { VideoPrivacy } from '../../../shared/models/videos' 3import { VideoPrivacy, VideoRateType } from '../../../shared/models/videos'
4import { activityPubCollectionPagination, activityPubContextify } from '../../helpers/activitypub' 4import { activityPubCollectionPagination, activityPubContextify } from '../../helpers/activitypub'
5import { pageToStartAndCount } from '../../helpers/core-utils' 5import { CONFIG, ROUTE_CACHE_LIFETIME } from '../../initializers'
6import { ACTIVITY_PUB, CONFIG, ROUTE_CACHE_LIFETIME } from '../../initializers'
7import { buildVideoAnnounce } from '../../lib/activitypub/send' 6import { buildVideoAnnounce } from '../../lib/activitypub/send'
8import { audiencify, getAudience } from '../../lib/activitypub/audience' 7import { audiencify, getAudience } from '../../lib/activitypub/audience'
9import { createActivityData } from '../../lib/activitypub/send/send-create' 8import { createActivityData } from '../../lib/activitypub/send/send-create'
@@ -18,6 +17,14 @@ import { VideoChannelModel } from '../../models/video/video-channel'
18import { VideoCommentModel } from '../../models/video/video-comment' 17import { VideoCommentModel } from '../../models/video/video-comment'
19import { VideoShareModel } from '../../models/video/video-share' 18import { VideoShareModel } from '../../models/video/video-share'
20import { cacheRoute } from '../../middlewares/cache' 19import { cacheRoute } from '../../middlewares/cache'
20import { activityPubResponse } from './utils'
21import { AccountVideoRateModel } from '../../models/account/account-video-rate'
22import {
23 getVideoCommentsActivityPubUrl,
24 getVideoDislikesActivityPubUrl,
25 getVideoLikesActivityPubUrl,
26 getVideoSharesActivityPubUrl
27} from '../../lib/activitypub'
21 28
22const activityPubClientRouter = express.Router() 29const activityPubClientRouter = express.Router()
23 30
@@ -116,10 +123,8 @@ async function accountFollowingController (req: express.Request, res: express.Re
116async function videoController (req: express.Request, res: express.Response, next: express.NextFunction) { 123async function videoController (req: express.Request, res: express.Response, next: express.NextFunction) {
117 const video: VideoModel = res.locals.video 124 const video: VideoModel = res.locals.video
118 125
119 // We need more attributes
120 const videoAll = await VideoModel.loadAndPopulateAll(video.id)
121 const audience = await getAudience(video.VideoChannel.Account.Actor, undefined, video.privacy === VideoPrivacy.PUBLIC) 126 const audience = await getAudience(video.VideoChannel.Account.Actor, undefined, video.privacy === VideoPrivacy.PUBLIC)
122 const videoObject = audiencify(videoAll.toActivityPubObject(), audience) 127 const videoObject = audiencify(video.toActivityPubObject(), audience)
123 128
124 if (req.path.endsWith('/activity')) { 129 if (req.path.endsWith('/activity')) {
125 const data = await createActivityData(video.url, video.VideoChannel.Account.Actor, videoObject, undefined, audience) 130 const data = await createActivityData(video.url, video.VideoChannel.Account.Actor, videoObject, undefined, audience)
@@ -139,41 +144,45 @@ async function videoAnnounceController (req: express.Request, res: express.Respo
139async function videoAnnouncesController (req: express.Request, res: express.Response, next: express.NextFunction) { 144async function videoAnnouncesController (req: express.Request, res: express.Response, next: express.NextFunction) {
140 const video: VideoModel = res.locals.video 145 const video: VideoModel = res.locals.video
141 146
142 // We need more attributes 147 const handler = async (start: number, count: number) => {
143 const videoAll = await VideoModel.loadAndPopulateAll(video.id) 148 const result = await VideoShareModel.listAndCountByVideoId(video.id, start, count)
144 const object = videoAll.toAnnouncesActivityPubObject() 149 return {
150 total: result.count,
151 data: result.rows.map(r => r.url)
152 }
153 }
154 const json = await activityPubCollectionPagination(getVideoSharesActivityPubUrl(video), handler, req.query.page)
145 155
146 return activityPubResponse(activityPubContextify(object), res) 156 return activityPubResponse(activityPubContextify(json), res)
147} 157}
148 158
149async function videoLikesController (req: express.Request, res: express.Response, next: express.NextFunction) { 159async function videoLikesController (req: express.Request, res: express.Response, next: express.NextFunction) {
150 const video: VideoModel = res.locals.video 160 const video: VideoModel = res.locals.video
161 const json = await videoRates(req, 'like', video, getVideoLikesActivityPubUrl(video))
151 162
152 // We need more attributes 163 return activityPubResponse(activityPubContextify(json), res)
153 const videoAll = await VideoModel.loadAndPopulateAll(video.id)
154 const { likesObject } = videoAll.toRatesActivityPubObjects()
155
156 return activityPubResponse(activityPubContextify(likesObject), res)
157} 164}
158 165
159async function videoDislikesController (req: express.Request, res: express.Response, next: express.NextFunction) { 166async function videoDislikesController (req: express.Request, res: express.Response, next: express.NextFunction) {
160 const video: VideoModel = res.locals.video 167 const video: VideoModel = res.locals.video
168 const json = await videoRates(req, 'dislike', video, getVideoDislikesActivityPubUrl(video))
161 169
162 // We need more attributes 170 return activityPubResponse(activityPubContextify(json), res)
163 const videoAll = await VideoModel.loadAndPopulateAll(video.id)
164 const { dislikesObject } = videoAll.toRatesActivityPubObjects()
165
166 return activityPubResponse(activityPubContextify(dislikesObject), res)
167} 171}
168 172
169async function videoCommentsController (req: express.Request, res: express.Response, next: express.NextFunction) { 173async function videoCommentsController (req: express.Request, res: express.Response, next: express.NextFunction) {
170 const video: VideoModel = res.locals.video 174 const video: VideoModel = res.locals.video
171 175
172 // We need more attributes 176 const handler = async (start: number, count: number) => {
173 const videoAll = await VideoModel.loadAndPopulateAll(video.id) 177 const result = await VideoCommentModel.listAndCountByVideoId(video.id, start, count)
174 const commentsObject = videoAll.toCommentsActivityPubObject() 178 return {
179 total: result.count,
180 data: result.rows.map(r => r.url)
181 }
182 }
183 const json = await activityPubCollectionPagination(getVideoCommentsActivityPubUrl(video), handler, req.query.page)
175 184
176 return activityPubResponse(activityPubContextify(commentsObject), res) 185 return activityPubResponse(activityPubContextify(json), res)
177} 186}
178 187
179async function videoChannelController (req: express.Request, res: express.Response, next: express.NextFunction) { 188async function videoChannelController (req: express.Request, res: express.Response, next: express.NextFunction) {
@@ -216,23 +225,28 @@ async function videoCommentController (req: express.Request, res: express.Respon
216// --------------------------------------------------------------------------- 225// ---------------------------------------------------------------------------
217 226
218async function actorFollowing (req: express.Request, actor: ActorModel) { 227async function actorFollowing (req: express.Request, actor: ActorModel) {
219 const page = req.query.page || 1 228 const handler = (start: number, count: number) => {
220 const { start, count } = pageToStartAndCount(page, ACTIVITY_PUB.COLLECTION_ITEMS_PER_PAGE) 229 return ActorFollowModel.listAcceptedFollowingUrlsForApi([ actor.id ], undefined, start, count)
230 }
221 231
222 const result = await ActorFollowModel.listAcceptedFollowingUrlsForApi([ actor.id ], undefined, start, count) 232 return activityPubCollectionPagination(CONFIG.WEBSERVER.URL + req.url, handler, req.query.page)
223 return activityPubCollectionPagination(CONFIG.WEBSERVER.URL + req.url, page, result)
224} 233}
225 234
226async function actorFollowers (req: express.Request, actor: ActorModel) { 235async function actorFollowers (req: express.Request, actor: ActorModel) {
227 const page = req.query.page || 1 236 const handler = (start: number, count: number) => {
228 const { start, count } = pageToStartAndCount(page, ACTIVITY_PUB.COLLECTION_ITEMS_PER_PAGE) 237 return ActorFollowModel.listAcceptedFollowerUrlsForApi([ actor.id ], undefined, start, count)
238 }
229 239
230 const result = await ActorFollowModel.listAcceptedFollowerUrlsForApi([ actor.id ], undefined, start, count) 240 return activityPubCollectionPagination(CONFIG.WEBSERVER.URL + req.url, handler, req.query.page)
231 return activityPubCollectionPagination(CONFIG.WEBSERVER.URL + req.url, page, result)
232} 241}
233 242
234function activityPubResponse (data: any, res: express.Response) { 243function videoRates (req: express.Request, rateType: VideoRateType, video: VideoModel, url: string) {
235 return res.type('application/activity+json; charset=utf-8') 244 const handler = async (start: number, count: number) => {
236 .json(data) 245 const result = await AccountVideoRateModel.listAndCountAccountUrlsByVideoId(rateType, video.id, start, count)
237 .end() 246 return {
247 total: result.count,
248 data: result.rows.map(r => r.Account.Actor.url)
249 }
250 }
251 return activityPubCollectionPagination(url, handler, req.query.page)
238} 252}
diff --git a/server/controllers/activitypub/outbox.ts b/server/controllers/activitypub/outbox.ts
index c9e087a13..97bf9b052 100644
--- a/server/controllers/activitypub/outbox.ts
+++ b/server/controllers/activitypub/outbox.ts
@@ -1,16 +1,15 @@
1import * as express from 'express' 1import * as express from 'express'
2import { Activity } from '../../../shared/models/activitypub/activity' 2import { Activity } from '../../../shared/models/activitypub/activity'
3import { VideoPrivacy } from '../../../shared/models/videos' 3import { VideoPrivacy } from '../../../shared/models/videos'
4import { activityPubCollectionPagination } from '../../helpers/activitypub' 4import { activityPubCollectionPagination, activityPubContextify } from '../../helpers/activitypub'
5import { pageToStartAndCount } from '../../helpers/core-utils'
6import { logger } from '../../helpers/logger' 5import { logger } from '../../helpers/logger'
7import { ACTIVITY_PUB } from '../../initializers/constants'
8import { announceActivityData, createActivityData } from '../../lib/activitypub/send' 6import { announceActivityData, createActivityData } from '../../lib/activitypub/send'
9import { buildAudience } from '../../lib/activitypub/audience' 7import { buildAudience } from '../../lib/activitypub/audience'
10import { asyncMiddleware, localAccountValidator } from '../../middlewares' 8import { asyncMiddleware, localAccountValidator } from '../../middlewares'
11import { AccountModel } from '../../models/account/account' 9import { AccountModel } from '../../models/account/account'
12import { ActorModel } from '../../models/activitypub/actor' 10import { ActorModel } from '../../models/activitypub/actor'
13import { VideoModel } from '../../models/video/video' 11import { VideoModel } from '../../models/video/video'
12import { activityPubResponse } from './utils'
14 13
15const outboxRouter = express.Router() 14const outboxRouter = express.Router()
16 15
@@ -30,10 +29,17 @@ export {
30async function outboxController (req: express.Request, res: express.Response, next: express.NextFunction) { 29async function outboxController (req: express.Request, res: express.Response, next: express.NextFunction) {
31 const account: AccountModel = res.locals.account 30 const account: AccountModel = res.locals.account
32 const actor = account.Actor 31 const actor = account.Actor
32 const actorOutboxUrl = account.Actor.url + '/outbox'
33
34 logger.info('Receiving outbox request for %s.', actorOutboxUrl)
33 35
34 const page = req.query.page || 1 36 const handler = (start: number, count: number) => buildActivities(actor, start, count)
35 const { start, count } = pageToStartAndCount(page, ACTIVITY_PUB.COLLECTION_ITEMS_PER_PAGE) 37 const json = await activityPubCollectionPagination(actorOutboxUrl, handler, req.query.page)
38
39 return activityPubResponse(activityPubContextify(json), res)
40}
36 41
42async function buildActivities (actor: ActorModel, start: number, count: number) {
37 const data = await VideoModel.listAllAndSharedByActorForOutbox(actor.id, start, count) 43 const data = await VideoModel.listAllAndSharedByActorForOutbox(actor.id, start, count)
38 const activities: Activity[] = [] 44 const activities: Activity[] = []
39 45
@@ -62,14 +68,8 @@ async function outboxController (req: express.Request, res: express.Response, ne
62 } 68 }
63 } 69 }
64 70
65 const newResult = { 71 return {
66 data: activities, 72 data: activities,
67 total: data.total 73 total: data.total
68 } 74 }
69 const actorOutboxUrl = account.Actor.url + '/outbox'
70 const json = activityPubCollectionPagination(actorOutboxUrl, page, newResult)
71
72 logger.info('Receiving outbox request for %s.', actorOutboxUrl)
73
74 return res.json(json).end()
75} 75}
diff --git a/server/controllers/activitypub/utils.ts b/server/controllers/activitypub/utils.ts
new file mode 100644
index 000000000..599cf48ab
--- /dev/null
+++ b/server/controllers/activitypub/utils.ts
@@ -0,0 +1,11 @@
1import * as express from 'express'
2
3function activityPubResponse (data: any, res: express.Response) {
4 return res.type('application/activity+json; charset=utf-8')
5 .json(data)
6 .end()
7}
8
9export {
10 activityPubResponse
11}
diff --git a/server/helpers/activitypub.ts b/server/helpers/activitypub.ts
index 1934fa0f0..d1f3ec02d 100644
--- a/server/helpers/activitypub.ts
+++ b/server/helpers/activitypub.ts
@@ -1,8 +1,11 @@
1import * as Bluebird from 'bluebird'
2import * as validator from 'validator'
1import { ResultList } from '../../shared/models' 3import { ResultList } from '../../shared/models'
2import { Activity, ActivityPubActor } from '../../shared/models/activitypub' 4import { Activity, ActivityPubActor } from '../../shared/models/activitypub'
3import { ACTIVITY_PUB } from '../initializers' 5import { ACTIVITY_PUB } from '../initializers'
4import { ActorModel } from '../models/activitypub/actor' 6import { ActorModel } from '../models/activitypub/actor'
5import { signObject } from './peertube-crypto' 7import { signObject } from './peertube-crypto'
8import { pageToStartAndCount } from './core-utils'
6 9
7function activityPubContextify <T> (data: T) { 10function activityPubContextify <T> (data: T) {
8 return Object.assign(data,{ 11 return Object.assign(data,{
@@ -44,16 +47,23 @@ function activityPubContextify <T> (data: T) {
44 }) 47 })
45} 48}
46 49
47function activityPubCollection (url: string, results: any[]) { 50type ActivityPubCollectionPaginationHandler = (start: number, count: number) => Bluebird<ResultList<any>> | Promise<ResultList<any>>
48 return { 51async function activityPubCollectionPagination (url: string, handler: ActivityPubCollectionPaginationHandler, page?: any) {
49 id: url, 52 if (!page || !validator.isInt(page)) {
50 type: 'OrderedCollection', 53 // We just display the first page URL, we only need the total items
51 totalItems: results.length, 54 const result = await handler(0, 1)
52 orderedItems: results 55
56 return {
57 id: url,
58 type: 'OrderedCollection',
59 totalItems: result.total,
60 first: url + '?page=1'
61 }
53 } 62 }
54}
55 63
56function activityPubCollectionPagination (url: string, page: any, result: ResultList<any>) { 64 const { start, count } = pageToStartAndCount(page, ACTIVITY_PUB.COLLECTION_ITEMS_PER_PAGE)
65 const result = await handler(start, count)
66
57 let next: string 67 let next: string
58 let prev: string 68 let prev: string
59 69
@@ -69,27 +79,16 @@ function activityPubCollectionPagination (url: string, page: any, result: Result
69 prev = url + '?page=' + (page - 1) 79 prev = url + '?page=' + (page - 1)
70 } 80 }
71 81
72 const orderedCollectionPagination = { 82 return {
73 id: url + '?page=' + page, 83 id: url + '?page=' + page,
74 type: 'OrderedCollectionPage', 84 type: 'OrderedCollectionPage',
75 prev, 85 prev,
76 next, 86 next,
77 partOf: url, 87 partOf: url,
78 orderedItems: result.data 88 orderedItems: result.data,
79 } 89 totalItems: result.total
80
81 if (page === 1) {
82 return activityPubContextify({
83 id: url,
84 type: 'OrderedCollection',
85 totalItems: result.total,
86 first: orderedCollectionPagination
87 })
88 } else {
89 orderedCollectionPagination['totalItems'] = result.total
90 } 90 }
91 91
92 return orderedCollectionPagination
93} 92}
94 93
95function buildSignedActivity (byActor: ActorModel, data: Object) { 94function buildSignedActivity (byActor: ActorModel, data: Object) {
@@ -110,6 +109,5 @@ export {
110 getActorUrl, 109 getActorUrl,
111 activityPubContextify, 110 activityPubContextify,
112 activityPubCollectionPagination, 111 activityPubCollectionPagination,
113 activityPubCollection,
114 buildSignedActivity 112 buildSignedActivity
115} 113}
diff --git a/server/lib/activitypub/crawl.ts b/server/lib/activitypub/crawl.ts
new file mode 100644
index 000000000..7305b3969
--- /dev/null
+++ b/server/lib/activitypub/crawl.ts
@@ -0,0 +1,40 @@
1import { ACTIVITY_PUB, JOB_REQUEST_TIMEOUT } from '../../initializers'
2import { doRequest } from '../../helpers/requests'
3import { logger } from '../../helpers/logger'
4
5async function crawlCollectionPage <T> (uri: string, handler: (items: T[]) => Promise<any>) {
6 logger.info('Crawling ActivityPub data on %s.', uri)
7
8 const options = {
9 method: 'GET',
10 uri,
11 json: true,
12 activityPub: true,
13 timeout: JOB_REQUEST_TIMEOUT
14 }
15
16 const response = await doRequest(options)
17 const firstBody = response.body
18
19 let limit = ACTIVITY_PUB.FETCH_PAGE_LIMIT
20 let i = 0
21 let nextLink = firstBody.first
22 while (nextLink && i < limit) {
23 options.uri = nextLink
24
25 const { body } = await doRequest(options)
26 nextLink = body.next
27 i++
28
29 if (Array.isArray(body.orderedItems)) {
30 const items = body.orderedItems
31 logger.info('Processing %i ActivityPub items for %s.', items.length, nextLink)
32
33 await handler(items)
34 }
35 }
36}
37
38export {
39 crawlCollectionPage
40}
diff --git a/server/lib/activitypub/video-comments.ts b/server/lib/activitypub/video-comments.ts
index 60c9179a6..fd03710c2 100644
--- a/server/lib/activitypub/video-comments.ts
+++ b/server/lib/activitypub/video-comments.ts
@@ -37,7 +37,7 @@ async function videoCommentActivityObjectToDBAttributes (video: VideoModel, acto
37 } 37 }
38} 38}
39 39
40async function addVideoComments (instance: VideoModel, commentUrls: string[]) { 40async function addVideoComments (commentUrls: string[], instance: VideoModel) {
41 for (const commentUrl of commentUrls) { 41 for (const commentUrl of commentUrls) {
42 await addVideoComment(instance, commentUrl) 42 await addVideoComment(instance, commentUrl)
43 } 43 }
diff --git a/server/lib/activitypub/videos.ts b/server/lib/activitypub/videos.ts
index dbd7385a4..be6794cef 100644
--- a/server/lib/activitypub/videos.ts
+++ b/server/lib/activitypub/videos.ts
@@ -20,6 +20,7 @@ import { VideoFileModel } from '../../models/video/video-file'
20import { VideoShareModel } from '../../models/video/video-share' 20import { VideoShareModel } from '../../models/video/video-share'
21import { getOrCreateActorAndServerAndModel } from './actor' 21import { getOrCreateActorAndServerAndModel } from './actor'
22import { addVideoComments } from './video-comments' 22import { addVideoComments } from './video-comments'
23import { crawlCollectionPage } from './crawl'
23 24
24function fetchRemoteVideoPreview (video: VideoModel, reject: Function) { 25function fetchRemoteVideoPreview (video: VideoModel, reject: Function) {
25 const host = video.VideoChannel.Account.Actor.Server.host 26 const host = video.VideoChannel.Account.Actor.Server.host
@@ -216,25 +217,17 @@ async function getOrCreateAccountAndVideoAndChannel (videoObject: VideoTorrentOb
216 const video = await retryTransactionWrapper(getOrCreateVideo, options) 217 const video = await retryTransactionWrapper(getOrCreateVideo, options)
217 218
218 // Process outside the transaction because we could fetch remote data 219 // Process outside the transaction because we could fetch remote data
219 if (videoObject.likes && Array.isArray(videoObject.likes.orderedItems)) { 220 logger.info('Adding likes of video %s.', video.uuid)
220 logger.info('Adding likes of video %s.', video.uuid) 221 await crawlCollectionPage<string>(videoObject.likes, (items) => createRates(items, video, 'like'))
221 await createRates(videoObject.likes.orderedItems, video, 'like')
222 }
223 222
224 if (videoObject.dislikes && Array.isArray(videoObject.dislikes.orderedItems)) { 223 logger.info('Adding dislikes of video %s.', video.uuid)
225 logger.info('Adding dislikes of video %s.', video.uuid) 224 await crawlCollectionPage<string>(videoObject.dislikes, (items) => createRates(items, video, 'dislike'))
226 await createRates(videoObject.dislikes.orderedItems, video, 'dislike')
227 }
228 225
229 if (videoObject.shares && Array.isArray(videoObject.shares.orderedItems)) { 226 logger.info('Adding shares of video %s.', video.uuid)
230 logger.info('Adding shares of video %s.', video.uuid) 227 await crawlCollectionPage<string>(videoObject.shares, (items) => addVideoShares(items, video))
231 await addVideoShares(video, videoObject.shares.orderedItems)
232 }
233 228
234 if (videoObject.comments && Array.isArray(videoObject.comments.orderedItems)) { 229 logger.info('Adding comments of video %s.', video.uuid)
235 logger.info('Adding comments of video %s.', video.uuid) 230 await crawlCollectionPage<string>(videoObject.comments, (items) => addVideoComments(items, video))
236 await addVideoComments(video, videoObject.comments.orderedItems)
237 }
238 231
239 return { actor, channelActor, video } 232 return { actor, channelActor, video }
240} 233}
@@ -266,7 +259,7 @@ async function createRates (actorUrls: string[], video: VideoModel, rate: VideoR
266 return 259 return
267} 260}
268 261
269async function addVideoShares (instance: VideoModel, shareUrls: string[]) { 262async function addVideoShares (shareUrls: string[], instance: VideoModel) {
270 for (const shareUrl of shareUrls) { 263 for (const shareUrl of shareUrls) {
271 // Fetch url 264 // Fetch url
272 const { body } = await doRequest({ 265 const { body } = await doRequest({
diff --git a/server/lib/job-queue/handlers/activitypub-http-fetcher.ts b/server/lib/job-queue/handlers/activitypub-http-fetcher.ts
index 4683beb2f..10c0e606f 100644
--- a/server/lib/job-queue/handlers/activitypub-http-fetcher.ts
+++ b/server/lib/job-queue/handlers/activitypub-http-fetcher.ts
@@ -1,9 +1,9 @@
1import * as kue from 'kue' 1import * as kue from 'kue'
2import { logger } from '../../../helpers/logger' 2import { logger } from '../../../helpers/logger'
3import { doRequest } from '../../../helpers/requests'
4import { ACTIVITY_PUB, JOB_REQUEST_TIMEOUT } from '../../../initializers'
5import { processActivities } from '../../activitypub/process' 3import { processActivities } from '../../activitypub/process'
6import { ActivitypubHttpBroadcastPayload } from './activitypub-http-broadcast' 4import { ActivitypubHttpBroadcastPayload } from './activitypub-http-broadcast'
5import { crawlCollectionPage } from '../../activitypub/crawl'
6import { Activity } from '../../../../shared/models/activitypub'
7 7
8export type ActivitypubHttpFetcherPayload = { 8export type ActivitypubHttpFetcherPayload = {
9 uris: string[] 9 uris: string[]
@@ -14,46 +14,8 @@ async function processActivityPubHttpFetcher (job: kue.Job) {
14 14
15 const payload = job.data as ActivitypubHttpBroadcastPayload 15 const payload = job.data as ActivitypubHttpBroadcastPayload
16 16
17 const options = {
18 method: 'GET',
19 uri: '',
20 json: true,
21 activityPub: true,
22 timeout: JOB_REQUEST_TIMEOUT
23 }
24
25 for (const uri of payload.uris) { 17 for (const uri of payload.uris) {
26 options.uri = uri 18 await crawlCollectionPage<Activity>(uri, (items) => processActivities(items))
27 logger.info('Fetching ActivityPub data on %s.', uri)
28
29 const response = await doRequest(options)
30 const firstBody = response.body
31
32 if (firstBody.first && Array.isArray(firstBody.first.orderedItems)) {
33 const activities = firstBody.first.orderedItems
34
35 logger.info('Processing %i items ActivityPub fetcher for %s.', activities.length, options.uri)
36
37 await processActivities(activities)
38 }
39
40 let limit = ACTIVITY_PUB.FETCH_PAGE_LIMIT
41 let i = 0
42 let nextLink = firstBody.first.next
43 while (nextLink && i < limit) {
44 options.uri = nextLink
45
46 const { body } = await doRequest(options)
47 nextLink = body.next
48 i++
49
50 if (Array.isArray(body.orderedItems)) {
51 const activities = body.orderedItems
52 logger.info('Processing %i items ActivityPub fetcher for %s.', activities.length, options.uri)
53
54 await processActivities(activities)
55 }
56 }
57 } 19 }
58} 20}
59 21
diff --git a/server/models/account/account-video-rate.ts b/server/models/account/account-video-rate.ts
index e969e4a43..508ab814f 100644
--- a/server/models/account/account-video-rate.ts
+++ b/server/models/account/account-video-rate.ts
@@ -6,6 +6,7 @@ import { VideoRateType } from '../../../shared/models/videos'
6import { VIDEO_RATE_TYPES } from '../../initializers' 6import { VIDEO_RATE_TYPES } from '../../initializers'
7import { VideoModel } from '../video/video' 7import { VideoModel } from '../video/video'
8import { AccountModel } from './account' 8import { AccountModel } from './account'
9import { ActorModel } from '../activitypub/actor'
9 10
10/* 11/*
11 Account rates per video. 12 Account rates per video.
@@ -66,4 +67,32 @@ export class AccountVideoRateModel extends Model<AccountVideoRateModel> {
66 67
67 return AccountVideoRateModel.findOne(options) 68 return AccountVideoRateModel.findOne(options)
68 } 69 }
70
71 static listAndCountAccountUrlsByVideoId (rateType: VideoRateType, videoId: number, start: number, count: number, t?: Transaction) {
72 const query = {
73 start,
74 count,
75 where: {
76 videoId,
77 type: rateType
78 },
79 transaction: t,
80 include: [
81 {
82 attributes: [ 'actorId' ],
83 model: AccountModel.unscoped(),
84 required: true,
85 include: [
86 {
87 attributes: [ 'url' ],
88 model: ActorModel.unscoped(),
89 required: true
90 }
91 ]
92 }
93 ]
94 }
95
96 return AccountVideoRateModel.findAndCountAll(query)
97 }
69} 98}
diff --git a/server/models/activitypub/actor-follow.ts b/server/models/activitypub/actor-follow.ts
index c97f4cead..b8ce6de1d 100644
--- a/server/models/activitypub/actor-follow.ts
+++ b/server/models/activitypub/actor-follow.ts
@@ -335,8 +335,7 @@ export class ActorFollowModel extends Model<ActorFollowModel> {
335 tasks.push(ActorFollowModel.sequelize.query(query, options)) 335 tasks.push(ActorFollowModel.sequelize.query(query, options))
336 } 336 }
337 337
338 const [ followers, [ { total } ] ] = await 338 const [ followers, [ { total } ] ] = await Promise.all(tasks)
339 Promise.all(tasks)
340 const urls: string[] = followers.map(f => f.url) 339 const urls: string[] = followers.map(f => f.url)
341 340
342 return { 341 return {
diff --git a/server/models/video/video-comment.ts b/server/models/video/video-comment.ts
index 5386a10aa..18398905e 100644
--- a/server/models/video/video-comment.ts
+++ b/server/models/video/video-comment.ts
@@ -326,6 +326,20 @@ export class VideoCommentModel extends Model<VideoCommentModel> {
326 .findAll(query) 326 .findAll(query)
327 } 327 }
328 328
329 static listAndCountByVideoId (videoId: number, start: number, count: number, t?: Sequelize.Transaction, order: 'ASC' | 'DESC' = 'ASC') {
330 const query = {
331 order: [ [ 'createdAt', order ] ],
332 start,
333 count,
334 where: {
335 videoId
336 },
337 transaction: t
338 }
339
340 return VideoCommentModel.findAndCountAll(query)
341 }
342
329 static async getStats () { 343 static async getStats () {
330 const totalLocalVideoComments = await VideoCommentModel.count({ 344 const totalLocalVideoComments = await VideoCommentModel.count({
331 include: [ 345 include: [
diff --git a/server/models/video/video-share.ts b/server/models/video/video-share.ts
index 602cc69b9..adadf5dea 100644
--- a/server/models/video/video-share.ts
+++ b/server/models/video/video-share.ts
@@ -187,4 +187,17 @@ export class VideoShareModel extends Model<VideoShareModel> {
187 .findAll(query) 187 .findAll(query)
188 .then(res => res.map(r => r.Actor)) 188 .then(res => res.map(r => r.Actor))
189 } 189 }
190
191 static listAndCountByVideoId (videoId: number, start: number, count: number, t?: Sequelize.Transaction) {
192 const query = {
193 start,
194 count,
195 where: {
196 videoId
197 },
198 transaction: t
199 }
200
201 return VideoShareModel.findAndCountAll(query)
202 }
190} 203}
diff --git a/server/models/video/video.ts b/server/models/video/video.ts
index 012a758ee..f4689fe12 100644
--- a/server/models/video/video.ts
+++ b/server/models/video/video.ts
@@ -29,7 +29,7 @@ import { VideoPrivacy, VideoResolution } from '../../../shared'
29import { VideoTorrentObject } from '../../../shared/models/activitypub/objects' 29import { VideoTorrentObject } from '../../../shared/models/activitypub/objects'
30import { Video, VideoDetails, VideoFile } from '../../../shared/models/videos' 30import { Video, VideoDetails, VideoFile } from '../../../shared/models/videos'
31import { VideoFilter } from '../../../shared/models/videos/video-query.type' 31import { VideoFilter } from '../../../shared/models/videos/video-query.type'
32import { activityPubCollection } from '../../helpers/activitypub' 32import { activityPubCollectionPagination } from '../../helpers/activitypub'
33import { 33import {
34 createTorrentPromise, 34 createTorrentPromise,
35 peertubeTruncate, 35 peertubeTruncate,
@@ -602,18 +602,6 @@ export class VideoModel extends Model<VideoModel> {
602 attributes: [ 'id', 'url' ], 602 attributes: [ 'id', 'url' ],
603 model: VideoShareModel.unscoped(), 603 model: VideoShareModel.unscoped(),
604 required: false, 604 required: false,
605 where: {
606 [Sequelize.Op.and]: [
607 {
608 id: {
609 [Sequelize.Op.not]: null
610 }
611 },
612 {
613 actorId
614 }
615 ]
616 },
617 include: [ 605 include: [
618 { 606 {
619 attributes: [ 'id', 'url' ], 607 attributes: [ 'id', 'url' ],
@@ -644,35 +632,6 @@ export class VideoModel extends Model<VideoModel> {
644 } 632 }
645 ] 633 ]
646 }, 634 },
647 {
648 attributes: [ 'type' ],
649 model: AccountVideoRateModel,
650 required: false,
651 include: [
652 {
653 attributes: [ 'id' ],
654 model: AccountModel.unscoped(),
655 include: [
656 {
657 attributes: [ 'url' ],
658 model: ActorModel.unscoped(),
659 include: [
660 {
661 attributes: [ 'host' ],
662 model: ServerModel,
663 required: false
664 }
665 ]
666 }
667 ]
668 }
669 ]
670 },
671 {
672 attributes: [ 'url' ],
673 model: VideoCommentModel,
674 required: false
675 },
676 VideoFileModel, 635 VideoFileModel,
677 TagModel 636 TagModel
678 ] 637 ]
@@ -897,26 +856,6 @@ export class VideoModel extends Model<VideoModel> {
897 .findOne(options) 856 .findOne(options)
898 } 857 }
899 858
900 static loadAndPopulateAll (id: number) {
901 const options = {
902 order: [ [ 'Tags', 'name', 'ASC' ] ],
903 where: {
904 id
905 }
906 }
907
908 return VideoModel
909 .scope([
910 ScopeNames.WITH_RATES,
911 ScopeNames.WITH_SHARES,
912 ScopeNames.WITH_TAGS,
913 ScopeNames.WITH_FILES,
914 ScopeNames.WITH_ACCOUNT_DETAILS,
915 ScopeNames.WITH_COMMENTS
916 ])
917 .findOne(options)
918 }
919
920 static async getStats () { 859 static async getStats () {
921 const totalLocalVideos = await VideoModel.count({ 860 const totalLocalVideos = await VideoModel.count({
922 where: { 861 where: {
@@ -1203,25 +1142,6 @@ export class VideoModel extends Model<VideoModel> {
1203 } 1142 }
1204 } 1143 }
1205 1144
1206 let likesObject
1207 let dislikesObject
1208
1209 if (Array.isArray(this.AccountVideoRates)) {
1210 const res = this.toRatesActivityPubObjects()
1211 likesObject = res.likesObject
1212 dislikesObject = res.dislikesObject
1213 }
1214
1215 let sharesObject
1216 if (Array.isArray(this.VideoShares)) {
1217 sharesObject = this.toAnnouncesActivityPubObject()
1218 }
1219
1220 let commentsObject
1221 if (Array.isArray(this.VideoComments)) {
1222 commentsObject = this.toCommentsActivityPubObject()
1223 }
1224
1225 const url = [] 1145 const url = []
1226 for (const file of this.VideoFiles) { 1146 for (const file of this.VideoFiles) {
1227 url.push({ 1147 url.push({
@@ -1280,10 +1200,10 @@ export class VideoModel extends Model<VideoModel> {
1280 height: THUMBNAILS_SIZE.height 1200 height: THUMBNAILS_SIZE.height
1281 }, 1201 },
1282 url, 1202 url,
1283 likes: likesObject, 1203 likes: getVideoLikesActivityPubUrl(this),
1284 dislikes: dislikesObject, 1204 dislikes: getVideoDislikesActivityPubUrl(this),
1285 shares: sharesObject, 1205 shares: getVideoSharesActivityPubUrl(this),
1286 comments: commentsObject, 1206 comments: getVideoCommentsActivityPubUrl(this),
1287 attributedTo: [ 1207 attributedTo: [
1288 { 1208 {
1289 type: 'Person', 1209 type: 'Person',
@@ -1297,44 +1217,6 @@ export class VideoModel extends Model<VideoModel> {
1297 } 1217 }
1298 } 1218 }
1299 1219
1300 toAnnouncesActivityPubObject () {
1301 const shares: string[] = []
1302
1303 for (const videoShare of this.VideoShares) {
1304 shares.push(videoShare.url)
1305 }
1306
1307 return activityPubCollection(getVideoSharesActivityPubUrl(this), shares)
1308 }
1309
1310 toCommentsActivityPubObject () {
1311 const comments: string[] = []
1312
1313 for (const videoComment of this.VideoComments) {
1314 comments.push(videoComment.url)
1315 }
1316
1317 return activityPubCollection(getVideoCommentsActivityPubUrl(this), comments)
1318 }
1319
1320 toRatesActivityPubObjects () {
1321 const likes: string[] = []
1322 const dislikes: string[] = []
1323
1324 for (const rate of this.AccountVideoRates) {
1325 if (rate.type === 'like') {
1326 likes.push(rate.Account.Actor.url)
1327 } else if (rate.type === 'dislike') {
1328 dislikes.push(rate.Account.Actor.url)
1329 }
1330 }
1331
1332 const likesObject = activityPubCollection(getVideoLikesActivityPubUrl(this), likes)
1333 const dislikesObject = activityPubCollection(getVideoDislikesActivityPubUrl(this), dislikes)
1334
1335 return { likesObject, dislikesObject }
1336 }
1337
1338 getTruncatedDescription () { 1220 getTruncatedDescription () {
1339 if (!this.description) return null 1221 if (!this.description) return null
1340 1222
diff --git a/shared/models/activitypub/objects/video-torrent-object.ts b/shared/models/activitypub/objects/video-torrent-object.ts
index 02820a4cb..767b6a2d0 100644
--- a/shared/models/activitypub/objects/video-torrent-object.ts
+++ b/shared/models/activitypub/objects/video-torrent-object.ts
@@ -26,10 +26,10 @@ export interface VideoTorrentObject {
26 support: string 26 support: string
27 icon: ActivityIconObject 27 icon: ActivityIconObject
28 url: ActivityUrlObject[] 28 url: ActivityUrlObject[]
29 likes?: ActivityPubOrderedCollection<string> 29 likes: string
30 dislikes?: ActivityPubOrderedCollection<string> 30 dislikes: string
31 shares?: ActivityPubOrderedCollection<string> 31 shares: string
32 comments?: ActivityPubOrderedCollection<string> 32 comments: string
33 attributedTo: ActivityPubAttributedTo[] 33 attributedTo: ActivityPubAttributedTo[]
34 to?: string[] 34 to?: string[]
35 cc?: string[] 35 cc?: string[]