diff options
author | Chocobozzz <me@florianbigard.com> | 2018-05-25 16:21:16 +0200 |
---|---|---|
committer | Chocobozzz <me@florianbigard.com> | 2018-05-25 16:21:16 +0200 |
commit | 8fffe21a7bc96d08b229293d66ddba576e609790 (patch) | |
tree | 5ebd5f5198a59084c5338ce197d7e836b39200a4 | |
parent | e251f170b00b2014ac4e823113c6ff40e3fb1471 (diff) | |
download | PeerTube-8fffe21a7bc96d08b229293d66ddba576e609790.tar.gz PeerTube-8fffe21a7bc96d08b229293d66ddba576e609790.tar.zst PeerTube-8fffe21a7bc96d08b229293d66ddba576e609790.zip |
Refractor and optimize AP collections
Only display urls in general object, and paginate video comments, shares, likes and
dislikes
-rw-r--r-- | server/controllers/activitypub/client.ts | 86 | ||||
-rw-r--r-- | server/controllers/activitypub/outbox.ts | 24 | ||||
-rw-r--r-- | server/controllers/activitypub/utils.ts | 11 | ||||
-rw-r--r-- | server/helpers/activitypub.ts | 44 | ||||
-rw-r--r-- | server/lib/activitypub/crawl.ts | 40 | ||||
-rw-r--r-- | server/lib/activitypub/video-comments.ts | 2 | ||||
-rw-r--r-- | server/lib/activitypub/videos.ts | 27 | ||||
-rw-r--r-- | server/lib/job-queue/handlers/activitypub-http-fetcher.ts | 44 | ||||
-rw-r--r-- | server/models/account/account-video-rate.ts | 29 | ||||
-rw-r--r-- | server/models/activitypub/actor-follow.ts | 3 | ||||
-rw-r--r-- | server/models/video/video-comment.ts | 14 | ||||
-rw-r--r-- | server/models/video/video-share.ts | 13 | ||||
-rw-r--r-- | server/models/video/video.ts | 128 | ||||
-rw-r--r-- | shared/models/activitypub/objects/video-torrent-object.ts | 8 |
14 files changed, 214 insertions, 259 deletions
diff --git a/server/controllers/activitypub/client.ts b/server/controllers/activitypub/client.ts index 767fde5d9..1c780783c 100644 --- a/server/controllers/activitypub/client.ts +++ b/server/controllers/activitypub/client.ts | |||
@@ -1,9 +1,8 @@ | |||
1 | // Intercept ActivityPub client requests | 1 | // Intercept ActivityPub client requests |
2 | import * as express from 'express' | 2 | import * as express from 'express' |
3 | import { VideoPrivacy } from '../../../shared/models/videos' | 3 | import { VideoPrivacy, VideoRateType } from '../../../shared/models/videos' |
4 | import { activityPubCollectionPagination, activityPubContextify } from '../../helpers/activitypub' | 4 | import { activityPubCollectionPagination, activityPubContextify } from '../../helpers/activitypub' |
5 | import { pageToStartAndCount } from '../../helpers/core-utils' | 5 | import { CONFIG, ROUTE_CACHE_LIFETIME } from '../../initializers' |
6 | import { ACTIVITY_PUB, CONFIG, ROUTE_CACHE_LIFETIME } from '../../initializers' | ||
7 | import { buildVideoAnnounce } from '../../lib/activitypub/send' | 6 | import { buildVideoAnnounce } from '../../lib/activitypub/send' |
8 | import { audiencify, getAudience } from '../../lib/activitypub/audience' | 7 | import { audiencify, getAudience } from '../../lib/activitypub/audience' |
9 | import { createActivityData } from '../../lib/activitypub/send/send-create' | 8 | import { createActivityData } from '../../lib/activitypub/send/send-create' |
@@ -18,6 +17,14 @@ import { VideoChannelModel } from '../../models/video/video-channel' | |||
18 | import { VideoCommentModel } from '../../models/video/video-comment' | 17 | import { VideoCommentModel } from '../../models/video/video-comment' |
19 | import { VideoShareModel } from '../../models/video/video-share' | 18 | import { VideoShareModel } from '../../models/video/video-share' |
20 | import { cacheRoute } from '../../middlewares/cache' | 19 | import { cacheRoute } from '../../middlewares/cache' |
20 | import { activityPubResponse } from './utils' | ||
21 | import { AccountVideoRateModel } from '../../models/account/account-video-rate' | ||
22 | import { | ||
23 | getVideoCommentsActivityPubUrl, | ||
24 | getVideoDislikesActivityPubUrl, | ||
25 | getVideoLikesActivityPubUrl, | ||
26 | getVideoSharesActivityPubUrl | ||
27 | } from '../../lib/activitypub' | ||
21 | 28 | ||
22 | const activityPubClientRouter = express.Router() | 29 | const activityPubClientRouter = express.Router() |
23 | 30 | ||
@@ -116,10 +123,8 @@ async function accountFollowingController (req: express.Request, res: express.Re | |||
116 | async function videoController (req: express.Request, res: express.Response, next: express.NextFunction) { | 123 | async function videoController (req: express.Request, res: express.Response, next: express.NextFunction) { |
117 | const video: VideoModel = res.locals.video | 124 | const video: VideoModel = res.locals.video |
118 | 125 | ||
119 | // We need more attributes | ||
120 | const videoAll = await VideoModel.loadAndPopulateAll(video.id) | ||
121 | const audience = await getAudience(video.VideoChannel.Account.Actor, undefined, video.privacy === VideoPrivacy.PUBLIC) | 126 | const audience = await getAudience(video.VideoChannel.Account.Actor, undefined, video.privacy === VideoPrivacy.PUBLIC) |
122 | const videoObject = audiencify(videoAll.toActivityPubObject(), audience) | 127 | const videoObject = audiencify(video.toActivityPubObject(), audience) |
123 | 128 | ||
124 | if (req.path.endsWith('/activity')) { | 129 | if (req.path.endsWith('/activity')) { |
125 | const data = await createActivityData(video.url, video.VideoChannel.Account.Actor, videoObject, undefined, audience) | 130 | const data = await createActivityData(video.url, video.VideoChannel.Account.Actor, videoObject, undefined, audience) |
@@ -139,41 +144,45 @@ async function videoAnnounceController (req: express.Request, res: express.Respo | |||
139 | async function videoAnnouncesController (req: express.Request, res: express.Response, next: express.NextFunction) { | 144 | async function videoAnnouncesController (req: express.Request, res: express.Response, next: express.NextFunction) { |
140 | const video: VideoModel = res.locals.video | 145 | const video: VideoModel = res.locals.video |
141 | 146 | ||
142 | // We need more attributes | 147 | const handler = async (start: number, count: number) => { |
143 | const videoAll = await VideoModel.loadAndPopulateAll(video.id) | 148 | const result = await VideoShareModel.listAndCountByVideoId(video.id, start, count) |
144 | const object = videoAll.toAnnouncesActivityPubObject() | 149 | return { |
150 | total: result.count, | ||
151 | data: result.rows.map(r => r.url) | ||
152 | } | ||
153 | } | ||
154 | const json = await activityPubCollectionPagination(getVideoSharesActivityPubUrl(video), handler, req.query.page) | ||
145 | 155 | ||
146 | return activityPubResponse(activityPubContextify(object), res) | 156 | return activityPubResponse(activityPubContextify(json), res) |
147 | } | 157 | } |
148 | 158 | ||
149 | async function videoLikesController (req: express.Request, res: express.Response, next: express.NextFunction) { | 159 | async function videoLikesController (req: express.Request, res: express.Response, next: express.NextFunction) { |
150 | const video: VideoModel = res.locals.video | 160 | const video: VideoModel = res.locals.video |
161 | const json = await videoRates(req, 'like', video, getVideoLikesActivityPubUrl(video)) | ||
151 | 162 | ||
152 | // We need more attributes | 163 | return activityPubResponse(activityPubContextify(json), res) |
153 | const videoAll = await VideoModel.loadAndPopulateAll(video.id) | ||
154 | const { likesObject } = videoAll.toRatesActivityPubObjects() | ||
155 | |||
156 | return activityPubResponse(activityPubContextify(likesObject), res) | ||
157 | } | 164 | } |
158 | 165 | ||
159 | async function videoDislikesController (req: express.Request, res: express.Response, next: express.NextFunction) { | 166 | async function videoDislikesController (req: express.Request, res: express.Response, next: express.NextFunction) { |
160 | const video: VideoModel = res.locals.video | 167 | const video: VideoModel = res.locals.video |
168 | const json = await videoRates(req, 'dislike', video, getVideoDislikesActivityPubUrl(video)) | ||
161 | 169 | ||
162 | // We need more attributes | 170 | return activityPubResponse(activityPubContextify(json), res) |
163 | const videoAll = await VideoModel.loadAndPopulateAll(video.id) | ||
164 | const { dislikesObject } = videoAll.toRatesActivityPubObjects() | ||
165 | |||
166 | return activityPubResponse(activityPubContextify(dislikesObject), res) | ||
167 | } | 171 | } |
168 | 172 | ||
169 | async function videoCommentsController (req: express.Request, res: express.Response, next: express.NextFunction) { | 173 | async function videoCommentsController (req: express.Request, res: express.Response, next: express.NextFunction) { |
170 | const video: VideoModel = res.locals.video | 174 | const video: VideoModel = res.locals.video |
171 | 175 | ||
172 | // We need more attributes | 176 | const handler = async (start: number, count: number) => { |
173 | const videoAll = await VideoModel.loadAndPopulateAll(video.id) | 177 | const result = await VideoCommentModel.listAndCountByVideoId(video.id, start, count) |
174 | const commentsObject = videoAll.toCommentsActivityPubObject() | 178 | return { |
179 | total: result.count, | ||
180 | data: result.rows.map(r => r.url) | ||
181 | } | ||
182 | } | ||
183 | const json = await activityPubCollectionPagination(getVideoCommentsActivityPubUrl(video), handler, req.query.page) | ||
175 | 184 | ||
176 | return activityPubResponse(activityPubContextify(commentsObject), res) | 185 | return activityPubResponse(activityPubContextify(json), res) |
177 | } | 186 | } |
178 | 187 | ||
179 | async function videoChannelController (req: express.Request, res: express.Response, next: express.NextFunction) { | 188 | async function videoChannelController (req: express.Request, res: express.Response, next: express.NextFunction) { |
@@ -216,23 +225,28 @@ async function videoCommentController (req: express.Request, res: express.Respon | |||
216 | // --------------------------------------------------------------------------- | 225 | // --------------------------------------------------------------------------- |
217 | 226 | ||
218 | async function actorFollowing (req: express.Request, actor: ActorModel) { | 227 | async function actorFollowing (req: express.Request, actor: ActorModel) { |
219 | const page = req.query.page || 1 | 228 | const handler = (start: number, count: number) => { |
220 | const { start, count } = pageToStartAndCount(page, ACTIVITY_PUB.COLLECTION_ITEMS_PER_PAGE) | 229 | return ActorFollowModel.listAcceptedFollowingUrlsForApi([ actor.id ], undefined, start, count) |
230 | } | ||
221 | 231 | ||
222 | const result = await ActorFollowModel.listAcceptedFollowingUrlsForApi([ actor.id ], undefined, start, count) | 232 | return activityPubCollectionPagination(CONFIG.WEBSERVER.URL + req.url, handler, req.query.page) |
223 | return activityPubCollectionPagination(CONFIG.WEBSERVER.URL + req.url, page, result) | ||
224 | } | 233 | } |
225 | 234 | ||
226 | async function actorFollowers (req: express.Request, actor: ActorModel) { | 235 | async function actorFollowers (req: express.Request, actor: ActorModel) { |
227 | const page = req.query.page || 1 | 236 | const handler = (start: number, count: number) => { |
228 | const { start, count } = pageToStartAndCount(page, ACTIVITY_PUB.COLLECTION_ITEMS_PER_PAGE) | 237 | return ActorFollowModel.listAcceptedFollowerUrlsForApi([ actor.id ], undefined, start, count) |
238 | } | ||
229 | 239 | ||
230 | const result = await ActorFollowModel.listAcceptedFollowerUrlsForApi([ actor.id ], undefined, start, count) | 240 | return activityPubCollectionPagination(CONFIG.WEBSERVER.URL + req.url, handler, req.query.page) |
231 | return activityPubCollectionPagination(CONFIG.WEBSERVER.URL + req.url, page, result) | ||
232 | } | 241 | } |
233 | 242 | ||
234 | function activityPubResponse (data: any, res: express.Response) { | 243 | function videoRates (req: express.Request, rateType: VideoRateType, video: VideoModel, url: string) { |
235 | return res.type('application/activity+json; charset=utf-8') | 244 | const handler = async (start: number, count: number) => { |
236 | .json(data) | 245 | const result = await AccountVideoRateModel.listAndCountAccountUrlsByVideoId(rateType, video.id, start, count) |
237 | .end() | 246 | return { |
247 | total: result.count, | ||
248 | data: result.rows.map(r => r.Account.Actor.url) | ||
249 | } | ||
250 | } | ||
251 | return activityPubCollectionPagination(url, handler, req.query.page) | ||
238 | } | 252 | } |
diff --git a/server/controllers/activitypub/outbox.ts b/server/controllers/activitypub/outbox.ts index c9e087a13..97bf9b052 100644 --- a/server/controllers/activitypub/outbox.ts +++ b/server/controllers/activitypub/outbox.ts | |||
@@ -1,16 +1,15 @@ | |||
1 | import * as express from 'express' | 1 | import * as express from 'express' |
2 | import { Activity } from '../../../shared/models/activitypub/activity' | 2 | import { Activity } from '../../../shared/models/activitypub/activity' |
3 | import { VideoPrivacy } from '../../../shared/models/videos' | 3 | import { VideoPrivacy } from '../../../shared/models/videos' |
4 | import { activityPubCollectionPagination } from '../../helpers/activitypub' | 4 | import { activityPubCollectionPagination, activityPubContextify } from '../../helpers/activitypub' |
5 | import { pageToStartAndCount } from '../../helpers/core-utils' | ||
6 | import { logger } from '../../helpers/logger' | 5 | import { logger } from '../../helpers/logger' |
7 | import { ACTIVITY_PUB } from '../../initializers/constants' | ||
8 | import { announceActivityData, createActivityData } from '../../lib/activitypub/send' | 6 | import { announceActivityData, createActivityData } from '../../lib/activitypub/send' |
9 | import { buildAudience } from '../../lib/activitypub/audience' | 7 | import { buildAudience } from '../../lib/activitypub/audience' |
10 | import { asyncMiddleware, localAccountValidator } from '../../middlewares' | 8 | import { asyncMiddleware, localAccountValidator } from '../../middlewares' |
11 | import { AccountModel } from '../../models/account/account' | 9 | import { AccountModel } from '../../models/account/account' |
12 | import { ActorModel } from '../../models/activitypub/actor' | 10 | import { ActorModel } from '../../models/activitypub/actor' |
13 | import { VideoModel } from '../../models/video/video' | 11 | import { VideoModel } from '../../models/video/video' |
12 | import { activityPubResponse } from './utils' | ||
14 | 13 | ||
15 | const outboxRouter = express.Router() | 14 | const outboxRouter = express.Router() |
16 | 15 | ||
@@ -30,10 +29,17 @@ export { | |||
30 | async function outboxController (req: express.Request, res: express.Response, next: express.NextFunction) { | 29 | async function outboxController (req: express.Request, res: express.Response, next: express.NextFunction) { |
31 | const account: AccountModel = res.locals.account | 30 | const account: AccountModel = res.locals.account |
32 | const actor = account.Actor | 31 | const actor = account.Actor |
32 | const actorOutboxUrl = account.Actor.url + '/outbox' | ||
33 | |||
34 | logger.info('Receiving outbox request for %s.', actorOutboxUrl) | ||
33 | 35 | ||
34 | const page = req.query.page || 1 | 36 | const handler = (start: number, count: number) => buildActivities(actor, start, count) |
35 | const { start, count } = pageToStartAndCount(page, ACTIVITY_PUB.COLLECTION_ITEMS_PER_PAGE) | 37 | const json = await activityPubCollectionPagination(actorOutboxUrl, handler, req.query.page) |
38 | |||
39 | return activityPubResponse(activityPubContextify(json), res) | ||
40 | } | ||
36 | 41 | ||
42 | async function buildActivities (actor: ActorModel, start: number, count: number) { | ||
37 | const data = await VideoModel.listAllAndSharedByActorForOutbox(actor.id, start, count) | 43 | const data = await VideoModel.listAllAndSharedByActorForOutbox(actor.id, start, count) |
38 | const activities: Activity[] = [] | 44 | const activities: Activity[] = [] |
39 | 45 | ||
@@ -62,14 +68,8 @@ async function outboxController (req: express.Request, res: express.Response, ne | |||
62 | } | 68 | } |
63 | } | 69 | } |
64 | 70 | ||
65 | const newResult = { | 71 | return { |
66 | data: activities, | 72 | data: activities, |
67 | total: data.total | 73 | total: data.total |
68 | } | 74 | } |
69 | const actorOutboxUrl = account.Actor.url + '/outbox' | ||
70 | const json = activityPubCollectionPagination(actorOutboxUrl, page, newResult) | ||
71 | |||
72 | logger.info('Receiving outbox request for %s.', actorOutboxUrl) | ||
73 | |||
74 | return res.json(json).end() | ||
75 | } | 75 | } |
diff --git a/server/controllers/activitypub/utils.ts b/server/controllers/activitypub/utils.ts new file mode 100644 index 000000000..599cf48ab --- /dev/null +++ b/server/controllers/activitypub/utils.ts | |||
@@ -0,0 +1,11 @@ | |||
1 | import * as express from 'express' | ||
2 | |||
3 | function activityPubResponse (data: any, res: express.Response) { | ||
4 | return res.type('application/activity+json; charset=utf-8') | ||
5 | .json(data) | ||
6 | .end() | ||
7 | } | ||
8 | |||
9 | export { | ||
10 | activityPubResponse | ||
11 | } | ||
diff --git a/server/helpers/activitypub.ts b/server/helpers/activitypub.ts index 1934fa0f0..d1f3ec02d 100644 --- a/server/helpers/activitypub.ts +++ b/server/helpers/activitypub.ts | |||
@@ -1,8 +1,11 @@ | |||
1 | import * as Bluebird from 'bluebird' | ||
2 | import * as validator from 'validator' | ||
1 | import { ResultList } from '../../shared/models' | 3 | import { ResultList } from '../../shared/models' |
2 | import { Activity, ActivityPubActor } from '../../shared/models/activitypub' | 4 | import { Activity, ActivityPubActor } from '../../shared/models/activitypub' |
3 | import { ACTIVITY_PUB } from '../initializers' | 5 | import { ACTIVITY_PUB } from '../initializers' |
4 | import { ActorModel } from '../models/activitypub/actor' | 6 | import { ActorModel } from '../models/activitypub/actor' |
5 | import { signObject } from './peertube-crypto' | 7 | import { signObject } from './peertube-crypto' |
8 | import { pageToStartAndCount } from './core-utils' | ||
6 | 9 | ||
7 | function activityPubContextify <T> (data: T) { | 10 | function activityPubContextify <T> (data: T) { |
8 | return Object.assign(data,{ | 11 | return Object.assign(data,{ |
@@ -44,16 +47,23 @@ function activityPubContextify <T> (data: T) { | |||
44 | }) | 47 | }) |
45 | } | 48 | } |
46 | 49 | ||
47 | function activityPubCollection (url: string, results: any[]) { | 50 | type ActivityPubCollectionPaginationHandler = (start: number, count: number) => Bluebird<ResultList<any>> | Promise<ResultList<any>> |
48 | return { | 51 | async function activityPubCollectionPagination (url: string, handler: ActivityPubCollectionPaginationHandler, page?: any) { |
49 | id: url, | 52 | if (!page || !validator.isInt(page)) { |
50 | type: 'OrderedCollection', | 53 | // We just display the first page URL, we only need the total items |
51 | totalItems: results.length, | 54 | const result = await handler(0, 1) |
52 | orderedItems: results | 55 | |
56 | return { | ||
57 | id: url, | ||
58 | type: 'OrderedCollection', | ||
59 | totalItems: result.total, | ||
60 | first: url + '?page=1' | ||
61 | } | ||
53 | } | 62 | } |
54 | } | ||
55 | 63 | ||
56 | function activityPubCollectionPagination (url: string, page: any, result: ResultList<any>) { | 64 | const { start, count } = pageToStartAndCount(page, ACTIVITY_PUB.COLLECTION_ITEMS_PER_PAGE) |
65 | const result = await handler(start, count) | ||
66 | |||
57 | let next: string | 67 | let next: string |
58 | let prev: string | 68 | let prev: string |
59 | 69 | ||
@@ -69,27 +79,16 @@ function activityPubCollectionPagination (url: string, page: any, result: Result | |||
69 | prev = url + '?page=' + (page - 1) | 79 | prev = url + '?page=' + (page - 1) |
70 | } | 80 | } |
71 | 81 | ||
72 | const orderedCollectionPagination = { | 82 | return { |
73 | id: url + '?page=' + page, | 83 | id: url + '?page=' + page, |
74 | type: 'OrderedCollectionPage', | 84 | type: 'OrderedCollectionPage', |
75 | prev, | 85 | prev, |
76 | next, | 86 | next, |
77 | partOf: url, | 87 | partOf: url, |
78 | orderedItems: result.data | 88 | orderedItems: result.data, |
79 | } | 89 | totalItems: result.total |
80 | |||
81 | if (page === 1) { | ||
82 | return activityPubContextify({ | ||
83 | id: url, | ||
84 | type: 'OrderedCollection', | ||
85 | totalItems: result.total, | ||
86 | first: orderedCollectionPagination | ||
87 | }) | ||
88 | } else { | ||
89 | orderedCollectionPagination['totalItems'] = result.total | ||
90 | } | 90 | } |
91 | 91 | ||
92 | return orderedCollectionPagination | ||
93 | } | 92 | } |
94 | 93 | ||
95 | function buildSignedActivity (byActor: ActorModel, data: Object) { | 94 | function buildSignedActivity (byActor: ActorModel, data: Object) { |
@@ -110,6 +109,5 @@ export { | |||
110 | getActorUrl, | 109 | getActorUrl, |
111 | activityPubContextify, | 110 | activityPubContextify, |
112 | activityPubCollectionPagination, | 111 | activityPubCollectionPagination, |
113 | activityPubCollection, | ||
114 | buildSignedActivity | 112 | buildSignedActivity |
115 | } | 113 | } |
diff --git a/server/lib/activitypub/crawl.ts b/server/lib/activitypub/crawl.ts new file mode 100644 index 000000000..7305b3969 --- /dev/null +++ b/server/lib/activitypub/crawl.ts | |||
@@ -0,0 +1,40 @@ | |||
1 | import { ACTIVITY_PUB, JOB_REQUEST_TIMEOUT } from '../../initializers' | ||
2 | import { doRequest } from '../../helpers/requests' | ||
3 | import { logger } from '../../helpers/logger' | ||
4 | |||
5 | async function crawlCollectionPage <T> (uri: string, handler: (items: T[]) => Promise<any>) { | ||
6 | logger.info('Crawling ActivityPub data on %s.', uri) | ||
7 | |||
8 | const options = { | ||
9 | method: 'GET', | ||
10 | uri, | ||
11 | json: true, | ||
12 | activityPub: true, | ||
13 | timeout: JOB_REQUEST_TIMEOUT | ||
14 | } | ||
15 | |||
16 | const response = await doRequest(options) | ||
17 | const firstBody = response.body | ||
18 | |||
19 | let limit = ACTIVITY_PUB.FETCH_PAGE_LIMIT | ||
20 | let i = 0 | ||
21 | let nextLink = firstBody.first | ||
22 | while (nextLink && i < limit) { | ||
23 | options.uri = nextLink | ||
24 | |||
25 | const { body } = await doRequest(options) | ||
26 | nextLink = body.next | ||
27 | i++ | ||
28 | |||
29 | if (Array.isArray(body.orderedItems)) { | ||
30 | const items = body.orderedItems | ||
31 | logger.info('Processing %i ActivityPub items for %s.', items.length, nextLink) | ||
32 | |||
33 | await handler(items) | ||
34 | } | ||
35 | } | ||
36 | } | ||
37 | |||
38 | export { | ||
39 | crawlCollectionPage | ||
40 | } | ||
diff --git a/server/lib/activitypub/video-comments.ts b/server/lib/activitypub/video-comments.ts index 60c9179a6..fd03710c2 100644 --- a/server/lib/activitypub/video-comments.ts +++ b/server/lib/activitypub/video-comments.ts | |||
@@ -37,7 +37,7 @@ async function videoCommentActivityObjectToDBAttributes (video: VideoModel, acto | |||
37 | } | 37 | } |
38 | } | 38 | } |
39 | 39 | ||
40 | async function addVideoComments (instance: VideoModel, commentUrls: string[]) { | 40 | async function addVideoComments (commentUrls: string[], instance: VideoModel) { |
41 | for (const commentUrl of commentUrls) { | 41 | for (const commentUrl of commentUrls) { |
42 | await addVideoComment(instance, commentUrl) | 42 | await addVideoComment(instance, commentUrl) |
43 | } | 43 | } |
diff --git a/server/lib/activitypub/videos.ts b/server/lib/activitypub/videos.ts index dbd7385a4..be6794cef 100644 --- a/server/lib/activitypub/videos.ts +++ b/server/lib/activitypub/videos.ts | |||
@@ -20,6 +20,7 @@ import { VideoFileModel } from '../../models/video/video-file' | |||
20 | import { VideoShareModel } from '../../models/video/video-share' | 20 | import { VideoShareModel } from '../../models/video/video-share' |
21 | import { getOrCreateActorAndServerAndModel } from './actor' | 21 | import { getOrCreateActorAndServerAndModel } from './actor' |
22 | import { addVideoComments } from './video-comments' | 22 | import { addVideoComments } from './video-comments' |
23 | import { crawlCollectionPage } from './crawl' | ||
23 | 24 | ||
24 | function fetchRemoteVideoPreview (video: VideoModel, reject: Function) { | 25 | function fetchRemoteVideoPreview (video: VideoModel, reject: Function) { |
25 | const host = video.VideoChannel.Account.Actor.Server.host | 26 | const host = video.VideoChannel.Account.Actor.Server.host |
@@ -216,25 +217,17 @@ async function getOrCreateAccountAndVideoAndChannel (videoObject: VideoTorrentOb | |||
216 | const video = await retryTransactionWrapper(getOrCreateVideo, options) | 217 | const video = await retryTransactionWrapper(getOrCreateVideo, options) |
217 | 218 | ||
218 | // Process outside the transaction because we could fetch remote data | 219 | // Process outside the transaction because we could fetch remote data |
219 | if (videoObject.likes && Array.isArray(videoObject.likes.orderedItems)) { | 220 | logger.info('Adding likes of video %s.', video.uuid) |
220 | logger.info('Adding likes of video %s.', video.uuid) | 221 | await crawlCollectionPage<string>(videoObject.likes, (items) => createRates(items, video, 'like')) |
221 | await createRates(videoObject.likes.orderedItems, video, 'like') | ||
222 | } | ||
223 | 222 | ||
224 | if (videoObject.dislikes && Array.isArray(videoObject.dislikes.orderedItems)) { | 223 | logger.info('Adding dislikes of video %s.', video.uuid) |
225 | logger.info('Adding dislikes of video %s.', video.uuid) | 224 | await crawlCollectionPage<string>(videoObject.dislikes, (items) => createRates(items, video, 'dislike')) |
226 | await createRates(videoObject.dislikes.orderedItems, video, 'dislike') | ||
227 | } | ||
228 | 225 | ||
229 | if (videoObject.shares && Array.isArray(videoObject.shares.orderedItems)) { | 226 | logger.info('Adding shares of video %s.', video.uuid) |
230 | logger.info('Adding shares of video %s.', video.uuid) | 227 | await crawlCollectionPage<string>(videoObject.shares, (items) => addVideoShares(items, video)) |
231 | await addVideoShares(video, videoObject.shares.orderedItems) | ||
232 | } | ||
233 | 228 | ||
234 | if (videoObject.comments && Array.isArray(videoObject.comments.orderedItems)) { | 229 | logger.info('Adding comments of video %s.', video.uuid) |
235 | logger.info('Adding comments of video %s.', video.uuid) | 230 | await crawlCollectionPage<string>(videoObject.comments, (items) => addVideoComments(items, video)) |
236 | await addVideoComments(video, videoObject.comments.orderedItems) | ||
237 | } | ||
238 | 231 | ||
239 | return { actor, channelActor, video } | 232 | return { actor, channelActor, video } |
240 | } | 233 | } |
@@ -266,7 +259,7 @@ async function createRates (actorUrls: string[], video: VideoModel, rate: VideoR | |||
266 | return | 259 | return |
267 | } | 260 | } |
268 | 261 | ||
269 | async function addVideoShares (instance: VideoModel, shareUrls: string[]) { | 262 | async function addVideoShares (shareUrls: string[], instance: VideoModel) { |
270 | for (const shareUrl of shareUrls) { | 263 | for (const shareUrl of shareUrls) { |
271 | // Fetch url | 264 | // Fetch url |
272 | const { body } = await doRequest({ | 265 | const { body } = await doRequest({ |
diff --git a/server/lib/job-queue/handlers/activitypub-http-fetcher.ts b/server/lib/job-queue/handlers/activitypub-http-fetcher.ts index 4683beb2f..10c0e606f 100644 --- a/server/lib/job-queue/handlers/activitypub-http-fetcher.ts +++ b/server/lib/job-queue/handlers/activitypub-http-fetcher.ts | |||
@@ -1,9 +1,9 @@ | |||
1 | import * as kue from 'kue' | 1 | import * as kue from 'kue' |
2 | import { logger } from '../../../helpers/logger' | 2 | import { logger } from '../../../helpers/logger' |
3 | import { doRequest } from '../../../helpers/requests' | ||
4 | import { ACTIVITY_PUB, JOB_REQUEST_TIMEOUT } from '../../../initializers' | ||
5 | import { processActivities } from '../../activitypub/process' | 3 | import { processActivities } from '../../activitypub/process' |
6 | import { ActivitypubHttpBroadcastPayload } from './activitypub-http-broadcast' | 4 | import { ActivitypubHttpBroadcastPayload } from './activitypub-http-broadcast' |
5 | import { crawlCollectionPage } from '../../activitypub/crawl' | ||
6 | import { Activity } from '../../../../shared/models/activitypub' | ||
7 | 7 | ||
8 | export type ActivitypubHttpFetcherPayload = { | 8 | export type ActivitypubHttpFetcherPayload = { |
9 | uris: string[] | 9 | uris: string[] |
@@ -14,46 +14,8 @@ async function processActivityPubHttpFetcher (job: kue.Job) { | |||
14 | 14 | ||
15 | const payload = job.data as ActivitypubHttpBroadcastPayload | 15 | const payload = job.data as ActivitypubHttpBroadcastPayload |
16 | 16 | ||
17 | const options = { | ||
18 | method: 'GET', | ||
19 | uri: '', | ||
20 | json: true, | ||
21 | activityPub: true, | ||
22 | timeout: JOB_REQUEST_TIMEOUT | ||
23 | } | ||
24 | |||
25 | for (const uri of payload.uris) { | 17 | for (const uri of payload.uris) { |
26 | options.uri = uri | 18 | await crawlCollectionPage<Activity>(uri, (items) => processActivities(items)) |
27 | logger.info('Fetching ActivityPub data on %s.', uri) | ||
28 | |||
29 | const response = await doRequest(options) | ||
30 | const firstBody = response.body | ||
31 | |||
32 | if (firstBody.first && Array.isArray(firstBody.first.orderedItems)) { | ||
33 | const activities = firstBody.first.orderedItems | ||
34 | |||
35 | logger.info('Processing %i items ActivityPub fetcher for %s.', activities.length, options.uri) | ||
36 | |||
37 | await processActivities(activities) | ||
38 | } | ||
39 | |||
40 | let limit = ACTIVITY_PUB.FETCH_PAGE_LIMIT | ||
41 | let i = 0 | ||
42 | let nextLink = firstBody.first.next | ||
43 | while (nextLink && i < limit) { | ||
44 | options.uri = nextLink | ||
45 | |||
46 | const { body } = await doRequest(options) | ||
47 | nextLink = body.next | ||
48 | i++ | ||
49 | |||
50 | if (Array.isArray(body.orderedItems)) { | ||
51 | const activities = body.orderedItems | ||
52 | logger.info('Processing %i items ActivityPub fetcher for %s.', activities.length, options.uri) | ||
53 | |||
54 | await processActivities(activities) | ||
55 | } | ||
56 | } | ||
57 | } | 19 | } |
58 | } | 20 | } |
59 | 21 | ||
diff --git a/server/models/account/account-video-rate.ts b/server/models/account/account-video-rate.ts index e969e4a43..508ab814f 100644 --- a/server/models/account/account-video-rate.ts +++ b/server/models/account/account-video-rate.ts | |||
@@ -6,6 +6,7 @@ import { VideoRateType } from '../../../shared/models/videos' | |||
6 | import { VIDEO_RATE_TYPES } from '../../initializers' | 6 | import { VIDEO_RATE_TYPES } from '../../initializers' |
7 | import { VideoModel } from '../video/video' | 7 | import { VideoModel } from '../video/video' |
8 | import { AccountModel } from './account' | 8 | import { AccountModel } from './account' |
9 | import { ActorModel } from '../activitypub/actor' | ||
9 | 10 | ||
10 | /* | 11 | /* |
11 | Account rates per video. | 12 | Account rates per video. |
@@ -66,4 +67,32 @@ export class AccountVideoRateModel extends Model<AccountVideoRateModel> { | |||
66 | 67 | ||
67 | return AccountVideoRateModel.findOne(options) | 68 | return AccountVideoRateModel.findOne(options) |
68 | } | 69 | } |
70 | |||
71 | static listAndCountAccountUrlsByVideoId (rateType: VideoRateType, videoId: number, start: number, count: number, t?: Transaction) { | ||
72 | const query = { | ||
73 | start, | ||
74 | count, | ||
75 | where: { | ||
76 | videoId, | ||
77 | type: rateType | ||
78 | }, | ||
79 | transaction: t, | ||
80 | include: [ | ||
81 | { | ||
82 | attributes: [ 'actorId' ], | ||
83 | model: AccountModel.unscoped(), | ||
84 | required: true, | ||
85 | include: [ | ||
86 | { | ||
87 | attributes: [ 'url' ], | ||
88 | model: ActorModel.unscoped(), | ||
89 | required: true | ||
90 | } | ||
91 | ] | ||
92 | } | ||
93 | ] | ||
94 | } | ||
95 | |||
96 | return AccountVideoRateModel.findAndCountAll(query) | ||
97 | } | ||
69 | } | 98 | } |
diff --git a/server/models/activitypub/actor-follow.ts b/server/models/activitypub/actor-follow.ts index c97f4cead..b8ce6de1d 100644 --- a/server/models/activitypub/actor-follow.ts +++ b/server/models/activitypub/actor-follow.ts | |||
@@ -335,8 +335,7 @@ export class ActorFollowModel extends Model<ActorFollowModel> { | |||
335 | tasks.push(ActorFollowModel.sequelize.query(query, options)) | 335 | tasks.push(ActorFollowModel.sequelize.query(query, options)) |
336 | } | 336 | } |
337 | 337 | ||
338 | const [ followers, [ { total } ] ] = await | 338 | const [ followers, [ { total } ] ] = await Promise.all(tasks) |
339 | Promise.all(tasks) | ||
340 | const urls: string[] = followers.map(f => f.url) | 339 | const urls: string[] = followers.map(f => f.url) |
341 | 340 | ||
342 | return { | 341 | return { |
diff --git a/server/models/video/video-comment.ts b/server/models/video/video-comment.ts index 5386a10aa..18398905e 100644 --- a/server/models/video/video-comment.ts +++ b/server/models/video/video-comment.ts | |||
@@ -326,6 +326,20 @@ export class VideoCommentModel extends Model<VideoCommentModel> { | |||
326 | .findAll(query) | 326 | .findAll(query) |
327 | } | 327 | } |
328 | 328 | ||
329 | static listAndCountByVideoId (videoId: number, start: number, count: number, t?: Sequelize.Transaction, order: 'ASC' | 'DESC' = 'ASC') { | ||
330 | const query = { | ||
331 | order: [ [ 'createdAt', order ] ], | ||
332 | start, | ||
333 | count, | ||
334 | where: { | ||
335 | videoId | ||
336 | }, | ||
337 | transaction: t | ||
338 | } | ||
339 | |||
340 | return VideoCommentModel.findAndCountAll(query) | ||
341 | } | ||
342 | |||
329 | static async getStats () { | 343 | static async getStats () { |
330 | const totalLocalVideoComments = await VideoCommentModel.count({ | 344 | const totalLocalVideoComments = await VideoCommentModel.count({ |
331 | include: [ | 345 | include: [ |
diff --git a/server/models/video/video-share.ts b/server/models/video/video-share.ts index 602cc69b9..adadf5dea 100644 --- a/server/models/video/video-share.ts +++ b/server/models/video/video-share.ts | |||
@@ -187,4 +187,17 @@ export class VideoShareModel extends Model<VideoShareModel> { | |||
187 | .findAll(query) | 187 | .findAll(query) |
188 | .then(res => res.map(r => r.Actor)) | 188 | .then(res => res.map(r => r.Actor)) |
189 | } | 189 | } |
190 | |||
191 | static listAndCountByVideoId (videoId: number, start: number, count: number, t?: Sequelize.Transaction) { | ||
192 | const query = { | ||
193 | start, | ||
194 | count, | ||
195 | where: { | ||
196 | videoId | ||
197 | }, | ||
198 | transaction: t | ||
199 | } | ||
200 | |||
201 | return VideoShareModel.findAndCountAll(query) | ||
202 | } | ||
190 | } | 203 | } |
diff --git a/server/models/video/video.ts b/server/models/video/video.ts index 012a758ee..f4689fe12 100644 --- a/server/models/video/video.ts +++ b/server/models/video/video.ts | |||
@@ -29,7 +29,7 @@ import { VideoPrivacy, VideoResolution } from '../../../shared' | |||
29 | import { VideoTorrentObject } from '../../../shared/models/activitypub/objects' | 29 | import { VideoTorrentObject } from '../../../shared/models/activitypub/objects' |
30 | import { Video, VideoDetails, VideoFile } from '../../../shared/models/videos' | 30 | import { Video, VideoDetails, VideoFile } from '../../../shared/models/videos' |
31 | import { VideoFilter } from '../../../shared/models/videos/video-query.type' | 31 | import { VideoFilter } from '../../../shared/models/videos/video-query.type' |
32 | import { activityPubCollection } from '../../helpers/activitypub' | 32 | import { activityPubCollectionPagination } from '../../helpers/activitypub' |
33 | import { | 33 | import { |
34 | createTorrentPromise, | 34 | createTorrentPromise, |
35 | peertubeTruncate, | 35 | peertubeTruncate, |
@@ -602,18 +602,6 @@ export class VideoModel extends Model<VideoModel> { | |||
602 | attributes: [ 'id', 'url' ], | 602 | attributes: [ 'id', 'url' ], |
603 | model: VideoShareModel.unscoped(), | 603 | model: VideoShareModel.unscoped(), |
604 | required: false, | 604 | required: false, |
605 | where: { | ||
606 | [Sequelize.Op.and]: [ | ||
607 | { | ||
608 | id: { | ||
609 | [Sequelize.Op.not]: null | ||
610 | } | ||
611 | }, | ||
612 | { | ||
613 | actorId | ||
614 | } | ||
615 | ] | ||
616 | }, | ||
617 | include: [ | 605 | include: [ |
618 | { | 606 | { |
619 | attributes: [ 'id', 'url' ], | 607 | attributes: [ 'id', 'url' ], |
@@ -644,35 +632,6 @@ export class VideoModel extends Model<VideoModel> { | |||
644 | } | 632 | } |
645 | ] | 633 | ] |
646 | }, | 634 | }, |
647 | { | ||
648 | attributes: [ 'type' ], | ||
649 | model: AccountVideoRateModel, | ||
650 | required: false, | ||
651 | include: [ | ||
652 | { | ||
653 | attributes: [ 'id' ], | ||
654 | model: AccountModel.unscoped(), | ||
655 | include: [ | ||
656 | { | ||
657 | attributes: [ 'url' ], | ||
658 | model: ActorModel.unscoped(), | ||
659 | include: [ | ||
660 | { | ||
661 | attributes: [ 'host' ], | ||
662 | model: ServerModel, | ||
663 | required: false | ||
664 | } | ||
665 | ] | ||
666 | } | ||
667 | ] | ||
668 | } | ||
669 | ] | ||
670 | }, | ||
671 | { | ||
672 | attributes: [ 'url' ], | ||
673 | model: VideoCommentModel, | ||
674 | required: false | ||
675 | }, | ||
676 | VideoFileModel, | 635 | VideoFileModel, |
677 | TagModel | 636 | TagModel |
678 | ] | 637 | ] |
@@ -897,26 +856,6 @@ export class VideoModel extends Model<VideoModel> { | |||
897 | .findOne(options) | 856 | .findOne(options) |
898 | } | 857 | } |
899 | 858 | ||
900 | static loadAndPopulateAll (id: number) { | ||
901 | const options = { | ||
902 | order: [ [ 'Tags', 'name', 'ASC' ] ], | ||
903 | where: { | ||
904 | id | ||
905 | } | ||
906 | } | ||
907 | |||
908 | return VideoModel | ||
909 | .scope([ | ||
910 | ScopeNames.WITH_RATES, | ||
911 | ScopeNames.WITH_SHARES, | ||
912 | ScopeNames.WITH_TAGS, | ||
913 | ScopeNames.WITH_FILES, | ||
914 | ScopeNames.WITH_ACCOUNT_DETAILS, | ||
915 | ScopeNames.WITH_COMMENTS | ||
916 | ]) | ||
917 | .findOne(options) | ||
918 | } | ||
919 | |||
920 | static async getStats () { | 859 | static async getStats () { |
921 | const totalLocalVideos = await VideoModel.count({ | 860 | const totalLocalVideos = await VideoModel.count({ |
922 | where: { | 861 | where: { |
@@ -1203,25 +1142,6 @@ export class VideoModel extends Model<VideoModel> { | |||
1203 | } | 1142 | } |
1204 | } | 1143 | } |
1205 | 1144 | ||
1206 | let likesObject | ||
1207 | let dislikesObject | ||
1208 | |||
1209 | if (Array.isArray(this.AccountVideoRates)) { | ||
1210 | const res = this.toRatesActivityPubObjects() | ||
1211 | likesObject = res.likesObject | ||
1212 | dislikesObject = res.dislikesObject | ||
1213 | } | ||
1214 | |||
1215 | let sharesObject | ||
1216 | if (Array.isArray(this.VideoShares)) { | ||
1217 | sharesObject = this.toAnnouncesActivityPubObject() | ||
1218 | } | ||
1219 | |||
1220 | let commentsObject | ||
1221 | if (Array.isArray(this.VideoComments)) { | ||
1222 | commentsObject = this.toCommentsActivityPubObject() | ||
1223 | } | ||
1224 | |||
1225 | const url = [] | 1145 | const url = [] |
1226 | for (const file of this.VideoFiles) { | 1146 | for (const file of this.VideoFiles) { |
1227 | url.push({ | 1147 | url.push({ |
@@ -1280,10 +1200,10 @@ export class VideoModel extends Model<VideoModel> { | |||
1280 | height: THUMBNAILS_SIZE.height | 1200 | height: THUMBNAILS_SIZE.height |
1281 | }, | 1201 | }, |
1282 | url, | 1202 | url, |
1283 | likes: likesObject, | 1203 | likes: getVideoLikesActivityPubUrl(this), |
1284 | dislikes: dislikesObject, | 1204 | dislikes: getVideoDislikesActivityPubUrl(this), |
1285 | shares: sharesObject, | 1205 | shares: getVideoSharesActivityPubUrl(this), |
1286 | comments: commentsObject, | 1206 | comments: getVideoCommentsActivityPubUrl(this), |
1287 | attributedTo: [ | 1207 | attributedTo: [ |
1288 | { | 1208 | { |
1289 | type: 'Person', | 1209 | type: 'Person', |
@@ -1297,44 +1217,6 @@ export class VideoModel extends Model<VideoModel> { | |||
1297 | } | 1217 | } |
1298 | } | 1218 | } |
1299 | 1219 | ||
1300 | toAnnouncesActivityPubObject () { | ||
1301 | const shares: string[] = [] | ||
1302 | |||
1303 | for (const videoShare of this.VideoShares) { | ||
1304 | shares.push(videoShare.url) | ||
1305 | } | ||
1306 | |||
1307 | return activityPubCollection(getVideoSharesActivityPubUrl(this), shares) | ||
1308 | } | ||
1309 | |||
1310 | toCommentsActivityPubObject () { | ||
1311 | const comments: string[] = [] | ||
1312 | |||
1313 | for (const videoComment of this.VideoComments) { | ||
1314 | comments.push(videoComment.url) | ||
1315 | } | ||
1316 | |||
1317 | return activityPubCollection(getVideoCommentsActivityPubUrl(this), comments) | ||
1318 | } | ||
1319 | |||
1320 | toRatesActivityPubObjects () { | ||
1321 | const likes: string[] = [] | ||
1322 | const dislikes: string[] = [] | ||
1323 | |||
1324 | for (const rate of this.AccountVideoRates) { | ||
1325 | if (rate.type === 'like') { | ||
1326 | likes.push(rate.Account.Actor.url) | ||
1327 | } else if (rate.type === 'dislike') { | ||
1328 | dislikes.push(rate.Account.Actor.url) | ||
1329 | } | ||
1330 | } | ||
1331 | |||
1332 | const likesObject = activityPubCollection(getVideoLikesActivityPubUrl(this), likes) | ||
1333 | const dislikesObject = activityPubCollection(getVideoDislikesActivityPubUrl(this), dislikes) | ||
1334 | |||
1335 | return { likesObject, dislikesObject } | ||
1336 | } | ||
1337 | |||
1338 | getTruncatedDescription () { | 1220 | getTruncatedDescription () { |
1339 | if (!this.description) return null | 1221 | if (!this.description) return null |
1340 | 1222 | ||
diff --git a/shared/models/activitypub/objects/video-torrent-object.ts b/shared/models/activitypub/objects/video-torrent-object.ts index 02820a4cb..767b6a2d0 100644 --- a/shared/models/activitypub/objects/video-torrent-object.ts +++ b/shared/models/activitypub/objects/video-torrent-object.ts | |||
@@ -26,10 +26,10 @@ export interface VideoTorrentObject { | |||
26 | support: string | 26 | support: string |
27 | icon: ActivityIconObject | 27 | icon: ActivityIconObject |
28 | url: ActivityUrlObject[] | 28 | url: ActivityUrlObject[] |
29 | likes?: ActivityPubOrderedCollection<string> | 29 | likes: string |
30 | dislikes?: ActivityPubOrderedCollection<string> | 30 | dislikes: string |
31 | shares?: ActivityPubOrderedCollection<string> | 31 | shares: string |
32 | comments?: ActivityPubOrderedCollection<string> | 32 | comments: string |
33 | attributedTo: ActivityPubAttributedTo[] | 33 | attributedTo: ActivityPubAttributedTo[] |
34 | to?: string[] | 34 | to?: string[] |
35 | cc?: string[] | 35 | cc?: string[] |