From 54e740594bc2eacd8026b5d2d6cfdfc06416a65b Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Thu, 18 Jan 2018 14:59:27 +0100 Subject: [PATCH] Avoid too many requests and fetching outbox --- server/controllers/activitypub/outbox.ts | 19 +++++++-- server/lib/activitypub/send/misc.ts | 5 +++ server/models/activitypub/actor-follow.ts | 3 +- server/models/activitypub/actor.ts | 51 ++++++++++++++++++++--- server/models/video/video.ts | 4 +- server/tests/api/server/follows.ts | 10 +---- 6 files changed, 72 insertions(+), 20 deletions(-) diff --git a/server/controllers/activitypub/outbox.ts b/server/controllers/activitypub/outbox.ts index 620f9ee83..ab12a7c4b 100644 --- a/server/controllers/activitypub/outbox.ts +++ b/server/controllers/activitypub/outbox.ts @@ -4,9 +4,11 @@ import { activityPubCollectionPagination } from '../../helpers/activitypub' import { pageToStartAndCount } from '../../helpers/core-utils' import { ACTIVITY_PUB } from '../../initializers/constants' import { announceActivityData, createActivityData } from '../../lib/activitypub/send' +import { buildAudience } from '../../lib/activitypub/send/misc' import { getAnnounceActivityPubUrl } from '../../lib/activitypub/url' import { asyncMiddleware, localAccountValidator } from '../../middlewares' import { AccountModel } from '../../models/account/account' +import { ActorModel } from '../../models/activitypub/actor' import { VideoModel } from '../../models/video/video' const outboxRouter = express.Router() @@ -34,20 +36,29 @@ async function outboxController (req: express.Request, res: express.Response, ne const data = await VideoModel.listAllAndSharedByActorForOutbox(actor.id, start, count) const activities: Activity[] = [] + // Avoid too many SQL requests + const actors = data.data.map(v => v.VideoChannel.Account.Actor) + actors.push(actor) + + const followersMatrix = await ActorModel.getActorsFollowerSharedInboxUrls(actors, undefined) + for (const video of data.data) { const videoObject = video.toActivityPubObject() - const videoChannel = video.VideoChannel + const byActor = video.VideoChannel.Account.Actor + const createActivityAudience = buildAudience(followersMatrix[byActor.id]) + // This is a shared video if (video.VideoShares !== undefined && video.VideoShares.length !== 0) { - const createActivity = await createActivityData(video.url, videoChannel.Account.Actor, videoObject, undefined) + const createActivity = await createActivityData(video.url, byActor, videoObject, undefined, createActivityAudience) + const announceAudience = buildAudience(followersMatrix[actor.id]) const url = getAnnounceActivityPubUrl(video.url, actor) - const announceActivity = await announceActivityData(url, actor, createActivity, undefined) + const announceActivity = await announceActivityData(url, actor, createActivity, undefined, announceAudience) activities.push(announceActivity) } else { - const createActivity = await createActivityData(video.url, videoChannel.Account.Actor, videoObject, undefined) + const createActivity = await createActivityData(video.url, byActor, videoObject, undefined, createActivityAudience) activities.push(createActivity) } diff --git a/server/lib/activitypub/send/misc.ts b/server/lib/activitypub/send/misc.ts index 261586ae4..dc0d3de57 100644 --- a/server/lib/activitypub/send/misc.ts +++ b/server/lib/activitypub/send/misc.ts @@ -143,6 +143,10 @@ async function getActorsInvolvedInVideo (video: VideoModel, t: Transaction) { async function getAudience (actorSender: ActorModel, t: Transaction, isPublic = true) { const followerInboxUrls = await actorSender.getFollowerSharedInboxUrls(t) + return buildAudience(followerInboxUrls, isPublic) +} + +function buildAudience (followerInboxUrls: string[], isPublic = true) { // Thanks Mastodon: https://github.com/tootsuite/mastodon/blob/master/app/lib/activitypub/tag_manager.rb#L47 let to = [] let cc = [] @@ -183,6 +187,7 @@ async function computeUris (toActors: ActorModel[], actorsException: ActorModel[ export { broadcastToFollowers, unicastTo, + buildAudience, getAudience, getOriginVideoAudience, getActorsInvolvedInVideo, diff --git a/server/models/activitypub/actor-follow.ts b/server/models/activitypub/actor-follow.ts index ced481547..416496607 100644 --- a/server/models/activitypub/actor-follow.ts +++ b/server/models/activitypub/actor-follow.ts @@ -375,7 +375,8 @@ export class ActorFollowModel extends Model { score: { [Sequelize.Op.lte]: 0 } - } + }, + logger: false } return ActorFollowModel.findAll(query) diff --git a/server/models/activitypub/actor.ts b/server/models/activitypub/actor.ts index 408d4df23..269149a31 100644 --- a/server/models/activitypub/actor.ts +++ b/server/models/activitypub/actor.ts @@ -167,17 +167,17 @@ export class ActorModel extends Model { }, onDelete: 'cascade' }) - AccountFollowing: ActorFollowModel[] + ActorFollowing: ActorFollowModel[] @HasMany(() => ActorFollowModel, { foreignKey: { name: 'targetActorId', allowNull: false }, - as: 'followers', + as: 'ActorFollowers', onDelete: 'cascade' }) - AccountFollowers: ActorFollowModel[] + ActorFollowers: ActorFollowModel[] @ForeignKey(() => ServerModel) @Column @@ -277,6 +277,45 @@ export class ActorModel extends Model { }) } + static async getActorsFollowerSharedInboxUrls (actors: ActorModel[], t: Sequelize.Transaction) { + const query = { + // attribute: [], + where: { + id: { + [Sequelize.Op.in]: actors.map(a => a.id) + } + }, + include: [ + { + // attributes: [ ], + model: ActorFollowModel.unscoped(), + required: true, + as: 'ActorFollowers', + where: { + state: 'accepted' + }, + include: [ + { + attributes: [ 'sharedInboxUrl' ], + model: ActorModel.unscoped(), + as: 'ActorFollower', + required: true + } + ] + } + ], + transaction: t + } + + const hash: { [ id: number ]: string[] } = {} + const res = await ActorModel.findAll(query) + for (const actor of res) { + hash[actor.id] = actor.ActorFollowers.map(follow => follow.ActorFollower.sharedInboxUrl) + } + + return hash + } + toFormattedJSON () { let avatar: Avatar = null if (this.Avatar) { @@ -347,10 +386,12 @@ export class ActorModel extends Model { attributes: [ 'sharedInboxUrl' ], include: [ { - model: ActorFollowModel, + attribute: [], + model: ActorFollowModel.unscoped(), required: true, - as: 'followers', + as: 'ActorFollowers', where: { + state: 'accepted', targetActorId: this.id } } diff --git a/server/models/video/video.ts b/server/models/video/video.ts index 514edfd9c..0d115367f 100644 --- a/server/models/video/video.ts +++ b/server/models/video/video.ts @@ -67,7 +67,7 @@ enum ScopeNames { '$VideoChannel.Account.Actor.serverId$': null }, { - '$VideoChannel.Account.Actor.followers.actorId$': actorId + '$VideoChannel.Account.Actor.ActorFollowers.actorId$': actorId }, { id: { @@ -106,7 +106,7 @@ enum ScopeNames { { attributes: [ ], model: ActorFollowModel.unscoped(), - as: 'followers', + as: 'ActorFollowers', required: false } ] diff --git a/server/tests/api/server/follows.ts b/server/tests/api/server/follows.ts index ac614d605..c0115e534 100644 --- a/server/tests/api/server/follows.ts +++ b/server/tests/api/server/follows.ts @@ -4,7 +4,7 @@ import * as chai from 'chai' import 'mocha' import { Video, VideoPrivacy } from '../../../../shared/models/videos' import { VideoComment, VideoCommentThreadTree } from '../../../../shared/models/videos/video-comment.model' -import { checkVideoFilesWereRemoved, completeVideoCheck, getVideoChannelsList } from '../../utils' +import { checkVideoFilesWereRemoved, completeVideoCheck } from '../../utils' import { flushAndRunMultipleServers, flushTests, getVideosList, killallServers, ServerInfo, setAccessTokensToServers, uploadVideo, @@ -12,7 +12,7 @@ import { } from '../../utils/index' import { dateIsValid } from '../../utils/miscs/miscs' import { follow, getFollowersListPaginationAndSort, getFollowingListPaginationAndSort, unfollow } from '../../utils/server/follows' -import { expectAccountFollows, getAccountsList } from '../../utils/users/accounts' +import { expectAccountFollows } from '../../utils/users/accounts' import { userLogin } from '../../utils/users/login' import { createUser } from '../../utils/users/users' import { @@ -354,12 +354,6 @@ describe('Test follows', function () { let res = await getVideosList(servers[ 0 ].url) expect(res.body.total).to.equal(1) - res = await getVideoChannelsList(servers[0].url, 0, 1) - expect(res.body.total).to.equal(2) - - res = await getAccountsList(servers[0].url) - expect(res.body.total).to.equal(2) - await checkVideoFilesWereRemoved(video4.uuid, servers[0].serverNumber) }) -- 2.41.0