diff options
author | Chocobozzz <me@florianbigard.com> | 2022-06-17 10:49:37 +0200 |
---|---|---|
committer | Chocobozzz <me@florianbigard.com> | 2022-06-17 11:55:03 +0200 |
commit | 3396e6534592865f184ee2db32a75957c42cb887 (patch) | |
tree | 324aeab92a27f2ffd1e8a16b09f0b5509bf8f477 | |
parent | b9e49a45f5441fd1a2d49a10b88aaf4d425dc992 (diff) | |
download | PeerTube-3396e6534592865f184ee2db32a75957c42cb887.tar.gz PeerTube-3396e6534592865f184ee2db32a75957c42cb887.tar.zst PeerTube-3396e6534592865f184ee2db32a75957c42cb887.zip |
Optimize broadcast job creation
-rw-r--r-- | scripts/simulate-many-viewers.ts | 14 | ||||
-rw-r--r-- | server/lib/activitypub/send/shared/audience-utils.ts | 6 | ||||
-rw-r--r-- | server/lib/activitypub/send/shared/send-utils.ts | 4 | ||||
-rw-r--r-- | server/lib/views/shared/video-viewer-counters.ts | 1 | ||||
-rw-r--r-- | server/models/actor/actor.ts | 61 | ||||
-rw-r--r-- | server/models/video/video-share.ts | 25 |
6 files changed, 49 insertions, 62 deletions
diff --git a/scripts/simulate-many-viewers.ts b/scripts/simulate-many-viewers.ts index 4ec928aa8..fb666c318 100644 --- a/scripts/simulate-many-viewers.ts +++ b/scripts/simulate-many-viewers.ts | |||
@@ -1,6 +1,13 @@ | |||
1 | import Bluebird from 'bluebird' | 1 | import Bluebird from 'bluebird' |
2 | import { wait } from '@shared/core-utils' | 2 | import { wait } from '@shared/core-utils' |
3 | import { createSingleServer, doubleFollow, PeerTubeServer, setAccessTokensToServers, waitJobs } from '@shared/server-commands' | 3 | import { |
4 | createSingleServer, | ||
5 | doubleFollow, | ||
6 | killallServers, | ||
7 | PeerTubeServer, | ||
8 | setAccessTokensToServers, | ||
9 | waitJobs | ||
10 | } from '@shared/server-commands' | ||
4 | 11 | ||
5 | let servers: PeerTubeServer[] | 12 | let servers: PeerTubeServer[] |
6 | const viewers: { xForwardedFor: string }[] = [] | 13 | const viewers: { xForwardedFor: string }[] = [] |
@@ -9,6 +16,7 @@ let videoId: string | |||
9 | run() | 16 | run() |
10 | .then(() => process.exit(0)) | 17 | .then(() => process.exit(0)) |
11 | .catch(err => console.error(err)) | 18 | .catch(err => console.error(err)) |
19 | .finally(() => killallServers(servers)) | ||
12 | 20 | ||
13 | async function run () { | 21 | async function run () { |
14 | await prepare() | 22 | await prepare() |
@@ -69,9 +77,13 @@ async function prepare () { | |||
69 | async function runViewers () { | 77 | async function runViewers () { |
70 | console.log('Will run views of %d viewers.', viewers.length) | 78 | console.log('Will run views of %d viewers.', viewers.length) |
71 | 79 | ||
80 | const before = new Date().getTime() | ||
81 | |||
72 | await Bluebird.map(viewers, viewer => { | 82 | await Bluebird.map(viewers, viewer => { |
73 | return servers[0].views.simulateView({ id: videoId, xForwardedFor: viewer.xForwardedFor }) | 83 | return servers[0].views.simulateView({ id: videoId, xForwardedFor: viewer.xForwardedFor }) |
74 | }, { concurrency: 100 }) | 84 | }, { concurrency: 100 }) |
75 | 85 | ||
86 | console.log('Finished to run views in %d seconds.', (new Date().getTime() - before) / 1000) | ||
87 | |||
76 | await wait(5000) | 88 | await wait(5000) |
77 | } | 89 | } |
diff --git a/server/lib/activitypub/send/shared/audience-utils.ts b/server/lib/activitypub/send/shared/audience-utils.ts index ba4be487c..2f6b0741d 100644 --- a/server/lib/activitypub/send/shared/audience-utils.ts +++ b/server/lib/activitypub/send/shared/audience-utils.ts | |||
@@ -3,7 +3,7 @@ import { ACTIVITY_PUB } from '@server/initializers/constants' | |||
3 | import { ActorModel } from '@server/models/actor/actor' | 3 | import { ActorModel } from '@server/models/actor/actor' |
4 | import { VideoModel } from '@server/models/video/video' | 4 | import { VideoModel } from '@server/models/video/video' |
5 | import { VideoShareModel } from '@server/models/video/video-share' | 5 | import { VideoShareModel } from '@server/models/video/video-share' |
6 | import { MActorFollowersUrl, MActorLight, MActorUrl, MCommentOwner, MCommentOwnerVideo, MVideoId } from '@server/types/models' | 6 | import { MActorFollowersUrl, MActorUrl, MCommentOwner, MCommentOwnerVideo, MVideoId } from '@server/types/models' |
7 | import { ActivityAudience } from '@shared/models' | 7 | import { ActivityAudience } from '@shared/models' |
8 | 8 | ||
9 | function getOriginVideoAudience (accountActor: MActorUrl, actorsInvolvedInVideo: MActorFollowersUrl[] = []): ActivityAudience { | 9 | function getOriginVideoAudience (accountActor: MActorUrl, actorsInvolvedInVideo: MActorFollowersUrl[] = []): ActivityAudience { |
@@ -51,13 +51,13 @@ function getAudienceFromFollowersOf (actorsInvolvedInObject: MActorFollowersUrl[ | |||
51 | } | 51 | } |
52 | 52 | ||
53 | async function getActorsInvolvedInVideo (video: MVideoId, t: Transaction) { | 53 | async function getActorsInvolvedInVideo (video: MVideoId, t: Transaction) { |
54 | const actors: MActorLight[] = await VideoShareModel.loadActorsByShare(video.id, t) | 54 | const actors = await VideoShareModel.listActorIdsAndFollowerUrlsByShare(video.id, t) |
55 | 55 | ||
56 | const videoAll = video as VideoModel | 56 | const videoAll = video as VideoModel |
57 | 57 | ||
58 | const videoActor = videoAll.VideoChannel?.Account | 58 | const videoActor = videoAll.VideoChannel?.Account |
59 | ? videoAll.VideoChannel.Account.Actor | 59 | ? videoAll.VideoChannel.Account.Actor |
60 | : await ActorModel.loadFromAccountByVideoId(video.id, t) | 60 | : await ActorModel.loadAccountActorFollowerUrlByVideoId(video.id, t) |
61 | 61 | ||
62 | actors.push(videoActor) | 62 | actors.push(videoActor) |
63 | 63 | ||
diff --git a/server/lib/activitypub/send/shared/send-utils.ts b/server/lib/activitypub/send/shared/send-utils.ts index dbcde91ee..72368c762 100644 --- a/server/lib/activitypub/send/shared/send-utils.ts +++ b/server/lib/activitypub/send/shared/send-utils.ts | |||
@@ -19,13 +19,13 @@ async function sendVideoRelatedActivity (activityBuilder: (audience: ActivityAud | |||
19 | }) { | 19 | }) { |
20 | const { byActor, video, transaction, contextType } = options | 20 | const { byActor, video, transaction, contextType } = options |
21 | 21 | ||
22 | const actorsInvolvedInVideo = await getActorsInvolvedInVideo(video, transaction) | ||
23 | |||
24 | // Send to origin | 22 | // Send to origin |
25 | if (video.isOwned() === false) { | 23 | if (video.isOwned() === false) { |
26 | return sendVideoActivityToOrigin(activityBuilder, options) | 24 | return sendVideoActivityToOrigin(activityBuilder, options) |
27 | } | 25 | } |
28 | 26 | ||
27 | const actorsInvolvedInVideo = await getActorsInvolvedInVideo(video, transaction) | ||
28 | |||
29 | // Send to followers | 29 | // Send to followers |
30 | const audience = getAudienceFromFollowersOf(actorsInvolvedInVideo) | 30 | const audience = getAudienceFromFollowersOf(actorsInvolvedInVideo) |
31 | const activity = activityBuilder(audience) | 31 | const activity = activityBuilder(audience) |
diff --git a/server/lib/views/shared/video-viewer-counters.ts b/server/lib/views/shared/video-viewer-counters.ts index 941b62ed7..999ab7d8d 100644 --- a/server/lib/views/shared/video-viewer-counters.ts +++ b/server/lib/views/shared/video-viewer-counters.ts | |||
@@ -1,4 +1,3 @@ | |||
1 | |||
2 | import { isTestInstance } from '@server/helpers/core-utils' | 1 | import { isTestInstance } from '@server/helpers/core-utils' |
3 | import { logger, loggerTagsFactory } from '@server/helpers/logger' | 2 | import { logger, loggerTagsFactory } from '@server/helpers/logger' |
4 | import { VIEW_LIFETIME } from '@server/initializers/constants' | 3 | import { VIEW_LIFETIME } from '@server/initializers/constants' |
diff --git a/server/models/actor/actor.ts b/server/models/actor/actor.ts index 93145b8ae..943b7364f 100644 --- a/server/models/actor/actor.ts +++ b/server/models/actor/actor.ts | |||
@@ -1,5 +1,5 @@ | |||
1 | import { values } from 'lodash' | 1 | import { values } from 'lodash' |
2 | import { literal, Op, Transaction } from 'sequelize' | 2 | import { literal, Op, QueryTypes, Transaction } from 'sequelize' |
3 | import { | 3 | import { |
4 | AllowNull, | 4 | AllowNull, |
5 | BelongsTo, | 5 | BelongsTo, |
@@ -43,15 +43,18 @@ import { | |||
43 | MActorAccountChannelId, | 43 | MActorAccountChannelId, |
44 | MActorAPAccount, | 44 | MActorAPAccount, |
45 | MActorAPChannel, | 45 | MActorAPChannel, |
46 | MActorFollowersUrl, | ||
46 | MActorFormattable, | 47 | MActorFormattable, |
47 | MActorFull, | 48 | MActorFull, |
48 | MActorHost, | 49 | MActorHost, |
50 | MActorId, | ||
49 | MActorServer, | 51 | MActorServer, |
50 | MActorSummaryFormattable, | 52 | MActorSummaryFormattable, |
51 | MActorUrl, | 53 | MActorUrl, |
52 | MActorWithInboxes | 54 | MActorWithInboxes |
53 | } from '../../types/models' | 55 | } from '../../types/models' |
54 | import { AccountModel } from '../account/account' | 56 | import { AccountModel } from '../account/account' |
57 | import { getServerActor } from '../application/application' | ||
55 | import { ServerModel } from '../server/server' | 58 | import { ServerModel } from '../server/server' |
56 | import { isOutdated, throwIfNotValid } from '../utils' | 59 | import { isOutdated, throwIfNotValid } from '../utils' |
57 | import { VideoModel } from '../video/video' | 60 | import { VideoModel } from '../video/video' |
@@ -304,7 +307,10 @@ export class ActorModel extends Model<Partial<AttributesOnly<ActorModel>>> { | |||
304 | }) | 307 | }) |
305 | VideoChannel: VideoChannelModel | 308 | VideoChannel: VideoChannelModel |
306 | 309 | ||
307 | static load (id: number): Promise<MActor> { | 310 | static async load (id: number): Promise<MActor> { |
311 | const actorServer = await getServerActor() | ||
312 | if (id === actorServer.id) return actorServer | ||
313 | |||
308 | return ActorModel.unscoped().findByPk(id) | 314 | return ActorModel.unscoped().findByPk(id) |
309 | } | 315 | } |
310 | 316 | ||
@@ -312,48 +318,21 @@ export class ActorModel extends Model<Partial<AttributesOnly<ActorModel>>> { | |||
312 | return ActorModel.scope(ScopeNames.FULL).findByPk(id) | 318 | return ActorModel.scope(ScopeNames.FULL).findByPk(id) |
313 | } | 319 | } |
314 | 320 | ||
315 | static loadFromAccountByVideoId (videoId: number, transaction: Transaction): Promise<MActor> { | 321 | static loadAccountActorFollowerUrlByVideoId (videoId: number, transaction: Transaction) { |
316 | const query = { | 322 | const query = `SELECT "actor"."id" AS "id", "actor"."followersUrl" AS "followersUrl" ` + |
317 | include: [ | 323 | `FROM "actor" ` + |
318 | { | 324 | `INNER JOIN "account" ON "actor"."id" = "account"."actorId" ` + |
319 | attributes: [ 'id' ], | 325 | `INNER JOIN "videoChannel" ON "videoChannel"."accountId" = "account"."id" ` + |
320 | model: AccountModel.unscoped(), | 326 | `INNER JOIN "video" ON "video"."channelId" = "videoChannel"."id" AND "video"."id" = :videoId` |
321 | required: true, | 327 | |
322 | include: [ | 328 | const options = { |
323 | { | 329 | type: QueryTypes.SELECT as QueryTypes.SELECT, |
324 | attributes: [ 'id' ], | 330 | replacements: { videoId }, |
325 | model: VideoChannelModel.unscoped(), | 331 | plain: true as true, |
326 | required: true, | ||
327 | include: [ | ||
328 | { | ||
329 | attributes: [ 'id' ], | ||
330 | model: VideoModel.unscoped(), | ||
331 | required: true, | ||
332 | where: { | ||
333 | id: videoId | ||
334 | } | ||
335 | } | ||
336 | ] | ||
337 | } | ||
338 | ] | ||
339 | } | ||
340 | ], | ||
341 | transaction | 332 | transaction |
342 | } | 333 | } |
343 | 334 | ||
344 | return ActorModel.unscoped().findOne(query) | 335 | return ActorModel.sequelize.query<MActorId & MActorFollowersUrl>(query, options) |
345 | } | ||
346 | |||
347 | static isActorUrlExist (url: string) { | ||
348 | const query = { | ||
349 | raw: true, | ||
350 | where: { | ||
351 | url | ||
352 | } | ||
353 | } | ||
354 | |||
355 | return ActorModel.unscoped().findOne(query) | ||
356 | .then(a => !!a) | ||
357 | } | 336 | } |
358 | 337 | ||
359 | static listByFollowersUrls (followersUrls: string[], transaction?: Transaction): Promise<MActorFull[]> { | 338 | static listByFollowersUrls (followersUrls: string[], transaction?: Transaction): Promise<MActorFull[]> { |
diff --git a/server/models/video/video-share.ts b/server/models/video/video-share.ts index ad95dec6e..ca63bb2d9 100644 --- a/server/models/video/video-share.ts +++ b/server/models/video/video-share.ts | |||
@@ -3,7 +3,7 @@ import { AllowNull, BelongsTo, Column, CreatedAt, DataType, ForeignKey, Is, Mode | |||
3 | import { AttributesOnly } from '@shared/typescript-utils' | 3 | import { AttributesOnly } from '@shared/typescript-utils' |
4 | import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc' | 4 | import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc' |
5 | import { CONSTRAINTS_FIELDS } from '../../initializers/constants' | 5 | import { CONSTRAINTS_FIELDS } from '../../initializers/constants' |
6 | import { MActorDefault } from '../../types/models' | 6 | import { MActorDefault, MActorFollowersUrl, MActorId } from '../../types/models' |
7 | import { MVideoShareActor, MVideoShareFull } from '../../types/models/video' | 7 | import { MVideoShareActor, MVideoShareFull } from '../../types/models/video' |
8 | import { ActorModel } from '../actor/actor' | 8 | import { ActorModel } from '../actor/actor' |
9 | import { buildLocalActorIdsIn, throwIfNotValid } from '../utils' | 9 | import { buildLocalActorIdsIn, throwIfNotValid } from '../utils' |
@@ -107,22 +107,19 @@ export class VideoShareModel extends Model<Partial<AttributesOnly<VideoShareMode | |||
107 | }) | 107 | }) |
108 | } | 108 | } |
109 | 109 | ||
110 | static loadActorsByShare (videoId: number, t: Transaction): Promise<MActorDefault[]> { | 110 | static listActorIdsAndFollowerUrlsByShare (videoId: number, t: Transaction) { |
111 | const query = { | 111 | const query = `SELECT "actor"."id" AS "id", "actor"."followersUrl" AS "followersUrl" ` + |
112 | where: { | 112 | `FROM "videoShare" ` + |
113 | videoId | 113 | `INNER JOIN "actor" ON "actor"."id" = "videoShare"."actorId" ` + |
114 | }, | 114 | `WHERE "videoShare"."videoId" = :videoId` |
115 | include: [ | 115 | |
116 | { | 116 | const options = { |
117 | model: ActorModel, | 117 | type: QueryTypes.SELECT as QueryTypes.SELECT, |
118 | required: true | 118 | replacements: { videoId }, |
119 | } | ||
120 | ], | ||
121 | transaction: t | 119 | transaction: t |
122 | } | 120 | } |
123 | 121 | ||
124 | return VideoShareModel.scope(ScopeNames.FULL).findAll(query) | 122 | return VideoShareModel.sequelize.query<MActorId & MActorFollowersUrl>(query, options) |
125 | .then((res: MVideoShareFull[]) => res.map(r => r.Actor)) | ||
126 | } | 123 | } |
127 | 124 | ||
128 | static loadActorsWhoSharedVideosOf (actorOwnerId: number, t: Transaction): Promise<MActorDefault[]> { | 125 | static loadActorsWhoSharedVideosOf (actorOwnerId: number, t: Transaction): Promise<MActorDefault[]> { |