aboutsummaryrefslogtreecommitdiffhomepage
diff options
context:
space:
mode:
authorChocobozzz <me@florianbigard.com>2022-06-17 10:49:37 +0200
committerChocobozzz <me@florianbigard.com>2022-06-17 11:55:03 +0200
commit3396e6534592865f184ee2db32a75957c42cb887 (patch)
tree324aeab92a27f2ffd1e8a16b09f0b5509bf8f477
parentb9e49a45f5441fd1a2d49a10b88aaf4d425dc992 (diff)
downloadPeerTube-3396e6534592865f184ee2db32a75957c42cb887.tar.gz
PeerTube-3396e6534592865f184ee2db32a75957c42cb887.tar.zst
PeerTube-3396e6534592865f184ee2db32a75957c42cb887.zip
Optimize broadcast job creation
-rw-r--r--scripts/simulate-many-viewers.ts14
-rw-r--r--server/lib/activitypub/send/shared/audience-utils.ts6
-rw-r--r--server/lib/activitypub/send/shared/send-utils.ts4
-rw-r--r--server/lib/views/shared/video-viewer-counters.ts1
-rw-r--r--server/models/actor/actor.ts61
-rw-r--r--server/models/video/video-share.ts25
6 files changed, 49 insertions, 62 deletions
diff --git a/scripts/simulate-many-viewers.ts b/scripts/simulate-many-viewers.ts
index 4ec928aa8..fb666c318 100644
--- a/scripts/simulate-many-viewers.ts
+++ b/scripts/simulate-many-viewers.ts
@@ -1,6 +1,13 @@
1import Bluebird from 'bluebird' 1import Bluebird from 'bluebird'
2import { wait } from '@shared/core-utils' 2import { wait } from '@shared/core-utils'
3import { createSingleServer, doubleFollow, PeerTubeServer, setAccessTokensToServers, waitJobs } from '@shared/server-commands' 3import {
4 createSingleServer,
5 doubleFollow,
6 killallServers,
7 PeerTubeServer,
8 setAccessTokensToServers,
9 waitJobs
10} from '@shared/server-commands'
4 11
5let servers: PeerTubeServer[] 12let servers: PeerTubeServer[]
6const viewers: { xForwardedFor: string }[] = [] 13const viewers: { xForwardedFor: string }[] = []
@@ -9,6 +16,7 @@ let videoId: string
9run() 16run()
10 .then(() => process.exit(0)) 17 .then(() => process.exit(0))
11 .catch(err => console.error(err)) 18 .catch(err => console.error(err))
19 .finally(() => killallServers(servers))
12 20
13async function run () { 21async function run () {
14 await prepare() 22 await prepare()
@@ -69,9 +77,13 @@ async function prepare () {
69async function runViewers () { 77async function runViewers () {
70 console.log('Will run views of %d viewers.', viewers.length) 78 console.log('Will run views of %d viewers.', viewers.length)
71 79
80 const before = new Date().getTime()
81
72 await Bluebird.map(viewers, viewer => { 82 await Bluebird.map(viewers, viewer => {
73 return servers[0].views.simulateView({ id: videoId, xForwardedFor: viewer.xForwardedFor }) 83 return servers[0].views.simulateView({ id: videoId, xForwardedFor: viewer.xForwardedFor })
74 }, { concurrency: 100 }) 84 }, { concurrency: 100 })
75 85
86 console.log('Finished to run views in %d seconds.', (new Date().getTime() - before) / 1000)
87
76 await wait(5000) 88 await wait(5000)
77} 89}
diff --git a/server/lib/activitypub/send/shared/audience-utils.ts b/server/lib/activitypub/send/shared/audience-utils.ts
index ba4be487c..2f6b0741d 100644
--- a/server/lib/activitypub/send/shared/audience-utils.ts
+++ b/server/lib/activitypub/send/shared/audience-utils.ts
@@ -3,7 +3,7 @@ import { ACTIVITY_PUB } from '@server/initializers/constants'
3import { ActorModel } from '@server/models/actor/actor' 3import { ActorModel } from '@server/models/actor/actor'
4import { VideoModel } from '@server/models/video/video' 4import { VideoModel } from '@server/models/video/video'
5import { VideoShareModel } from '@server/models/video/video-share' 5import { VideoShareModel } from '@server/models/video/video-share'
6import { MActorFollowersUrl, MActorLight, MActorUrl, MCommentOwner, MCommentOwnerVideo, MVideoId } from '@server/types/models' 6import { MActorFollowersUrl, MActorUrl, MCommentOwner, MCommentOwnerVideo, MVideoId } from '@server/types/models'
7import { ActivityAudience } from '@shared/models' 7import { ActivityAudience } from '@shared/models'
8 8
9function getOriginVideoAudience (accountActor: MActorUrl, actorsInvolvedInVideo: MActorFollowersUrl[] = []): ActivityAudience { 9function getOriginVideoAudience (accountActor: MActorUrl, actorsInvolvedInVideo: MActorFollowersUrl[] = []): ActivityAudience {
@@ -51,13 +51,13 @@ function getAudienceFromFollowersOf (actorsInvolvedInObject: MActorFollowersUrl[
51} 51}
52 52
53async function getActorsInvolvedInVideo (video: MVideoId, t: Transaction) { 53async function getActorsInvolvedInVideo (video: MVideoId, t: Transaction) {
54 const actors: MActorLight[] = await VideoShareModel.loadActorsByShare(video.id, t) 54 const actors = await VideoShareModel.listActorIdsAndFollowerUrlsByShare(video.id, t)
55 55
56 const videoAll = video as VideoModel 56 const videoAll = video as VideoModel
57 57
58 const videoActor = videoAll.VideoChannel?.Account 58 const videoActor = videoAll.VideoChannel?.Account
59 ? videoAll.VideoChannel.Account.Actor 59 ? videoAll.VideoChannel.Account.Actor
60 : await ActorModel.loadFromAccountByVideoId(video.id, t) 60 : await ActorModel.loadAccountActorFollowerUrlByVideoId(video.id, t)
61 61
62 actors.push(videoActor) 62 actors.push(videoActor)
63 63
diff --git a/server/lib/activitypub/send/shared/send-utils.ts b/server/lib/activitypub/send/shared/send-utils.ts
index dbcde91ee..72368c762 100644
--- a/server/lib/activitypub/send/shared/send-utils.ts
+++ b/server/lib/activitypub/send/shared/send-utils.ts
@@ -19,13 +19,13 @@ async function sendVideoRelatedActivity (activityBuilder: (audience: ActivityAud
19}) { 19}) {
20 const { byActor, video, transaction, contextType } = options 20 const { byActor, video, transaction, contextType } = options
21 21
22 const actorsInvolvedInVideo = await getActorsInvolvedInVideo(video, transaction)
23
24 // Send to origin 22 // Send to origin
25 if (video.isOwned() === false) { 23 if (video.isOwned() === false) {
26 return sendVideoActivityToOrigin(activityBuilder, options) 24 return sendVideoActivityToOrigin(activityBuilder, options)
27 } 25 }
28 26
27 const actorsInvolvedInVideo = await getActorsInvolvedInVideo(video, transaction)
28
29 // Send to followers 29 // Send to followers
30 const audience = getAudienceFromFollowersOf(actorsInvolvedInVideo) 30 const audience = getAudienceFromFollowersOf(actorsInvolvedInVideo)
31 const activity = activityBuilder(audience) 31 const activity = activityBuilder(audience)
diff --git a/server/lib/views/shared/video-viewer-counters.ts b/server/lib/views/shared/video-viewer-counters.ts
index 941b62ed7..999ab7d8d 100644
--- a/server/lib/views/shared/video-viewer-counters.ts
+++ b/server/lib/views/shared/video-viewer-counters.ts
@@ -1,4 +1,3 @@
1
2import { isTestInstance } from '@server/helpers/core-utils' 1import { isTestInstance } from '@server/helpers/core-utils'
3import { logger, loggerTagsFactory } from '@server/helpers/logger' 2import { logger, loggerTagsFactory } from '@server/helpers/logger'
4import { VIEW_LIFETIME } from '@server/initializers/constants' 3import { VIEW_LIFETIME } from '@server/initializers/constants'
diff --git a/server/models/actor/actor.ts b/server/models/actor/actor.ts
index 93145b8ae..943b7364f 100644
--- a/server/models/actor/actor.ts
+++ b/server/models/actor/actor.ts
@@ -1,5 +1,5 @@
1import { values } from 'lodash' 1import { values } from 'lodash'
2import { literal, Op, Transaction } from 'sequelize' 2import { literal, Op, QueryTypes, Transaction } from 'sequelize'
3import { 3import {
4 AllowNull, 4 AllowNull,
5 BelongsTo, 5 BelongsTo,
@@ -43,15 +43,18 @@ import {
43 MActorAccountChannelId, 43 MActorAccountChannelId,
44 MActorAPAccount, 44 MActorAPAccount,
45 MActorAPChannel, 45 MActorAPChannel,
46 MActorFollowersUrl,
46 MActorFormattable, 47 MActorFormattable,
47 MActorFull, 48 MActorFull,
48 MActorHost, 49 MActorHost,
50 MActorId,
49 MActorServer, 51 MActorServer,
50 MActorSummaryFormattable, 52 MActorSummaryFormattable,
51 MActorUrl, 53 MActorUrl,
52 MActorWithInboxes 54 MActorWithInboxes
53} from '../../types/models' 55} from '../../types/models'
54import { AccountModel } from '../account/account' 56import { AccountModel } from '../account/account'
57import { getServerActor } from '../application/application'
55import { ServerModel } from '../server/server' 58import { ServerModel } from '../server/server'
56import { isOutdated, throwIfNotValid } from '../utils' 59import { isOutdated, throwIfNotValid } from '../utils'
57import { VideoModel } from '../video/video' 60import { VideoModel } from '../video/video'
@@ -304,7 +307,10 @@ export class ActorModel extends Model<Partial<AttributesOnly<ActorModel>>> {
304 }) 307 })
305 VideoChannel: VideoChannelModel 308 VideoChannel: VideoChannelModel
306 309
307 static load (id: number): Promise<MActor> { 310 static async load (id: number): Promise<MActor> {
311 const actorServer = await getServerActor()
312 if (id === actorServer.id) return actorServer
313
308 return ActorModel.unscoped().findByPk(id) 314 return ActorModel.unscoped().findByPk(id)
309 } 315 }
310 316
@@ -312,48 +318,21 @@ export class ActorModel extends Model<Partial<AttributesOnly<ActorModel>>> {
312 return ActorModel.scope(ScopeNames.FULL).findByPk(id) 318 return ActorModel.scope(ScopeNames.FULL).findByPk(id)
313 } 319 }
314 320
315 static loadFromAccountByVideoId (videoId: number, transaction: Transaction): Promise<MActor> { 321 static loadAccountActorFollowerUrlByVideoId (videoId: number, transaction: Transaction) {
316 const query = { 322 const query = `SELECT "actor"."id" AS "id", "actor"."followersUrl" AS "followersUrl" ` +
317 include: [ 323 `FROM "actor" ` +
318 { 324 `INNER JOIN "account" ON "actor"."id" = "account"."actorId" ` +
319 attributes: [ 'id' ], 325 `INNER JOIN "videoChannel" ON "videoChannel"."accountId" = "account"."id" ` +
320 model: AccountModel.unscoped(), 326 `INNER JOIN "video" ON "video"."channelId" = "videoChannel"."id" AND "video"."id" = :videoId`
321 required: true, 327
322 include: [ 328 const options = {
323 { 329 type: QueryTypes.SELECT as QueryTypes.SELECT,
324 attributes: [ 'id' ], 330 replacements: { videoId },
325 model: VideoChannelModel.unscoped(), 331 plain: true as true,
326 required: true,
327 include: [
328 {
329 attributes: [ 'id' ],
330 model: VideoModel.unscoped(),
331 required: true,
332 where: {
333 id: videoId
334 }
335 }
336 ]
337 }
338 ]
339 }
340 ],
341 transaction 332 transaction
342 } 333 }
343 334
344 return ActorModel.unscoped().findOne(query) 335 return ActorModel.sequelize.query<MActorId & MActorFollowersUrl>(query, options)
345 }
346
347 static isActorUrlExist (url: string) {
348 const query = {
349 raw: true,
350 where: {
351 url
352 }
353 }
354
355 return ActorModel.unscoped().findOne(query)
356 .then(a => !!a)
357 } 336 }
358 337
359 static listByFollowersUrls (followersUrls: string[], transaction?: Transaction): Promise<MActorFull[]> { 338 static listByFollowersUrls (followersUrls: string[], transaction?: Transaction): Promise<MActorFull[]> {
diff --git a/server/models/video/video-share.ts b/server/models/video/video-share.ts
index ad95dec6e..ca63bb2d9 100644
--- a/server/models/video/video-share.ts
+++ b/server/models/video/video-share.ts
@@ -3,7 +3,7 @@ import { AllowNull, BelongsTo, Column, CreatedAt, DataType, ForeignKey, Is, Mode
3import { AttributesOnly } from '@shared/typescript-utils' 3import { AttributesOnly } from '@shared/typescript-utils'
4import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc' 4import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc'
5import { CONSTRAINTS_FIELDS } from '../../initializers/constants' 5import { CONSTRAINTS_FIELDS } from '../../initializers/constants'
6import { MActorDefault } from '../../types/models' 6import { MActorDefault, MActorFollowersUrl, MActorId } from '../../types/models'
7import { MVideoShareActor, MVideoShareFull } from '../../types/models/video' 7import { MVideoShareActor, MVideoShareFull } from '../../types/models/video'
8import { ActorModel } from '../actor/actor' 8import { ActorModel } from '../actor/actor'
9import { buildLocalActorIdsIn, throwIfNotValid } from '../utils' 9import { buildLocalActorIdsIn, throwIfNotValid } from '../utils'
@@ -107,22 +107,19 @@ export class VideoShareModel extends Model<Partial<AttributesOnly<VideoShareMode
107 }) 107 })
108 } 108 }
109 109
110 static loadActorsByShare (videoId: number, t: Transaction): Promise<MActorDefault[]> { 110 static listActorIdsAndFollowerUrlsByShare (videoId: number, t: Transaction) {
111 const query = { 111 const query = `SELECT "actor"."id" AS "id", "actor"."followersUrl" AS "followersUrl" ` +
112 where: { 112 `FROM "videoShare" ` +
113 videoId 113 `INNER JOIN "actor" ON "actor"."id" = "videoShare"."actorId" ` +
114 }, 114 `WHERE "videoShare"."videoId" = :videoId`
115 include: [ 115
116 { 116 const options = {
117 model: ActorModel, 117 type: QueryTypes.SELECT as QueryTypes.SELECT,
118 required: true 118 replacements: { videoId },
119 }
120 ],
121 transaction: t 119 transaction: t
122 } 120 }
123 121
124 return VideoShareModel.scope(ScopeNames.FULL).findAll(query) 122 return VideoShareModel.sequelize.query<MActorId & MActorFollowersUrl>(query, options)
125 .then((res: MVideoShareFull[]) => res.map(r => r.Actor))
126 } 123 }
127 124
128 static loadActorsWhoSharedVideosOf (actorOwnerId: number, t: Transaction): Promise<MActorDefault[]> { 125 static loadActorsWhoSharedVideosOf (actorOwnerId: number, t: Transaction): Promise<MActorDefault[]> {