From 68d6e57870dd5ea87a03d13adcfcc72786d8348a Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Fri, 17 Jun 2022 09:03:18 +0200 Subject: Fix transaction when processing local viewer --- server/lib/activitypub/local-video-viewer.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'server') diff --git a/server/lib/activitypub/local-video-viewer.ts b/server/lib/activitypub/local-video-viewer.ts index 738083adc..bdd746791 100644 --- a/server/lib/activitypub/local-video-viewer.ts +++ b/server/lib/activitypub/local-video-viewer.ts @@ -23,7 +23,7 @@ async function createOrUpdateLocalVideoViewer (watchAction: WatchActionObject, v : null, videoId: video.id - }) + }, { transaction: t }) await LocalVideoViewerWatchSectionModel.bulkCreateSections({ localVideoViewerId: localVideoViewer.id, @@ -31,7 +31,9 @@ async function createOrUpdateLocalVideoViewer (watchAction: WatchActionObject, v watchSections: watchAction.watchSections.map(s => ({ start: s.startTimestamp, end: s.endTimestamp - })) + })), + + transaction: t }) } -- cgit v1.2.3 From 42e6a7a747f5349ac3434ee44273e50966630440 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Fri, 17 Jun 2022 09:04:45 +0200 Subject: Increase max watch sections --- server/initializers/constants.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'server') diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index 909fffdb6..6034da1c6 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts @@ -371,7 +371,7 @@ const VIEW_LIFETIME = { VIEWER_STATS: 60000 * 60 // 1 hour } -const MAX_LOCAL_VIEWER_WATCH_SECTIONS = 10 +const MAX_LOCAL_VIEWER_WATCH_SECTIONS = 100 let CONTACT_FORM_LIFETIME = 60000 * 60 // 1 hour -- cgit v1.2.3 From 3396e6534592865f184ee2db32a75957c42cb887 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Fri, 17 Jun 2022 10:49:37 +0200 Subject: Optimize broadcast job creation --- .../lib/activitypub/send/shared/audience-utils.ts | 6 +-- server/lib/activitypub/send/shared/send-utils.ts | 4 +- server/lib/views/shared/video-viewer-counters.ts | 1 - server/models/actor/actor.ts | 61 +++++++--------------- server/models/video/video-share.ts | 25 ++++----- 5 files changed, 36 insertions(+), 61 deletions(-) (limited to 'server') diff --git a/server/lib/activitypub/send/shared/audience-utils.ts b/server/lib/activitypub/send/shared/audience-utils.ts index ba4be487c..2f6b0741d 100644 --- a/server/lib/activitypub/send/shared/audience-utils.ts +++ b/server/lib/activitypub/send/shared/audience-utils.ts @@ -3,7 +3,7 @@ import { ACTIVITY_PUB } from '@server/initializers/constants' import { ActorModel } from '@server/models/actor/actor' import { VideoModel } from '@server/models/video/video' import { VideoShareModel } from '@server/models/video/video-share' -import { MActorFollowersUrl, MActorLight, MActorUrl, MCommentOwner, MCommentOwnerVideo, MVideoId } from '@server/types/models' +import { MActorFollowersUrl, MActorUrl, MCommentOwner, MCommentOwnerVideo, MVideoId } from '@server/types/models' import { ActivityAudience } from '@shared/models' function getOriginVideoAudience (accountActor: MActorUrl, actorsInvolvedInVideo: MActorFollowersUrl[] = []): ActivityAudience { @@ -51,13 +51,13 @@ function getAudienceFromFollowersOf (actorsInvolvedInObject: MActorFollowersUrl[ } async function getActorsInvolvedInVideo (video: MVideoId, t: Transaction) { - const actors: MActorLight[] = await VideoShareModel.loadActorsByShare(video.id, t) + const actors = await VideoShareModel.listActorIdsAndFollowerUrlsByShare(video.id, t) const videoAll = video as VideoModel const videoActor = videoAll.VideoChannel?.Account ? videoAll.VideoChannel.Account.Actor - : await ActorModel.loadFromAccountByVideoId(video.id, t) + : await ActorModel.loadAccountActorFollowerUrlByVideoId(video.id, t) actors.push(videoActor) diff --git a/server/lib/activitypub/send/shared/send-utils.ts b/server/lib/activitypub/send/shared/send-utils.ts index dbcde91ee..72368c762 100644 --- a/server/lib/activitypub/send/shared/send-utils.ts +++ b/server/lib/activitypub/send/shared/send-utils.ts @@ -19,13 +19,13 @@ async function sendVideoRelatedActivity (activityBuilder: (audience: ActivityAud }) { const { byActor, video, transaction, contextType } = options - const actorsInvolvedInVideo = await getActorsInvolvedInVideo(video, transaction) - // Send to origin if (video.isOwned() === false) { return sendVideoActivityToOrigin(activityBuilder, options) } + const actorsInvolvedInVideo = await getActorsInvolvedInVideo(video, transaction) + // Send to followers const audience = getAudienceFromFollowersOf(actorsInvolvedInVideo) const activity = activityBuilder(audience) diff --git a/server/lib/views/shared/video-viewer-counters.ts b/server/lib/views/shared/video-viewer-counters.ts index 941b62ed7..999ab7d8d 100644 --- a/server/lib/views/shared/video-viewer-counters.ts +++ b/server/lib/views/shared/video-viewer-counters.ts @@ -1,4 +1,3 @@ - import { isTestInstance } from '@server/helpers/core-utils' import { logger, loggerTagsFactory } from '@server/helpers/logger' import { VIEW_LIFETIME } from '@server/initializers/constants' diff --git a/server/models/actor/actor.ts b/server/models/actor/actor.ts index 93145b8ae..943b7364f 100644 --- a/server/models/actor/actor.ts +++ b/server/models/actor/actor.ts @@ -1,5 +1,5 @@ import { values } from 'lodash' -import { literal, Op, Transaction } from 'sequelize' +import { literal, Op, QueryTypes, Transaction } from 'sequelize' import { AllowNull, BelongsTo, @@ -43,15 +43,18 @@ import { MActorAccountChannelId, MActorAPAccount, MActorAPChannel, + MActorFollowersUrl, MActorFormattable, MActorFull, MActorHost, + MActorId, MActorServer, MActorSummaryFormattable, MActorUrl, MActorWithInboxes } from '../../types/models' import { AccountModel } from '../account/account' +import { getServerActor } from '../application/application' import { ServerModel } from '../server/server' import { isOutdated, throwIfNotValid } from '../utils' import { VideoModel } from '../video/video' @@ -304,7 +307,10 @@ export class ActorModel extends Model>> { }) VideoChannel: VideoChannelModel - static load (id: number): Promise { + static async load (id: number): Promise { + const actorServer = await getServerActor() + if (id === actorServer.id) return actorServer + return ActorModel.unscoped().findByPk(id) } @@ -312,48 +318,21 @@ export class ActorModel extends Model>> { return ActorModel.scope(ScopeNames.FULL).findByPk(id) } - static loadFromAccountByVideoId (videoId: number, transaction: Transaction): Promise { - const query = { - include: [ - { - attributes: [ 'id' ], - model: AccountModel.unscoped(), - required: true, - include: [ - { - attributes: [ 'id' ], - model: VideoChannelModel.unscoped(), - required: true, - include: [ - { - attributes: [ 'id' ], - model: VideoModel.unscoped(), - required: true, - where: { - id: videoId - } - } - ] - } - ] - } - ], + static loadAccountActorFollowerUrlByVideoId (videoId: number, transaction: Transaction) { + const query = `SELECT "actor"."id" AS "id", "actor"."followersUrl" AS "followersUrl" ` + + `FROM "actor" ` + + `INNER JOIN "account" ON "actor"."id" = "account"."actorId" ` + + `INNER JOIN "videoChannel" ON "videoChannel"."accountId" = "account"."id" ` + + `INNER JOIN "video" ON "video"."channelId" = "videoChannel"."id" AND "video"."id" = :videoId` + + const options = { + type: QueryTypes.SELECT as QueryTypes.SELECT, + replacements: { videoId }, + plain: true as true, transaction } - return ActorModel.unscoped().findOne(query) - } - - static isActorUrlExist (url: string) { - const query = { - raw: true, - where: { - url - } - } - - return ActorModel.unscoped().findOne(query) - .then(a => !!a) + return ActorModel.sequelize.query(query, options) } static listByFollowersUrls (followersUrls: string[], transaction?: Transaction): Promise { diff --git a/server/models/video/video-share.ts b/server/models/video/video-share.ts index ad95dec6e..ca63bb2d9 100644 --- a/server/models/video/video-share.ts +++ b/server/models/video/video-share.ts @@ -3,7 +3,7 @@ import { AllowNull, BelongsTo, Column, CreatedAt, DataType, ForeignKey, Is, Mode import { AttributesOnly } from '@shared/typescript-utils' import { isActivityPubUrlValid } from '../../helpers/custom-validators/activitypub/misc' import { CONSTRAINTS_FIELDS } from '../../initializers/constants' -import { MActorDefault } from '../../types/models' +import { MActorDefault, MActorFollowersUrl, MActorId } from '../../types/models' import { MVideoShareActor, MVideoShareFull } from '../../types/models/video' import { ActorModel } from '../actor/actor' import { buildLocalActorIdsIn, throwIfNotValid } from '../utils' @@ -107,22 +107,19 @@ export class VideoShareModel extends Model { - const query = { - where: { - videoId - }, - include: [ - { - model: ActorModel, - required: true - } - ], + static listActorIdsAndFollowerUrlsByShare (videoId: number, t: Transaction) { + const query = `SELECT "actor"."id" AS "id", "actor"."followersUrl" AS "followersUrl" ` + + `FROM "videoShare" ` + + `INNER JOIN "actor" ON "actor"."id" = "videoShare"."actorId" ` + + `WHERE "videoShare"."videoId" = :videoId` + + const options = { + type: QueryTypes.SELECT as QueryTypes.SELECT, + replacements: { videoId }, transaction: t } - return VideoShareModel.scope(ScopeNames.FULL).findAll(query) - .then((res: MVideoShareFull[]) => res.map(r => r.Actor)) + return VideoShareModel.sequelize.query(query, options) } static loadActorsWhoSharedVideosOf (actorOwnerId: number, t: Transaction): Promise { -- cgit v1.2.3 From f27b7a750f1876632e84d594608d3d64ce974efc Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Fri, 17 Jun 2022 14:08:13 +0200 Subject: Send views in a dedicated queue --- server/initializers/constants.ts | 3 +++ server/lib/activitypub/send/send-view.ts | 2 +- server/lib/activitypub/send/shared/send-utils.ts | 19 +++++++++++++++---- server/lib/job-queue/job-queue.ts | 3 +++ 4 files changed, 22 insertions(+), 5 deletions(-) (limited to 'server') diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index 6034da1c6..cd2cc33d3 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts @@ -139,6 +139,7 @@ const REMOTE_SCHEME = { const JOB_ATTEMPTS: { [id in JobType]: number } = { 'activitypub-http-broadcast': 1, + 'activitypub-http-broadcast-parallel': 1, 'activitypub-http-unicast': 1, 'activitypub-http-fetcher': 2, 'activitypub-follow': 5, @@ -159,6 +160,7 @@ const JOB_ATTEMPTS: { [id in JobType]: number } = { // Excluded keys are jobs that can be configured by admins const JOB_CONCURRENCY: { [id in Exclude]: number } = { 'activitypub-http-broadcast': 1, + 'activitypub-http-broadcast-parallel': 30, 'activitypub-http-unicast': 10, 'activitypub-http-fetcher': 3, 'activitypub-cleaner': 1, @@ -176,6 +178,7 @@ const JOB_CONCURRENCY: { [id in Exclude Promise } = { 'activitypub-http-broadcast': processActivityPubHttpBroadcast, + 'activitypub-http-broadcast-parallel': processActivityPubHttpBroadcast, 'activitypub-http-unicast': processActivityPubHttpUnicast, 'activitypub-http-fetcher': processActivityPubHttpFetcher, 'activitypub-cleaner': processActivityPubCleaner, @@ -89,6 +91,7 @@ const handlers: { [id in JobType]: (job: Job) => Promise } = { const jobTypes: JobType[] = [ 'activitypub-follow', 'activitypub-http-broadcast', + 'activitypub-http-broadcast-parallel', 'activitypub-http-fetcher', 'activitypub-http-unicast', 'activitypub-cleaner', -- cgit v1.2.3