From aa2ce188d102ab38452df316d06286040b5d9075 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Fri, 17 Jun 2022 14:34:37 +0200 Subject: Optimize view endpoint --- scripts/simulate-many-viewers.ts | 10 +++++---- server/controllers/api/videos/view.ts | 2 +- server/initializers/constants.ts | 6 ++++-- server/lib/video.ts | 24 ++++++++++++++++++++-- server/lib/views/shared/video-viewer-counters.ts | 8 ++++---- server/lib/views/shared/video-viewer-stats.ts | 6 +++--- server/lib/views/shared/video-views.ts | 17 ++++++++------- server/lib/views/video-views-manager.ts | 4 ++-- server/middlewares/validators/videos/video-view.ts | 13 ++++++------ 9 files changed, 58 insertions(+), 32 deletions(-) diff --git a/scripts/simulate-many-viewers.ts b/scripts/simulate-many-viewers.ts index fb666c318..a993e175a 100644 --- a/scripts/simulate-many-viewers.ts +++ b/scripts/simulate-many-viewers.ts @@ -46,10 +46,12 @@ async function prepare () { } } + const env = { PRODUCTION_CONSTANTS: 'true' } + servers = await Promise.all([ - createSingleServer(1, config, { nodeArgs: [ '--inspect' ] }), - createSingleServer(2, config), - createSingleServer(3, config) + createSingleServer(1, config, { env, nodeArgs: [ '--inspect' ] }), + createSingleServer(2, config, { env }), + createSingleServer(3, config, { env }) ]) await setAccessTokensToServers(servers) @@ -81,7 +83,7 @@ async function runViewers () { await Bluebird.map(viewers, viewer => { return servers[0].views.simulateView({ id: videoId, xForwardedFor: viewer.xForwardedFor }) - }, { concurrency: 100 }) + }, { concurrency: 500 }) console.log('Finished to run views in %d seconds.', (new Date().getTime() - before) / 1000) diff --git a/server/controllers/api/videos/view.ts b/server/controllers/api/videos/view.ts index db1091f2d..dee1ec67c 100644 --- a/server/controllers/api/videos/view.ts +++ b/server/controllers/api/videos/view.ts @@ -26,7 +26,7 @@ export { // --------------------------------------------------------------------------- async function viewVideo (req: express.Request, res: express.Response) { - const video = res.locals.onlyVideo + const video = res.locals.onlyImmutableVideo const body = req.body as VideoView diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index 75ccbc458..52007d212 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts @@ -734,12 +734,14 @@ const VIDEO_LIVE = { const MEMOIZE_TTL = { OVERVIEWS_SAMPLE: 1000 * 3600 * 4, // 4 hours INFO_HASH_EXISTS: 1000 * 3600 * 12, // 12 hours + VIDEO_DURATION: 1000 * 10, // 10 seconds LIVE_ABLE_TO_UPLOAD: 1000 * 60, // 1 minute LIVE_CHECK_SOCKET_HEALTH: 1000 * 60 // 1 minute } const MEMOIZE_LENGTH = { - INFO_HASH_EXISTS: 200 + INFO_HASH_EXISTS: 200, + VIDEO_DURATION: 200 } const QUEUE_CONCURRENCY = { @@ -812,7 +814,7 @@ const STATS_TIMESERIE = { // --------------------------------------------------------------------------- // Special constants for a test instance -if (isTestInstance() === true) { +if (isTestInstance() === true && process.env.PRODUCTION_CONSTANTS !== 'true') { PRIVATE_RSA_KEY_SIZE = 1024 ACTOR_FOLLOW_SCORE.BASE = 20 diff --git a/server/lib/video.ts b/server/lib/video.ts index a98e45c60..86718abbe 100644 --- a/server/lib/video.ts +++ b/server/lib/video.ts @@ -1,6 +1,6 @@ import { UploadFiles } from 'express' import { Transaction } from 'sequelize/types' -import { DEFAULT_AUDIO_RESOLUTION, JOB_PRIORITY } from '@server/initializers/constants' +import { DEFAULT_AUDIO_RESOLUTION, JOB_PRIORITY, MEMOIZE_LENGTH, MEMOIZE_TTL } from '@server/initializers/constants' import { TagModel } from '@server/models/video/tag' import { VideoModel } from '@server/models/video/video' import { VideoJobInfoModel } from '@server/models/video/video-job-info' @@ -10,6 +10,7 @@ import { ThumbnailType, VideoCreate, VideoPrivacy, VideoState, VideoTranscodingP import { CreateJobOptions, JobQueue } from './job-queue/job-queue' import { updateVideoMiniatureFromExisting } from './thumbnail' import { CONFIG } from '@server/initializers/config' +import memoizee from 'memoizee' function buildLocalVideoFromReq (videoInfo: VideoCreate, channelId: number): FilteredModelAttributes { return { @@ -150,6 +151,24 @@ async function addMoveToObjectStorageJob (options: { // --------------------------------------------------------------------------- +async function getVideoDuration (videoId: number | string) { + const video = await VideoModel.load(videoId) + + const duration = video.isLive + ? undefined + : video.duration + + return { duration, isLive: video.isLive } +} + +const getCachedVideoDuration = memoizee(getVideoDuration, { + promise: true, + max: MEMOIZE_LENGTH.VIDEO_DURATION, + maxAge: MEMOIZE_TTL.VIDEO_DURATION +}) + +// --------------------------------------------------------------------------- + export { buildLocalVideoFromReq, buildVideoThumbnailsFromReq, @@ -157,5 +176,6 @@ export { addOptimizeOrMergeAudioJob, addTranscodingJob, addMoveToObjectStorageJob, - getTranscodingJobPriority + getTranscodingJobPriority, + getCachedVideoDuration } diff --git a/server/lib/views/shared/video-viewer-counters.ts b/server/lib/views/shared/video-viewer-counters.ts index 999ab7d8d..587621320 100644 --- a/server/lib/views/shared/video-viewer-counters.ts +++ b/server/lib/views/shared/video-viewer-counters.ts @@ -5,7 +5,7 @@ import { sendView } from '@server/lib/activitypub/send/send-view' import { PeerTubeSocket } from '@server/lib/peertube-socket' import { getServerActor } from '@server/models/application/application' import { VideoModel } from '@server/models/video/video' -import { MVideo } from '@server/types/models' +import { MVideo, MVideoImmutable } from '@server/types/models' import { buildUUID, sha256 } from '@shared/extra-utils' const lTags = loggerTagsFactory('views') @@ -33,7 +33,7 @@ export class VideoViewerCounters { // --------------------------------------------------------------------------- async addLocalViewer (options: { - video: MVideo + video: MVideoImmutable ip: string }) { const { video, ip } = options @@ -86,7 +86,7 @@ export class VideoViewerCounters { // --------------------------------------------------------------------------- private async addViewerToVideo (options: { - video: MVideo + video: MVideoImmutable viewerId: string viewerExpires?: Date }) { @@ -162,7 +162,7 @@ export class VideoViewerCounters { return sha256(this.salt + '-' + ip + '-' + videoUUID) } - private async federateViewerIfNeeded (video: MVideo, viewer: Viewer) { + private async federateViewerIfNeeded (video: MVideoImmutable, viewer: Viewer) { // Federate the viewer if it's been a "long" time we did not const now = new Date().getTime() const federationLimit = now - (VIEW_LIFETIME.VIEWER_COUNTER / 2) diff --git a/server/lib/views/shared/video-viewer-stats.ts b/server/lib/views/shared/video-viewer-stats.ts index a9ba25b47..a56c20559 100644 --- a/server/lib/views/shared/video-viewer-stats.ts +++ b/server/lib/views/shared/video-viewer-stats.ts @@ -10,7 +10,7 @@ import { Redis } from '@server/lib/redis' import { VideoModel } from '@server/models/video/video' import { LocalVideoViewerModel } from '@server/models/view/local-video-viewer' import { LocalVideoViewerWatchSectionModel } from '@server/models/view/local-video-viewer-watch-section' -import { MVideo } from '@server/types/models' +import { MVideo, MVideoImmutable } from '@server/types/models' import { VideoViewEvent } from '@shared/models' const lTags = loggerTagsFactory('views') @@ -41,7 +41,7 @@ export class VideoViewerStats { // --------------------------------------------------------------------------- async addLocalViewer (options: { - video: MVideo + video: MVideoImmutable currentTime: number ip: string viewEvent?: VideoViewEvent @@ -64,7 +64,7 @@ export class VideoViewerStats { // --------------------------------------------------------------------------- private async updateLocalViewerStats (options: { - video: MVideo + video: MVideoImmutable ip: string currentTime: number viewEvent?: VideoViewEvent diff --git a/server/lib/views/shared/video-views.ts b/server/lib/views/shared/video-views.ts index 275f7a014..e563287e1 100644 --- a/server/lib/views/shared/video-views.ts +++ b/server/lib/views/shared/video-views.ts @@ -1,7 +1,8 @@ import { logger, loggerTagsFactory } from '@server/helpers/logger' import { sendView } from '@server/lib/activitypub/send/send-view' +import { getCachedVideoDuration } from '@server/lib/video' import { getServerActor } from '@server/models/application/application' -import { MVideo } from '@server/types/models' +import { MVideo, MVideoImmutable } from '@server/types/models' import { buildUUID } from '@shared/extra-utils' import { Redis } from '../../redis' @@ -10,7 +11,7 @@ const lTags = loggerTagsFactory('views') export class VideoViews { async addLocalView (options: { - video: MVideo + video: MVideoImmutable ip: string watchTime: number }) { @@ -18,7 +19,7 @@ export class VideoViews { logger.debug('Adding local view to video %s.', video.uuid, { watchTime, ...lTags(video.uuid) }) - if (!this.hasEnoughWatchTime(video, watchTime)) return false + if (!await this.hasEnoughWatchTime(video, watchTime)) return false const viewExists = await Redis.Instance.doesVideoIPViewExist(ip, video.uuid) if (viewExists) return false @@ -46,7 +47,7 @@ export class VideoViews { // --------------------------------------------------------------------------- - private async addView (video: MVideo) { + private async addView (video: MVideoImmutable) { const promises: Promise[] = [] if (video.isOwned()) { @@ -58,10 +59,12 @@ export class VideoViews { await Promise.all(promises) } - private hasEnoughWatchTime (video: MVideo, watchTime: number) { - if (video.isLive || video.duration >= 30) return watchTime >= 30 + private async hasEnoughWatchTime (video: MVideoImmutable, watchTime: number) { + const { duration, isLive } = await getCachedVideoDuration(video.id) + + if (isLive || duration >= 30) return watchTime >= 30 // Check more than 50% of the video is watched - return video.duration / watchTime < 2 + return duration / watchTime < 2 } } diff --git a/server/lib/views/video-views-manager.ts b/server/lib/views/video-views-manager.ts index ea3b35c6c..86758e8d8 100644 --- a/server/lib/views/video-views-manager.ts +++ b/server/lib/views/video-views-manager.ts @@ -1,5 +1,5 @@ import { logger, loggerTagsFactory } from '@server/helpers/logger' -import { MVideo } from '@server/types/models' +import { MVideo, MVideoImmutable } from '@server/types/models' import { VideoViewEvent } from '@shared/models' import { VideoViewerCounters, VideoViewerStats, VideoViews } from './shared' @@ -41,7 +41,7 @@ export class VideoViewsManager { } async processLocalView (options: { - video: MVideo + video: MVideoImmutable currentTime: number ip: string | null viewEvent?: VideoViewEvent diff --git a/server/middlewares/validators/videos/video-view.ts b/server/middlewares/validators/videos/video-view.ts index 7a4994e8a..2edcd140f 100644 --- a/server/middlewares/validators/videos/video-view.ts +++ b/server/middlewares/validators/videos/video-view.ts @@ -6,6 +6,7 @@ import { HttpStatusCode } from '../../../../shared/models/http/http-error-codes' import { exists, isIdValid, isIntOrNull, toIntOrNull } from '../../../helpers/custom-validators/misc' import { logger } from '../../../helpers/logger' import { areValidationErrors, doesVideoExist, isValidVideoIdParam } from '../shared' +import { getCachedVideoDuration } from '@server/lib/video' const getVideoLocalViewerValidator = [ param('localViewerId') @@ -42,20 +43,18 @@ const videoViewValidator = [ logger.debug('Checking videoView parameters', { parameters: req.body }) if (areValidationErrors(req, res)) return - if (!await doesVideoExist(req.params.videoId, res, 'only-video')) return + if (!await doesVideoExist(req.params.videoId, res, 'only-immutable-attributes')) return - const video = res.locals.onlyVideo - const videoDuration = video.isLive - ? undefined - : video.duration + const video = res.locals.onlyImmutableVideo + const { duration } = await getCachedVideoDuration(video.id) if (!exists(req.body.currentTime)) { // TODO: remove in a few versions, introduced in 4.2 - req.body.currentTime = Math.min(videoDuration ?? 0, 30) + req.body.currentTime = Math.min(duration ?? 0, 30) } const currentTime: number = req.body.currentTime - if (!isVideoTimeValid(currentTime, videoDuration)) { + if (!isVideoTimeValid(currentTime, duration)) { return res.fail({ status: HttpStatusCode.BAD_REQUEST_400, message: 'Current time is invalid' -- cgit v1.2.3