From 7348b1fd84dee869b3c36554aea6797f09d4ceed Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Fri, 14 Sep 2018 11:52:23 +0200 Subject: Speed up overviews route --- server/controllers/api/overviews.ts | 26 ++++++++++++++------- server/helpers/utils.ts | 24 +++++-------------- server/initializers/constants.ts | 5 ++++ .../lib/schedulers/videos-redundancy-scheduler.ts | 2 +- server/models/video/video.ts | 27 +++++++++++----------- 5 files changed, 44 insertions(+), 40 deletions(-) (limited to 'server') diff --git a/server/controllers/api/overviews.ts b/server/controllers/api/overviews.ts index da941c0ac..cc3cc54a7 100644 --- a/server/controllers/api/overviews.ts +++ b/server/controllers/api/overviews.ts @@ -4,8 +4,9 @@ import { VideoModel } from '../../models/video/video' import { asyncMiddleware } from '../../middlewares' import { TagModel } from '../../models/video/tag' import { VideosOverview } from '../../../shared/models/overviews' -import { OVERVIEWS, ROUTE_CACHE_LIFETIME } from '../../initializers' +import { MEMOIZE_TTL, OVERVIEWS, ROUTE_CACHE_LIFETIME } from '../../initializers' import { cacheRoute } from '../../middlewares/cache' +import * as memoizee from 'memoizee' const overviewsRouter = express.Router() @@ -23,10 +24,17 @@ export { overviewsRouter } // This endpoint could be quite long, but we cache it async function getVideosOverview (req: express.Request, res: express.Response) { const attributes = await buildSamples() + + const [ categories, channels, tags ] = await Promise.all([ + Promise.all(attributes.categories.map(c => getVideosByCategory(c, res))), + Promise.all(attributes.channels.map(c => getVideosByChannel(c, res))), + Promise.all(attributes.tags.map(t => getVideosByTag(t, res))) + ]) + const result: VideosOverview = { - categories: await Promise.all(attributes.categories.map(c => getVideosByCategory(c, res))), - channels: await Promise.all(attributes.channels.map(c => getVideosByChannel(c, res))), - tags: await Promise.all(attributes.tags.map(t => getVideosByTag(t, res))) + categories, + channels, + tags } // Cleanup our object @@ -37,7 +45,7 @@ async function getVideosOverview (req: express.Request, res: express.Response) { return res.json(result) } -async function buildSamples () { +const buildSamples = memoizee(async function () { const [ categories, channels, tags ] = await Promise.all([ VideoModel.getRandomFieldSamples('category', OVERVIEWS.VIDEOS.SAMPLE_THRESHOLD, OVERVIEWS.VIDEOS.SAMPLES_COUNT), VideoModel.getRandomFieldSamples('channelId', OVERVIEWS.VIDEOS.SAMPLE_THRESHOLD ,OVERVIEWS.VIDEOS.SAMPLES_COUNT), @@ -45,7 +53,7 @@ async function buildSamples () { ]) return { categories, channels, tags } -} +}, { maxAge: MEMOIZE_TTL.OVERVIEWS_SAMPLE }) async function getVideosByTag (tag: string, res: express.Response) { const videos = await getVideos(res, { tagsOneOf: [ tag ] }) @@ -84,14 +92,16 @@ async function getVideos ( res: express.Response, where: { videoChannelId?: number, tagsOneOf?: string[], categoryOneOf?: number[] } ) { - const { data } = await VideoModel.listForApi(Object.assign({ + const query = Object.assign({ start: 0, count: 10, sort: '-createdAt', includeLocalVideos: true, nsfw: buildNSFWFilter(res), withFiles: false - }, where)) + }, where) + + const { data } = await VideoModel.listForApi(query, false) return data.map(d => d.toFormattedJSON()) } diff --git a/server/helpers/utils.ts b/server/helpers/utils.ts index a1ed8e72d..a42474417 100644 --- a/server/helpers/utils.ts +++ b/server/helpers/utils.ts @@ -1,12 +1,12 @@ import { ResultList } from '../../shared' import { CONFIG } from '../initializers' -import { ActorModel } from '../models/activitypub/actor' import { ApplicationModel } from '../models/application/application' import { pseudoRandomBytesPromise, sha256 } from './core-utils' import { logger } from './logger' import { join } from 'path' import { Instance as ParseTorrent } from 'parse-torrent' import { remove } from 'fs-extra' +import * as memoizee from 'memoizee' function deleteFileAsync (path: string) { remove(path) @@ -36,24 +36,12 @@ function getFormattedObjects (objects: T[], obje } as ResultList } -async function getServerActor () { - if (getServerActor.serverActor === undefined) { - const application = await ApplicationModel.load() - if (!application) throw Error('Could not load Application from database.') +const getServerActor = memoizee(async function () { + const application = await ApplicationModel.load() + if (!application) throw Error('Could not load Application from database.') - getServerActor.serverActor = application.Account.Actor - } - - if (!getServerActor.serverActor) { - logger.error('Cannot load server actor.') - process.exit(0) - } - - return Promise.resolve(getServerActor.serverActor) -} -namespace getServerActor { - export let serverActor: ActorModel -} + return application.Account.Actor +}) function generateVideoTmpPath (target: string | ParseTorrent) { const id = typeof target === 'string' ? target : target.infoHash diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index 5f7bcbd48..9cccb0919 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts @@ -592,6 +592,10 @@ const CACHE = { } } +const MEMOIZE_TTL = { + OVERVIEWS_SAMPLE: 1000 * 3600 * 4 // 4 hours +} + const REDUNDANCY = { VIDEOS: { EXPIRES_AFTER_MS: 48 * 3600 * 1000, // 2 days @@ -708,6 +712,7 @@ export { VIDEO_ABUSE_STATES, JOB_REQUEST_TIMEOUT, USER_PASSWORD_RESET_LIFETIME, + MEMOIZE_TTL, USER_EMAIL_VERIFY_LIFETIME, IMAGE_MIMETYPE_EXT, OVERVIEWS, diff --git a/server/lib/schedulers/videos-redundancy-scheduler.ts b/server/lib/schedulers/videos-redundancy-scheduler.ts index 8b91d750b..7079600a9 100644 --- a/server/lib/schedulers/videos-redundancy-scheduler.ts +++ b/server/lib/schedulers/videos-redundancy-scheduler.ts @@ -81,7 +81,7 @@ export class VideosRedundancyScheduler extends AbstractScheduler { } if (cache.strategy === 'recently-added') { - const minViews = (cache as RecentlyAddedStrategy).minViews + const minViews = cache.minViews return VideoRedundancyModel.findRecentlyAddedToDuplicate(REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR, minViews) } } diff --git a/server/models/video/video.ts b/server/models/video/video.ts index 23d1dedd6..b7d3f184f 100644 --- a/server/models/video/video.ts +++ b/server/models/video/video.ts @@ -929,7 +929,7 @@ export class VideoModel extends Model { videoChannelId?: number, actorId?: number trendingDays?: number - }) { + }, countVideos = true) { const query: IFindOptions = { offset: options.start, limit: options.count, @@ -962,7 +962,7 @@ export class VideoModel extends Model { trendingDays } - return VideoModel.getAvailableForApi(query, queryOptions) + return VideoModel.getAvailableForApi(query, queryOptions, countVideos) } static async searchAndPopulateAccountAndServer (options: { @@ -1164,7 +1164,14 @@ export class VideoModel extends Model { } // threshold corresponds to how many video the field should have to be returned - static getRandomFieldSamples (field: 'category' | 'channelId', threshold: number, count: number) { + static async getRandomFieldSamples (field: 'category' | 'channelId', threshold: number, count: number) { + const actorId = (await getServerActor()).id + + const scopeOptions = { + actorId, + includeLocalVideos: true + } + const query: IFindOptions = { attributes: [ field ], limit: count, @@ -1172,17 +1179,11 @@ export class VideoModel extends Model { having: Sequelize.where(Sequelize.fn('COUNT', Sequelize.col(field)), { [ Sequelize.Op.gte ]: threshold }) as any, // FIXME: typings - where: { - [ field ]: { - [ Sequelize.Op.not ]: null - }, - privacy: VideoPrivacy.PUBLIC, - state: VideoState.PUBLISHED - }, order: [ this.sequelize.random() ] } - return VideoModel.findAll(query) + return VideoModel.scope({ method: [ ScopeNames.AVAILABLE_FOR_LIST_IDS, scopeOptions ] }) + .findAll(query) .then(rows => rows.map(r => r[ field ])) } @@ -1210,7 +1211,7 @@ export class VideoModel extends Model { return {} } - private static async getAvailableForApi (query: IFindOptions, options: AvailableForListIDsOptions) { + private static async getAvailableForApi (query: IFindOptions, options: AvailableForListIDsOptions, countVideos = true) { const idsScope = { method: [ ScopeNames.AVAILABLE_FOR_LIST_IDS, options @@ -1227,7 +1228,7 @@ export class VideoModel extends Model { } const [ count, rowsId ] = await Promise.all([ - VideoModel.scope(countScope).count(countQuery), + countVideos ? VideoModel.scope(countScope).count(countQuery) : Promise.resolve(undefined), VideoModel.scope(idsScope).findAll(query) ]) const ids = rowsId.map(r => r.id) -- cgit v1.2.3