Speed up overviews route

5 files changed, 44 insertions, 40 deletions

diff --git a/server/controllers/api/overviews.ts b/server/controllers/api/overviews.ts
index da941c0ac..cc3cc54a7 100644
--- a/server/controllers/api/overviews.ts
+++ b/server/controllers/api/overviews.ts
@@ -4,8 +4,9 @@ import { VideoModel } from '../../models/video/video'
 import { asyncMiddleware } from '../../middlewares'
 import { TagModel } from '../../models/video/tag'
 import { VideosOverview } from '../../../shared/models/overviews'
-import { OVERVIEWS, ROUTE_CACHE_LIFETIME } from '../../initializers'
+import { MEMOIZE_TTL, OVERVIEWS, ROUTE_CACHE_LIFETIME } from '../../initializers'
 import { cacheRoute } from '../../middlewares/cache'
+import * as memoizee from 'memoizee'
 
 const overviewsRouter = express.Router()
 
@@ -23,10 +24,17 @@ export { overviewsRouter }
 // This endpoint could be quite long, but we cache it
 async function getVideosOverview (req: express.Request, res: express.Response) {
   const attributes = await buildSamples()
+
+  const [ categories, channels, tags ] = await Promise.all([
+    Promise.all(attributes.categories.map(c => getVideosByCategory(c, res))),
+    Promise.all(attributes.channels.map(c => getVideosByChannel(c, res))),
+    Promise.all(attributes.tags.map(t => getVideosByTag(t, res)))
+  ])
+
   const result: VideosOverview = {
-    categories: await Promise.all(attributes.categories.map(c => getVideosByCategory(c, res))),
-    channels: await Promise.all(attributes.channels.map(c => getVideosByChannel(c, res))),
-    tags: await Promise.all(attributes.tags.map(t => getVideosByTag(t, res)))
+    categories,
+    channels,
+    tags
   }
 
   // Cleanup our object
@@ -37,7 +45,7 @@ async function getVideosOverview (req: express.Request, res: express.Response) {
   return res.json(result)
 }
 
-async function buildSamples () {
+const buildSamples = memoizee(async function () {
   const [ categories, channels, tags ] = await Promise.all([
     VideoModel.getRandomFieldSamples('category', OVERVIEWS.VIDEOS.SAMPLE_THRESHOLD, OVERVIEWS.VIDEOS.SAMPLES_COUNT),
     VideoModel.getRandomFieldSamples('channelId', OVERVIEWS.VIDEOS.SAMPLE_THRESHOLD ,OVERVIEWS.VIDEOS.SAMPLES_COUNT),
@@ -45,7 +53,7 @@ async function buildSamples () {
   ])
 
   return { categories, channels, tags }
-}
+}, { maxAge: MEMOIZE_TTL.OVERVIEWS_SAMPLE })
 
 async function getVideosByTag (tag: string, res: express.Response) {
   const videos = await getVideos(res, { tagsOneOf: [ tag ] })
@@ -84,14 +92,16 @@ async function getVideos (
   res: express.Response,
   where: { videoChannelId?: number, tagsOneOf?: string[], categoryOneOf?: number[] }
 ) {
-  const { data } = await VideoModel.listForApi(Object.assign({
+  const query = Object.assign({
     start: 0,
     count: 10,
     sort: '-createdAt',
     includeLocalVideos: true,
     nsfw: buildNSFWFilter(res),
     withFiles: false
-  }, where))
+  }, where)
+
+  const { data } = await VideoModel.listForApi(query, false)
 
   return data.map(d => d.toFormattedJSON())
 }
diff --git a/server/helpers/utils.ts b/server/helpers/utils.ts
index a1ed8e72d..a42474417 100644
--- a/server/helpers/utils.ts
+++ b/server/helpers/utils.ts
@@ -1,12 +1,12 @@
 import { ResultList } from '../../shared'
 import { CONFIG } from '../initializers'
-import { ActorModel } from '../models/activitypub/actor'
 import { ApplicationModel } from '../models/application/application'
 import { pseudoRandomBytesPromise, sha256 } from './core-utils'
 import { logger } from './logger'
 import { join } from 'path'
 import { Instance as ParseTorrent } from 'parse-torrent'
 import { remove } from 'fs-extra'
+import * as memoizee from 'memoizee'
 
 function deleteFileAsync (path: string) {
   remove(path)
@@ -36,24 +36,12 @@ function getFormattedObjects<U, T extends FormattableToJSON> (objects: T[], obje
   } as ResultList<U>
 }
 
-async function getServerActor () {
-  if (getServerActor.serverActor === undefined) {
-    const application = await ApplicationModel.load()
-    if (!application) throw Error('Could not load Application from database.')
+const getServerActor = memoizee(async function () {
+  const application = await ApplicationModel.load()
+  if (!application) throw Error('Could not load Application from database.')
 
-    getServerActor.serverActor = application.Account.Actor
-  }
-
-  if (!getServerActor.serverActor) {
-    logger.error('Cannot load server actor.')
-    process.exit(0)
-  }
-
-  return Promise.resolve(getServerActor.serverActor)
-}
-namespace getServerActor {
-  export let serverActor: ActorModel
-}
+  return application.Account.Actor
+})
 
 function generateVideoTmpPath (target: string | ParseTorrent) {
   const id = typeof target === 'string' ? target : target.infoHash
diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts
index 5f7bcbd48..9cccb0919 100644
--- a/server/initializers/constants.ts
+++ b/server/initializers/constants.ts
@@ -592,6 +592,10 @@ const CACHE = {
   }
 }
 
+const MEMOIZE_TTL = {
+  OVERVIEWS_SAMPLE: 1000 * 3600 * 4 // 4 hours
+}
+
 const REDUNDANCY = {
   VIDEOS: {
     EXPIRES_AFTER_MS: 48 * 3600 * 1000, // 2 days
@@ -708,6 +712,7 @@ export {
   VIDEO_ABUSE_STATES,
   JOB_REQUEST_TIMEOUT,
   USER_PASSWORD_RESET_LIFETIME,
+  MEMOIZE_TTL,
   USER_EMAIL_VERIFY_LIFETIME,
   IMAGE_MIMETYPE_EXT,
   OVERVIEWS,
diff --git a/server/lib/schedulers/videos-redundancy-scheduler.ts b/server/lib/schedulers/videos-redundancy-scheduler.ts
index 8b91d750b..7079600a9 100644
--- a/server/lib/schedulers/videos-redundancy-scheduler.ts
+++ b/server/lib/schedulers/videos-redundancy-scheduler.ts
@@ -81,7 +81,7 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
     }
 
     if (cache.strategy === 'recently-added') {
-      const minViews = (cache as RecentlyAddedStrategy).minViews
+      const minViews = cache.minViews
       return VideoRedundancyModel.findRecentlyAddedToDuplicate(REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR, minViews)
     }
   }
diff --git a/server/models/video/video.ts b/server/models/video/video.ts
index 23d1dedd6..b7d3f184f 100644
--- a/server/models/video/video.ts
+++ b/server/models/video/video.ts
@@ -929,7 +929,7 @@ export class VideoModel extends Model<VideoModel> {
     videoChannelId?: number,
     actorId?: number
     trendingDays?: number
-  }) {
+  }, countVideos = true) {
     const query: IFindOptions<VideoModel> = {
       offset: options.start,
       limit: options.count,
@@ -962,7 +962,7 @@ export class VideoModel extends Model<VideoModel> {
       trendingDays
     }
 
-    return VideoModel.getAvailableForApi(query, queryOptions)
+    return VideoModel.getAvailableForApi(query, queryOptions, countVideos)
   }
 
   static async searchAndPopulateAccountAndServer (options: {
@@ -1164,7 +1164,14 @@ export class VideoModel extends Model<VideoModel> {
   }
 
   // threshold corresponds to how many video the field should have to be returned
-  static getRandomFieldSamples (field: 'category' | 'channelId', threshold: number, count: number) {
+  static async getRandomFieldSamples (field: 'category' | 'channelId', threshold: number, count: number) {
+    const actorId = (await getServerActor()).id
+
+    const scopeOptions = {
+      actorId,
+      includeLocalVideos: true
+    }
+
     const query: IFindOptions<VideoModel> = {
       attributes: [ field ],
       limit: count,
@@ -1172,17 +1179,11 @@ export class VideoModel extends Model<VideoModel> {
       having: Sequelize.where(Sequelize.fn('COUNT', Sequelize.col(field)), {
         [ Sequelize.Op.gte ]: threshold
       }) as any, // FIXME: typings
-      where: {
-        [ field ]: {
-          [ Sequelize.Op.not ]: null
-        },
-        privacy: VideoPrivacy.PUBLIC,
-        state: VideoState.PUBLISHED
-      },
       order: [ this.sequelize.random() ]
     }
 
-    return VideoModel.findAll(query)
+    return VideoModel.scope({ method: [ ScopeNames.AVAILABLE_FOR_LIST_IDS, scopeOptions ] })
+                     .findAll(query)
                      .then(rows => rows.map(r => r[ field ]))
   }
 
@@ -1210,7 +1211,7 @@ export class VideoModel extends Model<VideoModel> {
     return {}
   }
 
-  private static async getAvailableForApi (query: IFindOptions<VideoModel>, options: AvailableForListIDsOptions) {
+  private static async getAvailableForApi (query: IFindOptions<VideoModel>, options: AvailableForListIDsOptions, countVideos = true) {
     const idsScope = {
       method: [
         ScopeNames.AVAILABLE_FOR_LIST_IDS, options
@@ -1227,7 +1228,7 @@ export class VideoModel extends Model<VideoModel> {
     }
 
     const [ count, rowsId ] = await Promise.all([
-      VideoModel.scope(countScope).count(countQuery),
+      countVideos ? VideoModel.scope(countScope).count(countQuery) : Promise.resolve(undefined),
       VideoModel.scope(idsScope).findAll(query)
     ])
     const ids = rowsId.map(r => r.id)