From f9f899b9f803eb5159a67781f10649a0cf040677 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Wed, 19 Sep 2018 16:21:09 +0200 Subject: [PATCH] Add redundancy check interval in config --- config/default.yaml | 1 + config/production.yaml.example | 1 + config/test.yaml | 1 + server/initializers/checker.ts | 2 +- server/initializers/constants.ts | 7 +++---- .../schedulers/videos-redundancy-scheduler.ts | 18 +++++++++++------- 6 files changed, 18 insertions(+), 12 deletions(-) diff --git a/config/default.yaml b/config/default.yaml index 00eeaea8c..fa1fb628a 100644 --- a/config/default.yaml +++ b/config/default.yaml @@ -71,6 +71,7 @@ trending: # Once you have defined your strategies, choose which instances you want to cache in admin -> manage follows -> following redundancy: videos: + check_interval: '1 hour' # How often you want to check new videos to cache strategies: # - # size: '10GB' diff --git a/config/production.yaml.example b/config/production.yaml.example index 28770e480..4d8752206 100644 --- a/config/production.yaml.example +++ b/config/production.yaml.example @@ -72,6 +72,7 @@ trending: # Once you have defined your strategies, choose which instances you want to cache in admin -> manage follows -> following redundancy: videos: + check_interval: '1 hour' # How often you want to check new videos to cache strategies: # - # size: '10GB' diff --git a/config/test.yaml b/config/test.yaml index d36d90bbd..ad94b00cd 100644 --- a/config/test.yaml +++ b/config/test.yaml @@ -23,6 +23,7 @@ log: redundancy: videos: + check_interval: '5 seconds' strategies: - size: '10MB' diff --git a/server/initializers/checker.ts b/server/initializers/checker.ts index 8b5280848..a54f6155b 100644 --- a/server/initializers/checker.ts +++ b/server/initializers/checker.ts @@ -75,7 +75,7 @@ function checkMissedConfig () { 'cache.previews.size', 'admin.email', 'signup.enabled', 'signup.limit', 'signup.requires_email_verification', 'signup.filters.cidr.whitelist', 'signup.filters.cidr.blacklist', - 'redundancy.videos.strategies', + 'redundancy.videos.strategies', 'redundancy.videos.check_interval', 'transcoding.enabled', 'transcoding.threads', 'import.videos.http.enabled', 'import.videos.torrent.enabled', 'trending.videos.interval_days', diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index 881978753..03424ffb8 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts @@ -5,7 +5,7 @@ import { ActivityPubActorType } from '../../shared/models/activitypub' import { FollowState } from '../../shared/models/actors' import { VideoAbuseState, VideoImportState, VideoPrivacy } from '../../shared/models/videos' // Do not use barrels, remain constants as independent as possible -import { buildPath, isTestInstance, root, sanitizeHost, sanitizeUrl } from '../helpers/core-utils' +import { buildPath, isTestInstance, parseDuration, root, sanitizeHost, sanitizeUrl } from '../helpers/core-utils' import { NSFWPolicyType } from '../../shared/models/videos/nsfw-policy.type' import { invert } from 'lodash' import { CronRepeatOptions, EveryRepeatOptions } from 'bull' @@ -139,8 +139,7 @@ let SCHEDULER_INTERVALS_MS = { badActorFollow: 60000 * 60, // 1 hour removeOldJobs: 60000 * 60, // 1 hour updateVideos: 60000, // 1 minute - youtubeDLUpdate: 60000 * 60 * 24, // 1 day - videosRedundancy: 60000 * 2 // 2 hours + youtubeDLUpdate: 60000 * 60 * 24 // 1 day } // --------------------------------------------------------------------------- @@ -213,6 +212,7 @@ const CONFIG = { }, REDUNDANCY: { VIDEOS: { + CHECK_INTERVAL: parseDuration(config.get('redundancy.videos.check_interval')), STRATEGIES: buildVideosRedundancy(config.get('redundancy.videos.strategies')) } }, @@ -651,7 +651,6 @@ if (isTestInstance() === true) { SCHEDULER_INTERVALS_MS.badActorFollow = 10000 SCHEDULER_INTERVALS_MS.removeOldJobs = 10000 SCHEDULER_INTERVALS_MS.updateVideos = 5000 - SCHEDULER_INTERVALS_MS.videosRedundancy = 5000 REPEAT_JOBS['videos-views'] = { every: 5000 } REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR = 1 diff --git a/server/lib/schedulers/videos-redundancy-scheduler.ts b/server/lib/schedulers/videos-redundancy-scheduler.ts index 5f9fd9911..960651712 100644 --- a/server/lib/schedulers/videos-redundancy-scheduler.ts +++ b/server/lib/schedulers/videos-redundancy-scheduler.ts @@ -18,7 +18,7 @@ export class VideosRedundancyScheduler extends AbstractScheduler { private static instance: AbstractScheduler private executing = false - protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.videosRedundancy + protected schedulerIntervalMs = CONFIG.REDUNDANCY.VIDEOS.CHECK_INTERVAL private constructor () { super() @@ -50,6 +50,16 @@ export class VideosRedundancyScheduler extends AbstractScheduler { } } + await this.removeExpired() + + this.executing = false + } + + static get Instance () { + return this.instance || (this.instance = new this()) + } + + private async removeExpired () { const expired = await VideoRedundancyModel.listAllExpired() for (const m of expired) { @@ -61,12 +71,6 @@ export class VideosRedundancyScheduler extends AbstractScheduler { logger.error('Cannot remove %s video from our redundancy system.', this.buildEntryLogId(m)) } } - - this.executing = false - } - - static get Instance () { - return this.instance || (this.instance = new this()) } private findVideoToDuplicate (cache: VideosRedundancy) { -- 2.41.0