Improve redundancy: add 'min_lifetime' configuration

5 files changed, 135 insertions, 104 deletions

diff --git a/server/initializers/checker-after-init.ts b/server/initializers/checker-after-init.ts
new file mode 100644
index 000000000..588526235
--- /dev/null
+++ b/server/initializers/checker-after-init.ts
@@ -0,0 +1,115 @@
+import * as config from 'config'
+import { promisify0, isProdInstance, parseDuration, isTestInstance } from '../helpers/core-utils'
+import { UserModel } from '../models/account/user'
+import { ApplicationModel } from '../models/application/application'
+import { OAuthClientModel } from '../models/oauth/oauth-client'
+import { parse } from 'url'
+import { CONFIG } from './constants'
+import { logger } from '../helpers/logger'
+import { getServerActor } from '../helpers/utils'
+import { RecentlyAddedStrategy, VideosRedundancy } from '../../shared/models/redundancy'
+import { isArray } from '../helpers/custom-validators/misc'
+import { uniq } from 'lodash'
+
+async function checkActivityPubUrls () {
+  const actor = await getServerActor()
+
+  const parsed = parse(actor.url)
+  if (CONFIG.WEBSERVER.HOST !== parsed.host) {
+    const NODE_ENV = config.util.getEnv('NODE_ENV')
+    const NODE_CONFIG_DIR = config.util.getEnv('NODE_CONFIG_DIR')
+
+    logger.warn(
+      'It seems PeerTube was started (and created some data) with another domain name. ' +
+      'This means you will not be able to federate! ' +
+      'Please use %s %s npm run update-host to fix this.',
+      NODE_CONFIG_DIR ? `NODE_CONFIG_DIR=${NODE_CONFIG_DIR}` : '',
+      NODE_ENV ? `NODE_ENV=${NODE_ENV}` : ''
+    )
+  }
+}
+
+// Some checks on configuration files
+// Return an error message, or null if everything is okay
+function checkConfig () {
+  const defaultNSFWPolicy = CONFIG.INSTANCE.DEFAULT_NSFW_POLICY
+
+  // NSFW policy
+  {
+    const available = [ 'do_not_list', 'blur', 'display' ]
+    if (available.indexOf(defaultNSFWPolicy) === -1) {
+      return 'NSFW policy setting should be ' + available.join(' or ') + ' instead of ' + defaultNSFWPolicy
+    }
+  }
+
+  // Redundancies
+  const redundancyVideos = CONFIG.REDUNDANCY.VIDEOS.STRATEGIES
+  if (isArray(redundancyVideos)) {
+    const available = [ 'most-views', 'trending', 'recently-added' ]
+    for (const r of redundancyVideos) {
+      if (available.indexOf(r.strategy) === -1) {
+        return 'Videos redundancy should have ' + available.join(' or ') + ' strategy instead of ' + r.strategy
+      }
+
+      // Lifetime should not be < 10 hours
+      if (!isTestInstance() && r.minLifetime < 1000 * 3600 * 10) {
+        return 'Video redundancy minimum lifetime should be >= 10 hours for strategy ' + r.strategy
+      }
+    }
+
+    const filtered = uniq(redundancyVideos.map(r => r.strategy))
+    if (filtered.length !== redundancyVideos.length) {
+      return 'Redundancy video entries should have unique strategies'
+    }
+
+    const recentlyAddedStrategy = redundancyVideos.find(r => r.strategy === 'recently-added') as RecentlyAddedStrategy
+    if (recentlyAddedStrategy && isNaN(recentlyAddedStrategy.minViews)) {
+      return 'Min views in recently added strategy is not a number'
+    }
+  }
+
+  if (isProdInstance()) {
+    const configStorage = config.get('storage')
+    for (const key of Object.keys(configStorage)) {
+      if (configStorage[key].startsWith('storage/')) {
+        logger.warn(
+          'Directory of %s should not be in the production directory of PeerTube. Please check your production configuration file.',
+          key
+        )
+      }
+    }
+  }
+
+  return null
+}
+
+// We get db by param to not import it in this file (import orders)
+async function clientsExist () {
+  const totalClients = await OAuthClientModel.countTotal()
+
+  return totalClients !== 0
+}
+
+// We get db by param to not import it in this file (import orders)
+async function usersExist () {
+  const totalUsers = await UserModel.countTotal()
+
+  return totalUsers !== 0
+}
+
+// We get db by param to not import it in this file (import orders)
+async function applicationExist () {
+  const totalApplication = await ApplicationModel.countTotal()
+
+  return totalApplication !== 0
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+  checkConfig,
+  clientsExist,
+  usersExist,
+  applicationExist,
+  checkActivityPubUrls
+}
diff --git a/server/initializers/checker.ts b/server/initializers/checker-before-init.ts
index 5b068caa1..4f46d406a 100644
--- a/server/initializers/checker.ts
+++ b/server/initializers/checker-before-init.ts
@@ -1,78 +1,8 @@
 import * as config from 'config'
-import { promisify0, isProdInstance } from '../helpers/core-utils'
-import { UserModel } from '../models/account/user'
-import { ApplicationModel } from '../models/application/application'
-import { OAuthClientModel } from '../models/oauth/oauth-client'
-import { parse } from 'url'
-import { CONFIG } from './constants'
-import { logger } from '../helpers/logger'
-import { getServerActor } from '../helpers/utils'
-import { RecentlyAddedStrategy, VideosRedundancy } from '../../shared/models/redundancy'
+import { promisify0 } from '../helpers/core-utils'
 import { isArray } from '../helpers/custom-validators/misc'
-import { uniq } from 'lodash'
-
-async function checkActivityPubUrls () {
-  const actor = await getServerActor()
-
-  const parsed = parse(actor.url)
-  if (CONFIG.WEBSERVER.HOST !== parsed.host) {
-    const NODE_ENV = config.util.getEnv('NODE_ENV')
-    const NODE_CONFIG_DIR = config.util.getEnv('NODE_CONFIG_DIR')
-
-    logger.warn(
-      'It seems PeerTube was started (and created some data) with another domain name. ' +
-      'This means you will not be able to federate! ' +
-      'Please use %s %s npm run update-host to fix this.',
-      NODE_CONFIG_DIR ? `NODE_CONFIG_DIR=${NODE_CONFIG_DIR}` : '',
-      NODE_ENV ? `NODE_ENV=${NODE_ENV}` : ''
-    )
-  }
-}
 
-// Some checks on configuration files
-// Return an error message, or null if everything is okay
-function checkConfig () {
-  const defaultNSFWPolicy = config.get<string>('instance.default_nsfw_policy')
-
-  // NSFW policy
-  if ([ 'do_not_list', 'blur', 'display' ].indexOf(defaultNSFWPolicy) === -1) {
-    return 'NSFW policy setting should be "do_not_list" or "blur" or "display" instead of ' + defaultNSFWPolicy
-  }
-
-  // Redundancies
-  const redundancyVideos = config.get<VideosRedundancy[]>('redundancy.videos.strategies')
-  if (isArray(redundancyVideos)) {
-    for (const r of redundancyVideos) {
-      if ([ 'most-views', 'trending', 'recently-added' ].indexOf(r.strategy) === -1) {
-        return 'Redundancy video entries should have "most-views" strategy instead of ' + r.strategy
-      }
-    }
-
-    const filtered = uniq(redundancyVideos.map(r => r.strategy))
-    if (filtered.length !== redundancyVideos.length) {
-      return 'Redundancy video entries should have unique strategies'
-    }
-
-    const recentlyAddedStrategy = redundancyVideos.find(r => r.strategy === 'recently-added') as RecentlyAddedStrategy
-    if (recentlyAddedStrategy && isNaN(recentlyAddedStrategy.minViews)) {
-      return 'Min views in recently added strategy is not a number'
-    }
-  }
-
-  if (isProdInstance()) {
-    const configStorage = config.get('storage')
-    for (const key of Object.keys(configStorage)) {
-      if (configStorage[key].startsWith('storage/')) {
-        logger.warn(
-          'Directory of %s should not be in the production directory of PeerTube. Please check your production configuration file.',
-          key
-        )
-      }
-    }
-  }
-
-  return null
-}
+// ONLY USE CORE MODULES IN THIS FILE!
 
 // Check the config files
 function checkMissedConfig () {
@@ -109,6 +39,14 @@ function checkMissedConfig () {
     }
   }
 
+  const redundancyVideos = config.get<any>('redundancy.videos.strategies')
+  if (isArray(redundancyVideos)) {
+    for (const r of redundancyVideos) {
+      if (!r.size) miss.push('redundancy.videos.strategies.size')
+      if (!r.min_lifetime) miss.push('redundancy.videos.strategies.min_lifetime')
+    }
+  }
+
   const missingAlternatives = requiredAlternatives.filter(
     set => !set.find(alternative => !alternative.find(key => !config.has(key)))
   )
@@ -163,36 +101,10 @@ async function checkFFmpegEncoders (): Promise<Map<string, boolean>> {
   }
 }
 
-// We get db by param to not import it in this file (import orders)
-async function clientsExist () {
-  const totalClients = await OAuthClientModel.countTotal()
-
-  return totalClients !== 0
-}
-
-// We get db by param to not import it in this file (import orders)
-async function usersExist () {
-  const totalUsers = await UserModel.countTotal()
-
-  return totalUsers !== 0
-}
-
-// We get db by param to not import it in this file (import orders)
-async function applicationExist () {
-  const totalApplication = await ApplicationModel.countTotal()
-
-  return totalApplication !== 0
-}
-
 // ---------------------------------------------------------------------------
 
 export {
-  checkConfig,
   checkFFmpeg,
   checkFFmpegEncoders,
-  checkMissedConfig,
-  clientsExist,
-  usersExist,
-  applicationExist,
-  checkActivityPubUrls
+  checkMissedConfig
 }
diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts
index 03424ffb8..947cbda28 100644
--- a/server/initializers/constants.ts
+++ b/server/initializers/constants.ts
@@ -601,7 +601,6 @@ const MEMOIZE_TTL = {
 
 const REDUNDANCY = {
   VIDEOS: {
-    EXPIRES_AFTER_MS: 48 * 3600 * 1000, // 2 days
     RANDOMIZED_FACTOR: 5
   }
 }
@@ -750,10 +749,16 @@ function updateWebserverConfig () {
   CONFIG.WEBSERVER.HOST = sanitizeHost(CONFIG.WEBSERVER.HOSTNAME + ':' + CONFIG.WEBSERVER.PORT, REMOTE_SCHEME.HTTP)
 }
 
-function buildVideosRedundancy (objs: VideosRedundancy[]): VideosRedundancy[] {
+function buildVideosRedundancy (objs: any[]): VideosRedundancy[] {
   if (!objs) return []
 
-  return objs.map(obj => Object.assign(obj, { size: bytes.parse(obj.size) }))
+  return objs.map(obj => {
+    return Object.assign(obj, {
+      minLifetime: parseDuration(obj.min_lifetime),
+      size: bytes.parse(obj.size),
+      minViews: obj.min_views
+    })
+  })
 }
 
 function buildLanguages () {
diff --git a/server/initializers/index.ts b/server/initializers/index.ts
index 332702774..fe9190a9c 100644
--- a/server/initializers/index.ts
+++ b/server/initializers/index.ts
@@ -1,6 +1,5 @@
 // Constants first, database in second!
 export * from './constants'
 export * from './database'
-export * from './checker'
 export * from './installer'
 export * from './migrator'
diff --git a/server/initializers/installer.ts b/server/initializers/installer.ts
index 818bb04a2..c952ad46c 100644
--- a/server/initializers/installer.ts
+++ b/server/initializers/installer.ts
@@ -5,7 +5,7 @@ import { createApplicationActor, createUserAccountAndChannel } from '../lib/user
 import { UserModel } from '../models/account/user'
 import { ApplicationModel } from '../models/application/application'
 import { OAuthClientModel } from '../models/oauth/oauth-client'
-import { applicationExist, clientsExist, usersExist } from './checker'
+import { applicationExist, clientsExist, usersExist } from './checker-after-init'
 import { CACHE, CONFIG, LAST_MIGRATION_VERSION } from './constants'
 import { sequelizeTypescript } from './database'
 import { remove, ensureDir } from 'fs-extra'