Basic video redundancy implementation

4 files changed, 77 insertions, 4 deletions

diff --git a/server/initializers/checker.ts b/server/initializers/checker.ts
index 9dd104035..6a2badd35 100644
--- a/server/initializers/checker.ts
+++ b/server/initializers/checker.ts
@@ -7,6 +7,9 @@ import { parse } from 'url'
 import { CONFIG } from './constants'
 import { logger } from '../helpers/logger'
 import { getServerActor } from '../helpers/utils'
+import { VideosRedundancy } from '../../shared/models/redundancy'
+import { isArray } from '../helpers/custom-validators/misc'
+import { uniq } from 'lodash'
 
 async function checkActivityPubUrls () {
   const actor = await getServerActor()
@@ -35,6 +38,20 @@ function checkConfig () {
     return 'NSFW policy setting should be "do_not_list" or "blur" or "display" instead of ' + defaultNSFWPolicy
   }
 
+  const redundancyVideos = config.get<VideosRedundancy[]>('redundancy.videos')
+  if (isArray(redundancyVideos)) {
+    for (const r of redundancyVideos) {
+      if ([ 'most-views' ].indexOf(r.strategy) === -1) {
+        return 'Redundancy video entries should have "most-views" strategy instead of ' + r.strategy
+      }
+    }
+
+    const filtered = uniq(redundancyVideos.map(r => r.strategy))
+    if (filtered.length !== redundancyVideos.length) {
+      return 'Redundancy video entries should have uniq strategies'
+    }
+  }
+
   return null
 }
 
diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts
index 5b7ea5d6c..6b4afbfd8 100644
--- a/server/initializers/constants.ts
+++ b/server/initializers/constants.ts
@@ -1,6 +1,6 @@
 import { IConfig } from 'config'
 import { dirname, join } from 'path'
-import { JobType, VideoRateType, VideoState } from '../../shared/models'
+import { JobType, VideoRateType, VideoRedundancyStrategy, VideoState, VideosRedundancy } from '../../shared/models'
 import { ActivityPubActorType } from '../../shared/models/activitypub'
 import { FollowState } from '../../shared/models/actors'
 import { VideoAbuseState, VideoImportState, VideoPrivacy } from '../../shared/models/videos'
@@ -9,13 +9,14 @@ import { buildPath, isTestInstance, root, sanitizeHost, sanitizeUrl } from '../h
 import { NSFWPolicyType } from '../../shared/models/videos/nsfw-policy.type'
 import { invert } from 'lodash'
 import { CronRepeatOptions, EveryRepeatOptions } from 'bull'
+import * as bytes from 'bytes'
 
 // Use a variable to reload the configuration if we need
 let config: IConfig = require('config')
 
 // ---------------------------------------------------------------------------
 
-const LAST_MIGRATION_VERSION = 265
+const LAST_MIGRATION_VERSION = 270
 
 // ---------------------------------------------------------------------------
 
@@ -137,7 +138,8 @@ let SCHEDULER_INTERVALS_MS = {
   badActorFollow: 60000 * 60, // 1 hour
   removeOldJobs: 60000 * 60, // 1 hour
   updateVideos: 60000, // 1 minute
-  youtubeDLUpdate: 60000 * 60 * 24 // 1 day
+  youtubeDLUpdate: 60000 * 60 * 24, // 1 day
+  videosRedundancy: 60000 * 2 // 2 hours
 }
 
 // ---------------------------------------------------------------------------
@@ -208,6 +210,9 @@ const CONFIG = {
       INTERVAL_DAYS: config.get<number>('trending.videos.interval_days')
     }
   },
+  REDUNDANCY: {
+    VIDEOS: buildVideosRedundancy(config.get<any[]>('redundancy.videos'))
+  },
   ADMIN: {
     get EMAIL () { return config.get<string>('admin.email') }
   },
@@ -321,6 +326,9 @@ const CONSTRAINTS_FIELDS = {
       }
     }
   },
+  VIDEOS_REDUNDANCY: {
+    URL: { min: 3, max: 2000 } // Length
+  },
   VIDEOS: {
     NAME: { min: 3, max: 120 }, // Length
     LANGUAGE: { min: 1, max: 10 }, // Length
@@ -584,6 +592,13 @@ const CACHE = {
   }
 }
 
+const REDUNDANCY = {
+  VIDEOS: {
+    EXPIRES_AFTER_MS: 48 * 3600 * 1000, // 2 days
+    RANDOMIZED_FACTOR: 5
+  }
+}
+
 const ACCEPT_HEADERS = [ 'html', 'application/json' ].concat(ACTIVITY_PUB.POTENTIAL_ACCEPT_HEADERS)
 
 // ---------------------------------------------------------------------------
@@ -629,8 +644,11 @@ if (isTestInstance() === true) {
   SCHEDULER_INTERVALS_MS.badActorFollow = 10000
   SCHEDULER_INTERVALS_MS.removeOldJobs = 10000
   SCHEDULER_INTERVALS_MS.updateVideos = 5000
+  SCHEDULER_INTERVALS_MS.videosRedundancy = 5000
   REPEAT_JOBS['videos-views'] = { every: 5000 }
 
+  REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR = 1
+
   VIDEO_VIEW_LIFETIME = 1000 // 1 second
 
   JOB_ATTEMPTS['email'] = 1
@@ -653,6 +671,7 @@ export {
   CONFIG,
   CONSTRAINTS_FIELDS,
   EMBED_SIZE,
+  REDUNDANCY,
   JOB_CONCURRENCY,
   JOB_ATTEMPTS,
   LAST_MIGRATION_VERSION,
@@ -722,6 +741,17 @@ function updateWebserverConfig () {
   CONFIG.WEBSERVER.HOST = sanitizeHost(CONFIG.WEBSERVER.HOSTNAME + ':' + CONFIG.WEBSERVER.PORT, REMOTE_SCHEME.HTTP)
 }
 
+function buildVideosRedundancy (objs: { strategy: VideoRedundancyStrategy, size: string }[]): VideosRedundancy[] {
+  if (!objs) return []
+
+  return objs.map(obj => {
+    return {
+      strategy: obj.strategy,
+      size: bytes.parse(obj.size)
+    }
+  })
+}
+
 function buildLanguages () {
   const iso639 = require('iso-639-3')
 
diff --git a/server/initializers/database.ts b/server/initializers/database.ts
index b68e1a882..4d57bf8aa 100644
--- a/server/initializers/database.ts
+++ b/server/initializers/database.ts
@@ -27,6 +27,7 @@ import { VideoCaptionModel } from '../models/video/video-caption'
 import { VideoImportModel } from '../models/video/video-import'
 import { VideoViewModel } from '../models/video/video-views'
 import { VideoChangeOwnershipModel } from '../models/video/video-change-ownership'
+import { VideoRedundancyModel } from '../models/redundancy/video-redundancy'
 
 require('pg').defaults.parseInt8 = true // Avoid BIGINT to be converted to string
 
@@ -87,7 +88,8 @@ async function initDatabaseModels (silent: boolean) {
     VideoCommentModel,
     ScheduleVideoUpdateModel,
     VideoImportModel,
-    VideoViewModel
+    VideoViewModel,
+    VideoRedundancyModel
   ])
 
   // Check extensions exist in the database
diff --git a/server/initializers/migrations/0270-server-redundancy.ts b/server/initializers/migrations/0270-server-redundancy.ts
new file mode 100644
index 000000000..903ba8a85
--- /dev/null
+++ b/server/initializers/migrations/0270-server-redundancy.ts
@@ -0,0 +1,24 @@
+import * as Sequelize from 'sequelize'
+
+async function up (utils: {
+  transaction: Sequelize.Transaction
+  queryInterface: Sequelize.QueryInterface
+  sequelize: Sequelize.Sequelize
+}): Promise<any> {
+  {
+    const data = {
+      type: Sequelize.BOOLEAN,
+      allowNull: false,
+      defaultValue: false
+    }
+
+    await utils.queryInterface.addColumn('server', 'redundancyAllowed', data)
+  }
+
+}
+
+function down (options) {
+  throw new Error('Not implemented.')
+}
+
+export { up, down }