2 files changed, 32 insertions, 97 deletions

diff --git a/server/lib/schedulers/videos-redundancy-scheduler.ts b/server/lib/schedulers/videos-redundancy-scheduler.ts
index ee9ba1766..960651712 100644
--- a/server/lib/schedulers/videos-redundancy-scheduler.ts
+++ b/server/lib/schedulers/videos-redundancy-scheduler.ts
@@ -1,10 +1,9 @@
 import { AbstractScheduler } from './abstract-scheduler'
 import { CONFIG, JOB_TTL, REDUNDANCY, SCHEDULER_INTERVALS_MS } from '../../initializers'
 import { logger } from '../../helpers/logger'
-import { VideoRedundancyStrategy } from '../../../shared/models/redundancy'
+import { VideoRedundancyStrategy, VideosRedundancy } from '../../../shared/models/redundancy'
 import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy'
 import { VideoFileModel } from '../../models/video/video-file'
-import { sortBy } from 'lodash'
 import { downloadWebTorrentVideo } from '../../helpers/webtorrent'
 import { join } from 'path'
 import { rename } from 'fs-extra'
@@ -12,7 +11,6 @@ import { getServerActor } from '../../helpers/utils'
 import { sendCreateCacheFile, sendUpdateCacheFile } from '../activitypub/send'
 import { VideoModel } from '../../models/video/video'
 import { getVideoCacheFileActivityPubUrl } from '../activitypub/url'
-import { removeVideoRedundancy } from '../redundancy'
 import { isTestInstance } from '../../helpers/core-utils'
 
 export class VideosRedundancyScheduler extends AbstractScheduler {
@@ -20,7 +18,7 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
   private static instance: AbstractScheduler
   private executing = false
 
-  protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.videosRedundancy
+  protected schedulerIntervalMs = CONFIG.REDUNDANCY.VIDEOS.CHECK_INTERVAL
 
   private constructor () {
     super()
@@ -31,17 +29,15 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
 
     this.executing = true
 
-    for (const obj of CONFIG.REDUNDANCY.VIDEOS) {
-
+    for (const obj of CONFIG.REDUNDANCY.VIDEOS.STRATEGIES) {
       try {
-        const videoToDuplicate = await this.findVideoToDuplicate(obj.strategy)
+        const videoToDuplicate = await this.findVideoToDuplicate(obj)
         if (!videoToDuplicate) continue
 
         const videoFiles = videoToDuplicate.VideoFiles
         videoFiles.forEach(f => f.Video = videoToDuplicate)
 
-        const videosRedundancy = await VideoRedundancyModel.getVideoFiles(obj.strategy)
-        if (this.isTooHeavy(videosRedundancy, videoFiles, obj.size)) {
+        if (await this.isTooHeavy(obj.strategy, videoFiles, obj.size)) {
           if (!isTestInstance()) logger.info('Video %s is too big for our cache, skipping.', videoToDuplicate.url)
           continue
         }
@@ -54,6 +50,16 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
       }
     }
 
+    await this.removeExpired()
+
+    this.executing = false
+  }
+
+  static get Instance () {
+    return this.instance || (this.instance = new this())
+  }
+
+  private async removeExpired () {
     const expired = await VideoRedundancyModel.listAllExpired()
 
     for (const m of expired) {
@@ -65,16 +71,21 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
         logger.error('Cannot remove %s video from our redundancy system.', this.buildEntryLogId(m))
       }
     }
-
-    this.executing = false
   }
 
-  static get Instance () {
-    return this.instance || (this.instance = new this())
-  }
+  private findVideoToDuplicate (cache: VideosRedundancy) {
+    if (cache.strategy === 'most-views') {
+      return VideoRedundancyModel.findMostViewToDuplicate(REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR)
+    }
+
+    if (cache.strategy === 'trending') {
+      return VideoRedundancyModel.findTrendingToDuplicate(REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR)
+    }
 
-  private findVideoToDuplicate (strategy: VideoRedundancyStrategy) {
-    if (strategy === 'most-views') return VideoRedundancyModel.findMostViewToDuplicate(REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR)
+    if (cache.strategy === 'recently-added') {
+      const minViews = cache.minViews
+      return VideoRedundancyModel.findRecentlyAddedToDuplicate(REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR, minViews)
+    }
   }
 
   private async createVideoRedundancy (strategy: VideoRedundancyStrategy, filesToDuplicate: VideoFileModel[]) {
@@ -120,27 +131,10 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
     }
   }
 
-  // Unused, but could be useful in the future, with a custom strategy
-  private async purgeVideosIfNeeded (videosRedundancy: VideoRedundancyModel[], filesToDuplicate: VideoFileModel[], maxSize: number) {
-    const sortedVideosRedundancy = sortBy(videosRedundancy, 'createdAt')
-
-    while (this.isTooHeavy(sortedVideosRedundancy, filesToDuplicate, maxSize)) {
-      const toDelete = sortedVideosRedundancy.shift()
-
-      const videoFile = toDelete.VideoFile
-      logger.info('Purging video %s (resolution %d) from our redundancy system.', videoFile.Video.url, videoFile.resolution)
-
-      await removeVideoRedundancy(toDelete, undefined)
-    }
-
-    return sortedVideosRedundancy
-  }
-
-  private isTooHeavy (videosRedundancy: VideoRedundancyModel[], filesToDuplicate: VideoFileModel[], maxSizeArg: number) {
+  private async isTooHeavy (strategy: VideoRedundancyStrategy, filesToDuplicate: VideoFileModel[], maxSizeArg: number) {
     const maxSize = maxSizeArg - this.getTotalFileSizes(filesToDuplicate)
 
-    const redundancyReducer = (previous: number, current: VideoRedundancyModel) => previous + current.VideoFile.size
-    const totalDuplicated = videosRedundancy.reduce(redundancyReducer, 0)
+    const totalDuplicated = await VideoRedundancyModel.getTotalDuplicated(strategy)
 
     return totalDuplicated > maxSize
   }
diff --git a/server/lib/schedulers/youtube-dl-update-scheduler.ts b/server/lib/schedulers/youtube-dl-update-scheduler.ts
index faadb4334..461cd045e 100644
--- a/server/lib/schedulers/youtube-dl-update-scheduler.ts
+++ b/server/lib/schedulers/youtube-dl-update-scheduler.ts
@@ -1,13 +1,6 @@
-// Thanks: https://github.com/przemyslawpluta/node-youtube-dl/blob/master/lib/downloader.js
-// We rewrote it to avoid sync calls
-
 import { AbstractScheduler } from './abstract-scheduler'
 import { SCHEDULER_INTERVALS_MS } from '../../initializers'
-import { logger } from '../../helpers/logger'
-import * as request from 'request'
-import { createWriteStream, ensureDir, writeFile } from 'fs-extra'
-import { join } from 'path'
-import { root } from '../../helpers/core-utils'
+import { updateYoutubeDLBinary } from '../../helpers/youtube-dl'
 
 export class YoutubeDlUpdateScheduler extends AbstractScheduler {
 
@@ -19,60 +12,8 @@ export class YoutubeDlUpdateScheduler extends AbstractScheduler {
     super()
   }
 
-  async execute () {
-    logger.info('Updating youtubeDL binary.')
-
-    const binDirectory = join(root(), 'node_modules', 'youtube-dl', 'bin')
-    const bin = join(binDirectory, 'youtube-dl')
-    const detailsPath = join(binDirectory, 'details')
-    const url = 'https://yt-dl.org/downloads/latest/youtube-dl'
-
-    await ensureDir(binDirectory)
-
-    return new Promise(res => {
-      request.get(url, { followRedirect: false }, (err, result) => {
-        if (err) {
-          logger.error('Cannot update youtube-dl.', { err })
-          return res()
-        }
-
-        if (result.statusCode !== 302) {
-          logger.error('youtube-dl update error: did not get redirect for the latest version link. Status %d', result.statusCode)
-          return res()
-        }
-
-        const url = result.headers.location
-        const downloadFile = request.get(url)
-        const newVersion = /yt-dl\.org\/downloads\/(\d{4}\.\d\d\.\d\d(\.\d)?)\/youtube-dl/.exec(url)[ 1 ]
-
-        downloadFile.on('response', result => {
-          if (result.statusCode !== 200) {
-            logger.error('Cannot update youtube-dl: new version response is not 200, it\'s %d.', result.statusCode)
-            return res()
-          }
-
-          downloadFile.pipe(createWriteStream(bin, { mode: 493 }))
-        })
-
-        downloadFile.on('error', err => {
-          logger.error('youtube-dl update error.', { err })
-          return res()
-        })
-
-        downloadFile.on('end', () => {
-          const details = JSON.stringify({ version: newVersion, path: bin, exec: 'youtube-dl' })
-          writeFile(detailsPath, details, { encoding: 'utf8' }, err => {
-            if (err) {
-              logger.error('youtube-dl update error: cannot write details.', { err })
-              return res()
-            }
-
-            logger.info('youtube-dl updated to version %s.', newVersion)
-            return res()
-          })
-        })
-      })
-    })
+  execute () {
+    return updateYoutubeDLBinary()
   }
 
   static get Instance () {