Add ability to list redundancies

6 files changed, 80 insertions, 25 deletions

diff --git a/server/lib/activitypub/cache-file.ts b/server/lib/activitypub/cache-file.ts
index 65b2dcb49..8252e95e9 100644
--- a/server/lib/activitypub/cache-file.ts
+++ b/server/lib/activitypub/cache-file.ts
@@ -13,7 +13,7 @@ function cacheFileActivityObjectToDBAttributes (cacheFileObject: CacheFileObject
     if (!playlist) throw new Error('Cannot find HLS playlist of video ' + video.url)
 
     return {
-      expiresOn: new Date(cacheFileObject.expires),
+      expiresOn: cacheFileObject.expires ? new Date(cacheFileObject.expires) : null,
       url: cacheFileObject.id,
       fileUrl: url.href,
       strategy: null,
@@ -30,7 +30,7 @@ function cacheFileActivityObjectToDBAttributes (cacheFileObject: CacheFileObject
   if (!videoFile) throw new Error(`Cannot find video file ${url.height} ${url.fps} of video ${video.url}`)
 
   return {
-    expiresOn: new Date(cacheFileObject.expires),
+    expiresOn: cacheFileObject.expires ? new Date(cacheFileObject.expires) : null,
     url: cacheFileObject.id,
     fileUrl: url.href,
     strategy: null,
diff --git a/server/lib/job-queue/handlers/video-redundancy.ts b/server/lib/job-queue/handlers/video-redundancy.ts
new file mode 100644
index 000000000..319d7090e
--- /dev/null
+++ b/server/lib/job-queue/handlers/video-redundancy.ts
@@ -0,0 +1,20 @@
+import * as Bull from 'bull'
+import { logger } from '../../../helpers/logger'
+import { VideosRedundancyScheduler } from '@server/lib/schedulers/videos-redundancy-scheduler'
+
+export type VideoRedundancyPayload = {
+  videoId: number
+}
+
+async function processVideoRedundancy (job: Bull.Job) {
+  const payload = job.data as VideoRedundancyPayload
+  logger.info('Processing video redundancy in job %d.', job.id)
+
+  return VideosRedundancyScheduler.Instance.createManualRedundancy(payload.videoId)
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+  processVideoRedundancy
+}
diff --git a/server/lib/job-queue/job-queue.ts b/server/lib/job-queue/job-queue.ts
index ec601e9ea..8bbf58f2b 100644
--- a/server/lib/job-queue/job-queue.ts
+++ b/server/lib/job-queue/job-queue.ts
@@ -13,6 +13,7 @@ import { processVideoImport, VideoImportPayload } from './handlers/video-import'
 import { processVideosViews } from './handlers/video-views'
 import { refreshAPObject, RefreshPayload } from './handlers/activitypub-refresher'
 import { processVideoFileImport, VideoFileImportPayload } from './handlers/video-file-import'
+import { processVideoRedundancy, VideoRedundancyPayload } from '@server/lib/job-queue/handlers/video-redundancy'
 
 type CreateJobArgument =
   { type: 'activitypub-http-broadcast', payload: ActivitypubHttpBroadcastPayload } |
@@ -24,20 +25,21 @@ type CreateJobArgument =
   { type: 'email', payload: EmailPayload } |
   { type: 'video-import', payload: VideoImportPayload } |
   { type: 'activitypub-refresher', payload: RefreshPayload } |
-  { type: 'videos-views', payload: {} }
+  { type: 'videos-views', payload: {} } |
+  { type: 'video-redundancy', payload: VideoRedundancyPayload }
 
-const handlers: { [ id in (JobType | 'video-file') ]: (job: Bull.Job) => Promise<any>} = {
+const handlers: { [ id in JobType ]: (job: Bull.Job) => Promise<any>} = {
   'activitypub-http-broadcast': processActivityPubHttpBroadcast,
   'activitypub-http-unicast': processActivityPubHttpUnicast,
   'activitypub-http-fetcher': processActivityPubHttpFetcher,
   'activitypub-follow': processActivityPubFollow,
   'video-file-import': processVideoFileImport,
   'video-transcoding': processVideoTranscoding,
-  'video-file': processVideoTranscoding, // TODO: remove it (changed in 1.3)
   'email': processEmail,
   'video-import': processVideoImport,
   'videos-views': processVideosViews,
-  'activitypub-refresher': refreshAPObject
+  'activitypub-refresher': refreshAPObject,
+  'video-redundancy': processVideoRedundancy
 }
 
 const jobTypes: JobType[] = [
@@ -50,7 +52,8 @@ const jobTypes: JobType[] = [
   'video-file-import',
   'video-import',
   'videos-views',
-  'activitypub-refresher'
+  'activitypub-refresher',
+  'video-redundancy'
 ]
 
 class JobQueue {
diff --git a/server/lib/redundancy.ts b/server/lib/redundancy.ts
index 1b4ecd7c0..78d84e02e 100644
--- a/server/lib/redundancy.ts
+++ b/server/lib/redundancy.ts
@@ -13,10 +13,10 @@ async function removeVideoRedundancy (videoRedundancy: MVideoRedundancyVideo, t?
   await videoRedundancy.destroy({ transaction: t })
 }
 
-async function removeRedundancyOf (serverId: number) {
-  const videosRedundancy = await VideoRedundancyModel.listLocalOfServer(serverId)
+async function removeRedundanciesOfServer (serverId: number) {
+  const redundancies = await VideoRedundancyModel.listLocalOfServer(serverId)
 
-  for (const redundancy of videosRedundancy) {
+  for (const redundancy of redundancies) {
     await removeVideoRedundancy(redundancy)
   }
 }
@@ -24,6 +24,6 @@ async function removeRedundancyOf (serverId: number) {
 // ---------------------------------------------------------------------------
 
 export {
-  removeRedundancyOf,
+  removeRedundanciesOfServer,
   removeVideoRedundancy
 }
diff --git a/server/lib/schedulers/update-videos-scheduler.ts b/server/lib/schedulers/update-videos-scheduler.ts
index 350a335d3..956780a77 100644
--- a/server/lib/schedulers/update-videos-scheduler.ts
+++ b/server/lib/schedulers/update-videos-scheduler.ts
@@ -4,7 +4,6 @@ import { ScheduleVideoUpdateModel } from '../../models/video/schedule-video-upda
 import { retryTransactionWrapper } from '../../helpers/database-utils'
 import { federateVideoIfNeeded } from '../activitypub'
 import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants'
-import { VideoPrivacy } from '../../../shared/models/videos'
 import { Notifier } from '../notifier'
 import { sequelizeTypescript } from '../../initializers/database'
 import { MVideoFullLight } from '@server/typings/models'
diff --git a/server/lib/schedulers/videos-redundancy-scheduler.ts b/server/lib/schedulers/videos-redundancy-scheduler.ts
index c1c91b656..6e61cbe7d 100644
--- a/server/lib/schedulers/videos-redundancy-scheduler.ts
+++ b/server/lib/schedulers/videos-redundancy-scheduler.ts
@@ -1,7 +1,7 @@
 import { AbstractScheduler } from './abstract-scheduler'
 import { HLS_REDUNDANCY_DIRECTORY, REDUNDANCY, VIDEO_IMPORT_TIMEOUT, WEBSERVER } from '../../initializers/constants'
 import { logger } from '../../helpers/logger'
-import { VideosRedundancy } from '../../../shared/models/redundancy'
+import { VideosRedundancyStrategy } from '../../../shared/models/redundancy'
 import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy'
 import { downloadWebTorrentVideo, generateMagnetUri } from '../../helpers/webtorrent'
 import { join } from 'path'
@@ -25,9 +25,10 @@ import {
   MVideoWithAllFiles
 } from '@server/typings/models'
 import { getVideoFilename } from '../video-paths'
+import { VideoModel } from '@server/models/video/video'
 
 type CandidateToDuplicate = {
-  redundancy: VideosRedundancy,
+  redundancy: VideosRedundancyStrategy,
   video: MVideoWithAllFiles,
   files: MVideoFile[],
   streamingPlaylists: MStreamingPlaylistFiles[]
@@ -41,7 +42,7 @@ function isMVideoRedundancyFileVideo (
 
 export class VideosRedundancyScheduler extends AbstractScheduler {
 
-  private static instance: AbstractScheduler
+  private static instance: VideosRedundancyScheduler
 
   protected schedulerIntervalMs = CONFIG.REDUNDANCY.VIDEOS.CHECK_INTERVAL
 
@@ -49,6 +50,22 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
     super()
   }
 
+  async createManualRedundancy (videoId: number) {
+    const videoToDuplicate = await VideoModel.loadWithFiles(videoId)
+
+    if (!videoToDuplicate) {
+      logger.warn('Video to manually duplicate %d does not exist anymore.', videoId)
+      return
+    }
+
+    return this.createVideoRedundancies({
+      video: videoToDuplicate,
+      redundancy: null,
+      files: videoToDuplicate.VideoFiles,
+      streamingPlaylists: videoToDuplicate.VideoStreamingPlaylists
+    })
+  }
+
   protected async internalExecute () {
     for (const redundancyConfig of CONFIG.REDUNDANCY.VIDEOS.STRATEGIES) {
       logger.info('Running redundancy scheduler for strategy %s.', redundancyConfig.strategy)
@@ -94,7 +111,7 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
     for (const redundancyModel of expired) {
       try {
         const redundancyConfig = CONFIG.REDUNDANCY.VIDEOS.STRATEGIES.find(s => s.strategy === redundancyModel.strategy)
-        const candidate = {
+        const candidate: CandidateToDuplicate = {
           redundancy: redundancyConfig,
           video: null,
           files: [],
@@ -140,7 +157,7 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
     }
   }
 
-  private findVideoToDuplicate (cache: VideosRedundancy) {
+  private findVideoToDuplicate (cache: VideosRedundancyStrategy) {
     if (cache.strategy === 'most-views') {
       return VideoRedundancyModel.findMostViewToDuplicate(REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR)
     }
@@ -187,13 +204,21 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
     }
   }
 
-  private async createVideoFileRedundancy (redundancy: VideosRedundancy, video: MVideoAccountLight, fileArg: MVideoFile) {
+  private async createVideoFileRedundancy (redundancy: VideosRedundancyStrategy | null, video: MVideoAccountLight, fileArg: MVideoFile) {
+    let strategy = 'manual'
+    let expiresOn: Date = null
+
+    if (redundancy) {
+      strategy = redundancy.strategy
+      expiresOn = this.buildNewExpiration(redundancy.minLifetime)
+    }
+
     const file = fileArg as MVideoFileVideo
     file.Video = video
 
     const serverActor = await getServerActor()
 
-    logger.info('Duplicating %s - %d in videos redundancy with "%s" strategy.', video.url, file.resolution, redundancy.strategy)
+    logger.info('Duplicating %s - %d in videos redundancy with "%s" strategy.', video.url, file.resolution, strategy)
 
     const { baseUrlHttp, baseUrlWs } = video.getBaseUrls()
     const magnetUri = generateMagnetUri(video, file, baseUrlHttp, baseUrlWs)
@@ -204,10 +229,10 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
     await move(tmpPath, destPath, { overwrite: true })
 
     const createdModel: MVideoRedundancyFileVideo = await VideoRedundancyModel.create({
-      expiresOn: this.buildNewExpiration(redundancy.minLifetime),
+      expiresOn,
       url: getVideoCacheFileActivityPubUrl(file),
       fileUrl: video.getVideoRedundancyUrl(file, WEBSERVER.URL),
-      strategy: redundancy.strategy,
+      strategy,
       videoFileId: file.id,
       actorId: serverActor.id
     })
@@ -220,25 +245,33 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
   }
 
   private async createStreamingPlaylistRedundancy (
-    redundancy: VideosRedundancy,
+    redundancy: VideosRedundancyStrategy,
     video: MVideoAccountLight,
     playlistArg: MStreamingPlaylist
   ) {
+    let strategy = 'manual'
+    let expiresOn: Date = null
+
+    if (redundancy) {
+      strategy = redundancy.strategy
+      expiresOn = this.buildNewExpiration(redundancy.minLifetime)
+    }
+
     const playlist = playlistArg as MStreamingPlaylistVideo
     playlist.Video = video
 
     const serverActor = await getServerActor()
 
-    logger.info('Duplicating %s streaming playlist in videos redundancy with "%s" strategy.', video.url, redundancy.strategy)
+    logger.info('Duplicating %s streaming playlist in videos redundancy with "%s" strategy.', video.url, strategy)
 
     const destDirectory = join(HLS_REDUNDANCY_DIRECTORY, video.uuid)
     await downloadPlaylistSegments(playlist.playlistUrl, destDirectory, VIDEO_IMPORT_TIMEOUT)
 
     const createdModel: MVideoRedundancyStreamingPlaylistVideo = await VideoRedundancyModel.create({
-      expiresOn: this.buildNewExpiration(redundancy.minLifetime),
+      expiresOn,
       url: getVideoCacheStreamingPlaylistActivityPubUrl(video, playlist),
       fileUrl: playlist.getVideoRedundancyUrl(WEBSERVER.URL),
-      strategy: redundancy.strategy,
+      strategy,
       videoStreamingPlaylistId: playlist.id,
       actorId: serverActor.id
     })