Merge branch 'release/v1.2.0'

[github/Chocobozzz/PeerTube.git] / server / lib / schedulers / videos-redundancy-scheduler.ts

diff --git a/server/lib/schedulers/videos-redundancy-scheduler.ts b/server/lib/schedulers/videos-redundancy-scheduler.ts
index 11ee05a537a5f447d55045116b2f96393a16bcce..f643ee2268d38969e2403614949cd29a6b39eb42 100644 (file)
--- a/server/lib/schedulers/videos-redundancy-scheduler.ts
+++ b/server/lib/schedulers/videos-redundancy-scheduler.ts
@@ -1,15 +1,14 @@
 import { AbstractScheduler } from './abstract-scheduler'
-import { CONFIG, JOB_TTL, REDUNDANCY } from '../../initializers'
+import { CONFIG, REDUNDANCY, VIDEO_IMPORT_TIMEOUT } from '../../initializers'
 import { logger } from '../../helpers/logger'
 import { VideosRedundancy } from '../../../shared/models/redundancy'
 import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy'
 import { VideoFileModel } from '../../models/video/video-file'
 import { downloadWebTorrentVideo } from '../../helpers/webtorrent'
 import { join } from 'path'
-import { rename } from 'fs-extra'
+import { move } from 'fs-extra'
 import { getServerActor } from '../../helpers/utils'
 import { sendCreateCacheFile, sendUpdateCacheFile } from '../activitypub/send'
-import { VideoModel } from '../../models/video/video'
 import { getVideoCacheFileActivityPubUrl } from '../activitypub/url'
 import { removeVideoRedundancy } from '../redundancy'
 import { getOrCreateVideoAndAccountAndChannel } from '../activitypub'
@@ -17,7 +16,6 @@ import { getOrCreateVideoAndAccountAndChannel } from '../activitypub'
 export class VideosRedundancyScheduler extends AbstractScheduler {
 
   private static instance: AbstractScheduler
-  private executing = false
 
   protected schedulerIntervalMs = CONFIG.REDUNDANCY.VIDEOS.CHECK_INTERVAL
 
@@ -25,11 +23,7 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
     super()
   }
 
-  async execute () {
-    if (this.executing) return
-
-    this.executing = true
-
+  protected async internalExecute () {
     for (const obj of CONFIG.REDUNDANCY.VIDEOS.STRATEGIES) {
       logger.info('Running redundancy scheduler for strategy %s.', obj.strategy)
 
@@ -58,8 +52,6 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
     await this.extendsLocalExpiration()
 
     await this.purgeRemoteExpired()
-
-    this.executing = false
   }
 
   static get Instance () {
@@ -71,14 +63,28 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
 
     for (const redundancyModel of expired) {
       try {
-        const redundancy = CONFIG.REDUNDANCY.VIDEOS.STRATEGIES.find(s => s.strategy === redundancyModel.strategy)
-        await this.extendsExpirationOf(redundancyModel, redundancy.minLifetime)
+        await this.extendsOrDeleteRedundancy(redundancyModel)
       } catch (err) {
         logger.error('Cannot extend expiration of %s video from our redundancy system.', this.buildEntryLogId(redundancyModel))
       }
     }
   }
 
+  private async extendsOrDeleteRedundancy (redundancyModel: VideoRedundancyModel) {
+    // Refresh the video, maybe it was deleted
+    const video = await this.loadAndRefreshVideo(redundancyModel.VideoFile.Video.url)
+
+    if (!video) {
+      logger.info('Destroying existing redundancy %s, because the associated video does not exist anymore.', redundancyModel.url)
+
+      await redundancyModel.destroy()
+      return
+    }
+
+    const redundancy = CONFIG.REDUNDANCY.VIDEOS.STRATEGIES.find(s => s.strategy === redundancyModel.strategy)
+    await this.extendsExpirationOf(redundancyModel, redundancy.minLifetime)
+  }
+
   private async purgeRemoteExpired () {
     const expired = await VideoRedundancyModel.listRemoteExpired()
 
@@ -110,23 +116,11 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
     const serverActor = await getServerActor()
 
     for (const file of filesToDuplicate) {
-      // We need more attributes and check if the video still exists
-      const getVideoOptions = {
-        videoObject: file.Video.url,
-        syncParam: { likes: false, dislikes: false, shares: false, comments: false, thumbnail: false, refreshVideo: true },
-        fetchType: 'only-video' as 'only-video'
-      }
-      const { video } = await getOrCreateVideoAndAccountAndChannel(getVideoOptions)
-
-      const existing = await VideoRedundancyModel.loadLocalByFileId(file.id)
-      if (existing) {
-        if (video) {
-          await this.extendsExpirationOf(existing, redundancy.minLifetime)
-        } else {
-          logger.info('Destroying existing redundancy %s, because the associated video does not exist anymore.', existing.url)
+      const video = await this.loadAndRefreshVideo(file.Video.url)
 
-          await existing.destroy()
-        }
+      const existingRedundancy = await VideoRedundancyModel.loadLocalByFileId(file.id)
+      if (existingRedundancy) {
+        await this.extendsOrDeleteRedundancy(existingRedundancy)
 
         continue
       }
@@ -142,15 +136,15 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
       const { baseUrlHttp, baseUrlWs } = video.getBaseUrls()
       const magnetUri = video.generateMagnetUri(file, baseUrlHttp, baseUrlWs)
 
-      const tmpPath = await downloadWebTorrentVideo({ magnetUri }, JOB_TTL['video-import'])
+      const tmpPath = await downloadWebTorrentVideo({ magnetUri }, VIDEO_IMPORT_TIMEOUT)
 
-      const destPath = join(CONFIG.STORAGE.VIDEOS_DIR, video.getVideoFilename(file))
-      await rename(tmpPath, destPath)
+      const destPath = join(CONFIG.STORAGE.REDUNDANCY_DIR, video.getVideoFilename(file))
+      await move(tmpPath, destPath)
 
       const createdModel = await VideoRedundancyModel.create({
         expiresOn: this.buildNewExpiration(redundancy.minLifetime),
         url: getVideoCacheFileActivityPubUrl(file),
-        fileUrl: video.getVideoFileUrl(file, CONFIG.WEBSERVER.URL),
+        fileUrl: video.getVideoRedundancyUrl(file, CONFIG.WEBSERVER.URL),
         strategy: redundancy.strategy,
         videoFileId: file.id,
         actorId: serverActor.id
@@ -158,6 +152,8 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
       createdModel.VideoFile = file
 
       await sendCreateCacheFile(serverActor, createdModel)
+
+      logger.info('Duplicated %s - %d -> %s.', video.url, file.resolution, createdModel.url)
     }
   }
 
@@ -182,11 +178,12 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
   }
 
   private async isTooHeavy (redundancy: VideosRedundancy, filesToDuplicate: VideoFileModel[]) {
-    const maxSize = redundancy.size - this.getTotalFileSizes(filesToDuplicate)
+    const maxSize = redundancy.size
 
     const totalDuplicated = await VideoRedundancyModel.getTotalDuplicated(redundancy.strategy)
+    const totalWillDuplicate = totalDuplicated + this.getTotalFileSizes(filesToDuplicate)
 
-    return totalDuplicated > maxSize
+    return totalWillDuplicate > maxSize
   }
 
   private buildNewExpiration (expiresAfterMs: number) {
@@ -202,4 +199,16 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
 
     return files.reduce(fileReducer, 0)
   }
+
+  private async loadAndRefreshVideo (videoUrl: string) {
+    // We need more attributes and check if the video still exists
+    const getVideoOptions = {
+      videoObject: videoUrl,
+      syncParam: { likes: false, dislikes: false, shares: false, comments: false, thumbnail: false, refreshVideo: true },
+      fetchType: 'all' as 'all'
+    }
+    const { video } = await getOrCreateVideoAndAccountAndChannel(getVideoOptions)
+
+    return video
+  }
 }