Merge branch 'release/v1.2.0'

[github/Chocobozzz/PeerTube.git] / server / lib / schedulers / videos-redundancy-scheduler.ts

diff --git a/server/lib/schedulers/videos-redundancy-scheduler.ts b/server/lib/schedulers/videos-redundancy-scheduler.ts
index d324adfd03a5a36286cdd9240e3b7a6a8c34c146..f643ee2268d38969e2403614949cd29a6b39eb42 100644 (file)
--- a/server/lib/schedulers/videos-redundancy-scheduler.ts
+++ b/server/lib/schedulers/videos-redundancy-scheduler.ts
@@ -1,22 +1,21 @@
 import { AbstractScheduler } from './abstract-scheduler'
-import { CONFIG, JOB_TTL, REDUNDANCY } from '../../initializers'
+import { CONFIG, REDUNDANCY, VIDEO_IMPORT_TIMEOUT } from '../../initializers'
 import { logger } from '../../helpers/logger'
 import { VideosRedundancy } from '../../../shared/models/redundancy'
 import { VideoRedundancyModel } from '../../models/redundancy/video-redundancy'
 import { VideoFileModel } from '../../models/video/video-file'
 import { downloadWebTorrentVideo } from '../../helpers/webtorrent'
 import { join } from 'path'
-import { rename } from 'fs-extra'
+import { move } from 'fs-extra'
 import { getServerActor } from '../../helpers/utils'
 import { sendCreateCacheFile, sendUpdateCacheFile } from '../activitypub/send'
-import { VideoModel } from '../../models/video/video'
 import { getVideoCacheFileActivityPubUrl } from '../activitypub/url'
 import { removeVideoRedundancy } from '../redundancy'
+import { getOrCreateVideoAndAccountAndChannel } from '../activitypub'
 
 export class VideosRedundancyScheduler extends AbstractScheduler {
 
   private static instance: AbstractScheduler
-  private executing = false
 
   protected schedulerIntervalMs = CONFIG.REDUNDANCY.VIDEOS.CHECK_INTERVAL
 
@@ -24,11 +23,7 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
     super()
   }
 
-  async execute () {
-    if (this.executing) return
-
-    this.executing = true
-
+  protected async internalExecute () {
     for (const obj of CONFIG.REDUNDANCY.VIDEOS.STRATEGIES) {
       logger.info('Running redundancy scheduler for strategy %s.', obj.strategy)
 
@@ -57,8 +52,6 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
     await this.extendsLocalExpiration()
 
     await this.purgeRemoteExpired()
-
-    this.executing = false
   }
 
   static get Instance () {
@@ -70,14 +63,28 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
 
     for (const redundancyModel of expired) {
       try {
-        const redundancy = CONFIG.REDUNDANCY.VIDEOS.STRATEGIES.find(s => s.strategy === redundancyModel.strategy)
-        await this.extendsExpirationOf(redundancyModel, redundancy.minLifetime)
+        await this.extendsOrDeleteRedundancy(redundancyModel)
       } catch (err) {
         logger.error('Cannot extend expiration of %s video from our redundancy system.', this.buildEntryLogId(redundancyModel))
       }
     }
   }
 
+  private async extendsOrDeleteRedundancy (redundancyModel: VideoRedundancyModel) {
+    // Refresh the video, maybe it was deleted
+    const video = await this.loadAndRefreshVideo(redundancyModel.VideoFile.Video.url)
+
+    if (!video) {
+      logger.info('Destroying existing redundancy %s, because the associated video does not exist anymore.', redundancyModel.url)
+
+      await redundancyModel.destroy()
+      return
+    }
+
+    const redundancy = CONFIG.REDUNDANCY.VIDEOS.STRATEGIES.find(s => s.strategy === redundancyModel.strategy)
+    await this.extendsExpirationOf(redundancyModel, redundancy.minLifetime)
+  }
+
   private async purgeRemoteExpired () {
     const expired = await VideoRedundancyModel.listRemoteExpired()
 
@@ -109,31 +116,35 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
     const serverActor = await getServerActor()
 
     for (const file of filesToDuplicate) {
-      const existing = await VideoRedundancyModel.loadLocalByFileId(file.id)
-      if (existing) {
-        await this.extendsExpirationOf(existing, redundancy.minLifetime)
+      const video = await this.loadAndRefreshVideo(file.Video.url)
+
+      const existingRedundancy = await VideoRedundancyModel.loadLocalByFileId(file.id)
+      if (existingRedundancy) {
+        await this.extendsOrDeleteRedundancy(existingRedundancy)
 
         continue
       }
 
-      // We need more attributes and check if the video still exists
-      const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(file.Video.id)
-      if (!video) continue
+      if (!video) {
+        logger.info('Video %s we want to duplicate does not existing anymore, skipping.', file.Video.url)
+
+        continue
+      }
 
       logger.info('Duplicating %s - %d in videos redundancy with "%s" strategy.', video.url, file.resolution, redundancy.strategy)
 
       const { baseUrlHttp, baseUrlWs } = video.getBaseUrls()
       const magnetUri = video.generateMagnetUri(file, baseUrlHttp, baseUrlWs)
 
-      const tmpPath = await downloadWebTorrentVideo({ magnetUri }, JOB_TTL['video-import'])
+      const tmpPath = await downloadWebTorrentVideo({ magnetUri }, VIDEO_IMPORT_TIMEOUT)
 
-      const destPath = join(CONFIG.STORAGE.VIDEOS_DIR, video.getVideoFilename(file))
-      await rename(tmpPath, destPath)
+      const destPath = join(CONFIG.STORAGE.REDUNDANCY_DIR, video.getVideoFilename(file))
+      await move(tmpPath, destPath)
 
       const createdModel = await VideoRedundancyModel.create({
         expiresOn: this.buildNewExpiration(redundancy.minLifetime),
         url: getVideoCacheFileActivityPubUrl(file),
-        fileUrl: video.getVideoFileUrl(file, CONFIG.WEBSERVER.URL),
+        fileUrl: video.getVideoRedundancyUrl(file, CONFIG.WEBSERVER.URL),
         strategy: redundancy.strategy,
         videoFileId: file.id,
         actorId: serverActor.id
@@ -141,6 +152,8 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
       createdModel.VideoFile = file
 
       await sendCreateCacheFile(serverActor, createdModel)
+
+      logger.info('Duplicated %s - %d -> %s.', video.url, file.resolution, createdModel.url)
     }
   }
 
@@ -165,11 +178,12 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
   }
 
   private async isTooHeavy (redundancy: VideosRedundancy, filesToDuplicate: VideoFileModel[]) {
-    const maxSize = redundancy.size - this.getTotalFileSizes(filesToDuplicate)
+    const maxSize = redundancy.size
 
     const totalDuplicated = await VideoRedundancyModel.getTotalDuplicated(redundancy.strategy)
+    const totalWillDuplicate = totalDuplicated + this.getTotalFileSizes(filesToDuplicate)
 
-    return totalDuplicated > maxSize
+    return totalWillDuplicate > maxSize
   }
 
   private buildNewExpiration (expiresAfterMs: number) {
@@ -185,4 +199,16 @@ export class VideosRedundancyScheduler extends AbstractScheduler {
 
     return files.reduce(fileReducer, 0)
   }
+
+  private async loadAndRefreshVideo (videoUrl: string) {
+    // We need more attributes and check if the video still exists
+    const getVideoOptions = {
+      videoObject: videoUrl,
+      syncParam: { likes: false, dislikes: false, shares: false, comments: false, thumbnail: false, refreshVideo: true },
+      fetchType: 'all' as 'all'
+    }
+    const { video } = await getOrCreateVideoAndAccountAndChannel(getVideoOptions)
+
+    return video
+  }
 }