Improve redundancy: add 'min_lifetime' configuration

3 files changed, 86 insertions, 15 deletions

diff --git a/server/models/activitypub/actor.ts b/server/models/activitypub/actor.ts
index f8bb59323..12b83916e 100644
--- a/server/models/activitypub/actor.ts
+++ b/server/models/activitypub/actor.ts
@@ -37,6 +37,7 @@ import { ServerModel } from '../server/server'
 import { throwIfNotValid } from '../utils'
 import { VideoChannelModel } from '../video/video-channel'
 import { ActorFollowModel } from './actor-follow'
+import { VideoModel } from '../video/video'
 
 enum ScopeNames {
   FULL = 'FULL'
@@ -266,6 +267,36 @@ export class ActorModel extends Model<ActorModel> {
     return ActorModel.unscoped().findById(id)
   }
 
+  static loadAccountActorByVideoId (videoId: number, transaction: Sequelize.Transaction) {
+    const query = {
+      include: [
+        {
+          attributes: [ 'id' ],
+          model: AccountModel.unscoped(),
+          required: true,
+          include: [
+            {
+              attributes: [ 'id' ],
+              model: VideoChannelModel.unscoped(),
+              required: true,
+              include: {
+                attributes: [ 'id' ],
+                model: VideoModel.unscoped(),
+                required: true,
+                where: {
+                  id: videoId
+                }
+              }
+            }
+          ]
+        }
+      ],
+      transaction
+    }
+
+    return ActorModel.unscoped().findOne(query as any) // FIXME: typings
+  }
+
   static isActorUrlExist (url: string) {
     const query = {
       raw: true,
diff --git a/server/models/redundancy/video-redundancy.ts b/server/models/redundancy/video-redundancy.ts
index fb07287a8..970d2fe06 100644
--- a/server/models/redundancy/video-redundancy.ts
+++ b/server/models/redundancy/video-redundancy.ts
@@ -9,7 +9,6 @@ import {
   Is,
   Model,
   Scopes,
-  Sequelize,
   Table,
   UpdatedAt
 } from 'sequelize-typescript'
@@ -28,6 +27,7 @@ import { ServerModel } from '../server/server'
 import { sample } from 'lodash'
 import { isTestInstance } from '../../helpers/core-utils'
 import * as Bluebird from 'bluebird'
+import * as Sequelize from 'sequelize'
 
 export enum ScopeNames {
   WITH_VIDEO = 'WITH_VIDEO'
@@ -116,11 +116,11 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> {
   Actor: ActorModel
 
   @AfterDestroy
-  static removeFilesAndSendDelete (instance: VideoRedundancyModel) {
+  static removeFile (instance: VideoRedundancyModel) {
     // Not us
     if (!instance.strategy) return
 
-    logger.info('Removing video file %s-.', instance.VideoFile.Video.uuid, instance.VideoFile.resolution)
+    logger.info('Removing duplicated video file %s-%s.', instance.VideoFile.Video.uuid, instance.VideoFile.resolution)
 
     return instance.VideoFile.Video.removeFile(instance.VideoFile)
   }
@@ -135,11 +135,12 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> {
     return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO).findOne(query)
   }
 
-  static loadByUrl (url: string) {
+  static loadByUrl (url: string, transaction?: Sequelize.Transaction) {
     const query = {
       where: {
         url
-      }
+      },
+      transaction
     }
 
     return VideoRedundancyModel.findOne(query)
@@ -157,7 +158,6 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> {
     // On VideoModel!
     const query = {
       attributes: [ 'id', 'views' ],
-      logging: !isTestInstance(),
       limit: randomizedFactor,
       order: getVideoSort('-views'),
       include: [
@@ -174,7 +174,6 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> {
     const query = {
       attributes: [ 'id', 'views' ],
       subQuery: false,
-      logging: !isTestInstance(),
       group: 'VideoModel.id',
       limit: randomizedFactor,
       order: getVideoSort('-trending'),
@@ -193,7 +192,6 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> {
     // On VideoModel!
     const query = {
       attributes: [ 'id', 'publishedAt' ],
-      logging: !isTestInstance(),
       limit: randomizedFactor,
       order: getVideoSort('-publishedAt'),
       where: {
@@ -210,11 +208,29 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> {
     return VideoRedundancyModel.getVideoSample(VideoModel.unscoped().findAll(query))
   }
 
+  static async loadOldestLocalThatAlreadyExpired (strategy: VideoRedundancyStrategy, expiresAfterMs: number) {
+    const expiredDate = new Date()
+    expiredDate.setMilliseconds(expiredDate.getMilliseconds() - expiresAfterMs)
+
+    const actor = await getServerActor()
+
+    const query = {
+      where: {
+        actorId: actor.id,
+        strategy,
+        createdAt: {
+          [ Sequelize.Op.lt ]: expiredDate
+        }
+      }
+    }
+
+    return VideoRedundancyModel.scope([ ScopeNames.WITH_VIDEO ]).findOne(query)
+  }
+
   static async getTotalDuplicated (strategy: VideoRedundancyStrategy) {
     const actor = await getServerActor()
 
     const options = {
-      logging: !isTestInstance(),
       include: [
         {
           attributes: [],
@@ -228,21 +244,39 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> {
       ]
     }
 
-    return VideoFileModel.sum('size', options)
+    return VideoFileModel.sum('size', options as any) // FIXME: typings
   }
 
-  static listAllExpired () {
+  static async listLocalExpired () {
+    const actor = await getServerActor()
+
+    const query = {
+      where: {
+        actorId: actor.id,
+        expiresOn: {
+          [ Sequelize.Op.lt ]: new Date()
+        }
+      }
+    }
+
+    return VideoRedundancyModel.scope([ ScopeNames.WITH_VIDEO ]).findAll(query)
+  }
+
+  static async listRemoteExpired () {
+    const actor = await getServerActor()
+
     const query = {
-      logging: !isTestInstance(),
       where: {
+        actorId: {
+          [Sequelize.Op.ne]: actor.id
+        },
         expiresOn: {
           [ Sequelize.Op.lt ]: new Date()
         }
       }
     }
 
-    return VideoRedundancyModel.scope(ScopeNames.WITH_VIDEO)
-                               .findAll(query)
+    return VideoRedundancyModel.scope([ ScopeNames.WITH_VIDEO ]).findAll(query)
   }
 
   static async getStats (strategy: VideoRedundancyStrategy) {
@@ -299,7 +333,7 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> {
 
     const notIn = Sequelize.literal(
       '(' +
-        `SELECT "videoFileId" FROM "videoRedundancy" WHERE "actorId" = ${actor.id} AND "expiresOn" >= NOW()` +
+        `SELECT "videoFileId" FROM "videoRedundancy" WHERE "actorId" = ${actor.id}` +
       ')'
     )
 
diff --git a/server/models/video/video-file.ts b/server/models/video/video-file.ts
index 0907ea569..0887a3738 100644
--- a/server/models/video/video-file.ts
+++ b/server/models/video/video-file.ts
@@ -106,4 +106,10 @@ export class VideoFileModel extends Model<VideoFileModel> {
                 return results.length === 1
               })
   }
+
+  hasSameUniqueKeysThan (other: VideoFileModel) {
+    return this.fps === other.fps &&
+      this.resolution === other.resolution &&
+      this.videoId === other.videoId
+  }
 }