Fix redundancy with HLS only files

1 files changed, 54 insertions, 45 deletions

diff --git a/server/models/redundancy/video-redundancy.ts b/server/models/redundancy/video-redundancy.ts
index c536c288b..d3b839cfe 100644
--- a/server/models/redundancy/video-redundancy.ts
+++ b/server/models/redundancy/video-redundancy.ts
@@ -1,5 +1,5 @@
 import { sample } from 'lodash'
-import { col, FindOptions, fn, literal, Op, Transaction, WhereOptions } from 'sequelize'
+import { col, FindOptions, fn, literal, Op, QueryTypes, Transaction, WhereOptions } from 'sequelize'
 import {
   AllowNull,
   BeforeDestroy,
@@ -15,7 +15,7 @@ import {
   UpdatedAt
 } from 'sequelize-typescript'
 import { getServerActor } from '@server/models/application/application'
-import { MVideoForRedundancyAPI, MVideoRedundancy, MVideoRedundancyAP, MVideoRedundancyVideo } from '@server/types/models'
+import { MActor, MVideoForRedundancyAPI, MVideoRedundancy, MVideoRedundancyAP, MVideoRedundancyVideo } from '@server/types/models'
 import { VideoRedundanciesTarget } from '@shared/models/redundancy/video-redundancies-filters.model'
 import {
   FileRedundancyInformation,
@@ -36,6 +36,7 @@ import { VideoModel } from '../video/video'
 import { VideoChannelModel } from '../video/video-channel'
 import { VideoFileModel } from '../video/video-file'
 import { VideoStreamingPlaylistModel } from '../video/video-streaming-playlist'
+import { forEachSeries } from 'async'
 
 export enum ScopeNames {
   WITH_VIDEO = 'WITH_VIDEO'
@@ -261,6 +262,8 @@ export class VideoRedundancyModel extends Model {
   }
 
   static async findMostViewToDuplicate (randomizedFactor: number) {
+    const peertubeActor = await getServerActor()
+
     // On VideoModel!
     const query = {
       attributes: [ 'id', 'views' ],
@@ -268,10 +271,10 @@ export class VideoRedundancyModel extends Model {
       order: getVideoSort('-views'),
       where: {
         privacy: VideoPrivacy.PUBLIC,
-        isLive: false
+        isLive: false,
+        ...this.buildVideoIdsForDuplication(peertubeActor)
       },
       include: [
-        await VideoRedundancyModel.buildVideoFileForDuplication(),
         VideoRedundancyModel.buildServerRedundancyInclude()
       ]
     }
@@ -280,6 +283,8 @@ export class VideoRedundancyModel extends Model {
   }
 
   static async findTrendingToDuplicate (randomizedFactor: number) {
+    const peertubeActor = await getServerActor()
+
     // On VideoModel!
     const query = {
       attributes: [ 'id', 'views' ],
@@ -289,10 +294,10 @@ export class VideoRedundancyModel extends Model {
       order: getVideoSort('-trending'),
       where: {
         privacy: VideoPrivacy.PUBLIC,
-        isLive: false
+        isLive: false,
+        ...this.buildVideoIdsForDuplication(peertubeActor)
       },
       include: [
-        await VideoRedundancyModel.buildVideoFileForDuplication(),
         VideoRedundancyModel.buildServerRedundancyInclude(),
 
         VideoModel.buildTrendingQuery(CONFIG.TRENDING.VIDEOS.INTERVAL_DAYS)
@@ -303,6 +308,8 @@ export class VideoRedundancyModel extends Model {
   }
 
   static async findRecentlyAddedToDuplicate (randomizedFactor: number, minViews: number) {
+    const peertubeActor = await getServerActor()
+
     // On VideoModel!
     const query = {
       attributes: [ 'id', 'publishedAt' ],
@@ -313,10 +320,10 @@ export class VideoRedundancyModel extends Model {
         isLive: false,
         views: {
           [Op.gte]: minViews
-        }
+        },
+        ...this.buildVideoIdsForDuplication(peertubeActor)
       },
       include: [
-        await VideoRedundancyModel.buildVideoFileForDuplication(),
         VideoRedundancyModel.buildServerRedundancyInclude()
       ]
     }
@@ -573,32 +580,35 @@ export class VideoRedundancyModel extends Model {
   static async getStats (strategy: VideoRedundancyStrategyWithManual) {
     const actor = await getServerActor()
 
-    const query: FindOptions = {
-      raw: true,
-      attributes: [
-        [ fn('COALESCE', fn('SUM', col('VideoFile.size')), '0'), 'totalUsed' ],
-        [ fn('COUNT', fn('DISTINCT', col('videoId'))), 'totalVideos' ],
-        [ fn('COUNT', col('videoFileId')), 'totalVideoFiles' ]
-      ],
-      where: {
-        strategy,
-        actorId: actor.id
-      },
-      include: [
-        {
-          attributes: [],
-          model: VideoFileModel,
-          required: true
-        }
-      ]
-    }
-
-    return VideoRedundancyModel.findOne(query)
-                               .then((r: any) => ({
-                                 totalUsed: parseAggregateResult(r.totalUsed),
-                                 totalVideos: r.totalVideos,
-                                 totalVideoFiles: r.totalVideoFiles
-                               }))
+    const sql = `WITH "tmp" AS ` +
+      `(` +
+        `SELECT "videoFile"."size" AS "videoFileSize", "videoStreamingFile"."size" AS "videoStreamingFileSize", ` +
+          `"videoFile"."videoId" AS "videoFileVideoId", "videoStreamingPlaylist"."videoId" AS "videoStreamingVideoId"` +
+        `FROM "videoRedundancy" AS "videoRedundancy" ` +
+        `LEFT JOIN "videoFile" AS "videoFile" ON "videoRedundancy"."videoFileId" = "videoFile"."id" ` +
+        `LEFT JOIN "videoStreamingPlaylist" ON "videoRedundancy"."videoStreamingPlaylistId" = "videoStreamingPlaylist"."id" ` +
+        `LEFT JOIN "videoFile" AS "videoStreamingFile" ` +
+          `ON "videoStreamingPlaylist"."id" = "videoStreamingFile"."videoStreamingPlaylistId" ` +
+        `WHERE "videoRedundancy"."strategy" = :strategy AND "videoRedundancy"."actorId" = :actorId` +
+      `), ` +
+      `"videoIds" AS (` +
+        `SELECT "videoFileVideoId" AS "videoId" FROM "tmp" ` +
+        `UNION SELECT "videoStreamingVideoId" AS "videoId" FROM "tmp" ` +
+      `) ` +
+      `SELECT ` +
+      `COALESCE(SUM("videoFileSize"), '0') + COALESCE(SUM("videoStreamingFileSize"), '0') AS "totalUsed", ` +
+      `(SELECT COUNT("videoIds"."videoId") FROM "videoIds") AS "totalVideos", ` +
+      `COUNT(*) AS "totalVideoFiles" ` +
+      `FROM "tmp"`
+
+    return VideoRedundancyModel.sequelize.query<any>(sql, {
+      replacements: { strategy, actorId: actor.id },
+      type: QueryTypes.SELECT
+    }).then(([ row ]) => ({
+      totalUsed: parseAggregateResult(row.totalUsed),
+      totalVideos: row.totalVideos,
+      totalVideoFiles: row.totalVideoFiles
+    }))
   }
 
   static toFormattedJSONStatic (video: MVideoForRedundancyAPI): VideoRedundancy {
@@ -692,23 +702,22 @@ export class VideoRedundancyModel extends Model {
   }
 
   // Don't include video files we already duplicated
-  private static async buildVideoFileForDuplication () {
-    const actor = await getServerActor()
-
+  private static buildVideoIdsForDuplication (peertubeActor: MActor) {
     const notIn = literal(
       '(' +
-        `SELECT "videoFileId" FROM "videoRedundancy" WHERE "actorId" = ${actor.id} AND "videoFileId" IS NOT NULL` +
+        `SELECT "videoFile"."videoId" AS "videoId" FROM "videoRedundancy" ` +
+        `INNER JOIN "videoFile" ON "videoFile"."id" = "videoRedundancy"."videoFileId" ` +
+        `WHERE "videoRedundancy"."actorId" = ${peertubeActor.id} ` +
+        `UNION ` +
+        `SELECT "videoStreamingPlaylist"."videoId" AS "videoId" FROM "videoRedundancy" ` +
+        `INNER JOIN "videoStreamingPlaylist" ON "videoStreamingPlaylist"."id" = "videoRedundancy"."videoStreamingPlaylistId" ` +
+        `WHERE "videoRedundancy"."actorId" = ${peertubeActor.id} ` +
       ')'
     )
 
     return {
-      attributes: [],
-      model: VideoFileModel,
-      required: true,
-      where: {
-        id: {
-          [Op.notIn]: notIn
-        }
+      id: {
+        [Op.notIn]: notIn
       }
     }
   }