Fix redundancy with HLS only files

3 files changed, 132 insertions, 59 deletions

diff --git a/server/models/redundancy/video-redundancy.ts b/server/models/redundancy/video-redundancy.ts
index c536c288b..d3b839cfe 100644
--- a/server/models/redundancy/video-redundancy.ts
+++ b/server/models/redundancy/video-redundancy.ts
@@ -1,5 +1,5 @@
 import { sample } from 'lodash'
-import { col, FindOptions, fn, literal, Op, Transaction, WhereOptions } from 'sequelize'
+import { col, FindOptions, fn, literal, Op, QueryTypes, Transaction, WhereOptions } from 'sequelize'
 import {
   AllowNull,
   BeforeDestroy,
@@ -15,7 +15,7 @@ import {
   UpdatedAt
 } from 'sequelize-typescript'
 import { getServerActor } from '@server/models/application/application'
-import { MVideoForRedundancyAPI, MVideoRedundancy, MVideoRedundancyAP, MVideoRedundancyVideo } from '@server/types/models'
+import { MActor, MVideoForRedundancyAPI, MVideoRedundancy, MVideoRedundancyAP, MVideoRedundancyVideo } from '@server/types/models'
 import { VideoRedundanciesTarget } from '@shared/models/redundancy/video-redundancies-filters.model'
 import {
   FileRedundancyInformation,
@@ -36,6 +36,7 @@ import { VideoModel } from '../video/video'
 import { VideoChannelModel } from '../video/video-channel'
 import { VideoFileModel } from '../video/video-file'
 import { VideoStreamingPlaylistModel } from '../video/video-streaming-playlist'
+import { forEachSeries } from 'async'
 
 export enum ScopeNames {
   WITH_VIDEO = 'WITH_VIDEO'
@@ -261,6 +262,8 @@ export class VideoRedundancyModel extends Model {
   }
 
   static async findMostViewToDuplicate (randomizedFactor: number) {
+    const peertubeActor = await getServerActor()
+
     // On VideoModel!
     const query = {
       attributes: [ 'id', 'views' ],
@@ -268,10 +271,10 @@ export class VideoRedundancyModel extends Model {
       order: getVideoSort('-views'),
       where: {
         privacy: VideoPrivacy.PUBLIC,
-        isLive: false
+        isLive: false,
+        ...this.buildVideoIdsForDuplication(peertubeActor)
       },
       include: [
-        await VideoRedundancyModel.buildVideoFileForDuplication(),
         VideoRedundancyModel.buildServerRedundancyInclude()
       ]
     }
@@ -280,6 +283,8 @@ export class VideoRedundancyModel extends Model {
   }
 
   static async findTrendingToDuplicate (randomizedFactor: number) {
+    const peertubeActor = await getServerActor()
+
     // On VideoModel!
     const query = {
       attributes: [ 'id', 'views' ],
@@ -289,10 +294,10 @@ export class VideoRedundancyModel extends Model {
       order: getVideoSort('-trending'),
       where: {
         privacy: VideoPrivacy.PUBLIC,
-        isLive: false
+        isLive: false,
+        ...this.buildVideoIdsForDuplication(peertubeActor)
       },
       include: [
-        await VideoRedundancyModel.buildVideoFileForDuplication(),
         VideoRedundancyModel.buildServerRedundancyInclude(),
 
         VideoModel.buildTrendingQuery(CONFIG.TRENDING.VIDEOS.INTERVAL_DAYS)
@@ -303,6 +308,8 @@ export class VideoRedundancyModel extends Model {
   }
 
   static async findRecentlyAddedToDuplicate (randomizedFactor: number, minViews: number) {
+    const peertubeActor = await getServerActor()
+
     // On VideoModel!
     const query = {
       attributes: [ 'id', 'publishedAt' ],
@@ -313,10 +320,10 @@ export class VideoRedundancyModel extends Model {
         isLive: false,
         views: {
           [Op.gte]: minViews
-        }
+        },
+        ...this.buildVideoIdsForDuplication(peertubeActor)
       },
       include: [
-        await VideoRedundancyModel.buildVideoFileForDuplication(),
         VideoRedundancyModel.buildServerRedundancyInclude()
       ]
     }
@@ -573,32 +580,35 @@ export class VideoRedundancyModel extends Model {
   static async getStats (strategy: VideoRedundancyStrategyWithManual) {
     const actor = await getServerActor()
 
-    const query: FindOptions = {
-      raw: true,
-      attributes: [
-        [ fn('COALESCE', fn('SUM', col('VideoFile.size')), '0'), 'totalUsed' ],
-        [ fn('COUNT', fn('DISTINCT', col('videoId'))), 'totalVideos' ],
-        [ fn('COUNT', col('videoFileId')), 'totalVideoFiles' ]
-      ],
-      where: {
-        strategy,
-        actorId: actor.id
-      },
-      include: [
-        {
-          attributes: [],
-          model: VideoFileModel,
-          required: true
-        }
-      ]
-    }
-
-    return VideoRedundancyModel.findOne(query)
-                               .then((r: any) => ({
-                                 totalUsed: parseAggregateResult(r.totalUsed),
-                                 totalVideos: r.totalVideos,
-                                 totalVideoFiles: r.totalVideoFiles
-                               }))
+    const sql = `WITH "tmp" AS ` +
+      `(` +
+        `SELECT "videoFile"."size" AS "videoFileSize", "videoStreamingFile"."size" AS "videoStreamingFileSize", ` +
+          `"videoFile"."videoId" AS "videoFileVideoId", "videoStreamingPlaylist"."videoId" AS "videoStreamingVideoId"` +
+        `FROM "videoRedundancy" AS "videoRedundancy" ` +
+        `LEFT JOIN "videoFile" AS "videoFile" ON "videoRedundancy"."videoFileId" = "videoFile"."id" ` +
+        `LEFT JOIN "videoStreamingPlaylist" ON "videoRedundancy"."videoStreamingPlaylistId" = "videoStreamingPlaylist"."id" ` +
+        `LEFT JOIN "videoFile" AS "videoStreamingFile" ` +
+          `ON "videoStreamingPlaylist"."id" = "videoStreamingFile"."videoStreamingPlaylistId" ` +
+        `WHERE "videoRedundancy"."strategy" = :strategy AND "videoRedundancy"."actorId" = :actorId` +
+      `), ` +
+      `"videoIds" AS (` +
+        `SELECT "videoFileVideoId" AS "videoId" FROM "tmp" ` +
+        `UNION SELECT "videoStreamingVideoId" AS "videoId" FROM "tmp" ` +
+      `) ` +
+      `SELECT ` +
+      `COALESCE(SUM("videoFileSize"), '0') + COALESCE(SUM("videoStreamingFileSize"), '0') AS "totalUsed", ` +
+      `(SELECT COUNT("videoIds"."videoId") FROM "videoIds") AS "totalVideos", ` +
+      `COUNT(*) AS "totalVideoFiles" ` +
+      `FROM "tmp"`
+
+    return VideoRedundancyModel.sequelize.query<any>(sql, {
+      replacements: { strategy, actorId: actor.id },
+      type: QueryTypes.SELECT
+    }).then(([ row ]) => ({
+      totalUsed: parseAggregateResult(row.totalUsed),
+      totalVideos: row.totalVideos,
+      totalVideoFiles: row.totalVideoFiles
+    }))
   }
 
   static toFormattedJSONStatic (video: MVideoForRedundancyAPI): VideoRedundancy {
@@ -692,23 +702,22 @@ export class VideoRedundancyModel extends Model {
   }
 
   // Don't include video files we already duplicated
-  private static async buildVideoFileForDuplication () {
-    const actor = await getServerActor()
-
+  private static buildVideoIdsForDuplication (peertubeActor: MActor) {
     const notIn = literal(
       '(' +
-        `SELECT "videoFileId" FROM "videoRedundancy" WHERE "actorId" = ${actor.id} AND "videoFileId" IS NOT NULL` +
+        `SELECT "videoFile"."videoId" AS "videoId" FROM "videoRedundancy" ` +
+        `INNER JOIN "videoFile" ON "videoFile"."id" = "videoRedundancy"."videoFileId" ` +
+        `WHERE "videoRedundancy"."actorId" = ${peertubeActor.id} ` +
+        `UNION ` +
+        `SELECT "videoStreamingPlaylist"."videoId" AS "videoId" FROM "videoRedundancy" ` +
+        `INNER JOIN "videoStreamingPlaylist" ON "videoStreamingPlaylist"."id" = "videoRedundancy"."videoStreamingPlaylistId" ` +
+        `WHERE "videoRedundancy"."actorId" = ${peertubeActor.id} ` +
       ')'
     )
 
     return {
-      attributes: [],
-      model: VideoFileModel,
-      required: true,
-      where: {
-        id: {
-          [Op.notIn]: notIn
-        }
+      id: {
+        [Op.notIn]: notIn
       }
     }
   }
diff --git a/server/tests/api/redundancy/redundancy.ts b/server/tests/api/redundancy/redundancy.ts
index 7cfcf70e1..8da0ba72a 100644
--- a/server/tests/api/redundancy/redundancy.ts
+++ b/server/tests/api/redundancy/redundancy.ts
@@ -21,6 +21,8 @@ import {
   ServerInfo,
   setAccessTokensToServers,
   unfollow,
+  updateCustomConfig,
+  updateCustomSubConfig,
   uploadVideo,
   viewVideo,
   wait,
@@ -60,7 +62,7 @@ function checkMagnetWebseeds (file: { magnetUri: string, resolution: { id: numbe
   expect(parsed.urlList).to.have.lengthOf(baseWebseeds.length)
 }
 
-async function flushAndRunServers (strategy: VideoRedundancyStrategy | null, additionalParams: any = {}) {
+async function flushAndRunServers (strategy: VideoRedundancyStrategy | null, additionalParams: any = {}, withWebtorrent = true) {
   const strategies: any[] = []
 
   if (strategy !== null) {
@@ -75,6 +77,9 @@ async function flushAndRunServers (strategy: VideoRedundancyStrategy | null, add
 
   const config = {
     transcoding: {
+      webtorrent: {
+        enabled: withWebtorrent
+      },
       hls: {
         enabled: true
       }
@@ -253,7 +258,7 @@ async function checkStatsGlobal (strategy: VideoRedundancyStrategyWithManual) {
   return stat
 }
 
-async function checkStatsWith2Webseed (strategy: VideoRedundancyStrategyWithManual) {
+async function checkStatsWith1Redundancy (strategy: VideoRedundancyStrategyWithManual) {
   const stat = await checkStatsGlobal(strategy)
 
   expect(stat.totalUsed).to.be.at.least(1).and.below(409601)
@@ -261,7 +266,7 @@ async function checkStatsWith2Webseed (strategy: VideoRedundancyStrategyWithManu
   expect(stat.totalVideos).to.equal(1)
 }
 
-async function checkStatsWith1Webseed (strategy: VideoRedundancyStrategyWithManual) {
+async function checkStatsWithoutRedundancy (strategy: VideoRedundancyStrategyWithManual) {
   const stat = await checkStatsGlobal(strategy)
 
   expect(stat.totalUsed).to.equal(0)
@@ -313,7 +318,7 @@ describe('Test videos redundancy', function () {
     it('Should have 1 webseed on the first video', async function () {
       await check1WebSeed()
       await check0PlaylistRedundancies()
-      await checkStatsWith1Webseed(strategy)
+      await checkStatsWithoutRedundancy(strategy)
     })
 
     it('Should enable redundancy on server 1', function () {
@@ -329,7 +334,7 @@ describe('Test videos redundancy', function () {
 
       await check2Webseeds()
       await check1PlaylistRedundancies()
-      await checkStatsWith2Webseed(strategy)
+      await checkStatsWith1Redundancy(strategy)
     })
 
     it('Should undo redundancy on server 1 and remove duplicated videos', async function () {
@@ -363,7 +368,7 @@ describe('Test videos redundancy', function () {
     it('Should have 1 webseed on the first video', async function () {
       await check1WebSeed()
       await check0PlaylistRedundancies()
-      await checkStatsWith1Webseed(strategy)
+      await checkStatsWithoutRedundancy(strategy)
     })
 
     it('Should enable redundancy on server 1', function () {
@@ -379,7 +384,7 @@ describe('Test videos redundancy', function () {
 
       await check2Webseeds()
       await check1PlaylistRedundancies()
-      await checkStatsWith2Webseed(strategy)
+      await checkStatsWith1Redundancy(strategy)
     })
 
     it('Should unfollow on server 1 and remove duplicated videos', async function () {
@@ -413,7 +418,7 @@ describe('Test videos redundancy', function () {
     it('Should have 1 webseed on the first video', async function () {
       await check1WebSeed()
       await check0PlaylistRedundancies()
-      await checkStatsWith1Webseed(strategy)
+      await checkStatsWithoutRedundancy(strategy)
     })
 
     it('Should enable redundancy on server 1', function () {
@@ -429,7 +434,7 @@ describe('Test videos redundancy', function () {
 
       await check1WebSeed()
       await check0PlaylistRedundancies()
-      await checkStatsWith1Webseed(strategy)
+      await checkStatsWithoutRedundancy(strategy)
     })
 
     it('Should view 2 times the first video to have > min_views config', async function () {
@@ -451,7 +456,7 @@ describe('Test videos redundancy', function () {
 
       await check2Webseeds()
       await check1PlaylistRedundancies()
-      await checkStatsWith2Webseed(strategy)
+      await checkStatsWith1Redundancy(strategy)
     })
 
     it('Should remove the video and the redundancy files', async function () {
@@ -471,6 +476,65 @@ describe('Test videos redundancy', function () {
     })
   })
 
+  describe('With only HLS files', function () {
+    const strategy = 'recently-added'
+
+    before(async function () {
+      this.timeout(120000)
+
+      await flushAndRunServers(strategy, { min_views: 3 }, false)
+    })
+
+    it('Should have 0 playlist redundancy on the first video', async function () {
+      await check1WebSeed()
+      await check0PlaylistRedundancies()
+    })
+
+    it('Should enable redundancy on server 1', function () {
+      return enableRedundancyOnServer1()
+    })
+
+    it('Should still have 0 redundancy on the first video', async function () {
+      this.timeout(80000)
+
+      await waitJobs(servers)
+      await wait(15000)
+      await waitJobs(servers)
+
+      await check0PlaylistRedundancies()
+      await checkStatsWithoutRedundancy(strategy)
+    })
+
+    it('Should have 1 redundancy on the first video', async function () {
+      this.timeout(160000)
+
+      await viewVideo(servers[0].url, video1Server2UUID)
+      await viewVideo(servers[2].url, video1Server2UUID)
+
+      await wait(10000)
+      await waitJobs(servers)
+
+      await waitJobs(servers)
+      await waitUntilLog(servers[0], 'Duplicated ', 1)
+      await waitJobs(servers)
+
+      await check1PlaylistRedundancies()
+      await checkStatsWith1Redundancy(strategy)
+    })
+
+    it('Should remove the video and the redundancy files', async function () {
+      this.timeout(20000)
+
+      await removeVideo(servers[1].url, servers[1].accessToken, video1Server2UUID)
+
+      await waitJobs(servers)
+
+      for (const server of servers) {
+        await checkVideoFilesWereRemoved(video1Server2UUID, server.internalServerNumber)
+      }
+    })
+  })
+
   describe('With manual strategy', function () {
     before(function () {
       this.timeout(120000)
@@ -481,7 +545,7 @@ describe('Test videos redundancy', function () {
     it('Should have 1 webseed on the first video', async function () {
       await check1WebSeed()
       await check0PlaylistRedundancies()
-      await checkStatsWith1Webseed('manual')
+      await checkStatsWithoutRedundancy('manual')
     })
 
     it('Should create a redundancy on first video', async function () {
@@ -501,7 +565,7 @@ describe('Test videos redundancy', function () {
 
       await check2Webseeds()
       await check1PlaylistRedundancies()
-      await checkStatsWith2Webseed('manual')
+      await checkStatsWith1Redundancy('manual')
     })
 
     it('Should manually remove redundancies on server 1 and remove duplicated videos', async function () {
@@ -619,7 +683,7 @@ describe('Test videos redundancy', function () {
 
       await check2Webseeds()
       await check1PlaylistRedundancies()
-      await checkStatsWith2Webseed(strategy)
+      await checkStatsWith1Redundancy(strategy)
 
       const res = await uploadVideo(servers[1].url, servers[1].accessToken, { name: 'video 2 server 2' })
       video2Server2UUID = res.body.video.uuid
diff --git a/shared/extra-utils/videos/videos.ts b/shared/extra-utils/videos/videos.ts
index 110552c77..f94fa233c 100644
--- a/shared/extra-utils/videos/videos.ts
+++ b/shared/extra-utils/videos/videos.ts
@@ -338,7 +338,7 @@ async function checkVideoFilesWereRemoved (
 
     const files = await readdir(directoryPath)
     for (const file of files) {
-      expect(file).to.not.contain(videoUUID)
+      expect(file, `File ${file} should not exist in ${directoryPath}`).to.not.contain(videoUUID)
     }
   }
 }