Remove duplicated videos on unfollow/delete redundancy

5 files changed, 114 insertions, 16 deletions

diff --git a/server/controllers/api/server/follows.ts b/server/controllers/api/server/follows.ts
index a4eae6b45..d62400e42 100644
--- a/server/controllers/api/server/follows.ts
+++ b/server/controllers/api/server/follows.ts
@@ -17,6 +17,7 @@ import {
 import { followersSortValidator, followingSortValidator, followValidator } from '../../../middlewares/validators'
 import { ActorFollowModel } from '../../../models/activitypub/actor-follow'
 import { JobQueue } from '../../../lib/job-queue'
+import { removeRedundancyOf } from '../../../lib/redundancy'
 
 const serverFollowsRouter = express.Router()
 serverFollowsRouter.get('/following',
@@ -101,6 +102,10 @@ async function removeFollow (req: express.Request, res: express.Response, next:
     server.redundancyAllowed = false
     await server.save({ transaction: t })
 
+    // Async, could be long
+    removeRedundancyOf(server.id)
+      .catch(err => logger.error('Cannot remove redundancy of %s.', server.host, err))
+
     await follow.destroy({ transaction: t })
   })
 
diff --git a/server/controllers/api/server/redundancy.ts b/server/controllers/api/server/redundancy.ts
index 4216b9e35..4140c4991 100644
--- a/server/controllers/api/server/redundancy.ts
+++ b/server/controllers/api/server/redundancy.ts
@@ -3,6 +3,8 @@ import { UserRight } from '../../../../shared/models/users'
 import { asyncMiddleware, authenticate, ensureUserHasRight } from '../../../middlewares'
 import { updateServerRedundancyValidator } from '../../../middlewares/validators/redundancy'
 import { ServerModel } from '../../../models/server/server'
+import { removeRedundancyOf } from '../../../lib/redundancy'
+import { logger } from '../../../helpers/logger'
 
 const serverRedundancyRouter = express.Router()
 
@@ -28,5 +30,9 @@ async function updateRedundancy (req: express.Request, res: express.Response, ne
 
   await server.save()
 
+  // Async, could be long
+  removeRedundancyOf(server.id)
+    .catch(err => logger.error('Cannot remove redundancy of %s.', server.host, err))
+
   return res.sendStatus(204)
 }
diff --git a/server/lib/redundancy.ts b/server/lib/redundancy.ts
index 16b122658..04d3ded8f 100644
--- a/server/lib/redundancy.ts
+++ b/server/lib/redundancy.ts
@@ -12,8 +12,17 @@ async function removeVideoRedundancy (videoRedundancy: VideoRedundancyModel, t?:
   await videoRedundancy.destroy({ transaction: t })
 }
 
+async function removeRedundancyOf (serverId: number) {
+  const videosRedundancy = await VideoRedundancyModel.listLocalOfServer(serverId)
+
+  for (const redundancy of videosRedundancy) {
+    await removeVideoRedundancy(redundancy)
+  }
+}
+
 // ---------------------------------------------------------------------------
 
 export {
+  removeRedundancyOf,
   removeVideoRedundancy
 }
diff --git a/server/models/redundancy/video-redundancy.ts b/server/models/redundancy/video-redundancy.ts
index 3c87ec2c1..e67164802 100644
--- a/server/models/redundancy/video-redundancy.ts
+++ b/server/models/redundancy/video-redundancy.ts
@@ -286,6 +286,47 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> {
     return VideoRedundancyModel.scope([ ScopeNames.WITH_VIDEO ]).findAll(query)
   }
 
+  static async listLocalOfServer (serverId: number) {
+    const actor = await getServerActor()
+
+    const query = {
+      where: {
+        actorId: actor.id
+      },
+      include: [
+        {
+          model: VideoFileModel,
+          required: true,
+          include: [
+            {
+              model: VideoModel,
+              required: true,
+              include: [
+                {
+                  attributes: [],
+                  model: VideoChannelModel.unscoped(),
+                  required: true,
+                  include: [
+                    {
+                      attributes: [],
+                      model: ActorModel.unscoped(),
+                      required: true,
+                      where: {
+                        serverId
+                      }
+                    }
+                  ]
+                }
+              ]
+            }
+          ]
+        }
+      ]
+    }
+
+    return VideoRedundancyModel.findAll(query)
+  }
+
   static async getStats (strategy: VideoRedundancyStrategy) {
     const actor = await getServerActor()
 
diff --git a/server/tests/api/server/redundancy.ts b/server/tests/api/server/redundancy.ts
index 9b3b1b4ad..e1709891d 100644
--- a/server/tests/api/server/redundancy.ts
+++ b/server/tests/api/server/redundancy.ts
@@ -12,7 +12,7 @@ import {
   killallServers, makeGetRequest,
   root,
   ServerInfo,
-  setAccessTokensToServers,
+  setAccessTokensToServers, unfollow,
   uploadVideo,
   viewVideo,
   wait
@@ -39,6 +39,8 @@ function checkMagnetWebseeds (file: { magnetUri: string, resolution: { id: numbe
     const found = parsed.urlList.find(url => url === `${ws}-${file.resolution.id}.mp4`)
     expect(found, `Webseed ${ws} not found in ${file.magnetUri} on server ${server.url}`).to.not.be.undefined
   }
+
+  expect(parsed.urlList).to.have.lengthOf(baseWebseeds.length)
 }
 
 async function runServers (strategy: VideoRedundancyStrategy, additionalParams: any = {}) {
@@ -136,23 +138,21 @@ async function check2Webseeds (strategy: VideoRedundancyStrategy, videoUUID?: st
   ]
 
   for (const server of servers) {
-    {
-      const res = await getVideo(server.url, videoUUID)
+    const res = await getVideo(server.url, videoUUID)
 
-      const video: VideoDetails = res.body
+    const video: VideoDetails = res.body
 
-      for (const file of video.files) {
-        checkMagnetWebseeds(file, webseeds, server)
-
-        // Only servers 1 and 2 have the video
-        if (server.serverNumber !== 3) {
-          await makeGetRequest({
-            url: server.url,
-            statusCodeExpected: 200,
-            path: '/static/webseed/' + `${videoUUID}-${file.resolution.id}.mp4`,
-            contentType: null
-          })
-        }
+    for (const file of video.files) {
+      checkMagnetWebseeds(file, webseeds, server)
+
+      // Only servers 1 and 2 have the video
+      if (server.serverNumber !== 3) {
+        await makeGetRequest({
+          url: server.url,
+          statusCodeExpected: 200,
+          path: '/static/webseed/' + `${videoUUID}-${file.resolution.id}.mp4`,
+          contentType: null
+        })
       }
     }
   }
@@ -182,6 +182,21 @@ async function enableRedundancyOnServer1 () {
   expect(server2.following.hostRedundancyAllowed).to.be.true
 }
 
+async function disableRedundancyOnServer1 () {
+  await updateRedundancy(servers[ 0 ].url, servers[ 0 ].accessToken, servers[ 1 ].host, false)
+
+  const res = await getFollowingListPaginationAndSort(servers[ 0 ].url, 0, 5, '-createdAt')
+  const follows: ActorFollow[] = res.body.data
+  const server2 = follows.find(f => f.following.host === 'localhost:9002')
+  const server3 = follows.find(f => f.following.host === 'localhost:9003')
+
+  expect(server3).to.not.be.undefined
+  expect(server3.following.hostRedundancyAllowed).to.be.false
+
+  expect(server2).to.not.be.undefined
+  expect(server2.following.hostRedundancyAllowed).to.be.false
+}
+
 async function cleanServers () {
   killallServers(servers)
 }
@@ -217,6 +232,17 @@ describe('Test videos redundancy', function () {
       await checkStatsWith2Webseed(strategy)
     })
 
+    it('Should undo redundancy on server 1 and remove duplicated videos', async function () {
+      this.timeout(40000)
+
+      await disableRedundancyOnServer1()
+
+      await waitJobs(servers)
+      await wait(5000)
+
+      await check1WebSeed(strategy)
+    })
+
     after(function () {
       return cleanServers()
     })
@@ -251,6 +277,17 @@ describe('Test videos redundancy', function () {
       await checkStatsWith2Webseed(strategy)
     })
 
+    it('Should unfollow on server 1 and remove duplicated videos', async function () {
+      this.timeout(40000)
+
+      await unfollow(servers[0].url, servers[0].accessToken, servers[1])
+
+      await waitJobs(servers)
+      await wait(5000)
+
+      await check1WebSeed(strategy)
+    })
+
     after(function () {
       return cleanServers()
     })