Improve redundancy: add 'min_lifetime' configuration

2 files changed, 193 insertions, 75 deletions

diff --git a/server/tests/api/server/redundancy.ts b/server/tests/api/server/redundancy.ts
index 6ce4b9dd1..a773e3de4 100644
--- a/server/tests/api/server/redundancy.ts
+++ b/server/tests/api/server/redundancy.ts
@@ -31,14 +31,13 @@ const expect = chai.expect
 
 let servers: ServerInfo[] = []
 let video1Server2UUID: string
-let video2Server2UUID: string
 
-function checkMagnetWebseeds (file: { magnetUri: string, resolution: { id: number } }, baseWebseeds: string[]) {
+function checkMagnetWebseeds (file: { magnetUri: string, resolution: { id: number } }, baseWebseeds: string[], server: ServerInfo) {
   const parsed = magnetUtil.decode(file.magnetUri)
 
   for (const ws of baseWebseeds) {
     const found = parsed.urlList.find(url => url === `${ws}-${file.resolution.id}.mp4`)
-    expect(found, `Webseed ${ws} not found in ${file.magnetUri}`).to.not.be.undefined
+    expect(found, `Webseed ${ws} not found in ${file.magnetUri} on server ${server.url}`).to.not.be.undefined
   }
 }
 
@@ -49,6 +48,7 @@ async function runServers (strategy: VideoRedundancyStrategy, additionalParams:
         check_interval: '5 seconds',
         strategies: [
           immutableAssign({
+            min_lifetime: '1 hour',
             strategy: strategy,
             size: '100KB'
           }, additionalParams)
@@ -68,11 +68,6 @@ async function runServers (strategy: VideoRedundancyStrategy, additionalParams:
     await viewVideo(servers[ 1 ].url, video1Server2UUID)
   }
 
-  {
-    const res = await uploadVideo(servers[ 1 ].url, servers[ 1 ].accessToken, { name: 'video 2 server 2' })
-    video2Server2UUID = res.body.video.uuid
-  }
-
   await waitJobs(servers)
 
   // Server 1 and server 2 follow each other
@@ -85,68 +80,69 @@ async function runServers (strategy: VideoRedundancyStrategy, additionalParams:
   await waitJobs(servers)
 }
 
-async function check1WebSeed (strategy: VideoRedundancyStrategy) {
+async function check1WebSeed (strategy: VideoRedundancyStrategy, videoUUID?: string) {
+  if (!videoUUID) videoUUID = video1Server2UUID
+
   const webseeds = [
-    'http://localhost:9002/static/webseed/' + video1Server2UUID
+    'http://localhost:9002/static/webseed/' + videoUUID
   ]
 
   for (const server of servers) {
     {
-      const res = await getVideo(server.url, video1Server2UUID)
+      const res = await getVideo(server.url, videoUUID)
 
       const video: VideoDetails = res.body
-      video.files.forEach(f => checkMagnetWebseeds(f, webseeds))
+      for (const f of video.files) {
+        checkMagnetWebseeds(f, webseeds, server)
+      }
     }
+  }
+}
 
-    {
-      const res = await getStats(server.url)
-      const data: ServerStats = res.body
+async function checkStatsWith2Webseed (strategy: VideoRedundancyStrategy) {
+  const res = await getStats(servers[0].url)
+  const data: ServerStats = res.body
 
-      expect(data.videosRedundancy).to.have.lengthOf(1)
+  expect(data.videosRedundancy).to.have.lengthOf(1)
+  const stat = data.videosRedundancy[0]
 
-      const stat = data.videosRedundancy[0]
-      expect(stat.strategy).to.equal(strategy)
-      expect(stat.totalSize).to.equal(102400)
-      expect(stat.totalUsed).to.equal(0)
-      expect(stat.totalVideoFiles).to.equal(0)
-      expect(stat.totalVideos).to.equal(0)
-    }
-  }
+  expect(stat.strategy).to.equal(strategy)
+  expect(stat.totalSize).to.equal(102400)
+  expect(stat.totalUsed).to.be.at.least(1).and.below(102401)
+  expect(stat.totalVideoFiles).to.equal(4)
+  expect(stat.totalVideos).to.equal(1)
 }
 
-async function enableRedundancy () {
-  await updateRedundancy(servers[ 0 ].url, servers[ 0 ].accessToken, servers[ 1 ].host, true)
+async function checkStatsWith1Webseed (strategy: VideoRedundancyStrategy) {
+  const res = await getStats(servers[0].url)
+  const data: ServerStats = res.body
 
-  const res = await getFollowingListPaginationAndSort(servers[ 0 ].url, 0, 5, '-createdAt')
-  const follows: ActorFollow[] = res.body.data
-  const server2 = follows.find(f => f.following.host === 'localhost:9002')
-  const server3 = follows.find(f => f.following.host === 'localhost:9003')
-
-  expect(server3).to.not.be.undefined
-  expect(server3.following.hostRedundancyAllowed).to.be.false
+  expect(data.videosRedundancy).to.have.lengthOf(1)
 
-  expect(server2).to.not.be.undefined
-  expect(server2.following.hostRedundancyAllowed).to.be.true
+  const stat = data.videosRedundancy[0]
+  expect(stat.strategy).to.equal(strategy)
+  expect(stat.totalSize).to.equal(102400)
+  expect(stat.totalUsed).to.equal(0)
+  expect(stat.totalVideoFiles).to.equal(0)
+  expect(stat.totalVideos).to.equal(0)
 }
 
-async function check2Webseeds (strategy: VideoRedundancyStrategy) {
-  await waitJobs(servers)
-  await wait(15000)
-  await waitJobs(servers)
+async function check2Webseeds (strategy: VideoRedundancyStrategy, videoUUID?: string) {
+  if (!videoUUID) videoUUID = video1Server2UUID
 
   const webseeds = [
-    'http://localhost:9001/static/webseed/' + video1Server2UUID,
-    'http://localhost:9002/static/webseed/' + video1Server2UUID
+    'http://localhost:9001/static/webseed/' + videoUUID,
+    'http://localhost:9002/static/webseed/' + videoUUID
   ]
 
   for (const server of servers) {
     {
-      const res = await getVideo(server.url, video1Server2UUID)
+      const res = await getVideo(server.url, videoUUID)
 
       const video: VideoDetails = res.body
 
       for (const file of video.files) {
-        checkMagnetWebseeds(file, webseeds)
+        checkMagnetWebseeds(file, webseeds, server)
       }
     }
   }
@@ -155,22 +151,23 @@ async function check2Webseeds (strategy: VideoRedundancyStrategy) {
   expect(files).to.have.lengthOf(4)
 
   for (const resolution of [ 240, 360, 480, 720 ]) {
-    expect(files.find(f => f === `${video1Server2UUID}-${resolution}.mp4`)).to.not.be.undefined
+    expect(files.find(f => f === `${videoUUID}-${resolution}.mp4`)).to.not.be.undefined
   }
+}
 
-  {
-    const res = await getStats(servers[0].url)
-    const data: ServerStats = res.body
+async function enableRedundancyOnServer1 () {
+  await updateRedundancy(servers[ 0 ].url, servers[ 0 ].accessToken, servers[ 1 ].host, true)
 
-    expect(data.videosRedundancy).to.have.lengthOf(1)
-    const stat = data.videosRedundancy[0]
+  const res = await getFollowingListPaginationAndSort(servers[ 0 ].url, 0, 5, '-createdAt')
+  const follows: ActorFollow[] = res.body.data
+  const server2 = follows.find(f => f.following.host === 'localhost:9002')
+  const server3 = follows.find(f => f.following.host === 'localhost:9003')
 
-    expect(stat.strategy).to.equal(strategy)
-    expect(stat.totalSize).to.equal(102400)
-    expect(stat.totalUsed).to.be.at.least(1).and.below(102401)
-    expect(stat.totalVideoFiles).to.equal(4)
-    expect(stat.totalVideos).to.equal(1)
-  }
+  expect(server3).to.not.be.undefined
+  expect(server3.following.hostRedundancyAllowed).to.be.false
+
+  expect(server2).to.not.be.undefined
+  expect(server2.following.hostRedundancyAllowed).to.be.true
 }
 
 async function cleanServers () {
@@ -188,18 +185,24 @@ describe('Test videos redundancy', function () {
       return runServers(strategy)
     })
 
-    it('Should have 1 webseed on the first video', function () {
-      return check1WebSeed(strategy)
+    it('Should have 1 webseed on the first video', async function () {
+      await check1WebSeed(strategy)
+      await checkStatsWith1Webseed(strategy)
     })
 
     it('Should enable redundancy on server 1', function () {
-      return enableRedundancy()
+      return enableRedundancyOnServer1()
     })
 
-    it('Should have 2 webseed on the first video', function () {
+    it('Should have 2 webseed on the first video', async function () {
       this.timeout(40000)
 
-      return check2Webseeds(strategy)
+      await waitJobs(servers)
+      await wait(15000)
+      await waitJobs(servers)
+
+      await check2Webseeds(strategy)
+      await checkStatsWith2Webseed(strategy)
     })
 
     after(function () {
@@ -216,18 +219,24 @@ describe('Test videos redundancy', function () {
       return runServers(strategy)
     })
 
-    it('Should have 1 webseed on the first video', function () {
-      return check1WebSeed(strategy)
+    it('Should have 1 webseed on the first video', async function () {
+      await check1WebSeed(strategy)
+      await checkStatsWith1Webseed(strategy)
     })
 
     it('Should enable redundancy on server 1', function () {
-      return enableRedundancy()
+      return enableRedundancyOnServer1()
     })
 
-    it('Should have 2 webseed on the first video', function () {
+    it('Should have 2 webseed on the first video', async function () {
       this.timeout(40000)
 
-      return check2Webseeds(strategy)
+      await waitJobs(servers)
+      await wait(15000)
+      await waitJobs(servers)
+
+      await check2Webseeds(strategy)
+      await checkStatsWith2Webseed(strategy)
     })
 
     after(function () {
@@ -241,15 +250,16 @@ describe('Test videos redundancy', function () {
     before(function () {
       this.timeout(120000)
 
-      return runServers(strategy, { minViews: 3 })
+      return runServers(strategy, { min_views: 3 })
     })
 
-    it('Should have 1 webseed on the first video', function () {
-      return check1WebSeed(strategy)
+    it('Should have 1 webseed on the first video', async function () {
+      await check1WebSeed(strategy)
+      await checkStatsWith1Webseed(strategy)
     })
 
     it('Should enable redundancy on server 1', function () {
-      return enableRedundancy()
+      return enableRedundancyOnServer1()
     })
 
     it('Should still have 1 webseed on the first video', async function () {
@@ -259,10 +269,11 @@ describe('Test videos redundancy', function () {
       await wait(15000)
       await waitJobs(servers)
 
-      return check1WebSeed(strategy)
+      await check1WebSeed(strategy)
+      await checkStatsWith1Webseed(strategy)
     })
 
-    it('Should view 2 times the first video', async function () {
+    it('Should view 2 times the first video to have > min_views config', async function () {
       this.timeout(40000)
 
       await viewVideo(servers[ 0 ].url, video1Server2UUID)
@@ -272,10 +283,117 @@ describe('Test videos redundancy', function () {
       await waitJobs(servers)
     })
 
-    it('Should have 2 webseed on the first video', function () {
+    it('Should have 2 webseed on the first video', async function () {
       this.timeout(40000)
 
-      return check2Webseeds(strategy)
+      await waitJobs(servers)
+      await wait(15000)
+      await waitJobs(servers)
+
+      await check2Webseeds(strategy)
+      await checkStatsWith2Webseed(strategy)
+    })
+
+    after(function () {
+      return cleanServers()
+    })
+  })
+
+  describe('Test expiration', function () {
+    const strategy = 'recently-added'
+
+    async function checkContains (servers: ServerInfo[], str: string) {
+      for (const server of servers) {
+        const res = await getVideo(server.url, video1Server2UUID)
+        const video: VideoDetails = res.body
+
+        for (const f of video.files) {
+          expect(f.magnetUri).to.contain(str)
+        }
+      }
+    }
+
+    async function checkNotContains (servers: ServerInfo[], str: string) {
+      for (const server of servers) {
+        const res = await getVideo(server.url, video1Server2UUID)
+        const video: VideoDetails = res.body
+
+        for (const f of video.files) {
+          expect(f.magnetUri).to.not.contain(str)
+        }
+      }
+    }
+
+    before(async function () {
+      this.timeout(120000)
+
+      await runServers(strategy, { min_lifetime: '7 seconds', min_views: 0 })
+
+      await enableRedundancyOnServer1()
+    })
+
+    it('Should still have 2 webseeds after 10 seconds', async function () {
+      this.timeout(40000)
+
+      await wait(10000)
+
+      try {
+        await checkContains(servers, 'http%3A%2F%2Flocalhost%3A9001')
+      } catch {
+        // Maybe a server deleted a redundancy in the scheduler
+        await wait(2000)
+
+        await checkContains(servers, 'http%3A%2F%2Flocalhost%3A9001')
+      }
+    })
+
+    it('Should stop server 1 and expire video redundancy', async function () {
+      this.timeout(40000)
+
+      killallServers([ servers[0] ])
+
+      await wait(10000)
+
+      await checkNotContains([ servers[1], servers[2] ], 'http%3A%2F%2Flocalhost%3A9001')
+    })
+
+    after(function () {
+      return killallServers([ servers[1], servers[2] ])
+    })
+  })
+
+  describe('Test file replacement', function () {
+    let video2Server2UUID: string
+    const strategy = 'recently-added'
+
+    before(async function () {
+      this.timeout(120000)
+
+      await runServers(strategy, { min_lifetime: '7 seconds', min_views: 0 })
+
+      await enableRedundancyOnServer1()
+
+      await waitJobs(servers)
+      await wait(5000)
+      await waitJobs(servers)
+
+      await check2Webseeds(strategy)
+      await checkStatsWith2Webseed(strategy)
+
+      const res = await uploadVideo(servers[ 1 ].url, servers[ 1 ].accessToken, { name: 'video 2 server 2' })
+      video2Server2UUID = res.body.video.uuid
+    })
+
+    it('Should cache video 2 webseed on the first video', async function () {
+      this.timeout(40000)
+      this.retries(3)
+
+      await waitJobs(servers)
+
+      await wait(7000)
+
+      await check1WebSeed(strategy, video1Server2UUID)
+      await check2Webseeds(strategy, video2Server2UUID)
     })
 
     after(function () {
diff --git a/server/tests/utils/server/servers.ts b/server/tests/utils/server/servers.ts
index 26ab4e1bb..fbfc83ca1 100644
--- a/server/tests/utils/server/servers.ts
+++ b/server/tests/utils/server/servers.ts
@@ -144,8 +144,8 @@ function runServer (serverNumber: number, configOverride?: Object) {
   })
 }
 
-async function reRunServer (server: ServerInfo) {
-  const newServer = await runServer(server.serverNumber)
+async function reRunServer (server: ServerInfo, configOverride?: any) {
+  const newServer = await runServer(server.serverNumber, configOverride)
   server.app = newServer.app
 
   return server