Merge branch 'develop' into cli-wrapper

5 files changed, 227 insertions, 78 deletions

diff --git a/server/tests/api/server/jobs.ts b/server/tests/api/server/jobs.ts
index b2922c5da..f5a19c5ea 100644
--- a/server/tests/api/server/jobs.ts
+++ b/server/tests/api/server/jobs.ts
@@ -45,7 +45,9 @@ describe('Test jobs', function () {
     expect(res.body.total).to.be.above(2)
     expect(res.body.data).to.have.lengthOf(1)
 
-    const job = res.body.data[0]
+    let job = res.body.data[0]
+    // Skip repeat jobs
+    if (job.type === 'videos-views') job = res.body.data[1]
 
     expect(job.state).to.equal('completed')
     expect(job.type).to.equal('activitypub-follow')
diff --git a/server/tests/api/server/redundancy.ts b/server/tests/api/server/redundancy.ts
index c0ec75a45..6ce4b9dd1 100644
--- a/server/tests/api/server/redundancy.ts
+++ b/server/tests/api/server/redundancy.ts
@@ -6,15 +6,16 @@ import { VideoDetails } from '../../../../shared/models/videos'
 import {
   doubleFollow,
   flushAndRunMultipleServers,
-  flushTests,
   getFollowingListPaginationAndSort,
   getVideo,
+  immutableAssign,
   killallServers,
+  root,
   ServerInfo,
   setAccessTokensToServers,
   uploadVideo,
-  wait,
-  root, viewVideo
+  viewVideo,
+  wait
 } from '../../utils'
 import { waitJobs } from '../../utils/server/jobs'
 import * as magnetUtil from 'magnet-uri'
@@ -22,9 +23,16 @@ import { updateRedundancy } from '../../utils/server/redundancy'
 import { ActorFollow } from '../../../../shared/models/actors'
 import { readdir } from 'fs-extra'
 import { join } from 'path'
+import { VideoRedundancyStrategy } from '../../../../shared/models/redundancy'
+import { getStats } from '../../utils/server/stats'
+import { ServerStats } from '../../../../shared/models/server/server-stats.model'
 
 const expect = chai.expect
 
+let servers: ServerInfo[] = []
+let video1Server2UUID: string
+let video2Server2UUID: string
+
 function checkMagnetWebseeds (file: { magnetUri: string, resolution: { id: number } }, baseWebseeds: string[]) {
   const parsed = magnetUtil.decode(file.magnetUri)
 
@@ -34,84 +42,105 @@ function checkMagnetWebseeds (file: { magnetUri: string, resolution: { id: numbe
   }
 }
 
-describe('Test videos redundancy', function () {
-  let servers: ServerInfo[] = []
-  let video1Server2UUID: string
-  let video2Server2UUID: string
-
-  before(async function () {
-    this.timeout(120000)
-
-    servers = await flushAndRunMultipleServers(3)
+async function runServers (strategy: VideoRedundancyStrategy, additionalParams: any = {}) {
+  const config = {
+    redundancy: {
+      videos: {
+        check_interval: '5 seconds',
+        strategies: [
+          immutableAssign({
+            strategy: strategy,
+            size: '100KB'
+          }, additionalParams)
+        ]
+      }
+    }
+  }
+  servers = await flushAndRunMultipleServers(3, config)
 
-    // Get the access tokens
-    await setAccessTokensToServers(servers)
+  // Get the access tokens
+  await setAccessTokensToServers(servers)
 
-    {
-      const res = await uploadVideo(servers[ 1 ].url, servers[ 1 ].accessToken, { name: 'video 1 server 2' })
-      video1Server2UUID = res.body.video.uuid
+  {
+    const res = await uploadVideo(servers[ 1 ].url, servers[ 1 ].accessToken, { name: 'video 1 server 2' })
+    video1Server2UUID = res.body.video.uuid
 
-      await viewVideo(servers[1].url, video1Server2UUID)
-    }
+    await viewVideo(servers[ 1 ].url, video1Server2UUID)
+  }
 
-    {
-      const res = await uploadVideo(servers[ 1 ].url, servers[ 1 ].accessToken, { name: 'video 2 server 2' })
-      video2Server2UUID = res.body.video.uuid
-    }
+  {
+    const res = await uploadVideo(servers[ 1 ].url, servers[ 1 ].accessToken, { name: 'video 2 server 2' })
+    video2Server2UUID = res.body.video.uuid
+  }
 
-    await waitJobs(servers)
+  await waitJobs(servers)
 
-    // Server 1 and server 2 follow each other
-    await doubleFollow(servers[0], servers[1])
-    // Server 1 and server 3 follow each other
-    await doubleFollow(servers[0], servers[2])
-    // Server 2 and server 3 follow each other
-    await doubleFollow(servers[1], servers[2])
+  // Server 1 and server 2 follow each other
+  await doubleFollow(servers[ 0 ], servers[ 1 ])
+  // Server 1 and server 3 follow each other
+  await doubleFollow(servers[ 0 ], servers[ 2 ])
+  // Server 2 and server 3 follow each other
+  await doubleFollow(servers[ 1 ], servers[ 2 ])
 
-    await waitJobs(servers)
-  })
+  await waitJobs(servers)
+}
 
-  it('Should have 1 webseed on the first video', async function () {
-    const webseeds = [
-      'http://localhost:9002/static/webseed/' + video1Server2UUID
-    ]
+async function check1WebSeed (strategy: VideoRedundancyStrategy) {
+  const webseeds = [
+    'http://localhost:9002/static/webseed/' + video1Server2UUID
+  ]
 
-    for (const server of servers) {
+  for (const server of servers) {
+    {
       const res = await getVideo(server.url, video1Server2UUID)
 
       const video: VideoDetails = res.body
       video.files.forEach(f => checkMagnetWebseeds(f, webseeds))
     }
-  })
 
-  it('Should enable redundancy on server 1', async function () {
-    await updateRedundancy(servers[0].url, servers[0].accessToken, servers[1].host, true)
+    {
+      const res = await getStats(server.url)
+      const data: ServerStats = res.body
 
-    const res = await getFollowingListPaginationAndSort(servers[0].url, 0, 5, '-createdAt')
-    const follows: ActorFollow[] = res.body.data
-    const server2 = follows.find(f => f.following.host === 'localhost:9002')
-    const server3 = follows.find(f => f.following.host === 'localhost:9003')
+      expect(data.videosRedundancy).to.have.lengthOf(1)
 
-    expect(server3).to.not.be.undefined
-    expect(server3.following.hostRedundancyAllowed).to.be.false
+      const stat = data.videosRedundancy[0]
+      expect(stat.strategy).to.equal(strategy)
+      expect(stat.totalSize).to.equal(102400)
+      expect(stat.totalUsed).to.equal(0)
+      expect(stat.totalVideoFiles).to.equal(0)
+      expect(stat.totalVideos).to.equal(0)
+    }
+  }
+}
 
-    expect(server2).to.not.be.undefined
-    expect(server2.following.hostRedundancyAllowed).to.be.true
-  })
+async function enableRedundancy () {
+  await updateRedundancy(servers[ 0 ].url, servers[ 0 ].accessToken, servers[ 1 ].host, true)
 
-  it('Should have 2 webseed on the first video', async function () {
-    this.timeout(40000)
+  const res = await getFollowingListPaginationAndSort(servers[ 0 ].url, 0, 5, '-createdAt')
+  const follows: ActorFollow[] = res.body.data
+  const server2 = follows.find(f => f.following.host === 'localhost:9002')
+  const server3 = follows.find(f => f.following.host === 'localhost:9003')
 
-    await waitJobs(servers)
-    await wait(15000)
-    await waitJobs(servers)
+  expect(server3).to.not.be.undefined
+  expect(server3.following.hostRedundancyAllowed).to.be.false
 
-    const webseeds = [
-      'http://localhost:9001/static/webseed/' + video1Server2UUID,
-      'http://localhost:9002/static/webseed/' + video1Server2UUID
-    ]
+  expect(server2).to.not.be.undefined
+  expect(server2.following.hostRedundancyAllowed).to.be.true
+}
+
+async function check2Webseeds (strategy: VideoRedundancyStrategy) {
+  await waitJobs(servers)
+  await wait(15000)
+  await waitJobs(servers)
 
-    for (const server of servers) {
+  const webseeds = [
+    'http://localhost:9001/static/webseed/' + video1Server2UUID,
+    'http://localhost:9002/static/webseed/' + video1Server2UUID
+  ]
+
+  for (const server of servers) {
+    {
       const res = await getVideo(server.url, video1Server2UUID)
 
       const video: VideoDetails = res.body
@@ -120,21 +149,137 @@ describe('Test videos redundancy', function () {
         checkMagnetWebseeds(file, webseeds)
       }
     }
+  }
 
-    const files = await readdir(join(root(), 'test1', 'videos'))
-    expect(files).to.have.lengthOf(4)
+  const files = await readdir(join(root(), 'test1', 'videos'))
+  expect(files).to.have.lengthOf(4)
 
-    for (const resolution of [ 240, 360, 480, 720 ]) {
-      expect(files.find(f => f === `${video1Server2UUID}-${resolution}.mp4`)).to.not.be.undefined
-    }
+  for (const resolution of [ 240, 360, 480, 720 ]) {
+    expect(files.find(f => f === `${video1Server2UUID}-${resolution}.mp4`)).to.not.be.undefined
+  }
+
+  {
+    const res = await getStats(servers[0].url)
+    const data: ServerStats = res.body
+
+    expect(data.videosRedundancy).to.have.lengthOf(1)
+    const stat = data.videosRedundancy[0]
+
+    expect(stat.strategy).to.equal(strategy)
+    expect(stat.totalSize).to.equal(102400)
+    expect(stat.totalUsed).to.be.at.least(1).and.below(102401)
+    expect(stat.totalVideoFiles).to.equal(4)
+    expect(stat.totalVideos).to.equal(1)
+  }
+}
+
+async function cleanServers () {
+  killallServers(servers)
+}
+
+describe('Test videos redundancy', function () {
+
+  describe('With most-views strategy', function () {
+    const strategy = 'most-views'
+
+    before(function () {
+      this.timeout(120000)
+
+      return runServers(strategy)
+    })
+
+    it('Should have 1 webseed on the first video', function () {
+      return check1WebSeed(strategy)
+    })
+
+    it('Should enable redundancy on server 1', function () {
+      return enableRedundancy()
+    })
+
+    it('Should have 2 webseed on the first video', function () {
+      this.timeout(40000)
+
+      return check2Webseeds(strategy)
+    })
+
+    after(function () {
+      return cleanServers()
+    })
   })
 
-  after(async function () {
-    killallServers(servers)
+  describe('With trending strategy', function () {
+    const strategy = 'trending'
 
-    // Keep the logs if the test failed
-    if (this['ok']) {
-      await flushTests()
-    }
+    before(function () {
+      this.timeout(120000)
+
+      return runServers(strategy)
+    })
+
+    it('Should have 1 webseed on the first video', function () {
+      return check1WebSeed(strategy)
+    })
+
+    it('Should enable redundancy on server 1', function () {
+      return enableRedundancy()
+    })
+
+    it('Should have 2 webseed on the first video', function () {
+      this.timeout(40000)
+
+      return check2Webseeds(strategy)
+    })
+
+    after(function () {
+      return cleanServers()
+    })
+  })
+
+  describe('With recently added strategy', function () {
+    const strategy = 'recently-added'
+
+    before(function () {
+      this.timeout(120000)
+
+      return runServers(strategy, { minViews: 3 })
+    })
+
+    it('Should have 1 webseed on the first video', function () {
+      return check1WebSeed(strategy)
+    })
+
+    it('Should enable redundancy on server 1', function () {
+      return enableRedundancy()
+    })
+
+    it('Should still have 1 webseed on the first video', async function () {
+      this.timeout(40000)
+
+      await waitJobs(servers)
+      await wait(15000)
+      await waitJobs(servers)
+
+      return check1WebSeed(strategy)
+    })
+
+    it('Should view 2 times the first video', async function () {
+      this.timeout(40000)
+
+      await viewVideo(servers[ 0 ].url, video1Server2UUID)
+      await viewVideo(servers[ 2 ].url, video1Server2UUID)
+
+      await wait(10000)
+      await waitJobs(servers)
+    })
+
+    it('Should have 2 webseed on the first video', function () {
+      this.timeout(40000)
+
+      return check2Webseeds(strategy)
+    })
+
+    after(function () {
+      return cleanServers()
+    })
   })
 })
diff --git a/server/tests/api/server/stats.ts b/server/tests/api/server/stats.ts
index fc9b88805..cb229e876 100644
--- a/server/tests/api/server/stats.ts
+++ b/server/tests/api/server/stats.ts
@@ -21,7 +21,7 @@ import { waitJobs } from '../../utils/server/jobs'
 
 const expect = chai.expect
 
-describe('Test stats', function () {
+describe('Test stats (excluding redundancy)', function () {
   let servers: ServerInfo[] = []
 
   before(async function () {
diff --git a/server/tests/utils/server/servers.ts b/server/tests/utils/server/servers.ts
index 1372c03c3..26ab4e1bb 100644
--- a/server/tests/utils/server/servers.ts
+++ b/server/tests/utils/server/servers.ts
@@ -35,7 +35,7 @@ interface ServerInfo {
   }
 }
 
-function flushAndRunMultipleServers (totalServers) {
+function flushAndRunMultipleServers (totalServers: number, configOverride?: Object) {
   let apps = []
   let i = 0
 
@@ -51,10 +51,7 @@ function flushAndRunMultipleServers (totalServers) {
     flushTests()
       .then(() => {
         for (let j = 1; j <= totalServers; j++) {
-          // For the virtual buffer
-          setTimeout(() => {
-            runServer(j).then(app => anotherServerDone(j, app))
-          }, 1000 * (j - 1))
+          runServer(j, configOverride).then(app => anotherServerDone(j, app))
         }
       })
   })
diff --git a/server/tests/utils/server/stats.ts b/server/tests/utils/server/stats.ts
index 9cdec6cff..01989d952 100644
--- a/server/tests/utils/server/stats.ts
+++ b/server/tests/utils/server/stats.ts
@@ -1,11 +1,16 @@
 import { makeGetRequest } from '../'
 
-function getStats (url: string) {
+function getStats (url: string, useCache = false) {
   const path = '/api/v1/server/stats'
 
+  const query = {
+    t: useCache ? undefined : new Date().getTime()
+  }
+
   return makeGetRequest({
     url,
     path,
+    query,
     statusCodeExpected: 200
   })
 }