Add redundancy stats

7 files changed, 122 insertions, 33 deletions

diff --git a/server/controllers/api/overviews.ts b/server/controllers/api/overviews.ts
index cc3cc54a7..8b6773056 100644
--- a/server/controllers/api/overviews.ts
+++ b/server/controllers/api/overviews.ts
@@ -21,6 +21,16 @@ export { overviewsRouter }
 
 // ---------------------------------------------------------------------------
 
+const buildSamples = memoizee(async function () {
+  const [ categories, channels, tags ] = await Promise.all([
+    VideoModel.getRandomFieldSamples('category', OVERVIEWS.VIDEOS.SAMPLE_THRESHOLD, OVERVIEWS.VIDEOS.SAMPLES_COUNT),
+    VideoModel.getRandomFieldSamples('channelId', OVERVIEWS.VIDEOS.SAMPLE_THRESHOLD ,OVERVIEWS.VIDEOS.SAMPLES_COUNT),
+    TagModel.getRandomSamples(OVERVIEWS.VIDEOS.SAMPLE_THRESHOLD, OVERVIEWS.VIDEOS.SAMPLES_COUNT)
+  ])
+
+  return { categories, channels, tags }
+}, { maxAge: MEMOIZE_TTL.OVERVIEWS_SAMPLE })
+
 // This endpoint could be quite long, but we cache it
 async function getVideosOverview (req: express.Request, res: express.Response) {
   const attributes = await buildSamples()
@@ -45,16 +55,6 @@ async function getVideosOverview (req: express.Request, res: express.Response) {
   return res.json(result)
 }
 
-const buildSamples = memoizee(async function () {
-  const [ categories, channels, tags ] = await Promise.all([
-    VideoModel.getRandomFieldSamples('category', OVERVIEWS.VIDEOS.SAMPLE_THRESHOLD, OVERVIEWS.VIDEOS.SAMPLES_COUNT),
-    VideoModel.getRandomFieldSamples('channelId', OVERVIEWS.VIDEOS.SAMPLE_THRESHOLD ,OVERVIEWS.VIDEOS.SAMPLES_COUNT),
-    TagModel.getRandomSamples(OVERVIEWS.VIDEOS.SAMPLE_THRESHOLD, OVERVIEWS.VIDEOS.SAMPLES_COUNT)
-  ])
-
-  return { categories, channels, tags }
-}, { maxAge: MEMOIZE_TTL.OVERVIEWS_SAMPLE })
-
 async function getVideosByTag (tag: string, res: express.Response) {
   const videos = await getVideos(res, { tagsOneOf: [ tag ] })
 
diff --git a/server/controllers/api/server/stats.ts b/server/controllers/api/server/stats.ts
index 6f4fe938c..bb6311e81 100644
--- a/server/controllers/api/server/stats.ts
+++ b/server/controllers/api/server/stats.ts
@@ -5,10 +5,14 @@ import { UserModel } from '../../../models/account/user'
 import { ActorFollowModel } from '../../../models/activitypub/actor-follow'
 import { VideoModel } from '../../../models/video/video'
 import { VideoCommentModel } from '../../../models/video/video-comment'
+import { VideoRedundancyModel } from '../../../models/redundancy/video-redundancy'
+import { CONFIG, ROUTE_CACHE_LIFETIME } from '../../../initializers/constants'
+import { cacheRoute } from '../../../middlewares/cache'
 
 const statsRouter = express.Router()
 
 statsRouter.get('/stats',
+  asyncMiddleware(cacheRoute(ROUTE_CACHE_LIFETIME.STATS)),
   asyncMiddleware(getStats)
 )
 
@@ -18,6 +22,13 @@ async function getStats (req: express.Request, res: express.Response, next: expr
   const { totalUsers } = await UserModel.getStats()
   const { totalInstanceFollowers, totalInstanceFollowing } = await ActorFollowModel.getStats()
 
+  const videosRedundancyStats = await Promise.all(
+    CONFIG.REDUNDANCY.VIDEOS.map(r => {
+      return VideoRedundancyModel.getStats(r.strategy)
+        .then(stats => Object.assign(stats, { strategy: r.strategy, totalSize: r.size }))
+    })
+  )
+
   const data: ServerStats = {
     totalLocalVideos,
     totalLocalVideoViews,
@@ -26,7 +37,8 @@ async function getStats (req: express.Request, res: express.Response, next: expr
     totalVideoComments,
     totalUsers,
     totalInstanceFollowers,
-    totalInstanceFollowing
+    totalInstanceFollowing,
+    videosRedundancy: videosRedundancyStats
   }
 
   return res.json(data).end()
diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts
index 9cccb0919..e8dab21db 100644
--- a/server/initializers/constants.ts
+++ b/server/initializers/constants.ts
@@ -66,7 +66,8 @@ const ROUTE_CACHE_LIFETIME = {
   },
   ACTIVITY_PUB: {
     VIDEOS: '1 second' // 1 second, cache concurrent requests after a broadcast for example
-  }
+  },
+  STATS: '4 hours'
 }
 
 // ---------------------------------------------------------------------------
diff --git a/server/models/redundancy/video-redundancy.ts b/server/models/redundancy/video-redundancy.ts
index b7454c617..6ae02efb9 100644
--- a/server/models/redundancy/video-redundancy.ts
+++ b/server/models/redundancy/video-redundancy.ts
@@ -245,6 +245,37 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> {
                                .findAll(query)
   }
 
+  static async getStats (strategy: VideoRedundancyStrategy) {
+    const actor = await getServerActor()
+
+    const query = {
+      raw: true,
+      attributes: [
+        [ Sequelize.fn('COALESCE', Sequelize.fn('SUM', Sequelize.col('VideoFile.size')), '0'), 'totalUsed' ],
+        [ Sequelize.fn('COUNT', Sequelize.fn('DISTINCT', 'videoId')), 'totalVideos' ],
+        [ Sequelize.fn('COUNT', 'videoFileId'), 'totalVideoFiles' ]
+      ],
+      where: {
+        strategy,
+        actorId: actor.id
+      },
+      include: [
+        {
+          attributes: [],
+          model: VideoFileModel,
+          required: true
+        }
+      ]
+    }
+
+    return VideoRedundancyModel.find(query as any) // FIXME: typings
+      .then((r: any) => ({
+        totalUsed: parseInt(r.totalUsed.toString(), 10),
+        totalVideos: r.totalVideos,
+        totalVideoFiles: r.totalVideoFiles
+      }))
+  }
+
   toActivityPubObject (): CacheFileObject {
     return {
       id: this.url,
diff --git a/server/tests/api/server/redundancy.ts b/server/tests/api/server/redundancy.ts
index 6574e8ea9..c0ab251e6 100644
--- a/server/tests/api/server/redundancy.ts
+++ b/server/tests/api/server/redundancy.ts
@@ -23,6 +23,8 @@ import { ActorFollow } from '../../../../shared/models/actors'
 import { readdir } from 'fs-extra'
 import { join } from 'path'
 import { VideoRedundancyStrategy } from '../../../../shared/models/redundancy'
+import { getStats } from '../../utils/server/stats'
+import { ServerStats } from '../../../../shared/models/server/server-stats.model'
 
 const expect = chai.expect
 
@@ -79,16 +81,32 @@ async function runServers (strategy: VideoRedundancyStrategy, additionalParams:
   await waitJobs(servers)
 }
 
-async function check1WebSeed () {
+async function check1WebSeed (strategy: VideoRedundancyStrategy) {
   const webseeds = [
     'http://localhost:9002/static/webseed/' + video1Server2UUID
   ]
 
   for (const server of servers) {
-    const res = await getVideo(server.url, video1Server2UUID)
+    {
+      const res = await getVideo(server.url, video1Server2UUID)
 
-    const video: VideoDetails = res.body
-    video.files.forEach(f => checkMagnetWebseeds(f, webseeds))
+      const video: VideoDetails = res.body
+      video.files.forEach(f => checkMagnetWebseeds(f, webseeds))
+    }
+
+    {
+      const res = await getStats(server.url)
+      const data: ServerStats = res.body
+
+      expect(data.videosRedundancy).to.have.lengthOf(1)
+
+      const stat = data.videosRedundancy[0]
+      expect(stat.strategy).to.equal(strategy)
+      expect(stat.totalSize).to.equal(102400)
+      expect(stat.totalUsed).to.equal(0)
+      expect(stat.totalVideoFiles).to.equal(0)
+      expect(stat.totalVideos).to.equal(0)
+    }
   }
 }
 
@@ -107,7 +125,7 @@ async function enableRedundancy () {
   expect(server2.following.hostRedundancyAllowed).to.be.true
 }
 
-async function check2Webseeds () {
+async function check2Webseeds (strategy: VideoRedundancyStrategy) {
   await waitJobs(servers)
   await wait(15000)
   await waitJobs(servers)
@@ -118,12 +136,14 @@ async function check2Webseeds () {
   ]
 
   for (const server of servers) {
-    const res = await getVideo(server.url, video1Server2UUID)
+    {
+      const res = await getVideo(server.url, video1Server2UUID)
 
-    const video: VideoDetails = res.body
+      const video: VideoDetails = res.body
 
-    for (const file of video.files) {
-      checkMagnetWebseeds(file, webseeds)
+      for (const file of video.files) {
+        checkMagnetWebseeds(file, webseeds)
+      }
     }
   }
 
@@ -133,6 +153,20 @@ async function check2Webseeds () {
   for (const resolution of [ 240, 360, 480, 720 ]) {
     expect(files.find(f => f === `${video1Server2UUID}-${resolution}.mp4`)).to.not.be.undefined
   }
+
+  {
+    const res = await getStats(servers[0].url)
+    const data: ServerStats = res.body
+
+    expect(data.videosRedundancy).to.have.lengthOf(1)
+    const stat = data.videosRedundancy[0]
+
+    expect(stat.strategy).to.equal(strategy)
+    expect(stat.totalSize).to.equal(102400)
+    expect(stat.totalUsed).to.be.at.least(1).and.below(102401)
+    expect(stat.totalVideoFiles).to.equal(4)
+    expect(stat.totalVideos).to.equal(1)
+  }
 }
 
 async function cleanServers () {
@@ -142,15 +176,16 @@ async function cleanServers () {
 describe('Test videos redundancy', function () {
 
   describe('With most-views strategy', function () {
+    const strategy = 'most-views'
 
     before(function () {
       this.timeout(120000)
 
-      return runServers('most-views')
+      return runServers(strategy)
     })
 
     it('Should have 1 webseed on the first video', function () {
-      return check1WebSeed()
+      return check1WebSeed(strategy)
     })
 
     it('Should enable redundancy on server 1', function () {
@@ -160,7 +195,7 @@ describe('Test videos redundancy', function () {
     it('Should have 2 webseed on the first video', function () {
       this.timeout(40000)
 
-      return check2Webseeds()
+      return check2Webseeds(strategy)
     })
 
     after(function () {
@@ -169,15 +204,16 @@ describe('Test videos redundancy', function () {
   })
 
   describe('With trending strategy', function () {
+    const strategy = 'trending'
 
     before(function () {
       this.timeout(120000)
 
-      return runServers('trending')
+      return runServers(strategy)
     })
 
     it('Should have 1 webseed on the first video', function () {
-      return check1WebSeed()
+      return check1WebSeed(strategy)
     })
 
     it('Should enable redundancy on server 1', function () {
@@ -187,7 +223,7 @@ describe('Test videos redundancy', function () {
     it('Should have 2 webseed on the first video', function () {
       this.timeout(40000)
 
-      return check2Webseeds()
+      return check2Webseeds(strategy)
     })
 
     after(function () {
@@ -196,15 +232,16 @@ describe('Test videos redundancy', function () {
   })
 
   describe('With recently added strategy', function () {
+    const strategy = 'recently-added'
 
     before(function () {
       this.timeout(120000)
 
-      return runServers('recently-added', { minViews: 3 })
+      return runServers(strategy, { minViews: 3 })
     })
 
     it('Should have 1 webseed on the first video', function () {
-      return check1WebSeed()
+      return check1WebSeed(strategy)
     })
 
     it('Should enable redundancy on server 1', function () {
@@ -218,7 +255,7 @@ describe('Test videos redundancy', function () {
       await wait(15000)
       await waitJobs(servers)
 
-      return check1WebSeed()
+      return check1WebSeed(strategy)
     })
 
     it('Should view 2 times the first video', async function () {
@@ -234,7 +271,7 @@ describe('Test videos redundancy', function () {
     it('Should have 2 webseed on the first video', function () {
       this.timeout(40000)
 
-      return check2Webseeds()
+      return check2Webseeds(strategy)
     })
 
     after(function () {
diff --git a/server/tests/api/server/stats.ts b/server/tests/api/server/stats.ts
index fc9b88805..d8a3268bb 100644
--- a/server/tests/api/server/stats.ts
+++ b/server/tests/api/server/stats.ts
@@ -21,7 +21,7 @@ import { waitJobs } from '../../utils/server/jobs'
 
 const expect = chai.expect
 
-describe('Test stats', function () {
+describe('Test stats (excluding redundancy)', function () {
   let servers: ServerInfo[] = []
 
   before(async function () {
@@ -65,6 +65,7 @@ describe('Test stats', function () {
     expect(data.totalVideos).to.equal(1)
     expect(data.totalInstanceFollowers).to.equal(2)
     expect(data.totalInstanceFollowing).to.equal(1)
+    expect(data.videosRedundancy).to.have.lengthOf(0)
   })
 
   it('Should have the correct stats on instance 2', async function () {
@@ -79,6 +80,7 @@ describe('Test stats', function () {
     expect(data.totalVideos).to.equal(1)
     expect(data.totalInstanceFollowers).to.equal(1)
     expect(data.totalInstanceFollowing).to.equal(1)
+    expect(data.videosRedundancy).to.have.lengthOf(0)
   })
 
   it('Should have the correct stats on instance 3', async function () {
@@ -93,6 +95,7 @@ describe('Test stats', function () {
     expect(data.totalVideos).to.equal(1)
     expect(data.totalInstanceFollowing).to.equal(1)
     expect(data.totalInstanceFollowers).to.equal(0)
+    expect(data.videosRedundancy).to.have.lengthOf(0)
   })
 
   after(async function () {
diff --git a/server/tests/utils/server/stats.ts b/server/tests/utils/server/stats.ts
index 9cdec6cff..01989d952 100644
--- a/server/tests/utils/server/stats.ts
+++ b/server/tests/utils/server/stats.ts
@@ -1,11 +1,16 @@
 import { makeGetRequest } from '../'
 
-function getStats (url: string) {
+function getStats (url: string, useCache = false) {
   const path = '/api/v1/server/stats'
 
+  const query = {
+    t: useCache ? undefined : new Date().getTime()
+  }
+
   return makeGetRequest({
     url,
     path,
+    query,
     statusCodeExpected: 200
   })
 }