From 4b5384f6e7be62d072d21d8d964951ba572ab10e Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Fri, 14 Sep 2018 14:57:59 +0200 Subject: Add redundancy stats --- server/controllers/api/overviews.ts | 20 ++++---- server/controllers/api/server/stats.ts | 14 +++++- server/initializers/constants.ts | 3 +- server/models/redundancy/video-redundancy.ts | 31 ++++++++++++ server/tests/api/server/redundancy.ts | 75 +++++++++++++++++++++------- server/tests/api/server/stats.ts | 5 +- server/tests/utils/server/stats.ts | 7 ++- 7 files changed, 122 insertions(+), 33 deletions(-) (limited to 'server') diff --git a/server/controllers/api/overviews.ts b/server/controllers/api/overviews.ts index cc3cc54a7..8b6773056 100644 --- a/server/controllers/api/overviews.ts +++ b/server/controllers/api/overviews.ts @@ -21,6 +21,16 @@ export { overviewsRouter } // --------------------------------------------------------------------------- +const buildSamples = memoizee(async function () { + const [ categories, channels, tags ] = await Promise.all([ + VideoModel.getRandomFieldSamples('category', OVERVIEWS.VIDEOS.SAMPLE_THRESHOLD, OVERVIEWS.VIDEOS.SAMPLES_COUNT), + VideoModel.getRandomFieldSamples('channelId', OVERVIEWS.VIDEOS.SAMPLE_THRESHOLD ,OVERVIEWS.VIDEOS.SAMPLES_COUNT), + TagModel.getRandomSamples(OVERVIEWS.VIDEOS.SAMPLE_THRESHOLD, OVERVIEWS.VIDEOS.SAMPLES_COUNT) + ]) + + return { categories, channels, tags } +}, { maxAge: MEMOIZE_TTL.OVERVIEWS_SAMPLE }) + // This endpoint could be quite long, but we cache it async function getVideosOverview (req: express.Request, res: express.Response) { const attributes = await buildSamples() @@ -45,16 +55,6 @@ async function getVideosOverview (req: express.Request, res: express.Response) { return res.json(result) } -const buildSamples = memoizee(async function () { - const [ categories, channels, tags ] = await Promise.all([ - VideoModel.getRandomFieldSamples('category', OVERVIEWS.VIDEOS.SAMPLE_THRESHOLD, OVERVIEWS.VIDEOS.SAMPLES_COUNT), - VideoModel.getRandomFieldSamples('channelId', OVERVIEWS.VIDEOS.SAMPLE_THRESHOLD ,OVERVIEWS.VIDEOS.SAMPLES_COUNT), - TagModel.getRandomSamples(OVERVIEWS.VIDEOS.SAMPLE_THRESHOLD, OVERVIEWS.VIDEOS.SAMPLES_COUNT) - ]) - - return { categories, channels, tags } -}, { maxAge: MEMOIZE_TTL.OVERVIEWS_SAMPLE }) - async function getVideosByTag (tag: string, res: express.Response) { const videos = await getVideos(res, { tagsOneOf: [ tag ] }) diff --git a/server/controllers/api/server/stats.ts b/server/controllers/api/server/stats.ts index 6f4fe938c..bb6311e81 100644 --- a/server/controllers/api/server/stats.ts +++ b/server/controllers/api/server/stats.ts @@ -5,10 +5,14 @@ import { UserModel } from '../../../models/account/user' import { ActorFollowModel } from '../../../models/activitypub/actor-follow' import { VideoModel } from '../../../models/video/video' import { VideoCommentModel } from '../../../models/video/video-comment' +import { VideoRedundancyModel } from '../../../models/redundancy/video-redundancy' +import { CONFIG, ROUTE_CACHE_LIFETIME } from '../../../initializers/constants' +import { cacheRoute } from '../../../middlewares/cache' const statsRouter = express.Router() statsRouter.get('/stats', + asyncMiddleware(cacheRoute(ROUTE_CACHE_LIFETIME.STATS)), asyncMiddleware(getStats) ) @@ -18,6 +22,13 @@ async function getStats (req: express.Request, res: express.Response, next: expr const { totalUsers } = await UserModel.getStats() const { totalInstanceFollowers, totalInstanceFollowing } = await ActorFollowModel.getStats() + const videosRedundancyStats = await Promise.all( + CONFIG.REDUNDANCY.VIDEOS.map(r => { + return VideoRedundancyModel.getStats(r.strategy) + .then(stats => Object.assign(stats, { strategy: r.strategy, totalSize: r.size })) + }) + ) + const data: ServerStats = { totalLocalVideos, totalLocalVideoViews, @@ -26,7 +37,8 @@ async function getStats (req: express.Request, res: express.Response, next: expr totalVideoComments, totalUsers, totalInstanceFollowers, - totalInstanceFollowing + totalInstanceFollowing, + videosRedundancy: videosRedundancyStats } return res.json(data).end() diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index 9cccb0919..e8dab21db 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts @@ -66,7 +66,8 @@ const ROUTE_CACHE_LIFETIME = { }, ACTIVITY_PUB: { VIDEOS: '1 second' // 1 second, cache concurrent requests after a broadcast for example - } + }, + STATS: '4 hours' } // --------------------------------------------------------------------------- diff --git a/server/models/redundancy/video-redundancy.ts b/server/models/redundancy/video-redundancy.ts index b7454c617..6ae02efb9 100644 --- a/server/models/redundancy/video-redundancy.ts +++ b/server/models/redundancy/video-redundancy.ts @@ -245,6 +245,37 @@ export class VideoRedundancyModel extends Model { .findAll(query) } + static async getStats (strategy: VideoRedundancyStrategy) { + const actor = await getServerActor() + + const query = { + raw: true, + attributes: [ + [ Sequelize.fn('COALESCE', Sequelize.fn('SUM', Sequelize.col('VideoFile.size')), '0'), 'totalUsed' ], + [ Sequelize.fn('COUNT', Sequelize.fn('DISTINCT', 'videoId')), 'totalVideos' ], + [ Sequelize.fn('COUNT', 'videoFileId'), 'totalVideoFiles' ] + ], + where: { + strategy, + actorId: actor.id + }, + include: [ + { + attributes: [], + model: VideoFileModel, + required: true + } + ] + } + + return VideoRedundancyModel.find(query as any) // FIXME: typings + .then((r: any) => ({ + totalUsed: parseInt(r.totalUsed.toString(), 10), + totalVideos: r.totalVideos, + totalVideoFiles: r.totalVideoFiles + })) + } + toActivityPubObject (): CacheFileObject { return { id: this.url, diff --git a/server/tests/api/server/redundancy.ts b/server/tests/api/server/redundancy.ts index 6574e8ea9..c0ab251e6 100644 --- a/server/tests/api/server/redundancy.ts +++ b/server/tests/api/server/redundancy.ts @@ -23,6 +23,8 @@ import { ActorFollow } from '../../../../shared/models/actors' import { readdir } from 'fs-extra' import { join } from 'path' import { VideoRedundancyStrategy } from '../../../../shared/models/redundancy' +import { getStats } from '../../utils/server/stats' +import { ServerStats } from '../../../../shared/models/server/server-stats.model' const expect = chai.expect @@ -79,16 +81,32 @@ async function runServers (strategy: VideoRedundancyStrategy, additionalParams: await waitJobs(servers) } -async function check1WebSeed () { +async function check1WebSeed (strategy: VideoRedundancyStrategy) { const webseeds = [ 'http://localhost:9002/static/webseed/' + video1Server2UUID ] for (const server of servers) { - const res = await getVideo(server.url, video1Server2UUID) + { + const res = await getVideo(server.url, video1Server2UUID) - const video: VideoDetails = res.body - video.files.forEach(f => checkMagnetWebseeds(f, webseeds)) + const video: VideoDetails = res.body + video.files.forEach(f => checkMagnetWebseeds(f, webseeds)) + } + + { + const res = await getStats(server.url) + const data: ServerStats = res.body + + expect(data.videosRedundancy).to.have.lengthOf(1) + + const stat = data.videosRedundancy[0] + expect(stat.strategy).to.equal(strategy) + expect(stat.totalSize).to.equal(102400) + expect(stat.totalUsed).to.equal(0) + expect(stat.totalVideoFiles).to.equal(0) + expect(stat.totalVideos).to.equal(0) + } } } @@ -107,7 +125,7 @@ async function enableRedundancy () { expect(server2.following.hostRedundancyAllowed).to.be.true } -async function check2Webseeds () { +async function check2Webseeds (strategy: VideoRedundancyStrategy) { await waitJobs(servers) await wait(15000) await waitJobs(servers) @@ -118,12 +136,14 @@ async function check2Webseeds () { ] for (const server of servers) { - const res = await getVideo(server.url, video1Server2UUID) + { + const res = await getVideo(server.url, video1Server2UUID) - const video: VideoDetails = res.body + const video: VideoDetails = res.body - for (const file of video.files) { - checkMagnetWebseeds(file, webseeds) + for (const file of video.files) { + checkMagnetWebseeds(file, webseeds) + } } } @@ -133,6 +153,20 @@ async function check2Webseeds () { for (const resolution of [ 240, 360, 480, 720 ]) { expect(files.find(f => f === `${video1Server2UUID}-${resolution}.mp4`)).to.not.be.undefined } + + { + const res = await getStats(servers[0].url) + const data: ServerStats = res.body + + expect(data.videosRedundancy).to.have.lengthOf(1) + const stat = data.videosRedundancy[0] + + expect(stat.strategy).to.equal(strategy) + expect(stat.totalSize).to.equal(102400) + expect(stat.totalUsed).to.be.at.least(1).and.below(102401) + expect(stat.totalVideoFiles).to.equal(4) + expect(stat.totalVideos).to.equal(1) + } } async function cleanServers () { @@ -142,15 +176,16 @@ async function cleanServers () { describe('Test videos redundancy', function () { describe('With most-views strategy', function () { + const strategy = 'most-views' before(function () { this.timeout(120000) - return runServers('most-views') + return runServers(strategy) }) it('Should have 1 webseed on the first video', function () { - return check1WebSeed() + return check1WebSeed(strategy) }) it('Should enable redundancy on server 1', function () { @@ -160,7 +195,7 @@ describe('Test videos redundancy', function () { it('Should have 2 webseed on the first video', function () { this.timeout(40000) - return check2Webseeds() + return check2Webseeds(strategy) }) after(function () { @@ -169,15 +204,16 @@ describe('Test videos redundancy', function () { }) describe('With trending strategy', function () { + const strategy = 'trending' before(function () { this.timeout(120000) - return runServers('trending') + return runServers(strategy) }) it('Should have 1 webseed on the first video', function () { - return check1WebSeed() + return check1WebSeed(strategy) }) it('Should enable redundancy on server 1', function () { @@ -187,7 +223,7 @@ describe('Test videos redundancy', function () { it('Should have 2 webseed on the first video', function () { this.timeout(40000) - return check2Webseeds() + return check2Webseeds(strategy) }) after(function () { @@ -196,15 +232,16 @@ describe('Test videos redundancy', function () { }) describe('With recently added strategy', function () { + const strategy = 'recently-added' before(function () { this.timeout(120000) - return runServers('recently-added', { minViews: 3 }) + return runServers(strategy, { minViews: 3 }) }) it('Should have 1 webseed on the first video', function () { - return check1WebSeed() + return check1WebSeed(strategy) }) it('Should enable redundancy on server 1', function () { @@ -218,7 +255,7 @@ describe('Test videos redundancy', function () { await wait(15000) await waitJobs(servers) - return check1WebSeed() + return check1WebSeed(strategy) }) it('Should view 2 times the first video', async function () { @@ -234,7 +271,7 @@ describe('Test videos redundancy', function () { it('Should have 2 webseed on the first video', function () { this.timeout(40000) - return check2Webseeds() + return check2Webseeds(strategy) }) after(function () { diff --git a/server/tests/api/server/stats.ts b/server/tests/api/server/stats.ts index fc9b88805..d8a3268bb 100644 --- a/server/tests/api/server/stats.ts +++ b/server/tests/api/server/stats.ts @@ -21,7 +21,7 @@ import { waitJobs } from '../../utils/server/jobs' const expect = chai.expect -describe('Test stats', function () { +describe('Test stats (excluding redundancy)', function () { let servers: ServerInfo[] = [] before(async function () { @@ -65,6 +65,7 @@ describe('Test stats', function () { expect(data.totalVideos).to.equal(1) expect(data.totalInstanceFollowers).to.equal(2) expect(data.totalInstanceFollowing).to.equal(1) + expect(data.videosRedundancy).to.have.lengthOf(0) }) it('Should have the correct stats on instance 2', async function () { @@ -79,6 +80,7 @@ describe('Test stats', function () { expect(data.totalVideos).to.equal(1) expect(data.totalInstanceFollowers).to.equal(1) expect(data.totalInstanceFollowing).to.equal(1) + expect(data.videosRedundancy).to.have.lengthOf(0) }) it('Should have the correct stats on instance 3', async function () { @@ -93,6 +95,7 @@ describe('Test stats', function () { expect(data.totalVideos).to.equal(1) expect(data.totalInstanceFollowing).to.equal(1) expect(data.totalInstanceFollowers).to.equal(0) + expect(data.videosRedundancy).to.have.lengthOf(0) }) after(async function () { diff --git a/server/tests/utils/server/stats.ts b/server/tests/utils/server/stats.ts index 9cdec6cff..01989d952 100644 --- a/server/tests/utils/server/stats.ts +++ b/server/tests/utils/server/stats.ts @@ -1,11 +1,16 @@ import { makeGetRequest } from '../' -function getStats (url: string) { +function getStats (url: string, useCache = false) { const path = '/api/v1/server/stats' + const query = { + t: useCache ? undefined : new Date().getTime() + } + return makeGetRequest({ url, path, + query, statusCodeExpected: 200 }) } -- cgit v1.2.3