aboutsummaryrefslogtreecommitdiffhomepage
diff options
context:
space:
mode:
authorChocobozzz <me@florianbigard.com>2018-09-14 14:57:59 +0200
committerChocobozzz <me@florianbigard.com>2018-09-14 14:57:59 +0200
commit4b5384f6e7be62d072d21d8d964951ba572ab10e (patch)
tree64dbd0c55096435ef804988c4750f717ad140633
parentcfc16a6db88378f83fa3a501170fa0fc5e7d6636 (diff)
downloadPeerTube-4b5384f6e7be62d072d21d8d964951ba572ab10e.tar.gz
PeerTube-4b5384f6e7be62d072d21d8d964951ba572ab10e.tar.zst
PeerTube-4b5384f6e7be62d072d21d8d964951ba572ab10e.zip
Add redundancy stats
-rw-r--r--server/controllers/api/overviews.ts20
-rw-r--r--server/controllers/api/server/stats.ts14
-rw-r--r--server/initializers/constants.ts3
-rw-r--r--server/models/redundancy/video-redundancy.ts31
-rw-r--r--server/tests/api/server/redundancy.ts75
-rw-r--r--server/tests/api/server/stats.ts5
-rw-r--r--server/tests/utils/server/stats.ts7
-rw-r--r--shared/models/server/server-stats.model.ts10
8 files changed, 132 insertions, 33 deletions
diff --git a/server/controllers/api/overviews.ts b/server/controllers/api/overviews.ts
index cc3cc54a7..8b6773056 100644
--- a/server/controllers/api/overviews.ts
+++ b/server/controllers/api/overviews.ts
@@ -21,6 +21,16 @@ export { overviewsRouter }
21 21
22// --------------------------------------------------------------------------- 22// ---------------------------------------------------------------------------
23 23
24const buildSamples = memoizee(async function () {
25 const [ categories, channels, tags ] = await Promise.all([
26 VideoModel.getRandomFieldSamples('category', OVERVIEWS.VIDEOS.SAMPLE_THRESHOLD, OVERVIEWS.VIDEOS.SAMPLES_COUNT),
27 VideoModel.getRandomFieldSamples('channelId', OVERVIEWS.VIDEOS.SAMPLE_THRESHOLD ,OVERVIEWS.VIDEOS.SAMPLES_COUNT),
28 TagModel.getRandomSamples(OVERVIEWS.VIDEOS.SAMPLE_THRESHOLD, OVERVIEWS.VIDEOS.SAMPLES_COUNT)
29 ])
30
31 return { categories, channels, tags }
32}, { maxAge: MEMOIZE_TTL.OVERVIEWS_SAMPLE })
33
24// This endpoint could be quite long, but we cache it 34// This endpoint could be quite long, but we cache it
25async function getVideosOverview (req: express.Request, res: express.Response) { 35async function getVideosOverview (req: express.Request, res: express.Response) {
26 const attributes = await buildSamples() 36 const attributes = await buildSamples()
@@ -45,16 +55,6 @@ async function getVideosOverview (req: express.Request, res: express.Response) {
45 return res.json(result) 55 return res.json(result)
46} 56}
47 57
48const buildSamples = memoizee(async function () {
49 const [ categories, channels, tags ] = await Promise.all([
50 VideoModel.getRandomFieldSamples('category', OVERVIEWS.VIDEOS.SAMPLE_THRESHOLD, OVERVIEWS.VIDEOS.SAMPLES_COUNT),
51 VideoModel.getRandomFieldSamples('channelId', OVERVIEWS.VIDEOS.SAMPLE_THRESHOLD ,OVERVIEWS.VIDEOS.SAMPLES_COUNT),
52 TagModel.getRandomSamples(OVERVIEWS.VIDEOS.SAMPLE_THRESHOLD, OVERVIEWS.VIDEOS.SAMPLES_COUNT)
53 ])
54
55 return { categories, channels, tags }
56}, { maxAge: MEMOIZE_TTL.OVERVIEWS_SAMPLE })
57
58async function getVideosByTag (tag: string, res: express.Response) { 58async function getVideosByTag (tag: string, res: express.Response) {
59 const videos = await getVideos(res, { tagsOneOf: [ tag ] }) 59 const videos = await getVideos(res, { tagsOneOf: [ tag ] })
60 60
diff --git a/server/controllers/api/server/stats.ts b/server/controllers/api/server/stats.ts
index 6f4fe938c..bb6311e81 100644
--- a/server/controllers/api/server/stats.ts
+++ b/server/controllers/api/server/stats.ts
@@ -5,10 +5,14 @@ import { UserModel } from '../../../models/account/user'
5import { ActorFollowModel } from '../../../models/activitypub/actor-follow' 5import { ActorFollowModel } from '../../../models/activitypub/actor-follow'
6import { VideoModel } from '../../../models/video/video' 6import { VideoModel } from '../../../models/video/video'
7import { VideoCommentModel } from '../../../models/video/video-comment' 7import { VideoCommentModel } from '../../../models/video/video-comment'
8import { VideoRedundancyModel } from '../../../models/redundancy/video-redundancy'
9import { CONFIG, ROUTE_CACHE_LIFETIME } from '../../../initializers/constants'
10import { cacheRoute } from '../../../middlewares/cache'
8 11
9const statsRouter = express.Router() 12const statsRouter = express.Router()
10 13
11statsRouter.get('/stats', 14statsRouter.get('/stats',
15 asyncMiddleware(cacheRoute(ROUTE_CACHE_LIFETIME.STATS)),
12 asyncMiddleware(getStats) 16 asyncMiddleware(getStats)
13) 17)
14 18
@@ -18,6 +22,13 @@ async function getStats (req: express.Request, res: express.Response, next: expr
18 const { totalUsers } = await UserModel.getStats() 22 const { totalUsers } = await UserModel.getStats()
19 const { totalInstanceFollowers, totalInstanceFollowing } = await ActorFollowModel.getStats() 23 const { totalInstanceFollowers, totalInstanceFollowing } = await ActorFollowModel.getStats()
20 24
25 const videosRedundancyStats = await Promise.all(
26 CONFIG.REDUNDANCY.VIDEOS.map(r => {
27 return VideoRedundancyModel.getStats(r.strategy)
28 .then(stats => Object.assign(stats, { strategy: r.strategy, totalSize: r.size }))
29 })
30 )
31
21 const data: ServerStats = { 32 const data: ServerStats = {
22 totalLocalVideos, 33 totalLocalVideos,
23 totalLocalVideoViews, 34 totalLocalVideoViews,
@@ -26,7 +37,8 @@ async function getStats (req: express.Request, res: express.Response, next: expr
26 totalVideoComments, 37 totalVideoComments,
27 totalUsers, 38 totalUsers,
28 totalInstanceFollowers, 39 totalInstanceFollowers,
29 totalInstanceFollowing 40 totalInstanceFollowing,
41 videosRedundancy: videosRedundancyStats
30 } 42 }
31 43
32 return res.json(data).end() 44 return res.json(data).end()
diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts
index 9cccb0919..e8dab21db 100644
--- a/server/initializers/constants.ts
+++ b/server/initializers/constants.ts
@@ -66,7 +66,8 @@ const ROUTE_CACHE_LIFETIME = {
66 }, 66 },
67 ACTIVITY_PUB: { 67 ACTIVITY_PUB: {
68 VIDEOS: '1 second' // 1 second, cache concurrent requests after a broadcast for example 68 VIDEOS: '1 second' // 1 second, cache concurrent requests after a broadcast for example
69 } 69 },
70 STATS: '4 hours'
70} 71}
71 72
72// --------------------------------------------------------------------------- 73// ---------------------------------------------------------------------------
diff --git a/server/models/redundancy/video-redundancy.ts b/server/models/redundancy/video-redundancy.ts
index b7454c617..6ae02efb9 100644
--- a/server/models/redundancy/video-redundancy.ts
+++ b/server/models/redundancy/video-redundancy.ts
@@ -245,6 +245,37 @@ export class VideoRedundancyModel extends Model<VideoRedundancyModel> {
245 .findAll(query) 245 .findAll(query)
246 } 246 }
247 247
248 static async getStats (strategy: VideoRedundancyStrategy) {
249 const actor = await getServerActor()
250
251 const query = {
252 raw: true,
253 attributes: [
254 [ Sequelize.fn('COALESCE', Sequelize.fn('SUM', Sequelize.col('VideoFile.size')), '0'), 'totalUsed' ],
255 [ Sequelize.fn('COUNT', Sequelize.fn('DISTINCT', 'videoId')), 'totalVideos' ],
256 [ Sequelize.fn('COUNT', 'videoFileId'), 'totalVideoFiles' ]
257 ],
258 where: {
259 strategy,
260 actorId: actor.id
261 },
262 include: [
263 {
264 attributes: [],
265 model: VideoFileModel,
266 required: true
267 }
268 ]
269 }
270
271 return VideoRedundancyModel.find(query as any) // FIXME: typings
272 .then((r: any) => ({
273 totalUsed: parseInt(r.totalUsed.toString(), 10),
274 totalVideos: r.totalVideos,
275 totalVideoFiles: r.totalVideoFiles
276 }))
277 }
278
248 toActivityPubObject (): CacheFileObject { 279 toActivityPubObject (): CacheFileObject {
249 return { 280 return {
250 id: this.url, 281 id: this.url,
diff --git a/server/tests/api/server/redundancy.ts b/server/tests/api/server/redundancy.ts
index 6574e8ea9..c0ab251e6 100644
--- a/server/tests/api/server/redundancy.ts
+++ b/server/tests/api/server/redundancy.ts
@@ -23,6 +23,8 @@ import { ActorFollow } from '../../../../shared/models/actors'
23import { readdir } from 'fs-extra' 23import { readdir } from 'fs-extra'
24import { join } from 'path' 24import { join } from 'path'
25import { VideoRedundancyStrategy } from '../../../../shared/models/redundancy' 25import { VideoRedundancyStrategy } from '../../../../shared/models/redundancy'
26import { getStats } from '../../utils/server/stats'
27import { ServerStats } from '../../../../shared/models/server/server-stats.model'
26 28
27const expect = chai.expect 29const expect = chai.expect
28 30
@@ -79,16 +81,32 @@ async function runServers (strategy: VideoRedundancyStrategy, additionalParams:
79 await waitJobs(servers) 81 await waitJobs(servers)
80} 82}
81 83
82async function check1WebSeed () { 84async function check1WebSeed (strategy: VideoRedundancyStrategy) {
83 const webseeds = [ 85 const webseeds = [
84 'http://localhost:9002/static/webseed/' + video1Server2UUID 86 'http://localhost:9002/static/webseed/' + video1Server2UUID
85 ] 87 ]
86 88
87 for (const server of servers) { 89 for (const server of servers) {
88 const res = await getVideo(server.url, video1Server2UUID) 90 {
91 const res = await getVideo(server.url, video1Server2UUID)
89 92
90 const video: VideoDetails = res.body 93 const video: VideoDetails = res.body
91 video.files.forEach(f => checkMagnetWebseeds(f, webseeds)) 94 video.files.forEach(f => checkMagnetWebseeds(f, webseeds))
95 }
96
97 {
98 const res = await getStats(server.url)
99 const data: ServerStats = res.body
100
101 expect(data.videosRedundancy).to.have.lengthOf(1)
102
103 const stat = data.videosRedundancy[0]
104 expect(stat.strategy).to.equal(strategy)
105 expect(stat.totalSize).to.equal(102400)
106 expect(stat.totalUsed).to.equal(0)
107 expect(stat.totalVideoFiles).to.equal(0)
108 expect(stat.totalVideos).to.equal(0)
109 }
92 } 110 }
93} 111}
94 112
@@ -107,7 +125,7 @@ async function enableRedundancy () {
107 expect(server2.following.hostRedundancyAllowed).to.be.true 125 expect(server2.following.hostRedundancyAllowed).to.be.true
108} 126}
109 127
110async function check2Webseeds () { 128async function check2Webseeds (strategy: VideoRedundancyStrategy) {
111 await waitJobs(servers) 129 await waitJobs(servers)
112 await wait(15000) 130 await wait(15000)
113 await waitJobs(servers) 131 await waitJobs(servers)
@@ -118,12 +136,14 @@ async function check2Webseeds () {
118 ] 136 ]
119 137
120 for (const server of servers) { 138 for (const server of servers) {
121 const res = await getVideo(server.url, video1Server2UUID) 139 {
140 const res = await getVideo(server.url, video1Server2UUID)
122 141
123 const video: VideoDetails = res.body 142 const video: VideoDetails = res.body
124 143
125 for (const file of video.files) { 144 for (const file of video.files) {
126 checkMagnetWebseeds(file, webseeds) 145 checkMagnetWebseeds(file, webseeds)
146 }
127 } 147 }
128 } 148 }
129 149
@@ -133,6 +153,20 @@ async function check2Webseeds () {
133 for (const resolution of [ 240, 360, 480, 720 ]) { 153 for (const resolution of [ 240, 360, 480, 720 ]) {
134 expect(files.find(f => f === `${video1Server2UUID}-${resolution}.mp4`)).to.not.be.undefined 154 expect(files.find(f => f === `${video1Server2UUID}-${resolution}.mp4`)).to.not.be.undefined
135 } 155 }
156
157 {
158 const res = await getStats(servers[0].url)
159 const data: ServerStats = res.body
160
161 expect(data.videosRedundancy).to.have.lengthOf(1)
162 const stat = data.videosRedundancy[0]
163
164 expect(stat.strategy).to.equal(strategy)
165 expect(stat.totalSize).to.equal(102400)
166 expect(stat.totalUsed).to.be.at.least(1).and.below(102401)
167 expect(stat.totalVideoFiles).to.equal(4)
168 expect(stat.totalVideos).to.equal(1)
169 }
136} 170}
137 171
138async function cleanServers () { 172async function cleanServers () {
@@ -142,15 +176,16 @@ async function cleanServers () {
142describe('Test videos redundancy', function () { 176describe('Test videos redundancy', function () {
143 177
144 describe('With most-views strategy', function () { 178 describe('With most-views strategy', function () {
179 const strategy = 'most-views'
145 180
146 before(function () { 181 before(function () {
147 this.timeout(120000) 182 this.timeout(120000)
148 183
149 return runServers('most-views') 184 return runServers(strategy)
150 }) 185 })
151 186
152 it('Should have 1 webseed on the first video', function () { 187 it('Should have 1 webseed on the first video', function () {
153 return check1WebSeed() 188 return check1WebSeed(strategy)
154 }) 189 })
155 190
156 it('Should enable redundancy on server 1', function () { 191 it('Should enable redundancy on server 1', function () {
@@ -160,7 +195,7 @@ describe('Test videos redundancy', function () {
160 it('Should have 2 webseed on the first video', function () { 195 it('Should have 2 webseed on the first video', function () {
161 this.timeout(40000) 196 this.timeout(40000)
162 197
163 return check2Webseeds() 198 return check2Webseeds(strategy)
164 }) 199 })
165 200
166 after(function () { 201 after(function () {
@@ -169,15 +204,16 @@ describe('Test videos redundancy', function () {
169 }) 204 })
170 205
171 describe('With trending strategy', function () { 206 describe('With trending strategy', function () {
207 const strategy = 'trending'
172 208
173 before(function () { 209 before(function () {
174 this.timeout(120000) 210 this.timeout(120000)
175 211
176 return runServers('trending') 212 return runServers(strategy)
177 }) 213 })
178 214
179 it('Should have 1 webseed on the first video', function () { 215 it('Should have 1 webseed on the first video', function () {
180 return check1WebSeed() 216 return check1WebSeed(strategy)
181 }) 217 })
182 218
183 it('Should enable redundancy on server 1', function () { 219 it('Should enable redundancy on server 1', function () {
@@ -187,7 +223,7 @@ describe('Test videos redundancy', function () {
187 it('Should have 2 webseed on the first video', function () { 223 it('Should have 2 webseed on the first video', function () {
188 this.timeout(40000) 224 this.timeout(40000)
189 225
190 return check2Webseeds() 226 return check2Webseeds(strategy)
191 }) 227 })
192 228
193 after(function () { 229 after(function () {
@@ -196,15 +232,16 @@ describe('Test videos redundancy', function () {
196 }) 232 })
197 233
198 describe('With recently added strategy', function () { 234 describe('With recently added strategy', function () {
235 const strategy = 'recently-added'
199 236
200 before(function () { 237 before(function () {
201 this.timeout(120000) 238 this.timeout(120000)
202 239
203 return runServers('recently-added', { minViews: 3 }) 240 return runServers(strategy, { minViews: 3 })
204 }) 241 })
205 242
206 it('Should have 1 webseed on the first video', function () { 243 it('Should have 1 webseed on the first video', function () {
207 return check1WebSeed() 244 return check1WebSeed(strategy)
208 }) 245 })
209 246
210 it('Should enable redundancy on server 1', function () { 247 it('Should enable redundancy on server 1', function () {
@@ -218,7 +255,7 @@ describe('Test videos redundancy', function () {
218 await wait(15000) 255 await wait(15000)
219 await waitJobs(servers) 256 await waitJobs(servers)
220 257
221 return check1WebSeed() 258 return check1WebSeed(strategy)
222 }) 259 })
223 260
224 it('Should view 2 times the first video', async function () { 261 it('Should view 2 times the first video', async function () {
@@ -234,7 +271,7 @@ describe('Test videos redundancy', function () {
234 it('Should have 2 webseed on the first video', function () { 271 it('Should have 2 webseed on the first video', function () {
235 this.timeout(40000) 272 this.timeout(40000)
236 273
237 return check2Webseeds() 274 return check2Webseeds(strategy)
238 }) 275 })
239 276
240 after(function () { 277 after(function () {
diff --git a/server/tests/api/server/stats.ts b/server/tests/api/server/stats.ts
index fc9b88805..d8a3268bb 100644
--- a/server/tests/api/server/stats.ts
+++ b/server/tests/api/server/stats.ts
@@ -21,7 +21,7 @@ import { waitJobs } from '../../utils/server/jobs'
21 21
22const expect = chai.expect 22const expect = chai.expect
23 23
24describe('Test stats', function () { 24describe('Test stats (excluding redundancy)', function () {
25 let servers: ServerInfo[] = [] 25 let servers: ServerInfo[] = []
26 26
27 before(async function () { 27 before(async function () {
@@ -65,6 +65,7 @@ describe('Test stats', function () {
65 expect(data.totalVideos).to.equal(1) 65 expect(data.totalVideos).to.equal(1)
66 expect(data.totalInstanceFollowers).to.equal(2) 66 expect(data.totalInstanceFollowers).to.equal(2)
67 expect(data.totalInstanceFollowing).to.equal(1) 67 expect(data.totalInstanceFollowing).to.equal(1)
68 expect(data.videosRedundancy).to.have.lengthOf(0)
68 }) 69 })
69 70
70 it('Should have the correct stats on instance 2', async function () { 71 it('Should have the correct stats on instance 2', async function () {
@@ -79,6 +80,7 @@ describe('Test stats', function () {
79 expect(data.totalVideos).to.equal(1) 80 expect(data.totalVideos).to.equal(1)
80 expect(data.totalInstanceFollowers).to.equal(1) 81 expect(data.totalInstanceFollowers).to.equal(1)
81 expect(data.totalInstanceFollowing).to.equal(1) 82 expect(data.totalInstanceFollowing).to.equal(1)
83 expect(data.videosRedundancy).to.have.lengthOf(0)
82 }) 84 })
83 85
84 it('Should have the correct stats on instance 3', async function () { 86 it('Should have the correct stats on instance 3', async function () {
@@ -93,6 +95,7 @@ describe('Test stats', function () {
93 expect(data.totalVideos).to.equal(1) 95 expect(data.totalVideos).to.equal(1)
94 expect(data.totalInstanceFollowing).to.equal(1) 96 expect(data.totalInstanceFollowing).to.equal(1)
95 expect(data.totalInstanceFollowers).to.equal(0) 97 expect(data.totalInstanceFollowers).to.equal(0)
98 expect(data.videosRedundancy).to.have.lengthOf(0)
96 }) 99 })
97 100
98 after(async function () { 101 after(async function () {
diff --git a/server/tests/utils/server/stats.ts b/server/tests/utils/server/stats.ts
index 9cdec6cff..01989d952 100644
--- a/server/tests/utils/server/stats.ts
+++ b/server/tests/utils/server/stats.ts
@@ -1,11 +1,16 @@
1import { makeGetRequest } from '../' 1import { makeGetRequest } from '../'
2 2
3function getStats (url: string) { 3function getStats (url: string, useCache = false) {
4 const path = '/api/v1/server/stats' 4 const path = '/api/v1/server/stats'
5 5
6 const query = {
7 t: useCache ? undefined : new Date().getTime()
8 }
9
6 return makeGetRequest({ 10 return makeGetRequest({
7 url, 11 url,
8 path, 12 path,
13 query,
9 statusCodeExpected: 200 14 statusCodeExpected: 200
10 }) 15 })
11} 16}
diff --git a/shared/models/server/server-stats.model.ts b/shared/models/server/server-stats.model.ts
index 5c1bf3468..a6bd2d4d3 100644
--- a/shared/models/server/server-stats.model.ts
+++ b/shared/models/server/server-stats.model.ts
@@ -1,3 +1,5 @@
1import { VideoRedundancyStrategy } from '../redundancy'
2
1export interface ServerStats { 3export interface ServerStats {
2 totalUsers: number 4 totalUsers: number
3 totalLocalVideos: number 5 totalLocalVideos: number
@@ -9,4 +11,12 @@ export interface ServerStats {
9 11
10 totalInstanceFollowers: number 12 totalInstanceFollowers: number
11 totalInstanceFollowing: number 13 totalInstanceFollowing: number
14
15 videosRedundancy: {
16 strategy: VideoRedundancyStrategy
17 totalSize: number
18 totalUsed: number
19 totalVideoFiles: number
20 totalVideos: number
21 }[]
12} 22}