aboutsummaryrefslogtreecommitdiffhomepage
diff options
context:
space:
mode:
-rw-r--r--config/default.yaml10
-rw-r--r--config/production.yaml.example11
-rwxr-xr-xscripts/parse-log.ts2
-rw-r--r--server.ts2
-rw-r--r--server/controllers/api/server/logs.ts2
-rw-r--r--server/initializers/checker-before-init.ts3
-rw-r--r--server/initializers/config.ts7
-rw-r--r--server/initializers/constants.ts2
-rw-r--r--server/lib/schedulers/remove-old-views-scheduler.ts33
-rw-r--r--server/models/video/video-views.ts14
-rw-r--r--server/tests/api/videos/index.ts1
-rw-r--r--server/tests/api/videos/videos-views-cleaner.ts113
-rw-r--r--shared/core-utils/logs/logs.ts25
-rw-r--r--shared/utils/logs/logs.ts23
-rw-r--r--shared/utils/miscs/sql.ts15
15 files changed, 237 insertions, 26 deletions
diff --git a/config/default.yaml b/config/default.yaml
index d45d84b90..70b10299d 100644
--- a/config/default.yaml
+++ b/config/default.yaml
@@ -118,6 +118,16 @@ history:
118 # Other values could be '6 months' or '30 days' etc (PeerTube will periodically delete old entries from database) 118 # Other values could be '6 months' or '30 days' etc (PeerTube will periodically delete old entries from database)
119 max_age: -1 119 max_age: -1
120 120
121views:
122 videos:
123 # PeerTube creates a database entry every hour for each video to track views over a period of time
124 # This is used in particular by the Trending page
125 # PeerTube could remove old remote video views if you want to reduce your database size (video view counter will not be altered)
126 # -1 means no cleanup
127 # Other values could be '6 months' or '30 days' etc (PeerTube will periodically delete old entries from database)
128 remote:
129 max_age: -1
130
121cache: 131cache:
122 previews: 132 previews:
123 size: 500 # Max number of previews you want to cache 133 size: 500 # Max number of previews you want to cache
diff --git a/config/production.yaml.example b/config/production.yaml.example
index b813a65e9..06baaf7d4 100644
--- a/config/production.yaml.example
+++ b/config/production.yaml.example
@@ -119,6 +119,17 @@ history:
119 # Other values could be '6 months' or '30 days' etc (PeerTube will periodically delete old entries from database) 119 # Other values could be '6 months' or '30 days' etc (PeerTube will periodically delete old entries from database)
120 max_age: -1 120 max_age: -1
121 121
122views:
123 videos:
124 # PeerTube creates a database entry every hour for each video to track views over a period of time
125 # This is used in particular by the Trending page
126 # PeerTube could remove old remote video views if you want to reduce your database size (video view counter will not be altered)
127 # -1 means no cleanup
128 # Other values could be '6 months' or '30 days' etc (PeerTube will periodically delete old entries from database)
129 remote:
130 max_age: -1
131
132
122############################################################################### 133###############################################################################
123# 134#
124# From this point, all the following keys can be overridden by the web interface 135# From this point, all the following keys can be overridden by the web interface
diff --git a/scripts/parse-log.ts b/scripts/parse-log.ts
index fe87db009..83ad45b72 100755
--- a/scripts/parse-log.ts
+++ b/scripts/parse-log.ts
@@ -5,7 +5,7 @@ import { createInterface } from 'readline'
5import * as winston from 'winston' 5import * as winston from 'winston'
6import { labelFormatter } from '../server/helpers/logger' 6import { labelFormatter } from '../server/helpers/logger'
7import { CONFIG } from '../server/initializers/config' 7import { CONFIG } from '../server/initializers/config'
8import { mtimeSortFilesDesc } from '../shared/utils/logs/logs' 8import { mtimeSortFilesDesc } from '../shared/core-utils/logs/logs'
9 9
10program 10program
11 .option('-l, --level [level]', 'Level log (debug/info/warn/error)') 11 .option('-l, --level [level]', 'Level log (debug/info/warn/error)')
diff --git a/server.ts b/server.ts
index f4f0c4d68..aa4382ee7 100644
--- a/server.ts
+++ b/server.ts
@@ -101,6 +101,7 @@ import {
101import { advertiseDoNotTrack } from './server/middlewares/dnt' 101import { advertiseDoNotTrack } from './server/middlewares/dnt'
102import { Redis } from './server/lib/redis' 102import { Redis } from './server/lib/redis'
103import { ActorFollowScheduler } from './server/lib/schedulers/actor-follow-scheduler' 103import { ActorFollowScheduler } from './server/lib/schedulers/actor-follow-scheduler'
104import { RemoveOldViewsScheduler } from './server/lib/schedulers/remove-old-views-scheduler'
104import { RemoveOldJobsScheduler } from './server/lib/schedulers/remove-old-jobs-scheduler' 105import { RemoveOldJobsScheduler } from './server/lib/schedulers/remove-old-jobs-scheduler'
105import { UpdateVideosScheduler } from './server/lib/schedulers/update-videos-scheduler' 106import { UpdateVideosScheduler } from './server/lib/schedulers/update-videos-scheduler'
106import { YoutubeDlUpdateScheduler } from './server/lib/schedulers/youtube-dl-update-scheduler' 107import { YoutubeDlUpdateScheduler } from './server/lib/schedulers/youtube-dl-update-scheduler'
@@ -242,6 +243,7 @@ async function startApplication () {
242 YoutubeDlUpdateScheduler.Instance.enable() 243 YoutubeDlUpdateScheduler.Instance.enable()
243 VideosRedundancyScheduler.Instance.enable() 244 VideosRedundancyScheduler.Instance.enable()
244 RemoveOldHistoryScheduler.Instance.enable() 245 RemoveOldHistoryScheduler.Instance.enable()
246 RemoveOldViewsScheduler.Instance.enable()
245 247
246 // Redis initialization 248 // Redis initialization
247 Redis.Instance.init() 249 Redis.Instance.init()
diff --git a/server/controllers/api/server/logs.ts b/server/controllers/api/server/logs.ts
index 03941cca7..e9d1f2efd 100644
--- a/server/controllers/api/server/logs.ts
+++ b/server/controllers/api/server/logs.ts
@@ -1,7 +1,7 @@
1import * as express from 'express' 1import * as express from 'express'
2import { UserRight } from '../../../../shared/models/users' 2import { UserRight } from '../../../../shared/models/users'
3import { asyncMiddleware, authenticate, ensureUserHasRight } from '../../../middlewares' 3import { asyncMiddleware, authenticate, ensureUserHasRight } from '../../../middlewares'
4import { mtimeSortFilesDesc } from '../../../../shared/utils/logs/logs' 4import { mtimeSortFilesDesc } from '../../../../shared/core-utils/logs/logs'
5import { readdir, readFile } from 'fs-extra' 5import { readdir, readFile } from 'fs-extra'
6import { MAX_LOGS_OUTPUT_CHARACTERS } from '../../../initializers/constants' 6import { MAX_LOGS_OUTPUT_CHARACTERS } from '../../../initializers/constants'
7import { join } from 'path' 7import { join } from 'path'
diff --git a/server/initializers/checker-before-init.ts b/server/initializers/checker-before-init.ts
index 6b43debfb..223ef8078 100644
--- a/server/initializers/checker-before-init.ts
+++ b/server/initializers/checker-before-init.ts
@@ -26,7 +26,8 @@ function checkMissedConfig () {
26 'instance.is_nsfw', 'instance.default_nsfw_policy', 'instance.robots', 'instance.securitytxt', 26 'instance.is_nsfw', 'instance.default_nsfw_policy', 'instance.robots', 'instance.securitytxt',
27 'services.twitter.username', 'services.twitter.whitelisted', 27 'services.twitter.username', 'services.twitter.whitelisted',
28 'followers.instance.enabled', 'followers.instance.manual_approval', 28 'followers.instance.enabled', 'followers.instance.manual_approval',
29 'tracker.enabled', 'tracker.private', 'tracker.reject_too_many_announces' 29 'tracker.enabled', 'tracker.private', 'tracker.reject_too_many_announces',
30 'history.videos.max_age', 'views.videos.remote.max_age'
30 ] 31 ]
31 const requiredAlternatives = [ 32 const requiredAlternatives = [
32 [ // set 33 [ // set
diff --git a/server/initializers/config.ts b/server/initializers/config.ts
index 1f374dea9..baf502305 100644
--- a/server/initializers/config.ts
+++ b/server/initializers/config.ts
@@ -99,6 +99,13 @@ const CONFIG = {
99 MAX_AGE: parseDurationToMs(config.get('history.videos.max_age')) 99 MAX_AGE: parseDurationToMs(config.get('history.videos.max_age'))
100 } 100 }
101 }, 101 },
102 VIEWS: {
103 VIDEOS: {
104 REMOTE: {
105 MAX_AGE: parseDurationToMs(config.get('views.videos.remote.max_age'))
106 }
107 }
108 },
102 ADMIN: { 109 ADMIN: {
103 get EMAIL () { return config.get<string>('admin.email') } 110 get EMAIL () { return config.get<string>('admin.email') }
104 }, 111 },
diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts
index f008cd291..8f6ef1a81 100644
--- a/server/initializers/constants.ts
+++ b/server/initializers/constants.ts
@@ -163,6 +163,7 @@ const SCHEDULER_INTERVALS_MS = {
163 removeOldJobs: 60000 * 60, // 1 hour 163 removeOldJobs: 60000 * 60, // 1 hour
164 updateVideos: 60000, // 1 minute 164 updateVideos: 60000, // 1 minute
165 youtubeDLUpdate: 60000 * 60 * 24, // 1 day 165 youtubeDLUpdate: 60000 * 60 * 24, // 1 day
166 removeOldViews: 60000 * 60 * 24, // 1 day
166 removeOldHistory: 60000 * 60 * 24 // 1 day 167 removeOldHistory: 60000 * 60 * 24 // 1 day
167} 168}
168 169
@@ -592,6 +593,7 @@ if (isTestInstance() === true) {
592 SCHEDULER_INTERVALS_MS.actorFollowScores = 1000 593 SCHEDULER_INTERVALS_MS.actorFollowScores = 1000
593 SCHEDULER_INTERVALS_MS.removeOldJobs = 10000 594 SCHEDULER_INTERVALS_MS.removeOldJobs = 10000
594 SCHEDULER_INTERVALS_MS.removeOldHistory = 5000 595 SCHEDULER_INTERVALS_MS.removeOldHistory = 5000
596 SCHEDULER_INTERVALS_MS.removeOldViews = 5000
595 SCHEDULER_INTERVALS_MS.updateVideos = 5000 597 SCHEDULER_INTERVALS_MS.updateVideos = 5000
596 REPEAT_JOBS[ 'videos-views' ] = { every: 5000 } 598 REPEAT_JOBS[ 'videos-views' ] = { every: 5000 }
597 599
diff --git a/server/lib/schedulers/remove-old-views-scheduler.ts b/server/lib/schedulers/remove-old-views-scheduler.ts
new file mode 100644
index 000000000..39fbb9163
--- /dev/null
+++ b/server/lib/schedulers/remove-old-views-scheduler.ts
@@ -0,0 +1,33 @@
1import { logger } from '../../helpers/logger'
2import { AbstractScheduler } from './abstract-scheduler'
3import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants'
4import { UserVideoHistoryModel } from '../../models/account/user-video-history'
5import { CONFIG } from '../../initializers/config'
6import { isTestInstance } from '../../helpers/core-utils'
7import { VideoViewModel } from '../../models/video/video-views'
8
9export class RemoveOldViewsScheduler extends AbstractScheduler {
10
11 private static instance: AbstractScheduler
12
13 protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.removeOldViews
14
15 private constructor () {
16 super()
17 }
18
19 protected internalExecute () {
20 if (CONFIG.VIEWS.VIDEOS.REMOTE.MAX_AGE === -1) return
21
22 logger.info('Removing old videos views.')
23
24 const now = new Date()
25 const beforeDate = new Date(now.getTime() - CONFIG.VIEWS.VIDEOS.REMOTE.MAX_AGE).toISOString()
26
27 return VideoViewModel.removeOldRemoteViewsHistory(beforeDate)
28 }
29
30 static get Instance () {
31 return this.instance || (this.instance = new this())
32 }
33}
diff --git a/server/models/video/video-views.ts b/server/models/video/video-views.ts
index fde5f7056..6071e8c22 100644
--- a/server/models/video/video-views.ts
+++ b/server/models/video/video-views.ts
@@ -41,4 +41,18 @@ export class VideoViewModel extends Model<VideoViewModel> {
41 }) 41 })
42 Video: VideoModel 42 Video: VideoModel
43 43
44 static removeOldRemoteViewsHistory (beforeDate: string) {
45 const query = {
46 where: {
47 startDate: {
48 [Sequelize.Op.lt]: beforeDate
49 },
50 videoId: {
51 [Sequelize.Op.in]: Sequelize.literal('(SELECT "id" FROM "video" WHERE "remote" IS TRUE)')
52 }
53 }
54 }
55
56 return VideoViewModel.destroy(query)
57 }
44} 58}
diff --git a/server/tests/api/videos/index.ts b/server/tests/api/videos/index.ts
index 4be12ad15..93e1f3e98 100644
--- a/server/tests/api/videos/index.ts
+++ b/server/tests/api/videos/index.ts
@@ -18,3 +18,4 @@ import './video-transcoder'
18import './videos-filter' 18import './videos-filter'
19import './videos-history' 19import './videos-history'
20import './videos-overview' 20import './videos-overview'
21import './videos-views-cleaner'
diff --git a/server/tests/api/videos/videos-views-cleaner.ts b/server/tests/api/videos/videos-views-cleaner.ts
new file mode 100644
index 000000000..9f268c8e6
--- /dev/null
+++ b/server/tests/api/videos/videos-views-cleaner.ts
@@ -0,0 +1,113 @@
1/* tslint:disable:no-unused-expression */
2
3import * as chai from 'chai'
4import 'mocha'
5import {
6 flushAndRunMultipleServers,
7 flushTests,
8 killallServers,
9 reRunServer,
10 runServer,
11 ServerInfo,
12 setAccessTokensToServers,
13 uploadVideo, uploadVideoAndGetId, viewVideo, wait, countVideoViewsOf, doubleFollow, waitJobs
14} from '../../../../shared/utils'
15import { getVideosOverview } from '../../../../shared/utils/overviews/overviews'
16import { VideosOverview } from '../../../../shared/models/overviews'
17import { listMyVideosHistory } from '../../../../shared/utils/videos/video-history'
18
19const expect = chai.expect
20
21describe('Test video views cleaner', function () {
22 let servers: ServerInfo[]
23
24 let videoIdServer1: string
25 let videoIdServer2: string
26
27 before(async function () {
28 this.timeout(50000)
29
30 await flushTests()
31
32 servers = await flushAndRunMultipleServers(2)
33 await setAccessTokensToServers(servers)
34
35 await doubleFollow(servers[0], servers[1])
36
37 videoIdServer1 = (await uploadVideoAndGetId({ server: servers[0], videoName: 'video server 1' })).uuid
38 videoIdServer2 = (await uploadVideoAndGetId({ server: servers[1], videoName: 'video server 2' })).uuid
39
40 await waitJobs(servers)
41
42 await viewVideo(servers[0].url, videoIdServer1)
43 await viewVideo(servers[1].url, videoIdServer1)
44 await viewVideo(servers[0].url, videoIdServer2)
45 await viewVideo(servers[1].url, videoIdServer2)
46
47 await waitJobs(servers)
48 })
49
50 it('Should not clean old video views', async function () {
51 this.timeout(50000)
52
53 killallServers([ servers[0] ])
54
55 await reRunServer(servers[0], { views: { videos: { remote: { max_age: '10 days' } } } })
56
57 await wait(6000)
58
59 // Should still have views
60
61 {
62 for (const server of servers) {
63 const total = await countVideoViewsOf(server.serverNumber, videoIdServer1)
64 expect(total).to.equal(2)
65 }
66 }
67
68 {
69 for (const server of servers) {
70 const total = await countVideoViewsOf(server.serverNumber, videoIdServer2)
71 expect(total).to.equal(2)
72 }
73 }
74 })
75
76 it('Should clean old video views', async function () {
77 this.timeout(50000)
78
79 this.timeout(50000)
80
81 killallServers([ servers[0] ])
82
83 await reRunServer(servers[0], { views: { videos: { remote: { max_age: '5 seconds' } } } })
84
85 await wait(6000)
86
87 // Should still have views
88
89 {
90 for (const server of servers) {
91 const total = await countVideoViewsOf(server.serverNumber, videoIdServer1)
92 expect(total).to.equal(2)
93 }
94 }
95
96 {
97 const totalServer1 = await countVideoViewsOf(servers[0].serverNumber, videoIdServer2)
98 expect(totalServer1).to.equal(0)
99
100 const totalServer2 = await countVideoViewsOf(servers[1].serverNumber, videoIdServer2)
101 expect(totalServer2).to.equal(2)
102 }
103 })
104
105 after(async function () {
106 killallServers(servers)
107
108 // Keep the logs if the test failed
109 if (this['ok']) {
110 await flushTests()
111 }
112 })
113})
diff --git a/shared/core-utils/logs/logs.ts b/shared/core-utils/logs/logs.ts
new file mode 100644
index 000000000..d0996cf55
--- /dev/null
+++ b/shared/core-utils/logs/logs.ts
@@ -0,0 +1,25 @@
1import { stat } from 'fs-extra'
2
3async function mtimeSortFilesDesc (files: string[], basePath: string) {
4 const promises = []
5 const out: { file: string, mtime: number }[] = []
6
7 for (const file of files) {
8 const p = stat(basePath + '/' + file)
9 .then(stats => {
10 if (stats.isFile()) out.push({ file, mtime: stats.mtime.getTime() })
11 })
12
13 promises.push(p)
14 }
15
16 await Promise.all(promises)
17
18 out.sort((a, b) => b.mtime - a.mtime)
19
20 return out
21}
22
23export {
24 mtimeSortFilesDesc
25}
diff --git a/shared/utils/logs/logs.ts b/shared/utils/logs/logs.ts
index 21adace82..cbb1afb93 100644
--- a/shared/utils/logs/logs.ts
+++ b/shared/utils/logs/logs.ts
@@ -1,28 +1,6 @@
1// Thanks: https://stackoverflow.com/a/37014317
2import { stat } from 'fs-extra'
3import { makeGetRequest } from '../requests/requests' 1import { makeGetRequest } from '../requests/requests'
4import { LogLevel } from '../../models/server/log-level.type' 2import { LogLevel } from '../../models/server/log-level.type'
5 3
6async function mtimeSortFilesDesc (files: string[], basePath: string) {
7 const promises = []
8 const out: { file: string, mtime: number }[] = []
9
10 for (const file of files) {
11 const p = stat(basePath + '/' + file)
12 .then(stats => {
13 if (stats.isFile()) out.push({ file, mtime: stats.mtime.getTime() })
14 })
15
16 promises.push(p)
17 }
18
19 await Promise.all(promises)
20
21 out.sort((a, b) => b.mtime - a.mtime)
22
23 return out
24}
25
26function getLogs (url: string, accessToken: string, startDate: Date, endDate?: Date, level?: LogLevel) { 4function getLogs (url: string, accessToken: string, startDate: Date, endDate?: Date, level?: LogLevel) {
27 const path = '/api/v1/server/logs' 5 const path = '/api/v1/server/logs'
28 6
@@ -36,6 +14,5 @@ function getLogs (url: string, accessToken: string, startDate: Date, endDate?: D
36} 14}
37 15
38export { 16export {
39 mtimeSortFilesDesc,
40 getLogs 17 getLogs
41} 18}
diff --git a/shared/utils/miscs/sql.ts b/shared/utils/miscs/sql.ts
index 1ce3d801a..b281471ce 100644
--- a/shared/utils/miscs/sql.ts
+++ b/shared/utils/miscs/sql.ts
@@ -48,6 +48,20 @@ function setPlaylistField (serverNumber: number, uuid: string, field: string, va
48 return seq.query(`UPDATE "videoPlaylist" SET "${field}" = '${value}' WHERE uuid = '${uuid}'`, options) 48 return seq.query(`UPDATE "videoPlaylist" SET "${field}" = '${value}' WHERE uuid = '${uuid}'`, options)
49} 49}
50 50
51async function countVideoViewsOf (serverNumber: number, uuid: string) {
52 const seq = getSequelize(serverNumber)
53
54 // tslint:disable
55 const query = `SELECT SUM("videoView"."views") AS "total" FROM "videoView" INNER JOIN "video" ON "video"."id" = "videoView"."videoId" WHERE "video"."uuid" = '${uuid}'`
56
57 const options = { type: Sequelize.QueryTypes.SELECT }
58 const [ { total } ] = await seq.query(query, options)
59
60 if (!total) return 0
61
62 return parseInt(total, 10)
63}
64
51async function closeAllSequelize (servers: any[]) { 65async function closeAllSequelize (servers: any[]) {
52 for (let i = 1; i <= servers.length; i++) { 66 for (let i = 1; i <= servers.length; i++) {
53 if (sequelizes[ i ]) { 67 if (sequelizes[ i ]) {
@@ -61,5 +75,6 @@ export {
61 setVideoField, 75 setVideoField,
62 setPlaylistField, 76 setPlaylistField,
63 setActorField, 77 setActorField,
78 countVideoViewsOf,
64 closeAllSequelize 79 closeAllSequelize
65} 80}