aboutsummaryrefslogtreecommitdiffhomepage
path: root/server
diff options
context:
space:
mode:
Diffstat (limited to 'server')
-rw-r--r--server/controllers/api/server/logs.ts2
-rw-r--r--server/initializers/checker-before-init.ts3
-rw-r--r--server/initializers/config.ts7
-rw-r--r--server/initializers/constants.ts2
-rw-r--r--server/lib/schedulers/remove-old-views-scheduler.ts33
-rw-r--r--server/models/video/video-views.ts14
-rw-r--r--server/tests/api/videos/index.ts1
-rw-r--r--server/tests/api/videos/videos-views-cleaner.ts113
8 files changed, 173 insertions, 2 deletions
diff --git a/server/controllers/api/server/logs.ts b/server/controllers/api/server/logs.ts
index 03941cca7..e9d1f2efd 100644
--- a/server/controllers/api/server/logs.ts
+++ b/server/controllers/api/server/logs.ts
@@ -1,7 +1,7 @@
1import * as express from 'express' 1import * as express from 'express'
2import { UserRight } from '../../../../shared/models/users' 2import { UserRight } from '../../../../shared/models/users'
3import { asyncMiddleware, authenticate, ensureUserHasRight } from '../../../middlewares' 3import { asyncMiddleware, authenticate, ensureUserHasRight } from '../../../middlewares'
4import { mtimeSortFilesDesc } from '../../../../shared/utils/logs/logs' 4import { mtimeSortFilesDesc } from '../../../../shared/core-utils/logs/logs'
5import { readdir, readFile } from 'fs-extra' 5import { readdir, readFile } from 'fs-extra'
6import { MAX_LOGS_OUTPUT_CHARACTERS } from '../../../initializers/constants' 6import { MAX_LOGS_OUTPUT_CHARACTERS } from '../../../initializers/constants'
7import { join } from 'path' 7import { join } from 'path'
diff --git a/server/initializers/checker-before-init.ts b/server/initializers/checker-before-init.ts
index 6b43debfb..223ef8078 100644
--- a/server/initializers/checker-before-init.ts
+++ b/server/initializers/checker-before-init.ts
@@ -26,7 +26,8 @@ function checkMissedConfig () {
26 'instance.is_nsfw', 'instance.default_nsfw_policy', 'instance.robots', 'instance.securitytxt', 26 'instance.is_nsfw', 'instance.default_nsfw_policy', 'instance.robots', 'instance.securitytxt',
27 'services.twitter.username', 'services.twitter.whitelisted', 27 'services.twitter.username', 'services.twitter.whitelisted',
28 'followers.instance.enabled', 'followers.instance.manual_approval', 28 'followers.instance.enabled', 'followers.instance.manual_approval',
29 'tracker.enabled', 'tracker.private', 'tracker.reject_too_many_announces' 29 'tracker.enabled', 'tracker.private', 'tracker.reject_too_many_announces',
30 'history.videos.max_age', 'views.videos.remote.max_age'
30 ] 31 ]
31 const requiredAlternatives = [ 32 const requiredAlternatives = [
32 [ // set 33 [ // set
diff --git a/server/initializers/config.ts b/server/initializers/config.ts
index 1f374dea9..baf502305 100644
--- a/server/initializers/config.ts
+++ b/server/initializers/config.ts
@@ -99,6 +99,13 @@ const CONFIG = {
99 MAX_AGE: parseDurationToMs(config.get('history.videos.max_age')) 99 MAX_AGE: parseDurationToMs(config.get('history.videos.max_age'))
100 } 100 }
101 }, 101 },
102 VIEWS: {
103 VIDEOS: {
104 REMOTE: {
105 MAX_AGE: parseDurationToMs(config.get('views.videos.remote.max_age'))
106 }
107 }
108 },
102 ADMIN: { 109 ADMIN: {
103 get EMAIL () { return config.get<string>('admin.email') } 110 get EMAIL () { return config.get<string>('admin.email') }
104 }, 111 },
diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts
index f008cd291..8f6ef1a81 100644
--- a/server/initializers/constants.ts
+++ b/server/initializers/constants.ts
@@ -163,6 +163,7 @@ const SCHEDULER_INTERVALS_MS = {
163 removeOldJobs: 60000 * 60, // 1 hour 163 removeOldJobs: 60000 * 60, // 1 hour
164 updateVideos: 60000, // 1 minute 164 updateVideos: 60000, // 1 minute
165 youtubeDLUpdate: 60000 * 60 * 24, // 1 day 165 youtubeDLUpdate: 60000 * 60 * 24, // 1 day
166 removeOldViews: 60000 * 60 * 24, // 1 day
166 removeOldHistory: 60000 * 60 * 24 // 1 day 167 removeOldHistory: 60000 * 60 * 24 // 1 day
167} 168}
168 169
@@ -592,6 +593,7 @@ if (isTestInstance() === true) {
592 SCHEDULER_INTERVALS_MS.actorFollowScores = 1000 593 SCHEDULER_INTERVALS_MS.actorFollowScores = 1000
593 SCHEDULER_INTERVALS_MS.removeOldJobs = 10000 594 SCHEDULER_INTERVALS_MS.removeOldJobs = 10000
594 SCHEDULER_INTERVALS_MS.removeOldHistory = 5000 595 SCHEDULER_INTERVALS_MS.removeOldHistory = 5000
596 SCHEDULER_INTERVALS_MS.removeOldViews = 5000
595 SCHEDULER_INTERVALS_MS.updateVideos = 5000 597 SCHEDULER_INTERVALS_MS.updateVideos = 5000
596 REPEAT_JOBS[ 'videos-views' ] = { every: 5000 } 598 REPEAT_JOBS[ 'videos-views' ] = { every: 5000 }
597 599
diff --git a/server/lib/schedulers/remove-old-views-scheduler.ts b/server/lib/schedulers/remove-old-views-scheduler.ts
new file mode 100644
index 000000000..39fbb9163
--- /dev/null
+++ b/server/lib/schedulers/remove-old-views-scheduler.ts
@@ -0,0 +1,33 @@
1import { logger } from '../../helpers/logger'
2import { AbstractScheduler } from './abstract-scheduler'
3import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants'
4import { UserVideoHistoryModel } from '../../models/account/user-video-history'
5import { CONFIG } from '../../initializers/config'
6import { isTestInstance } from '../../helpers/core-utils'
7import { VideoViewModel } from '../../models/video/video-views'
8
9export class RemoveOldViewsScheduler extends AbstractScheduler {
10
11 private static instance: AbstractScheduler
12
13 protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.removeOldViews
14
15 private constructor () {
16 super()
17 }
18
19 protected internalExecute () {
20 if (CONFIG.VIEWS.VIDEOS.REMOTE.MAX_AGE === -1) return
21
22 logger.info('Removing old videos views.')
23
24 const now = new Date()
25 const beforeDate = new Date(now.getTime() - CONFIG.VIEWS.VIDEOS.REMOTE.MAX_AGE).toISOString()
26
27 return VideoViewModel.removeOldRemoteViewsHistory(beforeDate)
28 }
29
30 static get Instance () {
31 return this.instance || (this.instance = new this())
32 }
33}
diff --git a/server/models/video/video-views.ts b/server/models/video/video-views.ts
index fde5f7056..6071e8c22 100644
--- a/server/models/video/video-views.ts
+++ b/server/models/video/video-views.ts
@@ -41,4 +41,18 @@ export class VideoViewModel extends Model<VideoViewModel> {
41 }) 41 })
42 Video: VideoModel 42 Video: VideoModel
43 43
44 static removeOldRemoteViewsHistory (beforeDate: string) {
45 const query = {
46 where: {
47 startDate: {
48 [Sequelize.Op.lt]: beforeDate
49 },
50 videoId: {
51 [Sequelize.Op.in]: Sequelize.literal('(SELECT "id" FROM "video" WHERE "remote" IS TRUE)')
52 }
53 }
54 }
55
56 return VideoViewModel.destroy(query)
57 }
44} 58}
diff --git a/server/tests/api/videos/index.ts b/server/tests/api/videos/index.ts
index 4be12ad15..93e1f3e98 100644
--- a/server/tests/api/videos/index.ts
+++ b/server/tests/api/videos/index.ts
@@ -18,3 +18,4 @@ import './video-transcoder'
18import './videos-filter' 18import './videos-filter'
19import './videos-history' 19import './videos-history'
20import './videos-overview' 20import './videos-overview'
21import './videos-views-cleaner'
diff --git a/server/tests/api/videos/videos-views-cleaner.ts b/server/tests/api/videos/videos-views-cleaner.ts
new file mode 100644
index 000000000..9f268c8e6
--- /dev/null
+++ b/server/tests/api/videos/videos-views-cleaner.ts
@@ -0,0 +1,113 @@
1/* tslint:disable:no-unused-expression */
2
3import * as chai from 'chai'
4import 'mocha'
5import {
6 flushAndRunMultipleServers,
7 flushTests,
8 killallServers,
9 reRunServer,
10 runServer,
11 ServerInfo,
12 setAccessTokensToServers,
13 uploadVideo, uploadVideoAndGetId, viewVideo, wait, countVideoViewsOf, doubleFollow, waitJobs
14} from '../../../../shared/utils'
15import { getVideosOverview } from '../../../../shared/utils/overviews/overviews'
16import { VideosOverview } from '../../../../shared/models/overviews'
17import { listMyVideosHistory } from '../../../../shared/utils/videos/video-history'
18
19const expect = chai.expect
20
21describe('Test video views cleaner', function () {
22 let servers: ServerInfo[]
23
24 let videoIdServer1: string
25 let videoIdServer2: string
26
27 before(async function () {
28 this.timeout(50000)
29
30 await flushTests()
31
32 servers = await flushAndRunMultipleServers(2)
33 await setAccessTokensToServers(servers)
34
35 await doubleFollow(servers[0], servers[1])
36
37 videoIdServer1 = (await uploadVideoAndGetId({ server: servers[0], videoName: 'video server 1' })).uuid
38 videoIdServer2 = (await uploadVideoAndGetId({ server: servers[1], videoName: 'video server 2' })).uuid
39
40 await waitJobs(servers)
41
42 await viewVideo(servers[0].url, videoIdServer1)
43 await viewVideo(servers[1].url, videoIdServer1)
44 await viewVideo(servers[0].url, videoIdServer2)
45 await viewVideo(servers[1].url, videoIdServer2)
46
47 await waitJobs(servers)
48 })
49
50 it('Should not clean old video views', async function () {
51 this.timeout(50000)
52
53 killallServers([ servers[0] ])
54
55 await reRunServer(servers[0], { views: { videos: { remote: { max_age: '10 days' } } } })
56
57 await wait(6000)
58
59 // Should still have views
60
61 {
62 for (const server of servers) {
63 const total = await countVideoViewsOf(server.serverNumber, videoIdServer1)
64 expect(total).to.equal(2)
65 }
66 }
67
68 {
69 for (const server of servers) {
70 const total = await countVideoViewsOf(server.serverNumber, videoIdServer2)
71 expect(total).to.equal(2)
72 }
73 }
74 })
75
76 it('Should clean old video views', async function () {
77 this.timeout(50000)
78
79 this.timeout(50000)
80
81 killallServers([ servers[0] ])
82
83 await reRunServer(servers[0], { views: { videos: { remote: { max_age: '5 seconds' } } } })
84
85 await wait(6000)
86
87 // Should still have views
88
89 {
90 for (const server of servers) {
91 const total = await countVideoViewsOf(server.serverNumber, videoIdServer1)
92 expect(total).to.equal(2)
93 }
94 }
95
96 {
97 const totalServer1 = await countVideoViewsOf(servers[0].serverNumber, videoIdServer2)
98 expect(totalServer1).to.equal(0)
99
100 const totalServer2 = await countVideoViewsOf(servers[1].serverNumber, videoIdServer2)
101 expect(totalServer2).to.equal(2)
102 }
103 })
104
105 after(async function () {
106 killallServers(servers)
107
108 // Keep the logs if the test failed
109 if (this['ok']) {
110 await flushTests()
111 }
112 })
113})