aboutsummaryrefslogtreecommitdiffhomepage
path: root/server
diff options
context:
space:
mode:
authorChocobozzz <me@florianbigard.com>2019-04-11 15:38:53 +0200
committerChocobozzz <me@florianbigard.com>2019-04-11 15:38:53 +0200
commit8f0bc73d7d5f4c88cbc5588a0ece12b3855c8f98 (patch)
treee04f1da52f9377cf6ce820425c0de6e57ab57fc6 /server
parent76062d9f96e06a23a2efc8a727ea9c5394d21466 (diff)
downloadPeerTube-8f0bc73d7d5f4c88cbc5588a0ece12b3855c8f98.tar.gz
PeerTube-8f0bc73d7d5f4c88cbc5588a0ece12b3855c8f98.tar.zst
PeerTube-8f0bc73d7d5f4c88cbc5588a0ece12b3855c8f98.zip
Add ability to limit videos history size
Diffstat (limited to 'server')
-rw-r--r--server/controllers/api/users/my-history.ts2
-rw-r--r--server/helpers/core-utils.ts2
-rw-r--r--server/helpers/custom-validators/videos.ts3
-rw-r--r--server/initializers/config.ts11
-rw-r--r--server/initializers/constants.ts9
-rw-r--r--server/lib/schedulers/abstract-scheduler.ts3
-rw-r--r--server/lib/schedulers/remove-old-history-scheduler.ts32
-rw-r--r--server/middlewares/cache.ts4
-rw-r--r--server/models/account/user-video-history.ts14
-rw-r--r--server/tests/api/videos/videos-history.ts34
10 files changed, 98 insertions, 16 deletions
diff --git a/server/controllers/api/users/my-history.ts b/server/controllers/api/users/my-history.ts
index b30d3aec2..7025c0ff1 100644
--- a/server/controllers/api/users/my-history.ts
+++ b/server/controllers/api/users/my-history.ts
@@ -48,7 +48,7 @@ async function removeUserHistory (req: express.Request, res: express.Response) {
48 const beforeDate = req.body.beforeDate || null 48 const beforeDate = req.body.beforeDate || null
49 49
50 await sequelizeTypescript.transaction(t => { 50 await sequelizeTypescript.transaction(t => {
51 return UserVideoHistoryModel.removeHistoryBefore(user, beforeDate, t) 51 return UserVideoHistoryModel.removeUserHistoryBefore(user, beforeDate, t)
52 }) 52 })
53 53
54 // Do not send the delete to other instances, we delete OUR copy of this video abuse 54 // Do not send the delete to other instances, we delete OUR copy of this video abuse
diff --git a/server/helpers/core-utils.ts b/server/helpers/core-utils.ts
index f6d90bfca..305d3b71e 100644
--- a/server/helpers/core-utils.ts
+++ b/server/helpers/core-utils.ts
@@ -40,7 +40,7 @@ const timeTable = {
40 month: 3600000 * 24 * 30 40 month: 3600000 * 24 * 30
41} 41}
42 42
43export function parseDuration (duration: number | string): number { 43export function parseDurationToMs (duration: number | string): number {
44 if (typeof duration === 'number') return duration 44 if (typeof duration === 'number') return duration
45 45
46 if (typeof duration === 'string') { 46 if (typeof duration === 'string') {
diff --git a/server/helpers/custom-validators/videos.ts b/server/helpers/custom-validators/videos.ts
index eb08ae4ad..214db17a1 100644
--- a/server/helpers/custom-validators/videos.ts
+++ b/server/helpers/custom-validators/videos.ts
@@ -5,7 +5,8 @@ import 'multer'
5import * as validator from 'validator' 5import * as validator from 'validator'
6import { UserRight, VideoFilter, VideoPrivacy, VideoRateType } from '../../../shared' 6import { UserRight, VideoFilter, VideoPrivacy, VideoRateType } from '../../../shared'
7import { 7import {
8 CONSTRAINTS_FIELDS, MIMETYPES, 8 CONSTRAINTS_FIELDS,
9 MIMETYPES,
9 VIDEO_CATEGORIES, 10 VIDEO_CATEGORIES,
10 VIDEO_LICENCES, 11 VIDEO_LICENCES,
11 VIDEO_PRIVACIES, 12 VIDEO_PRIVACIES,
diff --git a/server/initializers/config.ts b/server/initializers/config.ts
index 8dd62cba8..1f374dea9 100644
--- a/server/initializers/config.ts
+++ b/server/initializers/config.ts
@@ -2,7 +2,7 @@ import { IConfig } from 'config'
2import { dirname, join } from 'path' 2import { dirname, join } from 'path'
3import { VideosRedundancy } from '../../shared/models' 3import { VideosRedundancy } from '../../shared/models'
4// Do not use barrels, remain constants as independent as possible 4// Do not use barrels, remain constants as independent as possible
5import { buildPath, parseBytes, parseDuration, root } from '../helpers/core-utils' 5import { buildPath, parseBytes, parseDurationToMs, root } from '../helpers/core-utils'
6import { NSFWPolicyType } from '../../shared/models/videos/nsfw-policy.type' 6import { NSFWPolicyType } from '../../shared/models/videos/nsfw-policy.type'
7import * as bytes from 'bytes' 7import * as bytes from 'bytes'
8 8
@@ -80,7 +80,7 @@ const CONFIG = {
80 }, 80 },
81 REDUNDANCY: { 81 REDUNDANCY: {
82 VIDEOS: { 82 VIDEOS: {
83 CHECK_INTERVAL: parseDuration(config.get<string>('redundancy.videos.check_interval')), 83 CHECK_INTERVAL: parseDurationToMs(config.get<string>('redundancy.videos.check_interval')),
84 STRATEGIES: buildVideosRedundancy(config.get<any[]>('redundancy.videos.strategies')) 84 STRATEGIES: buildVideosRedundancy(config.get<any[]>('redundancy.videos.strategies'))
85 } 85 }
86 }, 86 },
@@ -94,6 +94,11 @@ const CONFIG = {
94 PRIVATE: config.get<boolean>('tracker.private'), 94 PRIVATE: config.get<boolean>('tracker.private'),
95 REJECT_TOO_MANY_ANNOUNCES: config.get<boolean>('tracker.reject_too_many_announces') 95 REJECT_TOO_MANY_ANNOUNCES: config.get<boolean>('tracker.reject_too_many_announces')
96 }, 96 },
97 HISTORY: {
98 VIDEOS: {
99 MAX_AGE: parseDurationToMs(config.get('history.videos.max_age'))
100 }
101 },
97 ADMIN: { 102 ADMIN: {
98 get EMAIL () { return config.get<string>('admin.email') } 103 get EMAIL () { return config.get<string>('admin.email') }
99 }, 104 },
@@ -216,7 +221,7 @@ function buildVideosRedundancy (objs: any[]): VideosRedundancy[] {
216 221
217 return objs.map(obj => { 222 return objs.map(obj => {
218 return Object.assign({}, obj, { 223 return Object.assign({}, obj, {
219 minLifetime: parseDuration(obj.min_lifetime), 224 minLifetime: parseDurationToMs(obj.min_lifetime),
220 size: bytes.parse(obj.size), 225 size: bytes.parse(obj.size),
221 minViews: obj.min_views 226 minViews: obj.min_views
222 }) 227 })
diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts
index a0609d7cd..f008cd291 100644
--- a/server/initializers/constants.ts
+++ b/server/initializers/constants.ts
@@ -158,12 +158,12 @@ const JOB_REQUEST_TIMEOUT = 3000 // 3 seconds
158const JOB_COMPLETED_LIFETIME = 60000 * 60 * 24 * 2 // 2 days 158const JOB_COMPLETED_LIFETIME = 60000 * 60 * 24 * 2 // 2 days
159const VIDEO_IMPORT_TIMEOUT = 1000 * 3600 // 1 hour 159const VIDEO_IMPORT_TIMEOUT = 1000 * 3600 // 1 hour
160 160
161// 1 hour 161const SCHEDULER_INTERVALS_MS = {
162let SCHEDULER_INTERVALS_MS = {
163 actorFollowScores: 60000 * 60, // 1 hour 162 actorFollowScores: 60000 * 60, // 1 hour
164 removeOldJobs: 60000 * 60, // 1 hour 163 removeOldJobs: 60000 * 60, // 1 hour
165 updateVideos: 60000, // 1 minute 164 updateVideos: 60000, // 1 minute
166 youtubeDLUpdate: 60000 * 60 * 24 // 1 day 165 youtubeDLUpdate: 60000 * 60 * 24, // 1 day
166 removeOldHistory: 60000 * 60 * 24 // 1 day
167} 167}
168 168
169// --------------------------------------------------------------------------- 169// ---------------------------------------------------------------------------
@@ -591,6 +591,7 @@ if (isTestInstance() === true) {
591 591
592 SCHEDULER_INTERVALS_MS.actorFollowScores = 1000 592 SCHEDULER_INTERVALS_MS.actorFollowScores = 1000
593 SCHEDULER_INTERVALS_MS.removeOldJobs = 10000 593 SCHEDULER_INTERVALS_MS.removeOldJobs = 10000
594 SCHEDULER_INTERVALS_MS.removeOldHistory = 5000
594 SCHEDULER_INTERVALS_MS.updateVideos = 5000 595 SCHEDULER_INTERVALS_MS.updateVideos = 5000
595 REPEAT_JOBS[ 'videos-views' ] = { every: 5000 } 596 REPEAT_JOBS[ 'videos-views' ] = { every: 5000 }
596 597
@@ -734,7 +735,7 @@ function buildVideosExtname () {
734} 735}
735 736
736function loadLanguages () { 737function loadLanguages () {
737 VIDEO_LANGUAGES = buildLanguages() 738 Object.assign(VIDEO_LANGUAGES, buildLanguages())
738} 739}
739 740
740function buildLanguages () { 741function buildLanguages () {
diff --git a/server/lib/schedulers/abstract-scheduler.ts b/server/lib/schedulers/abstract-scheduler.ts
index 86ea7aa38..0e6088911 100644
--- a/server/lib/schedulers/abstract-scheduler.ts
+++ b/server/lib/schedulers/abstract-scheduler.ts
@@ -1,4 +1,5 @@
1import { logger } from '../../helpers/logger' 1import { logger } from '../../helpers/logger'
2import * as Bluebird from 'bluebird'
2 3
3export abstract class AbstractScheduler { 4export abstract class AbstractScheduler {
4 5
@@ -30,5 +31,5 @@ export abstract class AbstractScheduler {
30 } 31 }
31 } 32 }
32 33
33 protected abstract internalExecute (): Promise<any> 34 protected abstract internalExecute (): Promise<any> | Bluebird<any>
34} 35}
diff --git a/server/lib/schedulers/remove-old-history-scheduler.ts b/server/lib/schedulers/remove-old-history-scheduler.ts
new file mode 100644
index 000000000..1b5ff8394
--- /dev/null
+++ b/server/lib/schedulers/remove-old-history-scheduler.ts
@@ -0,0 +1,32 @@
1import { logger } from '../../helpers/logger'
2import { AbstractScheduler } from './abstract-scheduler'
3import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants'
4import { UserVideoHistoryModel } from '../../models/account/user-video-history'
5import { CONFIG } from '../../initializers/config'
6import { isTestInstance } from '../../helpers/core-utils'
7
8export class RemoveOldHistoryScheduler extends AbstractScheduler {
9
10 private static instance: AbstractScheduler
11
12 protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.removeOldHistory
13
14 private constructor () {
15 super()
16 }
17
18 protected internalExecute () {
19 if (CONFIG.HISTORY.VIDEOS.MAX_AGE === -1) return
20
21 logger.info('Removing old videos history.')
22
23 const now = new Date()
24 const beforeDate = new Date(now.getTime() - CONFIG.HISTORY.VIDEOS.MAX_AGE).toISOString()
25
26 return UserVideoHistoryModel.removeOldHistory(beforeDate)
27 }
28
29 static get Instance () {
30 return this.instance || (this.instance = new this())
31 }
32}
diff --git a/server/middlewares/cache.ts b/server/middlewares/cache.ts
index 8ffe75700..e83d8d569 100644
--- a/server/middlewares/cache.ts
+++ b/server/middlewares/cache.ts
@@ -1,6 +1,6 @@
1import * as express from 'express' 1import * as express from 'express'
2import * as AsyncLock from 'async-lock' 2import * as AsyncLock from 'async-lock'
3import { parseDuration } from '../helpers/core-utils' 3import { parseDurationToMs } from '../helpers/core-utils'
4import { Redis } from '../lib/redis' 4import { Redis } from '../lib/redis'
5import { logger } from '../helpers/logger' 5import { logger } from '../helpers/logger'
6 6
@@ -24,7 +24,7 @@ function cacheRoute (lifetimeArg: string | number) {
24 res.send = (body) => { 24 res.send = (body) => {
25 if (res.statusCode >= 200 && res.statusCode < 400) { 25 if (res.statusCode >= 200 && res.statusCode < 400) {
26 const contentType = res.get('content-type') 26 const contentType = res.get('content-type')
27 const lifetime = parseDuration(lifetimeArg) 27 const lifetime = parseDurationToMs(lifetimeArg)
28 28
29 Redis.Instance.setCachedRoute(req, body, lifetime, contentType, res.statusCode) 29 Redis.Instance.setCachedRoute(req, body, lifetime, contentType, res.statusCode)
30 .then(() => done()) 30 .then(() => done())
diff --git a/server/models/account/user-video-history.ts b/server/models/account/user-video-history.ts
index 15cb399c9..49d2def81 100644
--- a/server/models/account/user-video-history.ts
+++ b/server/models/account/user-video-history.ts
@@ -67,7 +67,7 @@ export class UserVideoHistoryModel extends Model<UserVideoHistoryModel> {
67 }) 67 })
68 } 68 }
69 69
70 static removeHistoryBefore (user: UserModel, beforeDate: string, t: Transaction) { 70 static removeUserHistoryBefore (user: UserModel, beforeDate: string, t: Transaction) {
71 const query: DestroyOptions = { 71 const query: DestroyOptions = {
72 where: { 72 where: {
73 userId: user.id 73 userId: user.id
@@ -83,4 +83,16 @@ export class UserVideoHistoryModel extends Model<UserVideoHistoryModel> {
83 83
84 return UserVideoHistoryModel.destroy(query) 84 return UserVideoHistoryModel.destroy(query)
85 } 85 }
86
87 static removeOldHistory (beforeDate: string) {
88 const query: DestroyOptions = {
89 where: {
90 updatedAt: {
91 [Op.lt]: beforeDate
92 }
93 }
94 }
95
96 return UserVideoHistoryModel.destroy(query)
97 }
86} 98}
diff --git a/server/tests/api/videos/videos-history.ts b/server/tests/api/videos/videos-history.ts
index f654a422b..f7d3a6aeb 100644
--- a/server/tests/api/videos/videos-history.ts
+++ b/server/tests/api/videos/videos-history.ts
@@ -7,14 +7,15 @@ import {
7 flushTests, 7 flushTests,
8 getVideosListWithToken, 8 getVideosListWithToken,
9 getVideoWithToken, 9 getVideoWithToken,
10 killallServers, 10 killallServers, reRunServer,
11 runServer, 11 runServer,
12 searchVideoWithToken, 12 searchVideoWithToken,
13 ServerInfo, 13 ServerInfo,
14 setAccessTokensToServers, 14 setAccessTokensToServers,
15 updateMyUser, 15 updateMyUser,
16 uploadVideo, 16 uploadVideo,
17 userLogin 17 userLogin,
18 wait
18} from '../../../../shared/utils' 19} from '../../../../shared/utils'
19import { Video, VideoDetails } from '../../../../shared/models/videos' 20import { Video, VideoDetails } from '../../../../shared/models/videos'
20import { listMyVideosHistory, removeMyVideosHistory, userWatchVideo } from '../../../../shared/utils/videos/video-history' 21import { listMyVideosHistory, removeMyVideosHistory, userWatchVideo } from '../../../../shared/utils/videos/video-history'
@@ -192,6 +193,35 @@ describe('Test videos history', function () {
192 expect(videos[1].name).to.equal('video 3') 193 expect(videos[1].name).to.equal('video 3')
193 }) 194 })
194 195
196 it('Should not clean old history', async function () {
197 this.timeout(50000)
198
199 killallServers([ server ])
200
201 await reRunServer(server, { history: { videos: { max_age: '10 days' } } })
202
203 await wait(6000)
204
205 // Should still have history
206
207 const res = await listMyVideosHistory(server.url, server.accessToken)
208
209 expect(res.body.total).to.equal(2)
210 })
211
212 it('Should clean old history', async function () {
213 this.timeout(50000)
214
215 killallServers([ server ])
216
217 await reRunServer(server, { history: { videos: { max_age: '5 seconds' } } })
218
219 await wait(6000)
220
221 const res = await listMyVideosHistory(server.url, server.accessToken)
222 expect(res.body.total).to.equal(0)
223 })
224
195 after(async function () { 225 after(async function () {
196 killallServers([ server ]) 226 killallServers([ server ])
197 227