Add ability to limit videos history size

10 files changed, 98 insertions, 16 deletions

diff --git a/server/controllers/api/users/my-history.ts b/server/controllers/api/users/my-history.ts
index b30d3aec2..7025c0ff1 100644
--- a/server/controllers/api/users/my-history.ts
+++ b/server/controllers/api/users/my-history.ts
@@ -48,7 +48,7 @@ async function removeUserHistory (req: express.Request, res: express.Response) {
   const beforeDate = req.body.beforeDate || null
 
   await sequelizeTypescript.transaction(t => {
-    return UserVideoHistoryModel.removeHistoryBefore(user, beforeDate, t)
+    return UserVideoHistoryModel.removeUserHistoryBefore(user, beforeDate, t)
   })
 
   // Do not send the delete to other instances, we delete OUR copy of this video abuse
diff --git a/server/helpers/core-utils.ts b/server/helpers/core-utils.ts
index f6d90bfca..305d3b71e 100644
--- a/server/helpers/core-utils.ts
+++ b/server/helpers/core-utils.ts
@@ -40,7 +40,7 @@ const timeTable = {
   month:        3600000 * 24 * 30
 }
 
-export function parseDuration (duration: number | string): number {
+export function parseDurationToMs (duration: number | string): number {
   if (typeof duration === 'number') return duration
 
   if (typeof duration === 'string') {
diff --git a/server/helpers/custom-validators/videos.ts b/server/helpers/custom-validators/videos.ts
index eb08ae4ad..214db17a1 100644
--- a/server/helpers/custom-validators/videos.ts
+++ b/server/helpers/custom-validators/videos.ts
@@ -5,7 +5,8 @@ import 'multer'
 import * as validator from 'validator'
 import { UserRight, VideoFilter, VideoPrivacy, VideoRateType } from '../../../shared'
 import {
-  CONSTRAINTS_FIELDS, MIMETYPES,
+  CONSTRAINTS_FIELDS,
+  MIMETYPES,
   VIDEO_CATEGORIES,
   VIDEO_LICENCES,
   VIDEO_PRIVACIES,
diff --git a/server/initializers/config.ts b/server/initializers/config.ts
index 8dd62cba8..1f374dea9 100644
--- a/server/initializers/config.ts
+++ b/server/initializers/config.ts
@@ -2,7 +2,7 @@ import { IConfig } from 'config'
 import { dirname, join } from 'path'
 import { VideosRedundancy } from '../../shared/models'
 // Do not use barrels, remain constants as independent as possible
-import { buildPath, parseBytes, parseDuration, root } from '../helpers/core-utils'
+import { buildPath, parseBytes, parseDurationToMs, root } from '../helpers/core-utils'
 import { NSFWPolicyType } from '../../shared/models/videos/nsfw-policy.type'
 import * as bytes from 'bytes'
 
@@ -80,7 +80,7 @@ const CONFIG = {
   },
   REDUNDANCY: {
     VIDEOS: {
-      CHECK_INTERVAL: parseDuration(config.get<string>('redundancy.videos.check_interval')),
+      CHECK_INTERVAL: parseDurationToMs(config.get<string>('redundancy.videos.check_interval')),
       STRATEGIES: buildVideosRedundancy(config.get<any[]>('redundancy.videos.strategies'))
     }
   },
@@ -94,6 +94,11 @@ const CONFIG = {
     PRIVATE: config.get<boolean>('tracker.private'),
     REJECT_TOO_MANY_ANNOUNCES: config.get<boolean>('tracker.reject_too_many_announces')
   },
+  HISTORY: {
+    VIDEOS: {
+      MAX_AGE: parseDurationToMs(config.get('history.videos.max_age'))
+    }
+  },
   ADMIN: {
     get EMAIL () { return config.get<string>('admin.email') }
   },
@@ -216,7 +221,7 @@ function buildVideosRedundancy (objs: any[]): VideosRedundancy[] {
 
   return objs.map(obj => {
     return Object.assign({}, obj, {
-      minLifetime: parseDuration(obj.min_lifetime),
+      minLifetime: parseDurationToMs(obj.min_lifetime),
       size: bytes.parse(obj.size),
       minViews: obj.min_views
     })
diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts
index a0609d7cd..f008cd291 100644
--- a/server/initializers/constants.ts
+++ b/server/initializers/constants.ts
@@ -158,12 +158,12 @@ const JOB_REQUEST_TIMEOUT = 3000 // 3 seconds
 const JOB_COMPLETED_LIFETIME = 60000 * 60 * 24 * 2 // 2 days
 const VIDEO_IMPORT_TIMEOUT = 1000 * 3600 // 1 hour
 
-// 1 hour
-let SCHEDULER_INTERVALS_MS = {
+const SCHEDULER_INTERVALS_MS = {
   actorFollowScores: 60000 * 60, // 1 hour
   removeOldJobs: 60000 * 60, // 1 hour
   updateVideos: 60000, // 1 minute
-  youtubeDLUpdate: 60000 * 60 * 24 // 1 day
+  youtubeDLUpdate: 60000 * 60 * 24, // 1 day
+  removeOldHistory: 60000 * 60 * 24 // 1 day
 }
 
 // ---------------------------------------------------------------------------
@@ -591,6 +591,7 @@ if (isTestInstance() === true) {
 
   SCHEDULER_INTERVALS_MS.actorFollowScores = 1000
   SCHEDULER_INTERVALS_MS.removeOldJobs = 10000
+  SCHEDULER_INTERVALS_MS.removeOldHistory = 5000
   SCHEDULER_INTERVALS_MS.updateVideos = 5000
   REPEAT_JOBS[ 'videos-views' ] = { every: 5000 }
 
@@ -734,7 +735,7 @@ function buildVideosExtname () {
 }
 
 function loadLanguages () {
-  VIDEO_LANGUAGES = buildLanguages()
+  Object.assign(VIDEO_LANGUAGES, buildLanguages())
 }
 
 function buildLanguages () {
diff --git a/server/lib/schedulers/abstract-scheduler.ts b/server/lib/schedulers/abstract-scheduler.ts
index 86ea7aa38..0e6088911 100644
--- a/server/lib/schedulers/abstract-scheduler.ts
+++ b/server/lib/schedulers/abstract-scheduler.ts
@@ -1,4 +1,5 @@
 import { logger } from '../../helpers/logger'
+import * as Bluebird from 'bluebird'
 
 export abstract class AbstractScheduler {
 
@@ -30,5 +31,5 @@ export abstract class AbstractScheduler {
     }
   }
 
-  protected abstract internalExecute (): Promise<any>
+  protected abstract internalExecute (): Promise<any> | Bluebird<any>
 }
diff --git a/server/lib/schedulers/remove-old-history-scheduler.ts b/server/lib/schedulers/remove-old-history-scheduler.ts
new file mode 100644
index 000000000..1b5ff8394
--- /dev/null
+++ b/server/lib/schedulers/remove-old-history-scheduler.ts
@@ -0,0 +1,32 @@
+import { logger } from '../../helpers/logger'
+import { AbstractScheduler } from './abstract-scheduler'
+import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants'
+import { UserVideoHistoryModel } from '../../models/account/user-video-history'
+import { CONFIG } from '../../initializers/config'
+import { isTestInstance } from '../../helpers/core-utils'
+
+export class RemoveOldHistoryScheduler extends AbstractScheduler {
+
+  private static instance: AbstractScheduler
+
+  protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.removeOldHistory
+
+  private constructor () {
+    super()
+  }
+
+  protected internalExecute () {
+    if (CONFIG.HISTORY.VIDEOS.MAX_AGE === -1) return
+
+    logger.info('Removing old videos history.')
+
+    const now = new Date()
+    const beforeDate = new Date(now.getTime() - CONFIG.HISTORY.VIDEOS.MAX_AGE).toISOString()
+
+    return UserVideoHistoryModel.removeOldHistory(beforeDate)
+  }
+
+  static get Instance () {
+    return this.instance || (this.instance = new this())
+  }
+}
diff --git a/server/middlewares/cache.ts b/server/middlewares/cache.ts
index 8ffe75700..e83d8d569 100644
--- a/server/middlewares/cache.ts
+++ b/server/middlewares/cache.ts
@@ -1,6 +1,6 @@
 import * as express from 'express'
 import * as AsyncLock from 'async-lock'
-import { parseDuration } from '../helpers/core-utils'
+import { parseDurationToMs } from '../helpers/core-utils'
 import { Redis } from '../lib/redis'
 import { logger } from '../helpers/logger'
 
@@ -24,7 +24,7 @@ function cacheRoute (lifetimeArg: string | number) {
           res.send = (body) => {
             if (res.statusCode >= 200 && res.statusCode < 400) {
               const contentType = res.get('content-type')
-              const lifetime = parseDuration(lifetimeArg)
+              const lifetime = parseDurationToMs(lifetimeArg)
 
               Redis.Instance.setCachedRoute(req, body, lifetime, contentType, res.statusCode)
                    .then(() => done())
diff --git a/server/models/account/user-video-history.ts b/server/models/account/user-video-history.ts
index 15cb399c9..49d2def81 100644
--- a/server/models/account/user-video-history.ts
+++ b/server/models/account/user-video-history.ts
@@ -67,7 +67,7 @@ export class UserVideoHistoryModel extends Model<UserVideoHistoryModel> {
     })
   }
 
-  static removeHistoryBefore (user: UserModel, beforeDate: string, t: Transaction) {
+  static removeUserHistoryBefore (user: UserModel, beforeDate: string, t: Transaction) {
     const query: DestroyOptions = {
       where: {
         userId: user.id
@@ -83,4 +83,16 @@ export class UserVideoHistoryModel extends Model<UserVideoHistoryModel> {
 
     return UserVideoHistoryModel.destroy(query)
   }
+
+  static removeOldHistory (beforeDate: string) {
+    const query: DestroyOptions = {
+      where: {
+        updatedAt: {
+          [Op.lt]: beforeDate
+        }
+      }
+    }
+
+    return UserVideoHistoryModel.destroy(query)
+  }
 }
diff --git a/server/tests/api/videos/videos-history.ts b/server/tests/api/videos/videos-history.ts
index f654a422b..f7d3a6aeb 100644
--- a/server/tests/api/videos/videos-history.ts
+++ b/server/tests/api/videos/videos-history.ts
@@ -7,14 +7,15 @@ import {
   flushTests,
   getVideosListWithToken,
   getVideoWithToken,
-  killallServers,
+  killallServers, reRunServer,
   runServer,
   searchVideoWithToken,
   ServerInfo,
   setAccessTokensToServers,
   updateMyUser,
   uploadVideo,
-  userLogin
+  userLogin,
+  wait
 } from '../../../../shared/utils'
 import { Video, VideoDetails } from '../../../../shared/models/videos'
 import { listMyVideosHistory, removeMyVideosHistory, userWatchVideo } from '../../../../shared/utils/videos/video-history'
@@ -192,6 +193,35 @@ describe('Test videos history', function () {
     expect(videos[1].name).to.equal('video 3')
   })
 
+  it('Should not clean old history', async function () {
+    this.timeout(50000)
+
+    killallServers([ server ])
+
+    await reRunServer(server, { history: { videos: { max_age: '10 days' } } })
+
+    await wait(6000)
+
+    // Should still have history
+
+    const res = await listMyVideosHistory(server.url, server.accessToken)
+
+    expect(res.body.total).to.equal(2)
+  })
+
+  it('Should clean old history', async function () {
+    this.timeout(50000)
+
+    killallServers([ server ])
+
+    await reRunServer(server, { history: { videos: { max_age: '5 seconds' } } })
+
+    await wait(6000)
+
+    const res = await listMyVideosHistory(server.url, server.accessToken)
+    expect(res.body.total).to.equal(0)
+  })
+
   after(async function () {
     killallServers([ server ])