Add ability to delete old remote views

8 files changed, 173 insertions, 2 deletions

diff --git a/server/controllers/api/server/logs.ts b/server/controllers/api/server/logs.ts
index 03941cca7..e9d1f2efd 100644
--- a/server/controllers/api/server/logs.ts
+++ b/server/controllers/api/server/logs.ts
@@ -1,7 +1,7 @@
 import * as express from 'express'
 import { UserRight } from '../../../../shared/models/users'
 import { asyncMiddleware, authenticate, ensureUserHasRight } from '../../../middlewares'
-import { mtimeSortFilesDesc } from '../../../../shared/utils/logs/logs'
+import { mtimeSortFilesDesc } from '../../../../shared/core-utils/logs/logs'
 import { readdir, readFile } from 'fs-extra'
 import { MAX_LOGS_OUTPUT_CHARACTERS } from '../../../initializers/constants'
 import { join } from 'path'
diff --git a/server/initializers/checker-before-init.ts b/server/initializers/checker-before-init.ts
index 6b43debfb..223ef8078 100644
--- a/server/initializers/checker-before-init.ts
+++ b/server/initializers/checker-before-init.ts
@@ -26,7 +26,8 @@ function checkMissedConfig () {
     'instance.is_nsfw', 'instance.default_nsfw_policy', 'instance.robots', 'instance.securitytxt',
     'services.twitter.username', 'services.twitter.whitelisted',
     'followers.instance.enabled', 'followers.instance.manual_approval',
-    'tracker.enabled', 'tracker.private', 'tracker.reject_too_many_announces'
+    'tracker.enabled', 'tracker.private', 'tracker.reject_too_many_announces',
+    'history.videos.max_age', 'views.videos.remote.max_age'
   ]
   const requiredAlternatives = [
     [ // set
diff --git a/server/initializers/config.ts b/server/initializers/config.ts
index 1f374dea9..baf502305 100644
--- a/server/initializers/config.ts
+++ b/server/initializers/config.ts
@@ -99,6 +99,13 @@ const CONFIG = {
       MAX_AGE: parseDurationToMs(config.get('history.videos.max_age'))
     }
   },
+  VIEWS: {
+    VIDEOS: {
+      REMOTE: {
+        MAX_AGE: parseDurationToMs(config.get('views.videos.remote.max_age'))
+      }
+    }
+  },
   ADMIN: {
     get EMAIL () { return config.get<string>('admin.email') }
   },
diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts
index f008cd291..8f6ef1a81 100644
--- a/server/initializers/constants.ts
+++ b/server/initializers/constants.ts
@@ -163,6 +163,7 @@ const SCHEDULER_INTERVALS_MS = {
   removeOldJobs: 60000 * 60, // 1 hour
   updateVideos: 60000, // 1 minute
   youtubeDLUpdate: 60000 * 60 * 24, // 1 day
+  removeOldViews: 60000 * 60 * 24, // 1 day
   removeOldHistory: 60000 * 60 * 24 // 1 day
 }
 
@@ -592,6 +593,7 @@ if (isTestInstance() === true) {
   SCHEDULER_INTERVALS_MS.actorFollowScores = 1000
   SCHEDULER_INTERVALS_MS.removeOldJobs = 10000
   SCHEDULER_INTERVALS_MS.removeOldHistory = 5000
+  SCHEDULER_INTERVALS_MS.removeOldViews = 5000
   SCHEDULER_INTERVALS_MS.updateVideos = 5000
   REPEAT_JOBS[ 'videos-views' ] = { every: 5000 }
 
diff --git a/server/lib/schedulers/remove-old-views-scheduler.ts b/server/lib/schedulers/remove-old-views-scheduler.ts
new file mode 100644
index 000000000..39fbb9163
--- /dev/null
+++ b/server/lib/schedulers/remove-old-views-scheduler.ts
@@ -0,0 +1,33 @@
+import { logger } from '../../helpers/logger'
+import { AbstractScheduler } from './abstract-scheduler'
+import { SCHEDULER_INTERVALS_MS } from '../../initializers/constants'
+import { UserVideoHistoryModel } from '../../models/account/user-video-history'
+import { CONFIG } from '../../initializers/config'
+import { isTestInstance } from '../../helpers/core-utils'
+import { VideoViewModel } from '../../models/video/video-views'
+
+export class RemoveOldViewsScheduler extends AbstractScheduler {
+
+  private static instance: AbstractScheduler
+
+  protected schedulerIntervalMs = SCHEDULER_INTERVALS_MS.removeOldViews
+
+  private constructor () {
+    super()
+  }
+
+  protected internalExecute () {
+    if (CONFIG.VIEWS.VIDEOS.REMOTE.MAX_AGE === -1) return
+
+    logger.info('Removing old videos views.')
+
+    const now = new Date()
+    const beforeDate = new Date(now.getTime() - CONFIG.VIEWS.VIDEOS.REMOTE.MAX_AGE).toISOString()
+
+    return VideoViewModel.removeOldRemoteViewsHistory(beforeDate)
+  }
+
+  static get Instance () {
+    return this.instance || (this.instance = new this())
+  }
+}
diff --git a/server/models/video/video-views.ts b/server/models/video/video-views.ts
index fde5f7056..6071e8c22 100644
--- a/server/models/video/video-views.ts
+++ b/server/models/video/video-views.ts
@@ -41,4 +41,18 @@ export class VideoViewModel extends Model<VideoViewModel> {
   })
   Video: VideoModel
 
+  static removeOldRemoteViewsHistory (beforeDate: string) {
+    const query = {
+      where: {
+        startDate: {
+          [Sequelize.Op.lt]: beforeDate
+        },
+        videoId: {
+          [Sequelize.Op.in]: Sequelize.literal('(SELECT "id" FROM "video" WHERE "remote" IS TRUE)')
+        }
+      }
+    }
+
+    return VideoViewModel.destroy(query)
+  }
 }
diff --git a/server/tests/api/videos/index.ts b/server/tests/api/videos/index.ts
index 4be12ad15..93e1f3e98 100644
--- a/server/tests/api/videos/index.ts
+++ b/server/tests/api/videos/index.ts
@@ -18,3 +18,4 @@ import './video-transcoder'
 import './videos-filter'
 import './videos-history'
 import './videos-overview'
+import './videos-views-cleaner'
diff --git a/server/tests/api/videos/videos-views-cleaner.ts b/server/tests/api/videos/videos-views-cleaner.ts
new file mode 100644
index 000000000..9f268c8e6
--- /dev/null
+++ b/server/tests/api/videos/videos-views-cleaner.ts
@@ -0,0 +1,113 @@
+/* tslint:disable:no-unused-expression */
+
+import * as chai from 'chai'
+import 'mocha'
+import {
+  flushAndRunMultipleServers,
+  flushTests,
+  killallServers,
+  reRunServer,
+  runServer,
+  ServerInfo,
+  setAccessTokensToServers,
+  uploadVideo, uploadVideoAndGetId, viewVideo, wait, countVideoViewsOf, doubleFollow, waitJobs
+} from '../../../../shared/utils'
+import { getVideosOverview } from '../../../../shared/utils/overviews/overviews'
+import { VideosOverview } from '../../../../shared/models/overviews'
+import { listMyVideosHistory } from '../../../../shared/utils/videos/video-history'
+
+const expect = chai.expect
+
+describe('Test video views cleaner', function () {
+  let servers: ServerInfo[]
+
+  let videoIdServer1: string
+  let videoIdServer2: string
+
+  before(async function () {
+    this.timeout(50000)
+
+    await flushTests()
+
+    servers = await flushAndRunMultipleServers(2)
+    await setAccessTokensToServers(servers)
+
+    await doubleFollow(servers[0], servers[1])
+
+    videoIdServer1 = (await uploadVideoAndGetId({ server: servers[0], videoName: 'video server 1' })).uuid
+    videoIdServer2 = (await uploadVideoAndGetId({ server: servers[1], videoName: 'video server 2' })).uuid
+
+    await waitJobs(servers)
+
+    await viewVideo(servers[0].url, videoIdServer1)
+    await viewVideo(servers[1].url, videoIdServer1)
+    await viewVideo(servers[0].url, videoIdServer2)
+    await viewVideo(servers[1].url, videoIdServer2)
+
+    await waitJobs(servers)
+  })
+
+  it('Should not clean old video views', async function () {
+    this.timeout(50000)
+
+    killallServers([ servers[0] ])
+
+    await reRunServer(servers[0], { views: { videos: { remote: { max_age: '10 days' } } } })
+
+    await wait(6000)
+
+    // Should still have views
+
+    {
+      for (const server of servers) {
+        const total = await countVideoViewsOf(server.serverNumber, videoIdServer1)
+        expect(total).to.equal(2)
+      }
+    }
+
+    {
+      for (const server of servers) {
+        const total = await countVideoViewsOf(server.serverNumber, videoIdServer2)
+        expect(total).to.equal(2)
+      }
+    }
+  })
+
+  it('Should clean old video views', async function () {
+    this.timeout(50000)
+
+    this.timeout(50000)
+
+    killallServers([ servers[0] ])
+
+    await reRunServer(servers[0], { views: { videos: { remote: { max_age: '5 seconds' } } } })
+
+    await wait(6000)
+
+    // Should still have views
+
+    {
+      for (const server of servers) {
+        const total = await countVideoViewsOf(server.serverNumber, videoIdServer1)
+        expect(total).to.equal(2)
+      }
+    }
+
+    {
+      const totalServer1 = await countVideoViewsOf(servers[0].serverNumber, videoIdServer2)
+      expect(totalServer1).to.equal(0)
+
+      const totalServer2 = await countVideoViewsOf(servers[1].serverNumber, videoIdServer2)
+      expect(totalServer2).to.equal(2)
+    }
+  })
+
+  after(async function () {
+    killallServers(servers)
+
+    // Keep the logs if the test failed
+    if (this['ok']) {
+      await flushTests()
+    }
+  })
+})