Support video views/viewers stats in server

 * Add "currentTime" and "event" body params to view endpoint
 * Merge watching and view endpoints
 * Introduce WatchAction AP activity
 * Add tables to store viewer information of local videos
 * Add endpoints to fetch video views/viewers stats of local videos
 * Refactor views/viewers handlers
 * Support "views" and "viewers" counters for both VOD and live videos

6 files changed, 61 insertions, 175 deletions

diff --git a/server/tests/api/videos/index.ts b/server/tests/api/videos/index.ts
index 7dc826353..27b119f30 100644
--- a/server/tests/api/videos/index.ts
+++ b/server/tests/api/videos/index.ts
@@ -16,4 +16,3 @@ import './video-schedule-update'
 import './videos-common-filters'
 import './videos-history'
 import './videos-overview'
-import './videos-views-cleaner'
diff --git a/server/tests/api/videos/multiple-servers.ts b/server/tests/api/videos/multiple-servers.ts
index a9df262dc..84c1515a3 100644
--- a/server/tests/api/videos/multiple-servers.ts
+++ b/server/tests/api/videos/multiple-servers.ts
@@ -504,21 +504,22 @@ describe('Test multiple servers', function () {
     it('Should view multiple videos on owned servers', async function () {
       this.timeout(30000)
 
-      await servers[2].videos.view({ id: localVideosServer3[0] })
+      await servers[2].views.simulateView({ id: localVideosServer3[0] })
       await wait(1000)
 
-      await servers[2].videos.view({ id: localVideosServer3[0] })
-      await servers[2].videos.view({ id: localVideosServer3[1] })
+      await servers[2].views.simulateView({ id: localVideosServer3[0] })
+      await servers[2].views.simulateView({ id: localVideosServer3[1] })
 
       await wait(1000)
 
-      await servers[2].videos.view({ id: localVideosServer3[0] })
-      await servers[2].videos.view({ id: localVideosServer3[0] })
+      await servers[2].views.simulateView({ id: localVideosServer3[0] })
+      await servers[2].views.simulateView({ id: localVideosServer3[0] })
 
       await waitJobs(servers)
 
-      // Wait the repeatable job
-      await wait(6000)
+      for (const server of servers) {
+        await server.debug.sendCommand({ body: { command: 'process-video-views-buffer' } })
+      }
 
       await waitJobs(servers)
 
@@ -537,23 +538,24 @@ describe('Test multiple servers', function () {
       this.timeout(45000)
 
       const tasks: Promise<any>[] = []
-      tasks.push(servers[0].videos.view({ id: remoteVideosServer1[0] }))
-      tasks.push(servers[1].videos.view({ id: remoteVideosServer2[0] }))
-      tasks.push(servers[1].videos.view({ id: remoteVideosServer2[0] }))
-      tasks.push(servers[2].videos.view({ id: remoteVideosServer3[0] }))
-      tasks.push(servers[2].videos.view({ id: remoteVideosServer3[1] }))
-      tasks.push(servers[2].videos.view({ id: remoteVideosServer3[1] }))
-      tasks.push(servers[2].videos.view({ id: remoteVideosServer3[1] }))
-      tasks.push(servers[2].videos.view({ id: localVideosServer3[1] }))
-      tasks.push(servers[2].videos.view({ id: localVideosServer3[1] }))
-      tasks.push(servers[2].videos.view({ id: localVideosServer3[1] }))
+      tasks.push(servers[0].views.simulateView({ id: remoteVideosServer1[0] }))
+      tasks.push(servers[1].views.simulateView({ id: remoteVideosServer2[0] }))
+      tasks.push(servers[1].views.simulateView({ id: remoteVideosServer2[0] }))
+      tasks.push(servers[2].views.simulateView({ id: remoteVideosServer3[0] }))
+      tasks.push(servers[2].views.simulateView({ id: remoteVideosServer3[1] }))
+      tasks.push(servers[2].views.simulateView({ id: remoteVideosServer3[1] }))
+      tasks.push(servers[2].views.simulateView({ id: remoteVideosServer3[1] }))
+      tasks.push(servers[2].views.simulateView({ id: localVideosServer3[1] }))
+      tasks.push(servers[2].views.simulateView({ id: localVideosServer3[1] }))
+      tasks.push(servers[2].views.simulateView({ id: localVideosServer3[1] }))
 
       await Promise.all(tasks)
 
       await waitJobs(servers)
 
-      // Wait the repeatable job
-      await wait(16000)
+      for (const server of servers) {
+        await server.debug.sendCommand({ body: { command: 'process-video-views-buffer' } })
+      }
 
       await waitJobs(servers)
 
diff --git a/server/tests/api/videos/single-server.ts b/server/tests/api/videos/single-server.ts
index d37043aef..0e429fef7 100644
--- a/server/tests/api/videos/single-server.ts
+++ b/server/tests/api/videos/single-server.ts
@@ -179,22 +179,21 @@ describe('Test a single server', function () {
     it('Should have the views updated', async function () {
       this.timeout(20000)
 
-      await server.videos.view({ id: videoId })
-      await server.videos.view({ id: videoId })
-      await server.videos.view({ id: videoId })
+      await server.views.simulateView({ id: videoId })
+      await server.views.simulateView({ id: videoId })
+      await server.views.simulateView({ id: videoId })
 
       await wait(1500)
 
-      await server.videos.view({ id: videoId })
-      await server.videos.view({ id: videoId })
+      await server.views.simulateView({ id: videoId })
+      await server.views.simulateView({ id: videoId })
 
       await wait(1500)
 
-      await server.videos.view({ id: videoId })
-      await server.videos.view({ id: videoId })
+      await server.views.simulateView({ id: videoId })
+      await server.views.simulateView({ id: videoId })
 
-      // Wait the repeatable job
-      await wait(8000)
+      await server.debug.sendCommand({ body: { command: 'process-video-views-buffer' } })
 
       const video = await server.videos.get({ id: videoId })
       expect(video.views).to.equal(3)
diff --git a/server/tests/api/videos/video-channels.ts b/server/tests/api/videos/video-channels.ts
index 09a4bfa70..6f495c42d 100644
--- a/server/tests/api/videos/video-channels.ts
+++ b/server/tests/api/videos/video-channels.ts
@@ -466,8 +466,8 @@ describe('Test video channels', function () {
 
     {
       // video has been posted on channel servers[0].store.videoChannel.id since last update
-      await servers[0].videos.view({ id: videoUUID, xForwardedFor: '0.0.0.1,127.0.0.1' })
-      await servers[0].videos.view({ id: videoUUID, xForwardedFor: '0.0.0.2,127.0.0.1' })
+      await servers[0].views.simulateView({ id: videoUUID, xForwardedFor: '0.0.0.1,127.0.0.1' })
+      await servers[0].views.simulateView({ id: videoUUID, xForwardedFor: '0.0.0.2,127.0.0.1' })
 
       // Wait the repeatable job
       await wait(8000)
diff --git a/server/tests/api/videos/videos-history.ts b/server/tests/api/videos/videos-history.ts
index 8648c97f0..b1b3ff10a 100644
--- a/server/tests/api/videos/videos-history.ts
+++ b/server/tests/api/videos/videos-history.ts
@@ -3,15 +3,8 @@
 import 'mocha'
 import * as chai from 'chai'
 import { wait } from '@shared/core-utils'
-import { HttpStatusCode, Video } from '@shared/models'
-import {
-  cleanupTests,
-  createSingleServer,
-  HistoryCommand,
-  killallServers,
-  PeerTubeServer,
-  setAccessTokensToServers
-} from '@shared/server-commands'
+import { Video } from '@shared/models'
+import { cleanupTests, createSingleServer, killallServers, PeerTubeServer, setAccessTokensToServers } from '@shared/server-commands'
 
 const expect = chai.expect
 
@@ -23,7 +16,6 @@ describe('Test videos history', function () {
   let video3UUID: string
   let video3WatchedDate: Date
   let userAccessToken: string
-  let command: HistoryCommand
 
   before(async function () {
     this.timeout(30000)
@@ -32,30 +24,26 @@ describe('Test videos history', function () {
 
     await setAccessTokensToServers([ server ])
 
-    command = server.history
+    // 10 seconds long
+    const fixture = 'video_59fps.mp4'
 
     {
-      const { id, uuid } = await server.videos.upload({ attributes: { name: 'video 1' } })
+      const { id, uuid } = await server.videos.upload({ attributes: { name: 'video 1', fixture } })
       video1UUID = uuid
       video1Id = id
     }
 
     {
-      const { uuid } = await server.videos.upload({ attributes: { name: 'video 2' } })
+      const { uuid } = await server.videos.upload({ attributes: { name: 'video 2', fixture } })
       video2UUID = uuid
     }
 
     {
-      const { uuid } = await server.videos.upload({ attributes: { name: 'video 3' } })
+      const { uuid } = await server.videos.upload({ attributes: { name: 'video 3', fixture } })
       video3UUID = uuid
     }
 
-    const user = {
-      username: 'user_1',
-      password: 'super password'
-    }
-    await server.users.create({ username: user.username, password: user.password })
-    userAccessToken = await server.login.getAccessToken(user)
+    userAccessToken = await server.users.generateUserAndToken('user_1')
   })
 
   it('Should get videos, without watching history', async function () {
@@ -70,8 +58,8 @@ describe('Test videos history', function () {
   })
 
   it('Should watch the first and second video', async function () {
-    await command.watchVideo({ videoId: video2UUID, currentTime: 8 })
-    await command.watchVideo({ videoId: video1UUID, currentTime: 3 })
+    await server.views.view({ id: video2UUID, token: server.accessToken, currentTime: 8 })
+    await server.views.view({ id: video1UUID, token: server.accessToken, currentTime: 3 })
   })
 
   it('Should return the correct history when listing, searching and getting videos', async function () {
@@ -124,9 +112,9 @@ describe('Test videos history', function () {
 
   it('Should have these videos when listing my history', async function () {
     video3WatchedDate = new Date()
-    await command.watchVideo({ videoId: video3UUID, currentTime: 2 })
+    await server.views.view({ id: video3UUID, token: server.accessToken, currentTime: 2 })
 
-    const body = await command.list()
+    const body = await server.history.list()
 
     expect(body.total).to.equal(3)
 
@@ -137,14 +125,14 @@ describe('Test videos history', function () {
   })
 
   it('Should not have videos history on another user', async function () {
-    const body = await command.list({ token: userAccessToken })
+    const body = await server.history.list({ token: userAccessToken })
 
     expect(body.total).to.equal(0)
     expect(body.data).to.have.lengthOf(0)
   })
 
   it('Should be able to search through videos in my history', async function () {
-    const body = await command.list({ search: '2' })
+    const body = await server.history.list({ search: '2' })
     expect(body.total).to.equal(1)
 
     const videos = body.data
@@ -152,11 +140,11 @@ describe('Test videos history', function () {
   })
 
   it('Should clear my history', async function () {
-    await command.removeAll({ beforeDate: video3WatchedDate.toISOString() })
+    await server.history.removeAll({ beforeDate: video3WatchedDate.toISOString() })
   })
 
   it('Should have my history cleared', async function () {
-    const body = await command.list()
+    const body = await server.history.list()
     expect(body.total).to.equal(1)
 
     const videos = body.data
@@ -168,7 +156,10 @@ describe('Test videos history', function () {
       videosHistoryEnabled: false
     })
 
-    await command.watchVideo({ videoId: video2UUID, currentTime: 8, expectedStatus: HttpStatusCode.CONFLICT_409 })
+    await server.views.view({ id: video2UUID, token: server.accessToken, currentTime: 8 })
+
+    const { data } = await server.history.list()
+    expect(data[0].name).to.not.equal('video 2')
   })
 
   it('Should re-enable videos history', async function () {
@@ -176,14 +167,10 @@ describe('Test videos history', function () {
       videosHistoryEnabled: true
     })
 
-    await command.watchVideo({ videoId: video1UUID, currentTime: 8 })
+    await server.views.view({ id: video2UUID, token: server.accessToken, currentTime: 8 })
 
-    const body = await command.list()
-    expect(body.total).to.equal(2)
-
-    const videos = body.data
-    expect(videos[0].name).to.equal('video 1')
-    expect(videos[1].name).to.equal('video 3')
+    const { data } = await server.history.list()
+    expect(data[0].name).to.equal('video 2')
   })
 
   it('Should not clean old history', async function () {
@@ -197,7 +184,7 @@ describe('Test videos history', function () {
 
     // Should still have history
 
-    const body = await command.list()
+    const body = await server.history.list()
     expect(body.total).to.equal(2)
   })
 
@@ -210,25 +197,25 @@ describe('Test videos history', function () {
 
     await wait(6000)
 
-    const body = await command.list()
+    const body = await server.history.list()
     expect(body.total).to.equal(0)
   })
 
   it('Should delete a specific history element', async function () {
     {
-      await command.watchVideo({ videoId: video1UUID, currentTime: 4 })
-      await command.watchVideo({ videoId: video2UUID, currentTime: 8 })
+      await server.views.view({ id: video1UUID, token: server.accessToken, currentTime: 4 })
+      await server.views.view({ id: video2UUID, token: server.accessToken, currentTime: 8 })
     }
 
     {
-      const body = await command.list()
+      const body = await server.history.list()
       expect(body.total).to.equal(2)
     }
 
     {
-      await command.removeElement({ videoId: video1Id })
+      await server.history.removeElement({ videoId: video1Id })
 
-      const body = await command.list()
+      const body = await server.history.list()
       expect(body.total).to.equal(1)
       expect(body.data[0].uuid).to.equal(video2UUID)
     }
diff --git a/server/tests/api/videos/videos-views-cleaner.ts b/server/tests/api/videos/videos-views-cleaner.ts
deleted file mode 100644
index e6815a4a8..000000000
--- a/server/tests/api/videos/videos-views-cleaner.ts
+++ /dev/null
@@ -1,101 +0,0 @@
-/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
-
-import 'mocha'
-import * as chai from 'chai'
-import { wait } from '@shared/core-utils'
-import {
-  cleanupTests,
-  createMultipleServers,
-  doubleFollow,
-  killallServers,
-  PeerTubeServer,
-  setAccessTokensToServers,
-  waitJobs
-} from '@shared/server-commands'
-
-const expect = chai.expect
-
-describe('Test video views cleaner', function () {
-  let servers: PeerTubeServer[]
-
-  let videoIdServer1: string
-  let videoIdServer2: string
-
-  before(async function () {
-    this.timeout(120000)
-
-    servers = await createMultipleServers(2)
-    await setAccessTokensToServers(servers)
-
-    await doubleFollow(servers[0], servers[1])
-
-    videoIdServer1 = (await servers[0].videos.quickUpload({ name: 'video server 1' })).uuid
-    videoIdServer2 = (await servers[1].videos.quickUpload({ name: 'video server 2' })).uuid
-
-    await waitJobs(servers)
-
-    await servers[0].videos.view({ id: videoIdServer1 })
-    await servers[1].videos.view({ id: videoIdServer1 })
-    await servers[0].videos.view({ id: videoIdServer2 })
-    await servers[1].videos.view({ id: videoIdServer2 })
-
-    await waitJobs(servers)
-  })
-
-  it('Should not clean old video views', async function () {
-    this.timeout(50000)
-
-    await killallServers([ servers[0] ])
-
-    await servers[0].run({ views: { videos: { remote: { max_age: '10 days' } } } })
-
-    await wait(6000)
-
-    // Should still have views
-
-    {
-      for (const server of servers) {
-        const total = await server.sql.countVideoViewsOf(videoIdServer1)
-        expect(total).to.equal(2, 'Server ' + server.serverNumber + ' does not have the correct amount of views')
-      }
-    }
-
-    {
-      for (const server of servers) {
-        const total = await server.sql.countVideoViewsOf(videoIdServer2)
-        expect(total).to.equal(2, 'Server ' + server.serverNumber + ' does not have the correct amount of views')
-      }
-    }
-  })
-
-  it('Should clean old video views', async function () {
-    this.timeout(50000)
-
-    await killallServers([ servers[0] ])
-
-    await servers[0].run({ views: { videos: { remote: { max_age: '5 seconds' } } } })
-
-    await wait(6000)
-
-    // Should still have views
-
-    {
-      for (const server of servers) {
-        const total = await server.sql.countVideoViewsOf(videoIdServer1)
-        expect(total).to.equal(2)
-      }
-    }
-
-    {
-      const totalServer1 = await servers[0].sql.countVideoViewsOf(videoIdServer2)
-      expect(totalServer1).to.equal(0)
-
-      const totalServer2 = await servers[1].sql.countVideoViewsOf(videoIdServer2)
-      expect(totalServer2).to.equal(2)
-    }
-  })
-
-  after(async function () {
-    await cleanupTests(servers)
-  })
-})