Support video views/viewers stats in server

 * Add "currentTime" and "event" body params to view endpoint
 * Merge watching and view endpoints
 * Introduce WatchAction AP activity
 * Add tables to store viewer information of local videos
 * Add endpoints to fetch video views/viewers stats of local videos
 * Refactor views/viewers handlers
 * Support "views" and "viewers" counters for both VOD and live videos

6 files changed, 160 insertions, 73 deletions

diff --git a/server/controllers/activitypub/client.ts b/server/controllers/activitypub/client.ts
index d0f761009..8e064fb5b 100644
--- a/server/controllers/activitypub/client.ts
+++ b/server/controllers/activitypub/client.ts
@@ -27,7 +27,7 @@ import {
   videosShareValidator
 } from '../../middlewares'
 import { cacheRoute } from '../../middlewares/cache/cache'
-import { getAccountVideoRateValidatorFactory, videoCommentGetValidator } from '../../middlewares/validators'
+import { getAccountVideoRateValidatorFactory, getVideoLocalViewerValidator, videoCommentGetValidator } from '../../middlewares/validators'
 import { videoFileRedundancyGetValidator, videoPlaylistRedundancyGetValidator } from '../../middlewares/validators/redundancy'
 import { videoPlaylistElementAPGetValidator, videoPlaylistsGetValidator } from '../../middlewares/validators/videos/video-playlists'
 import { AccountModel } from '../../models/account/account'
@@ -175,6 +175,12 @@ activityPubClientRouter.get('/video-playlists/:playlistId/videos/:playlistElemen
   videoPlaylistElementController
 )
 
+activityPubClientRouter.get('/videos/local-viewer/:localViewerId',
+  executeIfActivityPub,
+  asyncMiddleware(getVideoLocalViewerValidator),
+  getVideoLocalViewerController
+)
+
 // ---------------------------------------------------------------------------
 
 export {
@@ -399,6 +405,12 @@ function videoPlaylistElementController (req: express.Request, res: express.Resp
   return activityPubResponse(activityPubContextify(json, 'Playlist'), res)
 }
 
+function getVideoLocalViewerController (req: express.Request, res: express.Response) {
+  const localViewer = res.locals.localViewerFull
+
+  return activityPubResponse(activityPubContextify(localViewer.toActivityPubObject(), 'WatchAction'), res)
+}
+
 // ---------------------------------------------------------------------------
 
 function actorFollowing (req: express.Request, actor: MActorId) {
diff --git a/server/controllers/api/server/debug.ts b/server/controllers/api/server/debug.ts
index 093e6a03c..6b6ff027c 100644
--- a/server/controllers/api/server/debug.ts
+++ b/server/controllers/api/server/debug.ts
@@ -1,6 +1,8 @@
 import express from 'express'
 import { InboxManager } from '@server/lib/activitypub/inbox-manager'
 import { RemoveDanglingResumableUploadsScheduler } from '@server/lib/schedulers/remove-dangling-resumable-uploads-scheduler'
+import { VideoViewsBufferScheduler } from '@server/lib/schedulers/video-views-buffer-scheduler'
+import { VideoViewsManager } from '@server/lib/views/video-views-manager'
 import { Debug, SendDebugCommand } from '@shared/models'
 import { HttpStatusCode } from '../../../../shared/models/http/http-error-codes'
 import { UserRight } from '../../../../shared/models/users'
@@ -38,9 +40,13 @@ function getDebug (req: express.Request, res: express.Response) {
 async function runCommand (req: express.Request, res: express.Response) {
   const body: SendDebugCommand = req.body
 
-  if (body.command === 'remove-dandling-resumable-uploads') {
-    await RemoveDanglingResumableUploadsScheduler.Instance.execute()
+  const processors: { [id in SendDebugCommand['command']]: () => Promise<any> } = {
+    'remove-dandling-resumable-uploads': () => RemoveDanglingResumableUploadsScheduler.Instance.execute(),
+    'process-video-views-buffer': () => VideoViewsBufferScheduler.Instance.execute(),
+    'process-video-viewers': () => VideoViewsManager.Instance.processViewers()
   }
 
+  await processors[body.command]()
+
   return res.status(HttpStatusCode.NO_CONTENT_204).end()
 }
diff --git a/server/controllers/api/videos/index.ts b/server/controllers/api/videos/index.ts
index c7617093c..be233722c 100644
--- a/server/controllers/api/videos/index.ts
+++ b/server/controllers/api/videos/index.ts
@@ -1,7 +1,6 @@
 import express from 'express'
 import { pickCommonVideoQuery } from '@server/helpers/query'
 import { doJSONRequest } from '@server/helpers/requests'
-import { VideoViews } from '@server/lib/video-views'
 import { openapiOperationDoc } from '@server/middlewares/doc'
 import { getServerActor } from '@server/models/application/application'
 import { guessAdditionalAttributesFromQuery } from '@server/models/video/formatter/video-format-utils'
@@ -13,7 +12,6 @@ import { logger } from '../../../helpers/logger'
 import { getFormattedObjects } from '../../../helpers/utils'
 import { REMOTE_SCHEME, VIDEO_CATEGORIES, VIDEO_LANGUAGES, VIDEO_LICENCES, VIDEO_PRIVACIES } from '../../../initializers/constants'
 import { sequelizeTypescript } from '../../../initializers/database'
-import { sendView } from '../../../lib/activitypub/send/send-view'
 import { JobQueue } from '../../../lib/job-queue'
 import { Hooks } from '../../../lib/plugins/hooks'
 import {
@@ -35,28 +33,30 @@ import { VideoModel } from '../../../models/video/video'
 import { blacklistRouter } from './blacklist'
 import { videoCaptionsRouter } from './captions'
 import { videoCommentRouter } from './comment'
-import { studioRouter } from './studio'
 import { filesRouter } from './files'
 import { videoImportsRouter } from './import'
 import { liveRouter } from './live'
 import { ownershipVideoRouter } from './ownership'
 import { rateVideoRouter } from './rate'
+import { statsRouter } from './stats'
+import { studioRouter } from './studio'
 import { transcodingRouter } from './transcoding'
 import { updateRouter } from './update'
 import { uploadRouter } from './upload'
-import { watchingRouter } from './watching'
+import { viewRouter } from './view'
 
 const auditLogger = auditLoggerFactory('videos')
 const videosRouter = express.Router()
 
 videosRouter.use('/', blacklistRouter)
+videosRouter.use('/', statsRouter)
 videosRouter.use('/', rateVideoRouter)
 videosRouter.use('/', videoCommentRouter)
 videosRouter.use('/', studioRouter)
 videosRouter.use('/', videoCaptionsRouter)
 videosRouter.use('/', videoImportsRouter)
 videosRouter.use('/', ownershipVideoRouter)
-videosRouter.use('/', watchingRouter)
+videosRouter.use('/', viewRouter)
 videosRouter.use('/', liveRouter)
 videosRouter.use('/', uploadRouter)
 videosRouter.use('/', updateRouter)
@@ -103,11 +103,6 @@ videosRouter.get('/:id',
   asyncMiddleware(checkVideoFollowConstraints),
   getVideo
 )
-videosRouter.post('/:id/views',
-  openapiOperationDoc({ operationId: 'addView' }),
-  asyncMiddleware(videosCustomGetValidator('only-video')),
-  asyncMiddleware(viewVideo)
-)
 
 videosRouter.delete('/:id',
   openapiOperationDoc({ operationId: 'delVideo' }),
@@ -150,22 +145,6 @@ function getVideo (_req: express.Request, res: express.Response) {
   return res.json(video.toFormattedDetailsJSON())
 }
 
-async function viewVideo (req: express.Request, res: express.Response) {
-  const video = res.locals.onlyVideo
-
-  const ip = req.ip
-  const success = await VideoViews.Instance.processView({ video, ip })
-
-  if (success) {
-    const serverActor = await getServerActor()
-    await sendView(serverActor, video, undefined)
-
-    Hooks.runAction('action:api.video.viewed', { video: video, ip, req, res })
-  }
-
-  return res.status(HttpStatusCode.NO_CONTENT_204).end()
-}
-
 async function getVideoDescription (req: express.Request, res: express.Response) {
   const videoInstance = res.locals.videoAll
 
diff --git a/server/controllers/api/videos/stats.ts b/server/controllers/api/videos/stats.ts
new file mode 100644
index 000000000..5f8513e9e
--- /dev/null
+++ b/server/controllers/api/videos/stats.ts
@@ -0,0 +1,66 @@
+import express from 'express'
+import { LocalVideoViewerModel } from '@server/models/view/local-video-viewer'
+import { VideoStatsTimeserieMetric } from '@shared/models'
+import {
+  asyncMiddleware,
+  authenticate,
+  videoOverallStatsValidator,
+  videoRetentionStatsValidator,
+  videoTimeserieStatsValidator
+} from '../../../middlewares'
+
+const statsRouter = express.Router()
+
+statsRouter.get('/:videoId/stats/overall',
+  authenticate,
+  asyncMiddleware(videoOverallStatsValidator),
+  asyncMiddleware(getOverallStats)
+)
+
+statsRouter.get('/:videoId/stats/timeseries/:metric',
+  authenticate,
+  asyncMiddleware(videoTimeserieStatsValidator),
+  asyncMiddleware(getTimeserieStats)
+)
+
+statsRouter.get('/:videoId/stats/retention',
+  authenticate,
+  asyncMiddleware(videoRetentionStatsValidator),
+  asyncMiddleware(getRetentionStats)
+)
+
+// ---------------------------------------------------------------------------
+
+export {
+  statsRouter
+}
+
+// ---------------------------------------------------------------------------
+
+async function getOverallStats (req: express.Request, res: express.Response) {
+  const video = res.locals.videoAll
+
+  const stats = await LocalVideoViewerModel.getOverallStats(video)
+
+  return res.json(stats)
+}
+
+async function getRetentionStats (req: express.Request, res: express.Response) {
+  const video = res.locals.videoAll
+
+  const stats = await LocalVideoViewerModel.getRetentionStats(video)
+
+  return res.json(stats)
+}
+
+async function getTimeserieStats (req: express.Request, res: express.Response) {
+  const video = res.locals.videoAll
+  const metric = req.params.metric as VideoStatsTimeserieMetric
+
+  const stats = await LocalVideoViewerModel.getTimeserieStats({
+    video,
+    metric
+  })
+
+  return res.json(stats)
+}
diff --git a/server/controllers/api/videos/view.ts b/server/controllers/api/videos/view.ts
new file mode 100644
index 000000000..e28cf371a
--- /dev/null
+++ b/server/controllers/api/videos/view.ts
@@ -0,0 +1,68 @@
+import express from 'express'
+import { sendView } from '@server/lib/activitypub/send/send-view'
+import { Hooks } from '@server/lib/plugins/hooks'
+import { VideoViewsManager } from '@server/lib/views/video-views-manager'
+import { getServerActor } from '@server/models/application/application'
+import { MVideoId } from '@server/types/models'
+import { HttpStatusCode, VideoView } from '@shared/models'
+import { asyncMiddleware, methodsValidator, openapiOperationDoc, optionalAuthenticate, videoViewValidator } from '../../../middlewares'
+import { UserVideoHistoryModel } from '../../../models/user/user-video-history'
+
+const viewRouter = express.Router()
+
+viewRouter.all(
+  [ '/:videoId/views', '/:videoId/watching' ],
+  openapiOperationDoc({ operationId: 'addView' }),
+  methodsValidator([ 'PUT', 'POST' ]),
+  optionalAuthenticate,
+  asyncMiddleware(videoViewValidator),
+  asyncMiddleware(viewVideo)
+)
+
+// ---------------------------------------------------------------------------
+
+export {
+  viewRouter
+}
+
+// ---------------------------------------------------------------------------
+
+async function viewVideo (req: express.Request, res: express.Response) {
+  const video = res.locals.onlyVideo
+
+  const body = req.body as VideoView
+
+  const ip = req.ip
+  const { successView, successViewer } = await VideoViewsManager.Instance.processLocalView({
+    video,
+    ip,
+    currentTime: body.currentTime,
+    viewEvent: body.viewEvent
+  })
+
+  if (successView) {
+    await sendView({ byActor: await getServerActor(), video, type: 'view' })
+
+    Hooks.runAction('action:api.video.viewed', { video: video, ip, req, res })
+  }
+
+  if (successViewer) {
+    await sendView({ byActor: await getServerActor(), video, type: 'viewer' })
+  }
+
+  await updateUserHistoryIfNeeded(body, video, res)
+
+  return res.status(HttpStatusCode.NO_CONTENT_204).end()
+}
+
+async function updateUserHistoryIfNeeded (body: VideoView, video: MVideoId, res: express.Response) {
+  const user = res.locals.oauth?.token.User
+  if (!user) return
+  if (user.videosHistoryEnabled !== true) return
+
+  await UserVideoHistoryModel.upsert({
+    videoId: video.id,
+    userId: user.id,
+    currentTime: body.currentTime
+  })
+}
diff --git a/server/controllers/api/videos/watching.ts b/server/controllers/api/videos/watching.ts
deleted file mode 100644
index 3fd22caac..000000000
--- a/server/controllers/api/videos/watching.ts
+++ /dev/null
@@ -1,44 +0,0 @@
-import express from 'express'
-import { HttpStatusCode, UserWatchingVideo } from '@shared/models'
-import {
-  asyncMiddleware,
-  asyncRetryTransactionMiddleware,
-  authenticate,
-  openapiOperationDoc,
-  videoWatchingValidator
-} from '../../../middlewares'
-import { UserVideoHistoryModel } from '../../../models/user/user-video-history'
-
-const watchingRouter = express.Router()
-
-watchingRouter.put('/:videoId/watching',
-  openapiOperationDoc({ operationId: 'setProgress' }),
-  authenticate,
-  asyncMiddleware(videoWatchingValidator),
-  asyncRetryTransactionMiddleware(userWatchVideo)
-)
-
-// ---------------------------------------------------------------------------
-
-export {
-  watchingRouter
-}
-
-// ---------------------------------------------------------------------------
-
-async function userWatchVideo (req: express.Request, res: express.Response) {
-  const user = res.locals.oauth.token.User
-
-  const body: UserWatchingVideo = req.body
-  const { id: videoId } = res.locals.videoId
-
-  await UserVideoHistoryModel.upsert({
-    videoId,
-    userId: user.id,
-    currentTime: body.currentTime
-  })
-
-  return res.type('json')
-            .status(HttpStatusCode.NO_CONTENT_204)
-            .end()
-}