Support video views/viewers stats in server

 * Add "currentTime" and "event" body params to view endpoint
 * Merge watching and view endpoints
 * Introduce WatchAction AP activity
 * Add tables to store viewer information of local videos
 * Add endpoints to fetch video views/viewers stats of local videos
 * Refactor views/viewers handlers
 * Support "views" and "viewers" counters for both VOD and live videos

9 files changed, 156 insertions, 25 deletions

diff --git a/server/lib/activitypub/activity.ts b/server/lib/activitypub/activity.ts
index cccb7b1c1..e6cec1ba7 100644
--- a/server/lib/activitypub/activity.ts
+++ b/server/lib/activitypub/activity.ts
@@ -4,6 +4,17 @@ function getAPId (object: string | { id: string }) {
   return object.id
 }
 
+function getActivityStreamDuration (duration: number) {
+  // https://www.w3.org/TR/activitystreams-vocabulary/#dfn-duration
+  return 'PT' + duration + 'S'
+}
+
+function getDurationFromActivityStream (duration: string) {
+  return parseInt(duration.replace(/[^\d]+/, ''))
+}
+
 export {
-  getAPId
+  getAPId,
+  getActivityStreamDuration,
+  getDurationFromActivityStream
 }
diff --git a/server/lib/activitypub/context.ts b/server/lib/activitypub/context.ts
index 3bc40e2aa..b452cf9b3 100644
--- a/server/lib/activitypub/context.ts
+++ b/server/lib/activitypub/context.ts
@@ -15,7 +15,7 @@ export {
 
 type ContextValue = { [ id: string ]: (string | { '@type': string, '@id': string }) }
 
-const contextStore = {
+const contextStore: { [ id in ContextType ]: (string | { [ id: string ]: string })[] } = {
   Video: buildContext({
     Hashtag: 'as:Hashtag',
     uuid: 'sc:identifier',
@@ -109,7 +109,8 @@ const contextStore = {
     stopTimestamp: {
       '@type': 'sc:Number',
       '@id': 'pt:stopTimestamp'
-    }
+    },
+    uuid: 'sc:identifier'
   }),
 
   CacheFile: buildContext({
@@ -128,6 +129,24 @@ const contextStore = {
     }
   }),
 
+  WatchAction: buildContext({
+    WatchAction: 'sc:WatchAction',
+    startTimestamp: {
+      '@type': 'sc:Number',
+      '@id': 'pt:startTimestamp'
+    },
+    stopTimestamp: {
+      '@type': 'sc:Number',
+      '@id': 'pt:stopTimestamp'
+    },
+    watchSection: {
+      '@type': 'sc:Number',
+      '@id': 'pt:stopTimestamp'
+    },
+    uuid: 'sc:identifier'
+  }),
+
+  Collection: buildContext(),
   Follow: buildContext(),
   Reject: buildContext(),
   Accept: buildContext(),
diff --git a/server/lib/activitypub/local-video-viewer.ts b/server/lib/activitypub/local-video-viewer.ts
new file mode 100644
index 000000000..738083adc
--- /dev/null
+++ b/server/lib/activitypub/local-video-viewer.ts
@@ -0,0 +1,42 @@
+import { Transaction } from 'sequelize'
+import { LocalVideoViewerModel } from '@server/models/view/local-video-viewer'
+import { LocalVideoViewerWatchSectionModel } from '@server/models/view/local-video-viewer-watch-section'
+import { MVideo } from '@server/types/models'
+import { WatchActionObject } from '@shared/models'
+import { getDurationFromActivityStream } from './activity'
+
+async function createOrUpdateLocalVideoViewer (watchAction: WatchActionObject, video: MVideo, t: Transaction) {
+  const stats = await LocalVideoViewerModel.loadByUrl(watchAction.id)
+  if (stats) await stats.destroy({ transaction: t })
+
+  const localVideoViewer = await LocalVideoViewerModel.create({
+    url: watchAction.id,
+    uuid: watchAction.uuid,
+
+    watchTime: getDurationFromActivityStream(watchAction.duration),
+
+    startDate: new Date(watchAction.startTime),
+    endDate: new Date(watchAction.endTime),
+
+    country: watchAction.location
+      ? watchAction.location.addressCountry
+      : null,
+
+    videoId: video.id
+  })
+
+  await LocalVideoViewerWatchSectionModel.bulkCreateSections({
+    localVideoViewerId: localVideoViewer.id,
+
+    watchSections: watchAction.watchSections.map(s => ({
+      start: s.startTimestamp,
+      end: s.endTimestamp
+    }))
+  })
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+  createOrUpdateLocalVideoViewer
+}
diff --git a/server/lib/activitypub/process/process-create.ts b/server/lib/activitypub/process/process-create.ts
index b5b1a0feb..3e7931bb2 100644
--- a/server/lib/activitypub/process/process-create.ts
+++ b/server/lib/activitypub/process/process-create.ts
@@ -1,6 +1,7 @@
 import { isBlockedByServerOrAccount } from '@server/lib/blocklist'
 import { isRedundancyAccepted } from '@server/lib/redundancy'
-import { ActivityCreate, CacheFileObject, PlaylistObject, VideoCommentObject, VideoObject } from '@shared/models'
+import { VideoModel } from '@server/models/video/video'
+import { ActivityCreate, CacheFileObject, PlaylistObject, VideoCommentObject, VideoObject, WatchActionObject } from '@shared/models'
 import { retryTransactionWrapper } from '../../../helpers/database-utils'
 import { logger } from '../../../helpers/logger'
 import { sequelizeTypescript } from '../../../initializers/database'
@@ -8,6 +9,7 @@ import { APProcessorOptions } from '../../../types/activitypub-processor.model'
 import { MActorSignature, MCommentOwnerVideo, MVideoAccountLightBlacklistAllFiles } from '../../../types/models'
 import { Notifier } from '../../notifier'
 import { createOrUpdateCacheFile } from '../cache-file'
+import { createOrUpdateLocalVideoViewer } from '../local-video-viewer'
 import { createOrUpdateVideoPlaylist } from '../playlists'
 import { forwardVideoRelatedActivity } from '../send/shared/send-utils'
 import { resolveThread } from '../video-comments'
@@ -32,6 +34,10 @@ async function processCreateActivity (options: APProcessorOptions<ActivityCreate
     return retryTransactionWrapper(processCreateVideoComment, activity, byActor, notify)
   }
 
+  if (activityType === 'WatchAction') {
+    return retryTransactionWrapper(processCreateWatchAction, activity)
+  }
+
   if (activityType === 'CacheFile') {
     return retryTransactionWrapper(processCreateCacheFile, activity, byActor)
   }
@@ -81,6 +87,19 @@ async function processCreateCacheFile (activity: ActivityCreate, byActor: MActor
   }
 }
 
+async function processCreateWatchAction (activity: ActivityCreate) {
+  const watchAction = activity.object as WatchActionObject
+
+  if (watchAction.actionStatus !== 'CompletedActionStatus') return
+
+  const video = await VideoModel.loadByUrl(watchAction.object)
+  if (video.remote) return
+
+  await sequelizeTypescript.transaction(async t => {
+    return createOrUpdateLocalVideoViewer(watchAction, video, t)
+  })
+}
+
 async function processCreateVideoComment (activity: ActivityCreate, byActor: MActorSignature, notify: boolean) {
   const commentObject = activity.object as VideoCommentObject
   const byAccount = byActor.Account
diff --git a/server/lib/activitypub/process/process-view.ts b/server/lib/activitypub/process/process-view.ts
index c59940164..bad079843 100644
--- a/server/lib/activitypub/process/process-view.ts
+++ b/server/lib/activitypub/process/process-view.ts
@@ -1,4 +1,4 @@
-import { VideoViews } from '@server/lib/video-views'
+import { VideoViewsManager } from '@server/lib/views/video-views-manager'
 import { ActivityView } from '../../../../shared/models/activitypub'
 import { APProcessorOptions } from '../../../types/activitypub-processor.model'
 import { MActorSignature } from '../../../types/models'
@@ -32,7 +32,7 @@ async function processCreateView (activity: ActivityView, byActor: MActorSignatu
     ? new Date(activity.expires)
     : undefined
 
-  await VideoViews.Instance.processView({ video, ip: null, viewerExpires })
+  await VideoViewsManager.Instance.processRemoteView({ video, viewerExpires })
 
   if (video.isOwned()) {
     // Forward the view but don't resend the activity to the sender
diff --git a/server/lib/activitypub/send/send-create.ts b/server/lib/activitypub/send/send-create.ts
index 5d8763495..7c3a6bdd0 100644
--- a/server/lib/activitypub/send/send-create.ts
+++ b/server/lib/activitypub/send/send-create.ts
@@ -6,6 +6,7 @@ import { VideoCommentModel } from '../../../models/video/video-comment'
 import {
   MActorLight,
   MCommentOwnerVideo,
+  MLocalVideoViewerWithWatchSections,
   MVideoAccountLight,
   MVideoAP,
   MVideoPlaylistFull,
@@ -19,6 +20,7 @@ import {
   getActorsInvolvedInVideo,
   getAudienceFromFollowersOf,
   getVideoCommentAudience,
+  sendVideoActivityToOrigin,
   sendVideoRelatedActivity,
   unicastTo
 } from './shared'
@@ -61,6 +63,18 @@ async function sendCreateCacheFile (
   })
 }
 
+async function sendCreateWatchAction (stats: MLocalVideoViewerWithWatchSections, transaction: Transaction) {
+  logger.info('Creating job to send create watch action %s.', stats.url, lTags(stats.uuid))
+
+  const byActor = await getServerActor()
+
+  const activityBuilder = (audience: ActivityAudience) => {
+    return buildCreateActivity(stats.url, byActor, stats.toActivityPubObject(), audience)
+  }
+
+  return sendVideoActivityToOrigin(activityBuilder, { byActor, video: stats.Video, transaction, contextType: 'WatchAction' })
+}
+
 async function sendCreateVideoPlaylist (playlist: MVideoPlaylistFull, transaction: Transaction) {
   if (playlist.privacy === VideoPlaylistPrivacy.PRIVATE) return undefined
 
@@ -175,7 +189,8 @@ export {
   buildCreateActivity,
   sendCreateVideoComment,
   sendCreateVideoPlaylist,
-  sendCreateCacheFile
+  sendCreateCacheFile,
+  sendCreateWatchAction
 }
 
 // ---------------------------------------------------------------------------
diff --git a/server/lib/activitypub/send/send-view.ts b/server/lib/activitypub/send/send-view.ts
index 1f97307b9..1088bf258 100644
--- a/server/lib/activitypub/send/send-view.ts
+++ b/server/lib/activitypub/send/send-view.ts
@@ -1,27 +1,49 @@
 import { Transaction } from 'sequelize'
-import { VideoViews } from '@server/lib/video-views'
-import { MActorAudience, MVideoImmutable, MVideoUrl } from '@server/types/models'
+import { VideoViewsManager } from '@server/lib/views/video-views-manager'
+import { MActorAudience, MActorLight, MVideoImmutable, MVideoUrl } from '@server/types/models'
 import { ActivityAudience, ActivityView } from '@shared/models'
 import { logger } from '../../../helpers/logger'
-import { ActorModel } from '../../../models/actor/actor'
 import { audiencify, getAudience } from '../audience'
 import { getLocalVideoViewActivityPubUrl } from '../url'
 import { sendVideoRelatedActivity } from './shared/send-utils'
 
-async function sendView (byActor: ActorModel, video: MVideoImmutable, t: Transaction) {
-  logger.info('Creating job to send view of %s.', video.url)
+type ViewType = 'view' | 'viewer'
+
+async function sendView (options: {
+  byActor: MActorLight
+  type: ViewType
+  video: MVideoImmutable
+  transaction?: Transaction
+}) {
+  const { byActor, type, video, transaction } = options
+
+  logger.info('Creating job to send %s of %s.', type, video.url)
 
   const activityBuilder = (audience: ActivityAudience) => {
     const url = getLocalVideoViewActivityPubUrl(byActor, video)
 
-    return buildViewActivity(url, byActor, video, audience)
+    return buildViewActivity({ url, byActor, video, audience, type })
   }
 
-  return sendVideoRelatedActivity(activityBuilder, { byActor, video, transaction: t, contextType: 'View' })
+  return sendVideoRelatedActivity(activityBuilder, { byActor, video, transaction, contextType: 'View' })
 }
 
-function buildViewActivity (url: string, byActor: MActorAudience, video: MVideoUrl, audience?: ActivityAudience): ActivityView {
-  if (!audience) audience = getAudience(byActor)
+// ---------------------------------------------------------------------------
+
+export {
+  sendView
+}
+
+// ---------------------------------------------------------------------------
+
+function buildViewActivity (options: {
+  url: string
+  byActor: MActorAudience
+  video: MVideoUrl
+  type: ViewType
+  audience?: ActivityAudience
+}): ActivityView {
+  const { url, byActor, type, video, audience = getAudience(byActor) } = options
 
   return audiencify(
     {
@@ -29,14 +51,11 @@ function buildViewActivity (url: string, byActor: MActorAudience, video: MVideoU
       type: 'View' as 'View',
       actor: byActor.url,
       object: video.url,
-      expires: new Date(VideoViews.Instance.buildViewerExpireTime()).toISOString()
+
+      expires: type === 'viewer'
+        ? new Date(VideoViewsManager.Instance.buildViewerExpireTime()).toISOString()
+        : undefined
     },
     audience
   )
 }
-
-// ---------------------------------------------------------------------------
-
-export {
-  sendView
-}
diff --git a/server/lib/activitypub/url.ts b/server/lib/activitypub/url.ts
index 50be4fac9..8443fef4c 100644
--- a/server/lib/activitypub/url.ts
+++ b/server/lib/activitypub/url.ts
@@ -7,6 +7,7 @@ import {
   MActorId,
   MActorUrl,
   MCommentId,
+  MLocalVideoViewer,
   MVideoId,
   MVideoPlaylistElement,
   MVideoUrl,
@@ -59,6 +60,10 @@ function getLocalVideoViewActivityPubUrl (byActor: MActorUrl, video: MVideoId) {
   return byActor.url + '/views/videos/' + video.id + '/' + new Date().toISOString()
 }
 
+function getLocalVideoViewerActivityPubUrl (stats: MLocalVideoViewer) {
+  return WEBSERVER.URL + '/videos/local-viewer/' + stats.uuid
+}
+
 function getVideoLikeActivityPubUrlByLocalActor (byActor: MActorUrl, video: MVideoId) {
   return byActor.url + '/likes/' + video.id
 }
@@ -167,6 +172,7 @@ export {
   getLocalVideoCommentsActivityPubUrl,
   getLocalVideoLikesActivityPubUrl,
   getLocalVideoDislikesActivityPubUrl,
+  getLocalVideoViewerActivityPubUrl,
 
   getAbuseTargetUrl,
   checkUrlsSameHost,
diff --git a/server/lib/activitypub/videos/shared/object-to-model-attributes.ts b/server/lib/activitypub/videos/shared/object-to-model-attributes.ts
index c97217669..f02b9cba6 100644
--- a/server/lib/activitypub/videos/shared/object-to-model-attributes.ts
+++ b/server/lib/activitypub/videos/shared/object-to-model-attributes.ts
@@ -24,6 +24,7 @@ import {
   VideoPrivacy,
   VideoStreamingPlaylistType
 } from '@shared/models'
+import { getDurationFromActivityStream } from '../../activity'
 
 function getThumbnailFromIcons (videoObject: VideoObject) {
   let validIcons = videoObject.icon.filter(i => i.width > THUMBNAILS_SIZE.minWidth)
@@ -170,7 +171,6 @@ function getVideoAttributesFromObject (videoChannel: MChannelId, videoObject: Vi
     ? VideoPrivacy.PUBLIC
     : VideoPrivacy.UNLISTED
 
-  const duration = videoObject.duration.replace(/[^\d]+/, '')
   const language = videoObject.language?.identifier
 
   const category = videoObject.category
@@ -200,7 +200,7 @@ function getVideoAttributesFromObject (videoChannel: MChannelId, videoObject: Vi
     isLive: videoObject.isLiveBroadcast,
     state: videoObject.state,
     channelId: videoChannel.id,
-    duration: parseInt(duration, 10),
+    duration: getDurationFromActivityStream(videoObject.duration),
     createdAt: new Date(videoObject.published),
     publishedAt: new Date(videoObject.published),