Improve viewer counter

More precise, avoid weird decrease, reuse an id to federate viewers

5 files changed, 214 insertions, 117 deletions

diff --git a/server/lib/views/shared/index.ts b/server/lib/views/shared/index.ts
index dd510f4e2..139471183 100644
--- a/server/lib/views/shared/index.ts
+++ b/server/lib/views/shared/index.ts
@@ -1,2 +1,3 @@
-export * from './video-viewers'
+export * from './video-viewer-counters'
+export * from './video-viewer-stats'
 export * from './video-views'
diff --git a/server/lib/views/shared/video-viewer-counters.ts b/server/lib/views/shared/video-viewer-counters.ts
new file mode 100644
index 000000000..941b62ed7
--- /dev/null
+++ b/server/lib/views/shared/video-viewer-counters.ts
@@ -0,0 +1,176 @@
+
+import { isTestInstance } from '@server/helpers/core-utils'
+import { logger, loggerTagsFactory } from '@server/helpers/logger'
+import { VIEW_LIFETIME } from '@server/initializers/constants'
+import { sendView } from '@server/lib/activitypub/send/send-view'
+import { PeerTubeSocket } from '@server/lib/peertube-socket'
+import { getServerActor } from '@server/models/application/application'
+import { VideoModel } from '@server/models/video/video'
+import { MVideo } from '@server/types/models'
+import { buildUUID, sha256 } from '@shared/extra-utils'
+
+const lTags = loggerTagsFactory('views')
+
+type Viewer = {
+  expires: number
+  id: string
+  lastFederation?: number
+}
+
+export class VideoViewerCounters {
+
+  // expires is new Date().getTime()
+  private readonly viewersPerVideo = new Map<number, Viewer[]>()
+  private readonly idToViewer = new Map<string, Viewer>()
+
+  private readonly salt = buildUUID()
+
+  private processingViewerCounters = false
+
+  constructor () {
+    setInterval(() => this.cleanViewerCounters(), VIEW_LIFETIME.VIEWER_COUNTER)
+  }
+
+  // ---------------------------------------------------------------------------
+
+  async addLocalViewer (options: {
+    video: MVideo
+    ip: string
+  }) {
+    const { video, ip } = options
+
+    logger.debug('Adding local viewer to video viewers counter %s.', video.uuid, { ...lTags(video.uuid) })
+
+    const viewerId = this.generateViewerId(ip, video.uuid)
+    const viewer = this.idToViewer.get(viewerId)
+
+    if (viewer) {
+      viewer.expires = this.buildViewerExpireTime()
+      await this.federateViewerIfNeeded(video, viewer)
+
+      return false
+    }
+
+    const newViewer = await this.addViewerToVideo({ viewerId, video })
+    await this.federateViewerIfNeeded(video, newViewer)
+
+    return true
+  }
+
+  async addRemoteViewer (options: {
+    video: MVideo
+    viewerId: string
+    viewerExpires: Date
+  }) {
+    const { video, viewerExpires, viewerId } = options
+
+    logger.debug('Adding remote viewer to video %s.', video.uuid, { ...lTags(video.uuid) })
+
+    await this.addViewerToVideo({ video, viewerExpires, viewerId })
+
+    return true
+  }
+
+  // ---------------------------------------------------------------------------
+
+  getViewers (video: MVideo) {
+    const viewers = this.viewersPerVideo.get(video.id)
+    if (!viewers) return 0
+
+    return viewers.length
+  }
+
+  buildViewerExpireTime () {
+    return new Date().getTime() + VIEW_LIFETIME.VIEWER_COUNTER
+  }
+
+  // ---------------------------------------------------------------------------
+
+  private async addViewerToVideo (options: {
+    video: MVideo
+    viewerId: string
+    viewerExpires?: Date
+  }) {
+    const { video, viewerExpires, viewerId } = options
+
+    let watchers = this.viewersPerVideo.get(video.id)
+
+    if (!watchers) {
+      watchers = []
+      this.viewersPerVideo.set(video.id, watchers)
+    }
+
+    const expires = viewerExpires
+      ? viewerExpires.getTime()
+      : this.buildViewerExpireTime()
+
+    const viewer = { id: viewerId, expires }
+    watchers.push(viewer)
+
+    this.idToViewer.set(viewerId, viewer)
+
+    await this.notifyClients(video.id, watchers.length)
+
+    return viewer
+  }
+
+  private async cleanViewerCounters () {
+    if (this.processingViewerCounters) return
+    this.processingViewerCounters = true
+
+    if (!isTestInstance()) logger.info('Cleaning video viewers.', lTags())
+
+    try {
+      for (const videoId of this.viewersPerVideo.keys()) {
+        const notBefore = new Date().getTime()
+
+        const viewers = this.viewersPerVideo.get(videoId)
+
+        // Only keep not expired viewers
+        const newViewers: Viewer[] = []
+
+        // Filter new viewers
+        for (const viewer of viewers) {
+          if (viewer.expires > notBefore) {
+            newViewers.push(viewer)
+          } else {
+            this.idToViewer.delete(viewer.id)
+          }
+        }
+
+        if (newViewers.length === 0) this.viewersPerVideo.delete(videoId)
+        else this.viewersPerVideo.set(videoId, newViewers)
+
+        await this.notifyClients(videoId, newViewers.length)
+      }
+    } catch (err) {
+      logger.error('Error in video clean viewers scheduler.', { err, ...lTags() })
+    }
+
+    this.processingViewerCounters = false
+  }
+
+  private async notifyClients (videoId: string | number, viewersLength: number) {
+    const video = await VideoModel.loadImmutableAttributes(videoId)
+    if (!video) return
+
+    PeerTubeSocket.Instance.sendVideoViewsUpdate(video, viewersLength)
+
+    logger.debug('Video viewers update for %s is %d.', video.url, viewersLength, lTags())
+  }
+
+  private generateViewerId (ip: string, videoUUID: string) {
+    return sha256(this.salt + '-' + ip + '-' + videoUUID)
+  }
+
+  private async federateViewerIfNeeded (video: MVideo, viewer: Viewer) {
+    // Federate the viewer if it's been a "long" time we did not
+    const now = new Date().getTime()
+    const federationLimit = now - (VIEW_LIFETIME.VIEWER_COUNTER / 2)
+
+    if (viewer.lastFederation && viewer.lastFederation > federationLimit) return
+
+    await sendView({ byActor: await getServerActor(), video, type: 'viewer', viewerIdentifier: viewer.id })
+    viewer.lastFederation = now
+  }
+}
diff --git a/server/lib/views/shared/video-viewers.ts b/server/lib/views/shared/video-viewer-stats.ts
index 4dad1f0e8..fd66fd5c7 100644
--- a/server/lib/views/shared/video-viewers.ts
+++ b/server/lib/views/shared/video-viewer-stats.ts
@@ -6,7 +6,6 @@ import { MAX_LOCAL_VIEWER_WATCH_SECTIONS, VIEW_LIFETIME } from '@server/initiali
 import { sequelizeTypescript } from '@server/initializers/database'
 import { sendCreateWatchAction } from '@server/lib/activitypub/send'
 import { getLocalVideoViewerActivityPubUrl } from '@server/lib/activitypub/url'
-import { PeerTubeSocket } from '@server/lib/peertube-socket'
 import { Redis } from '@server/lib/redis'
 import { VideoModel } from '@server/models/video/video'
 import { LocalVideoViewerModel } from '@server/models/view/local-video-viewer'
@@ -32,39 +31,15 @@ type LocalViewerStats = {
   videoId: number
 }
 
-export class VideoViewers {
-
-  // Values are Date().getTime()
-  private readonly viewersPerVideo = new Map<number, number[]>()
-
-  private processingViewerCounters = false
-  private processingViewerStats = false
+export class VideoViewerStats {
+  private processingViewersStats = false
 
   constructor () {
-    setInterval(() => this.cleanViewerCounters(), VIEW_LIFETIME.VIEWER_COUNTER)
-
     setInterval(() => this.processViewerStats(), VIEW_LIFETIME.VIEWER_STATS)
   }
 
   // ---------------------------------------------------------------------------
 
-  getViewers (video: MVideo) {
-    const viewers = this.viewersPerVideo.get(video.id)
-    if (!viewers) return 0
-
-    return viewers.length
-  }
-
-  buildViewerExpireTime () {
-    return new Date().getTime() + VIEW_LIFETIME.VIEWER_COUNTER
-  }
-
-  async getWatchTime (videoId: number, ip: string) {
-    const stats: LocalViewerStats = await Redis.Instance.getLocalVideoViewer({ ip, videoId })
-
-    return stats?.watchTime || 0
-  }
-
   async addLocalViewer (options: {
     video: MVideo
     currentTime: number
@@ -73,51 +48,20 @@ export class VideoViewers {
   }) {
     const { video, ip, viewEvent, currentTime } = options
 
-    logger.debug('Adding local viewer to video %s.', video.uuid, { currentTime, viewEvent, ...lTags(video.uuid) })
-
-    await this.updateLocalViewerStats({ video, viewEvent, currentTime, ip })
-
-    const viewExists = await Redis.Instance.doesVideoIPViewerExist(ip, video.uuid)
-    if (viewExists) return false
+    logger.debug('Adding local viewer to video stats %s.', video.uuid, { currentTime, viewEvent, ...lTags(video.uuid) })
 
-    await Redis.Instance.setIPVideoViewer(ip, video.uuid)
-
-    return this.addViewerToVideo({ video })
+    return this.updateLocalViewerStats({ video, viewEvent, currentTime, ip })
   }
 
-  async addRemoteViewer (options: {
-    video: MVideo
-    viewerExpires: Date
-  }) {
-    const { video, viewerExpires } = options
+  // ---------------------------------------------------------------------------
 
-    logger.debug('Adding remote viewer to video %s.', video.uuid, { ...lTags(video.uuid) })
+  async getWatchTime (videoId: number, ip: string) {
+    const stats: LocalViewerStats = await Redis.Instance.getLocalVideoViewer({ ip, videoId })
 
-    return this.addViewerToVideo({ video, viewerExpires })
+    return stats?.watchTime || 0
   }
 
-  private async addViewerToVideo (options: {
-    video: MVideo
-    viewerExpires?: Date
-  }) {
-    const { video, viewerExpires } = options
-
-    let watchers = this.viewersPerVideo.get(video.id)
-
-    if (!watchers) {
-      watchers = []
-      this.viewersPerVideo.set(video.id, watchers)
-    }
-
-    const expiration = viewerExpires
-      ? viewerExpires.getTime()
-      : this.buildViewerExpireTime()
-
-    watchers.push(expiration)
-    await this.notifyClients(video.id, watchers.length)
-
-    return true
-  }
+  // ---------------------------------------------------------------------------
 
   private async updateLocalViewerStats (options: {
     video: MVideo
@@ -170,45 +114,9 @@ export class VideoViewers {
     await Redis.Instance.setLocalVideoViewer(ip, video.id, stats)
   }
 
-  private async cleanViewerCounters () {
-    if (this.processingViewerCounters) return
-    this.processingViewerCounters = true
-
-    if (!isTestInstance()) logger.info('Cleaning video viewers.', lTags())
-
-    try {
-      for (const videoId of this.viewersPerVideo.keys()) {
-        const notBefore = new Date().getTime()
-
-        const viewers = this.viewersPerVideo.get(videoId)
-
-        // Only keep not expired viewers
-        const newViewers = viewers.filter(w => w > notBefore)
-
-        if (newViewers.length === 0) this.viewersPerVideo.delete(videoId)
-        else this.viewersPerVideo.set(videoId, newViewers)
-
-        await this.notifyClients(videoId, newViewers.length)
-      }
-    } catch (err) {
-      logger.error('Error in video clean viewers scheduler.', { err, ...lTags() })
-    }
-
-    this.processingViewerCounters = false
-  }
-
-  private async notifyClients (videoId: string | number, viewersLength: number) {
-    const video = await VideoModel.loadImmutableAttributes(videoId)
-    if (!video) return
-
-    PeerTubeSocket.Instance.sendVideoViewsUpdate(video, viewersLength)
-
-    logger.debug('Video viewers update for %s is %d.', video.url, viewersLength, lTags())
-  }
-
   async processViewerStats () {
-    if (this.processingViewerStats) return
-    this.processingViewerStats = true
+    if (this.processingViewersStats) return
+    this.processingViewersStats = true
 
     if (!isTestInstance()) logger.info('Processing viewer statistics.', lTags())
 
@@ -245,7 +153,7 @@ export class VideoViewers {
       logger.error('Error in video save viewers stats scheduler.', { err, ...lTags() })
     }
 
-    this.processingViewerStats = false
+    this.processingViewersStats = false
   }
 
   private async saveViewerStats (video: MVideo, stats: LocalViewerStats, transaction: Transaction) {
diff --git a/server/lib/views/shared/video-views.ts b/server/lib/views/shared/video-views.ts
index 19250f993..275f7a014 100644
--- a/server/lib/views/shared/video-views.ts
+++ b/server/lib/views/shared/video-views.ts
@@ -1,5 +1,8 @@
 import { logger, loggerTagsFactory } from '@server/helpers/logger'
+import { sendView } from '@server/lib/activitypub/send/send-view'
+import { getServerActor } from '@server/models/application/application'
 import { MVideo } from '@server/types/models'
+import { buildUUID } from '@shared/extra-utils'
 import { Redis } from '../../redis'
 
 const lTags = loggerTagsFactory('views')
@@ -24,6 +27,8 @@ export class VideoViews {
 
     await this.addView(video)
 
+    await sendView({ byActor: await getServerActor(), video, type: 'view', viewerIdentifier: buildUUID() })
+
     return true
   }
 
@@ -39,6 +44,8 @@ export class VideoViews {
     return true
   }
 
+  // ---------------------------------------------------------------------------
+
   private async addView (video: MVideo) {
     const promises: Promise<any>[] = []
 
diff --git a/server/lib/views/video-views-manager.ts b/server/lib/views/video-views-manager.ts
index 9382fb482..ea3b35c6c 100644
--- a/server/lib/views/video-views-manager.ts
+++ b/server/lib/views/video-views-manager.ts
@@ -1,7 +1,7 @@
 import { logger, loggerTagsFactory } from '@server/helpers/logger'
 import { MVideo } from '@server/types/models'
 import { VideoViewEvent } from '@shared/models'
-import { VideoViewers, VideoViews } from './shared'
+import { VideoViewerCounters, VideoViewerStats, VideoViews } from './shared'
 
 /**
  * If processing a local view:
@@ -27,14 +27,16 @@ export class VideoViewsManager {
 
   private static instance: VideoViewsManager
 
-  private videoViewers: VideoViewers
+  private videoViewerStats: VideoViewerStats
+  private videoViewerCounters: VideoViewerCounters
   private videoViews: VideoViews
 
   private constructor () {
   }
 
   init () {
-    this.videoViewers = new VideoViewers()
+    this.videoViewerStats = new VideoViewerStats()
+    this.videoViewerCounters = new VideoViewerCounters()
     this.videoViews = new VideoViews()
   }
 
@@ -48,10 +50,12 @@ export class VideoViewsManager {
 
     logger.debug('Processing local view for %s and ip %s.', video.url, ip, lTags())
 
-    const successViewer = await this.videoViewers.addLocalViewer({ video, ip, viewEvent, currentTime })
+    await this.videoViewerStats.addLocalViewer({ video, ip, viewEvent, currentTime })
+
+    const successViewer = await this.videoViewerCounters.addLocalViewer({ video, ip })
 
     // Do it after added local viewer to fetch updated information
-    const watchTime = await this.videoViewers.getWatchTime(video.id, ip)
+    const watchTime = await this.videoViewerStats.getWatchTime(video.id, ip)
 
     const successView = await this.videoViews.addLocalView({ video, watchTime, ip })
 
@@ -60,26 +64,27 @@ export class VideoViewsManager {
 
   async processRemoteView (options: {
     video: MVideo
+    viewerId: string | null
     viewerExpires?: Date
   }) {
-    const { video, viewerExpires } = options
+    const { video, viewerId, viewerExpires } = options
 
-    logger.debug('Processing remote view for %s.', video.url, { viewerExpires, ...lTags() })
+    logger.debug('Processing remote view for %s.', video.url, { viewerExpires, viewerId, ...lTags() })
 
-    if (viewerExpires) await this.videoViewers.addRemoteViewer({ video, viewerExpires })
+    if (viewerExpires) await this.videoViewerCounters.addRemoteViewer({ video, viewerId, viewerExpires })
     else await this.videoViews.addRemoteView({ video })
   }
 
   getViewers (video: MVideo) {
-    return this.videoViewers.getViewers(video)
+    return this.videoViewerCounters.getViewers(video)
   }
 
   buildViewerExpireTime () {
-    return this.videoViewers.buildViewerExpireTime()
+    return this.videoViewerCounters.buildViewerExpireTime()
   }
 
-  processViewers () {
-    return this.videoViewers.processViewerStats()
+  processViewerStats () {
+    return this.videoViewerStats.processViewerStats()
   }
 
   static get Instance () {