Support video views/viewers stats in server

 * Add "currentTime" and "event" body params to view endpoint
 * Merge watching and view endpoints
 * Introduce WatchAction AP activity
 * Add tables to store viewer information of local videos
 * Add endpoints to fetch video views/viewers stats of local videos
 * Refactor views/viewers handlers
 * Support "views" and "viewers" counters for both VOD and live videos

7 files changed, 158 insertions, 12 deletions

diff --git a/server/helpers/custom-validators/activitypub/activity.ts b/server/helpers/custom-validators/activitypub/activity.ts
index b5c96f6e7..90a918523 100644
--- a/server/helpers/custom-validators/activitypub/activity.ts
+++ b/server/helpers/custom-validators/activitypub/activity.ts
@@ -8,6 +8,7 @@ import { isActivityPubUrlValid, isBaseActivityValid, isObjectValid } from './mis
 import { isPlaylistObjectValid } from './playlist'
 import { sanitizeAndCheckVideoCommentObject } from './video-comments'
 import { sanitizeAndCheckVideoTorrentObject } from './videos'
+import { isWatchActionObjectValid } from './watch-action'
 
 function isRootActivityValid (activity: any) {
   return isCollection(activity) || isActivity(activity)
@@ -82,6 +83,7 @@ function isCreateActivityValid (activity: any) {
       isDislikeActivityValid(activity.object) ||
       isFlagActivityValid(activity.object) ||
       isPlaylistObjectValid(activity.object) ||
+      isWatchActionObjectValid(activity.object) ||
 
       isCacheFileObjectValid(activity.object) ||
       sanitizeAndCheckVideoCommentObject(activity.object) ||
diff --git a/server/helpers/custom-validators/activitypub/misc.ts b/server/helpers/custom-validators/activitypub/misc.ts
index 4ee8e6fee..9d823299f 100644
--- a/server/helpers/custom-validators/activitypub/misc.ts
+++ b/server/helpers/custom-validators/activitypub/misc.ts
@@ -57,10 +57,19 @@ function setValidAttributedTo (obj: any) {
   return true
 }
 
+function isActivityPubVideoDurationValid (value: string) {
+  // https://www.w3.org/TR/activitystreams-vocabulary/#dfn-duration
+  return exists(value) &&
+    typeof value === 'string' &&
+    value.startsWith('PT') &&
+    value.endsWith('S')
+}
+
 export {
   isUrlValid,
   isActivityPubUrlValid,
   isBaseActivityValid,
   setValidAttributedTo,
-  isObjectValid
+  isObjectValid,
+  isActivityPubVideoDurationValid
 }
diff --git a/server/helpers/custom-validators/activitypub/videos.ts b/server/helpers/custom-validators/activitypub/videos.ts
index 80a321117..2a2f008b9 100644
--- a/server/helpers/custom-validators/activitypub/videos.ts
+++ b/server/helpers/custom-validators/activitypub/videos.ts
@@ -4,7 +4,7 @@ import { ActivityTrackerUrlObject, ActivityVideoFileMetadataUrlObject } from '@s
 import { LiveVideoLatencyMode, VideoState } from '../../../../shared/models/videos'
 import { ACTIVITY_PUB, CONSTRAINTS_FIELDS } from '../../../initializers/constants'
 import { peertubeTruncate } from '../../core-utils'
-import { exists, isArray, isBooleanValid, isDateValid, isUUIDValid } from '../misc'
+import { isArray, isBooleanValid, isDateValid, isUUIDValid } from '../misc'
 import { isLiveLatencyModeValid } from '../video-lives'
 import {
   isVideoDurationValid,
@@ -14,22 +14,13 @@ import {
   isVideoTruncatedDescriptionValid,
   isVideoViewsValid
 } from '../videos'
-import { isActivityPubUrlValid, isBaseActivityValid, setValidAttributedTo } from './misc'
+import { isActivityPubUrlValid, isActivityPubVideoDurationValid, isBaseActivityValid, setValidAttributedTo } from './misc'
 
 function sanitizeAndCheckVideoTorrentUpdateActivity (activity: any) {
   return isBaseActivityValid(activity, 'Update') &&
     sanitizeAndCheckVideoTorrentObject(activity.object)
 }
 
-function isActivityPubVideoDurationValid (value: string) {
-  // https://www.w3.org/TR/activitystreams-vocabulary/#dfn-duration
-  return exists(value) &&
-    typeof value === 'string' &&
-    value.startsWith('PT') &&
-    value.endsWith('S') &&
-    isVideoDurationValid(value.replace(/[^0-9]+/g, ''))
-}
-
 function sanitizeAndCheckVideoTorrentObject (video: any) {
   if (!video || video.type !== 'Video') return false
 
@@ -71,6 +62,7 @@ function sanitizeAndCheckVideoTorrentObject (video: any) {
   return isActivityPubUrlValid(video.id) &&
     isVideoNameValid(video.name) &&
     isActivityPubVideoDurationValid(video.duration) &&
+    isVideoDurationValid(video.duration.replace(/[^0-9]+/g, '')) &&
     isUUIDValid(video.uuid) &&
     (!video.category || isRemoteNumberIdentifierValid(video.category)) &&
     (!video.licence || isRemoteNumberIdentifierValid(video.licence)) &&
diff --git a/server/helpers/custom-validators/activitypub/watch-action.ts b/server/helpers/custom-validators/activitypub/watch-action.ts
new file mode 100644
index 000000000..b9ffa63f6
--- /dev/null
+++ b/server/helpers/custom-validators/activitypub/watch-action.ts
@@ -0,0 +1,37 @@
+import { WatchActionObject } from '@shared/models'
+import { exists, isDateValid, isUUIDValid } from '../misc'
+import { isVideoTimeValid } from '../video-view'
+import { isActivityPubVideoDurationValid, isObjectValid } from './misc'
+
+function isWatchActionObjectValid (action: WatchActionObject) {
+  return exists(action) &&
+    action.type === 'WatchAction' &&
+    isObjectValid(action.id) &&
+    isActivityPubVideoDurationValid(action.duration) &&
+    isDateValid(action.startTime) &&
+    isDateValid(action.endTime) &&
+    isLocationValid(action.location) &&
+    isUUIDValid(action.uuid) &&
+    isObjectValid(action.object) &&
+    isWatchSectionsValid(action.watchSections)
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+  isWatchActionObjectValid
+}
+
+// ---------------------------------------------------------------------------
+
+function isLocationValid (location: any) {
+  if (!location) return true
+
+  return typeof location === 'object' && typeof location.addressCountry === 'string'
+}
+
+function isWatchSectionsValid (sections: WatchActionObject['watchSections']) {
+  return Array.isArray(sections) && sections.every(s => {
+    return isVideoTimeValid(s.startTimestamp) && isVideoTimeValid(s.endTimestamp)
+  })
+}
diff --git a/server/helpers/custom-validators/video-stats.ts b/server/helpers/custom-validators/video-stats.ts
new file mode 100644
index 000000000..1e22f0654
--- /dev/null
+++ b/server/helpers/custom-validators/video-stats.ts
@@ -0,0 +1,16 @@
+import { VideoStatsTimeserieMetric } from '@shared/models'
+
+const validMetrics = new Set<VideoStatsTimeserieMetric>([
+  'viewers',
+  'aggregateWatchTime'
+])
+
+function isValidStatTimeserieMetric (value: VideoStatsTimeserieMetric) {
+  return validMetrics.has(value)
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+  isValidStatTimeserieMetric
+}
diff --git a/server/helpers/custom-validators/video-view.ts b/server/helpers/custom-validators/video-view.ts
new file mode 100644
index 000000000..091c92083
--- /dev/null
+++ b/server/helpers/custom-validators/video-view.ts
@@ -0,0 +1,12 @@
+import { exists } from './misc'
+
+function isVideoTimeValid (value: number, videoDuration?: number) {
+  if (value < 0) return false
+  if (exists(videoDuration) && value > videoDuration) return false
+
+  return true
+}
+
+export {
+  isVideoTimeValid
+}
diff --git a/server/helpers/geo-ip.ts b/server/helpers/geo-ip.ts
new file mode 100644
index 000000000..4ba7011c2
--- /dev/null
+++ b/server/helpers/geo-ip.ts
@@ -0,0 +1,78 @@
+import { pathExists, writeFile } from 'fs-extra'
+import maxmind, { CountryResponse, Reader } from 'maxmind'
+import { join } from 'path'
+import { CONFIG } from '@server/initializers/config'
+import { logger, loggerTagsFactory } from './logger'
+import { isBinaryResponse, peertubeGot } from './requests'
+
+const lTags = loggerTagsFactory('geo-ip')
+
+const mmbdFilename = 'dbip-country-lite-latest.mmdb'
+const mmdbPath = join(CONFIG.STORAGE.BIN_DIR, mmbdFilename)
+
+export class GeoIP {
+  private static instance: GeoIP
+
+  private reader: Reader<CountryResponse>
+
+  private constructor () {
+  }
+
+  async safeCountryISOLookup (ip: string): Promise<string> {
+    if (CONFIG.GEO_IP.ENABLED === false) return null
+
+    await this.initReaderIfNeeded()
+
+    try {
+      const result = this.reader.get(ip)
+      if (!result) return null
+
+      return result.country.iso_code
+    } catch (err) {
+      logger.error('Cannot get country from IP.', { err })
+
+      return null
+    }
+  }
+
+  async updateDatabase () {
+    if (CONFIG.GEO_IP.ENABLED === false) return
+
+    const url = CONFIG.GEO_IP.COUNTRY.DATABASE_URL
+
+    logger.info('Updating GeoIP database from %s.', url, lTags())
+
+    const gotOptions = { context: { bodyKBLimit: 200_000 }, responseType: 'buffer' as 'buffer' }
+
+    try {
+      const gotResult = await peertubeGot(url, gotOptions)
+
+      if (!isBinaryResponse(gotResult)) {
+        throw new Error('Not a binary response')
+      }
+
+      await writeFile(mmdbPath, gotResult.body)
+
+      // Reini reader
+      this.reader = undefined
+
+      logger.info('GeoIP database updated %s.', mmdbPath, lTags())
+    } catch (err) {
+      logger.error('Cannot update GeoIP database from %s.', url, { err, ...lTags() })
+    }
+  }
+
+  private async initReaderIfNeeded () {
+    if (!this.reader) {
+      if (!await pathExists(mmdbPath)) {
+        await this.updateDatabase()
+      }
+
+      this.reader = await maxmind.open(mmdbPath)
+    }
+  }
+
+  static get Instance () {
+    return this.instance || (this.instance = new this())
+  }
+}