From b211106695bb82f6c32e53306081b5262c3d109d Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Thu, 24 Mar 2022 13:36:47 +0100 Subject: Support video views/viewers stats in server * Add "currentTime" and "event" body params to view endpoint * Merge watching and view endpoints * Introduce WatchAction AP activity * Add tables to store viewer information of local videos * Add endpoints to fetch video views/viewers stats of local videos * Refactor views/viewers handlers * Support "views" and "viewers" counters for both VOD and live videos --- .../custom-validators/activitypub/activity.ts | 2 + .../helpers/custom-validators/activitypub/misc.ts | 11 ++- .../custom-validators/activitypub/videos.ts | 14 +--- .../custom-validators/activitypub/watch-action.ts | 37 ++++++++++ server/helpers/custom-validators/video-stats.ts | 16 +++++ server/helpers/custom-validators/video-view.ts | 12 ++++ server/helpers/geo-ip.ts | 78 ++++++++++++++++++++++ 7 files changed, 158 insertions(+), 12 deletions(-) create mode 100644 server/helpers/custom-validators/activitypub/watch-action.ts create mode 100644 server/helpers/custom-validators/video-stats.ts create mode 100644 server/helpers/custom-validators/video-view.ts create mode 100644 server/helpers/geo-ip.ts (limited to 'server/helpers') diff --git a/server/helpers/custom-validators/activitypub/activity.ts b/server/helpers/custom-validators/activitypub/activity.ts index b5c96f6e7..90a918523 100644 --- a/server/helpers/custom-validators/activitypub/activity.ts +++ b/server/helpers/custom-validators/activitypub/activity.ts @@ -8,6 +8,7 @@ import { isActivityPubUrlValid, isBaseActivityValid, isObjectValid } from './mis import { isPlaylistObjectValid } from './playlist' import { sanitizeAndCheckVideoCommentObject } from './video-comments' import { sanitizeAndCheckVideoTorrentObject } from './videos' +import { isWatchActionObjectValid } from './watch-action' function isRootActivityValid (activity: any) { return isCollection(activity) || isActivity(activity) @@ -82,6 +83,7 @@ function isCreateActivityValid (activity: any) { isDislikeActivityValid(activity.object) || isFlagActivityValid(activity.object) || isPlaylistObjectValid(activity.object) || + isWatchActionObjectValid(activity.object) || isCacheFileObjectValid(activity.object) || sanitizeAndCheckVideoCommentObject(activity.object) || diff --git a/server/helpers/custom-validators/activitypub/misc.ts b/server/helpers/custom-validators/activitypub/misc.ts index 4ee8e6fee..9d823299f 100644 --- a/server/helpers/custom-validators/activitypub/misc.ts +++ b/server/helpers/custom-validators/activitypub/misc.ts @@ -57,10 +57,19 @@ function setValidAttributedTo (obj: any) { return true } +function isActivityPubVideoDurationValid (value: string) { + // https://www.w3.org/TR/activitystreams-vocabulary/#dfn-duration + return exists(value) && + typeof value === 'string' && + value.startsWith('PT') && + value.endsWith('S') +} + export { isUrlValid, isActivityPubUrlValid, isBaseActivityValid, setValidAttributedTo, - isObjectValid + isObjectValid, + isActivityPubVideoDurationValid } diff --git a/server/helpers/custom-validators/activitypub/videos.ts b/server/helpers/custom-validators/activitypub/videos.ts index 80a321117..2a2f008b9 100644 --- a/server/helpers/custom-validators/activitypub/videos.ts +++ b/server/helpers/custom-validators/activitypub/videos.ts @@ -4,7 +4,7 @@ import { ActivityTrackerUrlObject, ActivityVideoFileMetadataUrlObject } from '@s import { LiveVideoLatencyMode, VideoState } from '../../../../shared/models/videos' import { ACTIVITY_PUB, CONSTRAINTS_FIELDS } from '../../../initializers/constants' import { peertubeTruncate } from '../../core-utils' -import { exists, isArray, isBooleanValid, isDateValid, isUUIDValid } from '../misc' +import { isArray, isBooleanValid, isDateValid, isUUIDValid } from '../misc' import { isLiveLatencyModeValid } from '../video-lives' import { isVideoDurationValid, @@ -14,22 +14,13 @@ import { isVideoTruncatedDescriptionValid, isVideoViewsValid } from '../videos' -import { isActivityPubUrlValid, isBaseActivityValid, setValidAttributedTo } from './misc' +import { isActivityPubUrlValid, isActivityPubVideoDurationValid, isBaseActivityValid, setValidAttributedTo } from './misc' function sanitizeAndCheckVideoTorrentUpdateActivity (activity: any) { return isBaseActivityValid(activity, 'Update') && sanitizeAndCheckVideoTorrentObject(activity.object) } -function isActivityPubVideoDurationValid (value: string) { - // https://www.w3.org/TR/activitystreams-vocabulary/#dfn-duration - return exists(value) && - typeof value === 'string' && - value.startsWith('PT') && - value.endsWith('S') && - isVideoDurationValid(value.replace(/[^0-9]+/g, '')) -} - function sanitizeAndCheckVideoTorrentObject (video: any) { if (!video || video.type !== 'Video') return false @@ -71,6 +62,7 @@ function sanitizeAndCheckVideoTorrentObject (video: any) { return isActivityPubUrlValid(video.id) && isVideoNameValid(video.name) && isActivityPubVideoDurationValid(video.duration) && + isVideoDurationValid(video.duration.replace(/[^0-9]+/g, '')) && isUUIDValid(video.uuid) && (!video.category || isRemoteNumberIdentifierValid(video.category)) && (!video.licence || isRemoteNumberIdentifierValid(video.licence)) && diff --git a/server/helpers/custom-validators/activitypub/watch-action.ts b/server/helpers/custom-validators/activitypub/watch-action.ts new file mode 100644 index 000000000..b9ffa63f6 --- /dev/null +++ b/server/helpers/custom-validators/activitypub/watch-action.ts @@ -0,0 +1,37 @@ +import { WatchActionObject } from '@shared/models' +import { exists, isDateValid, isUUIDValid } from '../misc' +import { isVideoTimeValid } from '../video-view' +import { isActivityPubVideoDurationValid, isObjectValid } from './misc' + +function isWatchActionObjectValid (action: WatchActionObject) { + return exists(action) && + action.type === 'WatchAction' && + isObjectValid(action.id) && + isActivityPubVideoDurationValid(action.duration) && + isDateValid(action.startTime) && + isDateValid(action.endTime) && + isLocationValid(action.location) && + isUUIDValid(action.uuid) && + isObjectValid(action.object) && + isWatchSectionsValid(action.watchSections) +} + +// --------------------------------------------------------------------------- + +export { + isWatchActionObjectValid +} + +// --------------------------------------------------------------------------- + +function isLocationValid (location: any) { + if (!location) return true + + return typeof location === 'object' && typeof location.addressCountry === 'string' +} + +function isWatchSectionsValid (sections: WatchActionObject['watchSections']) { + return Array.isArray(sections) && sections.every(s => { + return isVideoTimeValid(s.startTimestamp) && isVideoTimeValid(s.endTimestamp) + }) +} diff --git a/server/helpers/custom-validators/video-stats.ts b/server/helpers/custom-validators/video-stats.ts new file mode 100644 index 000000000..1e22f0654 --- /dev/null +++ b/server/helpers/custom-validators/video-stats.ts @@ -0,0 +1,16 @@ +import { VideoStatsTimeserieMetric } from '@shared/models' + +const validMetrics = new Set([ + 'viewers', + 'aggregateWatchTime' +]) + +function isValidStatTimeserieMetric (value: VideoStatsTimeserieMetric) { + return validMetrics.has(value) +} + +// --------------------------------------------------------------------------- + +export { + isValidStatTimeserieMetric +} diff --git a/server/helpers/custom-validators/video-view.ts b/server/helpers/custom-validators/video-view.ts new file mode 100644 index 000000000..091c92083 --- /dev/null +++ b/server/helpers/custom-validators/video-view.ts @@ -0,0 +1,12 @@ +import { exists } from './misc' + +function isVideoTimeValid (value: number, videoDuration?: number) { + if (value < 0) return false + if (exists(videoDuration) && value > videoDuration) return false + + return true +} + +export { + isVideoTimeValid +} diff --git a/server/helpers/geo-ip.ts b/server/helpers/geo-ip.ts new file mode 100644 index 000000000..4ba7011c2 --- /dev/null +++ b/server/helpers/geo-ip.ts @@ -0,0 +1,78 @@ +import { pathExists, writeFile } from 'fs-extra' +import maxmind, { CountryResponse, Reader } from 'maxmind' +import { join } from 'path' +import { CONFIG } from '@server/initializers/config' +import { logger, loggerTagsFactory } from './logger' +import { isBinaryResponse, peertubeGot } from './requests' + +const lTags = loggerTagsFactory('geo-ip') + +const mmbdFilename = 'dbip-country-lite-latest.mmdb' +const mmdbPath = join(CONFIG.STORAGE.BIN_DIR, mmbdFilename) + +export class GeoIP { + private static instance: GeoIP + + private reader: Reader + + private constructor () { + } + + async safeCountryISOLookup (ip: string): Promise { + if (CONFIG.GEO_IP.ENABLED === false) return null + + await this.initReaderIfNeeded() + + try { + const result = this.reader.get(ip) + if (!result) return null + + return result.country.iso_code + } catch (err) { + logger.error('Cannot get country from IP.', { err }) + + return null + } + } + + async updateDatabase () { + if (CONFIG.GEO_IP.ENABLED === false) return + + const url = CONFIG.GEO_IP.COUNTRY.DATABASE_URL + + logger.info('Updating GeoIP database from %s.', url, lTags()) + + const gotOptions = { context: { bodyKBLimit: 200_000 }, responseType: 'buffer' as 'buffer' } + + try { + const gotResult = await peertubeGot(url, gotOptions) + + if (!isBinaryResponse(gotResult)) { + throw new Error('Not a binary response') + } + + await writeFile(mmdbPath, gotResult.body) + + // Reini reader + this.reader = undefined + + logger.info('GeoIP database updated %s.', mmdbPath, lTags()) + } catch (err) { + logger.error('Cannot update GeoIP database from %s.', url, { err, ...lTags() }) + } + } + + private async initReaderIfNeeded () { + if (!this.reader) { + if (!await pathExists(mmdbPath)) { + await this.updateDatabase() + } + + this.reader = await maxmind.open(mmdbPath) + } + } + + static get Instance () { + return this.instance || (this.instance = new this()) + } +} -- cgit v1.2.3