From b211106695bb82f6c32e53306081b5262c3d109d Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Thu, 24 Mar 2022 13:36:47 +0100 Subject: Support video views/viewers stats in server * Add "currentTime" and "event" body params to view endpoint * Merge watching and view endpoints * Introduce WatchAction AP activity * Add tables to store viewer information of local videos * Add endpoints to fetch video views/viewers stats of local videos * Refactor views/viewers handlers * Support "views" and "viewers" counters for both VOD and live videos --- server/initializers/checker-before-init.ts | 1 + server/initializers/config.ts | 6 +++ server/initializers/constants.ts | 17 ++++++- server/initializers/database.ts | 10 +++-- .../migrations/0705-local-video-viewers.ts | 52 ++++++++++++++++++++++ 5 files changed, 81 insertions(+), 5 deletions(-) create mode 100644 server/initializers/migrations/0705-local-video-viewers.ts (limited to 'server/initializers') diff --git a/server/initializers/checker-before-init.ts b/server/initializers/checker-before-init.ts index 0f23a2d73..f2ef3d567 100644 --- a/server/initializers/checker-before-init.ts +++ b/server/initializers/checker-before-init.ts @@ -44,6 +44,7 @@ function checkMissedConfig () { 'history.videos.max_age', 'views.videos.remote.max_age', 'views.videos.local_buffer_update_interval', 'views.videos.ip_view_expiration', 'rates_limit.login.window', 'rates_limit.login.max', 'rates_limit.ask_send_email.window', 'rates_limit.ask_send_email.max', 'theme.default', + 'geo_ip.enabled', 'geo_ip.country.database_url', 'remote_redundancy.videos.accept_from', 'federation.videos.federate_unlisted', 'federation.videos.cleanup_remote_interactions', 'peertube.check_latest_version.enabled', 'peertube.check_latest_version.url', diff --git a/server/initializers/config.ts b/server/initializers/config.ts index 122cb9472..d8f5f3496 100644 --- a/server/initializers/config.ts +++ b/server/initializers/config.ts @@ -215,6 +215,12 @@ const CONFIG = { IP_VIEW_EXPIRATION: parseDurationToMs(config.get('views.videos.ip_view_expiration')) } }, + GEO_IP: { + ENABLED: config.get('geo_ip.enabled'), + COUNTRY: { + DATABASE_URL: config.get('geo_ip.country.database_url') + } + }, PLUGINS: { INDEX: { ENABLED: config.get('plugins.index.enabled'), diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index 6bcefe0db..4929923dc 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts @@ -24,7 +24,7 @@ import { CONFIG, registerConfigChangedHandler } from './config' // --------------------------------------------------------------------------- -const LAST_MIGRATION_VERSION = 700 +const LAST_MIGRATION_VERSION = 705 // --------------------------------------------------------------------------- @@ -228,6 +228,7 @@ const SCHEDULER_INTERVALS_MS = { REMOVE_OLD_JOBS: 60000 * 60, // 1 hour UPDATE_VIDEOS: 60000, // 1 minute YOUTUBE_DL_UPDATE: 60000 * 60 * 24, // 1 day + GEO_IP_UPDATE: 60000 * 60 * 24, // 1 day VIDEO_VIEWS_BUFFER_UPDATE: CONFIG.VIEWS.VIDEOS.LOCAL_BUFFER_UPDATE_INTERVAL, CHECK_PLUGINS: CONFIG.PLUGINS.INDEX.CHECK_LATEST_VERSIONS_INTERVAL, CHECK_PEERTUBE_VERSION: 60000 * 60 * 24, // 1 day @@ -366,9 +367,12 @@ const CONSTRAINTS_FIELDS = { const VIEW_LIFETIME = { VIEW: CONFIG.VIEWS.VIDEOS.IP_VIEW_EXPIRATION, - VIEWER: 60000 * 5 // 5 minutes + VIEWER: 60000 * 5, // 5 minutes + VIEWER_STATS: 60000 * 60 // 1 hour } +const MAX_LOCAL_VIEWER_WATCH_SECTIONS = 10 + let CONTACT_FORM_LIFETIME = 60000 * 60 // 1 hour const VIDEO_TRANSCODING_FPS: VideoTranscodingFPS = { @@ -800,6 +804,12 @@ const SEARCH_INDEX = { // --------------------------------------------------------------------------- +const STATS_TIMESERIE = { + MAX_DAYS: 30 +} + +// --------------------------------------------------------------------------- + // Special constants for a test instance if (isTestInstance() === true) { PRIVATE_RSA_KEY_SIZE = 1024 @@ -836,6 +846,7 @@ if (isTestInstance() === true) { REDUNDANCY.VIDEOS.RANDOMIZED_FACTOR = 1 VIEW_LIFETIME.VIEWER = 1000 * 5 // 5 second + VIEW_LIFETIME.VIEWER_STATS = 1000 * 5 // 5 second CONTACT_FORM_LIFETIME = 1000 // 1 second JOB_ATTEMPTS['email'] = 1 @@ -907,6 +918,7 @@ export { LAST_MIGRATION_VERSION, OAUTH_LIFETIME, CUSTOM_HTML_TAG_COMMENTS, + STATS_TIMESERIE, BROADCAST_CONCURRENCY, AUDIT_LOG_FILENAME, PAGINATION, @@ -949,6 +961,7 @@ export { ABUSE_STATES, LRU_CACHE, REQUEST_TIMEOUTS, + MAX_LOCAL_VIEWER_WATCH_SECTIONS, USER_PASSWORD_RESET_LIFETIME, USER_PASSWORD_CREATE_LIFETIME, MEMOIZE_TTL, diff --git a/server/initializers/database.ts b/server/initializers/database.ts index 0e690f6ae..7a7ba61f4 100644 --- a/server/initializers/database.ts +++ b/server/initializers/database.ts @@ -1,10 +1,14 @@ import { QueryTypes, Transaction } from 'sequelize' import { Sequelize as SequelizeTypescript } from 'sequelize-typescript' +import { ActorCustomPageModel } from '@server/models/account/actor-custom-page' import { TrackerModel } from '@server/models/server/tracker' import { VideoTrackerModel } from '@server/models/server/video-tracker' import { UserModel } from '@server/models/user/user' import { UserNotificationModel } from '@server/models/user/user-notification' import { UserVideoHistoryModel } from '@server/models/user/user-video-history' +import { VideoJobInfoModel } from '@server/models/video/video-job-info' +import { LocalVideoViewerModel } from '@server/models/view/local-video-viewer' +import { LocalVideoViewerWatchSectionModel } from '@server/models/view/local-video-viewer-watch-section' import { isTestInstance } from '../helpers/core-utils' import { logger } from '../helpers/logger' import { AbuseModel } from '../models/abuse/abuse' @@ -42,10 +46,8 @@ import { VideoPlaylistElementModel } from '../models/video/video-playlist-elemen import { VideoShareModel } from '../models/video/video-share' import { VideoStreamingPlaylistModel } from '../models/video/video-streaming-playlist' import { VideoTagModel } from '../models/video/video-tag' -import { VideoViewModel } from '../models/video/video-view' +import { VideoViewModel } from '../models/view/video-view' import { CONFIG } from './config' -import { ActorCustomPageModel } from '@server/models/account/actor-custom-page' -import { VideoJobInfoModel } from '@server/models/video/video-job-info' require('pg').defaults.parseInt8 = true // Avoid BIGINT to be converted to string @@ -140,6 +142,8 @@ async function initDatabaseModels (silent: boolean) { VideoStreamingPlaylistModel, VideoPlaylistModel, VideoPlaylistElementModel, + LocalVideoViewerModel, + LocalVideoViewerWatchSectionModel, ThumbnailModel, TrackerModel, VideoTrackerModel, diff --git a/server/initializers/migrations/0705-local-video-viewers.ts b/server/initializers/migrations/0705-local-video-viewers.ts new file mode 100644 index 000000000..123402641 --- /dev/null +++ b/server/initializers/migrations/0705-local-video-viewers.ts @@ -0,0 +1,52 @@ +import * as Sequelize from 'sequelize' + +async function up (utils: { + transaction: Sequelize.Transaction + queryInterface: Sequelize.QueryInterface + sequelize: Sequelize.Sequelize + db: any +}): Promise { + const { transaction } = utils + + { + const query = ` + CREATE TABLE IF NOT EXISTS "localVideoViewer" ( + "id" serial, + "startDate" timestamp with time zone NOT NULL, + "endDate" timestamp with time zone NOT NULL, + "watchTime" integer NOT NULL, + "country" varchar(255), + "uuid" uuid NOT NULL, + "url" varchar(255) NOT NULL, + "videoId" integer NOT NULL REFERENCES "video" ("id") ON DELETE CASCADE ON UPDATE CASCADE, + "createdAt" timestamp with time zone NOT NULL, + PRIMARY KEY ("id") + ); + ` + await utils.sequelize.query(query, { transaction }) + } + + { + const query = ` + CREATE TABLE IF NOT EXISTS "localVideoViewerWatchSection" ( + "id" serial, + "watchStart" integer NOT NULL, + "watchEnd" integer NOT NULL, + "localVideoViewerId" integer NOT NULL REFERENCES "localVideoViewer" ("id") ON DELETE CASCADE ON UPDATE CASCADE, + "createdAt" timestamp with time zone NOT NULL, + PRIMARY KEY ("id") + ); + ` + await utils.sequelize.query(query, { transaction }) + } + +} + +function down () { + throw new Error('Not implemented.') +} + +export { + up, + down +} -- cgit v1.2.3