diff options
author | Chocobozzz <me@florianbigard.com> | 2022-04-06 08:50:43 +0200 |
---|---|---|
committer | Chocobozzz <chocobozzz@cpy.re> | 2022-04-15 09:49:35 +0200 |
commit | ac907dc7c158056e9b6a5cb58acd27df5c7c2670 (patch) | |
tree | f9d8bff22e0543a305c64e1a1808c0df6d512f1e /server | |
parent | dfbcefc20dc64f0814b1f2e8e782a4ea1bd24db2 (diff) | |
download | PeerTube-ac907dc7c158056e9b6a5cb58acd27df5c7c2670.tar.gz PeerTube-ac907dc7c158056e9b6a5cb58acd27df5c7c2670.tar.zst PeerTube-ac907dc7c158056e9b6a5cb58acd27df5c7c2670.zip |
Improve viewer counter
More precise, avoid weird decrease, reuse an id to federate viewers
Diffstat (limited to 'server')
-rw-r--r-- | server/controllers/api/server/debug.ts | 2 | ||||
-rw-r--r-- | server/controllers/api/videos/view.ts | 10 | ||||
-rw-r--r-- | server/initializers/constants.ts | 2 | ||||
-rw-r--r-- | server/lib/activitypub/process/process-view.ts | 2 | ||||
-rw-r--r-- | server/lib/activitypub/send/send-view.ts | 5 | ||||
-rw-r--r-- | server/lib/activitypub/url.ts | 4 | ||||
-rw-r--r-- | server/lib/redis.ts | 12 | ||||
-rw-r--r-- | server/lib/views/shared/index.ts | 3 | ||||
-rw-r--r-- | server/lib/views/shared/video-viewer-counters.ts | 176 | ||||
-rw-r--r-- | server/lib/views/shared/video-viewer-stats.ts (renamed from server/lib/views/shared/video-viewers.ts) | 116 | ||||
-rw-r--r-- | server/lib/views/shared/video-views.ts | 7 | ||||
-rw-r--r-- | server/lib/views/video-views-manager.ts | 29 | ||||
-rw-r--r-- | server/tests/api/check-params/views.ts | 2 | ||||
-rw-r--r-- | server/tests/api/views/video-views-counter.ts | 9 | ||||
-rw-r--r-- | server/tests/api/views/video-views-overall-stats.ts | 6 | ||||
-rw-r--r-- | server/tests/api/views/video-views-retention-stats.ts | 2 | ||||
-rw-r--r-- | server/tests/api/views/video-views-timeserie-stats.ts | 4 |
17 files changed, 235 insertions, 156 deletions
diff --git a/server/controllers/api/server/debug.ts b/server/controllers/api/server/debug.ts index 6b6ff027c..e09510dc3 100644 --- a/server/controllers/api/server/debug.ts +++ b/server/controllers/api/server/debug.ts | |||
@@ -43,7 +43,7 @@ async function runCommand (req: express.Request, res: express.Response) { | |||
43 | const processors: { [id in SendDebugCommand['command']]: () => Promise<any> } = { | 43 | const processors: { [id in SendDebugCommand['command']]: () => Promise<any> } = { |
44 | 'remove-dandling-resumable-uploads': () => RemoveDanglingResumableUploadsScheduler.Instance.execute(), | 44 | 'remove-dandling-resumable-uploads': () => RemoveDanglingResumableUploadsScheduler.Instance.execute(), |
45 | 'process-video-views-buffer': () => VideoViewsBufferScheduler.Instance.execute(), | 45 | 'process-video-views-buffer': () => VideoViewsBufferScheduler.Instance.execute(), |
46 | 'process-video-viewers': () => VideoViewsManager.Instance.processViewers() | 46 | 'process-video-viewers': () => VideoViewsManager.Instance.processViewerStats() |
47 | } | 47 | } |
48 | 48 | ||
49 | await processors[body.command]() | 49 | await processors[body.command]() |
diff --git a/server/controllers/api/videos/view.ts b/server/controllers/api/videos/view.ts index e28cf371a..db1091f2d 100644 --- a/server/controllers/api/videos/view.ts +++ b/server/controllers/api/videos/view.ts | |||
@@ -1,8 +1,6 @@ | |||
1 | import express from 'express' | 1 | import express from 'express' |
2 | import { sendView } from '@server/lib/activitypub/send/send-view' | ||
3 | import { Hooks } from '@server/lib/plugins/hooks' | 2 | import { Hooks } from '@server/lib/plugins/hooks' |
4 | import { VideoViewsManager } from '@server/lib/views/video-views-manager' | 3 | import { VideoViewsManager } from '@server/lib/views/video-views-manager' |
5 | import { getServerActor } from '@server/models/application/application' | ||
6 | import { MVideoId } from '@server/types/models' | 4 | import { MVideoId } from '@server/types/models' |
7 | import { HttpStatusCode, VideoView } from '@shared/models' | 5 | import { HttpStatusCode, VideoView } from '@shared/models' |
8 | import { asyncMiddleware, methodsValidator, openapiOperationDoc, optionalAuthenticate, videoViewValidator } from '../../../middlewares' | 6 | import { asyncMiddleware, methodsValidator, openapiOperationDoc, optionalAuthenticate, videoViewValidator } from '../../../middlewares' |
@@ -33,7 +31,7 @@ async function viewVideo (req: express.Request, res: express.Response) { | |||
33 | const body = req.body as VideoView | 31 | const body = req.body as VideoView |
34 | 32 | ||
35 | const ip = req.ip | 33 | const ip = req.ip |
36 | const { successView, successViewer } = await VideoViewsManager.Instance.processLocalView({ | 34 | const { successView } = await VideoViewsManager.Instance.processLocalView({ |
37 | video, | 35 | video, |
38 | ip, | 36 | ip, |
39 | currentTime: body.currentTime, | 37 | currentTime: body.currentTime, |
@@ -41,15 +39,9 @@ async function viewVideo (req: express.Request, res: express.Response) { | |||
41 | }) | 39 | }) |
42 | 40 | ||
43 | if (successView) { | 41 | if (successView) { |
44 | await sendView({ byActor: await getServerActor(), video, type: 'view' }) | ||
45 | |||
46 | Hooks.runAction('action:api.video.viewed', { video: video, ip, req, res }) | 42 | Hooks.runAction('action:api.video.viewed', { video: video, ip, req, res }) |
47 | } | 43 | } |
48 | 44 | ||
49 | if (successViewer) { | ||
50 | await sendView({ byActor: await getServerActor(), video, type: 'viewer' }) | ||
51 | } | ||
52 | |||
53 | await updateUserHistoryIfNeeded(body, video, res) | 45 | await updateUserHistoryIfNeeded(body, video, res) |
54 | 46 | ||
55 | return res.status(HttpStatusCode.NO_CONTENT_204).end() | 47 | return res.status(HttpStatusCode.NO_CONTENT_204).end() |
diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index 9afbc5aea..a4d8d8fe7 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts | |||
@@ -367,7 +367,7 @@ const CONSTRAINTS_FIELDS = { | |||
367 | 367 | ||
368 | const VIEW_LIFETIME = { | 368 | const VIEW_LIFETIME = { |
369 | VIEW: CONFIG.VIEWS.VIDEOS.IP_VIEW_EXPIRATION, | 369 | VIEW: CONFIG.VIEWS.VIDEOS.IP_VIEW_EXPIRATION, |
370 | VIEWER_COUNTER: 60000 * 5, // 5 minutes | 370 | VIEWER_COUNTER: 60000 * 1, // 1 minute |
371 | VIEWER_STATS: 60000 * 60 // 1 hour | 371 | VIEWER_STATS: 60000 * 60 // 1 hour |
372 | } | 372 | } |
373 | 373 | ||
diff --git a/server/lib/activitypub/process/process-view.ts b/server/lib/activitypub/process/process-view.ts index bad079843..e49506d82 100644 --- a/server/lib/activitypub/process/process-view.ts +++ b/server/lib/activitypub/process/process-view.ts | |||
@@ -32,7 +32,7 @@ async function processCreateView (activity: ActivityView, byActor: MActorSignatu | |||
32 | ? new Date(activity.expires) | 32 | ? new Date(activity.expires) |
33 | : undefined | 33 | : undefined |
34 | 34 | ||
35 | await VideoViewsManager.Instance.processRemoteView({ video, viewerExpires }) | 35 | await VideoViewsManager.Instance.processRemoteView({ video, viewerId: activity.id, viewerExpires }) |
36 | 36 | ||
37 | if (video.isOwned()) { | 37 | if (video.isOwned()) { |
38 | // Forward the view but don't resend the activity to the sender | 38 | // Forward the view but don't resend the activity to the sender |
diff --git a/server/lib/activitypub/send/send-view.ts b/server/lib/activitypub/send/send-view.ts index 1088bf258..25a20ec6d 100644 --- a/server/lib/activitypub/send/send-view.ts +++ b/server/lib/activitypub/send/send-view.ts | |||
@@ -13,14 +13,15 @@ async function sendView (options: { | |||
13 | byActor: MActorLight | 13 | byActor: MActorLight |
14 | type: ViewType | 14 | type: ViewType |
15 | video: MVideoImmutable | 15 | video: MVideoImmutable |
16 | viewerIdentifier: string | ||
16 | transaction?: Transaction | 17 | transaction?: Transaction |
17 | }) { | 18 | }) { |
18 | const { byActor, type, video, transaction } = options | 19 | const { byActor, type, video, viewerIdentifier, transaction } = options |
19 | 20 | ||
20 | logger.info('Creating job to send %s of %s.', type, video.url) | 21 | logger.info('Creating job to send %s of %s.', type, video.url) |
21 | 22 | ||
22 | const activityBuilder = (audience: ActivityAudience) => { | 23 | const activityBuilder = (audience: ActivityAudience) => { |
23 | const url = getLocalVideoViewActivityPubUrl(byActor, video) | 24 | const url = getLocalVideoViewActivityPubUrl(byActor, video, viewerIdentifier) |
24 | 25 | ||
25 | return buildViewActivity({ url, byActor, video, audience, type }) | 26 | return buildViewActivity({ url, byActor, video, audience, type }) |
26 | } | 27 | } |
diff --git a/server/lib/activitypub/url.ts b/server/lib/activitypub/url.ts index 8443fef4c..2f68f7a17 100644 --- a/server/lib/activitypub/url.ts +++ b/server/lib/activitypub/url.ts | |||
@@ -56,8 +56,8 @@ function getLocalAbuseActivityPubUrl (abuse: MAbuseId) { | |||
56 | return WEBSERVER.URL + '/admin/abuses/' + abuse.id | 56 | return WEBSERVER.URL + '/admin/abuses/' + abuse.id |
57 | } | 57 | } |
58 | 58 | ||
59 | function getLocalVideoViewActivityPubUrl (byActor: MActorUrl, video: MVideoId) { | 59 | function getLocalVideoViewActivityPubUrl (byActor: MActorUrl, video: MVideoId, viewerIdentifier: string) { |
60 | return byActor.url + '/views/videos/' + video.id + '/' + new Date().toISOString() | 60 | return byActor.url + '/views/videos/' + video.id + '/' + viewerIdentifier |
61 | } | 61 | } |
62 | 62 | ||
63 | function getLocalVideoViewerActivityPubUrl (stats: MLocalVideoViewer) { | 63 | function getLocalVideoViewerActivityPubUrl (stats: MLocalVideoViewer) { |
diff --git a/server/lib/redis.ts b/server/lib/redis.ts index f9cea57cd..d052de786 100644 --- a/server/lib/redis.ts +++ b/server/lib/redis.ts | |||
@@ -145,18 +145,10 @@ class Redis { | |||
145 | return this.setValue(this.generateIPViewKey(ip, videoUUID), '1', VIEW_LIFETIME.VIEW) | 145 | return this.setValue(this.generateIPViewKey(ip, videoUUID), '1', VIEW_LIFETIME.VIEW) |
146 | } | 146 | } |
147 | 147 | ||
148 | setIPVideoViewer (ip: string, videoUUID: string) { | ||
149 | return this.setValue(this.generateIPViewerKey(ip, videoUUID), '1', VIEW_LIFETIME.VIEWER_COUNTER) | ||
150 | } | ||
151 | |||
152 | async doesVideoIPViewExist (ip: string, videoUUID: string) { | 148 | async doesVideoIPViewExist (ip: string, videoUUID: string) { |
153 | return this.exists(this.generateIPViewKey(ip, videoUUID)) | 149 | return this.exists(this.generateIPViewKey(ip, videoUUID)) |
154 | } | 150 | } |
155 | 151 | ||
156 | async doesVideoIPViewerExist (ip: string, videoUUID: string) { | ||
157 | return this.exists(this.generateIPViewerKey(ip, videoUUID)) | ||
158 | } | ||
159 | |||
160 | /* ************ Tracker IP block ************ */ | 152 | /* ************ Tracker IP block ************ */ |
161 | 153 | ||
162 | setTrackerBlockIP (ip: string) { | 154 | setTrackerBlockIP (ip: string) { |
@@ -361,10 +353,6 @@ class Redis { | |||
361 | return `views-${videoUUID}-${ip}` | 353 | return `views-${videoUUID}-${ip}` |
362 | } | 354 | } |
363 | 355 | ||
364 | private generateIPViewerKey (ip: string, videoUUID: string) { | ||
365 | return `viewer-${videoUUID}-${ip}` | ||
366 | } | ||
367 | |||
368 | private generateTrackerBlockIPKey (ip: string) { | 356 | private generateTrackerBlockIPKey (ip: string) { |
369 | return `tracker-block-ip-${ip}` | 357 | return `tracker-block-ip-${ip}` |
370 | } | 358 | } |
diff --git a/server/lib/views/shared/index.ts b/server/lib/views/shared/index.ts index dd510f4e2..139471183 100644 --- a/server/lib/views/shared/index.ts +++ b/server/lib/views/shared/index.ts | |||
@@ -1,2 +1,3 @@ | |||
1 | export * from './video-viewers' | 1 | export * from './video-viewer-counters' |
2 | export * from './video-viewer-stats' | ||
2 | export * from './video-views' | 3 | export * from './video-views' |
diff --git a/server/lib/views/shared/video-viewer-counters.ts b/server/lib/views/shared/video-viewer-counters.ts new file mode 100644 index 000000000..941b62ed7 --- /dev/null +++ b/server/lib/views/shared/video-viewer-counters.ts | |||
@@ -0,0 +1,176 @@ | |||
1 | |||
2 | import { isTestInstance } from '@server/helpers/core-utils' | ||
3 | import { logger, loggerTagsFactory } from '@server/helpers/logger' | ||
4 | import { VIEW_LIFETIME } from '@server/initializers/constants' | ||
5 | import { sendView } from '@server/lib/activitypub/send/send-view' | ||
6 | import { PeerTubeSocket } from '@server/lib/peertube-socket' | ||
7 | import { getServerActor } from '@server/models/application/application' | ||
8 | import { VideoModel } from '@server/models/video/video' | ||
9 | import { MVideo } from '@server/types/models' | ||
10 | import { buildUUID, sha256 } from '@shared/extra-utils' | ||
11 | |||
12 | const lTags = loggerTagsFactory('views') | ||
13 | |||
14 | type Viewer = { | ||
15 | expires: number | ||
16 | id: string | ||
17 | lastFederation?: number | ||
18 | } | ||
19 | |||
20 | export class VideoViewerCounters { | ||
21 | |||
22 | // expires is new Date().getTime() | ||
23 | private readonly viewersPerVideo = new Map<number, Viewer[]>() | ||
24 | private readonly idToViewer = new Map<string, Viewer>() | ||
25 | |||
26 | private readonly salt = buildUUID() | ||
27 | |||
28 | private processingViewerCounters = false | ||
29 | |||
30 | constructor () { | ||
31 | setInterval(() => this.cleanViewerCounters(), VIEW_LIFETIME.VIEWER_COUNTER) | ||
32 | } | ||
33 | |||
34 | // --------------------------------------------------------------------------- | ||
35 | |||
36 | async addLocalViewer (options: { | ||
37 | video: MVideo | ||
38 | ip: string | ||
39 | }) { | ||
40 | const { video, ip } = options | ||
41 | |||
42 | logger.debug('Adding local viewer to video viewers counter %s.', video.uuid, { ...lTags(video.uuid) }) | ||
43 | |||
44 | const viewerId = this.generateViewerId(ip, video.uuid) | ||
45 | const viewer = this.idToViewer.get(viewerId) | ||
46 | |||
47 | if (viewer) { | ||
48 | viewer.expires = this.buildViewerExpireTime() | ||
49 | await this.federateViewerIfNeeded(video, viewer) | ||
50 | |||
51 | return false | ||
52 | } | ||
53 | |||
54 | const newViewer = await this.addViewerToVideo({ viewerId, video }) | ||
55 | await this.federateViewerIfNeeded(video, newViewer) | ||
56 | |||
57 | return true | ||
58 | } | ||
59 | |||
60 | async addRemoteViewer (options: { | ||
61 | video: MVideo | ||
62 | viewerId: string | ||
63 | viewerExpires: Date | ||
64 | }) { | ||
65 | const { video, viewerExpires, viewerId } = options | ||
66 | |||
67 | logger.debug('Adding remote viewer to video %s.', video.uuid, { ...lTags(video.uuid) }) | ||
68 | |||
69 | await this.addViewerToVideo({ video, viewerExpires, viewerId }) | ||
70 | |||
71 | return true | ||
72 | } | ||
73 | |||
74 | // --------------------------------------------------------------------------- | ||
75 | |||
76 | getViewers (video: MVideo) { | ||
77 | const viewers = this.viewersPerVideo.get(video.id) | ||
78 | if (!viewers) return 0 | ||
79 | |||
80 | return viewers.length | ||
81 | } | ||
82 | |||
83 | buildViewerExpireTime () { | ||
84 | return new Date().getTime() + VIEW_LIFETIME.VIEWER_COUNTER | ||
85 | } | ||
86 | |||
87 | // --------------------------------------------------------------------------- | ||
88 | |||
89 | private async addViewerToVideo (options: { | ||
90 | video: MVideo | ||
91 | viewerId: string | ||
92 | viewerExpires?: Date | ||
93 | }) { | ||
94 | const { video, viewerExpires, viewerId } = options | ||
95 | |||
96 | let watchers = this.viewersPerVideo.get(video.id) | ||
97 | |||
98 | if (!watchers) { | ||
99 | watchers = [] | ||
100 | this.viewersPerVideo.set(video.id, watchers) | ||
101 | } | ||
102 | |||
103 | const expires = viewerExpires | ||
104 | ? viewerExpires.getTime() | ||
105 | : this.buildViewerExpireTime() | ||
106 | |||
107 | const viewer = { id: viewerId, expires } | ||
108 | watchers.push(viewer) | ||
109 | |||
110 | this.idToViewer.set(viewerId, viewer) | ||
111 | |||
112 | await this.notifyClients(video.id, watchers.length) | ||
113 | |||
114 | return viewer | ||
115 | } | ||
116 | |||
117 | private async cleanViewerCounters () { | ||
118 | if (this.processingViewerCounters) return | ||
119 | this.processingViewerCounters = true | ||
120 | |||
121 | if (!isTestInstance()) logger.info('Cleaning video viewers.', lTags()) | ||
122 | |||
123 | try { | ||
124 | for (const videoId of this.viewersPerVideo.keys()) { | ||
125 | const notBefore = new Date().getTime() | ||
126 | |||
127 | const viewers = this.viewersPerVideo.get(videoId) | ||
128 | |||
129 | // Only keep not expired viewers | ||
130 | const newViewers: Viewer[] = [] | ||
131 | |||
132 | // Filter new viewers | ||
133 | for (const viewer of viewers) { | ||
134 | if (viewer.expires > notBefore) { | ||
135 | newViewers.push(viewer) | ||
136 | } else { | ||
137 | this.idToViewer.delete(viewer.id) | ||
138 | } | ||
139 | } | ||
140 | |||
141 | if (newViewers.length === 0) this.viewersPerVideo.delete(videoId) | ||
142 | else this.viewersPerVideo.set(videoId, newViewers) | ||
143 | |||
144 | await this.notifyClients(videoId, newViewers.length) | ||
145 | } | ||
146 | } catch (err) { | ||
147 | logger.error('Error in video clean viewers scheduler.', { err, ...lTags() }) | ||
148 | } | ||
149 | |||
150 | this.processingViewerCounters = false | ||
151 | } | ||
152 | |||
153 | private async notifyClients (videoId: string | number, viewersLength: number) { | ||
154 | const video = await VideoModel.loadImmutableAttributes(videoId) | ||
155 | if (!video) return | ||
156 | |||
157 | PeerTubeSocket.Instance.sendVideoViewsUpdate(video, viewersLength) | ||
158 | |||
159 | logger.debug('Video viewers update for %s is %d.', video.url, viewersLength, lTags()) | ||
160 | } | ||
161 | |||
162 | private generateViewerId (ip: string, videoUUID: string) { | ||
163 | return sha256(this.salt + '-' + ip + '-' + videoUUID) | ||
164 | } | ||
165 | |||
166 | private async federateViewerIfNeeded (video: MVideo, viewer: Viewer) { | ||
167 | // Federate the viewer if it's been a "long" time we did not | ||
168 | const now = new Date().getTime() | ||
169 | const federationLimit = now - (VIEW_LIFETIME.VIEWER_COUNTER / 2) | ||
170 | |||
171 | if (viewer.lastFederation && viewer.lastFederation > federationLimit) return | ||
172 | |||
173 | await sendView({ byActor: await getServerActor(), video, type: 'viewer', viewerIdentifier: viewer.id }) | ||
174 | viewer.lastFederation = now | ||
175 | } | ||
176 | } | ||
diff --git a/server/lib/views/shared/video-viewers.ts b/server/lib/views/shared/video-viewer-stats.ts index 4dad1f0e8..fd66fd5c7 100644 --- a/server/lib/views/shared/video-viewers.ts +++ b/server/lib/views/shared/video-viewer-stats.ts | |||
@@ -6,7 +6,6 @@ import { MAX_LOCAL_VIEWER_WATCH_SECTIONS, VIEW_LIFETIME } from '@server/initiali | |||
6 | import { sequelizeTypescript } from '@server/initializers/database' | 6 | import { sequelizeTypescript } from '@server/initializers/database' |
7 | import { sendCreateWatchAction } from '@server/lib/activitypub/send' | 7 | import { sendCreateWatchAction } from '@server/lib/activitypub/send' |
8 | import { getLocalVideoViewerActivityPubUrl } from '@server/lib/activitypub/url' | 8 | import { getLocalVideoViewerActivityPubUrl } from '@server/lib/activitypub/url' |
9 | import { PeerTubeSocket } from '@server/lib/peertube-socket' | ||
10 | import { Redis } from '@server/lib/redis' | 9 | import { Redis } from '@server/lib/redis' |
11 | import { VideoModel } from '@server/models/video/video' | 10 | import { VideoModel } from '@server/models/video/video' |
12 | import { LocalVideoViewerModel } from '@server/models/view/local-video-viewer' | 11 | import { LocalVideoViewerModel } from '@server/models/view/local-video-viewer' |
@@ -32,39 +31,15 @@ type LocalViewerStats = { | |||
32 | videoId: number | 31 | videoId: number |
33 | } | 32 | } |
34 | 33 | ||
35 | export class VideoViewers { | 34 | export class VideoViewerStats { |
36 | 35 | private processingViewersStats = false | |
37 | // Values are Date().getTime() | ||
38 | private readonly viewersPerVideo = new Map<number, number[]>() | ||
39 | |||
40 | private processingViewerCounters = false | ||
41 | private processingViewerStats = false | ||
42 | 36 | ||
43 | constructor () { | 37 | constructor () { |
44 | setInterval(() => this.cleanViewerCounters(), VIEW_LIFETIME.VIEWER_COUNTER) | ||
45 | |||
46 | setInterval(() => this.processViewerStats(), VIEW_LIFETIME.VIEWER_STATS) | 38 | setInterval(() => this.processViewerStats(), VIEW_LIFETIME.VIEWER_STATS) |
47 | } | 39 | } |
48 | 40 | ||
49 | // --------------------------------------------------------------------------- | 41 | // --------------------------------------------------------------------------- |
50 | 42 | ||
51 | getViewers (video: MVideo) { | ||
52 | const viewers = this.viewersPerVideo.get(video.id) | ||
53 | if (!viewers) return 0 | ||
54 | |||
55 | return viewers.length | ||
56 | } | ||
57 | |||
58 | buildViewerExpireTime () { | ||
59 | return new Date().getTime() + VIEW_LIFETIME.VIEWER_COUNTER | ||
60 | } | ||
61 | |||
62 | async getWatchTime (videoId: number, ip: string) { | ||
63 | const stats: LocalViewerStats = await Redis.Instance.getLocalVideoViewer({ ip, videoId }) | ||
64 | |||
65 | return stats?.watchTime || 0 | ||
66 | } | ||
67 | |||
68 | async addLocalViewer (options: { | 43 | async addLocalViewer (options: { |
69 | video: MVideo | 44 | video: MVideo |
70 | currentTime: number | 45 | currentTime: number |
@@ -73,51 +48,20 @@ export class VideoViewers { | |||
73 | }) { | 48 | }) { |
74 | const { video, ip, viewEvent, currentTime } = options | 49 | const { video, ip, viewEvent, currentTime } = options |
75 | 50 | ||
76 | logger.debug('Adding local viewer to video %s.', video.uuid, { currentTime, viewEvent, ...lTags(video.uuid) }) | 51 | logger.debug('Adding local viewer to video stats %s.', video.uuid, { currentTime, viewEvent, ...lTags(video.uuid) }) |
77 | |||
78 | await this.updateLocalViewerStats({ video, viewEvent, currentTime, ip }) | ||
79 | |||
80 | const viewExists = await Redis.Instance.doesVideoIPViewerExist(ip, video.uuid) | ||
81 | if (viewExists) return false | ||
82 | 52 | ||
83 | await Redis.Instance.setIPVideoViewer(ip, video.uuid) | 53 | return this.updateLocalViewerStats({ video, viewEvent, currentTime, ip }) |
84 | |||
85 | return this.addViewerToVideo({ video }) | ||
86 | } | 54 | } |
87 | 55 | ||
88 | async addRemoteViewer (options: { | 56 | // --------------------------------------------------------------------------- |
89 | video: MVideo | ||
90 | viewerExpires: Date | ||
91 | }) { | ||
92 | const { video, viewerExpires } = options | ||
93 | 57 | ||
94 | logger.debug('Adding remote viewer to video %s.', video.uuid, { ...lTags(video.uuid) }) | 58 | async getWatchTime (videoId: number, ip: string) { |
59 | const stats: LocalViewerStats = await Redis.Instance.getLocalVideoViewer({ ip, videoId }) | ||
95 | 60 | ||
96 | return this.addViewerToVideo({ video, viewerExpires }) | 61 | return stats?.watchTime || 0 |
97 | } | 62 | } |
98 | 63 | ||
99 | private async addViewerToVideo (options: { | 64 | // --------------------------------------------------------------------------- |
100 | video: MVideo | ||
101 | viewerExpires?: Date | ||
102 | }) { | ||
103 | const { video, viewerExpires } = options | ||
104 | |||
105 | let watchers = this.viewersPerVideo.get(video.id) | ||
106 | |||
107 | if (!watchers) { | ||
108 | watchers = [] | ||
109 | this.viewersPerVideo.set(video.id, watchers) | ||
110 | } | ||
111 | |||
112 | const expiration = viewerExpires | ||
113 | ? viewerExpires.getTime() | ||
114 | : this.buildViewerExpireTime() | ||
115 | |||
116 | watchers.push(expiration) | ||
117 | await this.notifyClients(video.id, watchers.length) | ||
118 | |||
119 | return true | ||
120 | } | ||
121 | 65 | ||
122 | private async updateLocalViewerStats (options: { | 66 | private async updateLocalViewerStats (options: { |
123 | video: MVideo | 67 | video: MVideo |
@@ -170,45 +114,9 @@ export class VideoViewers { | |||
170 | await Redis.Instance.setLocalVideoViewer(ip, video.id, stats) | 114 | await Redis.Instance.setLocalVideoViewer(ip, video.id, stats) |
171 | } | 115 | } |
172 | 116 | ||
173 | private async cleanViewerCounters () { | ||
174 | if (this.processingViewerCounters) return | ||
175 | this.processingViewerCounters = true | ||
176 | |||
177 | if (!isTestInstance()) logger.info('Cleaning video viewers.', lTags()) | ||
178 | |||
179 | try { | ||
180 | for (const videoId of this.viewersPerVideo.keys()) { | ||
181 | const notBefore = new Date().getTime() | ||
182 | |||
183 | const viewers = this.viewersPerVideo.get(videoId) | ||
184 | |||
185 | // Only keep not expired viewers | ||
186 | const newViewers = viewers.filter(w => w > notBefore) | ||
187 | |||
188 | if (newViewers.length === 0) this.viewersPerVideo.delete(videoId) | ||
189 | else this.viewersPerVideo.set(videoId, newViewers) | ||
190 | |||
191 | await this.notifyClients(videoId, newViewers.length) | ||
192 | } | ||
193 | } catch (err) { | ||
194 | logger.error('Error in video clean viewers scheduler.', { err, ...lTags() }) | ||
195 | } | ||
196 | |||
197 | this.processingViewerCounters = false | ||
198 | } | ||
199 | |||
200 | private async notifyClients (videoId: string | number, viewersLength: number) { | ||
201 | const video = await VideoModel.loadImmutableAttributes(videoId) | ||
202 | if (!video) return | ||
203 | |||
204 | PeerTubeSocket.Instance.sendVideoViewsUpdate(video, viewersLength) | ||
205 | |||
206 | logger.debug('Video viewers update for %s is %d.', video.url, viewersLength, lTags()) | ||
207 | } | ||
208 | |||
209 | async processViewerStats () { | 117 | async processViewerStats () { |
210 | if (this.processingViewerStats) return | 118 | if (this.processingViewersStats) return |
211 | this.processingViewerStats = true | 119 | this.processingViewersStats = true |
212 | 120 | ||
213 | if (!isTestInstance()) logger.info('Processing viewer statistics.', lTags()) | 121 | if (!isTestInstance()) logger.info('Processing viewer statistics.', lTags()) |
214 | 122 | ||
@@ -245,7 +153,7 @@ export class VideoViewers { | |||
245 | logger.error('Error in video save viewers stats scheduler.', { err, ...lTags() }) | 153 | logger.error('Error in video save viewers stats scheduler.', { err, ...lTags() }) |
246 | } | 154 | } |
247 | 155 | ||
248 | this.processingViewerStats = false | 156 | this.processingViewersStats = false |
249 | } | 157 | } |
250 | 158 | ||
251 | private async saveViewerStats (video: MVideo, stats: LocalViewerStats, transaction: Transaction) { | 159 | private async saveViewerStats (video: MVideo, stats: LocalViewerStats, transaction: Transaction) { |
diff --git a/server/lib/views/shared/video-views.ts b/server/lib/views/shared/video-views.ts index 19250f993..275f7a014 100644 --- a/server/lib/views/shared/video-views.ts +++ b/server/lib/views/shared/video-views.ts | |||
@@ -1,5 +1,8 @@ | |||
1 | import { logger, loggerTagsFactory } from '@server/helpers/logger' | 1 | import { logger, loggerTagsFactory } from '@server/helpers/logger' |
2 | import { sendView } from '@server/lib/activitypub/send/send-view' | ||
3 | import { getServerActor } from '@server/models/application/application' | ||
2 | import { MVideo } from '@server/types/models' | 4 | import { MVideo } from '@server/types/models' |
5 | import { buildUUID } from '@shared/extra-utils' | ||
3 | import { Redis } from '../../redis' | 6 | import { Redis } from '../../redis' |
4 | 7 | ||
5 | const lTags = loggerTagsFactory('views') | 8 | const lTags = loggerTagsFactory('views') |
@@ -24,6 +27,8 @@ export class VideoViews { | |||
24 | 27 | ||
25 | await this.addView(video) | 28 | await this.addView(video) |
26 | 29 | ||
30 | await sendView({ byActor: await getServerActor(), video, type: 'view', viewerIdentifier: buildUUID() }) | ||
31 | |||
27 | return true | 32 | return true |
28 | } | 33 | } |
29 | 34 | ||
@@ -39,6 +44,8 @@ export class VideoViews { | |||
39 | return true | 44 | return true |
40 | } | 45 | } |
41 | 46 | ||
47 | // --------------------------------------------------------------------------- | ||
48 | |||
42 | private async addView (video: MVideo) { | 49 | private async addView (video: MVideo) { |
43 | const promises: Promise<any>[] = [] | 50 | const promises: Promise<any>[] = [] |
44 | 51 | ||
diff --git a/server/lib/views/video-views-manager.ts b/server/lib/views/video-views-manager.ts index 9382fb482..ea3b35c6c 100644 --- a/server/lib/views/video-views-manager.ts +++ b/server/lib/views/video-views-manager.ts | |||
@@ -1,7 +1,7 @@ | |||
1 | import { logger, loggerTagsFactory } from '@server/helpers/logger' | 1 | import { logger, loggerTagsFactory } from '@server/helpers/logger' |
2 | import { MVideo } from '@server/types/models' | 2 | import { MVideo } from '@server/types/models' |
3 | import { VideoViewEvent } from '@shared/models' | 3 | import { VideoViewEvent } from '@shared/models' |
4 | import { VideoViewers, VideoViews } from './shared' | 4 | import { VideoViewerCounters, VideoViewerStats, VideoViews } from './shared' |
5 | 5 | ||
6 | /** | 6 | /** |
7 | * If processing a local view: | 7 | * If processing a local view: |
@@ -27,14 +27,16 @@ export class VideoViewsManager { | |||
27 | 27 | ||
28 | private static instance: VideoViewsManager | 28 | private static instance: VideoViewsManager |
29 | 29 | ||
30 | private videoViewers: VideoViewers | 30 | private videoViewerStats: VideoViewerStats |
31 | private videoViewerCounters: VideoViewerCounters | ||
31 | private videoViews: VideoViews | 32 | private videoViews: VideoViews |
32 | 33 | ||
33 | private constructor () { | 34 | private constructor () { |
34 | } | 35 | } |
35 | 36 | ||
36 | init () { | 37 | init () { |
37 | this.videoViewers = new VideoViewers() | 38 | this.videoViewerStats = new VideoViewerStats() |
39 | this.videoViewerCounters = new VideoViewerCounters() | ||
38 | this.videoViews = new VideoViews() | 40 | this.videoViews = new VideoViews() |
39 | } | 41 | } |
40 | 42 | ||
@@ -48,10 +50,12 @@ export class VideoViewsManager { | |||
48 | 50 | ||
49 | logger.debug('Processing local view for %s and ip %s.', video.url, ip, lTags()) | 51 | logger.debug('Processing local view for %s and ip %s.', video.url, ip, lTags()) |
50 | 52 | ||
51 | const successViewer = await this.videoViewers.addLocalViewer({ video, ip, viewEvent, currentTime }) | 53 | await this.videoViewerStats.addLocalViewer({ video, ip, viewEvent, currentTime }) |
54 | |||
55 | const successViewer = await this.videoViewerCounters.addLocalViewer({ video, ip }) | ||
52 | 56 | ||
53 | // Do it after added local viewer to fetch updated information | 57 | // Do it after added local viewer to fetch updated information |
54 | const watchTime = await this.videoViewers.getWatchTime(video.id, ip) | 58 | const watchTime = await this.videoViewerStats.getWatchTime(video.id, ip) |
55 | 59 | ||
56 | const successView = await this.videoViews.addLocalView({ video, watchTime, ip }) | 60 | const successView = await this.videoViews.addLocalView({ video, watchTime, ip }) |
57 | 61 | ||
@@ -60,26 +64,27 @@ export class VideoViewsManager { | |||
60 | 64 | ||
61 | async processRemoteView (options: { | 65 | async processRemoteView (options: { |
62 | video: MVideo | 66 | video: MVideo |
67 | viewerId: string | null | ||
63 | viewerExpires?: Date | 68 | viewerExpires?: Date |
64 | }) { | 69 | }) { |
65 | const { video, viewerExpires } = options | 70 | const { video, viewerId, viewerExpires } = options |
66 | 71 | ||
67 | logger.debug('Processing remote view for %s.', video.url, { viewerExpires, ...lTags() }) | 72 | logger.debug('Processing remote view for %s.', video.url, { viewerExpires, viewerId, ...lTags() }) |
68 | 73 | ||
69 | if (viewerExpires) await this.videoViewers.addRemoteViewer({ video, viewerExpires }) | 74 | if (viewerExpires) await this.videoViewerCounters.addRemoteViewer({ video, viewerId, viewerExpires }) |
70 | else await this.videoViews.addRemoteView({ video }) | 75 | else await this.videoViews.addRemoteView({ video }) |
71 | } | 76 | } |
72 | 77 | ||
73 | getViewers (video: MVideo) { | 78 | getViewers (video: MVideo) { |
74 | return this.videoViewers.getViewers(video) | 79 | return this.videoViewerCounters.getViewers(video) |
75 | } | 80 | } |
76 | 81 | ||
77 | buildViewerExpireTime () { | 82 | buildViewerExpireTime () { |
78 | return this.videoViewers.buildViewerExpireTime() | 83 | return this.videoViewerCounters.buildViewerExpireTime() |
79 | } | 84 | } |
80 | 85 | ||
81 | processViewers () { | 86 | processViewerStats () { |
82 | return this.videoViewers.processViewerStats() | 87 | return this.videoViewerStats.processViewerStats() |
83 | } | 88 | } |
84 | 89 | ||
85 | static get Instance () { | 90 | static get Instance () { |
diff --git a/server/tests/api/check-params/views.ts b/server/tests/api/check-params/views.ts index 185b04af1..ca4752345 100644 --- a/server/tests/api/check-params/views.ts +++ b/server/tests/api/check-params/views.ts | |||
@@ -19,7 +19,7 @@ describe('Test videos views', function () { | |||
19 | let userAccessToken: string | 19 | let userAccessToken: string |
20 | 20 | ||
21 | before(async function () { | 21 | before(async function () { |
22 | this.timeout(30000) | 22 | this.timeout(120000) |
23 | 23 | ||
24 | servers = await createMultipleServers(2) | 24 | servers = await createMultipleServers(2) |
25 | await setAccessTokensToServers(servers) | 25 | await setAccessTokensToServers(servers) |
diff --git a/server/tests/api/views/video-views-counter.ts b/server/tests/api/views/video-views-counter.ts index b68aaa350..b8969d52d 100644 --- a/server/tests/api/views/video-views-counter.ts +++ b/server/tests/api/views/video-views-counter.ts | |||
@@ -57,10 +57,11 @@ describe('Test video views/viewers counters', function () { | |||
57 | }) | 57 | }) |
58 | 58 | ||
59 | it('Should not view again this video with the same IP', async function () { | 59 | it('Should not view again this video with the same IP', async function () { |
60 | await servers[0].views.simulateViewer({ id: videoUUID, currentTimes: [ 1, 4 ] }) | 60 | await servers[0].views.simulateViewer({ id: videoUUID, xForwardedFor: '0.0.0.1,127.0.0.1', currentTimes: [ 1, 4 ] }) |
61 | await servers[0].views.simulateViewer({ id: videoUUID, xForwardedFor: '0.0.0.1,127.0.0.1', currentTimes: [ 1, 4 ] }) | ||
61 | await processViewsBuffer(servers) | 62 | await processViewsBuffer(servers) |
62 | 63 | ||
63 | await checkCounter('views', videoUUID, 1) | 64 | await checkCounter('views', videoUUID, 2) |
64 | }) | 65 | }) |
65 | 66 | ||
66 | it('Should view the video from server 2 and send the event', async function () { | 67 | it('Should view the video from server 2 and send the event', async function () { |
@@ -68,7 +69,7 @@ describe('Test video views/viewers counters', function () { | |||
68 | await waitJobs(servers) | 69 | await waitJobs(servers) |
69 | await processViewsBuffer(servers) | 70 | await processViewsBuffer(servers) |
70 | 71 | ||
71 | await checkCounter('views', videoUUID, 2) | 72 | await checkCounter('views', videoUUID, 3) |
72 | }) | 73 | }) |
73 | }) | 74 | }) |
74 | 75 | ||
@@ -78,7 +79,7 @@ describe('Test video views/viewers counters', function () { | |||
78 | let command: FfmpegCommand | 79 | let command: FfmpegCommand |
79 | 80 | ||
80 | before(async function () { | 81 | before(async function () { |
81 | this.timeout(60000); | 82 | this.timeout(120000); |
82 | 83 | ||
83 | ({ vodVideoId, liveVideoId, ffmpegCommand: command } = await prepareViewsVideos({ servers, live: true, vod: true })) | 84 | ({ vodVideoId, liveVideoId, ffmpegCommand: command } = await prepareViewsVideos({ servers, live: true, vod: true })) |
84 | }) | 85 | }) |
diff --git a/server/tests/api/views/video-views-overall-stats.ts b/server/tests/api/views/video-views-overall-stats.ts index 22761d6ec..d8de73cbc 100644 --- a/server/tests/api/views/video-views-overall-stats.ts +++ b/server/tests/api/views/video-views-overall-stats.ts | |||
@@ -21,7 +21,7 @@ describe('Test views overall stats', function () { | |||
21 | let vodVideoId: string | 21 | let vodVideoId: string |
22 | 22 | ||
23 | before(async function () { | 23 | before(async function () { |
24 | this.timeout(60000); | 24 | this.timeout(120000); |
25 | 25 | ||
26 | ({ vodVideoId } = await prepareViewsVideos({ servers, live: false, vod: true })) | 26 | ({ vodVideoId } = await prepareViewsVideos({ servers, live: false, vod: true })) |
27 | }) | 27 | }) |
@@ -74,7 +74,7 @@ describe('Test views overall stats', function () { | |||
74 | let command: FfmpegCommand | 74 | let command: FfmpegCommand |
75 | 75 | ||
76 | before(async function () { | 76 | before(async function () { |
77 | this.timeout(60000); | 77 | this.timeout(120000); |
78 | 78 | ||
79 | ({ vodVideoId, liveVideoId, ffmpegCommand: command } = await prepareViewsVideos({ servers, live: true, vod: true })) | 79 | ({ vodVideoId, liveVideoId, ffmpegCommand: command } = await prepareViewsVideos({ servers, live: true, vod: true })) |
80 | }) | 80 | }) |
@@ -189,7 +189,7 @@ describe('Test views overall stats', function () { | |||
189 | let videoUUID: string | 189 | let videoUUID: string |
190 | 190 | ||
191 | before(async function () { | 191 | before(async function () { |
192 | this.timeout(60000); | 192 | this.timeout(120000); |
193 | 193 | ||
194 | ({ vodVideoId: videoUUID } = await prepareViewsVideos({ servers, live: true, vod: true })) | 194 | ({ vodVideoId: videoUUID } = await prepareViewsVideos({ servers, live: true, vod: true })) |
195 | }) | 195 | }) |
diff --git a/server/tests/api/views/video-views-retention-stats.ts b/server/tests/api/views/video-views-retention-stats.ts index 98be7bfdb..a27141d68 100644 --- a/server/tests/api/views/video-views-retention-stats.ts +++ b/server/tests/api/views/video-views-retention-stats.ts | |||
@@ -20,7 +20,7 @@ describe('Test views retention stats', function () { | |||
20 | let vodVideoId: string | 20 | let vodVideoId: string |
21 | 21 | ||
22 | before(async function () { | 22 | before(async function () { |
23 | this.timeout(60000); | 23 | this.timeout(120000); |
24 | 24 | ||
25 | ({ vodVideoId } = await prepareViewsVideos({ servers, live: false, vod: true })) | 25 | ({ vodVideoId } = await prepareViewsVideos({ servers, live: false, vod: true })) |
26 | }) | 26 | }) |
diff --git a/server/tests/api/views/video-views-timeserie-stats.ts b/server/tests/api/views/video-views-timeserie-stats.ts index 98c041cdf..858edeff7 100644 --- a/server/tests/api/views/video-views-timeserie-stats.ts +++ b/server/tests/api/views/video-views-timeserie-stats.ts | |||
@@ -24,7 +24,7 @@ describe('Test views timeserie stats', function () { | |||
24 | let vodVideoId: string | 24 | let vodVideoId: string |
25 | 25 | ||
26 | before(async function () { | 26 | before(async function () { |
27 | this.timeout(60000); | 27 | this.timeout(120000); |
28 | 28 | ||
29 | ({ vodVideoId } = await prepareViewsVideos({ servers, live: false, vod: true })) | 29 | ({ vodVideoId } = await prepareViewsVideos({ servers, live: false, vod: true })) |
30 | }) | 30 | }) |
@@ -63,7 +63,7 @@ describe('Test views timeserie stats', function () { | |||
63 | } | 63 | } |
64 | 64 | ||
65 | before(async function () { | 65 | before(async function () { |
66 | this.timeout(60000); | 66 | this.timeout(120000); |
67 | 67 | ||
68 | ({ vodVideoId, liveVideoId, ffmpegCommand: command } = await prepareViewsVideos({ servers, live: true, vod: true })) | 68 | ({ vodVideoId, liveVideoId, ffmpegCommand: command } = await prepareViewsVideos({ servers, live: true, vod: true })) |
69 | }) | 69 | }) |