diff options
author | Chocobozzz <me@florianbigard.com> | 2018-08-29 16:26:25 +0200 |
---|---|---|
committer | Chocobozzz <me@florianbigard.com> | 2018-08-30 15:03:18 +0200 |
commit | 6b6168606bc86430f6b7821c9d5f1c80d0425ebf (patch) | |
tree | 9aea6cf0875c9fee30c373eb4924b12d47d1e23c | |
parent | 2d9fea161fd4fc73994fc77951bafdccdc2071fd (diff) | |
download | PeerTube-6b6168606bc86430f6b7821c9d5f1c80d0425ebf.tar.gz PeerTube-6b6168606bc86430f6b7821c9d5f1c80d0425ebf.tar.zst PeerTube-6b6168606bc86430f6b7821c9d5f1c80d0425ebf.zip |
Bufferize videos views in redis
-rw-r--r-- | .github/ISSUE_TEMPLATE.md | 4 | ||||
-rw-r--r-- | server/controllers/api/videos/index.ts | 8 | ||||
-rw-r--r-- | server/initializers/constants.ts | 20 | ||||
-rw-r--r-- | server/initializers/database.ts | 4 | ||||
-rw-r--r-- | server/lib/activitypub/process/process-create.ts | 4 | ||||
-rw-r--r-- | server/lib/job-queue/handlers/video-views.ts | 40 | ||||
-rw-r--r-- | server/lib/job-queue/job-queue.ts | 20 | ||||
-rw-r--r-- | server/lib/redis.ts | 88 | ||||
-rw-r--r-- | server/models/video/video-views.ts | 41 | ||||
-rw-r--r-- | server/models/video/video.ts | 9 | ||||
-rw-r--r-- | server/tests/api/server/reverse-proxy.ts | 33 | ||||
-rw-r--r-- | server/tests/api/server/stats.ts | 3 | ||||
-rw-r--r-- | server/tests/api/videos/multiple-servers.ts | 10 | ||||
-rw-r--r-- | server/tests/api/videos/single-server.ts | 5 | ||||
-rw-r--r-- | server/tests/utils/server/jobs.ts | 8 | ||||
-rw-r--r-- | shared/models/server/job.model.ts | 3 |
16 files changed, 274 insertions, 26 deletions
diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md index 6b34c2cf7..8edb00b9c 100644 --- a/.github/ISSUE_TEMPLATE.md +++ b/.github/ISSUE_TEMPLATE.md | |||
@@ -11,6 +11,6 @@ | |||
11 | * **What do you see instead?** | 11 | * **What do you see instead?** |
12 | 12 | ||
13 | 13 | ||
14 | * **Browser console log if useful (Gist/Pastebin...):** | 14 | * **Browser console log if useful:** |
15 | * **Server log if useful (Gist/Pastebin...):** | 15 | * **Server log if useful (journalctl or /var/www/peertube/storage/logs):** |
16 | 16 | ||
diff --git a/server/controllers/api/videos/index.ts b/server/controllers/api/videos/index.ts index a86cf4f99..be803490b 100644 --- a/server/controllers/api/videos/index.ts +++ b/server/controllers/api/videos/index.ts | |||
@@ -380,14 +380,16 @@ async function viewVideo (req: express.Request, res: express.Response) { | |||
380 | const videoInstance = res.locals.video | 380 | const videoInstance = res.locals.video |
381 | 381 | ||
382 | const ip = req.ip | 382 | const ip = req.ip |
383 | const exists = await Redis.Instance.isViewExists(ip, videoInstance.uuid) | 383 | const exists = await Redis.Instance.isVideoIPViewExists(ip, videoInstance.uuid) |
384 | if (exists) { | 384 | if (exists) { |
385 | logger.debug('View for ip %s and video %s already exists.', ip, videoInstance.uuid) | 385 | logger.debug('View for ip %s and video %s already exists.', ip, videoInstance.uuid) |
386 | return res.status(204).end() | 386 | return res.status(204).end() |
387 | } | 387 | } |
388 | 388 | ||
389 | await videoInstance.increment('views') | 389 | await Promise.all([ |
390 | await Redis.Instance.setView(ip, videoInstance.uuid) | 390 | Redis.Instance.addVideoView(videoInstance.id), |
391 | Redis.Instance.setIPVideoView(ip, videoInstance.uuid) | ||
392 | ]) | ||
391 | 393 | ||
392 | const serverAccount = await getServerActor() | 394 | const serverAccount = await getServerActor() |
393 | 395 | ||
diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts index 7f1b25654..2d9a2e670 100644 --- a/server/initializers/constants.ts +++ b/server/initializers/constants.ts | |||
@@ -3,11 +3,12 @@ import { dirname, join } from 'path' | |||
3 | import { JobType, VideoRateType, VideoState } from '../../shared/models' | 3 | import { JobType, VideoRateType, VideoState } from '../../shared/models' |
4 | import { ActivityPubActorType } from '../../shared/models/activitypub' | 4 | import { ActivityPubActorType } from '../../shared/models/activitypub' |
5 | import { FollowState } from '../../shared/models/actors' | 5 | import { FollowState } from '../../shared/models/actors' |
6 | import { VideoPrivacy, VideoAbuseState, VideoImportState } from '../../shared/models/videos' | 6 | import { VideoAbuseState, VideoImportState, VideoPrivacy } from '../../shared/models/videos' |
7 | // Do not use barrels, remain constants as independent as possible | 7 | // Do not use barrels, remain constants as independent as possible |
8 | import { buildPath, isTestInstance, root, sanitizeHost, sanitizeUrl } from '../helpers/core-utils' | 8 | import { buildPath, isTestInstance, root, sanitizeHost, sanitizeUrl } from '../helpers/core-utils' |
9 | import { NSFWPolicyType } from '../../shared/models/videos/nsfw-policy.type' | 9 | import { NSFWPolicyType } from '../../shared/models/videos/nsfw-policy.type' |
10 | import { invert } from 'lodash' | 10 | import { invert } from 'lodash' |
11 | import { CronRepeatOptions, EveryRepeatOptions } from 'bull' | ||
11 | 12 | ||
12 | // Use a variable to reload the configuration if we need | 13 | // Use a variable to reload the configuration if we need |
13 | let config: IConfig = require('config') | 14 | let config: IConfig = require('config') |
@@ -90,7 +91,8 @@ const JOB_ATTEMPTS: { [ id in JobType ]: number } = { | |||
90 | 'video-file-import': 1, | 91 | 'video-file-import': 1, |
91 | 'video-file': 1, | 92 | 'video-file': 1, |
92 | 'video-import': 1, | 93 | 'video-import': 1, |
93 | 'email': 5 | 94 | 'email': 5, |
95 | 'videos-views': 1 | ||
94 | } | 96 | } |
95 | const JOB_CONCURRENCY: { [ id in JobType ]: number } = { | 97 | const JOB_CONCURRENCY: { [ id in JobType ]: number } = { |
96 | 'activitypub-http-broadcast': 1, | 98 | 'activitypub-http-broadcast': 1, |
@@ -100,7 +102,8 @@ const JOB_CONCURRENCY: { [ id in JobType ]: number } = { | |||
100 | 'video-file-import': 1, | 102 | 'video-file-import': 1, |
101 | 'video-file': 1, | 103 | 'video-file': 1, |
102 | 'video-import': 1, | 104 | 'video-import': 1, |
103 | 'email': 5 | 105 | 'email': 5, |
106 | 'videos-views': 1 | ||
104 | } | 107 | } |
105 | const JOB_TTL: { [ id in JobType ]: number } = { | 108 | const JOB_TTL: { [ id in JobType ]: number } = { |
106 | 'activitypub-http-broadcast': 60000 * 10, // 10 minutes | 109 | 'activitypub-http-broadcast': 60000 * 10, // 10 minutes |
@@ -110,8 +113,15 @@ const JOB_TTL: { [ id in JobType ]: number } = { | |||
110 | 'video-file-import': 1000 * 3600, // 1 hour | 113 | 'video-file-import': 1000 * 3600, // 1 hour |
111 | 'video-file': 1000 * 3600 * 48, // 2 days, transcoding could be long | 114 | 'video-file': 1000 * 3600 * 48, // 2 days, transcoding could be long |
112 | 'video-import': 1000 * 3600 * 5, // 5 hours | 115 | 'video-import': 1000 * 3600 * 5, // 5 hours |
113 | 'email': 60000 * 10 // 10 minutes | 116 | 'email': 60000 * 10, // 10 minutes |
117 | 'videos-views': undefined // Unlimited | ||
114 | } | 118 | } |
119 | const REPEAT_JOBS: { [ id: string ]: EveryRepeatOptions | CronRepeatOptions } = { | ||
120 | 'videos-views': { | ||
121 | cron: '1 * * * *' // At 1 minutes past the hour | ||
122 | } | ||
123 | } | ||
124 | |||
115 | const BROADCAST_CONCURRENCY = 10 // How many requests in parallel we do in activitypub-http-broadcast job | 125 | const BROADCAST_CONCURRENCY = 10 // How many requests in parallel we do in activitypub-http-broadcast job |
116 | const CRAWL_REQUEST_CONCURRENCY = 1 // How many requests in parallel to fetch remote data (likes, shares...) | 126 | const CRAWL_REQUEST_CONCURRENCY = 1 // How many requests in parallel to fetch remote data (likes, shares...) |
117 | const JOB_REQUEST_TIMEOUT = 3000 // 3 seconds | 127 | const JOB_REQUEST_TIMEOUT = 3000 // 3 seconds |
@@ -591,6 +601,7 @@ if (isTestInstance() === true) { | |||
591 | SCHEDULER_INTERVALS_MS.badActorFollow = 10000 | 601 | SCHEDULER_INTERVALS_MS.badActorFollow = 10000 |
592 | SCHEDULER_INTERVALS_MS.removeOldJobs = 10000 | 602 | SCHEDULER_INTERVALS_MS.removeOldJobs = 10000 |
593 | SCHEDULER_INTERVALS_MS.updateVideos = 5000 | 603 | SCHEDULER_INTERVALS_MS.updateVideos = 5000 |
604 | REPEAT_JOBS['videos-views'] = { every: 5000 } | ||
594 | 605 | ||
595 | VIDEO_VIEW_LIFETIME = 1000 // 1 second | 606 | VIDEO_VIEW_LIFETIME = 1000 // 1 second |
596 | 607 | ||
@@ -652,6 +663,7 @@ export { | |||
652 | USER_PASSWORD_RESET_LIFETIME, | 663 | USER_PASSWORD_RESET_LIFETIME, |
653 | IMAGE_MIMETYPE_EXT, | 664 | IMAGE_MIMETYPE_EXT, |
654 | SCHEDULER_INTERVALS_MS, | 665 | SCHEDULER_INTERVALS_MS, |
666 | REPEAT_JOBS, | ||
655 | STATIC_DOWNLOAD_PATHS, | 667 | STATIC_DOWNLOAD_PATHS, |
656 | RATES_LIMIT, | 668 | RATES_LIMIT, |
657 | VIDEO_EXT_MIMETYPE, | 669 | VIDEO_EXT_MIMETYPE, |
diff --git a/server/initializers/database.ts b/server/initializers/database.ts index 0be752363..78bc8101c 100644 --- a/server/initializers/database.ts +++ b/server/initializers/database.ts | |||
@@ -25,6 +25,7 @@ import { CONFIG } from './constants' | |||
25 | import { ScheduleVideoUpdateModel } from '../models/video/schedule-video-update' | 25 | import { ScheduleVideoUpdateModel } from '../models/video/schedule-video-update' |
26 | import { VideoCaptionModel } from '../models/video/video-caption' | 26 | import { VideoCaptionModel } from '../models/video/video-caption' |
27 | import { VideoImportModel } from '../models/video/video-import' | 27 | import { VideoImportModel } from '../models/video/video-import' |
28 | import { VideoViewModel } from '../models/video/video-views' | ||
28 | 29 | ||
29 | require('pg').defaults.parseInt8 = true // Avoid BIGINT to be converted to string | 30 | require('pg').defaults.parseInt8 = true // Avoid BIGINT to be converted to string |
30 | 31 | ||
@@ -83,7 +84,8 @@ async function initDatabaseModels (silent: boolean) { | |||
83 | VideoModel, | 84 | VideoModel, |
84 | VideoCommentModel, | 85 | VideoCommentModel, |
85 | ScheduleVideoUpdateModel, | 86 | ScheduleVideoUpdateModel, |
86 | VideoImportModel | 87 | VideoImportModel, |
88 | VideoViewModel | ||
87 | ]) | 89 | ]) |
88 | 90 | ||
89 | // Check extensions exist in the database | 91 | // Check extensions exist in the database |
diff --git a/server/lib/activitypub/process/process-create.ts b/server/lib/activitypub/process/process-create.ts index 75f07d131..16f426e23 100644 --- a/server/lib/activitypub/process/process-create.ts +++ b/server/lib/activitypub/process/process-create.ts | |||
@@ -7,11 +7,11 @@ import { sequelizeTypescript } from '../../../initializers' | |||
7 | import { AccountVideoRateModel } from '../../../models/account/account-video-rate' | 7 | import { AccountVideoRateModel } from '../../../models/account/account-video-rate' |
8 | import { ActorModel } from '../../../models/activitypub/actor' | 8 | import { ActorModel } from '../../../models/activitypub/actor' |
9 | import { VideoAbuseModel } from '../../../models/video/video-abuse' | 9 | import { VideoAbuseModel } from '../../../models/video/video-abuse' |
10 | import { VideoCommentModel } from '../../../models/video/video-comment' | ||
11 | import { getOrCreateActorAndServerAndModel } from '../actor' | 10 | import { getOrCreateActorAndServerAndModel } from '../actor' |
12 | import { addVideoComment, resolveThread } from '../video-comments' | 11 | import { addVideoComment, resolveThread } from '../video-comments' |
13 | import { getOrCreateVideoAndAccountAndChannel } from '../videos' | 12 | import { getOrCreateVideoAndAccountAndChannel } from '../videos' |
14 | import { forwardActivity, forwardVideoRelatedActivity } from '../send/utils' | 13 | import { forwardActivity, forwardVideoRelatedActivity } from '../send/utils' |
14 | import { Redis } from '../../redis' | ||
15 | 15 | ||
16 | async function processCreateActivity (activity: ActivityCreate) { | 16 | async function processCreateActivity (activity: ActivityCreate) { |
17 | const activityObject = activity.object | 17 | const activityObject = activity.object |
@@ -88,7 +88,7 @@ async function processCreateView (byActor: ActorModel, activity: ActivityCreate) | |||
88 | const actor = await ActorModel.loadByUrl(view.actor) | 88 | const actor = await ActorModel.loadByUrl(view.actor) |
89 | if (!actor) throw new Error('Unknown actor ' + view.actor) | 89 | if (!actor) throw new Error('Unknown actor ' + view.actor) |
90 | 90 | ||
91 | await video.increment('views') | 91 | await Redis.Instance.addVideoView(video.id) |
92 | 92 | ||
93 | if (video.isOwned()) { | 93 | if (video.isOwned()) { |
94 | // Don't resend the activity to the sender | 94 | // Don't resend the activity to the sender |
diff --git a/server/lib/job-queue/handlers/video-views.ts b/server/lib/job-queue/handlers/video-views.ts new file mode 100644 index 000000000..875d8ab88 --- /dev/null +++ b/server/lib/job-queue/handlers/video-views.ts | |||
@@ -0,0 +1,40 @@ | |||
1 | import { Redis } from '../../redis' | ||
2 | import { logger } from '../../../helpers/logger' | ||
3 | import { VideoModel } from '../../../models/video/video' | ||
4 | import { VideoViewModel } from '../../../models/video/video-views' | ||
5 | |||
6 | async function processVideosViewsViews () { | ||
7 | const hour = new Date().getHours() | ||
8 | const startDate = new Date().setMinutes(0, 0, 0) | ||
9 | const endDate = new Date().setMinutes(59, 59, 999) | ||
10 | |||
11 | const videoIds = await Redis.Instance.getVideosIdViewed(hour) | ||
12 | if (videoIds.length === 0) return | ||
13 | |||
14 | logger.info('Processing videos views in job for hour %d.', hour) | ||
15 | |||
16 | for (const videoId of videoIds) { | ||
17 | const views = await Redis.Instance.getVideoViews(videoId, hour) | ||
18 | if (isNaN(views)) { | ||
19 | logger.error('Cannot process videos views of video %s in hour %d: views number is NaN.', videoId, hour) | ||
20 | } else { | ||
21 | logger.debug('Adding %d views to video %d in hour %d.', views, videoId, hour) | ||
22 | |||
23 | await VideoModel.incrementViews(videoId, views) | ||
24 | await VideoViewModel.create({ | ||
25 | startDate, | ||
26 | endDate, | ||
27 | views, | ||
28 | videoId | ||
29 | }) | ||
30 | } | ||
31 | |||
32 | await Redis.Instance.deleteVideoViews(videoId, hour) | ||
33 | } | ||
34 | } | ||
35 | |||
36 | // --------------------------------------------------------------------------- | ||
37 | |||
38 | export { | ||
39 | processVideosViewsViews | ||
40 | } | ||
diff --git a/server/lib/job-queue/job-queue.ts b/server/lib/job-queue/job-queue.ts index ddb357db5..0696ba43c 100644 --- a/server/lib/job-queue/job-queue.ts +++ b/server/lib/job-queue/job-queue.ts | |||
@@ -2,7 +2,7 @@ import * as Bull from 'bull' | |||
2 | import { JobState, JobType } from '../../../shared/models' | 2 | import { JobState, JobType } from '../../../shared/models' |
3 | import { logger } from '../../helpers/logger' | 3 | import { logger } from '../../helpers/logger' |
4 | import { Redis } from '../redis' | 4 | import { Redis } from '../redis' |
5 | import { CONFIG, JOB_ATTEMPTS, JOB_COMPLETED_LIFETIME, JOB_CONCURRENCY, JOB_TTL } from '../../initializers' | 5 | import { CONFIG, JOB_ATTEMPTS, JOB_COMPLETED_LIFETIME, JOB_CONCURRENCY, JOB_TTL, REPEAT_JOBS } from '../../initializers' |
6 | import { ActivitypubHttpBroadcastPayload, processActivityPubHttpBroadcast } from './handlers/activitypub-http-broadcast' | 6 | import { ActivitypubHttpBroadcastPayload, processActivityPubHttpBroadcast } from './handlers/activitypub-http-broadcast' |
7 | import { ActivitypubHttpFetcherPayload, processActivityPubHttpFetcher } from './handlers/activitypub-http-fetcher' | 7 | import { ActivitypubHttpFetcherPayload, processActivityPubHttpFetcher } from './handlers/activitypub-http-fetcher' |
8 | import { ActivitypubHttpUnicastPayload, processActivityPubHttpUnicast } from './handlers/activitypub-http-unicast' | 8 | import { ActivitypubHttpUnicastPayload, processActivityPubHttpUnicast } from './handlers/activitypub-http-unicast' |
@@ -10,6 +10,7 @@ import { EmailPayload, processEmail } from './handlers/email' | |||
10 | import { processVideoFile, processVideoFileImport, VideoFileImportPayload, VideoFilePayload } from './handlers/video-file' | 10 | import { processVideoFile, processVideoFileImport, VideoFileImportPayload, VideoFilePayload } from './handlers/video-file' |
11 | import { ActivitypubFollowPayload, processActivityPubFollow } from './handlers/activitypub-follow' | 11 | import { ActivitypubFollowPayload, processActivityPubFollow } from './handlers/activitypub-follow' |
12 | import { processVideoImport, VideoImportPayload } from './handlers/video-import' | 12 | import { processVideoImport, VideoImportPayload } from './handlers/video-import' |
13 | import { processVideosViewsViews } from './handlers/video-views' | ||
13 | 14 | ||
14 | type CreateJobArgument = | 15 | type CreateJobArgument = |
15 | { type: 'activitypub-http-broadcast', payload: ActivitypubHttpBroadcastPayload } | | 16 | { type: 'activitypub-http-broadcast', payload: ActivitypubHttpBroadcastPayload } | |
@@ -19,7 +20,8 @@ type CreateJobArgument = | |||
19 | { type: 'video-file-import', payload: VideoFileImportPayload } | | 20 | { type: 'video-file-import', payload: VideoFileImportPayload } | |
20 | { type: 'video-file', payload: VideoFilePayload } | | 21 | { type: 'video-file', payload: VideoFilePayload } | |
21 | { type: 'email', payload: EmailPayload } | | 22 | { type: 'email', payload: EmailPayload } | |
22 | { type: 'video-import', payload: VideoImportPayload } | 23 | { type: 'video-import', payload: VideoImportPayload } | |
24 | { type: 'videos-views', payload: {} } | ||
23 | 25 | ||
24 | const handlers: { [ id in JobType ]: (job: Bull.Job) => Promise<any>} = { | 26 | const handlers: { [ id in JobType ]: (job: Bull.Job) => Promise<any>} = { |
25 | 'activitypub-http-broadcast': processActivityPubHttpBroadcast, | 27 | 'activitypub-http-broadcast': processActivityPubHttpBroadcast, |
@@ -29,7 +31,8 @@ const handlers: { [ id in JobType ]: (job: Bull.Job) => Promise<any>} = { | |||
29 | 'video-file-import': processVideoFileImport, | 31 | 'video-file-import': processVideoFileImport, |
30 | 'video-file': processVideoFile, | 32 | 'video-file': processVideoFile, |
31 | 'email': processEmail, | 33 | 'email': processEmail, |
32 | 'video-import': processVideoImport | 34 | 'video-import': processVideoImport, |
35 | 'videos-views': processVideosViewsViews | ||
33 | } | 36 | } |
34 | 37 | ||
35 | const jobTypes: JobType[] = [ | 38 | const jobTypes: JobType[] = [ |
@@ -40,7 +43,8 @@ const jobTypes: JobType[] = [ | |||
40 | 'email', | 43 | 'email', |
41 | 'video-file', | 44 | 'video-file', |
42 | 'video-file-import', | 45 | 'video-file-import', |
43 | 'video-import' | 46 | 'video-import', |
47 | 'videos-views' | ||
44 | ] | 48 | ] |
45 | 49 | ||
46 | class JobQueue { | 50 | class JobQueue { |
@@ -85,6 +89,8 @@ class JobQueue { | |||
85 | 89 | ||
86 | this.queues[handlerName] = queue | 90 | this.queues[handlerName] = queue |
87 | } | 91 | } |
92 | |||
93 | this.addRepeatableJobs() | ||
88 | } | 94 | } |
89 | 95 | ||
90 | terminate () { | 96 | terminate () { |
@@ -163,6 +169,12 @@ class JobQueue { | |||
163 | } | 169 | } |
164 | } | 170 | } |
165 | 171 | ||
172 | private addRepeatableJobs () { | ||
173 | this.queues['videos-views'].add({}, { | ||
174 | repeat: REPEAT_JOBS['videos-views'] | ||
175 | }) | ||
176 | } | ||
177 | |||
166 | static get Instance () { | 178 | static get Instance () { |
167 | return this.instance || (this.instance = new this()) | 179 | return this.instance || (this.instance = new this()) |
168 | } | 180 | } |
diff --git a/server/lib/redis.ts b/server/lib/redis.ts index 941f7d557..0b4b41e4e 100644 --- a/server/lib/redis.ts +++ b/server/lib/redis.ts | |||
@@ -60,11 +60,11 @@ class Redis { | |||
60 | return this.getValue(this.generateResetPasswordKey(userId)) | 60 | return this.getValue(this.generateResetPasswordKey(userId)) |
61 | } | 61 | } |
62 | 62 | ||
63 | setView (ip: string, videoUUID: string) { | 63 | setIPVideoView (ip: string, videoUUID: string) { |
64 | return this.setValue(this.buildViewKey(ip, videoUUID), '1', VIDEO_VIEW_LIFETIME) | 64 | return this.setValue(this.buildViewKey(ip, videoUUID), '1', VIDEO_VIEW_LIFETIME) |
65 | } | 65 | } |
66 | 66 | ||
67 | async isViewExists (ip: string, videoUUID: string) { | 67 | async isVideoIPViewExists (ip: string, videoUUID: string) { |
68 | return this.exists(this.buildViewKey(ip, videoUUID)) | 68 | return this.exists(this.buildViewKey(ip, videoUUID)) |
69 | } | 69 | } |
70 | 70 | ||
@@ -85,6 +85,52 @@ class Redis { | |||
85 | return this.setObject(this.buildCachedRouteKey(req), cached, lifetime) | 85 | return this.setObject(this.buildCachedRouteKey(req), cached, lifetime) |
86 | } | 86 | } |
87 | 87 | ||
88 | addVideoView (videoId: number) { | ||
89 | const keyIncr = this.generateVideoViewKey(videoId) | ||
90 | const keySet = this.generateVideosViewKey() | ||
91 | |||
92 | return Promise.all([ | ||
93 | this.addToSet(keySet, videoId.toString()), | ||
94 | this.increment(keyIncr) | ||
95 | ]) | ||
96 | } | ||
97 | |||
98 | async getVideoViews (videoId: number, hour: number) { | ||
99 | const key = this.generateVideoViewKey(videoId, hour) | ||
100 | |||
101 | const valueString = await this.getValue(key) | ||
102 | return parseInt(valueString, 10) | ||
103 | } | ||
104 | |||
105 | async getVideosIdViewed (hour: number) { | ||
106 | const key = this.generateVideosViewKey(hour) | ||
107 | |||
108 | const stringIds = await this.getSet(key) | ||
109 | return stringIds.map(s => parseInt(s, 10)) | ||
110 | } | ||
111 | |||
112 | deleteVideoViews (videoId: number, hour: number) { | ||
113 | const keySet = this.generateVideosViewKey(hour) | ||
114 | const keyIncr = this.generateVideoViewKey(videoId, hour) | ||
115 | |||
116 | return Promise.all([ | ||
117 | this.deleteFromSet(keySet, videoId.toString()), | ||
118 | this.deleteKey(keyIncr) | ||
119 | ]) | ||
120 | } | ||
121 | |||
122 | generateVideosViewKey (hour?: number) { | ||
123 | if (!hour) hour = new Date().getHours() | ||
124 | |||
125 | return `videos-view-h${hour}` | ||
126 | } | ||
127 | |||
128 | generateVideoViewKey (videoId: number, hour?: number) { | ||
129 | if (!hour) hour = new Date().getHours() | ||
130 | |||
131 | return `video-view-${videoId}-h${hour}` | ||
132 | } | ||
133 | |||
88 | generateResetPasswordKey (userId: number) { | 134 | generateResetPasswordKey (userId: number) { |
89 | return 'reset-password-' + userId | 135 | return 'reset-password-' + userId |
90 | } | 136 | } |
@@ -107,6 +153,34 @@ class Redis { | |||
107 | }) | 153 | }) |
108 | } | 154 | } |
109 | 155 | ||
156 | private getSet (key: string) { | ||
157 | return new Promise<string[]>((res, rej) => { | ||
158 | this.client.smembers(this.prefix + key, (err, value) => { | ||
159 | if (err) return rej(err) | ||
160 | |||
161 | return res(value) | ||
162 | }) | ||
163 | }) | ||
164 | } | ||
165 | |||
166 | private addToSet (key: string, value: string) { | ||
167 | return new Promise<string[]>((res, rej) => { | ||
168 | this.client.sadd(this.prefix + key, value, err => err ? rej(err) : res()) | ||
169 | }) | ||
170 | } | ||
171 | |||
172 | private deleteFromSet (key: string, value: string) { | ||
173 | return new Promise<void>((res, rej) => { | ||
174 | this.client.srem(this.prefix + key, value, err => err ? rej(err) : res()) | ||
175 | }) | ||
176 | } | ||
177 | |||
178 | private deleteKey (key: string) { | ||
179 | return new Promise<void>((res, rej) => { | ||
180 | this.client.del(this.prefix + key, err => err ? rej(err) : res()) | ||
181 | }) | ||
182 | } | ||
183 | |||
110 | private setValue (key: string, value: string, expirationMilliseconds: number) { | 184 | private setValue (key: string, value: string, expirationMilliseconds: number) { |
111 | return new Promise<void>((res, rej) => { | 185 | return new Promise<void>((res, rej) => { |
112 | this.client.set(this.prefix + key, value, 'PX', expirationMilliseconds, (err, ok) => { | 186 | this.client.set(this.prefix + key, value, 'PX', expirationMilliseconds, (err, ok) => { |
@@ -145,6 +219,16 @@ class Redis { | |||
145 | }) | 219 | }) |
146 | } | 220 | } |
147 | 221 | ||
222 | private increment (key: string) { | ||
223 | return new Promise<number>((res, rej) => { | ||
224 | this.client.incr(this.prefix + key, (err, value) => { | ||
225 | if (err) return rej(err) | ||
226 | |||
227 | return res(value) | ||
228 | }) | ||
229 | }) | ||
230 | } | ||
231 | |||
148 | private exists (key: string) { | 232 | private exists (key: string) { |
149 | return new Promise<boolean>((res, rej) => { | 233 | return new Promise<boolean>((res, rej) => { |
150 | this.client.exists(this.prefix + key, (err, existsNumber) => { | 234 | this.client.exists(this.prefix + key, (err, existsNumber) => { |
diff --git a/server/models/video/video-views.ts b/server/models/video/video-views.ts new file mode 100644 index 000000000..90ce671fd --- /dev/null +++ b/server/models/video/video-views.ts | |||
@@ -0,0 +1,41 @@ | |||
1 | import { AllowNull, BelongsTo, Column, CreatedAt, ForeignKey, Model, Table } from 'sequelize-typescript' | ||
2 | import { VideoModel } from './video' | ||
3 | import * as Sequelize from 'sequelize' | ||
4 | |||
5 | @Table({ | ||
6 | tableName: 'videoView', | ||
7 | indexes: [ | ||
8 | { | ||
9 | fields: [ 'videoId' ] | ||
10 | } | ||
11 | ] | ||
12 | }) | ||
13 | export class VideoViewModel extends Model<VideoViewModel> { | ||
14 | @CreatedAt | ||
15 | createdAt: Date | ||
16 | |||
17 | @AllowNull(false) | ||
18 | @Column(Sequelize.DATE) | ||
19 | startDate: Date | ||
20 | |||
21 | @AllowNull(false) | ||
22 | @Column(Sequelize.DATE) | ||
23 | endDate: Date | ||
24 | |||
25 | @AllowNull(false) | ||
26 | @Column | ||
27 | views: number | ||
28 | |||
29 | @ForeignKey(() => VideoModel) | ||
30 | @Column | ||
31 | videoId: number | ||
32 | |||
33 | @BelongsTo(() => VideoModel, { | ||
34 | foreignKey: { | ||
35 | allowNull: false | ||
36 | }, | ||
37 | onDelete: 'CASCADE' | ||
38 | }) | ||
39 | Video: VideoModel | ||
40 | |||
41 | } | ||
diff --git a/server/models/video/video.ts b/server/models/video/video.ts index 6271db1b3..3410833c8 100644 --- a/server/models/video/video.ts +++ b/server/models/video/video.ts | |||
@@ -1074,6 +1074,15 @@ export class VideoModel extends Model<VideoModel> { | |||
1074 | } | 1074 | } |
1075 | } | 1075 | } |
1076 | 1076 | ||
1077 | static incrementViews (id: number, views: number) { | ||
1078 | return VideoModel.increment('views', { | ||
1079 | by: views, | ||
1080 | where: { | ||
1081 | id | ||
1082 | } | ||
1083 | }) | ||
1084 | } | ||
1085 | |||
1077 | private static buildActorWhereWithFilter (filter?: VideoFilter) { | 1086 | private static buildActorWhereWithFilter (filter?: VideoFilter) { |
1078 | if (filter && filter === 'local') { | 1087 | if (filter && filter === 'local') { |
1079 | return { | 1088 | return { |
diff --git a/server/tests/api/server/reverse-proxy.ts b/server/tests/api/server/reverse-proxy.ts index 908b4a68c..e2c2a293e 100644 --- a/server/tests/api/server/reverse-proxy.ts +++ b/server/tests/api/server/reverse-proxy.ts | |||
@@ -4,7 +4,18 @@ import 'mocha' | |||
4 | import * as chai from 'chai' | 4 | import * as chai from 'chai' |
5 | import { About } from '../../../../shared/models/server/about.model' | 5 | import { About } from '../../../../shared/models/server/about.model' |
6 | import { CustomConfig } from '../../../../shared/models/server/custom-config.model' | 6 | import { CustomConfig } from '../../../../shared/models/server/custom-config.model' |
7 | import { deleteCustomConfig, getAbout, getVideo, killallServers, login, reRunServer, uploadVideo, userLogin, viewVideo } from '../../utils' | 7 | import { |
8 | deleteCustomConfig, | ||
9 | getAbout, | ||
10 | getVideo, | ||
11 | killallServers, | ||
12 | login, | ||
13 | reRunServer, | ||
14 | uploadVideo, | ||
15 | userLogin, | ||
16 | viewVideo, | ||
17 | wait | ||
18 | } from '../../utils' | ||
8 | const expect = chai.expect | 19 | const expect = chai.expect |
9 | 20 | ||
10 | import { | 21 | import { |
@@ -30,33 +41,53 @@ describe('Test application behind a reverse proxy', function () { | |||
30 | }) | 41 | }) |
31 | 42 | ||
32 | it('Should view a video only once with the same IP by default', async function () { | 43 | it('Should view a video only once with the same IP by default', async function () { |
44 | this.timeout(20000) | ||
45 | |||
33 | await viewVideo(server.url, videoId) | 46 | await viewVideo(server.url, videoId) |
34 | await viewVideo(server.url, videoId) | 47 | await viewVideo(server.url, videoId) |
35 | 48 | ||
49 | // Wait the repeatable job | ||
50 | await wait(8000) | ||
51 | |||
36 | const { body } = await getVideo(server.url, videoId) | 52 | const { body } = await getVideo(server.url, videoId) |
37 | expect(body.views).to.equal(1) | 53 | expect(body.views).to.equal(1) |
38 | }) | 54 | }) |
39 | 55 | ||
40 | it('Should view a video 2 times with the X-Forwarded-For header set', async function () { | 56 | it('Should view a video 2 times with the X-Forwarded-For header set', async function () { |
57 | this.timeout(20000) | ||
58 | |||
41 | await viewVideo(server.url, videoId, 204, '0.0.0.1,127.0.0.1') | 59 | await viewVideo(server.url, videoId, 204, '0.0.0.1,127.0.0.1') |
42 | await viewVideo(server.url, videoId, 204, '0.0.0.2,127.0.0.1') | 60 | await viewVideo(server.url, videoId, 204, '0.0.0.2,127.0.0.1') |
43 | 61 | ||
62 | // Wait the repeatable job | ||
63 | await wait(8000) | ||
64 | |||
44 | const { body } = await getVideo(server.url, videoId) | 65 | const { body } = await getVideo(server.url, videoId) |
45 | expect(body.views).to.equal(3) | 66 | expect(body.views).to.equal(3) |
46 | }) | 67 | }) |
47 | 68 | ||
48 | it('Should view a video only once with the same client IP in the X-Forwarded-For header', async function () { | 69 | it('Should view a video only once with the same client IP in the X-Forwarded-For header', async function () { |
70 | this.timeout(20000) | ||
71 | |||
49 | await viewVideo(server.url, videoId, 204, '0.0.0.4,0.0.0.3,::ffff:127.0.0.1') | 72 | await viewVideo(server.url, videoId, 204, '0.0.0.4,0.0.0.3,::ffff:127.0.0.1') |
50 | await viewVideo(server.url, videoId, 204, '0.0.0.5,0.0.0.3,127.0.0.1') | 73 | await viewVideo(server.url, videoId, 204, '0.0.0.5,0.0.0.3,127.0.0.1') |
51 | 74 | ||
75 | // Wait the repeatable job | ||
76 | await wait(8000) | ||
77 | |||
52 | const { body } = await getVideo(server.url, videoId) | 78 | const { body } = await getVideo(server.url, videoId) |
53 | expect(body.views).to.equal(4) | 79 | expect(body.views).to.equal(4) |
54 | }) | 80 | }) |
55 | 81 | ||
56 | it('Should view a video two times with a different client IP in the X-Forwarded-For header', async function () { | 82 | it('Should view a video two times with a different client IP in the X-Forwarded-For header', async function () { |
83 | this.timeout(20000) | ||
84 | |||
57 | await viewVideo(server.url, videoId, 204, '0.0.0.8,0.0.0.6,127.0.0.1') | 85 | await viewVideo(server.url, videoId, 204, '0.0.0.8,0.0.0.6,127.0.0.1') |
58 | await viewVideo(server.url, videoId, 204, '0.0.0.8,0.0.0.7,127.0.0.1') | 86 | await viewVideo(server.url, videoId, 204, '0.0.0.8,0.0.0.7,127.0.0.1') |
59 | 87 | ||
88 | // Wait the repeatable job | ||
89 | await wait(8000) | ||
90 | |||
60 | const { body } = await getVideo(server.url, videoId) | 91 | const { body } = await getVideo(server.url, videoId) |
61 | expect(body.views).to.equal(6) | 92 | expect(body.views).to.equal(6) |
62 | }) | 93 | }) |
diff --git a/server/tests/api/server/stats.ts b/server/tests/api/server/stats.ts index e75089a14..fc9b88805 100644 --- a/server/tests/api/server/stats.ts +++ b/server/tests/api/server/stats.ts | |||
@@ -46,6 +46,9 @@ describe('Test stats', function () { | |||
46 | 46 | ||
47 | await viewVideo(servers[0].url, videoUUID) | 47 | await viewVideo(servers[0].url, videoUUID) |
48 | 48 | ||
49 | // Wait the video views repeatable job | ||
50 | await wait(8000) | ||
51 | |||
49 | await follow(servers[2].url, [ servers[0].url ], servers[2].accessToken) | 52 | await follow(servers[2].url, [ servers[0].url ], servers[2].accessToken) |
50 | await waitJobs(servers) | 53 | await waitJobs(servers) |
51 | }) | 54 | }) |
diff --git a/server/tests/api/videos/multiple-servers.ts b/server/tests/api/videos/multiple-servers.ts index c551ccc59..4553ee855 100644 --- a/server/tests/api/videos/multiple-servers.ts +++ b/server/tests/api/videos/multiple-servers.ts | |||
@@ -492,7 +492,7 @@ describe('Test multiple servers', function () { | |||
492 | }) | 492 | }) |
493 | 493 | ||
494 | it('Should view multiple videos on owned servers', async function () { | 494 | it('Should view multiple videos on owned servers', async function () { |
495 | this.timeout(15000) | 495 | this.timeout(30000) |
496 | 496 | ||
497 | const tasks: Promise<any>[] = [] | 497 | const tasks: Promise<any>[] = [] |
498 | await viewVideo(servers[2].url, localVideosServer3[0]) | 498 | await viewVideo(servers[2].url, localVideosServer3[0]) |
@@ -511,6 +511,9 @@ describe('Test multiple servers', function () { | |||
511 | 511 | ||
512 | await waitJobs(servers) | 512 | await waitJobs(servers) |
513 | 513 | ||
514 | // Wait the repeatable job | ||
515 | await wait(6000) | ||
516 | |||
514 | for (const server of servers) { | 517 | for (const server of servers) { |
515 | const res = await getVideosList(server.url) | 518 | const res = await getVideosList(server.url) |
516 | 519 | ||
@@ -524,7 +527,7 @@ describe('Test multiple servers', function () { | |||
524 | }) | 527 | }) |
525 | 528 | ||
526 | it('Should view multiple videos on each servers', async function () { | 529 | it('Should view multiple videos on each servers', async function () { |
527 | this.timeout(15000) | 530 | this.timeout(30000) |
528 | 531 | ||
529 | const tasks: Promise<any>[] = [] | 532 | const tasks: Promise<any>[] = [] |
530 | tasks.push(viewVideo(servers[0].url, remoteVideosServer1[0])) | 533 | tasks.push(viewVideo(servers[0].url, remoteVideosServer1[0])) |
@@ -542,6 +545,9 @@ describe('Test multiple servers', function () { | |||
542 | 545 | ||
543 | await waitJobs(servers) | 546 | await waitJobs(servers) |
544 | 547 | ||
548 | // Wait the repeatable job | ||
549 | await wait(8000) | ||
550 | |||
545 | let baseVideos = null | 551 | let baseVideos = null |
546 | 552 | ||
547 | for (const server of servers) { | 553 | for (const server of servers) { |
diff --git a/server/tests/api/videos/single-server.ts b/server/tests/api/videos/single-server.ts index a757ad9da..89408fec6 100644 --- a/server/tests/api/videos/single-server.ts +++ b/server/tests/api/videos/single-server.ts | |||
@@ -196,7 +196,7 @@ describe('Test a single server', function () { | |||
196 | }) | 196 | }) |
197 | 197 | ||
198 | it('Should have the views updated', async function () { | 198 | it('Should have the views updated', async function () { |
199 | this.timeout(10000) | 199 | this.timeout(20000) |
200 | 200 | ||
201 | await viewVideo(server.url, videoId) | 201 | await viewVideo(server.url, videoId) |
202 | await viewVideo(server.url, videoId) | 202 | await viewVideo(server.url, videoId) |
@@ -212,6 +212,9 @@ describe('Test a single server', function () { | |||
212 | await viewVideo(server.url, videoId) | 212 | await viewVideo(server.url, videoId) |
213 | await viewVideo(server.url, videoId) | 213 | await viewVideo(server.url, videoId) |
214 | 214 | ||
215 | // Wait the repeatable job | ||
216 | await wait(8000) | ||
217 | |||
215 | const res = await getVideo(server.url, videoId) | 218 | const res = await getVideo(server.url, videoId) |
216 | 219 | ||
217 | const video = res.body | 220 | const video = res.body |
diff --git a/server/tests/utils/server/jobs.ts b/server/tests/utils/server/jobs.ts index c9cb8d3a3..4c02cace5 100644 --- a/server/tests/utils/server/jobs.ts +++ b/server/tests/utils/server/jobs.ts | |||
@@ -1,5 +1,5 @@ | |||
1 | import * as request from 'supertest' | 1 | import * as request from 'supertest' |
2 | import { JobState } from '../../../../shared/models' | 2 | import { Job, JobState } from '../../../../shared/models' |
3 | import { ServerInfo, wait } from '../index' | 3 | import { ServerInfo, wait } from '../index' |
4 | 4 | ||
5 | function getJobsList (url: string, accessToken: string, state: JobState) { | 5 | function getJobsList (url: string, accessToken: string, state: JobState) { |
@@ -44,8 +44,10 @@ async function waitJobs (serversArg: ServerInfo[] | ServerInfo) { | |||
44 | for (const server of servers) { | 44 | for (const server of servers) { |
45 | for (const state of states) { | 45 | for (const state of states) { |
46 | const p = getJobsListPaginationAndSort(server.url, server.accessToken, state, 0, 10, '-createdAt') | 46 | const p = getJobsListPaginationAndSort(server.url, server.accessToken, state, 0, 10, '-createdAt') |
47 | .then(res => { | 47 | .then(res => res.body.data) |
48 | if (res.body.total > 0) pendingRequests = true | 48 | .then((jobs: Job[]) => jobs.filter(j => j.type !== 'videos-views')) |
49 | .then(jobs => { | ||
50 | if (jobs.length !== 0) pendingRequests = true | ||
49 | }) | 51 | }) |
50 | tasks.push(p) | 52 | tasks.push(p) |
51 | } | 53 | } |
diff --git a/shared/models/server/job.model.ts b/shared/models/server/job.model.ts index 2565479f6..4046297c4 100644 --- a/shared/models/server/job.model.ts +++ b/shared/models/server/job.model.ts | |||
@@ -7,7 +7,8 @@ export type JobType = 'activitypub-http-unicast' | | |||
7 | 'video-file-import' | | 7 | 'video-file-import' | |
8 | 'video-file' | | 8 | 'video-file' | |
9 | 'email' | | 9 | 'email' | |
10 | 'video-import' | 10 | 'video-import' | |
11 | 'videos-views' | ||
11 | 12 | ||
12 | export interface Job { | 13 | export interface Job { |
13 | id: number | 14 | id: number |