aboutsummaryrefslogtreecommitdiffhomepage
diff options
context:
space:
mode:
authorChocobozzz <me@florianbigard.com>2018-08-29 16:26:25 +0200
committerChocobozzz <me@florianbigard.com>2018-08-30 15:03:18 +0200
commit6b6168606bc86430f6b7821c9d5f1c80d0425ebf (patch)
tree9aea6cf0875c9fee30c373eb4924b12d47d1e23c
parent2d9fea161fd4fc73994fc77951bafdccdc2071fd (diff)
downloadPeerTube-6b6168606bc86430f6b7821c9d5f1c80d0425ebf.tar.gz
PeerTube-6b6168606bc86430f6b7821c9d5f1c80d0425ebf.tar.zst
PeerTube-6b6168606bc86430f6b7821c9d5f1c80d0425ebf.zip
Bufferize videos views in redis
-rw-r--r--.github/ISSUE_TEMPLATE.md4
-rw-r--r--server/controllers/api/videos/index.ts8
-rw-r--r--server/initializers/constants.ts20
-rw-r--r--server/initializers/database.ts4
-rw-r--r--server/lib/activitypub/process/process-create.ts4
-rw-r--r--server/lib/job-queue/handlers/video-views.ts40
-rw-r--r--server/lib/job-queue/job-queue.ts20
-rw-r--r--server/lib/redis.ts88
-rw-r--r--server/models/video/video-views.ts41
-rw-r--r--server/models/video/video.ts9
-rw-r--r--server/tests/api/server/reverse-proxy.ts33
-rw-r--r--server/tests/api/server/stats.ts3
-rw-r--r--server/tests/api/videos/multiple-servers.ts10
-rw-r--r--server/tests/api/videos/single-server.ts5
-rw-r--r--server/tests/utils/server/jobs.ts8
-rw-r--r--shared/models/server/job.model.ts3
16 files changed, 274 insertions, 26 deletions
diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md
index 6b34c2cf7..8edb00b9c 100644
--- a/.github/ISSUE_TEMPLATE.md
+++ b/.github/ISSUE_TEMPLATE.md
@@ -11,6 +11,6 @@
11* **What do you see instead?** 11* **What do you see instead?**
12 12
13 13
14* **Browser console log if useful (Gist/Pastebin...):** 14* **Browser console log if useful:**
15* **Server log if useful (Gist/Pastebin...):** 15* **Server log if useful (journalctl or /var/www/peertube/storage/logs):**
16 16
diff --git a/server/controllers/api/videos/index.ts b/server/controllers/api/videos/index.ts
index a86cf4f99..be803490b 100644
--- a/server/controllers/api/videos/index.ts
+++ b/server/controllers/api/videos/index.ts
@@ -380,14 +380,16 @@ async function viewVideo (req: express.Request, res: express.Response) {
380 const videoInstance = res.locals.video 380 const videoInstance = res.locals.video
381 381
382 const ip = req.ip 382 const ip = req.ip
383 const exists = await Redis.Instance.isViewExists(ip, videoInstance.uuid) 383 const exists = await Redis.Instance.isVideoIPViewExists(ip, videoInstance.uuid)
384 if (exists) { 384 if (exists) {
385 logger.debug('View for ip %s and video %s already exists.', ip, videoInstance.uuid) 385 logger.debug('View for ip %s and video %s already exists.', ip, videoInstance.uuid)
386 return res.status(204).end() 386 return res.status(204).end()
387 } 387 }
388 388
389 await videoInstance.increment('views') 389 await Promise.all([
390 await Redis.Instance.setView(ip, videoInstance.uuid) 390 Redis.Instance.addVideoView(videoInstance.id),
391 Redis.Instance.setIPVideoView(ip, videoInstance.uuid)
392 ])
391 393
392 const serverAccount = await getServerActor() 394 const serverAccount = await getServerActor()
393 395
diff --git a/server/initializers/constants.ts b/server/initializers/constants.ts
index 7f1b25654..2d9a2e670 100644
--- a/server/initializers/constants.ts
+++ b/server/initializers/constants.ts
@@ -3,11 +3,12 @@ import { dirname, join } from 'path'
3import { JobType, VideoRateType, VideoState } from '../../shared/models' 3import { JobType, VideoRateType, VideoState } from '../../shared/models'
4import { ActivityPubActorType } from '../../shared/models/activitypub' 4import { ActivityPubActorType } from '../../shared/models/activitypub'
5import { FollowState } from '../../shared/models/actors' 5import { FollowState } from '../../shared/models/actors'
6import { VideoPrivacy, VideoAbuseState, VideoImportState } from '../../shared/models/videos' 6import { VideoAbuseState, VideoImportState, VideoPrivacy } from '../../shared/models/videos'
7// Do not use barrels, remain constants as independent as possible 7// Do not use barrels, remain constants as independent as possible
8import { buildPath, isTestInstance, root, sanitizeHost, sanitizeUrl } from '../helpers/core-utils' 8import { buildPath, isTestInstance, root, sanitizeHost, sanitizeUrl } from '../helpers/core-utils'
9import { NSFWPolicyType } from '../../shared/models/videos/nsfw-policy.type' 9import { NSFWPolicyType } from '../../shared/models/videos/nsfw-policy.type'
10import { invert } from 'lodash' 10import { invert } from 'lodash'
11import { CronRepeatOptions, EveryRepeatOptions } from 'bull'
11 12
12// Use a variable to reload the configuration if we need 13// Use a variable to reload the configuration if we need
13let config: IConfig = require('config') 14let config: IConfig = require('config')
@@ -90,7 +91,8 @@ const JOB_ATTEMPTS: { [ id in JobType ]: number } = {
90 'video-file-import': 1, 91 'video-file-import': 1,
91 'video-file': 1, 92 'video-file': 1,
92 'video-import': 1, 93 'video-import': 1,
93 'email': 5 94 'email': 5,
95 'videos-views': 1
94} 96}
95const JOB_CONCURRENCY: { [ id in JobType ]: number } = { 97const JOB_CONCURRENCY: { [ id in JobType ]: number } = {
96 'activitypub-http-broadcast': 1, 98 'activitypub-http-broadcast': 1,
@@ -100,7 +102,8 @@ const JOB_CONCURRENCY: { [ id in JobType ]: number } = {
100 'video-file-import': 1, 102 'video-file-import': 1,
101 'video-file': 1, 103 'video-file': 1,
102 'video-import': 1, 104 'video-import': 1,
103 'email': 5 105 'email': 5,
106 'videos-views': 1
104} 107}
105const JOB_TTL: { [ id in JobType ]: number } = { 108const JOB_TTL: { [ id in JobType ]: number } = {
106 'activitypub-http-broadcast': 60000 * 10, // 10 minutes 109 'activitypub-http-broadcast': 60000 * 10, // 10 minutes
@@ -110,8 +113,15 @@ const JOB_TTL: { [ id in JobType ]: number } = {
110 'video-file-import': 1000 * 3600, // 1 hour 113 'video-file-import': 1000 * 3600, // 1 hour
111 'video-file': 1000 * 3600 * 48, // 2 days, transcoding could be long 114 'video-file': 1000 * 3600 * 48, // 2 days, transcoding could be long
112 'video-import': 1000 * 3600 * 5, // 5 hours 115 'video-import': 1000 * 3600 * 5, // 5 hours
113 'email': 60000 * 10 // 10 minutes 116 'email': 60000 * 10, // 10 minutes
117 'videos-views': undefined // Unlimited
114} 118}
119const REPEAT_JOBS: { [ id: string ]: EveryRepeatOptions | CronRepeatOptions } = {
120 'videos-views': {
121 cron: '1 * * * *' // At 1 minutes past the hour
122 }
123}
124
115const BROADCAST_CONCURRENCY = 10 // How many requests in parallel we do in activitypub-http-broadcast job 125const BROADCAST_CONCURRENCY = 10 // How many requests in parallel we do in activitypub-http-broadcast job
116const CRAWL_REQUEST_CONCURRENCY = 1 // How many requests in parallel to fetch remote data (likes, shares...) 126const CRAWL_REQUEST_CONCURRENCY = 1 // How many requests in parallel to fetch remote data (likes, shares...)
117const JOB_REQUEST_TIMEOUT = 3000 // 3 seconds 127const JOB_REQUEST_TIMEOUT = 3000 // 3 seconds
@@ -591,6 +601,7 @@ if (isTestInstance() === true) {
591 SCHEDULER_INTERVALS_MS.badActorFollow = 10000 601 SCHEDULER_INTERVALS_MS.badActorFollow = 10000
592 SCHEDULER_INTERVALS_MS.removeOldJobs = 10000 602 SCHEDULER_INTERVALS_MS.removeOldJobs = 10000
593 SCHEDULER_INTERVALS_MS.updateVideos = 5000 603 SCHEDULER_INTERVALS_MS.updateVideos = 5000
604 REPEAT_JOBS['videos-views'] = { every: 5000 }
594 605
595 VIDEO_VIEW_LIFETIME = 1000 // 1 second 606 VIDEO_VIEW_LIFETIME = 1000 // 1 second
596 607
@@ -652,6 +663,7 @@ export {
652 USER_PASSWORD_RESET_LIFETIME, 663 USER_PASSWORD_RESET_LIFETIME,
653 IMAGE_MIMETYPE_EXT, 664 IMAGE_MIMETYPE_EXT,
654 SCHEDULER_INTERVALS_MS, 665 SCHEDULER_INTERVALS_MS,
666 REPEAT_JOBS,
655 STATIC_DOWNLOAD_PATHS, 667 STATIC_DOWNLOAD_PATHS,
656 RATES_LIMIT, 668 RATES_LIMIT,
657 VIDEO_EXT_MIMETYPE, 669 VIDEO_EXT_MIMETYPE,
diff --git a/server/initializers/database.ts b/server/initializers/database.ts
index 0be752363..78bc8101c 100644
--- a/server/initializers/database.ts
+++ b/server/initializers/database.ts
@@ -25,6 +25,7 @@ import { CONFIG } from './constants'
25import { ScheduleVideoUpdateModel } from '../models/video/schedule-video-update' 25import { ScheduleVideoUpdateModel } from '../models/video/schedule-video-update'
26import { VideoCaptionModel } from '../models/video/video-caption' 26import { VideoCaptionModel } from '../models/video/video-caption'
27import { VideoImportModel } from '../models/video/video-import' 27import { VideoImportModel } from '../models/video/video-import'
28import { VideoViewModel } from '../models/video/video-views'
28 29
29require('pg').defaults.parseInt8 = true // Avoid BIGINT to be converted to string 30require('pg').defaults.parseInt8 = true // Avoid BIGINT to be converted to string
30 31
@@ -83,7 +84,8 @@ async function initDatabaseModels (silent: boolean) {
83 VideoModel, 84 VideoModel,
84 VideoCommentModel, 85 VideoCommentModel,
85 ScheduleVideoUpdateModel, 86 ScheduleVideoUpdateModel,
86 VideoImportModel 87 VideoImportModel,
88 VideoViewModel
87 ]) 89 ])
88 90
89 // Check extensions exist in the database 91 // Check extensions exist in the database
diff --git a/server/lib/activitypub/process/process-create.ts b/server/lib/activitypub/process/process-create.ts
index 75f07d131..16f426e23 100644
--- a/server/lib/activitypub/process/process-create.ts
+++ b/server/lib/activitypub/process/process-create.ts
@@ -7,11 +7,11 @@ import { sequelizeTypescript } from '../../../initializers'
7import { AccountVideoRateModel } from '../../../models/account/account-video-rate' 7import { AccountVideoRateModel } from '../../../models/account/account-video-rate'
8import { ActorModel } from '../../../models/activitypub/actor' 8import { ActorModel } from '../../../models/activitypub/actor'
9import { VideoAbuseModel } from '../../../models/video/video-abuse' 9import { VideoAbuseModel } from '../../../models/video/video-abuse'
10import { VideoCommentModel } from '../../../models/video/video-comment'
11import { getOrCreateActorAndServerAndModel } from '../actor' 10import { getOrCreateActorAndServerAndModel } from '../actor'
12import { addVideoComment, resolveThread } from '../video-comments' 11import { addVideoComment, resolveThread } from '../video-comments'
13import { getOrCreateVideoAndAccountAndChannel } from '../videos' 12import { getOrCreateVideoAndAccountAndChannel } from '../videos'
14import { forwardActivity, forwardVideoRelatedActivity } from '../send/utils' 13import { forwardActivity, forwardVideoRelatedActivity } from '../send/utils'
14import { Redis } from '../../redis'
15 15
16async function processCreateActivity (activity: ActivityCreate) { 16async function processCreateActivity (activity: ActivityCreate) {
17 const activityObject = activity.object 17 const activityObject = activity.object
@@ -88,7 +88,7 @@ async function processCreateView (byActor: ActorModel, activity: ActivityCreate)
88 const actor = await ActorModel.loadByUrl(view.actor) 88 const actor = await ActorModel.loadByUrl(view.actor)
89 if (!actor) throw new Error('Unknown actor ' + view.actor) 89 if (!actor) throw new Error('Unknown actor ' + view.actor)
90 90
91 await video.increment('views') 91 await Redis.Instance.addVideoView(video.id)
92 92
93 if (video.isOwned()) { 93 if (video.isOwned()) {
94 // Don't resend the activity to the sender 94 // Don't resend the activity to the sender
diff --git a/server/lib/job-queue/handlers/video-views.ts b/server/lib/job-queue/handlers/video-views.ts
new file mode 100644
index 000000000..875d8ab88
--- /dev/null
+++ b/server/lib/job-queue/handlers/video-views.ts
@@ -0,0 +1,40 @@
1import { Redis } from '../../redis'
2import { logger } from '../../../helpers/logger'
3import { VideoModel } from '../../../models/video/video'
4import { VideoViewModel } from '../../../models/video/video-views'
5
6async function processVideosViewsViews () {
7 const hour = new Date().getHours()
8 const startDate = new Date().setMinutes(0, 0, 0)
9 const endDate = new Date().setMinutes(59, 59, 999)
10
11 const videoIds = await Redis.Instance.getVideosIdViewed(hour)
12 if (videoIds.length === 0) return
13
14 logger.info('Processing videos views in job for hour %d.', hour)
15
16 for (const videoId of videoIds) {
17 const views = await Redis.Instance.getVideoViews(videoId, hour)
18 if (isNaN(views)) {
19 logger.error('Cannot process videos views of video %s in hour %d: views number is NaN.', videoId, hour)
20 } else {
21 logger.debug('Adding %d views to video %d in hour %d.', views, videoId, hour)
22
23 await VideoModel.incrementViews(videoId, views)
24 await VideoViewModel.create({
25 startDate,
26 endDate,
27 views,
28 videoId
29 })
30 }
31
32 await Redis.Instance.deleteVideoViews(videoId, hour)
33 }
34}
35
36// ---------------------------------------------------------------------------
37
38export {
39 processVideosViewsViews
40}
diff --git a/server/lib/job-queue/job-queue.ts b/server/lib/job-queue/job-queue.ts
index ddb357db5..0696ba43c 100644
--- a/server/lib/job-queue/job-queue.ts
+++ b/server/lib/job-queue/job-queue.ts
@@ -2,7 +2,7 @@ import * as Bull from 'bull'
2import { JobState, JobType } from '../../../shared/models' 2import { JobState, JobType } from '../../../shared/models'
3import { logger } from '../../helpers/logger' 3import { logger } from '../../helpers/logger'
4import { Redis } from '../redis' 4import { Redis } from '../redis'
5import { CONFIG, JOB_ATTEMPTS, JOB_COMPLETED_LIFETIME, JOB_CONCURRENCY, JOB_TTL } from '../../initializers' 5import { CONFIG, JOB_ATTEMPTS, JOB_COMPLETED_LIFETIME, JOB_CONCURRENCY, JOB_TTL, REPEAT_JOBS } from '../../initializers'
6import { ActivitypubHttpBroadcastPayload, processActivityPubHttpBroadcast } from './handlers/activitypub-http-broadcast' 6import { ActivitypubHttpBroadcastPayload, processActivityPubHttpBroadcast } from './handlers/activitypub-http-broadcast'
7import { ActivitypubHttpFetcherPayload, processActivityPubHttpFetcher } from './handlers/activitypub-http-fetcher' 7import { ActivitypubHttpFetcherPayload, processActivityPubHttpFetcher } from './handlers/activitypub-http-fetcher'
8import { ActivitypubHttpUnicastPayload, processActivityPubHttpUnicast } from './handlers/activitypub-http-unicast' 8import { ActivitypubHttpUnicastPayload, processActivityPubHttpUnicast } from './handlers/activitypub-http-unicast'
@@ -10,6 +10,7 @@ import { EmailPayload, processEmail } from './handlers/email'
10import { processVideoFile, processVideoFileImport, VideoFileImportPayload, VideoFilePayload } from './handlers/video-file' 10import { processVideoFile, processVideoFileImport, VideoFileImportPayload, VideoFilePayload } from './handlers/video-file'
11import { ActivitypubFollowPayload, processActivityPubFollow } from './handlers/activitypub-follow' 11import { ActivitypubFollowPayload, processActivityPubFollow } from './handlers/activitypub-follow'
12import { processVideoImport, VideoImportPayload } from './handlers/video-import' 12import { processVideoImport, VideoImportPayload } from './handlers/video-import'
13import { processVideosViewsViews } from './handlers/video-views'
13 14
14type CreateJobArgument = 15type CreateJobArgument =
15 { type: 'activitypub-http-broadcast', payload: ActivitypubHttpBroadcastPayload } | 16 { type: 'activitypub-http-broadcast', payload: ActivitypubHttpBroadcastPayload } |
@@ -19,7 +20,8 @@ type CreateJobArgument =
19 { type: 'video-file-import', payload: VideoFileImportPayload } | 20 { type: 'video-file-import', payload: VideoFileImportPayload } |
20 { type: 'video-file', payload: VideoFilePayload } | 21 { type: 'video-file', payload: VideoFilePayload } |
21 { type: 'email', payload: EmailPayload } | 22 { type: 'email', payload: EmailPayload } |
22 { type: 'video-import', payload: VideoImportPayload } 23 { type: 'video-import', payload: VideoImportPayload } |
24 { type: 'videos-views', payload: {} }
23 25
24const handlers: { [ id in JobType ]: (job: Bull.Job) => Promise<any>} = { 26const handlers: { [ id in JobType ]: (job: Bull.Job) => Promise<any>} = {
25 'activitypub-http-broadcast': processActivityPubHttpBroadcast, 27 'activitypub-http-broadcast': processActivityPubHttpBroadcast,
@@ -29,7 +31,8 @@ const handlers: { [ id in JobType ]: (job: Bull.Job) => Promise<any>} = {
29 'video-file-import': processVideoFileImport, 31 'video-file-import': processVideoFileImport,
30 'video-file': processVideoFile, 32 'video-file': processVideoFile,
31 'email': processEmail, 33 'email': processEmail,
32 'video-import': processVideoImport 34 'video-import': processVideoImport,
35 'videos-views': processVideosViewsViews
33} 36}
34 37
35const jobTypes: JobType[] = [ 38const jobTypes: JobType[] = [
@@ -40,7 +43,8 @@ const jobTypes: JobType[] = [
40 'email', 43 'email',
41 'video-file', 44 'video-file',
42 'video-file-import', 45 'video-file-import',
43 'video-import' 46 'video-import',
47 'videos-views'
44] 48]
45 49
46class JobQueue { 50class JobQueue {
@@ -85,6 +89,8 @@ class JobQueue {
85 89
86 this.queues[handlerName] = queue 90 this.queues[handlerName] = queue
87 } 91 }
92
93 this.addRepeatableJobs()
88 } 94 }
89 95
90 terminate () { 96 terminate () {
@@ -163,6 +169,12 @@ class JobQueue {
163 } 169 }
164 } 170 }
165 171
172 private addRepeatableJobs () {
173 this.queues['videos-views'].add({}, {
174 repeat: REPEAT_JOBS['videos-views']
175 })
176 }
177
166 static get Instance () { 178 static get Instance () {
167 return this.instance || (this.instance = new this()) 179 return this.instance || (this.instance = new this())
168 } 180 }
diff --git a/server/lib/redis.ts b/server/lib/redis.ts
index 941f7d557..0b4b41e4e 100644
--- a/server/lib/redis.ts
+++ b/server/lib/redis.ts
@@ -60,11 +60,11 @@ class Redis {
60 return this.getValue(this.generateResetPasswordKey(userId)) 60 return this.getValue(this.generateResetPasswordKey(userId))
61 } 61 }
62 62
63 setView (ip: string, videoUUID: string) { 63 setIPVideoView (ip: string, videoUUID: string) {
64 return this.setValue(this.buildViewKey(ip, videoUUID), '1', VIDEO_VIEW_LIFETIME) 64 return this.setValue(this.buildViewKey(ip, videoUUID), '1', VIDEO_VIEW_LIFETIME)
65 } 65 }
66 66
67 async isViewExists (ip: string, videoUUID: string) { 67 async isVideoIPViewExists (ip: string, videoUUID: string) {
68 return this.exists(this.buildViewKey(ip, videoUUID)) 68 return this.exists(this.buildViewKey(ip, videoUUID))
69 } 69 }
70 70
@@ -85,6 +85,52 @@ class Redis {
85 return this.setObject(this.buildCachedRouteKey(req), cached, lifetime) 85 return this.setObject(this.buildCachedRouteKey(req), cached, lifetime)
86 } 86 }
87 87
88 addVideoView (videoId: number) {
89 const keyIncr = this.generateVideoViewKey(videoId)
90 const keySet = this.generateVideosViewKey()
91
92 return Promise.all([
93 this.addToSet(keySet, videoId.toString()),
94 this.increment(keyIncr)
95 ])
96 }
97
98 async getVideoViews (videoId: number, hour: number) {
99 const key = this.generateVideoViewKey(videoId, hour)
100
101 const valueString = await this.getValue(key)
102 return parseInt(valueString, 10)
103 }
104
105 async getVideosIdViewed (hour: number) {
106 const key = this.generateVideosViewKey(hour)
107
108 const stringIds = await this.getSet(key)
109 return stringIds.map(s => parseInt(s, 10))
110 }
111
112 deleteVideoViews (videoId: number, hour: number) {
113 const keySet = this.generateVideosViewKey(hour)
114 const keyIncr = this.generateVideoViewKey(videoId, hour)
115
116 return Promise.all([
117 this.deleteFromSet(keySet, videoId.toString()),
118 this.deleteKey(keyIncr)
119 ])
120 }
121
122 generateVideosViewKey (hour?: number) {
123 if (!hour) hour = new Date().getHours()
124
125 return `videos-view-h${hour}`
126 }
127
128 generateVideoViewKey (videoId: number, hour?: number) {
129 if (!hour) hour = new Date().getHours()
130
131 return `video-view-${videoId}-h${hour}`
132 }
133
88 generateResetPasswordKey (userId: number) { 134 generateResetPasswordKey (userId: number) {
89 return 'reset-password-' + userId 135 return 'reset-password-' + userId
90 } 136 }
@@ -107,6 +153,34 @@ class Redis {
107 }) 153 })
108 } 154 }
109 155
156 private getSet (key: string) {
157 return new Promise<string[]>((res, rej) => {
158 this.client.smembers(this.prefix + key, (err, value) => {
159 if (err) return rej(err)
160
161 return res(value)
162 })
163 })
164 }
165
166 private addToSet (key: string, value: string) {
167 return new Promise<string[]>((res, rej) => {
168 this.client.sadd(this.prefix + key, value, err => err ? rej(err) : res())
169 })
170 }
171
172 private deleteFromSet (key: string, value: string) {
173 return new Promise<void>((res, rej) => {
174 this.client.srem(this.prefix + key, value, err => err ? rej(err) : res())
175 })
176 }
177
178 private deleteKey (key: string) {
179 return new Promise<void>((res, rej) => {
180 this.client.del(this.prefix + key, err => err ? rej(err) : res())
181 })
182 }
183
110 private setValue (key: string, value: string, expirationMilliseconds: number) { 184 private setValue (key: string, value: string, expirationMilliseconds: number) {
111 return new Promise<void>((res, rej) => { 185 return new Promise<void>((res, rej) => {
112 this.client.set(this.prefix + key, value, 'PX', expirationMilliseconds, (err, ok) => { 186 this.client.set(this.prefix + key, value, 'PX', expirationMilliseconds, (err, ok) => {
@@ -145,6 +219,16 @@ class Redis {
145 }) 219 })
146 } 220 }
147 221
222 private increment (key: string) {
223 return new Promise<number>((res, rej) => {
224 this.client.incr(this.prefix + key, (err, value) => {
225 if (err) return rej(err)
226
227 return res(value)
228 })
229 })
230 }
231
148 private exists (key: string) { 232 private exists (key: string) {
149 return new Promise<boolean>((res, rej) => { 233 return new Promise<boolean>((res, rej) => {
150 this.client.exists(this.prefix + key, (err, existsNumber) => { 234 this.client.exists(this.prefix + key, (err, existsNumber) => {
diff --git a/server/models/video/video-views.ts b/server/models/video/video-views.ts
new file mode 100644
index 000000000..90ce671fd
--- /dev/null
+++ b/server/models/video/video-views.ts
@@ -0,0 +1,41 @@
1import { AllowNull, BelongsTo, Column, CreatedAt, ForeignKey, Model, Table } from 'sequelize-typescript'
2import { VideoModel } from './video'
3import * as Sequelize from 'sequelize'
4
5@Table({
6 tableName: 'videoView',
7 indexes: [
8 {
9 fields: [ 'videoId' ]
10 }
11 ]
12})
13export class VideoViewModel extends Model<VideoViewModel> {
14 @CreatedAt
15 createdAt: Date
16
17 @AllowNull(false)
18 @Column(Sequelize.DATE)
19 startDate: Date
20
21 @AllowNull(false)
22 @Column(Sequelize.DATE)
23 endDate: Date
24
25 @AllowNull(false)
26 @Column
27 views: number
28
29 @ForeignKey(() => VideoModel)
30 @Column
31 videoId: number
32
33 @BelongsTo(() => VideoModel, {
34 foreignKey: {
35 allowNull: false
36 },
37 onDelete: 'CASCADE'
38 })
39 Video: VideoModel
40
41}
diff --git a/server/models/video/video.ts b/server/models/video/video.ts
index 6271db1b3..3410833c8 100644
--- a/server/models/video/video.ts
+++ b/server/models/video/video.ts
@@ -1074,6 +1074,15 @@ export class VideoModel extends Model<VideoModel> {
1074 } 1074 }
1075 } 1075 }
1076 1076
1077 static incrementViews (id: number, views: number) {
1078 return VideoModel.increment('views', {
1079 by: views,
1080 where: {
1081 id
1082 }
1083 })
1084 }
1085
1077 private static buildActorWhereWithFilter (filter?: VideoFilter) { 1086 private static buildActorWhereWithFilter (filter?: VideoFilter) {
1078 if (filter && filter === 'local') { 1087 if (filter && filter === 'local') {
1079 return { 1088 return {
diff --git a/server/tests/api/server/reverse-proxy.ts b/server/tests/api/server/reverse-proxy.ts
index 908b4a68c..e2c2a293e 100644
--- a/server/tests/api/server/reverse-proxy.ts
+++ b/server/tests/api/server/reverse-proxy.ts
@@ -4,7 +4,18 @@ import 'mocha'
4import * as chai from 'chai' 4import * as chai from 'chai'
5import { About } from '../../../../shared/models/server/about.model' 5import { About } from '../../../../shared/models/server/about.model'
6import { CustomConfig } from '../../../../shared/models/server/custom-config.model' 6import { CustomConfig } from '../../../../shared/models/server/custom-config.model'
7import { deleteCustomConfig, getAbout, getVideo, killallServers, login, reRunServer, uploadVideo, userLogin, viewVideo } from '../../utils' 7import {
8 deleteCustomConfig,
9 getAbout,
10 getVideo,
11 killallServers,
12 login,
13 reRunServer,
14 uploadVideo,
15 userLogin,
16 viewVideo,
17 wait
18} from '../../utils'
8const expect = chai.expect 19const expect = chai.expect
9 20
10import { 21import {
@@ -30,33 +41,53 @@ describe('Test application behind a reverse proxy', function () {
30 }) 41 })
31 42
32 it('Should view a video only once with the same IP by default', async function () { 43 it('Should view a video only once with the same IP by default', async function () {
44 this.timeout(20000)
45
33 await viewVideo(server.url, videoId) 46 await viewVideo(server.url, videoId)
34 await viewVideo(server.url, videoId) 47 await viewVideo(server.url, videoId)
35 48
49 // Wait the repeatable job
50 await wait(8000)
51
36 const { body } = await getVideo(server.url, videoId) 52 const { body } = await getVideo(server.url, videoId)
37 expect(body.views).to.equal(1) 53 expect(body.views).to.equal(1)
38 }) 54 })
39 55
40 it('Should view a video 2 times with the X-Forwarded-For header set', async function () { 56 it('Should view a video 2 times with the X-Forwarded-For header set', async function () {
57 this.timeout(20000)
58
41 await viewVideo(server.url, videoId, 204, '0.0.0.1,127.0.0.1') 59 await viewVideo(server.url, videoId, 204, '0.0.0.1,127.0.0.1')
42 await viewVideo(server.url, videoId, 204, '0.0.0.2,127.0.0.1') 60 await viewVideo(server.url, videoId, 204, '0.0.0.2,127.0.0.1')
43 61
62 // Wait the repeatable job
63 await wait(8000)
64
44 const { body } = await getVideo(server.url, videoId) 65 const { body } = await getVideo(server.url, videoId)
45 expect(body.views).to.equal(3) 66 expect(body.views).to.equal(3)
46 }) 67 })
47 68
48 it('Should view a video only once with the same client IP in the X-Forwarded-For header', async function () { 69 it('Should view a video only once with the same client IP in the X-Forwarded-For header', async function () {
70 this.timeout(20000)
71
49 await viewVideo(server.url, videoId, 204, '0.0.0.4,0.0.0.3,::ffff:127.0.0.1') 72 await viewVideo(server.url, videoId, 204, '0.0.0.4,0.0.0.3,::ffff:127.0.0.1')
50 await viewVideo(server.url, videoId, 204, '0.0.0.5,0.0.0.3,127.0.0.1') 73 await viewVideo(server.url, videoId, 204, '0.0.0.5,0.0.0.3,127.0.0.1')
51 74
75 // Wait the repeatable job
76 await wait(8000)
77
52 const { body } = await getVideo(server.url, videoId) 78 const { body } = await getVideo(server.url, videoId)
53 expect(body.views).to.equal(4) 79 expect(body.views).to.equal(4)
54 }) 80 })
55 81
56 it('Should view a video two times with a different client IP in the X-Forwarded-For header', async function () { 82 it('Should view a video two times with a different client IP in the X-Forwarded-For header', async function () {
83 this.timeout(20000)
84
57 await viewVideo(server.url, videoId, 204, '0.0.0.8,0.0.0.6,127.0.0.1') 85 await viewVideo(server.url, videoId, 204, '0.0.0.8,0.0.0.6,127.0.0.1')
58 await viewVideo(server.url, videoId, 204, '0.0.0.8,0.0.0.7,127.0.0.1') 86 await viewVideo(server.url, videoId, 204, '0.0.0.8,0.0.0.7,127.0.0.1')
59 87
88 // Wait the repeatable job
89 await wait(8000)
90
60 const { body } = await getVideo(server.url, videoId) 91 const { body } = await getVideo(server.url, videoId)
61 expect(body.views).to.equal(6) 92 expect(body.views).to.equal(6)
62 }) 93 })
diff --git a/server/tests/api/server/stats.ts b/server/tests/api/server/stats.ts
index e75089a14..fc9b88805 100644
--- a/server/tests/api/server/stats.ts
+++ b/server/tests/api/server/stats.ts
@@ -46,6 +46,9 @@ describe('Test stats', function () {
46 46
47 await viewVideo(servers[0].url, videoUUID) 47 await viewVideo(servers[0].url, videoUUID)
48 48
49 // Wait the video views repeatable job
50 await wait(8000)
51
49 await follow(servers[2].url, [ servers[0].url ], servers[2].accessToken) 52 await follow(servers[2].url, [ servers[0].url ], servers[2].accessToken)
50 await waitJobs(servers) 53 await waitJobs(servers)
51 }) 54 })
diff --git a/server/tests/api/videos/multiple-servers.ts b/server/tests/api/videos/multiple-servers.ts
index c551ccc59..4553ee855 100644
--- a/server/tests/api/videos/multiple-servers.ts
+++ b/server/tests/api/videos/multiple-servers.ts
@@ -492,7 +492,7 @@ describe('Test multiple servers', function () {
492 }) 492 })
493 493
494 it('Should view multiple videos on owned servers', async function () { 494 it('Should view multiple videos on owned servers', async function () {
495 this.timeout(15000) 495 this.timeout(30000)
496 496
497 const tasks: Promise<any>[] = [] 497 const tasks: Promise<any>[] = []
498 await viewVideo(servers[2].url, localVideosServer3[0]) 498 await viewVideo(servers[2].url, localVideosServer3[0])
@@ -511,6 +511,9 @@ describe('Test multiple servers', function () {
511 511
512 await waitJobs(servers) 512 await waitJobs(servers)
513 513
514 // Wait the repeatable job
515 await wait(6000)
516
514 for (const server of servers) { 517 for (const server of servers) {
515 const res = await getVideosList(server.url) 518 const res = await getVideosList(server.url)
516 519
@@ -524,7 +527,7 @@ describe('Test multiple servers', function () {
524 }) 527 })
525 528
526 it('Should view multiple videos on each servers', async function () { 529 it('Should view multiple videos on each servers', async function () {
527 this.timeout(15000) 530 this.timeout(30000)
528 531
529 const tasks: Promise<any>[] = [] 532 const tasks: Promise<any>[] = []
530 tasks.push(viewVideo(servers[0].url, remoteVideosServer1[0])) 533 tasks.push(viewVideo(servers[0].url, remoteVideosServer1[0]))
@@ -542,6 +545,9 @@ describe('Test multiple servers', function () {
542 545
543 await waitJobs(servers) 546 await waitJobs(servers)
544 547
548 // Wait the repeatable job
549 await wait(8000)
550
545 let baseVideos = null 551 let baseVideos = null
546 552
547 for (const server of servers) { 553 for (const server of servers) {
diff --git a/server/tests/api/videos/single-server.ts b/server/tests/api/videos/single-server.ts
index a757ad9da..89408fec6 100644
--- a/server/tests/api/videos/single-server.ts
+++ b/server/tests/api/videos/single-server.ts
@@ -196,7 +196,7 @@ describe('Test a single server', function () {
196 }) 196 })
197 197
198 it('Should have the views updated', async function () { 198 it('Should have the views updated', async function () {
199 this.timeout(10000) 199 this.timeout(20000)
200 200
201 await viewVideo(server.url, videoId) 201 await viewVideo(server.url, videoId)
202 await viewVideo(server.url, videoId) 202 await viewVideo(server.url, videoId)
@@ -212,6 +212,9 @@ describe('Test a single server', function () {
212 await viewVideo(server.url, videoId) 212 await viewVideo(server.url, videoId)
213 await viewVideo(server.url, videoId) 213 await viewVideo(server.url, videoId)
214 214
215 // Wait the repeatable job
216 await wait(8000)
217
215 const res = await getVideo(server.url, videoId) 218 const res = await getVideo(server.url, videoId)
216 219
217 const video = res.body 220 const video = res.body
diff --git a/server/tests/utils/server/jobs.ts b/server/tests/utils/server/jobs.ts
index c9cb8d3a3..4c02cace5 100644
--- a/server/tests/utils/server/jobs.ts
+++ b/server/tests/utils/server/jobs.ts
@@ -1,5 +1,5 @@
1import * as request from 'supertest' 1import * as request from 'supertest'
2import { JobState } from '../../../../shared/models' 2import { Job, JobState } from '../../../../shared/models'
3import { ServerInfo, wait } from '../index' 3import { ServerInfo, wait } from '../index'
4 4
5function getJobsList (url: string, accessToken: string, state: JobState) { 5function getJobsList (url: string, accessToken: string, state: JobState) {
@@ -44,8 +44,10 @@ async function waitJobs (serversArg: ServerInfo[] | ServerInfo) {
44 for (const server of servers) { 44 for (const server of servers) {
45 for (const state of states) { 45 for (const state of states) {
46 const p = getJobsListPaginationAndSort(server.url, server.accessToken, state, 0, 10, '-createdAt') 46 const p = getJobsListPaginationAndSort(server.url, server.accessToken, state, 0, 10, '-createdAt')
47 .then(res => { 47 .then(res => res.body.data)
48 if (res.body.total > 0) pendingRequests = true 48 .then((jobs: Job[]) => jobs.filter(j => j.type !== 'videos-views'))
49 .then(jobs => {
50 if (jobs.length !== 0) pendingRequests = true
49 }) 51 })
50 tasks.push(p) 52 tasks.push(p)
51 } 53 }
diff --git a/shared/models/server/job.model.ts b/shared/models/server/job.model.ts
index 2565479f6..4046297c4 100644
--- a/shared/models/server/job.model.ts
+++ b/shared/models/server/job.model.ts
@@ -7,7 +7,8 @@ export type JobType = 'activitypub-http-unicast' |
7 'video-file-import' | 7 'video-file-import' |
8 'video-file' | 8 'video-file' |
9 'email' | 9 'email' |
10 'video-import' 10 'video-import' |
11 'videos-views'
11 12
12export interface Job { 13export interface Job {
13 id: number 14 id: number