aboutsummaryrefslogtreecommitdiffhomepage
path: root/server
diff options
context:
space:
mode:
authorChocobozzz <florian.bigard@gmail.com>2017-02-21 21:35:59 +0100
committerChocobozzz <florian.bigard@gmail.com>2017-02-26 20:01:26 +0100
commit9e167724f7e933f41d9ea2e1c31772bf4c560a28 (patch)
tree093cb7c1b088f35aaf847f859a313a121c8cd233 /server
parent0150b17e51df3e9fad8a59133d828c68f8ba672b (diff)
downloadPeerTube-9e167724f7e933f41d9ea2e1c31772bf4c560a28.tar.gz
PeerTube-9e167724f7e933f41d9ea2e1c31772bf4c560a28.tar.zst
PeerTube-9e167724f7e933f41d9ea2e1c31772bf4c560a28.zip
Server: make a basic "quick and dirty update" for videos
This system will be useful to to update some int video attributes (likes, dislikes, views...) The classic system is not used because we need some optimization for scaling
Diffstat (limited to 'server')
-rw-r--r--server/controllers/api/remote/videos.js74
-rw-r--r--server/controllers/api/videos.js16
-rw-r--r--server/helpers/custom-validators/remote/videos.js25
-rw-r--r--server/helpers/custom-validators/videos.js17
-rw-r--r--server/helpers/requests.js2
-rw-r--r--server/initializers/constants.js20
-rw-r--r--server/initializers/migrations/0015-video-views.js19
-rw-r--r--server/lib/base-request-scheduler.js140
-rw-r--r--server/lib/friends.js26
-rw-r--r--server/lib/request-scheduler.js178
-rw-r--r--server/lib/request-video-qadu-scheduler.js116
-rw-r--r--server/middlewares/validators/remote/videos.js11
-rw-r--r--server/models/pod.js63
-rw-r--r--server/models/request-to-pod.js4
-rw-r--r--server/models/request-video-qadu.js154
-rw-r--r--server/models/request.js63
-rw-r--r--server/models/video.js13
-rw-r--r--server/tests/api/multiple-pods.js60
-rw-r--r--server/tests/api/single-pod.js11
19 files changed, 796 insertions, 216 deletions
diff --git a/server/controllers/api/remote/videos.js b/server/controllers/api/remote/videos.js
index f8b4949cd..79b503d4d 100644
--- a/server/controllers/api/remote/videos.js
+++ b/server/controllers/api/remote/videos.js
@@ -31,6 +31,13 @@ router.post('/',
31 remoteVideos 31 remoteVideos
32) 32)
33 33
34router.post('/qadu',
35 signatureValidators.signature,
36 secureMiddleware.checkSignature,
37 videosValidators.remoteQaduVideos,
38 remoteVideosQadu
39)
40
34// --------------------------------------------------------------------------- 41// ---------------------------------------------------------------------------
35 42
36module.exports = router 43module.exports = router
@@ -62,6 +69,73 @@ function remoteVideos (req, res, next) {
62 return res.type('json').status(204).end() 69 return res.type('json').status(204).end()
63} 70}
64 71
72function remoteVideosQadu (req, res, next) {
73 const requests = req.body.data
74 const fromPod = res.locals.secure.pod
75
76 eachSeries(requests, function (request, callbackEach) {
77 const videoData = request.data
78
79 quickAndDirtyUpdateVideoRetryWrapper(videoData, fromPod, callbackEach)
80 }, function (err) {
81 if (err) logger.error('Error managing remote videos.', { error: err })
82 })
83
84 return res.type('json').status(204).end()
85}
86
87function quickAndDirtyUpdateVideoRetryWrapper (videoData, fromPod, finalCallback) {
88 const options = {
89 arguments: [ videoData, fromPod ],
90 errorMessage: 'Cannot update quick and dirty the remote video with many retries.'
91 }
92
93 databaseUtils.retryTransactionWrapper(quickAndDirtyUpdateVideo, options, finalCallback)
94}
95
96function quickAndDirtyUpdateVideo (videoData, fromPod, finalCallback) {
97 waterfall([
98 databaseUtils.startSerializableTransaction,
99
100 function findVideo (t, callback) {
101 fetchVideo(fromPod.host, videoData.remoteId, function (err, videoInstance) {
102 return callback(err, t, videoInstance)
103 })
104 },
105
106 function updateVideoIntoDB (t, videoInstance, callback) {
107 const options = { transaction: t }
108
109 if (videoData.views) {
110 videoInstance.set('views', videoData.views)
111 }
112
113 if (videoData.likes) {
114 videoInstance.set('likes', videoData.likes)
115 }
116
117 if (videoData.dislikes) {
118 videoInstance.set('dislikes', videoData.dislikes)
119 }
120
121 videoInstance.save(options).asCallback(function (err) {
122 return callback(err, t)
123 })
124 },
125
126 databaseUtils.commitTransaction
127
128 ], function (err, t) {
129 if (err) {
130 logger.debug('Cannot quick and dirty update the remote video.', { error: err })
131 return databaseUtils.rollbackTransaction(err, t, finalCallback)
132 }
133
134 logger.info('Remote video %s quick and dirty updated', videoData.name)
135 return finalCallback(null)
136 })
137}
138
65// Handle retries on fail 139// Handle retries on fail
66function addRemoteVideoRetryWrapper (videoToCreateData, fromPod, finalCallback) { 140function addRemoteVideoRetryWrapper (videoToCreateData, fromPod, finalCallback) {
67 const options = { 141 const options = {
diff --git a/server/controllers/api/videos.js b/server/controllers/api/videos.js
index c936105e7..9f4bbb7b7 100644
--- a/server/controllers/api/videos.js
+++ b/server/controllers/api/videos.js
@@ -320,6 +320,22 @@ function updateVideo (req, res, finalCallback) {
320 320
321function getVideo (req, res, next) { 321function getVideo (req, res, next) {
322 const videoInstance = res.locals.video 322 const videoInstance = res.locals.video
323
324 if (videoInstance.isOwned()) {
325 // The increment is done directly in the database, not using the instance value
326 videoInstance.increment('views').asCallback(function (err) {
327 if (err) {
328 logger.error('Cannot add view to video %d.', videoInstance.id)
329 return
330 }
331
332 // FIXME: make a real view system
333 // For example, only add a view when a user watch a video during 30s etc
334 friends.quickAndDirtyUpdateVideoToFriends(videoInstance.id, constants.REQUEST_VIDEO_QADU_TYPES.VIEWS)
335 })
336 }
337
338 // Do not wait the view system
323 res.json(videoInstance.toFormatedJSON()) 339 res.json(videoInstance.toFormatedJSON())
324} 340}
325 341
diff --git a/server/helpers/custom-validators/remote/videos.js b/server/helpers/custom-validators/remote/videos.js
index ee68ebc10..2e9cf822e 100644
--- a/server/helpers/custom-validators/remote/videos.js
+++ b/server/helpers/custom-validators/remote/videos.js
@@ -1,5 +1,7 @@
1'use strict' 1'use strict'
2 2
3const has = require('lodash/has')
4
3const constants = require('../../../initializers/constants') 5const constants = require('../../../initializers/constants')
4const videosValidators = require('../videos') 6const videosValidators = require('../videos')
5const miscValidators = require('../misc') 7const miscValidators = require('../misc')
@@ -7,7 +9,8 @@ const miscValidators = require('../misc')
7const ENDPOINT_ACTIONS = constants.REQUEST_ENDPOINT_ACTIONS[constants.REQUEST_ENDPOINTS.VIDEOS] 9const ENDPOINT_ACTIONS = constants.REQUEST_ENDPOINT_ACTIONS[constants.REQUEST_ENDPOINTS.VIDEOS]
8 10
9const remoteVideosValidators = { 11const remoteVideosValidators = {
10 isEachRemoteRequestVideosValid 12 isEachRemoteRequestVideosValid,
13 isEachRemoteRequestVideosQaduValid
11} 14}
12 15
13function isEachRemoteRequestVideosValid (requests) { 16function isEachRemoteRequestVideosValid (requests) {
@@ -16,13 +19,13 @@ function isEachRemoteRequestVideosValid (requests) {
16 const video = request.data 19 const video = request.data
17 return ( 20 return (
18 isRequestTypeAddValid(request.type) && 21 isRequestTypeAddValid(request.type) &&
19 isCommonVideoAttrbiutesValid(video) && 22 isCommonVideoAttributesValid(video) &&
20 videosValidators.isVideoAuthorValid(video.author) && 23 videosValidators.isVideoAuthorValid(video.author) &&
21 videosValidators.isVideoThumbnailDataValid(video.thumbnailData) 24 videosValidators.isVideoThumbnailDataValid(video.thumbnailData)
22 ) || 25 ) ||
23 ( 26 (
24 isRequestTypeUpdateValid(request.type) && 27 isRequestTypeUpdateValid(request.type) &&
25 isCommonVideoAttrbiutesValid(video) 28 isCommonVideoAttributesValid(video)
26 ) || 29 ) ||
27 ( 30 (
28 isRequestTypeRemoveValid(request.type) && 31 isRequestTypeRemoveValid(request.type) &&
@@ -37,13 +40,27 @@ function isEachRemoteRequestVideosValid (requests) {
37 }) 40 })
38} 41}
39 42
43function isEachRemoteRequestVideosQaduValid (requests) {
44 return miscValidators.isArray(requests) &&
45 requests.every(function (request) {
46 const video = request.data
47
48 return (
49 videosValidators.isVideoRemoteIdValid(video.remoteId) &&
50 (has(video, 'views') === false || videosValidators.isVideoViewsValid) &&
51 (has(video, 'likes') === false || videosValidators.isVideoLikesValid) &&
52 (has(video, 'dislikes') === false || videosValidators.isVideoDislikesValid)
53 )
54 })
55}
56
40// --------------------------------------------------------------------------- 57// ---------------------------------------------------------------------------
41 58
42module.exports = remoteVideosValidators 59module.exports = remoteVideosValidators
43 60
44// --------------------------------------------------------------------------- 61// ---------------------------------------------------------------------------
45 62
46function isCommonVideoAttrbiutesValid (video) { 63function isCommonVideoAttributesValid (video) {
47 return videosValidators.isVideoDateValid(video.createdAt) && 64 return videosValidators.isVideoDateValid(video.createdAt) &&
48 videosValidators.isVideoDateValid(video.updatedAt) && 65 videosValidators.isVideoDateValid(video.updatedAt) &&
49 videosValidators.isVideoDescriptionValid(video.description) && 66 videosValidators.isVideoDescriptionValid(video.description) &&
diff --git a/server/helpers/custom-validators/videos.js b/server/helpers/custom-validators/videos.js
index e2d2c8e6d..1d844118b 100644
--- a/server/helpers/custom-validators/videos.js
+++ b/server/helpers/custom-validators/videos.js
@@ -22,7 +22,10 @@ const videosValidators = {
22 isVideoRemoteIdValid, 22 isVideoRemoteIdValid,
23 isVideoAbuseReasonValid, 23 isVideoAbuseReasonValid,
24 isVideoAbuseReporterUsernameValid, 24 isVideoAbuseReporterUsernameValid,
25 isVideoFile 25 isVideoFile,
26 isVideoViewsValid,
27 isVideoLikesValid,
28 isVideoDislikesValid
26} 29}
27 30
28function isVideoAuthorValid (value) { 31function isVideoAuthorValid (value) {
@@ -82,6 +85,18 @@ function isVideoAbuseReporterUsernameValid (value) {
82 return usersValidators.isUserUsernameValid(value) 85 return usersValidators.isUserUsernameValid(value)
83} 86}
84 87
88function isVideoViewsValid (value) {
89 return validator.isInt(value, { min: 0 })
90}
91
92function isVideoLikesValid (value) {
93 return validator.isInt(value, { min: 0 })
94}
95
96function isVideoDislikesValid (value) {
97 return validator.isInt(value, { min: 0 })
98}
99
85function isVideoFile (value, files) { 100function isVideoFile (value, files) {
86 // Should have files 101 // Should have files
87 if (!files) return false 102 if (!files) return false
diff --git a/server/helpers/requests.js b/server/helpers/requests.js
index 095b95e1c..427864117 100644
--- a/server/helpers/requests.js
+++ b/server/helpers/requests.js
@@ -58,6 +58,8 @@ function makeSecureRequest (params, callback) {
58 requestParams.json.data = params.data 58 requestParams.json.data = params.data
59 } 59 }
60 60
61 console.log(requestParams.json.data)
62
61 request.post(requestParams, callback) 63 request.post(requestParams, callback)
62} 64}
63 65
diff --git a/server/initializers/constants.js b/server/initializers/constants.js
index 821580893..668bfe56c 100644
--- a/server/initializers/constants.js
+++ b/server/initializers/constants.js
@@ -5,7 +5,7 @@ const path = require('path')
5 5
6// --------------------------------------------------------------------------- 6// ---------------------------------------------------------------------------
7 7
8const LAST_MIGRATION_VERSION = 10 8const LAST_MIGRATION_VERSION = 15
9 9
10// --------------------------------------------------------------------------- 10// ---------------------------------------------------------------------------
11 11
@@ -24,7 +24,7 @@ const SEARCHABLE_COLUMNS = {
24const SORTABLE_COLUMNS = { 24const SORTABLE_COLUMNS = {
25 USERS: [ 'id', '-id', 'username', '-username', 'createdAt', '-createdAt' ], 25 USERS: [ 'id', '-id', 'username', '-username', 'createdAt', '-createdAt' ],
26 VIDEO_ABUSES: [ 'id', '-id', 'createdAt', '-createdAt' ], 26 VIDEO_ABUSES: [ 'id', '-id', 'createdAt', '-createdAt' ],
27 VIDEOS: [ 'name', '-name', 'duration', '-duration', 'createdAt', '-createdAt' ] 27 VIDEOS: [ 'name', '-name', 'duration', '-duration', 'createdAt', '-createdAt', 'views', '-views' ]
28} 28}
29 29
30const OAUTH_LIFETIME = { 30const OAUTH_LIFETIME = {
@@ -116,11 +116,16 @@ const REQUESTS_LIMIT_PODS = 10
116// How many requests we send to a pod per interval 116// How many requests we send to a pod per interval
117const REQUESTS_LIMIT_PER_POD = 5 117const REQUESTS_LIMIT_PER_POD = 5
118 118
119const REQUESTS_VIDEO_QADU_LIMIT_PODS = 10
120// The QADU requests are not big
121const REQUESTS_VIDEO_QADU_LIMIT_PER_POD = 50
122
119// Number of requests to retry for replay requests module 123// Number of requests to retry for replay requests module
120const RETRY_REQUESTS = 5 124const RETRY_REQUESTS = 5
121 125
122const REQUEST_ENDPOINTS = { 126const REQUEST_ENDPOINTS = {
123 VIDEOS: 'videos' 127 VIDEOS: 'videos',
128 QADU: 'videos/qadu'
124} 129}
125const REQUEST_ENDPOINT_ACTIONS = {} 130const REQUEST_ENDPOINT_ACTIONS = {}
126REQUEST_ENDPOINT_ACTIONS[REQUEST_ENDPOINTS.VIDEOS] = { 131REQUEST_ENDPOINT_ACTIONS[REQUEST_ENDPOINTS.VIDEOS] = {
@@ -130,6 +135,12 @@ REQUEST_ENDPOINT_ACTIONS[REQUEST_ENDPOINTS.VIDEOS] = {
130 REPORT_ABUSE: 'report-abuse' 135 REPORT_ABUSE: 'report-abuse'
131} 136}
132 137
138const REQUEST_VIDEO_QADU_TYPES = {
139 LIKES: 'likes',
140 DISLIKES: 'dislikes',
141 VIEWS: 'views'
142}
143
133const REMOTE_SCHEME = { 144const REMOTE_SCHEME = {
134 HTTP: 'https', 145 HTTP: 'https',
135 WS: 'wss' 146 WS: 'wss'
@@ -199,10 +210,13 @@ module.exports = {
199 REMOTE_SCHEME, 210 REMOTE_SCHEME,
200 REQUEST_ENDPOINT_ACTIONS, 211 REQUEST_ENDPOINT_ACTIONS,
201 REQUEST_ENDPOINTS, 212 REQUEST_ENDPOINTS,
213 REQUEST_VIDEO_QADU_TYPES,
202 REQUESTS_IN_PARALLEL, 214 REQUESTS_IN_PARALLEL,
203 REQUESTS_INTERVAL, 215 REQUESTS_INTERVAL,
204 REQUESTS_LIMIT_PER_POD, 216 REQUESTS_LIMIT_PER_POD,
205 REQUESTS_LIMIT_PODS, 217 REQUESTS_LIMIT_PODS,
218 REQUESTS_VIDEO_QADU_LIMIT_PER_POD,
219 REQUESTS_VIDEO_QADU_LIMIT_PODS,
206 RETRY_REQUESTS, 220 RETRY_REQUESTS,
207 SEARCHABLE_COLUMNS, 221 SEARCHABLE_COLUMNS,
208 SIGNATURE_ALGORITHM, 222 SIGNATURE_ALGORITHM,
diff --git a/server/initializers/migrations/0015-video-views.js b/server/initializers/migrations/0015-video-views.js
new file mode 100644
index 000000000..ae49fe73c
--- /dev/null
+++ b/server/initializers/migrations/0015-video-views.js
@@ -0,0 +1,19 @@
1'use strict'
2
3// utils = { transaction, queryInterface, sequelize, Sequelize }
4exports.up = function (utils, finalCallback) {
5 const q = utils.queryInterface
6 const Sequelize = utils.Sequelize
7
8 const data = {
9 type: Sequelize.INTEGER,
10 allowNull: false,
11 defaultValue: 0
12 }
13
14 q.addColumn('Videos', 'views', data, { transaction: utils.transaction }).asCallback(finalCallback)
15}
16
17exports.down = function (options, callback) {
18 throw new Error('Not implemented.')
19}
diff --git a/server/lib/base-request-scheduler.js b/server/lib/base-request-scheduler.js
new file mode 100644
index 000000000..d15680c25
--- /dev/null
+++ b/server/lib/base-request-scheduler.js
@@ -0,0 +1,140 @@
1'use strict'
2
3const eachLimit = require('async/eachLimit')
4
5const constants = require('../initializers/constants')
6const db = require('../initializers/database')
7const logger = require('../helpers/logger')
8const requests = require('../helpers/requests')
9
10module.exports = class BaseRequestScheduler {
11
12 constructor (options) {
13 this.lastRequestTimestamp = 0
14 this.timer = null
15 }
16
17 activate () {
18 logger.info('Requests scheduler activated.')
19 this.lastRequestTimestamp = Date.now()
20
21 this.timer = setInterval(() => {
22 this.lastRequestTimestamp = Date.now()
23 this.makeRequests()
24 }, constants.REQUESTS_INTERVAL)
25 }
26
27 deactivate () {
28 logger.info('Requests scheduler deactivated.')
29 clearInterval(this.timer)
30 this.timer = null
31 }
32
33 forceSend () {
34 logger.info('Force requests scheduler sending.')
35 this.makeRequests()
36 }
37
38 remainingMilliSeconds () {
39 if (this.timer === null) return -1
40
41 return constants.REQUESTS_INTERVAL - (Date.now() - this.lastRequestTimestamp)
42 }
43
44 // ---------------------------------------------------------------------------
45
46 // Make a requests to friends of a certain type
47 makeRequest (toPod, requestEndpoint, requestsToMake, callback) {
48 if (!callback) callback = function () {}
49
50 const params = {
51 toPod: toPod,
52 sign: true, // Prove our identity
53 method: 'POST',
54 path: '/api/' + constants.API_VERSION + '/remote/' + requestEndpoint,
55 data: requestsToMake // Requests we need to make
56 }
57
58 // Make multiple retry requests to all of pods
59 // The function fire some useful callbacks
60 requests.makeSecureRequest(params, (err, res) => {
61 if (err || (res.statusCode !== 200 && res.statusCode !== 201 && res.statusCode !== 204)) {
62 err = err ? err.message : 'Status code not 20x : ' + res.statusCode
63 logger.error('Error sending secure request to %s pod.', toPod.host, { error: err })
64
65 return callback(false)
66 }
67
68 return callback(true)
69 })
70 }
71
72 // Make all the requests of the scheduler
73 makeRequests () {
74 this.getRequestModel().listWithLimitAndRandom(this.limitPods, this.limitPerPod, (err, requests) => {
75 if (err) {
76 logger.error('Cannot get the list of "%s".', this.description, { err: err })
77 return // Abort
78 }
79
80 // If there are no requests, abort
81 if (requests.length === 0) {
82 logger.info('No "%s" to make.', this.description)
83 return
84 }
85
86 // We want to group requests by destinations pod and endpoint
87 const requestsToMakeGrouped = this.buildRequestObjects(requests)
88
89 logger.info('Making "%s" to friends.', this.description)
90
91 const goodPods = []
92 const badPods = []
93
94 eachLimit(Object.keys(requestsToMakeGrouped), constants.REQUESTS_IN_PARALLEL, (hashKey, callbackEach) => {
95 const requestToMake = requestsToMakeGrouped[hashKey]
96 const toPod = requestToMake.toPod
97
98 // Maybe the pod is not our friend anymore so simply remove it
99 if (!toPod) {
100 const requestIdsToDelete = requestToMake.ids
101
102 logger.info('Removing %d "%s" of unexisting pod %s.', requestIdsToDelete.length, this.description, requestToMake.toPod.id)
103 return this.getRequestToPodModel().removePodOf(requestIdsToDelete, requestToMake.toPod.id, callbackEach)
104 }
105
106 this.makeRequest(toPod, requestToMake.endpoint, requestToMake.datas, (success) => {
107 if (success === false) {
108 badPods.push(requestToMake.toPod.id)
109 return callbackEach()
110 }
111
112 logger.debug('Removing requests for pod %s.', requestToMake.toPod.id, { requestsIds: requestToMake.ids })
113 goodPods.push(requestToMake.toPod.id)
114
115 // Remove the pod id of these request ids
116 this.getRequestToPodModel().removeByRequestIdsAndPod(requestToMake.ids, requestToMake.toPod.id, callbackEach)
117
118 this.afterRequestHook()
119 })
120 }, () => {
121 // All the requests were made, we update the pods score
122 db.Pod.updatePodsScore(goodPods, badPods)
123
124 this.afterRequestsHook()
125 })
126 })
127 }
128
129 flush (callback) {
130 this.getRequestModel().removeAll(callback)
131 }
132
133 afterRequestHook () {
134 // Nothing to do, let children reimplement it
135 }
136
137 afterRequestsHook () {
138 // Nothing to do, let children reimplement it
139 }
140}
diff --git a/server/lib/friends.js b/server/lib/friends.js
index d53ab4553..424a30801 100644
--- a/server/lib/friends.js
+++ b/server/lib/friends.js
@@ -12,15 +12,19 @@ const logger = require('../helpers/logger')
12const peertubeCrypto = require('../helpers/peertube-crypto') 12const peertubeCrypto = require('../helpers/peertube-crypto')
13const requests = require('../helpers/requests') 13const requests = require('../helpers/requests')
14const RequestScheduler = require('./request-scheduler') 14const RequestScheduler = require('./request-scheduler')
15const RequestVideoQaduScheduler = require('./request-video-qadu-scheduler')
15 16
16const ENDPOINT_ACTIONS = constants.REQUEST_ENDPOINT_ACTIONS[constants.REQUEST_ENDPOINTS.VIDEOS] 17const ENDPOINT_ACTIONS = constants.REQUEST_ENDPOINT_ACTIONS[constants.REQUEST_ENDPOINTS.VIDEOS]
18
17const requestScheduler = new RequestScheduler() 19const requestScheduler = new RequestScheduler()
20const requestSchedulerVideoQadu = new RequestVideoQaduScheduler()
18 21
19const friends = { 22const friends = {
20 activate, 23 activate,
21 addVideoToFriends, 24 addVideoToFriends,
22 updateVideoToFriends, 25 updateVideoToFriends,
23 reportAbuseVideoToFriend, 26 reportAbuseVideoToFriend,
27 quickAndDirtyUpdateVideoToFriends,
24 hasFriends, 28 hasFriends,
25 makeFriends, 29 makeFriends,
26 quitFriends, 30 quitFriends,
@@ -30,6 +34,7 @@ const friends = {
30 34
31function activate () { 35function activate () {
32 requestScheduler.activate() 36 requestScheduler.activate()
37 requestSchedulerVideoQadu.activate()
33} 38}
34 39
35function addVideoToFriends (videoData, transaction, callback) { 40function addVideoToFriends (videoData, transaction, callback) {
@@ -71,6 +76,15 @@ function reportAbuseVideoToFriend (reportData, video) {
71 createRequest(options) 76 createRequest(options)
72} 77}
73 78
79function quickAndDirtyUpdateVideoToFriends (videoId, type, transaction, callback) {
80 const options = {
81 videoId,
82 type,
83 transaction
84 }
85 return createVideoQaduRequest(options, callback)
86}
87
74function hasFriends (callback) { 88function hasFriends (callback) {
75 db.Pod.countAll(function (err, count) { 89 db.Pod.countAll(function (err, count) {
76 if (err) return callback(err) 90 if (err) return callback(err)
@@ -110,7 +124,11 @@ function quitFriends (callback) {
110 124
111 waterfall([ 125 waterfall([
112 function flushRequests (callbackAsync) { 126 function flushRequests (callbackAsync) {
113 requestScheduler.flush(callbackAsync) 127 requestScheduler.flush(err => callbackAsync(err))
128 },
129
130 function flushVideoQaduRequests (callbackAsync) {
131 requestSchedulerVideoQadu.flush(err => callbackAsync(err))
114 }, 132 },
115 133
116 function getPodsList (callbackAsync) { 134 function getPodsList (callbackAsync) {
@@ -310,6 +328,12 @@ function createRequest (options, callback) {
310 }) 328 })
311} 329}
312 330
331function createVideoQaduRequest (options, callback) {
332 if (!callback) callback = function () {}
333
334 requestSchedulerVideoQadu.createRequest(options, callback)
335}
336
313function isMe (host) { 337function isMe (host) {
314 return host === constants.CONFIG.WEBSERVER.HOST 338 return host === constants.CONFIG.WEBSERVER.HOST
315} 339}
diff --git a/server/lib/request-scheduler.js b/server/lib/request-scheduler.js
index 28dabe339..6b6535519 100644
--- a/server/lib/request-scheduler.js
+++ b/server/lib/request-scheduler.js
@@ -1,44 +1,54 @@
1'use strict' 1'use strict'
2 2
3const eachLimit = require('async/eachLimit')
4
5const constants = require('../initializers/constants') 3const constants = require('../initializers/constants')
4const BaseRequestScheduler = require('./base-request-scheduler')
6const db = require('../initializers/database') 5const db = require('../initializers/database')
7const logger = require('../helpers/logger') 6const logger = require('../helpers/logger')
8const requests = require('../helpers/requests')
9 7
10module.exports = class RequestScheduler { 8module.exports = class RequestScheduler extends BaseRequestScheduler {
11 9
12 constructor () { 10 constructor () {
13 this.lastRequestTimestamp = 0 11 super()
14 this.timer = null
15 }
16 12
17 activate () { 13 // We limit the size of the requests
18 logger.info('Requests scheduler activated.') 14 this.limitPods = constants.REQUESTS_LIMIT_PODS
19 this.lastRequestTimestamp = Date.now() 15 this.limitPerPod = constants.REQUESTS_LIMIT_PER_POD
20 16
21 this.timer = setInterval(() => { 17 this.description = 'requests'
22 this.lastRequestTimestamp = Date.now()
23 this.makeRequests()
24 }, constants.REQUESTS_INTERVAL)
25 } 18 }
26 19
27 deactivate () { 20 getRequestModel () {
28 logger.info('Requests scheduler deactivated.') 21 return db.Request
29 clearInterval(this.timer)
30 this.timer = null
31 } 22 }
32 23
33 forceSend () { 24 getRequestToPodModel () {
34 logger.info('Force requests scheduler sending.') 25 return db.RequestToPod
35 this.makeRequests()
36 } 26 }
37 27
38 remainingMilliSeconds () { 28 buildRequestObjects (requests) {
39 if (this.timer === null) return -1 29 const requestsToMakeGrouped = {}
30
31 Object.keys(requests).forEach(toPodId => {
32 requests[toPodId].forEach(data => {
33 const request = data.request
34 const pod = data.pod
35 const hashKey = toPodId + request.endpoint
36
37 if (!requestsToMakeGrouped[hashKey]) {
38 requestsToMakeGrouped[hashKey] = {
39 toPod: pod,
40 endpoint: request.endpoint,
41 ids: [], // request ids, to delete them from the DB in the future
42 datas: [] // requests data,
43 }
44 }
45
46 requestsToMakeGrouped[hashKey].ids.push(request.id)
47 requestsToMakeGrouped[hashKey].datas.push(request.request)
48 })
49 })
40 50
41 return constants.REQUESTS_INTERVAL - (Date.now() - this.lastRequestTimestamp) 51 return requestsToMakeGrouped
42 } 52 }
43 53
44 // { type, endpoint, data, toIds, transaction } 54 // { type, endpoint, data, toIds, transaction }
@@ -79,122 +89,10 @@ module.exports = class RequestScheduler {
79 89
80 // --------------------------------------------------------------------------- 90 // ---------------------------------------------------------------------------
81 91
82 // Make all the requests of the scheduler 92 afterRequestsHook () {
83 makeRequests () { 93 // Flush requests with no pod
84 // We limit the size of the requests 94 this.getRequestModel().removeWithEmptyTo(err => {
85 // We don't want to stuck with the same failing requests so we get a random list 95 if (err) logger.error('Error when removing requests with no pods.', { error: err })
86 db.Request.listWithLimitAndRandom(constants.REQUESTS_LIMIT_PODS, constants.REQUESTS_LIMIT_PER_POD, (err, requests) => {
87 if (err) {
88 logger.error('Cannot get the list of requests.', { err: err })
89 return // Abort
90 }
91
92 // If there are no requests, abort
93 if (requests.length === 0) {
94 logger.info('No requests to make.')
95 return
96 }
97
98 // We want to group requests by destinations pod and endpoint
99 const requestsToMakeGrouped = this.buildRequestObjects(requests)
100
101 logger.info('Making requests to friends.')
102
103 const goodPods = []
104 const badPods = []
105
106 eachLimit(Object.keys(requestsToMakeGrouped), constants.REQUESTS_IN_PARALLEL, (hashKey, callbackEach) => {
107 const requestToMake = requestsToMakeGrouped[hashKey]
108 const toPod = requestToMake.toPod
109
110 // Maybe the pod is not our friend anymore so simply remove it
111 if (!toPod) {
112 const requestIdsToDelete = requestToMake.ids
113
114 logger.info('Removing %d requests of unexisting pod %s.', requestIdsToDelete.length, requestToMake.toPod.id)
115 return db.RequestToPod.removePodOf(requestIdsToDelete, requestToMake.toPod.id, callbackEach)
116 }
117
118 this.makeRequest(toPod, requestToMake.endpoint, requestToMake.datas, (success) => {
119 if (success === false) {
120 badPods.push(requestToMake.toPod.id)
121 return callbackEach()
122 }
123
124 logger.debug('Removing requests for pod %s.', requestToMake.toPod.id, { requestsIds: requestToMake.ids })
125 goodPods.push(requestToMake.toPod.id)
126
127 // Remove the pod id of these request ids
128 db.RequestToPod.removePodOf(requestToMake.ids, requestToMake.toPod.id, callbackEach)
129 })
130 }, () => {
131 // All the requests were made, we update the pods score
132 db.Request.updatePodsScore(goodPods, badPods)
133 // Flush requests with no pod
134 db.Request.removeWithEmptyTo(err => {
135 if (err) logger.error('Error when removing requests with no pods.', { error: err })
136 })
137 })
138 })
139 }
140
141 // Make a requests to friends of a certain type
142 makeRequest (toPod, requestEndpoint, requestsToMake, callback) {
143 if (!callback) callback = function () {}
144
145 const params = {
146 toPod: toPod,
147 sign: true, // Prove our identity
148 method: 'POST',
149 path: '/api/' + constants.API_VERSION + '/remote/' + requestEndpoint,
150 data: requestsToMake // Requests we need to make
151 }
152
153 // Make multiple retry requests to all of pods
154 // The function fire some useful callbacks
155 requests.makeSecureRequest(params, (err, res) => {
156 if (err || (res.statusCode !== 200 && res.statusCode !== 201 && res.statusCode !== 204)) {
157 err = err ? err.message : 'Status code not 20x : ' + res.statusCode
158 logger.error('Error sending secure request to %s pod.', toPod.host, { error: err })
159
160 return callback(false)
161 }
162
163 return callback(true)
164 })
165 }
166
167 buildRequestObjects (requests) {
168 const requestsToMakeGrouped = {}
169
170 Object.keys(requests).forEach(toPodId => {
171 requests[toPodId].forEach(data => {
172 const request = data.request
173 const pod = data.pod
174 const hashKey = toPodId + request.endpoint
175
176 if (!requestsToMakeGrouped[hashKey]) {
177 requestsToMakeGrouped[hashKey] = {
178 toPod: pod,
179 endpoint: request.endpoint,
180 ids: [], // request ids, to delete them from the DB in the future
181 datas: [] // requests data,
182 }
183 }
184
185 requestsToMakeGrouped[hashKey].ids.push(request.id)
186 requestsToMakeGrouped[hashKey].datas.push(request.request)
187 })
188 })
189
190 return requestsToMakeGrouped
191 }
192
193 flush (callback) {
194 db.Request.removeAll(err => {
195 if (err) logger.error('Cannot flush the requests.', { error: err })
196
197 return callback(err)
198 }) 96 })
199 } 97 }
200} 98}
diff --git a/server/lib/request-video-qadu-scheduler.js b/server/lib/request-video-qadu-scheduler.js
new file mode 100644
index 000000000..401b2fb44
--- /dev/null
+++ b/server/lib/request-video-qadu-scheduler.js
@@ -0,0 +1,116 @@
1'use strict'
2
3const BaseRequestScheduler = require('./base-request-scheduler')
4const constants = require('../initializers/constants')
5const db = require('../initializers/database')
6const logger = require('../helpers/logger')
7
8module.exports = class RequestVideoQaduScheduler extends BaseRequestScheduler {
9
10 constructor () {
11 super()
12
13 // We limit the size of the requests
14 this.limitPods = constants.REQUESTS_VIDEO_QADU_LIMIT_PODS
15 this.limitPerPod = constants.REQUESTS_VIDEO_QADU_LIMIT_PODS
16
17 this.description = 'video QADU requests'
18 }
19
20 getRequestModel () {
21 return db.RequestVideoQadu
22 }
23
24 getRequestToPodModel () {
25 return db.RequestVideoQadu
26 }
27
28 buildRequestObjects (requests) {
29 const requestsToMakeGrouped = {}
30
31 Object.keys(requests).forEach(toPodId => {
32 requests[toPodId].forEach(data => {
33 const request = data.request
34 const video = data.video
35 const pod = data.pod
36 const hashKey = toPodId
37
38 if (!requestsToMakeGrouped[hashKey]) {
39 requestsToMakeGrouped[hashKey] = {
40 toPod: pod,
41 endpoint: constants.REQUEST_ENDPOINTS.QADU,
42 ids: [], // request ids, to delete them from the DB in the future
43 datas: [], // requests data
44 videos: {}
45 }
46 }
47
48 if (!requestsToMakeGrouped[hashKey].videos[video.id]) {
49 requestsToMakeGrouped[hashKey].videos[video.id] = {}
50 }
51
52 const videoData = requestsToMakeGrouped[hashKey].videos[video.id]
53
54 switch (request.type) {
55 case constants.REQUEST_VIDEO_QADU_TYPES.LIKES:
56 videoData.likes = video.likes
57 break
58
59 case constants.REQUEST_VIDEO_QADU_TYPES.DISLIKES:
60 videoData.likes = video.dislikes
61 break
62
63 case constants.REQUEST_VIDEO_QADU_TYPES.VIEWS:
64 videoData.views = video.views
65 break
66
67 default:
68 logger.error('Unknown request video QADU type %s.', request.type)
69 return
70 }
71
72 // Do not forget the remoteId so the remote pod can identify the video
73 videoData.remoteId = video.id
74 requestsToMakeGrouped[hashKey].ids.push(request.id)
75 requestsToMakeGrouped[hashKey].videos[video.id] = videoData
76 })
77 })
78
79 Object.keys(requestsToMakeGrouped).forEach(hashKey => {
80 Object.keys(requestsToMakeGrouped[hashKey].videos).forEach(videoId => {
81 const videoData = requestsToMakeGrouped[hashKey].videos[videoId]
82
83 requestsToMakeGrouped[hashKey].datas.push({
84 data: videoData
85 })
86 })
87
88 // We don't need it anymore, it was just to build our datas array
89 delete requestsToMakeGrouped[hashKey].videos
90 })
91
92 return requestsToMakeGrouped
93 }
94
95 // { type, videoId, transaction? }
96 createRequest (options, callback) {
97 const type = options.type
98 const videoId = options.videoId
99 const transaction = options.transaction
100
101 const dbRequestOptions = {}
102 if (transaction) dbRequestOptions.transaction = transaction
103
104 // Send the update to all our friends
105 db.Pod.listAllIds(options.transaction, function (err, podIds) {
106 if (err) return callback(err)
107
108 const queries = []
109 podIds.forEach(podId => {
110 queries.push({ type, videoId, podId })
111 })
112
113 return db.RequestVideoQadu.bulkCreate(queries, dbRequestOptions).asCallback(callback)
114 })
115 }
116}
diff --git a/server/middlewares/validators/remote/videos.js b/server/middlewares/validators/remote/videos.js
index cf9925b6c..ddc274c45 100644
--- a/server/middlewares/validators/remote/videos.js
+++ b/server/middlewares/validators/remote/videos.js
@@ -4,7 +4,8 @@ const checkErrors = require('../utils').checkErrors
4const logger = require('../../../helpers/logger') 4const logger = require('../../../helpers/logger')
5 5
6const validatorsRemoteVideos = { 6const validatorsRemoteVideos = {
7 remoteVideos 7 remoteVideos,
8 remoteQaduVideos
8} 9}
9 10
10function remoteVideos (req, res, next) { 11function remoteVideos (req, res, next) {
@@ -15,6 +16,14 @@ function remoteVideos (req, res, next) {
15 checkErrors(req, res, next) 16 checkErrors(req, res, next)
16} 17}
17 18
19function remoteQaduVideos (req, res, next) {
20 req.checkBody('data').isEachRemoteRequestVideosQaduValid()
21
22 logger.debug('Checking remoteVideosQadu parameters', { parameters: req.body })
23
24 checkErrors(req, res, next)
25}
26
18// --------------------------------------------------------------------------- 27// ---------------------------------------------------------------------------
19 28
20module.exports = validatorsRemoteVideos 29module.exports = validatorsRemoteVideos
diff --git a/server/models/pod.js b/server/models/pod.js
index 79afb737a..14814708e 100644
--- a/server/models/pod.js
+++ b/server/models/pod.js
@@ -1,8 +1,11 @@
1'use strict' 1'use strict'
2 2
3const each = require('async/each')
3const map = require('lodash/map') 4const map = require('lodash/map')
5const waterfall = require('async/waterfall')
4 6
5const constants = require('../initializers/constants') 7const constants = require('../initializers/constants')
8const logger = require('../helpers/logger')
6const customPodsValidators = require('../helpers/custom-validators').pods 9const customPodsValidators = require('../helpers/custom-validators').pods
7 10
8// --------------------------------------------------------------------------- 11// ---------------------------------------------------------------------------
@@ -62,6 +65,7 @@ module.exports = function (sequelize, DataTypes) {
62 listBadPods, 65 listBadPods,
63 load, 66 load,
64 loadByHost, 67 loadByHost,
68 updatePodsScore,
65 removeAll 69 removeAll
66 }, 70 },
67 instanceMethods: { 71 instanceMethods: {
@@ -144,7 +148,7 @@ function listAllIds (transaction, callback) {
144 }) 148 })
145} 149}
146 150
147function listRandomPodIdsWithRequest (limit, callback) { 151function listRandomPodIdsWithRequest (limit, tableRequestPod, callback) {
148 const self = this 152 const self = this
149 153
150 self.count().asCallback(function (err, count) { 154 self.count().asCallback(function (err, count) {
@@ -166,7 +170,7 @@ function listRandomPodIdsWithRequest (limit, callback) {
166 where: { 170 where: {
167 id: { 171 id: {
168 $in: [ 172 $in: [
169 this.sequelize.literal('SELECT "podId" FROM "RequestToPods"') 173 this.sequelize.literal('SELECT "podId" FROM "' + tableRequestPod + '"')
170 ] 174 ]
171 } 175 }
172 } 176 }
@@ -207,3 +211,58 @@ function loadByHost (host, callback) {
207function removeAll (callback) { 211function removeAll (callback) {
208 return this.destroy().asCallback(callback) 212 return this.destroy().asCallback(callback)
209} 213}
214
215function updatePodsScore (goodPods, badPods) {
216 const self = this
217
218 logger.info('Updating %d good pods and %d bad pods scores.', goodPods.length, badPods.length)
219
220 if (goodPods.length !== 0) {
221 this.incrementScores(goodPods, constants.PODS_SCORE.BONUS, function (err) {
222 if (err) logger.error('Cannot increment scores of good pods.', { error: err })
223 })
224 }
225
226 if (badPods.length !== 0) {
227 this.incrementScores(badPods, constants.PODS_SCORE.MALUS, function (err) {
228 if (err) logger.error('Cannot decrement scores of bad pods.', { error: err })
229 removeBadPods.call(self)
230 })
231 }
232}
233
234// ---------------------------------------------------------------------------
235
236// Remove pods with a score of 0 (too many requests where they were unreachable)
237function removeBadPods () {
238 const self = this
239
240 waterfall([
241 function findBadPods (callback) {
242 self.sequelize.models.Pod.listBadPods(function (err, pods) {
243 if (err) {
244 logger.error('Cannot find bad pods.', { error: err })
245 return callback(err)
246 }
247
248 return callback(null, pods)
249 })
250 },
251
252 function removeTheseBadPods (pods, callback) {
253 each(pods, function (pod, callbackEach) {
254 pod.destroy().asCallback(callbackEach)
255 }, function (err) {
256 return callback(err, pods.length)
257 })
258 }
259 ], function (err, numberOfPodsRemoved) {
260 if (err) {
261 logger.error('Cannot remove bad pods.', { error: err })
262 } else if (numberOfPodsRemoved) {
263 logger.info('Removed %d pods.', numberOfPodsRemoved)
264 } else {
265 logger.info('No need to remove bad pods.')
266 }
267 })
268}
diff --git a/server/models/request-to-pod.js b/server/models/request-to-pod.js
index f42a53458..0e01a842e 100644
--- a/server/models/request-to-pod.js
+++ b/server/models/request-to-pod.js
@@ -17,7 +17,7 @@ module.exports = function (sequelize, DataTypes) {
17 } 17 }
18 ], 18 ],
19 classMethods: { 19 classMethods: {
20 removePodOf 20 removeByRequestIdsAndPod
21 } 21 }
22 }) 22 })
23 23
@@ -26,7 +26,7 @@ module.exports = function (sequelize, DataTypes) {
26 26
27// --------------------------------------------------------------------------- 27// ---------------------------------------------------------------------------
28 28
29function removePodOf (requestsIds, podId, callback) { 29function removeByRequestIdsAndPod (requestsIds, podId, callback) {
30 if (!callback) callback = function () {} 30 if (!callback) callback = function () {}
31 31
32 const query = { 32 const query = {
diff --git a/server/models/request-video-qadu.js b/server/models/request-video-qadu.js
new file mode 100644
index 000000000..7010fc992
--- /dev/null
+++ b/server/models/request-video-qadu.js
@@ -0,0 +1,154 @@
1'use strict'
2
3/*
4 Request Video for Quick And Dirty Updates like:
5 - views
6 - likes
7 - dislikes
8
9 We can't put it in the same system than basic requests for efficiency.
10 Moreover we don't want to slow down the basic requests with a lot of views/likes/dislikes requests.
11 So we put it an independant request scheduler.
12*/
13
14const values = require('lodash/values')
15
16const constants = require('../initializers/constants')
17
18// ---------------------------------------------------------------------------
19
20module.exports = function (sequelize, DataTypes) {
21 const RequestVideoQadu = sequelize.define('RequestVideoQadu',
22 {
23 type: {
24 type: DataTypes.ENUM(values(constants.REQUEST_VIDEO_QADU_TYPES)),
25 allowNull: false
26 }
27 },
28 {
29 timestamps: false,
30 indexes: [
31 {
32 fields: [ 'podId' ]
33 },
34 {
35 fields: [ 'videoId' ]
36 }
37 ],
38 classMethods: {
39 associate,
40
41 listWithLimitAndRandom,
42
43 countTotalRequests,
44 removeAll,
45 removeByRequestIdsAndPod
46 }
47 }
48 )
49
50 return RequestVideoQadu
51}
52
53// ------------------------------ STATICS ------------------------------
54
55function associate (models) {
56 this.belongsTo(models.Pod, {
57 foreignKey: {
58 name: 'podId',
59 allowNull: false
60 },
61 onDelete: 'CASCADE'
62 })
63
64 this.belongsTo(models.Video, {
65 foreignKey: {
66 name: 'videoId',
67 allowNull: false
68 },
69 onDelete: 'CASCADE'
70 })
71}
72
73function countTotalRequests (callback) {
74 const query = {
75 include: [ this.sequelize.models.Pod ]
76 }
77
78 return this.count(query).asCallback(callback)
79}
80
81function listWithLimitAndRandom (limitPods, limitRequestsPerPod, callback) {
82 const self = this
83 const Pod = this.sequelize.models.Pod
84
85 Pod.listRandomPodIdsWithRequest(limitPods, 'RequestVideoQadus', function (err, podIds) {
86 if (err) return callback(err)
87
88 // We don't have friends that have requests
89 if (podIds.length === 0) return callback(null, [])
90
91 const query = {
92 include: [
93 {
94 model: self.sequelize.models.Pod,
95 where: {
96 id: {
97 $in: podIds
98 }
99 }
100 },
101 {
102 model: self.sequelize.models.Video
103 }
104 ]
105 }
106
107 self.findAll(query).asCallback(function (err, requests) {
108 if (err) return callback(err)
109
110 const requestsGrouped = groupAndTruncateRequests(requests, limitRequestsPerPod)
111 return callback(err, requestsGrouped)
112 })
113 })
114}
115
116function removeByRequestIdsAndPod (ids, podId, callback) {
117 const query = {
118 where: {
119 id: {
120 $in: ids
121 },
122 podId
123 }
124 }
125
126 this.destroy(query).asCallback(callback)
127}
128
129function removeAll (callback) {
130 // Delete all requests
131 this.truncate({ cascade: true }).asCallback(callback)
132}
133
134// ---------------------------------------------------------------------------
135
136function groupAndTruncateRequests (requests, limitRequestsPerPod) {
137 const requestsGrouped = {}
138
139 requests.forEach(function (request) {
140 const pod = request.Pod
141
142 if (!requestsGrouped[pod.id]) requestsGrouped[pod.id] = []
143
144 if (requestsGrouped[pod.id].length < limitRequestsPerPod) {
145 requestsGrouped[pod.id].push({
146 request: request,
147 video: request.Video,
148 pod
149 })
150 }
151 })
152
153 return requestsGrouped
154}
diff --git a/server/models/request.js b/server/models/request.js
index ca616d130..de73501fc 100644
--- a/server/models/request.js
+++ b/server/models/request.js
@@ -1,11 +1,8 @@
1'use strict' 1'use strict'
2 2
3const each = require('async/each')
4const waterfall = require('async/waterfall')
5const values = require('lodash/values') 3const values = require('lodash/values')
6 4
7const constants = require('../initializers/constants') 5const constants = require('../initializers/constants')
8const logger = require('../helpers/logger')
9 6
10// --------------------------------------------------------------------------- 7// ---------------------------------------------------------------------------
11 8
@@ -28,8 +25,6 @@ module.exports = function (sequelize, DataTypes) {
28 listWithLimitAndRandom, 25 listWithLimitAndRandom,
29 26
30 countTotalRequests, 27 countTotalRequests,
31 removeBadPods,
32 updatePodsScore,
33 removeAll, 28 removeAll,
34 removeWithEmptyTo 29 removeWithEmptyTo
35 } 30 }
@@ -60,71 +55,17 @@ function countTotalRequests (callback) {
60 return this.count(query).asCallback(callback) 55 return this.count(query).asCallback(callback)
61} 56}
62 57
63// Remove pods with a score of 0 (too many requests where they were unreachable)
64function removeBadPods () {
65 const self = this
66
67 waterfall([
68 function findBadPods (callback) {
69 self.sequelize.models.Pod.listBadPods(function (err, pods) {
70 if (err) {
71 logger.error('Cannot find bad pods.', { error: err })
72 return callback(err)
73 }
74
75 return callback(null, pods)
76 })
77 },
78
79 function removeTheseBadPods (pods, callback) {
80 each(pods, function (pod, callbackEach) {
81 pod.destroy().asCallback(callbackEach)
82 }, function (err) {
83 return callback(err, pods.length)
84 })
85 }
86 ], function (err, numberOfPodsRemoved) {
87 if (err) {
88 logger.error('Cannot remove bad pods.', { error: err })
89 } else if (numberOfPodsRemoved) {
90 logger.info('Removed %d pods.', numberOfPodsRemoved)
91 } else {
92 logger.info('No need to remove bad pods.')
93 }
94 })
95}
96
97function updatePodsScore (goodPods, badPods) {
98 const self = this
99 const Pod = this.sequelize.models.Pod
100
101 logger.info('Updating %d good pods and %d bad pods scores.', goodPods.length, badPods.length)
102
103 if (goodPods.length !== 0) {
104 Pod.incrementScores(goodPods, constants.PODS_SCORE.BONUS, function (err) {
105 if (err) logger.error('Cannot increment scores of good pods.', { error: err })
106 })
107 }
108
109 if (badPods.length !== 0) {
110 Pod.incrementScores(badPods, constants.PODS_SCORE.MALUS, function (err) {
111 if (err) logger.error('Cannot decrement scores of bad pods.', { error: err })
112 removeBadPods.call(self)
113 })
114 }
115}
116
117function listWithLimitAndRandom (limitPods, limitRequestsPerPod, callback) { 58function listWithLimitAndRandom (limitPods, limitRequestsPerPod, callback) {
118 const self = this 59 const self = this
119 const Pod = this.sequelize.models.Pod 60 const Pod = this.sequelize.models.Pod
120 61
121 Pod.listRandomPodIdsWithRequest(limitPods, function (err, podIds) { 62 Pod.listRandomPodIdsWithRequest(limitPods, 'RequestToPods', function (err, podIds) {
122 if (err) return callback(err) 63 if (err) return callback(err)
123 64
124 // We don't have friends that have requests 65 // We don't have friends that have requests
125 if (podIds.length === 0) return callback(null, []) 66 if (podIds.length === 0) return callback(null, [])
126 67
127 // The the first x requests of these pods 68 // The first x requests of these pods
128 // It is very important to sort by id ASC to keep the requests order! 69 // It is very important to sort by id ASC to keep the requests order!
129 const query = { 70 const query = {
130 order: [ 71 order: [
diff --git a/server/models/video.js b/server/models/video.js
index d0fd61eb4..daa273845 100644
--- a/server/models/video.js
+++ b/server/models/video.js
@@ -80,6 +80,15 @@ module.exports = function (sequelize, DataTypes) {
80 if (res === false) throw new Error('Video duration is not valid.') 80 if (res === false) throw new Error('Video duration is not valid.')
81 } 81 }
82 } 82 }
83 },
84 views: {
85 type: DataTypes.INTEGER,
86 allowNull: false,
87 defaultValue: 0,
88 validate: {
89 min: 0,
90 isInt: true
91 }
83 } 92 }
84 }, 93 },
85 { 94 {
@@ -101,6 +110,9 @@ module.exports = function (sequelize, DataTypes) {
101 }, 110 },
102 { 111 {
103 fields: [ 'infoHash' ] 112 fields: [ 'infoHash' ]
113 },
114 {
115 fields: [ 'views' ]
104 } 116 }
105 ], 117 ],
106 classMethods: { 118 classMethods: {
@@ -336,6 +348,7 @@ function toFormatedJSON () {
336 magnetUri: this.generateMagnetUri(), 348 magnetUri: this.generateMagnetUri(),
337 author: this.Author.name, 349 author: this.Author.name,
338 duration: this.duration, 350 duration: this.duration,
351 views: this.views,
339 tags: map(this.Tags, 'name'), 352 tags: map(this.Tags, 'name'),
340 thumbnailPath: pathUtils.join(constants.STATIC_PATHS.THUMBNAILS, this.getThumbnailName()), 353 thumbnailPath: pathUtils.join(constants.STATIC_PATHS.THUMBNAILS, this.getThumbnailName()),
341 createdAt: this.createdAt, 354 createdAt: this.createdAt,
diff --git a/server/tests/api/multiple-pods.js b/server/tests/api/multiple-pods.js
index df12ba0e9..871db54be 100644
--- a/server/tests/api/multiple-pods.js
+++ b/server/tests/api/multiple-pods.js
@@ -3,6 +3,7 @@
3const chai = require('chai') 3const chai = require('chai')
4const each = require('async/each') 4const each = require('async/each')
5const expect = chai.expect 5const expect = chai.expect
6const parallel = require('async/parallel')
6const series = require('async/series') 7const series = require('async/series')
7const WebTorrent = require('webtorrent') 8const WebTorrent = require('webtorrent')
8const webtorrent = new WebTorrent() 9const webtorrent = new WebTorrent()
@@ -375,6 +376,63 @@ describe('Test multiple pods', function () {
375 }) 376 })
376 }) 377 })
377 378
379 describe('Should update video views', function () {
380 let videoId1
381 let videoId2
382
383 before(function (done) {
384 videosUtils.getVideosList(servers[2].url, function (err, res) {
385 if (err) throw err
386
387 const videos = res.body.data.filter(video => video.isLocal === true)
388 videoId1 = videos[0].id
389 videoId2 = videos[1].id
390
391 done()
392 })
393 })
394
395 it('Should views multiple videos on owned servers', function (done) {
396 this.timeout(30000)
397
398 parallel([
399 function (callback) {
400 videosUtils.getVideo(servers[2].url, videoId1, callback)
401 },
402
403 function (callback) {
404 videosUtils.getVideo(servers[2].url, videoId1, callback)
405 },
406
407 function (callback) {
408 videosUtils.getVideo(servers[2].url, videoId1, callback)
409 },
410
411 function (callback) {
412 videosUtils.getVideo(servers[2].url, videoId2, callback)
413 }
414 ], function (err) {
415 if (err) throw err
416
417 setTimeout(done, 22000)
418 })
419 })
420
421 it('Should have views updated on each pod', function (done) {
422 each(servers, function (server, callback) {
423 videosUtils.getVideosList(server.url, function (err, res) {
424 if (err) throw err
425
426 const videos = res.body.data
427 expect(videos.find(video => video.views === 3)).to.be.exist
428 expect(videos.find(video => video.views === 1)).to.be.exist
429
430 callback()
431 })
432 }, done)
433 })
434 })
435/*
378 describe('Should manipulate these videos', function () { 436 describe('Should manipulate these videos', function () {
379 it('Should update the video 3 by asking pod 3', function (done) { 437 it('Should update the video 3 by asking pod 3', function (done) {
380 this.timeout(15000) 438 this.timeout(15000)
@@ -462,7 +520,7 @@ describe('Test multiple pods', function () {
462 }, done) 520 }, done)
463 }) 521 })
464 }) 522 })
465 523*/
466 after(function (done) { 524 after(function (done) {
467 servers.forEach(function (server) { 525 servers.forEach(function (server) {
468 process.kill(-server.app.pid) 526 process.kill(-server.app.pid)
diff --git a/server/tests/api/single-pod.js b/server/tests/api/single-pod.js
index 83a2b4411..40c33686f 100644
--- a/server/tests/api/single-pod.js
+++ b/server/tests/api/single-pod.js
@@ -129,6 +129,17 @@ describe('Test a single pod', function () {
129 }) 129 })
130 }) 130 })
131 131
132 it('Should have the views updated', function (done) {
133 videosUtils.getVideo(server.url, videoId, function (err, res) {
134 if (err) throw err
135
136 const video = res.body
137 expect(video.views).to.equal(1)
138
139 done()
140 })
141 })
142
132 it('Should search the video by name by default', function (done) { 143 it('Should search the video by name by default', function (done) {
133 videosUtils.searchVideo(server.url, 'my', function (err, res) { 144 videosUtils.searchVideo(server.url, 'my', function (err, res) {
134 if (err) throw err 145 if (err) throw err