aboutsummaryrefslogtreecommitdiffhomepage
diff options
context:
space:
mode:
-rw-r--r--server/controllers/api/remote/videos.js74
-rw-r--r--server/controllers/api/videos.js16
-rw-r--r--server/helpers/custom-validators/remote/videos.js25
-rw-r--r--server/helpers/custom-validators/videos.js17
-rw-r--r--server/helpers/requests.js2
-rw-r--r--server/initializers/constants.js20
-rw-r--r--server/initializers/migrations/0015-video-views.js19
-rw-r--r--server/lib/base-request-scheduler.js140
-rw-r--r--server/lib/friends.js26
-rw-r--r--server/lib/request-scheduler.js178
-rw-r--r--server/lib/request-video-qadu-scheduler.js116
-rw-r--r--server/middlewares/validators/remote/videos.js11
-rw-r--r--server/models/pod.js63
-rw-r--r--server/models/request-to-pod.js4
-rw-r--r--server/models/request-video-qadu.js154
-rw-r--r--server/models/request.js63
-rw-r--r--server/models/video.js13
-rw-r--r--server/tests/api/multiple-pods.js60
-rw-r--r--server/tests/api/single-pod.js11
19 files changed, 796 insertions, 216 deletions
diff --git a/server/controllers/api/remote/videos.js b/server/controllers/api/remote/videos.js
index f8b4949cd..79b503d4d 100644
--- a/server/controllers/api/remote/videos.js
+++ b/server/controllers/api/remote/videos.js
@@ -31,6 +31,13 @@ router.post('/',
31 remoteVideos 31 remoteVideos
32) 32)
33 33
34router.post('/qadu',
35 signatureValidators.signature,
36 secureMiddleware.checkSignature,
37 videosValidators.remoteQaduVideos,
38 remoteVideosQadu
39)
40
34// --------------------------------------------------------------------------- 41// ---------------------------------------------------------------------------
35 42
36module.exports = router 43module.exports = router
@@ -62,6 +69,73 @@ function remoteVideos (req, res, next) {
62 return res.type('json').status(204).end() 69 return res.type('json').status(204).end()
63} 70}
64 71
72function remoteVideosQadu (req, res, next) {
73 const requests = req.body.data
74 const fromPod = res.locals.secure.pod
75
76 eachSeries(requests, function (request, callbackEach) {
77 const videoData = request.data
78
79 quickAndDirtyUpdateVideoRetryWrapper(videoData, fromPod, callbackEach)
80 }, function (err) {
81 if (err) logger.error('Error managing remote videos.', { error: err })
82 })
83
84 return res.type('json').status(204).end()
85}
86
87function quickAndDirtyUpdateVideoRetryWrapper (videoData, fromPod, finalCallback) {
88 const options = {
89 arguments: [ videoData, fromPod ],
90 errorMessage: 'Cannot update quick and dirty the remote video with many retries.'
91 }
92
93 databaseUtils.retryTransactionWrapper(quickAndDirtyUpdateVideo, options, finalCallback)
94}
95
96function quickAndDirtyUpdateVideo (videoData, fromPod, finalCallback) {
97 waterfall([
98 databaseUtils.startSerializableTransaction,
99
100 function findVideo (t, callback) {
101 fetchVideo(fromPod.host, videoData.remoteId, function (err, videoInstance) {
102 return callback(err, t, videoInstance)
103 })
104 },
105
106 function updateVideoIntoDB (t, videoInstance, callback) {
107 const options = { transaction: t }
108
109 if (videoData.views) {
110 videoInstance.set('views', videoData.views)
111 }
112
113 if (videoData.likes) {
114 videoInstance.set('likes', videoData.likes)
115 }
116
117 if (videoData.dislikes) {
118 videoInstance.set('dislikes', videoData.dislikes)
119 }
120
121 videoInstance.save(options).asCallback(function (err) {
122 return callback(err, t)
123 })
124 },
125
126 databaseUtils.commitTransaction
127
128 ], function (err, t) {
129 if (err) {
130 logger.debug('Cannot quick and dirty update the remote video.', { error: err })
131 return databaseUtils.rollbackTransaction(err, t, finalCallback)
132 }
133
134 logger.info('Remote video %s quick and dirty updated', videoData.name)
135 return finalCallback(null)
136 })
137}
138
65// Handle retries on fail 139// Handle retries on fail
66function addRemoteVideoRetryWrapper (videoToCreateData, fromPod, finalCallback) { 140function addRemoteVideoRetryWrapper (videoToCreateData, fromPod, finalCallback) {
67 const options = { 141 const options = {
diff --git a/server/controllers/api/videos.js b/server/controllers/api/videos.js
index c936105e7..9f4bbb7b7 100644
--- a/server/controllers/api/videos.js
+++ b/server/controllers/api/videos.js
@@ -320,6 +320,22 @@ function updateVideo (req, res, finalCallback) {
320 320
321function getVideo (req, res, next) { 321function getVideo (req, res, next) {
322 const videoInstance = res.locals.video 322 const videoInstance = res.locals.video
323
324 if (videoInstance.isOwned()) {
325 // The increment is done directly in the database, not using the instance value
326 videoInstance.increment('views').asCallback(function (err) {
327 if (err) {
328 logger.error('Cannot add view to video %d.', videoInstance.id)
329 return
330 }
331
332 // FIXME: make a real view system
333 // For example, only add a view when a user watch a video during 30s etc
334 friends.quickAndDirtyUpdateVideoToFriends(videoInstance.id, constants.REQUEST_VIDEO_QADU_TYPES.VIEWS)
335 })
336 }
337
338 // Do not wait the view system
323 res.json(videoInstance.toFormatedJSON()) 339 res.json(videoInstance.toFormatedJSON())
324} 340}
325 341
diff --git a/server/helpers/custom-validators/remote/videos.js b/server/helpers/custom-validators/remote/videos.js
index ee68ebc10..2e9cf822e 100644
--- a/server/helpers/custom-validators/remote/videos.js
+++ b/server/helpers/custom-validators/remote/videos.js
@@ -1,5 +1,7 @@
1'use strict' 1'use strict'
2 2
3const has = require('lodash/has')
4
3const constants = require('../../../initializers/constants') 5const constants = require('../../../initializers/constants')
4const videosValidators = require('../videos') 6const videosValidators = require('../videos')
5const miscValidators = require('../misc') 7const miscValidators = require('../misc')
@@ -7,7 +9,8 @@ const miscValidators = require('../misc')
7const ENDPOINT_ACTIONS = constants.REQUEST_ENDPOINT_ACTIONS[constants.REQUEST_ENDPOINTS.VIDEOS] 9const ENDPOINT_ACTIONS = constants.REQUEST_ENDPOINT_ACTIONS[constants.REQUEST_ENDPOINTS.VIDEOS]
8 10
9const remoteVideosValidators = { 11const remoteVideosValidators = {
10 isEachRemoteRequestVideosValid 12 isEachRemoteRequestVideosValid,
13 isEachRemoteRequestVideosQaduValid
11} 14}
12 15
13function isEachRemoteRequestVideosValid (requests) { 16function isEachRemoteRequestVideosValid (requests) {
@@ -16,13 +19,13 @@ function isEachRemoteRequestVideosValid (requests) {
16 const video = request.data 19 const video = request.data
17 return ( 20 return (
18 isRequestTypeAddValid(request.type) && 21 isRequestTypeAddValid(request.type) &&
19 isCommonVideoAttrbiutesValid(video) && 22 isCommonVideoAttributesValid(video) &&
20 videosValidators.isVideoAuthorValid(video.author) && 23 videosValidators.isVideoAuthorValid(video.author) &&
21 videosValidators.isVideoThumbnailDataValid(video.thumbnailData) 24 videosValidators.isVideoThumbnailDataValid(video.thumbnailData)
22 ) || 25 ) ||
23 ( 26 (
24 isRequestTypeUpdateValid(request.type) && 27 isRequestTypeUpdateValid(request.type) &&
25 isCommonVideoAttrbiutesValid(video) 28 isCommonVideoAttributesValid(video)
26 ) || 29 ) ||
27 ( 30 (
28 isRequestTypeRemoveValid(request.type) && 31 isRequestTypeRemoveValid(request.type) &&
@@ -37,13 +40,27 @@ function isEachRemoteRequestVideosValid (requests) {
37 }) 40 })
38} 41}
39 42
43function isEachRemoteRequestVideosQaduValid (requests) {
44 return miscValidators.isArray(requests) &&
45 requests.every(function (request) {
46 const video = request.data
47
48 return (
49 videosValidators.isVideoRemoteIdValid(video.remoteId) &&
50 (has(video, 'views') === false || videosValidators.isVideoViewsValid) &&
51 (has(video, 'likes') === false || videosValidators.isVideoLikesValid) &&
52 (has(video, 'dislikes') === false || videosValidators.isVideoDislikesValid)
53 )
54 })
55}
56
40// --------------------------------------------------------------------------- 57// ---------------------------------------------------------------------------
41 58
42module.exports = remoteVideosValidators 59module.exports = remoteVideosValidators
43 60
44// --------------------------------------------------------------------------- 61// ---------------------------------------------------------------------------
45 62
46function isCommonVideoAttrbiutesValid (video) { 63function isCommonVideoAttributesValid (video) {
47 return videosValidators.isVideoDateValid(video.createdAt) && 64 return videosValidators.isVideoDateValid(video.createdAt) &&
48 videosValidators.isVideoDateValid(video.updatedAt) && 65 videosValidators.isVideoDateValid(video.updatedAt) &&
49 videosValidators.isVideoDescriptionValid(video.description) && 66 videosValidators.isVideoDescriptionValid(video.description) &&
diff --git a/server/helpers/custom-validators/videos.js b/server/helpers/custom-validators/videos.js
index e2d2c8e6d..1d844118b 100644
--- a/server/helpers/custom-validators/videos.js
+++ b/server/helpers/custom-validators/videos.js
@@ -22,7 +22,10 @@ const videosValidators = {
22 isVideoRemoteIdValid, 22 isVideoRemoteIdValid,
23 isVideoAbuseReasonValid, 23 isVideoAbuseReasonValid,
24 isVideoAbuseReporterUsernameValid, 24 isVideoAbuseReporterUsernameValid,
25 isVideoFile 25 isVideoFile,
26 isVideoViewsValid,
27 isVideoLikesValid,
28 isVideoDislikesValid
26} 29}
27 30
28function isVideoAuthorValid (value) { 31function isVideoAuthorValid (value) {
@@ -82,6 +85,18 @@ function isVideoAbuseReporterUsernameValid (value) {
82 return usersValidators.isUserUsernameValid(value) 85 return usersValidators.isUserUsernameValid(value)
83} 86}
84 87
88function isVideoViewsValid (value) {
89 return validator.isInt(value, { min: 0 })
90}
91
92function isVideoLikesValid (value) {
93 return validator.isInt(value, { min: 0 })
94}
95
96function isVideoDislikesValid (value) {
97 return validator.isInt(value, { min: 0 })
98}
99
85function isVideoFile (value, files) { 100function isVideoFile (value, files) {
86 // Should have files 101 // Should have files
87 if (!files) return false 102 if (!files) return false
diff --git a/server/helpers/requests.js b/server/helpers/requests.js
index 095b95e1c..427864117 100644
--- a/server/helpers/requests.js
+++ b/server/helpers/requests.js
@@ -58,6 +58,8 @@ function makeSecureRequest (params, callback) {
58 requestParams.json.data = params.data 58 requestParams.json.data = params.data
59 } 59 }
60 60
61 console.log(requestParams.json.data)
62
61 request.post(requestParams, callback) 63 request.post(requestParams, callback)
62} 64}
63 65
diff --git a/server/initializers/constants.js b/server/initializers/constants.js
index 821580893..668bfe56c 100644
--- a/server/initializers/constants.js
+++ b/server/initializers/constants.js
@@ -5,7 +5,7 @@ const path = require('path')
5 5
6// --------------------------------------------------------------------------- 6// ---------------------------------------------------------------------------
7 7
8const LAST_MIGRATION_VERSION = 10 8const LAST_MIGRATION_VERSION = 15
9 9
10// --------------------------------------------------------------------------- 10// ---------------------------------------------------------------------------
11 11
@@ -24,7 +24,7 @@ const SEARCHABLE_COLUMNS = {
24const SORTABLE_COLUMNS = { 24const SORTABLE_COLUMNS = {
25 USERS: [ 'id', '-id', 'username', '-username', 'createdAt', '-createdAt' ], 25 USERS: [ 'id', '-id', 'username', '-username', 'createdAt', '-createdAt' ],
26 VIDEO_ABUSES: [ 'id', '-id', 'createdAt', '-createdAt' ], 26 VIDEO_ABUSES: [ 'id', '-id', 'createdAt', '-createdAt' ],
27 VIDEOS: [ 'name', '-name', 'duration', '-duration', 'createdAt', '-createdAt' ] 27 VIDEOS: [ 'name', '-name', 'duration', '-duration', 'createdAt', '-createdAt', 'views', '-views' ]
28} 28}
29 29
30const OAUTH_LIFETIME = { 30const OAUTH_LIFETIME = {
@@ -116,11 +116,16 @@ const REQUESTS_LIMIT_PODS = 10
116// How many requests we send to a pod per interval 116// How many requests we send to a pod per interval
117const REQUESTS_LIMIT_PER_POD = 5 117const REQUESTS_LIMIT_PER_POD = 5
118 118
119const REQUESTS_VIDEO_QADU_LIMIT_PODS = 10
120// The QADU requests are not big
121const REQUESTS_VIDEO_QADU_LIMIT_PER_POD = 50
122
119// Number of requests to retry for replay requests module 123// Number of requests to retry for replay requests module
120const RETRY_REQUESTS = 5 124const RETRY_REQUESTS = 5
121 125
122const REQUEST_ENDPOINTS = { 126const REQUEST_ENDPOINTS = {
123 VIDEOS: 'videos' 127 VIDEOS: 'videos',
128 QADU: 'videos/qadu'
124} 129}
125const REQUEST_ENDPOINT_ACTIONS = {} 130const REQUEST_ENDPOINT_ACTIONS = {}
126REQUEST_ENDPOINT_ACTIONS[REQUEST_ENDPOINTS.VIDEOS] = { 131REQUEST_ENDPOINT_ACTIONS[REQUEST_ENDPOINTS.VIDEOS] = {
@@ -130,6 +135,12 @@ REQUEST_ENDPOINT_ACTIONS[REQUEST_ENDPOINTS.VIDEOS] = {
130 REPORT_ABUSE: 'report-abuse' 135 REPORT_ABUSE: 'report-abuse'
131} 136}
132 137
138const REQUEST_VIDEO_QADU_TYPES = {
139 LIKES: 'likes',
140 DISLIKES: 'dislikes',
141 VIEWS: 'views'
142}
143
133const REMOTE_SCHEME = { 144const REMOTE_SCHEME = {
134 HTTP: 'https', 145 HTTP: 'https',
135 WS: 'wss' 146 WS: 'wss'
@@ -199,10 +210,13 @@ module.exports = {
199 REMOTE_SCHEME, 210 REMOTE_SCHEME,
200 REQUEST_ENDPOINT_ACTIONS, 211 REQUEST_ENDPOINT_ACTIONS,
201 REQUEST_ENDPOINTS, 212 REQUEST_ENDPOINTS,
213 REQUEST_VIDEO_QADU_TYPES,
202 REQUESTS_IN_PARALLEL, 214 REQUESTS_IN_PARALLEL,
203 REQUESTS_INTERVAL, 215 REQUESTS_INTERVAL,
204 REQUESTS_LIMIT_PER_POD, 216 REQUESTS_LIMIT_PER_POD,
205 REQUESTS_LIMIT_PODS, 217 REQUESTS_LIMIT_PODS,
218 REQUESTS_VIDEO_QADU_LIMIT_PER_POD,
219 REQUESTS_VIDEO_QADU_LIMIT_PODS,
206 RETRY_REQUESTS, 220 RETRY_REQUESTS,
207 SEARCHABLE_COLUMNS, 221 SEARCHABLE_COLUMNS,
208 SIGNATURE_ALGORITHM, 222 SIGNATURE_ALGORITHM,
diff --git a/server/initializers/migrations/0015-video-views.js b/server/initializers/migrations/0015-video-views.js
new file mode 100644
index 000000000..ae49fe73c
--- /dev/null
+++ b/server/initializers/migrations/0015-video-views.js
@@ -0,0 +1,19 @@
1'use strict'
2
3// utils = { transaction, queryInterface, sequelize, Sequelize }
4exports.up = function (utils, finalCallback) {
5 const q = utils.queryInterface
6 const Sequelize = utils.Sequelize
7
8 const data = {
9 type: Sequelize.INTEGER,
10 allowNull: false,
11 defaultValue: 0
12 }
13
14 q.addColumn('Videos', 'views', data, { transaction: utils.transaction }).asCallback(finalCallback)
15}
16
17exports.down = function (options, callback) {
18 throw new Error('Not implemented.')
19}
diff --git a/server/lib/base-request-scheduler.js b/server/lib/base-request-scheduler.js
new file mode 100644
index 000000000..d15680c25
--- /dev/null
+++ b/server/lib/base-request-scheduler.js
@@ -0,0 +1,140 @@
1'use strict'
2
3const eachLimit = require('async/eachLimit')
4
5const constants = require('../initializers/constants')
6const db = require('../initializers/database')
7const logger = require('../helpers/logger')
8const requests = require('../helpers/requests')
9
10module.exports = class BaseRequestScheduler {
11
12 constructor (options) {
13 this.lastRequestTimestamp = 0
14 this.timer = null
15 }
16
17 activate () {
18 logger.info('Requests scheduler activated.')
19 this.lastRequestTimestamp = Date.now()
20
21 this.timer = setInterval(() => {
22 this.lastRequestTimestamp = Date.now()
23 this.makeRequests()
24 }, constants.REQUESTS_INTERVAL)
25 }
26
27 deactivate () {
28 logger.info('Requests scheduler deactivated.')
29 clearInterval(this.timer)
30 this.timer = null
31 }
32
33 forceSend () {
34 logger.info('Force requests scheduler sending.')
35 this.makeRequests()
36 }
37
38 remainingMilliSeconds () {
39 if (this.timer === null) return -1
40
41 return constants.REQUESTS_INTERVAL - (Date.now() - this.lastRequestTimestamp)
42 }
43
44 // ---------------------------------------------------------------------------
45
46 // Make a requests to friends of a certain type
47 makeRequest (toPod, requestEndpoint, requestsToMake, callback) {
48 if (!callback) callback = function () {}
49
50 const params = {
51 toPod: toPod,
52 sign: true, // Prove our identity
53 method: 'POST',
54 path: '/api/' + constants.API_VERSION + '/remote/' + requestEndpoint,
55 data: requestsToMake // Requests we need to make
56 }
57
58 // Make multiple retry requests to all of pods
59 // The function fire some useful callbacks
60 requests.makeSecureRequest(params, (err, res) => {
61 if (err || (res.statusCode !== 200 && res.statusCode !== 201 && res.statusCode !== 204)) {
62 err = err ? err.message : 'Status code not 20x : ' + res.statusCode
63 logger.error('Error sending secure request to %s pod.', toPod.host, { error: err })
64
65 return callback(false)
66 }
67
68 return callback(true)
69 })
70 }
71
72 // Make all the requests of the scheduler
73 makeRequests () {
74 this.getRequestModel().listWithLimitAndRandom(this.limitPods, this.limitPerPod, (err, requests) => {
75 if (err) {
76 logger.error('Cannot get the list of "%s".', this.description, { err: err })
77 return // Abort
78 }
79
80 // If there are no requests, abort
81 if (requests.length === 0) {
82 logger.info('No "%s" to make.', this.description)
83 return
84 }
85
86 // We want to group requests by destinations pod and endpoint
87 const requestsToMakeGrouped = this.buildRequestObjects(requests)
88
89 logger.info('Making "%s" to friends.', this.description)
90
91 const goodPods = []
92 const badPods = []
93
94 eachLimit(Object.keys(requestsToMakeGrouped), constants.REQUESTS_IN_PARALLEL, (hashKey, callbackEach) => {
95 const requestToMake = requestsToMakeGrouped[hashKey]
96 const toPod = requestToMake.toPod
97
98 // Maybe the pod is not our friend anymore so simply remove it
99 if (!toPod) {
100 const requestIdsToDelete = requestToMake.ids
101
102 logger.info('Removing %d "%s" of unexisting pod %s.', requestIdsToDelete.length, this.description, requestToMake.toPod.id)
103 return this.getRequestToPodModel().removePodOf(requestIdsToDelete, requestToMake.toPod.id, callbackEach)
104 }
105
106 this.makeRequest(toPod, requestToMake.endpoint, requestToMake.datas, (success) => {
107 if (success === false) {
108 badPods.push(requestToMake.toPod.id)
109 return callbackEach()
110 }
111
112 logger.debug('Removing requests for pod %s.', requestToMake.toPod.id, { requestsIds: requestToMake.ids })
113 goodPods.push(requestToMake.toPod.id)
114
115 // Remove the pod id of these request ids
116 this.getRequestToPodModel().removeByRequestIdsAndPod(requestToMake.ids, requestToMake.toPod.id, callbackEach)
117
118 this.afterRequestHook()
119 })
120 }, () => {
121 // All the requests were made, we update the pods score
122 db.Pod.updatePodsScore(goodPods, badPods)
123
124 this.afterRequestsHook()
125 })
126 })
127 }
128
129 flush (callback) {
130 this.getRequestModel().removeAll(callback)
131 }
132
133 afterRequestHook () {
134 // Nothing to do, let children reimplement it
135 }
136
137 afterRequestsHook () {
138 // Nothing to do, let children reimplement it
139 }
140}
diff --git a/server/lib/friends.js b/server/lib/friends.js
index d53ab4553..424a30801 100644
--- a/server/lib/friends.js
+++ b/server/lib/friends.js
@@ -12,15 +12,19 @@ const logger = require('../helpers/logger')
12const peertubeCrypto = require('../helpers/peertube-crypto') 12const peertubeCrypto = require('../helpers/peertube-crypto')
13const requests = require('../helpers/requests') 13const requests = require('../helpers/requests')
14const RequestScheduler = require('./request-scheduler') 14const RequestScheduler = require('./request-scheduler')
15const RequestVideoQaduScheduler = require('./request-video-qadu-scheduler')
15 16
16const ENDPOINT_ACTIONS = constants.REQUEST_ENDPOINT_ACTIONS[constants.REQUEST_ENDPOINTS.VIDEOS] 17const ENDPOINT_ACTIONS = constants.REQUEST_ENDPOINT_ACTIONS[constants.REQUEST_ENDPOINTS.VIDEOS]
18
17const requestScheduler = new RequestScheduler() 19const requestScheduler = new RequestScheduler()
20const requestSchedulerVideoQadu = new RequestVideoQaduScheduler()
18 21
19const friends = { 22const friends = {
20 activate, 23 activate,
21 addVideoToFriends, 24 addVideoToFriends,
22 updateVideoToFriends, 25 updateVideoToFriends,
23 reportAbuseVideoToFriend, 26 reportAbuseVideoToFriend,
27 quickAndDirtyUpdateVideoToFriends,
24 hasFriends, 28 hasFriends,
25 makeFriends, 29 makeFriends,
26 quitFriends, 30 quitFriends,
@@ -30,6 +34,7 @@ const friends = {
30 34
31function activate () { 35function activate () {
32 requestScheduler.activate() 36 requestScheduler.activate()
37 requestSchedulerVideoQadu.activate()
33} 38}
34 39
35function addVideoToFriends (videoData, transaction, callback) { 40function addVideoToFriends (videoData, transaction, callback) {
@@ -71,6 +76,15 @@ function reportAbuseVideoToFriend (reportData, video) {
71 createRequest(options) 76 createRequest(options)
72} 77}
73 78
79function quickAndDirtyUpdateVideoToFriends (videoId, type, transaction, callback) {
80 const options = {
81 videoId,
82 type,
83 transaction
84 }
85 return createVideoQaduRequest(options, callback)
86}
87
74function hasFriends (callback) { 88function hasFriends (callback) {
75 db.Pod.countAll(function (err, count) { 89 db.Pod.countAll(function (err, count) {
76 if (err) return callback(err) 90 if (err) return callback(err)
@@ -110,7 +124,11 @@ function quitFriends (callback) {
110 124
111 waterfall([ 125 waterfall([
112 function flushRequests (callbackAsync) { 126 function flushRequests (callbackAsync) {
113 requestScheduler.flush(callbackAsync) 127 requestScheduler.flush(err => callbackAsync(err))
128 },
129
130 function flushVideoQaduRequests (callbackAsync) {
131 requestSchedulerVideoQadu.flush(err => callbackAsync(err))
114 }, 132 },
115 133
116 function getPodsList (callbackAsync) { 134 function getPodsList (callbackAsync) {
@@ -310,6 +328,12 @@ function createRequest (options, callback) {
310 }) 328 })
311} 329}
312 330
331function createVideoQaduRequest (options, callback) {
332 if (!callback) callback = function () {}
333
334 requestSchedulerVideoQadu.createRequest(options, callback)
335}
336
313function isMe (host) { 337function isMe (host) {
314 return host === constants.CONFIG.WEBSERVER.HOST 338 return host === constants.CONFIG.WEBSERVER.HOST
315} 339}
diff --git a/server/lib/request-scheduler.js b/server/lib/request-scheduler.js
index 28dabe339..6b6535519 100644
--- a/server/lib/request-scheduler.js
+++ b/server/lib/request-scheduler.js
@@ -1,44 +1,54 @@
1'use strict' 1'use strict'
2 2
3const eachLimit = require('async/eachLimit')
4
5const constants = require('../initializers/constants') 3const constants = require('../initializers/constants')
4const BaseRequestScheduler = require('./base-request-scheduler')
6const db = require('../initializers/database') 5const db = require('../initializers/database')
7const logger = require('../helpers/logger') 6const logger = require('../helpers/logger')
8const requests = require('../helpers/requests')
9 7
10module.exports = class RequestScheduler { 8module.exports = class RequestScheduler extends BaseRequestScheduler {
11 9
12 constructor () { 10 constructor () {
13 this.lastRequestTimestamp = 0 11 super()
14 this.timer = null
15 }
16 12
17 activate () { 13 // We limit the size of the requests
18 logger.info('Requests scheduler activated.') 14 this.limitPods = constants.REQUESTS_LIMIT_PODS
19 this.lastRequestTimestamp = Date.now() 15 this.limitPerPod = constants.REQUESTS_LIMIT_PER_POD
20 16
21 this.timer = setInterval(() => { 17 this.description = 'requests'
22 this.lastRequestTimestamp = Date.now()
23 this.makeRequests()
24 }, constants.REQUESTS_INTERVAL)
25 } 18 }
26 19
27 deactivate () { 20 getRequestModel () {
28 logger.info('Requests scheduler deactivated.') 21 return db.Request
29 clearInterval(this.timer)
30 this.timer = null
31 } 22 }
32 23
33 forceSend () { 24 getRequestToPodModel () {
34 logger.info('Force requests scheduler sending.') 25 return db.RequestToPod
35 this.makeRequests()
36 } 26 }
37 27
38 remainingMilliSeconds () { 28 buildRequestObjects (requests) {
39 if (this.timer === null) return -1 29 const requestsToMakeGrouped = {}
30
31 Object.keys(requests).forEach(toPodId => {
32 requests[toPodId].forEach(data => {
33 const request = data.request
34 const pod = data.pod
35 const hashKey = toPodId + request.endpoint
36
37 if (!requestsToMakeGrouped[hashKey]) {
38 requestsToMakeGrouped[hashKey] = {
39 toPod: pod,
40 endpoint: request.endpoint,
41 ids: [], // request ids, to delete them from the DB in the future
42 datas: [] // requests data,
43 }
44 }
45
46 requestsToMakeGrouped[hashKey].ids.push(request.id)
47 requestsToMakeGrouped[hashKey].datas.push(request.request)
48 })
49 })
40 50
41 return constants.REQUESTS_INTERVAL - (Date.now() - this.lastRequestTimestamp) 51 return requestsToMakeGrouped
42 } 52 }
43 53
44 // { type, endpoint, data, toIds, transaction } 54 // { type, endpoint, data, toIds, transaction }
@@ -79,122 +89,10 @@ module.exports = class RequestScheduler {
79 89
80 // --------------------------------------------------------------------------- 90 // ---------------------------------------------------------------------------
81 91
82 // Make all the requests of the scheduler 92 afterRequestsHook () {
83 makeRequests () { 93 // Flush requests with no pod
84 // We limit the size of the requests 94 this.getRequestModel().removeWithEmptyTo(err => {
85 // We don't want to stuck with the same failing requests so we get a random list 95 if (err) logger.error('Error when removing requests with no pods.', { error: err })
86 db.Request.listWithLimitAndRandom(constants.REQUESTS_LIMIT_PODS, constants.REQUESTS_LIMIT_PER_POD, (err, requests) => {
87 if (err) {
88 logger.error('Cannot get the list of requests.', { err: err })
89 return // Abort
90 }
91
92 // If there are no requests, abort
93 if (requests.length === 0) {
94 logger.info('No requests to make.')
95 return
96 }
97
98 // We want to group requests by destinations pod and endpoint
99 const requestsToMakeGrouped = this.buildRequestObjects(requests)
100
101 logger.info('Making requests to friends.')
102
103 const goodPods = []
104 const badPods = []
105
106 eachLimit(Object.keys(requestsToMakeGrouped), constants.REQUESTS_IN_PARALLEL, (hashKey, callbackEach) => {
107 const requestToMake = requestsToMakeGrouped[hashKey]
108 const toPod = requestToMake.toPod
109
110 // Maybe the pod is not our friend anymore so simply remove it
111 if (!toPod) {
112 const requestIdsToDelete = requestToMake.ids
113
114 logger.info('Removing %d requests of unexisting pod %s.', requestIdsToDelete.length, requestToMake.toPod.id)
115 return db.RequestToPod.removePodOf(requestIdsToDelete, requestToMake.toPod.id, callbackEach)
116 }
117
118 this.makeRequest(toPod, requestToMake.endpoint, requestToMake.datas, (success) => {
119 if (success === false) {
120 badPods.push(requestToMake.toPod.id)
121 return callbackEach()
122 }
123
124 logger.debug('Removing requests for pod %s.', requestToMake.toPod.id, { requestsIds: requestToMake.ids })
125 goodPods.push(requestToMake.toPod.id)
126
127 // Remove the pod id of these request ids
128 db.RequestToPod.removePodOf(requestToMake.ids, requestToMake.toPod.id, callbackEach)
129 })
130 }, () => {
131 // All the requests were made, we update the pods score
132 db.Request.updatePodsScore(goodPods, badPods)
133 // Flush requests with no pod
134 db.Request.removeWithEmptyTo(err => {
135 if (err) logger.error('Error when removing requests with no pods.', { error: err })
136 })
137 })
138 })
139 }
140
141 // Make a requests to friends of a certain type
142 makeRequest (toPod, requestEndpoint, requestsToMake, callback) {
143 if (!callback) callback = function () {}
144
145 const params = {
146 toPod: toPod,
147 sign: true, // Prove our identity
148 method: 'POST',
149 path: '/api/' + constants.API_VERSION + '/remote/' + requestEndpoint,
150 data: requestsToMake // Requests we need to make
151 }
152
153 // Make multiple retry requests to all of pods
154 // The function fire some useful callbacks
155 requests.makeSecureRequest(params, (err, res) => {
156 if (err || (res.statusCode !== 200 && res.statusCode !== 201 && res.statusCode !== 204)) {
157 err = err ? err.message : 'Status code not 20x : ' + res.statusCode
158 logger.error('Error sending secure request to %s pod.', toPod.host, { error: err })
159
160 return callback(false)
161 }
162
163 return callback(true)
164 })
165 }
166
167 buildRequestObjects (requests) {
168 const requestsToMakeGrouped = {}
169
170 Object.keys(requests).forEach(toPodId => {
171 requests[toPodId].forEach(data => {
172 const request = data.request
173 const pod = data.pod
174 const hashKey = toPodId + request.endpoint
175
176 if (!requestsToMakeGrouped[hashKey]) {
177 requestsToMakeGrouped[hashKey] = {
178 toPod: pod,
179 endpoint: request.endpoint,
180 ids: [], // request ids, to delete them from the DB in the future
181 datas: [] // requests data,
182 }
183 }
184
185 requestsToMakeGrouped[hashKey].ids.push(request.id)
186 requestsToMakeGrouped[hashKey].datas.push(request.request)
187 })
188 })
189
190 return requestsToMakeGrouped
191 }
192
193 flush (callback) {
194 db.Request.removeAll(err => {
195 if (err) logger.error('Cannot flush the requests.', { error: err })
196
197 return callback(err)
198 }) 96 })
199 } 97 }
200} 98}
diff --git a/server/lib/request-video-qadu-scheduler.js b/server/lib/request-video-qadu-scheduler.js
new file mode 100644
index 000000000..401b2fb44
--- /dev/null
+++ b/server/lib/request-video-qadu-scheduler.js
@@ -0,0 +1,116 @@
1'use strict'
2
3const BaseRequestScheduler = require('./base-request-scheduler')
4const constants = require('../initializers/constants')
5const db = require('../initializers/database')
6const logger = require('../helpers/logger')
7
8module.exports = class RequestVideoQaduScheduler extends BaseRequestScheduler {
9
10 constructor () {
11 super()
12
13 // We limit the size of the requests
14 this.limitPods = constants.REQUESTS_VIDEO_QADU_LIMIT_PODS
15 this.limitPerPod = constants.REQUESTS_VIDEO_QADU_LIMIT_PODS
16
17 this.description = 'video QADU requests'
18 }
19
20 getRequestModel () {
21 return db.RequestVideoQadu
22 }
23
24 getRequestToPodModel () {
25 return db.RequestVideoQadu
26 }
27
28 buildRequestObjects (requests) {
29 const requestsToMakeGrouped = {}
30
31 Object.keys(requests).forEach(toPodId => {
32 requests[toPodId].forEach(data => {
33 const request = data.request
34 const video = data.video
35 const pod = data.pod
36 const hashKey = toPodId
37
38 if (!requestsToMakeGrouped[hashKey]) {
39 requestsToMakeGrouped[hashKey] = {
40 toPod: pod,
41 endpoint: constants.REQUEST_ENDPOINTS.QADU,
42 ids: [], // request ids, to delete them from the DB in the future
43 datas: [], // requests data
44 videos: {}
45 }
46 }
47
48 if (!requestsToMakeGrouped[hashKey].videos[video.id]) {
49 requestsToMakeGrouped[hashKey].videos[video.id] = {}
50 }
51
52 const videoData = requestsToMakeGrouped[hashKey].videos[video.id]
53
54 switch (request.type) {
55 case constants.REQUEST_VIDEO_QADU_TYPES.LIKES:
56 videoData.likes = video.likes
57 break
58
59 case constants.REQUEST_VIDEO_QADU_TYPES.DISLIKES:
60 videoData.likes = video.dislikes
61 break
62
63 case constants.REQUEST_VIDEO_QADU_TYPES.VIEWS:
64 videoData.views = video.views
65 break
66
67 default:
68 logger.error('Unknown request video QADU type %s.', request.type)
69 return
70 }
71
72 // Do not forget the remoteId so the remote pod can identify the video
73 videoData.remoteId = video.id
74 requestsToMakeGrouped[hashKey].ids.push(request.id)
75 requestsToMakeGrouped[hashKey].videos[video.id] = videoData
76 })
77 })
78
79 Object.keys(requestsToMakeGrouped).forEach(hashKey => {
80 Object.keys(requestsToMakeGrouped[hashKey].videos).forEach(videoId => {
81 const videoData = requestsToMakeGrouped[hashKey].videos[videoId]
82
83 requestsToMakeGrouped[hashKey].datas.push({
84 data: videoData
85 })
86 })
87
88 // We don't need it anymore, it was just to build our datas array
89 delete requestsToMakeGrouped[hashKey].videos
90 })
91
92 return requestsToMakeGrouped
93 }
94
95 // { type, videoId, transaction? }
96 createRequest (options, callback) {
97 const type = options.type
98 const videoId = options.videoId
99 const transaction = options.transaction
100
101 const dbRequestOptions = {}
102 if (transaction) dbRequestOptions.transaction = transaction
103
104 // Send the update to all our friends
105 db.Pod.listAllIds(options.transaction, function (err, podIds) {
106 if (err) return callback(err)
107
108 const queries = []
109 podIds.forEach(podId => {
110 queries.push({ type, videoId, podId })
111 })
112
113 return db.RequestVideoQadu.bulkCreate(queries, dbRequestOptions).asCallback(callback)
114 })
115 }
116}
diff --git a/server/middlewares/validators/remote/videos.js b/server/middlewares/validators/remote/videos.js
index cf9925b6c..ddc274c45 100644
--- a/server/middlewares/validators/remote/videos.js
+++ b/server/middlewares/validators/remote/videos.js
@@ -4,7 +4,8 @@ const checkErrors = require('../utils').checkErrors
4const logger = require('../../../helpers/logger') 4const logger = require('../../../helpers/logger')
5 5
6const validatorsRemoteVideos = { 6const validatorsRemoteVideos = {
7 remoteVideos 7 remoteVideos,
8 remoteQaduVideos
8} 9}
9 10
10function remoteVideos (req, res, next) { 11function remoteVideos (req, res, next) {
@@ -15,6 +16,14 @@ function remoteVideos (req, res, next) {
15 checkErrors(req, res, next) 16 checkErrors(req, res, next)
16} 17}
17 18
19function remoteQaduVideos (req, res, next) {
20 req.checkBody('data').isEachRemoteRequestVideosQaduValid()
21
22 logger.debug('Checking remoteVideosQadu parameters', { parameters: req.body })
23
24 checkErrors(req, res, next)
25}
26
18// --------------------------------------------------------------------------- 27// ---------------------------------------------------------------------------
19 28
20module.exports = validatorsRemoteVideos 29module.exports = validatorsRemoteVideos
diff --git a/server/models/pod.js b/server/models/pod.js
index 79afb737a..14814708e 100644
--- a/server/models/pod.js
+++ b/server/models/pod.js
@@ -1,8 +1,11 @@
1'use strict' 1'use strict'
2 2
3const each = require('async/each')
3const map = require('lodash/map') 4const map = require('lodash/map')
5const waterfall = require('async/waterfall')
4 6
5const constants = require('../initializers/constants') 7const constants = require('../initializers/constants')
8const logger = require('../helpers/logger')
6const customPodsValidators = require('../helpers/custom-validators').pods 9const customPodsValidators = require('../helpers/custom-validators').pods
7 10
8// --------------------------------------------------------------------------- 11// ---------------------------------------------------------------------------
@@ -62,6 +65,7 @@ module.exports = function (sequelize, DataTypes) {
62 listBadPods, 65 listBadPods,
63 load, 66 load,
64 loadByHost, 67 loadByHost,
68 updatePodsScore,
65 removeAll 69 removeAll
66 }, 70 },
67 instanceMethods: { 71 instanceMethods: {
@@ -144,7 +148,7 @@ function listAllIds (transaction, callback) {
144 }) 148 })
145} 149}
146 150
147function listRandomPodIdsWithRequest (limit, callback) { 151function listRandomPodIdsWithRequest (limit, tableRequestPod, callback) {
148 const self = this 152 const self = this
149 153
150 self.count().asCallback(function (err, count) { 154 self.count().asCallback(function (err, count) {
@@ -166,7 +170,7 @@ function listRandomPodIdsWithRequest (limit, callback) {
166 where: { 170 where: {
167 id: { 171 id: {
168 $in: [ 172 $in: [
169 this.sequelize.literal('SELECT "podId" FROM "RequestToPods"') 173 this.sequelize.literal('SELECT "podId" FROM "' + tableRequestPod + '"')
170 ] 174 ]
171 } 175 }
172 } 176 }
@@ -207,3 +211,58 @@ function loadByHost (host, callback) {
207function removeAll (callback) { 211function removeAll (callback) {
208 return this.destroy().asCallback(callback) 212 return this.destroy().asCallback(callback)
209} 213}
214
215function updatePodsScore (goodPods, badPods) {
216 const self = this
217
218 logger.info('Updating %d good pods and %d bad pods scores.', goodPods.length, badPods.length)
219
220 if (goodPods.length !== 0) {
221 this.incrementScores(goodPods, constants.PODS_SCORE.BONUS, function (err) {
222 if (err) logger.error('Cannot increment scores of good pods.', { error: err })
223 })
224 }
225
226 if (badPods.length !== 0) {
227 this.incrementScores(badPods, constants.PODS_SCORE.MALUS, function (err) {
228 if (err) logger.error('Cannot decrement scores of bad pods.', { error: err })
229 removeBadPods.call(self)
230 })
231 }
232}
233
234// ---------------------------------------------------------------------------
235
236// Remove pods with a score of 0 (too many requests where they were unreachable)
237function removeBadPods () {
238 const self = this
239
240 waterfall([
241 function findBadPods (callback) {
242 self.sequelize.models.Pod.listBadPods(function (err, pods) {
243 if (err) {
244 logger.error('Cannot find bad pods.', { error: err })
245 return callback(err)
246 }
247
248 return callback(null, pods)
249 })
250 },
251
252 function removeTheseBadPods (pods, callback) {
253 each(pods, function (pod, callbackEach) {
254 pod.destroy().asCallback(callbackEach)
255 }, function (err) {
256 return callback(err, pods.length)
257 })
258 }
259 ], function (err, numberOfPodsRemoved) {
260 if (err) {
261 logger.error('Cannot remove bad pods.', { error: err })
262 } else if (numberOfPodsRemoved) {
263 logger.info('Removed %d pods.', numberOfPodsRemoved)
264 } else {
265 logger.info('No need to remove bad pods.')
266 }
267 })
268}
diff --git a/server/models/request-to-pod.js b/server/models/request-to-pod.js
index f42a53458..0e01a842e 100644
--- a/server/models/request-to-pod.js
+++ b/server/models/request-to-pod.js
@@ -17,7 +17,7 @@ module.exports = function (sequelize, DataTypes) {
17 } 17 }
18 ], 18 ],
19 classMethods: { 19 classMethods: {
20 removePodOf 20 removeByRequestIdsAndPod
21 } 21 }
22 }) 22 })
23 23
@@ -26,7 +26,7 @@ module.exports = function (sequelize, DataTypes) {
26 26
27// --------------------------------------------------------------------------- 27// ---------------------------------------------------------------------------
28 28
29function removePodOf (requestsIds, podId, callback) { 29function removeByRequestIdsAndPod (requestsIds, podId, callback) {
30 if (!callback) callback = function () {} 30 if (!callback) callback = function () {}
31 31
32 const query = { 32 const query = {
diff --git a/server/models/request-video-qadu.js b/server/models/request-video-qadu.js
new file mode 100644
index 000000000..7010fc992
--- /dev/null
+++ b/server/models/request-video-qadu.js
@@ -0,0 +1,154 @@
1'use strict'
2
3/*
4 Request Video for Quick And Dirty Updates like:
5 - views
6 - likes
7 - dislikes
8
9 We can't put it in the same system than basic requests for efficiency.
10 Moreover we don't want to slow down the basic requests with a lot of views/likes/dislikes requests.
11 So we put it an independant request scheduler.
12*/
13
14const values = require('lodash/values')
15
16const constants = require('../initializers/constants')
17
18// ---------------------------------------------------------------------------
19
20module.exports = function (sequelize, DataTypes) {
21 const RequestVideoQadu = sequelize.define('RequestVideoQadu',
22 {
23 type: {
24 type: DataTypes.ENUM(values(constants.REQUEST_VIDEO_QADU_TYPES)),
25 allowNull: false
26 }
27 },
28 {
29 timestamps: false,
30 indexes: [
31 {
32 fields: [ 'podId' ]
33 },
34 {
35 fields: [ 'videoId' ]
36 }
37 ],
38 classMethods: {
39 associate,
40
41 listWithLimitAndRandom,
42
43 countTotalRequests,
44 removeAll,
45 removeByRequestIdsAndPod
46 }
47 }
48 )
49
50 return RequestVideoQadu
51}
52
53// ------------------------------ STATICS ------------------------------
54
55function associate (models) {
56 this.belongsTo(models.Pod, {
57 foreignKey: {
58 name: 'podId',
59 allowNull: false
60 },
61 onDelete: 'CASCADE'
62 })
63
64 this.belongsTo(models.Video, {
65 foreignKey: {
66 name: 'videoId',
67 allowNull: false
68 },
69 onDelete: 'CASCADE'
70 })
71}
72
73function countTotalRequests (callback) {
74 const query = {
75 include: [ this.sequelize.models.Pod ]
76 }
77
78 return this.count(query).asCallback(callback)
79}
80
81function listWithLimitAndRandom (limitPods, limitRequestsPerPod, callback) {
82 const self = this
83 const Pod = this.sequelize.models.Pod
84
85 Pod.listRandomPodIdsWithRequest(limitPods, 'RequestVideoQadus', function (err, podIds) {
86 if (err) return callback(err)
87
88 // We don't have friends that have requests
89 if (podIds.length === 0) return callback(null, [])
90
91 const query = {
92 include: [
93 {
94 model: self.sequelize.models.Pod,
95 where: {
96 id: {
97 $in: podIds
98 }
99 }
100 },
101 {
102 model: self.sequelize.models.Video
103 }
104 ]
105 }
106
107 self.findAll(query).asCallback(function (err, requests) {
108 if (err) return callback(err)
109
110 const requestsGrouped = groupAndTruncateRequests(requests, limitRequestsPerPod)
111 return callback(err, requestsGrouped)
112 })
113 })
114}
115
116function removeByRequestIdsAndPod (ids, podId, callback) {
117 const query = {
118 where: {
119 id: {
120 $in: ids
121 },
122 podId
123 }
124 }
125
126 this.destroy(query).asCallback(callback)
127}
128
129function removeAll (callback) {
130 // Delete all requests
131 this.truncate({ cascade: true }).asCallback(callback)
132}
133
134// ---------------------------------------------------------------------------
135
136function groupAndTruncateRequests (requests, limitRequestsPerPod) {
137 const requestsGrouped = {}
138
139 requests.forEach(function (request) {
140 const pod = request.Pod
141
142 if (!requestsGrouped[pod.id]) requestsGrouped[pod.id] = []
143
144 if (requestsGrouped[pod.id].length < limitRequestsPerPod) {
145 requestsGrouped[pod.id].push({
146 request: request,
147 video: request.Video,
148 pod
149 })
150 }
151 })
152
153 return requestsGrouped
154}
diff --git a/server/models/request.js b/server/models/request.js
index ca616d130..de73501fc 100644
--- a/server/models/request.js
+++ b/server/models/request.js
@@ -1,11 +1,8 @@
1'use strict' 1'use strict'
2 2
3const each = require('async/each')
4const waterfall = require('async/waterfall')
5const values = require('lodash/values') 3const values = require('lodash/values')
6 4
7const constants = require('../initializers/constants') 5const constants = require('../initializers/constants')
8const logger = require('../helpers/logger')
9 6
10// --------------------------------------------------------------------------- 7// ---------------------------------------------------------------------------
11 8
@@ -28,8 +25,6 @@ module.exports = function (sequelize, DataTypes) {
28 listWithLimitAndRandom, 25 listWithLimitAndRandom,
29 26
30 countTotalRequests, 27 countTotalRequests,
31 removeBadPods,
32 updatePodsScore,
33 removeAll, 28 removeAll,
34 removeWithEmptyTo 29 removeWithEmptyTo
35 } 30 }
@@ -60,71 +55,17 @@ function countTotalRequests (callback) {
60 return this.count(query).asCallback(callback) 55 return this.count(query).asCallback(callback)
61} 56}
62 57
63// Remove pods with a score of 0 (too many requests where they were unreachable)
64function removeBadPods () {
65 const self = this
66
67 waterfall([
68 function findBadPods (callback) {
69 self.sequelize.models.Pod.listBadPods(function (err, pods) {
70 if (err) {
71 logger.error('Cannot find bad pods.', { error: err })
72 return callback(err)
73 }
74
75 return callback(null, pods)
76 })
77 },
78
79 function removeTheseBadPods (pods, callback) {
80 each(pods, function (pod, callbackEach) {
81 pod.destroy().asCallback(callbackEach)
82 }, function (err) {
83 return callback(err, pods.length)
84 })
85 }
86 ], function (err, numberOfPodsRemoved) {
87 if (err) {
88 logger.error('Cannot remove bad pods.', { error: err })
89 } else if (numberOfPodsRemoved) {
90 logger.info('Removed %d pods.', numberOfPodsRemoved)
91 } else {
92 logger.info('No need to remove bad pods.')
93 }
94 })
95}
96
97function updatePodsScore (goodPods, badPods) {
98 const self = this
99 const Pod = this.sequelize.models.Pod
100
101 logger.info('Updating %d good pods and %d bad pods scores.', goodPods.length, badPods.length)
102
103 if (goodPods.length !== 0) {
104 Pod.incrementScores(goodPods, constants.PODS_SCORE.BONUS, function (err) {
105 if (err) logger.error('Cannot increment scores of good pods.', { error: err })
106 })
107 }
108
109 if (badPods.length !== 0) {
110 Pod.incrementScores(badPods, constants.PODS_SCORE.MALUS, function (err) {
111 if (err) logger.error('Cannot decrement scores of bad pods.', { error: err })
112 removeBadPods.call(self)
113 })
114 }
115}
116
117function listWithLimitAndRandom (limitPods, limitRequestsPerPod, callback) { 58function listWithLimitAndRandom (limitPods, limitRequestsPerPod, callback) {
118 const self = this 59 const self = this
119 const Pod = this.sequelize.models.Pod 60 const Pod = this.sequelize.models.Pod
120 61
121 Pod.listRandomPodIdsWithRequest(limitPods, function (err, podIds) { 62 Pod.listRandomPodIdsWithRequest(limitPods, 'RequestToPods', function (err, podIds) {
122 if (err) return callback(err) 63 if (err) return callback(err)
123 64
124 // We don't have friends that have requests 65 // We don't have friends that have requests
125 if (podIds.length === 0) return callback(null, []) 66 if (podIds.length === 0) return callback(null, [])
126 67
127 // The the first x requests of these pods 68 // The first x requests of these pods
128 // It is very important to sort by id ASC to keep the requests order! 69 // It is very important to sort by id ASC to keep the requests order!
129 const query = { 70 const query = {
130 order: [ 71 order: [
diff --git a/server/models/video.js b/server/models/video.js
index d0fd61eb4..daa273845 100644
--- a/server/models/video.js
+++ b/server/models/video.js
@@ -80,6 +80,15 @@ module.exports = function (sequelize, DataTypes) {
80 if (res === false) throw new Error('Video duration is not valid.') 80 if (res === false) throw new Error('Video duration is not valid.')
81 } 81 }
82 } 82 }
83 },
84 views: {
85 type: DataTypes.INTEGER,
86 allowNull: false,
87 defaultValue: 0,
88 validate: {
89 min: 0,
90 isInt: true
91 }
83 } 92 }
84 }, 93 },
85 { 94 {
@@ -101,6 +110,9 @@ module.exports = function (sequelize, DataTypes) {
101 }, 110 },
102 { 111 {
103 fields: [ 'infoHash' ] 112 fields: [ 'infoHash' ]
113 },
114 {
115 fields: [ 'views' ]
104 } 116 }
105 ], 117 ],
106 classMethods: { 118 classMethods: {
@@ -336,6 +348,7 @@ function toFormatedJSON () {
336 magnetUri: this.generateMagnetUri(), 348 magnetUri: this.generateMagnetUri(),
337 author: this.Author.name, 349 author: this.Author.name,
338 duration: this.duration, 350 duration: this.duration,
351 views: this.views,
339 tags: map(this.Tags, 'name'), 352 tags: map(this.Tags, 'name'),
340 thumbnailPath: pathUtils.join(constants.STATIC_PATHS.THUMBNAILS, this.getThumbnailName()), 353 thumbnailPath: pathUtils.join(constants.STATIC_PATHS.THUMBNAILS, this.getThumbnailName()),
341 createdAt: this.createdAt, 354 createdAt: this.createdAt,
diff --git a/server/tests/api/multiple-pods.js b/server/tests/api/multiple-pods.js
index df12ba0e9..871db54be 100644
--- a/server/tests/api/multiple-pods.js
+++ b/server/tests/api/multiple-pods.js
@@ -3,6 +3,7 @@
3const chai = require('chai') 3const chai = require('chai')
4const each = require('async/each') 4const each = require('async/each')
5const expect = chai.expect 5const expect = chai.expect
6const parallel = require('async/parallel')
6const series = require('async/series') 7const series = require('async/series')
7const WebTorrent = require('webtorrent') 8const WebTorrent = require('webtorrent')
8const webtorrent = new WebTorrent() 9const webtorrent = new WebTorrent()
@@ -375,6 +376,63 @@ describe('Test multiple pods', function () {
375 }) 376 })
376 }) 377 })
377 378
379 describe('Should update video views', function () {
380 let videoId1
381 let videoId2
382
383 before(function (done) {
384 videosUtils.getVideosList(servers[2].url, function (err, res) {
385 if (err) throw err
386
387 const videos = res.body.data.filter(video => video.isLocal === true)
388 videoId1 = videos[0].id
389 videoId2 = videos[1].id
390
391 done()
392 })
393 })
394
395 it('Should views multiple videos on owned servers', function (done) {
396 this.timeout(30000)
397
398 parallel([
399 function (callback) {
400 videosUtils.getVideo(servers[2].url, videoId1, callback)
401 },
402
403 function (callback) {
404 videosUtils.getVideo(servers[2].url, videoId1, callback)
405 },
406
407 function (callback) {
408 videosUtils.getVideo(servers[2].url, videoId1, callback)
409 },
410
411 function (callback) {
412 videosUtils.getVideo(servers[2].url, videoId2, callback)
413 }
414 ], function (err) {
415 if (err) throw err
416
417 setTimeout(done, 22000)
418 })
419 })
420
421 it('Should have views updated on each pod', function (done) {
422 each(servers, function (server, callback) {
423 videosUtils.getVideosList(server.url, function (err, res) {
424 if (err) throw err
425
426 const videos = res.body.data
427 expect(videos.find(video => video.views === 3)).to.be.exist
428 expect(videos.find(video => video.views === 1)).to.be.exist
429
430 callback()
431 })
432 }, done)
433 })
434 })
435/*
378 describe('Should manipulate these videos', function () { 436 describe('Should manipulate these videos', function () {
379 it('Should update the video 3 by asking pod 3', function (done) { 437 it('Should update the video 3 by asking pod 3', function (done) {
380 this.timeout(15000) 438 this.timeout(15000)
@@ -462,7 +520,7 @@ describe('Test multiple pods', function () {
462 }, done) 520 }, done)
463 }) 521 })
464 }) 522 })
465 523*/
466 after(function (done) { 524 after(function (done) {
467 servers.forEach(function (server) { 525 servers.forEach(function (server) {
468 process.kill(-server.app.pid) 526 process.kill(-server.app.pid)
diff --git a/server/tests/api/single-pod.js b/server/tests/api/single-pod.js
index 83a2b4411..40c33686f 100644
--- a/server/tests/api/single-pod.js
+++ b/server/tests/api/single-pod.js
@@ -129,6 +129,17 @@ describe('Test a single pod', function () {
129 }) 129 })
130 }) 130 })
131 131
132 it('Should have the views updated', function (done) {
133 videosUtils.getVideo(server.url, videoId, function (err, res) {
134 if (err) throw err
135
136 const video = res.body
137 expect(video.views).to.equal(1)
138
139 done()
140 })
141 })
142
132 it('Should search the video by name by default', function (done) { 143 it('Should search the video by name by default', function (done) {
133 videosUtils.searchVideo(server.url, 'my', function (err, res) { 144 videosUtils.searchVideo(server.url, 'my', function (err, res) {
134 if (err) throw err 145 if (err) throw err