aboutsummaryrefslogtreecommitdiffhomepage
path: root/server
diff options
context:
space:
mode:
Diffstat (limited to 'server')
-rw-r--r--server/controllers/api/remote/videos.js108
-rw-r--r--server/controllers/api/videos.js3
-rw-r--r--server/helpers/custom-validators/remote/videos.js24
-rw-r--r--server/helpers/custom-validators/videos.js14
-rw-r--r--server/helpers/utils.js7
-rw-r--r--server/initializers/constants.js24
-rw-r--r--server/lib/friends.js22
-rw-r--r--server/lib/request-video-event-scheduler.js109
-rw-r--r--server/lib/request-video-qadu-scheduler.js2
-rw-r--r--server/middlewares/validators/remote/videos.js12
-rw-r--r--server/models/pod.js9
-rw-r--r--server/models/request-video-event.js169
-rw-r--r--server/models/request-video-qadu.js5
-rw-r--r--server/models/request.js2
-rw-r--r--server/tests/api/multiple-pods.js150
15 files changed, 612 insertions, 48 deletions
diff --git a/server/controllers/api/remote/videos.js b/server/controllers/api/remote/videos.js
index 79b503d4d..39c9579c1 100644
--- a/server/controllers/api/remote/videos.js
+++ b/server/controllers/api/remote/videos.js
@@ -38,6 +38,13 @@ router.post('/qadu',
38 remoteVideosQadu 38 remoteVideosQadu
39) 39)
40 40
41router.post('/events',
42 signatureValidators.signature,
43 secureMiddleware.checkSignature,
44 videosValidators.remoteEventsVideos,
45 remoteVideosEvents
46)
47
41// --------------------------------------------------------------------------- 48// ---------------------------------------------------------------------------
42 49
43module.exports = router 50module.exports = router
@@ -84,6 +91,84 @@ function remoteVideosQadu (req, res, next) {
84 return res.type('json').status(204).end() 91 return res.type('json').status(204).end()
85} 92}
86 93
94function remoteVideosEvents (req, res, next) {
95 const requests = req.body.data
96 const fromPod = res.locals.secure.pod
97
98 eachSeries(requests, function (request, callbackEach) {
99 const eventData = request.data
100
101 processVideosEventsRetryWrapper(eventData, fromPod, callbackEach)
102 }, function (err) {
103 if (err) logger.error('Error managing remote videos.', { error: err })
104 })
105
106 return res.type('json').status(204).end()
107}
108
109function processVideosEventsRetryWrapper (eventData, fromPod, finalCallback) {
110 const options = {
111 arguments: [ eventData, fromPod ],
112 errorMessage: 'Cannot process videos events with many retries.'
113 }
114
115 databaseUtils.retryTransactionWrapper(processVideosEvents, options, finalCallback)
116}
117
118function processVideosEvents (eventData, fromPod, finalCallback) {
119 waterfall([
120 databaseUtils.startSerializableTransaction,
121
122 function findVideo (t, callback) {
123 fetchOwnedVideo(eventData.remoteId, function (err, videoInstance) {
124 return callback(err, t, videoInstance)
125 })
126 },
127
128 function updateVideoIntoDB (t, videoInstance, callback) {
129 const options = { transaction: t }
130
131 let columnToUpdate
132
133 switch (eventData.eventType) {
134 case constants.REQUEST_VIDEO_EVENT_TYPES.VIEWS:
135 columnToUpdate = 'views'
136 break
137
138 case constants.REQUEST_VIDEO_EVENT_TYPES.LIKES:
139 columnToUpdate = 'likes'
140 break
141
142 case constants.REQUEST_VIDEO_EVENT_TYPES.DISLIKES:
143 columnToUpdate = 'dislikes'
144 break
145
146 default:
147 return callback(new Error('Unknown video event type.'))
148 }
149
150 const query = {}
151 query[columnToUpdate] = eventData.count
152
153 videoInstance.increment(query, options).asCallback(function (err) {
154 return callback(err, t)
155 })
156 },
157
158 databaseUtils.commitTransaction
159
160 ], function (err, t) {
161 if (err) {
162 console.log(err)
163 logger.debug('Cannot process a video event.', { error: err })
164 return databaseUtils.rollbackTransaction(err, t, finalCallback)
165 }
166
167 logger.info('Remote video event processed for video %s.', eventData.remoteId)
168 return finalCallback(null)
169 })
170}
171
87function quickAndDirtyUpdateVideoRetryWrapper (videoData, fromPod, finalCallback) { 172function quickAndDirtyUpdateVideoRetryWrapper (videoData, fromPod, finalCallback) {
88 const options = { 173 const options = {
89 arguments: [ videoData, fromPod ], 174 arguments: [ videoData, fromPod ],
@@ -98,7 +183,7 @@ function quickAndDirtyUpdateVideo (videoData, fromPod, finalCallback) {
98 databaseUtils.startSerializableTransaction, 183 databaseUtils.startSerializableTransaction,
99 184
100 function findVideo (t, callback) { 185 function findVideo (t, callback) {
101 fetchVideo(fromPod.host, videoData.remoteId, function (err, videoInstance) { 186 fetchRemoteVideo(fromPod.host, videoData.remoteId, function (err, videoInstance) {
102 return callback(err, t, videoInstance) 187 return callback(err, t, videoInstance)
103 }) 188 })
104 }, 189 },
@@ -264,7 +349,7 @@ function updateRemoteVideo (videoAttributesToUpdate, fromPod, finalCallback) {
264 databaseUtils.startSerializableTransaction, 349 databaseUtils.startSerializableTransaction,
265 350
266 function findVideo (t, callback) { 351 function findVideo (t, callback) {
267 fetchVideo(fromPod.host, videoAttributesToUpdate.remoteId, function (err, videoInstance) { 352 fetchRemoteVideo(fromPod.host, videoAttributesToUpdate.remoteId, function (err, videoInstance) {
268 return callback(err, t, videoInstance) 353 return callback(err, t, videoInstance)
269 }) 354 })
270 }, 355 },
@@ -317,7 +402,7 @@ function updateRemoteVideo (videoAttributesToUpdate, fromPod, finalCallback) {
317 402
318function removeRemoteVideo (videoToRemoveData, fromPod, callback) { 403function removeRemoteVideo (videoToRemoveData, fromPod, callback) {
319 // We need the instance because we have to remove some other stuffs (thumbnail etc) 404 // We need the instance because we have to remove some other stuffs (thumbnail etc)
320 fetchVideo(fromPod.host, videoToRemoveData.remoteId, function (err, video) { 405 fetchRemoteVideo(fromPod.host, videoToRemoveData.remoteId, function (err, video) {
321 // Do not return the error, continue the process 406 // Do not return the error, continue the process
322 if (err) return callback(null) 407 if (err) return callback(null)
323 408
@@ -334,7 +419,7 @@ function removeRemoteVideo (videoToRemoveData, fromPod, callback) {
334} 419}
335 420
336function reportAbuseRemoteVideo (reportData, fromPod, callback) { 421function reportAbuseRemoteVideo (reportData, fromPod, callback) {
337 db.Video.load(reportData.videoRemoteId, function (err, video) { 422 fetchOwnedVideo(reportData.videoRemoteId, function (err, video) {
338 if (err || !video) { 423 if (err || !video) {
339 if (!err) err = new Error('video not found') 424 if (!err) err = new Error('video not found')
340 425
@@ -362,7 +447,20 @@ function reportAbuseRemoteVideo (reportData, fromPod, callback) {
362 }) 447 })
363} 448}
364 449
365function fetchVideo (podHost, remoteId, callback) { 450function fetchOwnedVideo (id, callback) {
451 db.Video.load(id, function (err, video) {
452 if (err || !video) {
453 if (!err) err = new Error('video not found')
454
455 logger.error('Cannot load owned video from id.', { error: err, id })
456 return callback(err)
457 }
458
459 return callback(null, video)
460 })
461}
462
463function fetchRemoteVideo (podHost, remoteId, callback) {
366 db.Video.loadByHostAndRemoteId(podHost, remoteId, function (err, video) { 464 db.Video.loadByHostAndRemoteId(podHost, remoteId, function (err, video) {
367 if (err || !video) { 465 if (err || !video) {
368 if (!err) err = new Error('video not found') 466 if (!err) err = new Error('video not found')
diff --git a/server/controllers/api/videos.js b/server/controllers/api/videos.js
index 9f4bbb7b7..d64ed4e4e 100644
--- a/server/controllers/api/videos.js
+++ b/server/controllers/api/videos.js
@@ -333,6 +333,9 @@ function getVideo (req, res, next) {
333 // For example, only add a view when a user watch a video during 30s etc 333 // For example, only add a view when a user watch a video during 30s etc
334 friends.quickAndDirtyUpdateVideoToFriends(videoInstance.id, constants.REQUEST_VIDEO_QADU_TYPES.VIEWS) 334 friends.quickAndDirtyUpdateVideoToFriends(videoInstance.id, constants.REQUEST_VIDEO_QADU_TYPES.VIEWS)
335 }) 335 })
336 } else {
337 // Just send the event to our friends
338 friends.addEventToRemoteVideo(videoInstance.id, constants.REQUEST_VIDEO_EVENT_TYPES.VIEWS)
336 } 339 }
337 340
338 // Do not wait the view system 341 // Do not wait the view system
diff --git a/server/helpers/custom-validators/remote/videos.js b/server/helpers/custom-validators/remote/videos.js
index 2e9cf822e..c1786014d 100644
--- a/server/helpers/custom-validators/remote/videos.js
+++ b/server/helpers/custom-validators/remote/videos.js
@@ -1,6 +1,7 @@
1'use strict' 1'use strict'
2 2
3const has = require('lodash/has') 3const has = require('lodash/has')
4const values = require('lodash/values')
4 5
5const constants = require('../../../initializers/constants') 6const constants = require('../../../initializers/constants')
6const videosValidators = require('../videos') 7const videosValidators = require('../videos')
@@ -10,13 +11,17 @@ const ENDPOINT_ACTIONS = constants.REQUEST_ENDPOINT_ACTIONS[constants.REQUEST_EN
10 11
11const remoteVideosValidators = { 12const remoteVideosValidators = {
12 isEachRemoteRequestVideosValid, 13 isEachRemoteRequestVideosValid,
13 isEachRemoteRequestVideosQaduValid 14 isEachRemoteRequestVideosQaduValid,
15 isEachRemoteRequestVideosEventsValid
14} 16}
15 17
16function isEachRemoteRequestVideosValid (requests) { 18function isEachRemoteRequestVideosValid (requests) {
17 return miscValidators.isArray(requests) && 19 return miscValidators.isArray(requests) &&
18 requests.every(function (request) { 20 requests.every(function (request) {
19 const video = request.data 21 const video = request.data
22
23 if (!video) return false
24
20 return ( 25 return (
21 isRequestTypeAddValid(request.type) && 26 isRequestTypeAddValid(request.type) &&
22 isCommonVideoAttributesValid(video) && 27 isCommonVideoAttributesValid(video) &&
@@ -45,6 +50,8 @@ function isEachRemoteRequestVideosQaduValid (requests) {
45 requests.every(function (request) { 50 requests.every(function (request) {
46 const video = request.data 51 const video = request.data
47 52
53 if (!video) return false
54
48 return ( 55 return (
49 videosValidators.isVideoRemoteIdValid(video.remoteId) && 56 videosValidators.isVideoRemoteIdValid(video.remoteId) &&
50 (has(video, 'views') === false || videosValidators.isVideoViewsValid) && 57 (has(video, 'views') === false || videosValidators.isVideoViewsValid) &&
@@ -54,6 +61,21 @@ function isEachRemoteRequestVideosQaduValid (requests) {
54 }) 61 })
55} 62}
56 63
64function isEachRemoteRequestVideosEventsValid (requests) {
65 return miscValidators.isArray(requests) &&
66 requests.every(function (request) {
67 const eventData = request.data
68
69 if (!eventData) return false
70
71 return (
72 videosValidators.isVideoRemoteIdValid(eventData.remoteId) &&
73 values(constants.REQUEST_VIDEO_EVENT_TYPES).indexOf(eventData.eventType) !== -1 &&
74 videosValidators.isVideoEventCountValid(eventData.count)
75 )
76 })
77}
78
57// --------------------------------------------------------------------------- 79// ---------------------------------------------------------------------------
58 80
59module.exports = remoteVideosValidators 81module.exports = remoteVideosValidators
diff --git a/server/helpers/custom-validators/videos.js b/server/helpers/custom-validators/videos.js
index 1d844118b..c5a1f3cb5 100644
--- a/server/helpers/custom-validators/videos.js
+++ b/server/helpers/custom-validators/videos.js
@@ -7,6 +7,7 @@ const usersValidators = require('./users')
7const miscValidators = require('./misc') 7const miscValidators = require('./misc')
8const VIDEOS_CONSTRAINTS_FIELDS = constants.CONSTRAINTS_FIELDS.VIDEOS 8const VIDEOS_CONSTRAINTS_FIELDS = constants.CONSTRAINTS_FIELDS.VIDEOS
9const VIDEO_ABUSES_CONSTRAINTS_FIELDS = constants.CONSTRAINTS_FIELDS.VIDEO_ABUSES 9const VIDEO_ABUSES_CONSTRAINTS_FIELDS = constants.CONSTRAINTS_FIELDS.VIDEO_ABUSES
10const VIDEO_EVENTS_CONSTRAINTS_FIELDS = constants.CONSTRAINTS_FIELDS.VIDEO_EVENTS
10 11
11const videosValidators = { 12const videosValidators = {
12 isVideoAuthorValid, 13 isVideoAuthorValid,
@@ -25,7 +26,8 @@ const videosValidators = {
25 isVideoFile, 26 isVideoFile,
26 isVideoViewsValid, 27 isVideoViewsValid,
27 isVideoLikesValid, 28 isVideoLikesValid,
28 isVideoDislikesValid 29 isVideoDislikesValid,
30 isVideoEventCountValid
29} 31}
30 32
31function isVideoAuthorValid (value) { 33function isVideoAuthorValid (value) {
@@ -86,15 +88,19 @@ function isVideoAbuseReporterUsernameValid (value) {
86} 88}
87 89
88function isVideoViewsValid (value) { 90function isVideoViewsValid (value) {
89 return validator.isInt(value, { min: 0 }) 91 return validator.isInt(value + '', VIDEOS_CONSTRAINTS_FIELDS.VIEWS)
90} 92}
91 93
92function isVideoLikesValid (value) { 94function isVideoLikesValid (value) {
93 return validator.isInt(value, { min: 0 }) 95 return validator.isInt(value + '', VIDEOS_CONSTRAINTS_FIELDS.LIKES)
94} 96}
95 97
96function isVideoDislikesValid (value) { 98function isVideoDislikesValid (value) {
97 return validator.isInt(value, { min: 0 }) 99 return validator.isInt(value + '', VIDEOS_CONSTRAINTS_FIELDS.DISLIKES)
100}
101
102function isVideoEventCountValid (value) {
103 return validator.isInt(value + '', VIDEO_EVENTS_CONSTRAINTS_FIELDS.COUNT)
98} 104}
99 105
100function isVideoFile (value, files) { 106function isVideoFile (value, files) {
diff --git a/server/helpers/utils.js b/server/helpers/utils.js
index 9f4b14582..6d40e8f3f 100644
--- a/server/helpers/utils.js
+++ b/server/helpers/utils.js
@@ -6,6 +6,7 @@ const logger = require('./logger')
6 6
7const utils = { 7const utils = {
8 badRequest, 8 badRequest,
9 createEmptyCallback,
9 cleanForExit, 10 cleanForExit,
10 generateRandomString, 11 generateRandomString,
11 isTestInstance, 12 isTestInstance,
@@ -29,6 +30,12 @@ function cleanForExit (webtorrentProcess) {
29 process.kill(-webtorrentProcess.pid) 30 process.kill(-webtorrentProcess.pid)
30} 31}
31 32
33function createEmptyCallback () {
34 return function (err) {
35 if (err) logger.error('Error in empty callback.', { error: err })
36 }
37}
38
32function isTestInstance () { 39function isTestInstance () {
33 return (process.env.NODE_ENV === 'test') 40 return (process.env.NODE_ENV === 'test')
34} 41}
diff --git a/server/initializers/constants.js b/server/initializers/constants.js
index 668bfe56c..b99186e13 100644
--- a/server/initializers/constants.js
+++ b/server/initializers/constants.js
@@ -85,7 +85,13 @@ const CONSTRAINTS_FIELDS = {
85 TAGS: { min: 1, max: 3 }, // Number of total tags 85 TAGS: { min: 1, max: 3 }, // Number of total tags
86 TAG: { min: 2, max: 10 }, // Length 86 TAG: { min: 2, max: 10 }, // Length
87 THUMBNAIL: { min: 2, max: 30 }, 87 THUMBNAIL: { min: 2, max: 30 },
88 THUMBNAIL_DATA: { min: 0, max: 20000 } // Bytes 88 THUMBNAIL_DATA: { min: 0, max: 20000 }, // Bytes
89 VIEWS: { min: 0 },
90 LIKES: { min: 0 },
91 DISLIKES: { min: 0 }
92 },
93 VIDEO_EVENTS: {
94 COUNT: { min: 0 }
89 } 95 }
90} 96}
91 97
@@ -120,12 +126,17 @@ const REQUESTS_VIDEO_QADU_LIMIT_PODS = 10
120// The QADU requests are not big 126// The QADU requests are not big
121const REQUESTS_VIDEO_QADU_LIMIT_PER_POD = 50 127const REQUESTS_VIDEO_QADU_LIMIT_PER_POD = 50
122 128
129const REQUESTS_VIDEO_EVENT_LIMIT_PODS = 10
130// The EVENTS requests are not big
131const REQUESTS_VIDEO_EVENT_LIMIT_PER_POD = 50
132
123// Number of requests to retry for replay requests module 133// Number of requests to retry for replay requests module
124const RETRY_REQUESTS = 5 134const RETRY_REQUESTS = 5
125 135
126const REQUEST_ENDPOINTS = { 136const REQUEST_ENDPOINTS = {
127 VIDEOS: 'videos', 137 VIDEOS: 'videos',
128 QADU: 'videos/qadu' 138 QADU: 'videos/qadu',
139 EVENT: 'videos/events'
129} 140}
130const REQUEST_ENDPOINT_ACTIONS = {} 141const REQUEST_ENDPOINT_ACTIONS = {}
131REQUEST_ENDPOINT_ACTIONS[REQUEST_ENDPOINTS.VIDEOS] = { 142REQUEST_ENDPOINT_ACTIONS[REQUEST_ENDPOINTS.VIDEOS] = {
@@ -141,6 +152,12 @@ const REQUEST_VIDEO_QADU_TYPES = {
141 VIEWS: 'views' 152 VIEWS: 'views'
142} 153}
143 154
155const REQUEST_VIDEO_EVENT_TYPES = {
156 LIKES: 'likes',
157 DISLIKES: 'dislikes',
158 VIEWS: 'views'
159}
160
144const REMOTE_SCHEME = { 161const REMOTE_SCHEME = {
145 HTTP: 'https', 162 HTTP: 'https',
146 WS: 'wss' 163 WS: 'wss'
@@ -210,6 +227,7 @@ module.exports = {
210 REMOTE_SCHEME, 227 REMOTE_SCHEME,
211 REQUEST_ENDPOINT_ACTIONS, 228 REQUEST_ENDPOINT_ACTIONS,
212 REQUEST_ENDPOINTS, 229 REQUEST_ENDPOINTS,
230 REQUEST_VIDEO_EVENT_TYPES,
213 REQUEST_VIDEO_QADU_TYPES, 231 REQUEST_VIDEO_QADU_TYPES,
214 REQUESTS_IN_PARALLEL, 232 REQUESTS_IN_PARALLEL,
215 REQUESTS_INTERVAL, 233 REQUESTS_INTERVAL,
@@ -217,6 +235,8 @@ module.exports = {
217 REQUESTS_LIMIT_PODS, 235 REQUESTS_LIMIT_PODS,
218 REQUESTS_VIDEO_QADU_LIMIT_PER_POD, 236 REQUESTS_VIDEO_QADU_LIMIT_PER_POD,
219 REQUESTS_VIDEO_QADU_LIMIT_PODS, 237 REQUESTS_VIDEO_QADU_LIMIT_PODS,
238 REQUESTS_VIDEO_EVENT_LIMIT_PER_POD,
239 REQUESTS_VIDEO_EVENT_LIMIT_PODS,
220 RETRY_REQUESTS, 240 RETRY_REQUESTS,
221 SEARCHABLE_COLUMNS, 241 SEARCHABLE_COLUMNS,
222 SIGNATURE_ALGORITHM, 242 SIGNATURE_ALGORITHM,
diff --git a/server/lib/friends.js b/server/lib/friends.js
index 424a30801..203f0e52c 100644
--- a/server/lib/friends.js
+++ b/server/lib/friends.js
@@ -11,13 +11,16 @@ const db = require('../initializers/database')
11const logger = require('../helpers/logger') 11const logger = require('../helpers/logger')
12const peertubeCrypto = require('../helpers/peertube-crypto') 12const peertubeCrypto = require('../helpers/peertube-crypto')
13const requests = require('../helpers/requests') 13const requests = require('../helpers/requests')
14const utils = require('../helpers/utils')
14const RequestScheduler = require('./request-scheduler') 15const RequestScheduler = require('./request-scheduler')
15const RequestVideoQaduScheduler = require('./request-video-qadu-scheduler') 16const RequestVideoQaduScheduler = require('./request-video-qadu-scheduler')
17const RequestVideoEventScheduler = require('./request-video-event-scheduler')
16 18
17const ENDPOINT_ACTIONS = constants.REQUEST_ENDPOINT_ACTIONS[constants.REQUEST_ENDPOINTS.VIDEOS] 19const ENDPOINT_ACTIONS = constants.REQUEST_ENDPOINT_ACTIONS[constants.REQUEST_ENDPOINTS.VIDEOS]
18 20
19const requestScheduler = new RequestScheduler() 21const requestScheduler = new RequestScheduler()
20const requestSchedulerVideoQadu = new RequestVideoQaduScheduler() 22const requestSchedulerVideoQadu = new RequestVideoQaduScheduler()
23const requestSchedulerVideoEvent = new RequestVideoEventScheduler()
21 24
22const friends = { 25const friends = {
23 activate, 26 activate,
@@ -25,6 +28,7 @@ const friends = {
25 updateVideoToFriends, 28 updateVideoToFriends,
26 reportAbuseVideoToFriend, 29 reportAbuseVideoToFriend,
27 quickAndDirtyUpdateVideoToFriends, 30 quickAndDirtyUpdateVideoToFriends,
31 addEventToRemoteVideo,
28 hasFriends, 32 hasFriends,
29 makeFriends, 33 makeFriends,
30 quitFriends, 34 quitFriends,
@@ -35,6 +39,7 @@ const friends = {
35function activate () { 39function activate () {
36 requestScheduler.activate() 40 requestScheduler.activate()
37 requestSchedulerVideoQadu.activate() 41 requestSchedulerVideoQadu.activate()
42 requestSchedulerVideoEvent.activate()
38} 43}
39 44
40function addVideoToFriends (videoData, transaction, callback) { 45function addVideoToFriends (videoData, transaction, callback) {
@@ -85,6 +90,15 @@ function quickAndDirtyUpdateVideoToFriends (videoId, type, transaction, callback
85 return createVideoQaduRequest(options, callback) 90 return createVideoQaduRequest(options, callback)
86} 91}
87 92
93function addEventToRemoteVideo (videoId, type, transaction, callback) {
94 const options = {
95 videoId,
96 type,
97 transaction
98 }
99 createVideoEventRequest(options, callback)
100}
101
88function hasFriends (callback) { 102function hasFriends (callback) {
89 db.Pod.countAll(function (err, count) { 103 db.Pod.countAll(function (err, count) {
90 if (err) return callback(err) 104 if (err) return callback(err)
@@ -329,11 +343,17 @@ function createRequest (options, callback) {
329} 343}
330 344
331function createVideoQaduRequest (options, callback) { 345function createVideoQaduRequest (options, callback) {
332 if (!callback) callback = function () {} 346 if (!callback) callback = utils.createEmptyCallback()
333 347
334 requestSchedulerVideoQadu.createRequest(options, callback) 348 requestSchedulerVideoQadu.createRequest(options, callback)
335} 349}
336 350
351function createVideoEventRequest (options, callback) {
352 if (!callback) callback = utils.createEmptyCallback()
353
354 requestSchedulerVideoEvent.createRequest(options, callback)
355}
356
337function isMe (host) { 357function isMe (host) {
338 return host === constants.CONFIG.WEBSERVER.HOST 358 return host === constants.CONFIG.WEBSERVER.HOST
339} 359}
diff --git a/server/lib/request-video-event-scheduler.js b/server/lib/request-video-event-scheduler.js
new file mode 100644
index 000000000..5ea5631b0
--- /dev/null
+++ b/server/lib/request-video-event-scheduler.js
@@ -0,0 +1,109 @@
1'use strict'
2
3const BaseRequestScheduler = require('./base-request-scheduler')
4const constants = require('../initializers/constants')
5const db = require('../initializers/database')
6
7module.exports = class RequestVideoEventScheduler extends BaseRequestScheduler {
8
9 constructor () {
10 super()
11
12 // We limit the size of the requests
13 this.limitPods = constants.REQUESTS_VIDEO_EVENT_LIMIT_PODS
14 this.limitPerPod = constants.REQUESTS_VIDEO_EVENT_LIMIT_PER_POD
15
16 this.description = 'video event requests'
17 }
18
19 getRequestModel () {
20 return db.RequestVideoEvent
21 }
22
23 getRequestToPodModel () {
24 return db.RequestVideoEvent
25 }
26
27 buildRequestObjects (eventsToProcess) {
28 const requestsToMakeGrouped = {}
29
30 /* Example:
31 {
32 pod1: {
33 video1: { views: 4, likes: 5 },
34 video2: { likes: 5 }
35 }
36 }
37 */
38 const eventsPerVideoPerPod = {}
39
40 // We group video events per video and per pod
41 // We add the counts of the same event types
42 Object.keys(eventsToProcess).forEach(toPodId => {
43 eventsToProcess[toPodId].forEach(eventToProcess => {
44 if (!eventsPerVideoPerPod[toPodId]) eventsPerVideoPerPod[toPodId] = {}
45
46 if (!requestsToMakeGrouped[toPodId]) {
47 requestsToMakeGrouped[toPodId] = {
48 toPod: eventToProcess.pod,
49 endpoint: constants.REQUEST_ENDPOINTS.EVENT,
50 ids: [], // request ids, to delete them from the DB in the future
51 datas: [] // requests data
52 }
53 }
54 requestsToMakeGrouped[toPodId].ids.push(eventToProcess.id)
55
56 const eventsPerVideo = eventsPerVideoPerPod[toPodId]
57 const remoteId = eventToProcess.video.remoteId
58 if (!eventsPerVideo[remoteId]) eventsPerVideo[remoteId] = {}
59
60 const events = eventsPerVideo[remoteId]
61 if (!events[eventToProcess.type]) events[eventToProcess.type] = 0
62
63 events[eventToProcess.type] += eventToProcess.count
64 })
65 })
66
67 // Now we build our requests array per pod
68 Object.keys(eventsPerVideoPerPod).forEach(toPodId => {
69 const eventsForPod = eventsPerVideoPerPod[toPodId]
70
71 Object.keys(eventsForPod).forEach(remoteId => {
72 const eventsForVideo = eventsForPod[remoteId]
73
74 Object.keys(eventsForVideo).forEach(eventType => {
75 requestsToMakeGrouped[toPodId].datas.push({
76 data: {
77 remoteId,
78 eventType,
79 count: eventsForVideo[eventType]
80 }
81 })
82 })
83 })
84 })
85
86 return requestsToMakeGrouped
87 }
88
89 // { type, videoId, count?, transaction? }
90 createRequest (options, callback) {
91 const type = options.type
92 const videoId = options.videoId
93 const transaction = options.transaction
94 let count = options.count
95
96 if (count === undefined) count = 1
97
98 const dbRequestOptions = {}
99 if (transaction) dbRequestOptions.transaction = transaction
100
101 const createQuery = {
102 type,
103 count,
104 videoId
105 }
106
107 return db.RequestVideoEvent.create(createQuery, dbRequestOptions).asCallback(callback)
108 }
109}
diff --git a/server/lib/request-video-qadu-scheduler.js b/server/lib/request-video-qadu-scheduler.js
index 401b2fb44..29e44a6c4 100644
--- a/server/lib/request-video-qadu-scheduler.js
+++ b/server/lib/request-video-qadu-scheduler.js
@@ -12,7 +12,7 @@ module.exports = class RequestVideoQaduScheduler extends BaseRequestScheduler {
12 12
13 // We limit the size of the requests 13 // We limit the size of the requests
14 this.limitPods = constants.REQUESTS_VIDEO_QADU_LIMIT_PODS 14 this.limitPods = constants.REQUESTS_VIDEO_QADU_LIMIT_PODS
15 this.limitPerPod = constants.REQUESTS_VIDEO_QADU_LIMIT_PODS 15 this.limitPerPod = constants.REQUESTS_VIDEO_QADU_LIMIT_PER_POD
16 16
17 this.description = 'video QADU requests' 17 this.description = 'video QADU requests'
18 } 18 }
diff --git a/server/middlewares/validators/remote/videos.js b/server/middlewares/validators/remote/videos.js
index ddc274c45..f2c6cba5e 100644
--- a/server/middlewares/validators/remote/videos.js
+++ b/server/middlewares/validators/remote/videos.js
@@ -5,7 +5,8 @@ const logger = require('../../../helpers/logger')
5 5
6const validatorsRemoteVideos = { 6const validatorsRemoteVideos = {
7 remoteVideos, 7 remoteVideos,
8 remoteQaduVideos 8 remoteQaduVideos,
9 remoteEventsVideos
9} 10}
10 11
11function remoteVideos (req, res, next) { 12function remoteVideos (req, res, next) {
@@ -19,11 +20,18 @@ function remoteVideos (req, res, next) {
19function remoteQaduVideos (req, res, next) { 20function remoteQaduVideos (req, res, next) {
20 req.checkBody('data').isEachRemoteRequestVideosQaduValid() 21 req.checkBody('data').isEachRemoteRequestVideosQaduValid()
21 22
22 logger.debug('Checking remoteVideosQadu parameters', { parameters: req.body }) 23 logger.debug('Checking remoteQaduVideos parameters', { parameters: req.body })
23 24
24 checkErrors(req, res, next) 25 checkErrors(req, res, next)
25} 26}
26 27
28function remoteEventsVideos (req, res, next) {
29 req.checkBody('data').isEachRemoteRequestVideosEventsValid()
30
31 logger.debug('Checking remoteEventsVideos parameters', { parameters: req.body })
32
33 checkErrors(req, res, next)
34}
27// --------------------------------------------------------------------------- 35// ---------------------------------------------------------------------------
28 36
29module.exports = validatorsRemoteVideos 37module.exports = validatorsRemoteVideos
diff --git a/server/models/pod.js b/server/models/pod.js
index 14814708e..8e2d488e1 100644
--- a/server/models/pod.js
+++ b/server/models/pod.js
@@ -148,7 +148,12 @@ function listAllIds (transaction, callback) {
148 }) 148 })
149} 149}
150 150
151function listRandomPodIdsWithRequest (limit, tableRequestPod, callback) { 151function listRandomPodIdsWithRequest (limit, tableWithPods, tableWithPodsJoins, callback) {
152 if (!callback) {
153 callback = tableWithPodsJoins
154 tableWithPodsJoins = ''
155 }
156
152 const self = this 157 const self = this
153 158
154 self.count().asCallback(function (err, count) { 159 self.count().asCallback(function (err, count) {
@@ -170,7 +175,7 @@ function listRandomPodIdsWithRequest (limit, tableRequestPod, callback) {
170 where: { 175 where: {
171 id: { 176 id: {
172 $in: [ 177 $in: [
173 this.sequelize.literal('SELECT "podId" FROM "' + tableRequestPod + '"') 178 this.sequelize.literal(`SELECT DISTINCT "${tableWithPods}"."podId" FROM "${tableWithPods}" ${tableWithPodsJoins}`)
174 ] 179 ]
175 } 180 }
176 } 181 }
diff --git a/server/models/request-video-event.js b/server/models/request-video-event.js
new file mode 100644
index 000000000..ef3ebcb3a
--- /dev/null
+++ b/server/models/request-video-event.js
@@ -0,0 +1,169 @@
1'use strict'
2
3/*
4 Request Video events (likes, dislikes, views...)
5*/
6
7const values = require('lodash/values')
8
9const constants = require('../initializers/constants')
10const customVideosValidators = require('../helpers/custom-validators').videos
11
12// ---------------------------------------------------------------------------
13
14module.exports = function (sequelize, DataTypes) {
15 const RequestVideoEvent = sequelize.define('RequestVideoEvent',
16 {
17 type: {
18 type: DataTypes.ENUM(values(constants.REQUEST_VIDEO_EVENT_TYPES)),
19 allowNull: false
20 },
21 count: {
22 type: DataTypes.INTEGER,
23 allowNull: false,
24 validate: {
25 countValid: function (value) {
26 const res = customVideosValidators.isVideoEventCountValid(value)
27 if (res === false) throw new Error('Video event count is not valid.')
28 }
29 }
30 }
31 },
32 {
33 updatedAt: false,
34 indexes: [
35 {
36 fields: [ 'videoId' ]
37 }
38 ],
39 classMethods: {
40 associate,
41
42 listWithLimitAndRandom,
43
44 countTotalRequests,
45 removeAll,
46 removeByRequestIdsAndPod
47 }
48 }
49 )
50
51 return RequestVideoEvent
52}
53
54// ------------------------------ STATICS ------------------------------
55
56function associate (models) {
57 this.belongsTo(models.Video, {
58 foreignKey: {
59 name: 'videoId',
60 allowNull: false
61 },
62 onDelete: 'CASCADE'
63 })
64}
65
66function countTotalRequests (callback) {
67 const query = {}
68 return this.count(query).asCallback(callback)
69}
70
71function listWithLimitAndRandom (limitPods, limitRequestsPerPod, callback) {
72 const self = this
73 const Pod = this.sequelize.models.Pod
74
75 // We make a join between videos and authors to find the podId of our video event requests
76 const podJoins = 'INNER JOIN "Videos" ON "Videos"."authorId" = "Authors"."id" ' +
77 'INNER JOIN "RequestVideoEvents" ON "RequestVideoEvents"."videoId" = "Videos"."id"'
78
79 Pod.listRandomPodIdsWithRequest(limitPods, 'Authors', podJoins, function (err, podIds) {
80 if (err) return callback(err)
81
82 // We don't have friends that have requests
83 if (podIds.length === 0) return callback(null, [])
84
85 const query = {
86 include: [
87 {
88 model: self.sequelize.models.Video,
89 include: [
90 {
91 model: self.sequelize.models.Author,
92 include: [
93 {
94 model: self.sequelize.models.Pod,
95 where: {
96 id: {
97 $in: podIds
98 }
99 }
100 }
101 ]
102 }
103 ]
104 }
105 ]
106 }
107
108 self.findAll(query).asCallback(function (err, requests) {
109 if (err) return callback(err)
110
111 const requestsGrouped = groupAndTruncateRequests(requests, limitRequestsPerPod)
112 return callback(err, requestsGrouped)
113 })
114 })
115}
116
117function removeByRequestIdsAndPod (ids, podId, callback) {
118 const query = {
119 where: {
120 id: {
121 $in: ids
122 }
123 },
124 include: [
125 {
126 model: this.sequelize.models.Video,
127 include: [
128 {
129 model: this.sequelize.models.Author,
130 where: {
131 podId
132 }
133 }
134 ]
135 }
136 ]
137 }
138
139 this.destroy(query).asCallback(callback)
140}
141
142function removeAll (callback) {
143 // Delete all requests
144 this.truncate({ cascade: true }).asCallback(callback)
145}
146
147// ---------------------------------------------------------------------------
148
149function groupAndTruncateRequests (events, limitRequestsPerPod) {
150 const eventsGrouped = {}
151
152 events.forEach(function (event) {
153 const pod = event.Video.Author.Pod
154
155 if (!eventsGrouped[pod.id]) eventsGrouped[pod.id] = []
156
157 if (eventsGrouped[pod.id].length < limitRequestsPerPod) {
158 eventsGrouped[pod.id].push({
159 id: event.id,
160 type: event.type,
161 count: event.count,
162 video: event.Video,
163 pod
164 })
165 }
166 })
167
168 return eventsGrouped
169}
diff --git a/server/models/request-video-qadu.js b/server/models/request-video-qadu.js
index 7010fc992..5d88738aa 100644
--- a/server/models/request-video-qadu.js
+++ b/server/models/request-video-qadu.js
@@ -71,10 +71,7 @@ function associate (models) {
71} 71}
72 72
73function countTotalRequests (callback) { 73function countTotalRequests (callback) {
74 const query = { 74 const query = {}
75 include: [ this.sequelize.models.Pod ]
76 }
77
78 return this.count(query).asCallback(callback) 75 return this.count(query).asCallback(callback)
79} 76}
80 77
diff --git a/server/models/request.js b/server/models/request.js
index de73501fc..3a047f7ee 100644
--- a/server/models/request.js
+++ b/server/models/request.js
@@ -48,6 +48,8 @@ function associate (models) {
48} 48}
49 49
50function countTotalRequests (callback) { 50function countTotalRequests (callback) {
51 // We need to include Pod because there are no cascade delete when a pod is removed
52 // So we could count requests that do not have existing pod anymore
51 const query = { 53 const query = {
52 include: [ this.sequelize.models.Pod ] 54 include: [ this.sequelize.models.Pod ]
53 } 55 }
diff --git a/server/tests/api/multiple-pods.js b/server/tests/api/multiple-pods.js
index 871db54be..94b62e27a 100644
--- a/server/tests/api/multiple-pods.js
+++ b/server/tests/api/multiple-pods.js
@@ -377,19 +377,44 @@ describe('Test multiple pods', function () {
377 }) 377 })
378 378
379 describe('Should update video views', function () { 379 describe('Should update video views', function () {
380 let videoId1 380 let localVideosPod3 = []
381 let videoId2 381 let remoteVideosPod1 = []
382 let remoteVideosPod2 = []
383 let remoteVideosPod3 = []
382 384
383 before(function (done) { 385 before(function (done) {
384 videosUtils.getVideosList(servers[2].url, function (err, res) { 386 parallel([
385 if (err) throw err 387 function (callback) {
388 videosUtils.getVideosList(servers[0].url, function (err, res) {
389 if (err) throw err
386 390
387 const videos = res.body.data.filter(video => video.isLocal === true) 391 remoteVideosPod1 = res.body.data.filter(video => video.isLocal === false).map(video => video.id)
388 videoId1 = videos[0].id
389 videoId2 = videos[1].id
390 392
391 done() 393 callback()
392 }) 394 })
395 },
396
397 function (callback) {
398 videosUtils.getVideosList(servers[1].url, function (err, res) {
399 if (err) throw err
400
401 remoteVideosPod2 = res.body.data.filter(video => video.isLocal === false).map(video => video.id)
402
403 callback()
404 })
405 },
406
407 function (callback) {
408 videosUtils.getVideosList(servers[2].url, function (err, res) {
409 if (err) throw err
410
411 localVideosPod3 = res.body.data.filter(video => video.isLocal === true).map(video => video.id)
412 remoteVideosPod3 = res.body.data.filter(video => video.isLocal === false).map(video => video.id)
413
414 callback()
415 })
416 }
417 ], done)
393 }) 418 })
394 419
395 it('Should views multiple videos on owned servers', function (done) { 420 it('Should views multiple videos on owned servers', function (done) {
@@ -397,42 +422,115 @@ describe('Test multiple pods', function () {
397 422
398 parallel([ 423 parallel([
399 function (callback) { 424 function (callback) {
400 videosUtils.getVideo(servers[2].url, videoId1, callback) 425 videosUtils.getVideo(servers[2].url, localVideosPod3[0], callback)
401 }, 426 },
402 427
403 function (callback) { 428 function (callback) {
404 videosUtils.getVideo(servers[2].url, videoId1, callback) 429 videosUtils.getVideo(servers[2].url, localVideosPod3[0], callback)
405 }, 430 },
406 431
407 function (callback) { 432 function (callback) {
408 videosUtils.getVideo(servers[2].url, videoId1, callback) 433 videosUtils.getVideo(servers[2].url, localVideosPod3[0], callback)
409 }, 434 },
410 435
411 function (callback) { 436 function (callback) {
412 videosUtils.getVideo(servers[2].url, videoId2, callback) 437 videosUtils.getVideo(servers[2].url, localVideosPod3[1], callback)
438 },
439
440 function (callback) {
441 setTimeout(done, 22000)
413 } 442 }
414 ], function (err) { 443 ], function (err) {
415 if (err) throw err 444 if (err) throw err
416 445
417 setTimeout(done, 22000) 446 each(servers, function (server, callback) {
447 videosUtils.getVideosList(server.url, function (err, res) {
448 if (err) throw err
449
450 const videos = res.body.data
451 expect(videos.find(video => video.views === 3)).to.be.exist
452 expect(videos.find(video => video.views === 1)).to.be.exist
453
454 callback()
455 })
456 }, done)
418 }) 457 })
419 }) 458 })
420 459
421 it('Should have views updated on each pod', function (done) { 460 it('Should views multiple videos on each servers', function (done) {
422 each(servers, function (server, callback) { 461 this.timeout(30000)
423 videosUtils.getVideosList(server.url, function (err, res) {
424 if (err) throw err
425 462
426 const videos = res.body.data 463 parallel([
427 expect(videos.find(video => video.views === 3)).to.be.exist 464 function (callback) {
428 expect(videos.find(video => video.views === 1)).to.be.exist 465 videosUtils.getVideo(servers[0].url, remoteVideosPod1[0], callback)
466 },
429 467
430 callback() 468 function (callback) {
431 }) 469 videosUtils.getVideo(servers[1].url, remoteVideosPod2[0], callback)
432 }, done) 470 },
471
472 function (callback) {
473 videosUtils.getVideo(servers[1].url, remoteVideosPod2[0], callback)
474 },
475
476 function (callback) {
477 videosUtils.getVideo(servers[2].url, remoteVideosPod3[0], callback)
478 },
479
480 function (callback) {
481 videosUtils.getVideo(servers[2].url, remoteVideosPod3[1], callback)
482 },
483
484 function (callback) {
485 videosUtils.getVideo(servers[2].url, remoteVideosPod3[1], callback)
486 },
487
488 function (callback) {
489 videosUtils.getVideo(servers[2].url, remoteVideosPod3[1], callback)
490 },
491
492 function (callback) {
493 videosUtils.getVideo(servers[2].url, localVideosPod3[1], callback)
494 },
495
496 function (callback) {
497 videosUtils.getVideo(servers[2].url, localVideosPod3[1], callback)
498 },
499
500 function (callback) {
501 videosUtils.getVideo(servers[2].url, localVideosPod3[1], callback)
502 },
503
504 function (callback) {
505 setTimeout(done, 22000)
506 }
507 ], function (err) {
508 if (err) throw err
509
510 let baseVideos = null
511 each(servers, function (server, callback) {
512 videosUtils.getVideosList(server.url, function (err, res) {
513 if (err) throw err
514
515 const videos = res.body
516
517 // Initialize base videos for future comparisons
518 if (baseVideos === null) {
519 baseVideos = videos
520 return callback()
521 }
522
523 for (let i = 0; i < baseVideos.length; i++) {
524 expect(baseVideos[i].views).to.equal(videos[i].views)
525 }
526
527 callback()
528 })
529 }, done)
530 })
433 }) 531 })
434 }) 532 })
435/* 533
436 describe('Should manipulate these videos', function () { 534 describe('Should manipulate these videos', function () {
437 it('Should update the video 3 by asking pod 3', function (done) { 535 it('Should update the video 3 by asking pod 3', function (done) {
438 this.timeout(15000) 536 this.timeout(15000)
@@ -520,7 +618,7 @@ describe('Test multiple pods', function () {
520 }, done) 618 }, done)
521 }) 619 })
522 }) 620 })
523*/ 621
524 after(function (done) { 622 after(function (done) {
525 servers.forEach(function (server) { 623 servers.forEach(function (server) {
526 process.kill(-server.app.pid) 624 process.kill(-server.app.pid)