diff options
-rw-r--r-- | server/controllers/api/remote/videos.js | 74 | ||||
-rw-r--r-- | server/controllers/api/videos.js | 16 | ||||
-rw-r--r-- | server/helpers/custom-validators/remote/videos.js | 25 | ||||
-rw-r--r-- | server/helpers/custom-validators/videos.js | 17 | ||||
-rw-r--r-- | server/helpers/requests.js | 2 | ||||
-rw-r--r-- | server/initializers/constants.js | 20 | ||||
-rw-r--r-- | server/initializers/migrations/0015-video-views.js | 19 | ||||
-rw-r--r-- | server/lib/base-request-scheduler.js | 140 | ||||
-rw-r--r-- | server/lib/friends.js | 26 | ||||
-rw-r--r-- | server/lib/request-scheduler.js | 178 | ||||
-rw-r--r-- | server/lib/request-video-qadu-scheduler.js | 116 | ||||
-rw-r--r-- | server/middlewares/validators/remote/videos.js | 11 | ||||
-rw-r--r-- | server/models/pod.js | 63 | ||||
-rw-r--r-- | server/models/request-to-pod.js | 4 | ||||
-rw-r--r-- | server/models/request-video-qadu.js | 154 | ||||
-rw-r--r-- | server/models/request.js | 63 | ||||
-rw-r--r-- | server/models/video.js | 13 | ||||
-rw-r--r-- | server/tests/api/multiple-pods.js | 60 | ||||
-rw-r--r-- | server/tests/api/single-pod.js | 11 |
19 files changed, 796 insertions, 216 deletions
diff --git a/server/controllers/api/remote/videos.js b/server/controllers/api/remote/videos.js index f8b4949cd..79b503d4d 100644 --- a/server/controllers/api/remote/videos.js +++ b/server/controllers/api/remote/videos.js | |||
@@ -31,6 +31,13 @@ router.post('/', | |||
31 | remoteVideos | 31 | remoteVideos |
32 | ) | 32 | ) |
33 | 33 | ||
34 | router.post('/qadu', | ||
35 | signatureValidators.signature, | ||
36 | secureMiddleware.checkSignature, | ||
37 | videosValidators.remoteQaduVideos, | ||
38 | remoteVideosQadu | ||
39 | ) | ||
40 | |||
34 | // --------------------------------------------------------------------------- | 41 | // --------------------------------------------------------------------------- |
35 | 42 | ||
36 | module.exports = router | 43 | module.exports = router |
@@ -62,6 +69,73 @@ function remoteVideos (req, res, next) { | |||
62 | return res.type('json').status(204).end() | 69 | return res.type('json').status(204).end() |
63 | } | 70 | } |
64 | 71 | ||
72 | function remoteVideosQadu (req, res, next) { | ||
73 | const requests = req.body.data | ||
74 | const fromPod = res.locals.secure.pod | ||
75 | |||
76 | eachSeries(requests, function (request, callbackEach) { | ||
77 | const videoData = request.data | ||
78 | |||
79 | quickAndDirtyUpdateVideoRetryWrapper(videoData, fromPod, callbackEach) | ||
80 | }, function (err) { | ||
81 | if (err) logger.error('Error managing remote videos.', { error: err }) | ||
82 | }) | ||
83 | |||
84 | return res.type('json').status(204).end() | ||
85 | } | ||
86 | |||
87 | function quickAndDirtyUpdateVideoRetryWrapper (videoData, fromPod, finalCallback) { | ||
88 | const options = { | ||
89 | arguments: [ videoData, fromPod ], | ||
90 | errorMessage: 'Cannot update quick and dirty the remote video with many retries.' | ||
91 | } | ||
92 | |||
93 | databaseUtils.retryTransactionWrapper(quickAndDirtyUpdateVideo, options, finalCallback) | ||
94 | } | ||
95 | |||
96 | function quickAndDirtyUpdateVideo (videoData, fromPod, finalCallback) { | ||
97 | waterfall([ | ||
98 | databaseUtils.startSerializableTransaction, | ||
99 | |||
100 | function findVideo (t, callback) { | ||
101 | fetchVideo(fromPod.host, videoData.remoteId, function (err, videoInstance) { | ||
102 | return callback(err, t, videoInstance) | ||
103 | }) | ||
104 | }, | ||
105 | |||
106 | function updateVideoIntoDB (t, videoInstance, callback) { | ||
107 | const options = { transaction: t } | ||
108 | |||
109 | if (videoData.views) { | ||
110 | videoInstance.set('views', videoData.views) | ||
111 | } | ||
112 | |||
113 | if (videoData.likes) { | ||
114 | videoInstance.set('likes', videoData.likes) | ||
115 | } | ||
116 | |||
117 | if (videoData.dislikes) { | ||
118 | videoInstance.set('dislikes', videoData.dislikes) | ||
119 | } | ||
120 | |||
121 | videoInstance.save(options).asCallback(function (err) { | ||
122 | return callback(err, t) | ||
123 | }) | ||
124 | }, | ||
125 | |||
126 | databaseUtils.commitTransaction | ||
127 | |||
128 | ], function (err, t) { | ||
129 | if (err) { | ||
130 | logger.debug('Cannot quick and dirty update the remote video.', { error: err }) | ||
131 | return databaseUtils.rollbackTransaction(err, t, finalCallback) | ||
132 | } | ||
133 | |||
134 | logger.info('Remote video %s quick and dirty updated', videoData.name) | ||
135 | return finalCallback(null) | ||
136 | }) | ||
137 | } | ||
138 | |||
65 | // Handle retries on fail | 139 | // Handle retries on fail |
66 | function addRemoteVideoRetryWrapper (videoToCreateData, fromPod, finalCallback) { | 140 | function addRemoteVideoRetryWrapper (videoToCreateData, fromPod, finalCallback) { |
67 | const options = { | 141 | const options = { |
diff --git a/server/controllers/api/videos.js b/server/controllers/api/videos.js index c936105e7..9f4bbb7b7 100644 --- a/server/controllers/api/videos.js +++ b/server/controllers/api/videos.js | |||
@@ -320,6 +320,22 @@ function updateVideo (req, res, finalCallback) { | |||
320 | 320 | ||
321 | function getVideo (req, res, next) { | 321 | function getVideo (req, res, next) { |
322 | const videoInstance = res.locals.video | 322 | const videoInstance = res.locals.video |
323 | |||
324 | if (videoInstance.isOwned()) { | ||
325 | // The increment is done directly in the database, not using the instance value | ||
326 | videoInstance.increment('views').asCallback(function (err) { | ||
327 | if (err) { | ||
328 | logger.error('Cannot add view to video %d.', videoInstance.id) | ||
329 | return | ||
330 | } | ||
331 | |||
332 | // FIXME: make a real view system | ||
333 | // For example, only add a view when a user watch a video during 30s etc | ||
334 | friends.quickAndDirtyUpdateVideoToFriends(videoInstance.id, constants.REQUEST_VIDEO_QADU_TYPES.VIEWS) | ||
335 | }) | ||
336 | } | ||
337 | |||
338 | // Do not wait the view system | ||
323 | res.json(videoInstance.toFormatedJSON()) | 339 | res.json(videoInstance.toFormatedJSON()) |
324 | } | 340 | } |
325 | 341 | ||
diff --git a/server/helpers/custom-validators/remote/videos.js b/server/helpers/custom-validators/remote/videos.js index ee68ebc10..2e9cf822e 100644 --- a/server/helpers/custom-validators/remote/videos.js +++ b/server/helpers/custom-validators/remote/videos.js | |||
@@ -1,5 +1,7 @@ | |||
1 | 'use strict' | 1 | 'use strict' |
2 | 2 | ||
3 | const has = require('lodash/has') | ||
4 | |||
3 | const constants = require('../../../initializers/constants') | 5 | const constants = require('../../../initializers/constants') |
4 | const videosValidators = require('../videos') | 6 | const videosValidators = require('../videos') |
5 | const miscValidators = require('../misc') | 7 | const miscValidators = require('../misc') |
@@ -7,7 +9,8 @@ const miscValidators = require('../misc') | |||
7 | const ENDPOINT_ACTIONS = constants.REQUEST_ENDPOINT_ACTIONS[constants.REQUEST_ENDPOINTS.VIDEOS] | 9 | const ENDPOINT_ACTIONS = constants.REQUEST_ENDPOINT_ACTIONS[constants.REQUEST_ENDPOINTS.VIDEOS] |
8 | 10 | ||
9 | const remoteVideosValidators = { | 11 | const remoteVideosValidators = { |
10 | isEachRemoteRequestVideosValid | 12 | isEachRemoteRequestVideosValid, |
13 | isEachRemoteRequestVideosQaduValid | ||
11 | } | 14 | } |
12 | 15 | ||
13 | function isEachRemoteRequestVideosValid (requests) { | 16 | function isEachRemoteRequestVideosValid (requests) { |
@@ -16,13 +19,13 @@ function isEachRemoteRequestVideosValid (requests) { | |||
16 | const video = request.data | 19 | const video = request.data |
17 | return ( | 20 | return ( |
18 | isRequestTypeAddValid(request.type) && | 21 | isRequestTypeAddValid(request.type) && |
19 | isCommonVideoAttrbiutesValid(video) && | 22 | isCommonVideoAttributesValid(video) && |
20 | videosValidators.isVideoAuthorValid(video.author) && | 23 | videosValidators.isVideoAuthorValid(video.author) && |
21 | videosValidators.isVideoThumbnailDataValid(video.thumbnailData) | 24 | videosValidators.isVideoThumbnailDataValid(video.thumbnailData) |
22 | ) || | 25 | ) || |
23 | ( | 26 | ( |
24 | isRequestTypeUpdateValid(request.type) && | 27 | isRequestTypeUpdateValid(request.type) && |
25 | isCommonVideoAttrbiutesValid(video) | 28 | isCommonVideoAttributesValid(video) |
26 | ) || | 29 | ) || |
27 | ( | 30 | ( |
28 | isRequestTypeRemoveValid(request.type) && | 31 | isRequestTypeRemoveValid(request.type) && |
@@ -37,13 +40,27 @@ function isEachRemoteRequestVideosValid (requests) { | |||
37 | }) | 40 | }) |
38 | } | 41 | } |
39 | 42 | ||
43 | function isEachRemoteRequestVideosQaduValid (requests) { | ||
44 | return miscValidators.isArray(requests) && | ||
45 | requests.every(function (request) { | ||
46 | const video = request.data | ||
47 | |||
48 | return ( | ||
49 | videosValidators.isVideoRemoteIdValid(video.remoteId) && | ||
50 | (has(video, 'views') === false || videosValidators.isVideoViewsValid) && | ||
51 | (has(video, 'likes') === false || videosValidators.isVideoLikesValid) && | ||
52 | (has(video, 'dislikes') === false || videosValidators.isVideoDislikesValid) | ||
53 | ) | ||
54 | }) | ||
55 | } | ||
56 | |||
40 | // --------------------------------------------------------------------------- | 57 | // --------------------------------------------------------------------------- |
41 | 58 | ||
42 | module.exports = remoteVideosValidators | 59 | module.exports = remoteVideosValidators |
43 | 60 | ||
44 | // --------------------------------------------------------------------------- | 61 | // --------------------------------------------------------------------------- |
45 | 62 | ||
46 | function isCommonVideoAttrbiutesValid (video) { | 63 | function isCommonVideoAttributesValid (video) { |
47 | return videosValidators.isVideoDateValid(video.createdAt) && | 64 | return videosValidators.isVideoDateValid(video.createdAt) && |
48 | videosValidators.isVideoDateValid(video.updatedAt) && | 65 | videosValidators.isVideoDateValid(video.updatedAt) && |
49 | videosValidators.isVideoDescriptionValid(video.description) && | 66 | videosValidators.isVideoDescriptionValid(video.description) && |
diff --git a/server/helpers/custom-validators/videos.js b/server/helpers/custom-validators/videos.js index e2d2c8e6d..1d844118b 100644 --- a/server/helpers/custom-validators/videos.js +++ b/server/helpers/custom-validators/videos.js | |||
@@ -22,7 +22,10 @@ const videosValidators = { | |||
22 | isVideoRemoteIdValid, | 22 | isVideoRemoteIdValid, |
23 | isVideoAbuseReasonValid, | 23 | isVideoAbuseReasonValid, |
24 | isVideoAbuseReporterUsernameValid, | 24 | isVideoAbuseReporterUsernameValid, |
25 | isVideoFile | 25 | isVideoFile, |
26 | isVideoViewsValid, | ||
27 | isVideoLikesValid, | ||
28 | isVideoDislikesValid | ||
26 | } | 29 | } |
27 | 30 | ||
28 | function isVideoAuthorValid (value) { | 31 | function isVideoAuthorValid (value) { |
@@ -82,6 +85,18 @@ function isVideoAbuseReporterUsernameValid (value) { | |||
82 | return usersValidators.isUserUsernameValid(value) | 85 | return usersValidators.isUserUsernameValid(value) |
83 | } | 86 | } |
84 | 87 | ||
88 | function isVideoViewsValid (value) { | ||
89 | return validator.isInt(value, { min: 0 }) | ||
90 | } | ||
91 | |||
92 | function isVideoLikesValid (value) { | ||
93 | return validator.isInt(value, { min: 0 }) | ||
94 | } | ||
95 | |||
96 | function isVideoDislikesValid (value) { | ||
97 | return validator.isInt(value, { min: 0 }) | ||
98 | } | ||
99 | |||
85 | function isVideoFile (value, files) { | 100 | function isVideoFile (value, files) { |
86 | // Should have files | 101 | // Should have files |
87 | if (!files) return false | 102 | if (!files) return false |
diff --git a/server/helpers/requests.js b/server/helpers/requests.js index 095b95e1c..427864117 100644 --- a/server/helpers/requests.js +++ b/server/helpers/requests.js | |||
@@ -58,6 +58,8 @@ function makeSecureRequest (params, callback) { | |||
58 | requestParams.json.data = params.data | 58 | requestParams.json.data = params.data |
59 | } | 59 | } |
60 | 60 | ||
61 | console.log(requestParams.json.data) | ||
62 | |||
61 | request.post(requestParams, callback) | 63 | request.post(requestParams, callback) |
62 | } | 64 | } |
63 | 65 | ||
diff --git a/server/initializers/constants.js b/server/initializers/constants.js index 821580893..668bfe56c 100644 --- a/server/initializers/constants.js +++ b/server/initializers/constants.js | |||
@@ -5,7 +5,7 @@ const path = require('path') | |||
5 | 5 | ||
6 | // --------------------------------------------------------------------------- | 6 | // --------------------------------------------------------------------------- |
7 | 7 | ||
8 | const LAST_MIGRATION_VERSION = 10 | 8 | const LAST_MIGRATION_VERSION = 15 |
9 | 9 | ||
10 | // --------------------------------------------------------------------------- | 10 | // --------------------------------------------------------------------------- |
11 | 11 | ||
@@ -24,7 +24,7 @@ const SEARCHABLE_COLUMNS = { | |||
24 | const SORTABLE_COLUMNS = { | 24 | const SORTABLE_COLUMNS = { |
25 | USERS: [ 'id', '-id', 'username', '-username', 'createdAt', '-createdAt' ], | 25 | USERS: [ 'id', '-id', 'username', '-username', 'createdAt', '-createdAt' ], |
26 | VIDEO_ABUSES: [ 'id', '-id', 'createdAt', '-createdAt' ], | 26 | VIDEO_ABUSES: [ 'id', '-id', 'createdAt', '-createdAt' ], |
27 | VIDEOS: [ 'name', '-name', 'duration', '-duration', 'createdAt', '-createdAt' ] | 27 | VIDEOS: [ 'name', '-name', 'duration', '-duration', 'createdAt', '-createdAt', 'views', '-views' ] |
28 | } | 28 | } |
29 | 29 | ||
30 | const OAUTH_LIFETIME = { | 30 | const OAUTH_LIFETIME = { |
@@ -116,11 +116,16 @@ const REQUESTS_LIMIT_PODS = 10 | |||
116 | // How many requests we send to a pod per interval | 116 | // How many requests we send to a pod per interval |
117 | const REQUESTS_LIMIT_PER_POD = 5 | 117 | const REQUESTS_LIMIT_PER_POD = 5 |
118 | 118 | ||
119 | const REQUESTS_VIDEO_QADU_LIMIT_PODS = 10 | ||
120 | // The QADU requests are not big | ||
121 | const REQUESTS_VIDEO_QADU_LIMIT_PER_POD = 50 | ||
122 | |||
119 | // Number of requests to retry for replay requests module | 123 | // Number of requests to retry for replay requests module |
120 | const RETRY_REQUESTS = 5 | 124 | const RETRY_REQUESTS = 5 |
121 | 125 | ||
122 | const REQUEST_ENDPOINTS = { | 126 | const REQUEST_ENDPOINTS = { |
123 | VIDEOS: 'videos' | 127 | VIDEOS: 'videos', |
128 | QADU: 'videos/qadu' | ||
124 | } | 129 | } |
125 | const REQUEST_ENDPOINT_ACTIONS = {} | 130 | const REQUEST_ENDPOINT_ACTIONS = {} |
126 | REQUEST_ENDPOINT_ACTIONS[REQUEST_ENDPOINTS.VIDEOS] = { | 131 | REQUEST_ENDPOINT_ACTIONS[REQUEST_ENDPOINTS.VIDEOS] = { |
@@ -130,6 +135,12 @@ REQUEST_ENDPOINT_ACTIONS[REQUEST_ENDPOINTS.VIDEOS] = { | |||
130 | REPORT_ABUSE: 'report-abuse' | 135 | REPORT_ABUSE: 'report-abuse' |
131 | } | 136 | } |
132 | 137 | ||
138 | const REQUEST_VIDEO_QADU_TYPES = { | ||
139 | LIKES: 'likes', | ||
140 | DISLIKES: 'dislikes', | ||
141 | VIEWS: 'views' | ||
142 | } | ||
143 | |||
133 | const REMOTE_SCHEME = { | 144 | const REMOTE_SCHEME = { |
134 | HTTP: 'https', | 145 | HTTP: 'https', |
135 | WS: 'wss' | 146 | WS: 'wss' |
@@ -199,10 +210,13 @@ module.exports = { | |||
199 | REMOTE_SCHEME, | 210 | REMOTE_SCHEME, |
200 | REQUEST_ENDPOINT_ACTIONS, | 211 | REQUEST_ENDPOINT_ACTIONS, |
201 | REQUEST_ENDPOINTS, | 212 | REQUEST_ENDPOINTS, |
213 | REQUEST_VIDEO_QADU_TYPES, | ||
202 | REQUESTS_IN_PARALLEL, | 214 | REQUESTS_IN_PARALLEL, |
203 | REQUESTS_INTERVAL, | 215 | REQUESTS_INTERVAL, |
204 | REQUESTS_LIMIT_PER_POD, | 216 | REQUESTS_LIMIT_PER_POD, |
205 | REQUESTS_LIMIT_PODS, | 217 | REQUESTS_LIMIT_PODS, |
218 | REQUESTS_VIDEO_QADU_LIMIT_PER_POD, | ||
219 | REQUESTS_VIDEO_QADU_LIMIT_PODS, | ||
206 | RETRY_REQUESTS, | 220 | RETRY_REQUESTS, |
207 | SEARCHABLE_COLUMNS, | 221 | SEARCHABLE_COLUMNS, |
208 | SIGNATURE_ALGORITHM, | 222 | SIGNATURE_ALGORITHM, |
diff --git a/server/initializers/migrations/0015-video-views.js b/server/initializers/migrations/0015-video-views.js new file mode 100644 index 000000000..ae49fe73c --- /dev/null +++ b/server/initializers/migrations/0015-video-views.js | |||
@@ -0,0 +1,19 @@ | |||
1 | 'use strict' | ||
2 | |||
3 | // utils = { transaction, queryInterface, sequelize, Sequelize } | ||
4 | exports.up = function (utils, finalCallback) { | ||
5 | const q = utils.queryInterface | ||
6 | const Sequelize = utils.Sequelize | ||
7 | |||
8 | const data = { | ||
9 | type: Sequelize.INTEGER, | ||
10 | allowNull: false, | ||
11 | defaultValue: 0 | ||
12 | } | ||
13 | |||
14 | q.addColumn('Videos', 'views', data, { transaction: utils.transaction }).asCallback(finalCallback) | ||
15 | } | ||
16 | |||
17 | exports.down = function (options, callback) { | ||
18 | throw new Error('Not implemented.') | ||
19 | } | ||
diff --git a/server/lib/base-request-scheduler.js b/server/lib/base-request-scheduler.js new file mode 100644 index 000000000..d15680c25 --- /dev/null +++ b/server/lib/base-request-scheduler.js | |||
@@ -0,0 +1,140 @@ | |||
1 | 'use strict' | ||
2 | |||
3 | const eachLimit = require('async/eachLimit') | ||
4 | |||
5 | const constants = require('../initializers/constants') | ||
6 | const db = require('../initializers/database') | ||
7 | const logger = require('../helpers/logger') | ||
8 | const requests = require('../helpers/requests') | ||
9 | |||
10 | module.exports = class BaseRequestScheduler { | ||
11 | |||
12 | constructor (options) { | ||
13 | this.lastRequestTimestamp = 0 | ||
14 | this.timer = null | ||
15 | } | ||
16 | |||
17 | activate () { | ||
18 | logger.info('Requests scheduler activated.') | ||
19 | this.lastRequestTimestamp = Date.now() | ||
20 | |||
21 | this.timer = setInterval(() => { | ||
22 | this.lastRequestTimestamp = Date.now() | ||
23 | this.makeRequests() | ||
24 | }, constants.REQUESTS_INTERVAL) | ||
25 | } | ||
26 | |||
27 | deactivate () { | ||
28 | logger.info('Requests scheduler deactivated.') | ||
29 | clearInterval(this.timer) | ||
30 | this.timer = null | ||
31 | } | ||
32 | |||
33 | forceSend () { | ||
34 | logger.info('Force requests scheduler sending.') | ||
35 | this.makeRequests() | ||
36 | } | ||
37 | |||
38 | remainingMilliSeconds () { | ||
39 | if (this.timer === null) return -1 | ||
40 | |||
41 | return constants.REQUESTS_INTERVAL - (Date.now() - this.lastRequestTimestamp) | ||
42 | } | ||
43 | |||
44 | // --------------------------------------------------------------------------- | ||
45 | |||
46 | // Make a requests to friends of a certain type | ||
47 | makeRequest (toPod, requestEndpoint, requestsToMake, callback) { | ||
48 | if (!callback) callback = function () {} | ||
49 | |||
50 | const params = { | ||
51 | toPod: toPod, | ||
52 | sign: true, // Prove our identity | ||
53 | method: 'POST', | ||
54 | path: '/api/' + constants.API_VERSION + '/remote/' + requestEndpoint, | ||
55 | data: requestsToMake // Requests we need to make | ||
56 | } | ||
57 | |||
58 | // Make multiple retry requests to all of pods | ||
59 | // The function fire some useful callbacks | ||
60 | requests.makeSecureRequest(params, (err, res) => { | ||
61 | if (err || (res.statusCode !== 200 && res.statusCode !== 201 && res.statusCode !== 204)) { | ||
62 | err = err ? err.message : 'Status code not 20x : ' + res.statusCode | ||
63 | logger.error('Error sending secure request to %s pod.', toPod.host, { error: err }) | ||
64 | |||
65 | return callback(false) | ||
66 | } | ||
67 | |||
68 | return callback(true) | ||
69 | }) | ||
70 | } | ||
71 | |||
72 | // Make all the requests of the scheduler | ||
73 | makeRequests () { | ||
74 | this.getRequestModel().listWithLimitAndRandom(this.limitPods, this.limitPerPod, (err, requests) => { | ||
75 | if (err) { | ||
76 | logger.error('Cannot get the list of "%s".', this.description, { err: err }) | ||
77 | return // Abort | ||
78 | } | ||
79 | |||
80 | // If there are no requests, abort | ||
81 | if (requests.length === 0) { | ||
82 | logger.info('No "%s" to make.', this.description) | ||
83 | return | ||
84 | } | ||
85 | |||
86 | // We want to group requests by destinations pod and endpoint | ||
87 | const requestsToMakeGrouped = this.buildRequestObjects(requests) | ||
88 | |||
89 | logger.info('Making "%s" to friends.', this.description) | ||
90 | |||
91 | const goodPods = [] | ||
92 | const badPods = [] | ||
93 | |||
94 | eachLimit(Object.keys(requestsToMakeGrouped), constants.REQUESTS_IN_PARALLEL, (hashKey, callbackEach) => { | ||
95 | const requestToMake = requestsToMakeGrouped[hashKey] | ||
96 | const toPod = requestToMake.toPod | ||
97 | |||
98 | // Maybe the pod is not our friend anymore so simply remove it | ||
99 | if (!toPod) { | ||
100 | const requestIdsToDelete = requestToMake.ids | ||
101 | |||
102 | logger.info('Removing %d "%s" of unexisting pod %s.', requestIdsToDelete.length, this.description, requestToMake.toPod.id) | ||
103 | return this.getRequestToPodModel().removePodOf(requestIdsToDelete, requestToMake.toPod.id, callbackEach) | ||
104 | } | ||
105 | |||
106 | this.makeRequest(toPod, requestToMake.endpoint, requestToMake.datas, (success) => { | ||
107 | if (success === false) { | ||
108 | badPods.push(requestToMake.toPod.id) | ||
109 | return callbackEach() | ||
110 | } | ||
111 | |||
112 | logger.debug('Removing requests for pod %s.', requestToMake.toPod.id, { requestsIds: requestToMake.ids }) | ||
113 | goodPods.push(requestToMake.toPod.id) | ||
114 | |||
115 | // Remove the pod id of these request ids | ||
116 | this.getRequestToPodModel().removeByRequestIdsAndPod(requestToMake.ids, requestToMake.toPod.id, callbackEach) | ||
117 | |||
118 | this.afterRequestHook() | ||
119 | }) | ||
120 | }, () => { | ||
121 | // All the requests were made, we update the pods score | ||
122 | db.Pod.updatePodsScore(goodPods, badPods) | ||
123 | |||
124 | this.afterRequestsHook() | ||
125 | }) | ||
126 | }) | ||
127 | } | ||
128 | |||
129 | flush (callback) { | ||
130 | this.getRequestModel().removeAll(callback) | ||
131 | } | ||
132 | |||
133 | afterRequestHook () { | ||
134 | // Nothing to do, let children reimplement it | ||
135 | } | ||
136 | |||
137 | afterRequestsHook () { | ||
138 | // Nothing to do, let children reimplement it | ||
139 | } | ||
140 | } | ||
diff --git a/server/lib/friends.js b/server/lib/friends.js index d53ab4553..424a30801 100644 --- a/server/lib/friends.js +++ b/server/lib/friends.js | |||
@@ -12,15 +12,19 @@ const logger = require('../helpers/logger') | |||
12 | const peertubeCrypto = require('../helpers/peertube-crypto') | 12 | const peertubeCrypto = require('../helpers/peertube-crypto') |
13 | const requests = require('../helpers/requests') | 13 | const requests = require('../helpers/requests') |
14 | const RequestScheduler = require('./request-scheduler') | 14 | const RequestScheduler = require('./request-scheduler') |
15 | const RequestVideoQaduScheduler = require('./request-video-qadu-scheduler') | ||
15 | 16 | ||
16 | const ENDPOINT_ACTIONS = constants.REQUEST_ENDPOINT_ACTIONS[constants.REQUEST_ENDPOINTS.VIDEOS] | 17 | const ENDPOINT_ACTIONS = constants.REQUEST_ENDPOINT_ACTIONS[constants.REQUEST_ENDPOINTS.VIDEOS] |
18 | |||
17 | const requestScheduler = new RequestScheduler() | 19 | const requestScheduler = new RequestScheduler() |
20 | const requestSchedulerVideoQadu = new RequestVideoQaduScheduler() | ||
18 | 21 | ||
19 | const friends = { | 22 | const friends = { |
20 | activate, | 23 | activate, |
21 | addVideoToFriends, | 24 | addVideoToFriends, |
22 | updateVideoToFriends, | 25 | updateVideoToFriends, |
23 | reportAbuseVideoToFriend, | 26 | reportAbuseVideoToFriend, |
27 | quickAndDirtyUpdateVideoToFriends, | ||
24 | hasFriends, | 28 | hasFriends, |
25 | makeFriends, | 29 | makeFriends, |
26 | quitFriends, | 30 | quitFriends, |
@@ -30,6 +34,7 @@ const friends = { | |||
30 | 34 | ||
31 | function activate () { | 35 | function activate () { |
32 | requestScheduler.activate() | 36 | requestScheduler.activate() |
37 | requestSchedulerVideoQadu.activate() | ||
33 | } | 38 | } |
34 | 39 | ||
35 | function addVideoToFriends (videoData, transaction, callback) { | 40 | function addVideoToFriends (videoData, transaction, callback) { |
@@ -71,6 +76,15 @@ function reportAbuseVideoToFriend (reportData, video) { | |||
71 | createRequest(options) | 76 | createRequest(options) |
72 | } | 77 | } |
73 | 78 | ||
79 | function quickAndDirtyUpdateVideoToFriends (videoId, type, transaction, callback) { | ||
80 | const options = { | ||
81 | videoId, | ||
82 | type, | ||
83 | transaction | ||
84 | } | ||
85 | return createVideoQaduRequest(options, callback) | ||
86 | } | ||
87 | |||
74 | function hasFriends (callback) { | 88 | function hasFriends (callback) { |
75 | db.Pod.countAll(function (err, count) { | 89 | db.Pod.countAll(function (err, count) { |
76 | if (err) return callback(err) | 90 | if (err) return callback(err) |
@@ -110,7 +124,11 @@ function quitFriends (callback) { | |||
110 | 124 | ||
111 | waterfall([ | 125 | waterfall([ |
112 | function flushRequests (callbackAsync) { | 126 | function flushRequests (callbackAsync) { |
113 | requestScheduler.flush(callbackAsync) | 127 | requestScheduler.flush(err => callbackAsync(err)) |
128 | }, | ||
129 | |||
130 | function flushVideoQaduRequests (callbackAsync) { | ||
131 | requestSchedulerVideoQadu.flush(err => callbackAsync(err)) | ||
114 | }, | 132 | }, |
115 | 133 | ||
116 | function getPodsList (callbackAsync) { | 134 | function getPodsList (callbackAsync) { |
@@ -310,6 +328,12 @@ function createRequest (options, callback) { | |||
310 | }) | 328 | }) |
311 | } | 329 | } |
312 | 330 | ||
331 | function createVideoQaduRequest (options, callback) { | ||
332 | if (!callback) callback = function () {} | ||
333 | |||
334 | requestSchedulerVideoQadu.createRequest(options, callback) | ||
335 | } | ||
336 | |||
313 | function isMe (host) { | 337 | function isMe (host) { |
314 | return host === constants.CONFIG.WEBSERVER.HOST | 338 | return host === constants.CONFIG.WEBSERVER.HOST |
315 | } | 339 | } |
diff --git a/server/lib/request-scheduler.js b/server/lib/request-scheduler.js index 28dabe339..6b6535519 100644 --- a/server/lib/request-scheduler.js +++ b/server/lib/request-scheduler.js | |||
@@ -1,44 +1,54 @@ | |||
1 | 'use strict' | 1 | 'use strict' |
2 | 2 | ||
3 | const eachLimit = require('async/eachLimit') | ||
4 | |||
5 | const constants = require('../initializers/constants') | 3 | const constants = require('../initializers/constants') |
4 | const BaseRequestScheduler = require('./base-request-scheduler') | ||
6 | const db = require('../initializers/database') | 5 | const db = require('../initializers/database') |
7 | const logger = require('../helpers/logger') | 6 | const logger = require('../helpers/logger') |
8 | const requests = require('../helpers/requests') | ||
9 | 7 | ||
10 | module.exports = class RequestScheduler { | 8 | module.exports = class RequestScheduler extends BaseRequestScheduler { |
11 | 9 | ||
12 | constructor () { | 10 | constructor () { |
13 | this.lastRequestTimestamp = 0 | 11 | super() |
14 | this.timer = null | ||
15 | } | ||
16 | 12 | ||
17 | activate () { | 13 | // We limit the size of the requests |
18 | logger.info('Requests scheduler activated.') | 14 | this.limitPods = constants.REQUESTS_LIMIT_PODS |
19 | this.lastRequestTimestamp = Date.now() | 15 | this.limitPerPod = constants.REQUESTS_LIMIT_PER_POD |
20 | 16 | ||
21 | this.timer = setInterval(() => { | 17 | this.description = 'requests' |
22 | this.lastRequestTimestamp = Date.now() | ||
23 | this.makeRequests() | ||
24 | }, constants.REQUESTS_INTERVAL) | ||
25 | } | 18 | } |
26 | 19 | ||
27 | deactivate () { | 20 | getRequestModel () { |
28 | logger.info('Requests scheduler deactivated.') | 21 | return db.Request |
29 | clearInterval(this.timer) | ||
30 | this.timer = null | ||
31 | } | 22 | } |
32 | 23 | ||
33 | forceSend () { | 24 | getRequestToPodModel () { |
34 | logger.info('Force requests scheduler sending.') | 25 | return db.RequestToPod |
35 | this.makeRequests() | ||
36 | } | 26 | } |
37 | 27 | ||
38 | remainingMilliSeconds () { | 28 | buildRequestObjects (requests) { |
39 | if (this.timer === null) return -1 | 29 | const requestsToMakeGrouped = {} |
30 | |||
31 | Object.keys(requests).forEach(toPodId => { | ||
32 | requests[toPodId].forEach(data => { | ||
33 | const request = data.request | ||
34 | const pod = data.pod | ||
35 | const hashKey = toPodId + request.endpoint | ||
36 | |||
37 | if (!requestsToMakeGrouped[hashKey]) { | ||
38 | requestsToMakeGrouped[hashKey] = { | ||
39 | toPod: pod, | ||
40 | endpoint: request.endpoint, | ||
41 | ids: [], // request ids, to delete them from the DB in the future | ||
42 | datas: [] // requests data, | ||
43 | } | ||
44 | } | ||
45 | |||
46 | requestsToMakeGrouped[hashKey].ids.push(request.id) | ||
47 | requestsToMakeGrouped[hashKey].datas.push(request.request) | ||
48 | }) | ||
49 | }) | ||
40 | 50 | ||
41 | return constants.REQUESTS_INTERVAL - (Date.now() - this.lastRequestTimestamp) | 51 | return requestsToMakeGrouped |
42 | } | 52 | } |
43 | 53 | ||
44 | // { type, endpoint, data, toIds, transaction } | 54 | // { type, endpoint, data, toIds, transaction } |
@@ -79,122 +89,10 @@ module.exports = class RequestScheduler { | |||
79 | 89 | ||
80 | // --------------------------------------------------------------------------- | 90 | // --------------------------------------------------------------------------- |
81 | 91 | ||
82 | // Make all the requests of the scheduler | 92 | afterRequestsHook () { |
83 | makeRequests () { | 93 | // Flush requests with no pod |
84 | // We limit the size of the requests | 94 | this.getRequestModel().removeWithEmptyTo(err => { |
85 | // We don't want to stuck with the same failing requests so we get a random list | 95 | if (err) logger.error('Error when removing requests with no pods.', { error: err }) |
86 | db.Request.listWithLimitAndRandom(constants.REQUESTS_LIMIT_PODS, constants.REQUESTS_LIMIT_PER_POD, (err, requests) => { | ||
87 | if (err) { | ||
88 | logger.error('Cannot get the list of requests.', { err: err }) | ||
89 | return // Abort | ||
90 | } | ||
91 | |||
92 | // If there are no requests, abort | ||
93 | if (requests.length === 0) { | ||
94 | logger.info('No requests to make.') | ||
95 | return | ||
96 | } | ||
97 | |||
98 | // We want to group requests by destinations pod and endpoint | ||
99 | const requestsToMakeGrouped = this.buildRequestObjects(requests) | ||
100 | |||
101 | logger.info('Making requests to friends.') | ||
102 | |||
103 | const goodPods = [] | ||
104 | const badPods = [] | ||
105 | |||
106 | eachLimit(Object.keys(requestsToMakeGrouped), constants.REQUESTS_IN_PARALLEL, (hashKey, callbackEach) => { | ||
107 | const requestToMake = requestsToMakeGrouped[hashKey] | ||
108 | const toPod = requestToMake.toPod | ||
109 | |||
110 | // Maybe the pod is not our friend anymore so simply remove it | ||
111 | if (!toPod) { | ||
112 | const requestIdsToDelete = requestToMake.ids | ||
113 | |||
114 | logger.info('Removing %d requests of unexisting pod %s.', requestIdsToDelete.length, requestToMake.toPod.id) | ||
115 | return db.RequestToPod.removePodOf(requestIdsToDelete, requestToMake.toPod.id, callbackEach) | ||
116 | } | ||
117 | |||
118 | this.makeRequest(toPod, requestToMake.endpoint, requestToMake.datas, (success) => { | ||
119 | if (success === false) { | ||
120 | badPods.push(requestToMake.toPod.id) | ||
121 | return callbackEach() | ||
122 | } | ||
123 | |||
124 | logger.debug('Removing requests for pod %s.', requestToMake.toPod.id, { requestsIds: requestToMake.ids }) | ||
125 | goodPods.push(requestToMake.toPod.id) | ||
126 | |||
127 | // Remove the pod id of these request ids | ||
128 | db.RequestToPod.removePodOf(requestToMake.ids, requestToMake.toPod.id, callbackEach) | ||
129 | }) | ||
130 | }, () => { | ||
131 | // All the requests were made, we update the pods score | ||
132 | db.Request.updatePodsScore(goodPods, badPods) | ||
133 | // Flush requests with no pod | ||
134 | db.Request.removeWithEmptyTo(err => { | ||
135 | if (err) logger.error('Error when removing requests with no pods.', { error: err }) | ||
136 | }) | ||
137 | }) | ||
138 | }) | ||
139 | } | ||
140 | |||
141 | // Make a requests to friends of a certain type | ||
142 | makeRequest (toPod, requestEndpoint, requestsToMake, callback) { | ||
143 | if (!callback) callback = function () {} | ||
144 | |||
145 | const params = { | ||
146 | toPod: toPod, | ||
147 | sign: true, // Prove our identity | ||
148 | method: 'POST', | ||
149 | path: '/api/' + constants.API_VERSION + '/remote/' + requestEndpoint, | ||
150 | data: requestsToMake // Requests we need to make | ||
151 | } | ||
152 | |||
153 | // Make multiple retry requests to all of pods | ||
154 | // The function fire some useful callbacks | ||
155 | requests.makeSecureRequest(params, (err, res) => { | ||
156 | if (err || (res.statusCode !== 200 && res.statusCode !== 201 && res.statusCode !== 204)) { | ||
157 | err = err ? err.message : 'Status code not 20x : ' + res.statusCode | ||
158 | logger.error('Error sending secure request to %s pod.', toPod.host, { error: err }) | ||
159 | |||
160 | return callback(false) | ||
161 | } | ||
162 | |||
163 | return callback(true) | ||
164 | }) | ||
165 | } | ||
166 | |||
167 | buildRequestObjects (requests) { | ||
168 | const requestsToMakeGrouped = {} | ||
169 | |||
170 | Object.keys(requests).forEach(toPodId => { | ||
171 | requests[toPodId].forEach(data => { | ||
172 | const request = data.request | ||
173 | const pod = data.pod | ||
174 | const hashKey = toPodId + request.endpoint | ||
175 | |||
176 | if (!requestsToMakeGrouped[hashKey]) { | ||
177 | requestsToMakeGrouped[hashKey] = { | ||
178 | toPod: pod, | ||
179 | endpoint: request.endpoint, | ||
180 | ids: [], // request ids, to delete them from the DB in the future | ||
181 | datas: [] // requests data, | ||
182 | } | ||
183 | } | ||
184 | |||
185 | requestsToMakeGrouped[hashKey].ids.push(request.id) | ||
186 | requestsToMakeGrouped[hashKey].datas.push(request.request) | ||
187 | }) | ||
188 | }) | ||
189 | |||
190 | return requestsToMakeGrouped | ||
191 | } | ||
192 | |||
193 | flush (callback) { | ||
194 | db.Request.removeAll(err => { | ||
195 | if (err) logger.error('Cannot flush the requests.', { error: err }) | ||
196 | |||
197 | return callback(err) | ||
198 | }) | 96 | }) |
199 | } | 97 | } |
200 | } | 98 | } |
diff --git a/server/lib/request-video-qadu-scheduler.js b/server/lib/request-video-qadu-scheduler.js new file mode 100644 index 000000000..401b2fb44 --- /dev/null +++ b/server/lib/request-video-qadu-scheduler.js | |||
@@ -0,0 +1,116 @@ | |||
1 | 'use strict' | ||
2 | |||
3 | const BaseRequestScheduler = require('./base-request-scheduler') | ||
4 | const constants = require('../initializers/constants') | ||
5 | const db = require('../initializers/database') | ||
6 | const logger = require('../helpers/logger') | ||
7 | |||
8 | module.exports = class RequestVideoQaduScheduler extends BaseRequestScheduler { | ||
9 | |||
10 | constructor () { | ||
11 | super() | ||
12 | |||
13 | // We limit the size of the requests | ||
14 | this.limitPods = constants.REQUESTS_VIDEO_QADU_LIMIT_PODS | ||
15 | this.limitPerPod = constants.REQUESTS_VIDEO_QADU_LIMIT_PODS | ||
16 | |||
17 | this.description = 'video QADU requests' | ||
18 | } | ||
19 | |||
20 | getRequestModel () { | ||
21 | return db.RequestVideoQadu | ||
22 | } | ||
23 | |||
24 | getRequestToPodModel () { | ||
25 | return db.RequestVideoQadu | ||
26 | } | ||
27 | |||
28 | buildRequestObjects (requests) { | ||
29 | const requestsToMakeGrouped = {} | ||
30 | |||
31 | Object.keys(requests).forEach(toPodId => { | ||
32 | requests[toPodId].forEach(data => { | ||
33 | const request = data.request | ||
34 | const video = data.video | ||
35 | const pod = data.pod | ||
36 | const hashKey = toPodId | ||
37 | |||
38 | if (!requestsToMakeGrouped[hashKey]) { | ||
39 | requestsToMakeGrouped[hashKey] = { | ||
40 | toPod: pod, | ||
41 | endpoint: constants.REQUEST_ENDPOINTS.QADU, | ||
42 | ids: [], // request ids, to delete them from the DB in the future | ||
43 | datas: [], // requests data | ||
44 | videos: {} | ||
45 | } | ||
46 | } | ||
47 | |||
48 | if (!requestsToMakeGrouped[hashKey].videos[video.id]) { | ||
49 | requestsToMakeGrouped[hashKey].videos[video.id] = {} | ||
50 | } | ||
51 | |||
52 | const videoData = requestsToMakeGrouped[hashKey].videos[video.id] | ||
53 | |||
54 | switch (request.type) { | ||
55 | case constants.REQUEST_VIDEO_QADU_TYPES.LIKES: | ||
56 | videoData.likes = video.likes | ||
57 | break | ||
58 | |||
59 | case constants.REQUEST_VIDEO_QADU_TYPES.DISLIKES: | ||
60 | videoData.likes = video.dislikes | ||
61 | break | ||
62 | |||
63 | case constants.REQUEST_VIDEO_QADU_TYPES.VIEWS: | ||
64 | videoData.views = video.views | ||
65 | break | ||
66 | |||
67 | default: | ||
68 | logger.error('Unknown request video QADU type %s.', request.type) | ||
69 | return | ||
70 | } | ||
71 | |||
72 | // Do not forget the remoteId so the remote pod can identify the video | ||
73 | videoData.remoteId = video.id | ||
74 | requestsToMakeGrouped[hashKey].ids.push(request.id) | ||
75 | requestsToMakeGrouped[hashKey].videos[video.id] = videoData | ||
76 | }) | ||
77 | }) | ||
78 | |||
79 | Object.keys(requestsToMakeGrouped).forEach(hashKey => { | ||
80 | Object.keys(requestsToMakeGrouped[hashKey].videos).forEach(videoId => { | ||
81 | const videoData = requestsToMakeGrouped[hashKey].videos[videoId] | ||
82 | |||
83 | requestsToMakeGrouped[hashKey].datas.push({ | ||
84 | data: videoData | ||
85 | }) | ||
86 | }) | ||
87 | |||
88 | // We don't need it anymore, it was just to build our datas array | ||
89 | delete requestsToMakeGrouped[hashKey].videos | ||
90 | }) | ||
91 | |||
92 | return requestsToMakeGrouped | ||
93 | } | ||
94 | |||
95 | // { type, videoId, transaction? } | ||
96 | createRequest (options, callback) { | ||
97 | const type = options.type | ||
98 | const videoId = options.videoId | ||
99 | const transaction = options.transaction | ||
100 | |||
101 | const dbRequestOptions = {} | ||
102 | if (transaction) dbRequestOptions.transaction = transaction | ||
103 | |||
104 | // Send the update to all our friends | ||
105 | db.Pod.listAllIds(options.transaction, function (err, podIds) { | ||
106 | if (err) return callback(err) | ||
107 | |||
108 | const queries = [] | ||
109 | podIds.forEach(podId => { | ||
110 | queries.push({ type, videoId, podId }) | ||
111 | }) | ||
112 | |||
113 | return db.RequestVideoQadu.bulkCreate(queries, dbRequestOptions).asCallback(callback) | ||
114 | }) | ||
115 | } | ||
116 | } | ||
diff --git a/server/middlewares/validators/remote/videos.js b/server/middlewares/validators/remote/videos.js index cf9925b6c..ddc274c45 100644 --- a/server/middlewares/validators/remote/videos.js +++ b/server/middlewares/validators/remote/videos.js | |||
@@ -4,7 +4,8 @@ const checkErrors = require('../utils').checkErrors | |||
4 | const logger = require('../../../helpers/logger') | 4 | const logger = require('../../../helpers/logger') |
5 | 5 | ||
6 | const validatorsRemoteVideos = { | 6 | const validatorsRemoteVideos = { |
7 | remoteVideos | 7 | remoteVideos, |
8 | remoteQaduVideos | ||
8 | } | 9 | } |
9 | 10 | ||
10 | function remoteVideos (req, res, next) { | 11 | function remoteVideos (req, res, next) { |
@@ -15,6 +16,14 @@ function remoteVideos (req, res, next) { | |||
15 | checkErrors(req, res, next) | 16 | checkErrors(req, res, next) |
16 | } | 17 | } |
17 | 18 | ||
19 | function remoteQaduVideos (req, res, next) { | ||
20 | req.checkBody('data').isEachRemoteRequestVideosQaduValid() | ||
21 | |||
22 | logger.debug('Checking remoteVideosQadu parameters', { parameters: req.body }) | ||
23 | |||
24 | checkErrors(req, res, next) | ||
25 | } | ||
26 | |||
18 | // --------------------------------------------------------------------------- | 27 | // --------------------------------------------------------------------------- |
19 | 28 | ||
20 | module.exports = validatorsRemoteVideos | 29 | module.exports = validatorsRemoteVideos |
diff --git a/server/models/pod.js b/server/models/pod.js index 79afb737a..14814708e 100644 --- a/server/models/pod.js +++ b/server/models/pod.js | |||
@@ -1,8 +1,11 @@ | |||
1 | 'use strict' | 1 | 'use strict' |
2 | 2 | ||
3 | const each = require('async/each') | ||
3 | const map = require('lodash/map') | 4 | const map = require('lodash/map') |
5 | const waterfall = require('async/waterfall') | ||
4 | 6 | ||
5 | const constants = require('../initializers/constants') | 7 | const constants = require('../initializers/constants') |
8 | const logger = require('../helpers/logger') | ||
6 | const customPodsValidators = require('../helpers/custom-validators').pods | 9 | const customPodsValidators = require('../helpers/custom-validators').pods |
7 | 10 | ||
8 | // --------------------------------------------------------------------------- | 11 | // --------------------------------------------------------------------------- |
@@ -62,6 +65,7 @@ module.exports = function (sequelize, DataTypes) { | |||
62 | listBadPods, | 65 | listBadPods, |
63 | load, | 66 | load, |
64 | loadByHost, | 67 | loadByHost, |
68 | updatePodsScore, | ||
65 | removeAll | 69 | removeAll |
66 | }, | 70 | }, |
67 | instanceMethods: { | 71 | instanceMethods: { |
@@ -144,7 +148,7 @@ function listAllIds (transaction, callback) { | |||
144 | }) | 148 | }) |
145 | } | 149 | } |
146 | 150 | ||
147 | function listRandomPodIdsWithRequest (limit, callback) { | 151 | function listRandomPodIdsWithRequest (limit, tableRequestPod, callback) { |
148 | const self = this | 152 | const self = this |
149 | 153 | ||
150 | self.count().asCallback(function (err, count) { | 154 | self.count().asCallback(function (err, count) { |
@@ -166,7 +170,7 @@ function listRandomPodIdsWithRequest (limit, callback) { | |||
166 | where: { | 170 | where: { |
167 | id: { | 171 | id: { |
168 | $in: [ | 172 | $in: [ |
169 | this.sequelize.literal('SELECT "podId" FROM "RequestToPods"') | 173 | this.sequelize.literal('SELECT "podId" FROM "' + tableRequestPod + '"') |
170 | ] | 174 | ] |
171 | } | 175 | } |
172 | } | 176 | } |
@@ -207,3 +211,58 @@ function loadByHost (host, callback) { | |||
207 | function removeAll (callback) { | 211 | function removeAll (callback) { |
208 | return this.destroy().asCallback(callback) | 212 | return this.destroy().asCallback(callback) |
209 | } | 213 | } |
214 | |||
215 | function updatePodsScore (goodPods, badPods) { | ||
216 | const self = this | ||
217 | |||
218 | logger.info('Updating %d good pods and %d bad pods scores.', goodPods.length, badPods.length) | ||
219 | |||
220 | if (goodPods.length !== 0) { | ||
221 | this.incrementScores(goodPods, constants.PODS_SCORE.BONUS, function (err) { | ||
222 | if (err) logger.error('Cannot increment scores of good pods.', { error: err }) | ||
223 | }) | ||
224 | } | ||
225 | |||
226 | if (badPods.length !== 0) { | ||
227 | this.incrementScores(badPods, constants.PODS_SCORE.MALUS, function (err) { | ||
228 | if (err) logger.error('Cannot decrement scores of bad pods.', { error: err }) | ||
229 | removeBadPods.call(self) | ||
230 | }) | ||
231 | } | ||
232 | } | ||
233 | |||
234 | // --------------------------------------------------------------------------- | ||
235 | |||
236 | // Remove pods with a score of 0 (too many requests where they were unreachable) | ||
237 | function removeBadPods () { | ||
238 | const self = this | ||
239 | |||
240 | waterfall([ | ||
241 | function findBadPods (callback) { | ||
242 | self.sequelize.models.Pod.listBadPods(function (err, pods) { | ||
243 | if (err) { | ||
244 | logger.error('Cannot find bad pods.', { error: err }) | ||
245 | return callback(err) | ||
246 | } | ||
247 | |||
248 | return callback(null, pods) | ||
249 | }) | ||
250 | }, | ||
251 | |||
252 | function removeTheseBadPods (pods, callback) { | ||
253 | each(pods, function (pod, callbackEach) { | ||
254 | pod.destroy().asCallback(callbackEach) | ||
255 | }, function (err) { | ||
256 | return callback(err, pods.length) | ||
257 | }) | ||
258 | } | ||
259 | ], function (err, numberOfPodsRemoved) { | ||
260 | if (err) { | ||
261 | logger.error('Cannot remove bad pods.', { error: err }) | ||
262 | } else if (numberOfPodsRemoved) { | ||
263 | logger.info('Removed %d pods.', numberOfPodsRemoved) | ||
264 | } else { | ||
265 | logger.info('No need to remove bad pods.') | ||
266 | } | ||
267 | }) | ||
268 | } | ||
diff --git a/server/models/request-to-pod.js b/server/models/request-to-pod.js index f42a53458..0e01a842e 100644 --- a/server/models/request-to-pod.js +++ b/server/models/request-to-pod.js | |||
@@ -17,7 +17,7 @@ module.exports = function (sequelize, DataTypes) { | |||
17 | } | 17 | } |
18 | ], | 18 | ], |
19 | classMethods: { | 19 | classMethods: { |
20 | removePodOf | 20 | removeByRequestIdsAndPod |
21 | } | 21 | } |
22 | }) | 22 | }) |
23 | 23 | ||
@@ -26,7 +26,7 @@ module.exports = function (sequelize, DataTypes) { | |||
26 | 26 | ||
27 | // --------------------------------------------------------------------------- | 27 | // --------------------------------------------------------------------------- |
28 | 28 | ||
29 | function removePodOf (requestsIds, podId, callback) { | 29 | function removeByRequestIdsAndPod (requestsIds, podId, callback) { |
30 | if (!callback) callback = function () {} | 30 | if (!callback) callback = function () {} |
31 | 31 | ||
32 | const query = { | 32 | const query = { |
diff --git a/server/models/request-video-qadu.js b/server/models/request-video-qadu.js new file mode 100644 index 000000000..7010fc992 --- /dev/null +++ b/server/models/request-video-qadu.js | |||
@@ -0,0 +1,154 @@ | |||
1 | 'use strict' | ||
2 | |||
3 | /* | ||
4 | Request Video for Quick And Dirty Updates like: | ||
5 | - views | ||
6 | - likes | ||
7 | - dislikes | ||
8 | |||
9 | We can't put it in the same system than basic requests for efficiency. | ||
10 | Moreover we don't want to slow down the basic requests with a lot of views/likes/dislikes requests. | ||
11 | So we put it an independant request scheduler. | ||
12 | */ | ||
13 | |||
14 | const values = require('lodash/values') | ||
15 | |||
16 | const constants = require('../initializers/constants') | ||
17 | |||
18 | // --------------------------------------------------------------------------- | ||
19 | |||
20 | module.exports = function (sequelize, DataTypes) { | ||
21 | const RequestVideoQadu = sequelize.define('RequestVideoQadu', | ||
22 | { | ||
23 | type: { | ||
24 | type: DataTypes.ENUM(values(constants.REQUEST_VIDEO_QADU_TYPES)), | ||
25 | allowNull: false | ||
26 | } | ||
27 | }, | ||
28 | { | ||
29 | timestamps: false, | ||
30 | indexes: [ | ||
31 | { | ||
32 | fields: [ 'podId' ] | ||
33 | }, | ||
34 | { | ||
35 | fields: [ 'videoId' ] | ||
36 | } | ||
37 | ], | ||
38 | classMethods: { | ||
39 | associate, | ||
40 | |||
41 | listWithLimitAndRandom, | ||
42 | |||
43 | countTotalRequests, | ||
44 | removeAll, | ||
45 | removeByRequestIdsAndPod | ||
46 | } | ||
47 | } | ||
48 | ) | ||
49 | |||
50 | return RequestVideoQadu | ||
51 | } | ||
52 | |||
53 | // ------------------------------ STATICS ------------------------------ | ||
54 | |||
55 | function associate (models) { | ||
56 | this.belongsTo(models.Pod, { | ||
57 | foreignKey: { | ||
58 | name: 'podId', | ||
59 | allowNull: false | ||
60 | }, | ||
61 | onDelete: 'CASCADE' | ||
62 | }) | ||
63 | |||
64 | this.belongsTo(models.Video, { | ||
65 | foreignKey: { | ||
66 | name: 'videoId', | ||
67 | allowNull: false | ||
68 | }, | ||
69 | onDelete: 'CASCADE' | ||
70 | }) | ||
71 | } | ||
72 | |||
73 | function countTotalRequests (callback) { | ||
74 | const query = { | ||
75 | include: [ this.sequelize.models.Pod ] | ||
76 | } | ||
77 | |||
78 | return this.count(query).asCallback(callback) | ||
79 | } | ||
80 | |||
81 | function listWithLimitAndRandom (limitPods, limitRequestsPerPod, callback) { | ||
82 | const self = this | ||
83 | const Pod = this.sequelize.models.Pod | ||
84 | |||
85 | Pod.listRandomPodIdsWithRequest(limitPods, 'RequestVideoQadus', function (err, podIds) { | ||
86 | if (err) return callback(err) | ||
87 | |||
88 | // We don't have friends that have requests | ||
89 | if (podIds.length === 0) return callback(null, []) | ||
90 | |||
91 | const query = { | ||
92 | include: [ | ||
93 | { | ||
94 | model: self.sequelize.models.Pod, | ||
95 | where: { | ||
96 | id: { | ||
97 | $in: podIds | ||
98 | } | ||
99 | } | ||
100 | }, | ||
101 | { | ||
102 | model: self.sequelize.models.Video | ||
103 | } | ||
104 | ] | ||
105 | } | ||
106 | |||
107 | self.findAll(query).asCallback(function (err, requests) { | ||
108 | if (err) return callback(err) | ||
109 | |||
110 | const requestsGrouped = groupAndTruncateRequests(requests, limitRequestsPerPod) | ||
111 | return callback(err, requestsGrouped) | ||
112 | }) | ||
113 | }) | ||
114 | } | ||
115 | |||
116 | function removeByRequestIdsAndPod (ids, podId, callback) { | ||
117 | const query = { | ||
118 | where: { | ||
119 | id: { | ||
120 | $in: ids | ||
121 | }, | ||
122 | podId | ||
123 | } | ||
124 | } | ||
125 | |||
126 | this.destroy(query).asCallback(callback) | ||
127 | } | ||
128 | |||
129 | function removeAll (callback) { | ||
130 | // Delete all requests | ||
131 | this.truncate({ cascade: true }).asCallback(callback) | ||
132 | } | ||
133 | |||
134 | // --------------------------------------------------------------------------- | ||
135 | |||
136 | function groupAndTruncateRequests (requests, limitRequestsPerPod) { | ||
137 | const requestsGrouped = {} | ||
138 | |||
139 | requests.forEach(function (request) { | ||
140 | const pod = request.Pod | ||
141 | |||
142 | if (!requestsGrouped[pod.id]) requestsGrouped[pod.id] = [] | ||
143 | |||
144 | if (requestsGrouped[pod.id].length < limitRequestsPerPod) { | ||
145 | requestsGrouped[pod.id].push({ | ||
146 | request: request, | ||
147 | video: request.Video, | ||
148 | pod | ||
149 | }) | ||
150 | } | ||
151 | }) | ||
152 | |||
153 | return requestsGrouped | ||
154 | } | ||
diff --git a/server/models/request.js b/server/models/request.js index ca616d130..de73501fc 100644 --- a/server/models/request.js +++ b/server/models/request.js | |||
@@ -1,11 +1,8 @@ | |||
1 | 'use strict' | 1 | 'use strict' |
2 | 2 | ||
3 | const each = require('async/each') | ||
4 | const waterfall = require('async/waterfall') | ||
5 | const values = require('lodash/values') | 3 | const values = require('lodash/values') |
6 | 4 | ||
7 | const constants = require('../initializers/constants') | 5 | const constants = require('../initializers/constants') |
8 | const logger = require('../helpers/logger') | ||
9 | 6 | ||
10 | // --------------------------------------------------------------------------- | 7 | // --------------------------------------------------------------------------- |
11 | 8 | ||
@@ -28,8 +25,6 @@ module.exports = function (sequelize, DataTypes) { | |||
28 | listWithLimitAndRandom, | 25 | listWithLimitAndRandom, |
29 | 26 | ||
30 | countTotalRequests, | 27 | countTotalRequests, |
31 | removeBadPods, | ||
32 | updatePodsScore, | ||
33 | removeAll, | 28 | removeAll, |
34 | removeWithEmptyTo | 29 | removeWithEmptyTo |
35 | } | 30 | } |
@@ -60,71 +55,17 @@ function countTotalRequests (callback) { | |||
60 | return this.count(query).asCallback(callback) | 55 | return this.count(query).asCallback(callback) |
61 | } | 56 | } |
62 | 57 | ||
63 | // Remove pods with a score of 0 (too many requests where they were unreachable) | ||
64 | function removeBadPods () { | ||
65 | const self = this | ||
66 | |||
67 | waterfall([ | ||
68 | function findBadPods (callback) { | ||
69 | self.sequelize.models.Pod.listBadPods(function (err, pods) { | ||
70 | if (err) { | ||
71 | logger.error('Cannot find bad pods.', { error: err }) | ||
72 | return callback(err) | ||
73 | } | ||
74 | |||
75 | return callback(null, pods) | ||
76 | }) | ||
77 | }, | ||
78 | |||
79 | function removeTheseBadPods (pods, callback) { | ||
80 | each(pods, function (pod, callbackEach) { | ||
81 | pod.destroy().asCallback(callbackEach) | ||
82 | }, function (err) { | ||
83 | return callback(err, pods.length) | ||
84 | }) | ||
85 | } | ||
86 | ], function (err, numberOfPodsRemoved) { | ||
87 | if (err) { | ||
88 | logger.error('Cannot remove bad pods.', { error: err }) | ||
89 | } else if (numberOfPodsRemoved) { | ||
90 | logger.info('Removed %d pods.', numberOfPodsRemoved) | ||
91 | } else { | ||
92 | logger.info('No need to remove bad pods.') | ||
93 | } | ||
94 | }) | ||
95 | } | ||
96 | |||
97 | function updatePodsScore (goodPods, badPods) { | ||
98 | const self = this | ||
99 | const Pod = this.sequelize.models.Pod | ||
100 | |||
101 | logger.info('Updating %d good pods and %d bad pods scores.', goodPods.length, badPods.length) | ||
102 | |||
103 | if (goodPods.length !== 0) { | ||
104 | Pod.incrementScores(goodPods, constants.PODS_SCORE.BONUS, function (err) { | ||
105 | if (err) logger.error('Cannot increment scores of good pods.', { error: err }) | ||
106 | }) | ||
107 | } | ||
108 | |||
109 | if (badPods.length !== 0) { | ||
110 | Pod.incrementScores(badPods, constants.PODS_SCORE.MALUS, function (err) { | ||
111 | if (err) logger.error('Cannot decrement scores of bad pods.', { error: err }) | ||
112 | removeBadPods.call(self) | ||
113 | }) | ||
114 | } | ||
115 | } | ||
116 | |||
117 | function listWithLimitAndRandom (limitPods, limitRequestsPerPod, callback) { | 58 | function listWithLimitAndRandom (limitPods, limitRequestsPerPod, callback) { |
118 | const self = this | 59 | const self = this |
119 | const Pod = this.sequelize.models.Pod | 60 | const Pod = this.sequelize.models.Pod |
120 | 61 | ||
121 | Pod.listRandomPodIdsWithRequest(limitPods, function (err, podIds) { | 62 | Pod.listRandomPodIdsWithRequest(limitPods, 'RequestToPods', function (err, podIds) { |
122 | if (err) return callback(err) | 63 | if (err) return callback(err) |
123 | 64 | ||
124 | // We don't have friends that have requests | 65 | // We don't have friends that have requests |
125 | if (podIds.length === 0) return callback(null, []) | 66 | if (podIds.length === 0) return callback(null, []) |
126 | 67 | ||
127 | // The the first x requests of these pods | 68 | // The first x requests of these pods |
128 | // It is very important to sort by id ASC to keep the requests order! | 69 | // It is very important to sort by id ASC to keep the requests order! |
129 | const query = { | 70 | const query = { |
130 | order: [ | 71 | order: [ |
diff --git a/server/models/video.js b/server/models/video.js index d0fd61eb4..daa273845 100644 --- a/server/models/video.js +++ b/server/models/video.js | |||
@@ -80,6 +80,15 @@ module.exports = function (sequelize, DataTypes) { | |||
80 | if (res === false) throw new Error('Video duration is not valid.') | 80 | if (res === false) throw new Error('Video duration is not valid.') |
81 | } | 81 | } |
82 | } | 82 | } |
83 | }, | ||
84 | views: { | ||
85 | type: DataTypes.INTEGER, | ||
86 | allowNull: false, | ||
87 | defaultValue: 0, | ||
88 | validate: { | ||
89 | min: 0, | ||
90 | isInt: true | ||
91 | } | ||
83 | } | 92 | } |
84 | }, | 93 | }, |
85 | { | 94 | { |
@@ -101,6 +110,9 @@ module.exports = function (sequelize, DataTypes) { | |||
101 | }, | 110 | }, |
102 | { | 111 | { |
103 | fields: [ 'infoHash' ] | 112 | fields: [ 'infoHash' ] |
113 | }, | ||
114 | { | ||
115 | fields: [ 'views' ] | ||
104 | } | 116 | } |
105 | ], | 117 | ], |
106 | classMethods: { | 118 | classMethods: { |
@@ -336,6 +348,7 @@ function toFormatedJSON () { | |||
336 | magnetUri: this.generateMagnetUri(), | 348 | magnetUri: this.generateMagnetUri(), |
337 | author: this.Author.name, | 349 | author: this.Author.name, |
338 | duration: this.duration, | 350 | duration: this.duration, |
351 | views: this.views, | ||
339 | tags: map(this.Tags, 'name'), | 352 | tags: map(this.Tags, 'name'), |
340 | thumbnailPath: pathUtils.join(constants.STATIC_PATHS.THUMBNAILS, this.getThumbnailName()), | 353 | thumbnailPath: pathUtils.join(constants.STATIC_PATHS.THUMBNAILS, this.getThumbnailName()), |
341 | createdAt: this.createdAt, | 354 | createdAt: this.createdAt, |
diff --git a/server/tests/api/multiple-pods.js b/server/tests/api/multiple-pods.js index df12ba0e9..871db54be 100644 --- a/server/tests/api/multiple-pods.js +++ b/server/tests/api/multiple-pods.js | |||
@@ -3,6 +3,7 @@ | |||
3 | const chai = require('chai') | 3 | const chai = require('chai') |
4 | const each = require('async/each') | 4 | const each = require('async/each') |
5 | const expect = chai.expect | 5 | const expect = chai.expect |
6 | const parallel = require('async/parallel') | ||
6 | const series = require('async/series') | 7 | const series = require('async/series') |
7 | const WebTorrent = require('webtorrent') | 8 | const WebTorrent = require('webtorrent') |
8 | const webtorrent = new WebTorrent() | 9 | const webtorrent = new WebTorrent() |
@@ -375,6 +376,63 @@ describe('Test multiple pods', function () { | |||
375 | }) | 376 | }) |
376 | }) | 377 | }) |
377 | 378 | ||
379 | describe('Should update video views', function () { | ||
380 | let videoId1 | ||
381 | let videoId2 | ||
382 | |||
383 | before(function (done) { | ||
384 | videosUtils.getVideosList(servers[2].url, function (err, res) { | ||
385 | if (err) throw err | ||
386 | |||
387 | const videos = res.body.data.filter(video => video.isLocal === true) | ||
388 | videoId1 = videos[0].id | ||
389 | videoId2 = videos[1].id | ||
390 | |||
391 | done() | ||
392 | }) | ||
393 | }) | ||
394 | |||
395 | it('Should views multiple videos on owned servers', function (done) { | ||
396 | this.timeout(30000) | ||
397 | |||
398 | parallel([ | ||
399 | function (callback) { | ||
400 | videosUtils.getVideo(servers[2].url, videoId1, callback) | ||
401 | }, | ||
402 | |||
403 | function (callback) { | ||
404 | videosUtils.getVideo(servers[2].url, videoId1, callback) | ||
405 | }, | ||
406 | |||
407 | function (callback) { | ||
408 | videosUtils.getVideo(servers[2].url, videoId1, callback) | ||
409 | }, | ||
410 | |||
411 | function (callback) { | ||
412 | videosUtils.getVideo(servers[2].url, videoId2, callback) | ||
413 | } | ||
414 | ], function (err) { | ||
415 | if (err) throw err | ||
416 | |||
417 | setTimeout(done, 22000) | ||
418 | }) | ||
419 | }) | ||
420 | |||
421 | it('Should have views updated on each pod', function (done) { | ||
422 | each(servers, function (server, callback) { | ||
423 | videosUtils.getVideosList(server.url, function (err, res) { | ||
424 | if (err) throw err | ||
425 | |||
426 | const videos = res.body.data | ||
427 | expect(videos.find(video => video.views === 3)).to.be.exist | ||
428 | expect(videos.find(video => video.views === 1)).to.be.exist | ||
429 | |||
430 | callback() | ||
431 | }) | ||
432 | }, done) | ||
433 | }) | ||
434 | }) | ||
435 | /* | ||
378 | describe('Should manipulate these videos', function () { | 436 | describe('Should manipulate these videos', function () { |
379 | it('Should update the video 3 by asking pod 3', function (done) { | 437 | it('Should update the video 3 by asking pod 3', function (done) { |
380 | this.timeout(15000) | 438 | this.timeout(15000) |
@@ -462,7 +520,7 @@ describe('Test multiple pods', function () { | |||
462 | }, done) | 520 | }, done) |
463 | }) | 521 | }) |
464 | }) | 522 | }) |
465 | 523 | */ | |
466 | after(function (done) { | 524 | after(function (done) { |
467 | servers.forEach(function (server) { | 525 | servers.forEach(function (server) { |
468 | process.kill(-server.app.pid) | 526 | process.kill(-server.app.pid) |
diff --git a/server/tests/api/single-pod.js b/server/tests/api/single-pod.js index 83a2b4411..40c33686f 100644 --- a/server/tests/api/single-pod.js +++ b/server/tests/api/single-pod.js | |||
@@ -129,6 +129,17 @@ describe('Test a single pod', function () { | |||
129 | }) | 129 | }) |
130 | }) | 130 | }) |
131 | 131 | ||
132 | it('Should have the views updated', function (done) { | ||
133 | videosUtils.getVideo(server.url, videoId, function (err, res) { | ||
134 | if (err) throw err | ||
135 | |||
136 | const video = res.body | ||
137 | expect(video.views).to.equal(1) | ||
138 | |||
139 | done() | ||
140 | }) | ||
141 | }) | ||
142 | |||
132 | it('Should search the video by name by default', function (done) { | 143 | it('Should search the video by name by default', function (done) { |
133 | videosUtils.searchVideo(server.url, 'my', function (err, res) { | 144 | videosUtils.searchVideo(server.url, 'my', function (err, res) { |
134 | if (err) throw err | 145 | if (err) throw err |