diff options
author | Chocobozzz <florian.bigard@gmail.com> | 2017-02-21 21:35:59 +0100 |
---|---|---|
committer | Chocobozzz <florian.bigard@gmail.com> | 2017-02-26 20:01:26 +0100 |
commit | 9e167724f7e933f41d9ea2e1c31772bf4c560a28 (patch) | |
tree | 093cb7c1b088f35aaf847f859a313a121c8cd233 | |
parent | 0150b17e51df3e9fad8a59133d828c68f8ba672b (diff) | |
download | PeerTube-9e167724f7e933f41d9ea2e1c31772bf4c560a28.tar.gz PeerTube-9e167724f7e933f41d9ea2e1c31772bf4c560a28.tar.zst PeerTube-9e167724f7e933f41d9ea2e1c31772bf4c560a28.zip |
Server: make a basic "quick and dirty update" for videos
This system will be useful to to update some int video attributes
(likes, dislikes, views...)
The classic system is not used because we need some optimization for
scaling
-rw-r--r-- | server/controllers/api/remote/videos.js | 74 | ||||
-rw-r--r-- | server/controllers/api/videos.js | 16 | ||||
-rw-r--r-- | server/helpers/custom-validators/remote/videos.js | 25 | ||||
-rw-r--r-- | server/helpers/custom-validators/videos.js | 17 | ||||
-rw-r--r-- | server/helpers/requests.js | 2 | ||||
-rw-r--r-- | server/initializers/constants.js | 20 | ||||
-rw-r--r-- | server/initializers/migrations/0015-video-views.js | 19 | ||||
-rw-r--r-- | server/lib/base-request-scheduler.js | 140 | ||||
-rw-r--r-- | server/lib/friends.js | 26 | ||||
-rw-r--r-- | server/lib/request-scheduler.js | 178 | ||||
-rw-r--r-- | server/lib/request-video-qadu-scheduler.js | 116 | ||||
-rw-r--r-- | server/middlewares/validators/remote/videos.js | 11 | ||||
-rw-r--r-- | server/models/pod.js | 63 | ||||
-rw-r--r-- | server/models/request-to-pod.js | 4 | ||||
-rw-r--r-- | server/models/request-video-qadu.js | 154 | ||||
-rw-r--r-- | server/models/request.js | 63 | ||||
-rw-r--r-- | server/models/video.js | 13 | ||||
-rw-r--r-- | server/tests/api/multiple-pods.js | 60 | ||||
-rw-r--r-- | server/tests/api/single-pod.js | 11 |
19 files changed, 796 insertions, 216 deletions
diff --git a/server/controllers/api/remote/videos.js b/server/controllers/api/remote/videos.js index f8b4949cd..79b503d4d 100644 --- a/server/controllers/api/remote/videos.js +++ b/server/controllers/api/remote/videos.js | |||
@@ -31,6 +31,13 @@ router.post('/', | |||
31 | remoteVideos | 31 | remoteVideos |
32 | ) | 32 | ) |
33 | 33 | ||
34 | router.post('/qadu', | ||
35 | signatureValidators.signature, | ||
36 | secureMiddleware.checkSignature, | ||
37 | videosValidators.remoteQaduVideos, | ||
38 | remoteVideosQadu | ||
39 | ) | ||
40 | |||
34 | // --------------------------------------------------------------------------- | 41 | // --------------------------------------------------------------------------- |
35 | 42 | ||
36 | module.exports = router | 43 | module.exports = router |
@@ -62,6 +69,73 @@ function remoteVideos (req, res, next) { | |||
62 | return res.type('json').status(204).end() | 69 | return res.type('json').status(204).end() |
63 | } | 70 | } |
64 | 71 | ||
72 | function remoteVideosQadu (req, res, next) { | ||
73 | const requests = req.body.data | ||
74 | const fromPod = res.locals.secure.pod | ||
75 | |||
76 | eachSeries(requests, function (request, callbackEach) { | ||
77 | const videoData = request.data | ||
78 | |||
79 | quickAndDirtyUpdateVideoRetryWrapper(videoData, fromPod, callbackEach) | ||
80 | }, function (err) { | ||
81 | if (err) logger.error('Error managing remote videos.', { error: err }) | ||
82 | }) | ||
83 | |||
84 | return res.type('json').status(204).end() | ||
85 | } | ||
86 | |||
87 | function quickAndDirtyUpdateVideoRetryWrapper (videoData, fromPod, finalCallback) { | ||
88 | const options = { | ||
89 | arguments: [ videoData, fromPod ], | ||
90 | errorMessage: 'Cannot update quick and dirty the remote video with many retries.' | ||
91 | } | ||
92 | |||
93 | databaseUtils.retryTransactionWrapper(quickAndDirtyUpdateVideo, options, finalCallback) | ||
94 | } | ||
95 | |||
96 | function quickAndDirtyUpdateVideo (videoData, fromPod, finalCallback) { | ||
97 | waterfall([ | ||
98 | databaseUtils.startSerializableTransaction, | ||
99 | |||
100 | function findVideo (t, callback) { | ||
101 | fetchVideo(fromPod.host, videoData.remoteId, function (err, videoInstance) { | ||
102 | return callback(err, t, videoInstance) | ||
103 | }) | ||
104 | }, | ||
105 | |||
106 | function updateVideoIntoDB (t, videoInstance, callback) { | ||
107 | const options = { transaction: t } | ||
108 | |||
109 | if (videoData.views) { | ||
110 | videoInstance.set('views', videoData.views) | ||
111 | } | ||
112 | |||
113 | if (videoData.likes) { | ||
114 | videoInstance.set('likes', videoData.likes) | ||
115 | } | ||
116 | |||
117 | if (videoData.dislikes) { | ||
118 | videoInstance.set('dislikes', videoData.dislikes) | ||
119 | } | ||
120 | |||
121 | videoInstance.save(options).asCallback(function (err) { | ||
122 | return callback(err, t) | ||
123 | }) | ||
124 | }, | ||
125 | |||
126 | databaseUtils.commitTransaction | ||
127 | |||
128 | ], function (err, t) { | ||
129 | if (err) { | ||
130 | logger.debug('Cannot quick and dirty update the remote video.', { error: err }) | ||
131 | return databaseUtils.rollbackTransaction(err, t, finalCallback) | ||
132 | } | ||
133 | |||
134 | logger.info('Remote video %s quick and dirty updated', videoData.name) | ||
135 | return finalCallback(null) | ||
136 | }) | ||
137 | } | ||
138 | |||
65 | // Handle retries on fail | 139 | // Handle retries on fail |
66 | function addRemoteVideoRetryWrapper (videoToCreateData, fromPod, finalCallback) { | 140 | function addRemoteVideoRetryWrapper (videoToCreateData, fromPod, finalCallback) { |
67 | const options = { | 141 | const options = { |
diff --git a/server/controllers/api/videos.js b/server/controllers/api/videos.js index c936105e7..9f4bbb7b7 100644 --- a/server/controllers/api/videos.js +++ b/server/controllers/api/videos.js | |||
@@ -320,6 +320,22 @@ function updateVideo (req, res, finalCallback) { | |||
320 | 320 | ||
321 | function getVideo (req, res, next) { | 321 | function getVideo (req, res, next) { |
322 | const videoInstance = res.locals.video | 322 | const videoInstance = res.locals.video |
323 | |||
324 | if (videoInstance.isOwned()) { | ||
325 | // The increment is done directly in the database, not using the instance value | ||
326 | videoInstance.increment('views').asCallback(function (err) { | ||
327 | if (err) { | ||
328 | logger.error('Cannot add view to video %d.', videoInstance.id) | ||
329 | return | ||
330 | } | ||
331 | |||
332 | // FIXME: make a real view system | ||
333 | // For example, only add a view when a user watch a video during 30s etc | ||
334 | friends.quickAndDirtyUpdateVideoToFriends(videoInstance.id, constants.REQUEST_VIDEO_QADU_TYPES.VIEWS) | ||
335 | }) | ||
336 | } | ||
337 | |||
338 | // Do not wait the view system | ||
323 | res.json(videoInstance.toFormatedJSON()) | 339 | res.json(videoInstance.toFormatedJSON()) |
324 | } | 340 | } |
325 | 341 | ||
diff --git a/server/helpers/custom-validators/remote/videos.js b/server/helpers/custom-validators/remote/videos.js index ee68ebc10..2e9cf822e 100644 --- a/server/helpers/custom-validators/remote/videos.js +++ b/server/helpers/custom-validators/remote/videos.js | |||
@@ -1,5 +1,7 @@ | |||
1 | 'use strict' | 1 | 'use strict' |
2 | 2 | ||
3 | const has = require('lodash/has') | ||
4 | |||
3 | const constants = require('../../../initializers/constants') | 5 | const constants = require('../../../initializers/constants') |
4 | const videosValidators = require('../videos') | 6 | const videosValidators = require('../videos') |
5 | const miscValidators = require('../misc') | 7 | const miscValidators = require('../misc') |
@@ -7,7 +9,8 @@ const miscValidators = require('../misc') | |||
7 | const ENDPOINT_ACTIONS = constants.REQUEST_ENDPOINT_ACTIONS[constants.REQUEST_ENDPOINTS.VIDEOS] | 9 | const ENDPOINT_ACTIONS = constants.REQUEST_ENDPOINT_ACTIONS[constants.REQUEST_ENDPOINTS.VIDEOS] |
8 | 10 | ||
9 | const remoteVideosValidators = { | 11 | const remoteVideosValidators = { |
10 | isEachRemoteRequestVideosValid | 12 | isEachRemoteRequestVideosValid, |
13 | isEachRemoteRequestVideosQaduValid | ||
11 | } | 14 | } |
12 | 15 | ||
13 | function isEachRemoteRequestVideosValid (requests) { | 16 | function isEachRemoteRequestVideosValid (requests) { |
@@ -16,13 +19,13 @@ function isEachRemoteRequestVideosValid (requests) { | |||
16 | const video = request.data | 19 | const video = request.data |
17 | return ( | 20 | return ( |
18 | isRequestTypeAddValid(request.type) && | 21 | isRequestTypeAddValid(request.type) && |
19 | isCommonVideoAttrbiutesValid(video) && | 22 | isCommonVideoAttributesValid(video) && |
20 | videosValidators.isVideoAuthorValid(video.author) && | 23 | videosValidators.isVideoAuthorValid(video.author) && |
21 | videosValidators.isVideoThumbnailDataValid(video.thumbnailData) | 24 | videosValidators.isVideoThumbnailDataValid(video.thumbnailData) |
22 | ) || | 25 | ) || |
23 | ( | 26 | ( |
24 | isRequestTypeUpdateValid(request.type) && | 27 | isRequestTypeUpdateValid(request.type) && |
25 | isCommonVideoAttrbiutesValid(video) | 28 | isCommonVideoAttributesValid(video) |
26 | ) || | 29 | ) || |
27 | ( | 30 | ( |
28 | isRequestTypeRemoveValid(request.type) && | 31 | isRequestTypeRemoveValid(request.type) && |
@@ -37,13 +40,27 @@ function isEachRemoteRequestVideosValid (requests) { | |||
37 | }) | 40 | }) |
38 | } | 41 | } |
39 | 42 | ||
43 | function isEachRemoteRequestVideosQaduValid (requests) { | ||
44 | return miscValidators.isArray(requests) && | ||
45 | requests.every(function (request) { | ||
46 | const video = request.data | ||
47 | |||
48 | return ( | ||
49 | videosValidators.isVideoRemoteIdValid(video.remoteId) && | ||
50 | (has(video, 'views') === false || videosValidators.isVideoViewsValid) && | ||
51 | (has(video, 'likes') === false || videosValidators.isVideoLikesValid) && | ||
52 | (has(video, 'dislikes') === false || videosValidators.isVideoDislikesValid) | ||
53 | ) | ||
54 | }) | ||
55 | } | ||
56 | |||
40 | // --------------------------------------------------------------------------- | 57 | // --------------------------------------------------------------------------- |
41 | 58 | ||
42 | module.exports = remoteVideosValidators | 59 | module.exports = remoteVideosValidators |
43 | 60 | ||
44 | // --------------------------------------------------------------------------- | 61 | // --------------------------------------------------------------------------- |
45 | 62 | ||
46 | function isCommonVideoAttrbiutesValid (video) { | 63 | function isCommonVideoAttributesValid (video) { |
47 | return videosValidators.isVideoDateValid(video.createdAt) && | 64 | return videosValidators.isVideoDateValid(video.createdAt) && |
48 | videosValidators.isVideoDateValid(video.updatedAt) && | 65 | videosValidators.isVideoDateValid(video.updatedAt) && |
49 | videosValidators.isVideoDescriptionValid(video.description) && | 66 | videosValidators.isVideoDescriptionValid(video.description) && |
diff --git a/server/helpers/custom-validators/videos.js b/server/helpers/custom-validators/videos.js index e2d2c8e6d..1d844118b 100644 --- a/server/helpers/custom-validators/videos.js +++ b/server/helpers/custom-validators/videos.js | |||
@@ -22,7 +22,10 @@ const videosValidators = { | |||
22 | isVideoRemoteIdValid, | 22 | isVideoRemoteIdValid, |
23 | isVideoAbuseReasonValid, | 23 | isVideoAbuseReasonValid, |
24 | isVideoAbuseReporterUsernameValid, | 24 | isVideoAbuseReporterUsernameValid, |
25 | isVideoFile | 25 | isVideoFile, |
26 | isVideoViewsValid, | ||
27 | isVideoLikesValid, | ||
28 | isVideoDislikesValid | ||
26 | } | 29 | } |
27 | 30 | ||
28 | function isVideoAuthorValid (value) { | 31 | function isVideoAuthorValid (value) { |
@@ -82,6 +85,18 @@ function isVideoAbuseReporterUsernameValid (value) { | |||
82 | return usersValidators.isUserUsernameValid(value) | 85 | return usersValidators.isUserUsernameValid(value) |
83 | } | 86 | } |
84 | 87 | ||
88 | function isVideoViewsValid (value) { | ||
89 | return validator.isInt(value, { min: 0 }) | ||
90 | } | ||
91 | |||
92 | function isVideoLikesValid (value) { | ||
93 | return validator.isInt(value, { min: 0 }) | ||
94 | } | ||
95 | |||
96 | function isVideoDislikesValid (value) { | ||
97 | return validator.isInt(value, { min: 0 }) | ||
98 | } | ||
99 | |||
85 | function isVideoFile (value, files) { | 100 | function isVideoFile (value, files) { |
86 | // Should have files | 101 | // Should have files |
87 | if (!files) return false | 102 | if (!files) return false |
diff --git a/server/helpers/requests.js b/server/helpers/requests.js index 095b95e1c..427864117 100644 --- a/server/helpers/requests.js +++ b/server/helpers/requests.js | |||
@@ -58,6 +58,8 @@ function makeSecureRequest (params, callback) { | |||
58 | requestParams.json.data = params.data | 58 | requestParams.json.data = params.data |
59 | } | 59 | } |
60 | 60 | ||
61 | console.log(requestParams.json.data) | ||
62 | |||
61 | request.post(requestParams, callback) | 63 | request.post(requestParams, callback) |
62 | } | 64 | } |
63 | 65 | ||
diff --git a/server/initializers/constants.js b/server/initializers/constants.js index 821580893..668bfe56c 100644 --- a/server/initializers/constants.js +++ b/server/initializers/constants.js | |||
@@ -5,7 +5,7 @@ const path = require('path') | |||
5 | 5 | ||
6 | // --------------------------------------------------------------------------- | 6 | // --------------------------------------------------------------------------- |
7 | 7 | ||
8 | const LAST_MIGRATION_VERSION = 10 | 8 | const LAST_MIGRATION_VERSION = 15 |
9 | 9 | ||
10 | // --------------------------------------------------------------------------- | 10 | // --------------------------------------------------------------------------- |
11 | 11 | ||
@@ -24,7 +24,7 @@ const SEARCHABLE_COLUMNS = { | |||
24 | const SORTABLE_COLUMNS = { | 24 | const SORTABLE_COLUMNS = { |
25 | USERS: [ 'id', '-id', 'username', '-username', 'createdAt', '-createdAt' ], | 25 | USERS: [ 'id', '-id', 'username', '-username', 'createdAt', '-createdAt' ], |
26 | VIDEO_ABUSES: [ 'id', '-id', 'createdAt', '-createdAt' ], | 26 | VIDEO_ABUSES: [ 'id', '-id', 'createdAt', '-createdAt' ], |
27 | VIDEOS: [ 'name', '-name', 'duration', '-duration', 'createdAt', '-createdAt' ] | 27 | VIDEOS: [ 'name', '-name', 'duration', '-duration', 'createdAt', '-createdAt', 'views', '-views' ] |
28 | } | 28 | } |
29 | 29 | ||
30 | const OAUTH_LIFETIME = { | 30 | const OAUTH_LIFETIME = { |
@@ -116,11 +116,16 @@ const REQUESTS_LIMIT_PODS = 10 | |||
116 | // How many requests we send to a pod per interval | 116 | // How many requests we send to a pod per interval |
117 | const REQUESTS_LIMIT_PER_POD = 5 | 117 | const REQUESTS_LIMIT_PER_POD = 5 |
118 | 118 | ||
119 | const REQUESTS_VIDEO_QADU_LIMIT_PODS = 10 | ||
120 | // The QADU requests are not big | ||
121 | const REQUESTS_VIDEO_QADU_LIMIT_PER_POD = 50 | ||
122 | |||
119 | // Number of requests to retry for replay requests module | 123 | // Number of requests to retry for replay requests module |
120 | const RETRY_REQUESTS = 5 | 124 | const RETRY_REQUESTS = 5 |
121 | 125 | ||
122 | const REQUEST_ENDPOINTS = { | 126 | const REQUEST_ENDPOINTS = { |
123 | VIDEOS: 'videos' | 127 | VIDEOS: 'videos', |
128 | QADU: 'videos/qadu' | ||
124 | } | 129 | } |
125 | const REQUEST_ENDPOINT_ACTIONS = {} | 130 | const REQUEST_ENDPOINT_ACTIONS = {} |
126 | REQUEST_ENDPOINT_ACTIONS[REQUEST_ENDPOINTS.VIDEOS] = { | 131 | REQUEST_ENDPOINT_ACTIONS[REQUEST_ENDPOINTS.VIDEOS] = { |
@@ -130,6 +135,12 @@ REQUEST_ENDPOINT_ACTIONS[REQUEST_ENDPOINTS.VIDEOS] = { | |||
130 | REPORT_ABUSE: 'report-abuse' | 135 | REPORT_ABUSE: 'report-abuse' |
131 | } | 136 | } |
132 | 137 | ||
138 | const REQUEST_VIDEO_QADU_TYPES = { | ||
139 | LIKES: 'likes', | ||
140 | DISLIKES: 'dislikes', | ||
141 | VIEWS: 'views' | ||
142 | } | ||
143 | |||
133 | const REMOTE_SCHEME = { | 144 | const REMOTE_SCHEME = { |
134 | HTTP: 'https', | 145 | HTTP: 'https', |
135 | WS: 'wss' | 146 | WS: 'wss' |
@@ -199,10 +210,13 @@ module.exports = { | |||
199 | REMOTE_SCHEME, | 210 | REMOTE_SCHEME, |
200 | REQUEST_ENDPOINT_ACTIONS, | 211 | REQUEST_ENDPOINT_ACTIONS, |
201 | REQUEST_ENDPOINTS, | 212 | REQUEST_ENDPOINTS, |
213 | REQUEST_VIDEO_QADU_TYPES, | ||
202 | REQUESTS_IN_PARALLEL, | 214 | REQUESTS_IN_PARALLEL, |
203 | REQUESTS_INTERVAL, | 215 | REQUESTS_INTERVAL, |
204 | REQUESTS_LIMIT_PER_POD, | 216 | REQUESTS_LIMIT_PER_POD, |
205 | REQUESTS_LIMIT_PODS, | 217 | REQUESTS_LIMIT_PODS, |
218 | REQUESTS_VIDEO_QADU_LIMIT_PER_POD, | ||
219 | REQUESTS_VIDEO_QADU_LIMIT_PODS, | ||
206 | RETRY_REQUESTS, | 220 | RETRY_REQUESTS, |
207 | SEARCHABLE_COLUMNS, | 221 | SEARCHABLE_COLUMNS, |
208 | SIGNATURE_ALGORITHM, | 222 | SIGNATURE_ALGORITHM, |
diff --git a/server/initializers/migrations/0015-video-views.js b/server/initializers/migrations/0015-video-views.js new file mode 100644 index 000000000..ae49fe73c --- /dev/null +++ b/server/initializers/migrations/0015-video-views.js | |||
@@ -0,0 +1,19 @@ | |||
1 | 'use strict' | ||
2 | |||
3 | // utils = { transaction, queryInterface, sequelize, Sequelize } | ||
4 | exports.up = function (utils, finalCallback) { | ||
5 | const q = utils.queryInterface | ||
6 | const Sequelize = utils.Sequelize | ||
7 | |||
8 | const data = { | ||
9 | type: Sequelize.INTEGER, | ||
10 | allowNull: false, | ||
11 | defaultValue: 0 | ||
12 | } | ||
13 | |||
14 | q.addColumn('Videos', 'views', data, { transaction: utils.transaction }).asCallback(finalCallback) | ||
15 | } | ||
16 | |||
17 | exports.down = function (options, callback) { | ||
18 | throw new Error('Not implemented.') | ||
19 | } | ||
diff --git a/server/lib/base-request-scheduler.js b/server/lib/base-request-scheduler.js new file mode 100644 index 000000000..d15680c25 --- /dev/null +++ b/server/lib/base-request-scheduler.js | |||
@@ -0,0 +1,140 @@ | |||
1 | 'use strict' | ||
2 | |||
3 | const eachLimit = require('async/eachLimit') | ||
4 | |||
5 | const constants = require('../initializers/constants') | ||
6 | const db = require('../initializers/database') | ||
7 | const logger = require('../helpers/logger') | ||
8 | const requests = require('../helpers/requests') | ||
9 | |||
10 | module.exports = class BaseRequestScheduler { | ||
11 | |||
12 | constructor (options) { | ||
13 | this.lastRequestTimestamp = 0 | ||
14 | this.timer = null | ||
15 | } | ||
16 | |||
17 | activate () { | ||
18 | logger.info('Requests scheduler activated.') | ||
19 | this.lastRequestTimestamp = Date.now() | ||
20 | |||
21 | this.timer = setInterval(() => { | ||
22 | this.lastRequestTimestamp = Date.now() | ||
23 | this.makeRequests() | ||
24 | }, constants.REQUESTS_INTERVAL) | ||
25 | } | ||
26 | |||
27 | deactivate () { | ||
28 | logger.info('Requests scheduler deactivated.') | ||
29 | clearInterval(this.timer) | ||
30 | this.timer = null | ||
31 | } | ||
32 | |||
33 | forceSend () { | ||
34 | logger.info('Force requests scheduler sending.') | ||
35 | this.makeRequests() | ||
36 | } | ||
37 | |||
38 | remainingMilliSeconds () { | ||
39 | if (this.timer === null) return -1 | ||
40 | |||
41 | return constants.REQUESTS_INTERVAL - (Date.now() - this.lastRequestTimestamp) | ||
42 | } | ||
43 | |||
44 | // --------------------------------------------------------------------------- | ||
45 | |||
46 | // Make a requests to friends of a certain type | ||
47 | makeRequest (toPod, requestEndpoint, requestsToMake, callback) { | ||
48 | if (!callback) callback = function () {} | ||
49 | |||
50 | const params = { | ||
51 | toPod: toPod, | ||
52 | sign: true, // Prove our identity | ||
53 | method: 'POST', | ||
54 | path: '/api/' + constants.API_VERSION + '/remote/' + requestEndpoint, | ||
55 | data: requestsToMake // Requests we need to make | ||
56 | } | ||
57 | |||
58 | // Make multiple retry requests to all of pods | ||
59 | // The function fire some useful callbacks | ||
60 | requests.makeSecureRequest(params, (err, res) => { | ||
61 | if (err || (res.statusCode !== 200 && res.statusCode !== 201 && res.statusCode !== 204)) { | ||
62 | err = err ? err.message : 'Status code not 20x : ' + res.statusCode | ||
63 | logger.error('Error sending secure request to %s pod.', toPod.host, { error: err }) | ||
64 | |||
65 | return callback(false) | ||
66 | } | ||
67 | |||
68 | return callback(true) | ||
69 | }) | ||
70 | } | ||
71 | |||
72 | // Make all the requests of the scheduler | ||
73 | makeRequests () { | ||
74 | this.getRequestModel().listWithLimitAndRandom(this.limitPods, this.limitPerPod, (err, requests) => { | ||
75 | if (err) { | ||
76 | logger.error('Cannot get the list of "%s".', this.description, { err: err }) | ||
77 | return // Abort | ||
78 | } | ||
79 | |||
80 | // If there are no requests, abort | ||
81 | if (requests.length === 0) { | ||
82 | logger.info('No "%s" to make.', this.description) | ||
83 | return | ||
84 | } | ||
85 | |||
86 | // We want to group requests by destinations pod and endpoint | ||
87 | const requestsToMakeGrouped = this.buildRequestObjects(requests) | ||
88 | |||
89 | logger.info('Making "%s" to friends.', this.description) | ||
90 | |||
91 | const goodPods = [] | ||
92 | const badPods = [] | ||
93 | |||
94 | eachLimit(Object.keys(requestsToMakeGrouped), constants.REQUESTS_IN_PARALLEL, (hashKey, callbackEach) => { | ||
95 | const requestToMake = requestsToMakeGrouped[hashKey] | ||
96 | const toPod = requestToMake.toPod | ||
97 | |||
98 | // Maybe the pod is not our friend anymore so simply remove it | ||
99 | if (!toPod) { | ||
100 | const requestIdsToDelete = requestToMake.ids | ||
101 | |||
102 | logger.info('Removing %d "%s" of unexisting pod %s.', requestIdsToDelete.length, this.description, requestToMake.toPod.id) | ||
103 | return this.getRequestToPodModel().removePodOf(requestIdsToDelete, requestToMake.toPod.id, callbackEach) | ||
104 | } | ||
105 | |||
106 | this.makeRequest(toPod, requestToMake.endpoint, requestToMake.datas, (success) => { | ||
107 | if (success === false) { | ||
108 | badPods.push(requestToMake.toPod.id) | ||
109 | return callbackEach() | ||
110 | } | ||
111 | |||
112 | logger.debug('Removing requests for pod %s.', requestToMake.toPod.id, { requestsIds: requestToMake.ids }) | ||
113 | goodPods.push(requestToMake.toPod.id) | ||
114 | |||
115 | // Remove the pod id of these request ids | ||
116 | this.getRequestToPodModel().removeByRequestIdsAndPod(requestToMake.ids, requestToMake.toPod.id, callbackEach) | ||
117 | |||
118 | this.afterRequestHook() | ||
119 | }) | ||
120 | }, () => { | ||
121 | // All the requests were made, we update the pods score | ||
122 | db.Pod.updatePodsScore(goodPods, badPods) | ||
123 | |||
124 | this.afterRequestsHook() | ||
125 | }) | ||
126 | }) | ||
127 | } | ||
128 | |||
129 | flush (callback) { | ||
130 | this.getRequestModel().removeAll(callback) | ||
131 | } | ||
132 | |||
133 | afterRequestHook () { | ||
134 | // Nothing to do, let children reimplement it | ||
135 | } | ||
136 | |||
137 | afterRequestsHook () { | ||
138 | // Nothing to do, let children reimplement it | ||
139 | } | ||
140 | } | ||
diff --git a/server/lib/friends.js b/server/lib/friends.js index d53ab4553..424a30801 100644 --- a/server/lib/friends.js +++ b/server/lib/friends.js | |||
@@ -12,15 +12,19 @@ const logger = require('../helpers/logger') | |||
12 | const peertubeCrypto = require('../helpers/peertube-crypto') | 12 | const peertubeCrypto = require('../helpers/peertube-crypto') |
13 | const requests = require('../helpers/requests') | 13 | const requests = require('../helpers/requests') |
14 | const RequestScheduler = require('./request-scheduler') | 14 | const RequestScheduler = require('./request-scheduler') |
15 | const RequestVideoQaduScheduler = require('./request-video-qadu-scheduler') | ||
15 | 16 | ||
16 | const ENDPOINT_ACTIONS = constants.REQUEST_ENDPOINT_ACTIONS[constants.REQUEST_ENDPOINTS.VIDEOS] | 17 | const ENDPOINT_ACTIONS = constants.REQUEST_ENDPOINT_ACTIONS[constants.REQUEST_ENDPOINTS.VIDEOS] |
18 | |||
17 | const requestScheduler = new RequestScheduler() | 19 | const requestScheduler = new RequestScheduler() |
20 | const requestSchedulerVideoQadu = new RequestVideoQaduScheduler() | ||
18 | 21 | ||
19 | const friends = { | 22 | const friends = { |
20 | activate, | 23 | activate, |
21 | addVideoToFriends, | 24 | addVideoToFriends, |
22 | updateVideoToFriends, | 25 | updateVideoToFriends, |
23 | reportAbuseVideoToFriend, | 26 | reportAbuseVideoToFriend, |
27 | quickAndDirtyUpdateVideoToFriends, | ||
24 | hasFriends, | 28 | hasFriends, |
25 | makeFriends, | 29 | makeFriends, |
26 | quitFriends, | 30 | quitFriends, |
@@ -30,6 +34,7 @@ const friends = { | |||
30 | 34 | ||
31 | function activate () { | 35 | function activate () { |
32 | requestScheduler.activate() | 36 | requestScheduler.activate() |
37 | requestSchedulerVideoQadu.activate() | ||
33 | } | 38 | } |
34 | 39 | ||
35 | function addVideoToFriends (videoData, transaction, callback) { | 40 | function addVideoToFriends (videoData, transaction, callback) { |
@@ -71,6 +76,15 @@ function reportAbuseVideoToFriend (reportData, video) { | |||
71 | createRequest(options) | 76 | createRequest(options) |
72 | } | 77 | } |
73 | 78 | ||
79 | function quickAndDirtyUpdateVideoToFriends (videoId, type, transaction, callback) { | ||
80 | const options = { | ||
81 | videoId, | ||
82 | type, | ||
83 | transaction | ||
84 | } | ||
85 | return createVideoQaduRequest(options, callback) | ||
86 | } | ||
87 | |||
74 | function hasFriends (callback) { | 88 | function hasFriends (callback) { |
75 | db.Pod.countAll(function (err, count) { | 89 | db.Pod.countAll(function (err, count) { |
76 | if (err) return callback(err) | 90 | if (err) return callback(err) |
@@ -110,7 +124,11 @@ function quitFriends (callback) { | |||
110 | 124 | ||
111 | waterfall([ | 125 | waterfall([ |
112 | function flushRequests (callbackAsync) { | 126 | function flushRequests (callbackAsync) { |
113 | requestScheduler.flush(callbackAsync) | 127 | requestScheduler.flush(err => callbackAsync(err)) |
128 | }, | ||
129 | |||
130 | function flushVideoQaduRequests (callbackAsync) { | ||
131 | requestSchedulerVideoQadu.flush(err => callbackAsync(err)) | ||
114 | }, | 132 | }, |
115 | 133 | ||
116 | function getPodsList (callbackAsync) { | 134 | function getPodsList (callbackAsync) { |
@@ -310,6 +328,12 @@ function createRequest (options, callback) { | |||
310 | }) | 328 | }) |
311 | } | 329 | } |
312 | 330 | ||
331 | function createVideoQaduRequest (options, callback) { | ||
332 | if (!callback) callback = function () {} | ||
333 | |||
334 | requestSchedulerVideoQadu.createRequest(options, callback) | ||
335 | } | ||
336 | |||
313 | function isMe (host) { | 337 | function isMe (host) { |
314 | return host === constants.CONFIG.WEBSERVER.HOST | 338 | return host === constants.CONFIG.WEBSERVER.HOST |
315 | } | 339 | } |
diff --git a/server/lib/request-scheduler.js b/server/lib/request-scheduler.js index 28dabe339..6b6535519 100644 --- a/server/lib/request-scheduler.js +++ b/server/lib/request-scheduler.js | |||
@@ -1,44 +1,54 @@ | |||
1 | 'use strict' | 1 | 'use strict' |
2 | 2 | ||
3 | const eachLimit = require('async/eachLimit') | ||
4 | |||
5 | const constants = require('../initializers/constants') | 3 | const constants = require('../initializers/constants') |
4 | const BaseRequestScheduler = require('./base-request-scheduler') | ||
6 | const db = require('../initializers/database') | 5 | const db = require('../initializers/database') |
7 | const logger = require('../helpers/logger') | 6 | const logger = require('../helpers/logger') |
8 | const requests = require('../helpers/requests') | ||
9 | 7 | ||
10 | module.exports = class RequestScheduler { | 8 | module.exports = class RequestScheduler extends BaseRequestScheduler { |
11 | 9 | ||
12 | constructor () { | 10 | constructor () { |
13 | this.lastRequestTimestamp = 0 | 11 | super() |
14 | this.timer = null | ||
15 | } | ||
16 | 12 | ||
17 | activate () { | 13 | // We limit the size of the requests |
18 | logger.info('Requests scheduler activated.') | 14 | this.limitPods = constants.REQUESTS_LIMIT_PODS |
19 | this.lastRequestTimestamp = Date.now() | 15 | this.limitPerPod = constants.REQUESTS_LIMIT_PER_POD |
20 | 16 | ||
21 | this.timer = setInterval(() => { | 17 | this.description = 'requests' |
22 | this.lastRequestTimestamp = Date.now() | ||
23 | this.makeRequests() | ||
24 | }, constants.REQUESTS_INTERVAL) | ||
25 | } | 18 | } |
26 | 19 | ||
27 | deactivate () { | 20 | getRequestModel () { |
28 | logger.info('Requests scheduler deactivated.') | 21 | return db.Request |
29 | clearInterval(this.timer) | ||
30 | this.timer = null | ||
31 | } | 22 | } |
32 | 23 | ||
33 | forceSend () { | 24 | getRequestToPodModel () { |
34 | logger.info('Force requests scheduler sending.') | 25 | return db.RequestToPod |
35 | this.makeRequests() | ||
36 | } | 26 | } |
37 | 27 | ||
38 | remainingMilliSeconds () { | 28 | buildRequestObjects (requests) { |
39 | if (this.timer === null) return -1 | 29 | const requestsToMakeGrouped = {} |
30 | |||
31 | Object.keys(requests).forEach(toPodId => { | ||
32 | requests[toPodId].forEach(data => { | ||
33 | const request = data.request | ||
34 | const pod = data.pod | ||
35 | const hashKey = toPodId + request.endpoint | ||
36 | |||
37 | if (!requestsToMakeGrouped[hashKey]) { | ||
38 | requestsToMakeGrouped[hashKey] = { | ||
39 | toPod: pod, | ||
40 | endpoint: request.endpoint, | ||
41 | ids: [], // request ids, to delete them from the DB in the future | ||
42 | datas: [] // requests data, | ||
43 | } | ||
44 | } | ||
45 | |||
46 | requestsToMakeGrouped[hashKey].ids.push(request.id) | ||
47 | requestsToMakeGrouped[hashKey].datas.push(request.request) | ||
48 | }) | ||
49 | }) | ||
40 | 50 | ||
41 | return constants.REQUESTS_INTERVAL - (Date.now() - this.lastRequestTimestamp) | 51 | return requestsToMakeGrouped |
42 | } | 52 | } |
43 | 53 | ||
44 | // { type, endpoint, data, toIds, transaction } | 54 | // { type, endpoint, data, toIds, transaction } |
@@ -79,122 +89,10 @@ module.exports = class RequestScheduler { | |||
79 | 89 | ||
80 | // --------------------------------------------------------------------------- | 90 | // --------------------------------------------------------------------------- |
81 | 91 | ||
82 | // Make all the requests of the scheduler | 92 | afterRequestsHook () { |
83 | makeRequests () { | 93 | // Flush requests with no pod |
84 | // We limit the size of the requests | 94 | this.getRequestModel().removeWithEmptyTo(err => { |
85 | // We don't want to stuck with the same failing requests so we get a random list | 95 | if (err) logger.error('Error when removing requests with no pods.', { error: err }) |
86 | db.Request.listWithLimitAndRandom(constants.REQUESTS_LIMIT_PODS, constants.REQUESTS_LIMIT_PER_POD, (err, requests) => { | ||
87 | if (err) { | ||
88 | logger.error('Cannot get the list of requests.', { err: err }) | ||
89 | return // Abort | ||
90 | } | ||
91 | |||
92 | // If there are no requests, abort | ||
93 | if (requests.length === 0) { | ||
94 | logger.info('No requests to make.') | ||
95 | return | ||
96 | } | ||
97 | |||
98 | // We want to group requests by destinations pod and endpoint | ||
99 | const requestsToMakeGrouped = this.buildRequestObjects(requests) | ||
100 | |||
101 | logger.info('Making requests to friends.') | ||
102 | |||
103 | const goodPods = [] | ||
104 | const badPods = [] | ||
105 | |||
106 | eachLimit(Object.keys(requestsToMakeGrouped), constants.REQUESTS_IN_PARALLEL, (hashKey, callbackEach) => { | ||
107 | const requestToMake = requestsToMakeGrouped[hashKey] | ||
108 | const toPod = requestToMake.toPod | ||
109 | |||
110 | // Maybe the pod is not our friend anymore so simply remove it | ||
111 | if (!toPod) { | ||
112 | const requestIdsToDelete = requestToMake.ids | ||
113 | |||
114 | logger.info('Removing %d requests of unexisting pod %s.', requestIdsToDelete.length, requestToMake.toPod.id) | ||
115 | return db.RequestToPod.removePodOf(requestIdsToDelete, requestToMake.toPod.id, callbackEach) | ||
116 | } | ||
117 | |||
118 | this.makeRequest(toPod, requestToMake.endpoint, requestToMake.datas, (success) => { | ||
119 | if (success === false) { | ||
120 | badPods.push(requestToMake.toPod.id) | ||
121 | return callbackEach() | ||
122 | } | ||
123 | |||
124 | logger.debug('Removing requests for pod %s.', requestToMake.toPod.id, { requestsIds: requestToMake.ids }) | ||
125 | goodPods.push(requestToMake.toPod.id) | ||
126 | |||
127 | // Remove the pod id of these request ids | ||
128 | db.RequestToPod.removePodOf(requestToMake.ids, requestToMake.toPod.id, callbackEach) | ||
129 | }) | ||
130 | }, () => { | ||
131 | // All the requests were made, we update the pods score | ||
132 | db.Request.updatePodsScore(goodPods, badPods) | ||
133 | // Flush requests with no pod | ||
134 | db.Request.removeWithEmptyTo(err => { | ||
135 | if (err) logger.error('Error when removing requests with no pods.', { error: err }) | ||
136 | }) | ||
137 | }) | ||
138 | }) | ||
139 | } | ||
140 | |||
141 | // Make a requests to friends of a certain type | ||
142 | makeRequest (toPod, requestEndpoint, requestsToMake, callback) { | ||
143 | if (!callback) callback = function () {} | ||
144 | |||
145 | const params = { | ||
146 | toPod: toPod, | ||
147 | sign: true, // Prove our identity | ||
148 | method: 'POST', | ||
149 | path: '/api/' + constants.API_VERSION + '/remote/' + requestEndpoint, | ||
150 | data: requestsToMake // Requests we need to make | ||
151 | } | ||
152 | |||
153 | // Make multiple retry requests to all of pods | ||
154 | // The function fire some useful callbacks | ||
155 | requests.makeSecureRequest(params, (err, res) => { | ||
156 | if (err || (res.statusCode !== 200 && res.statusCode !== 201 && res.statusCode !== 204)) { | ||
157 | err = err ? err.message : 'Status code not 20x : ' + res.statusCode | ||
158 | logger.error('Error sending secure request to %s pod.', toPod.host, { error: err }) | ||
159 | |||
160 | return callback(false) | ||
161 | } | ||
162 | |||
163 | return callback(true) | ||
164 | }) | ||
165 | } | ||
166 | |||
167 | buildRequestObjects (requests) { | ||
168 | const requestsToMakeGrouped = {} | ||
169 | |||
170 | Object.keys(requests).forEach(toPodId => { | ||
171 | requests[toPodId].forEach(data => { | ||
172 | const request = data.request | ||
173 | const pod = data.pod | ||
174 | const hashKey = toPodId + request.endpoint | ||
175 | |||
176 | if (!requestsToMakeGrouped[hashKey]) { | ||
177 | requestsToMakeGrouped[hashKey] = { | ||
178 | toPod: pod, | ||
179 | endpoint: request.endpoint, | ||
180 | ids: [], // request ids, to delete them from the DB in the future | ||
181 | datas: [] // requests data, | ||
182 | } | ||
183 | } | ||
184 | |||
185 | requestsToMakeGrouped[hashKey].ids.push(request.id) | ||
186 | requestsToMakeGrouped[hashKey].datas.push(request.request) | ||
187 | }) | ||
188 | }) | ||
189 | |||
190 | return requestsToMakeGrouped | ||
191 | } | ||
192 | |||
193 | flush (callback) { | ||
194 | db.Request.removeAll(err => { | ||
195 | if (err) logger.error('Cannot flush the requests.', { error: err }) | ||
196 | |||
197 | return callback(err) | ||
198 | }) | 96 | }) |
199 | } | 97 | } |
200 | } | 98 | } |
diff --git a/server/lib/request-video-qadu-scheduler.js b/server/lib/request-video-qadu-scheduler.js new file mode 100644 index 000000000..401b2fb44 --- /dev/null +++ b/server/lib/request-video-qadu-scheduler.js | |||
@@ -0,0 +1,116 @@ | |||
1 | 'use strict' | ||
2 | |||
3 | const BaseRequestScheduler = require('./base-request-scheduler') | ||
4 | const constants = require('../initializers/constants') | ||
5 | const db = require('../initializers/database') | ||
6 | const logger = require('../helpers/logger') | ||
7 | |||
8 | module.exports = class RequestVideoQaduScheduler extends BaseRequestScheduler { | ||
9 | |||
10 | constructor () { | ||
11 | super() | ||
12 | |||
13 | // We limit the size of the requests | ||
14 | this.limitPods = constants.REQUESTS_VIDEO_QADU_LIMIT_PODS | ||
15 | this.limitPerPod = constants.REQUESTS_VIDEO_QADU_LIMIT_PODS | ||
16 | |||
17 | this.description = 'video QADU requests' | ||
18 | } | ||
19 | |||
20 | getRequestModel () { | ||
21 | return db.RequestVideoQadu | ||
22 | } | ||
23 | |||
24 | getRequestToPodModel () { | ||
25 | return db.RequestVideoQadu | ||
26 | } | ||
27 | |||
28 | buildRequestObjects (requests) { | ||
29 | const requestsToMakeGrouped = {} | ||
30 | |||
31 | Object.keys(requests).forEach(toPodId => { | ||
32 | requests[toPodId].forEach(data => { | ||
33 | const request = data.request | ||
34 | const video = data.video | ||
35 | const pod = data.pod | ||
36 | const hashKey = toPodId | ||
37 | |||
38 | if (!requestsToMakeGrouped[hashKey]) { | ||
39 | requestsToMakeGrouped[hashKey] = { | ||
40 | toPod: pod, | ||
41 | endpoint: constants.REQUEST_ENDPOINTS.QADU, | ||
42 | ids: [], // request ids, to delete them from the DB in the future | ||
43 | datas: [], // requests data | ||
44 | videos: {} | ||
45 | } | ||
46 | } | ||
47 | |||
48 | if (!requestsToMakeGrouped[hashKey].videos[video.id]) { | ||
49 | requestsToMakeGrouped[hashKey].videos[video.id] = {} | ||
50 | } | ||
51 | |||
52 | const videoData = requestsToMakeGrouped[hashKey].videos[video.id] | ||
53 | |||
54 | switch (request.type) { | ||
55 | case constants.REQUEST_VIDEO_QADU_TYPES.LIKES: | ||
56 | videoData.likes = video.likes | ||
57 | break | ||
58 | |||
59 | case constants.REQUEST_VIDEO_QADU_TYPES.DISLIKES: | ||
60 | videoData.likes = video.dislikes | ||
61 | break | ||
62 | |||
63 | case constants.REQUEST_VIDEO_QADU_TYPES.VIEWS: | ||
64 | videoData.views = video.views | ||
65 | break | ||
66 | |||
67 | default: | ||
68 | logger.error('Unknown request video QADU type %s.', request.type) | ||
69 | return | ||
70 | } | ||
71 | |||
72 | // Do not forget the remoteId so the remote pod can identify the video | ||
73 | videoData.remoteId = video.id | ||
74 | requestsToMakeGrouped[hashKey].ids.push(request.id) | ||
75 | requestsToMakeGrouped[hashKey].videos[video.id] = videoData | ||
76 | }) | ||
77 | }) | ||
78 | |||
79 | Object.keys(requestsToMakeGrouped).forEach(hashKey => { | ||
80 | Object.keys(requestsToMakeGrouped[hashKey].videos).forEach(videoId => { | ||
81 | const videoData = requestsToMakeGrouped[hashKey].videos[videoId] | ||
82 | |||
83 | requestsToMakeGrouped[hashKey].datas.push({ | ||
84 | data: videoData | ||
85 | }) | ||
86 | }) | ||
87 | |||
88 | // We don't need it anymore, it was just to build our datas array | ||
89 | delete requestsToMakeGrouped[hashKey].videos | ||
90 | }) | ||
91 | |||
92 | return requestsToMakeGrouped | ||
93 | } | ||
94 | |||
95 | // { type, videoId, transaction? } | ||
96 | createRequest (options, callback) { | ||
97 | const type = options.type | ||
98 | const videoId = options.videoId | ||
99 | const transaction = options.transaction | ||
100 | |||
101 | const dbRequestOptions = {} | ||
102 | if (transaction) dbRequestOptions.transaction = transaction | ||
103 | |||
104 | // Send the update to all our friends | ||
105 | db.Pod.listAllIds(options.transaction, function (err, podIds) { | ||
106 | if (err) return callback(err) | ||
107 | |||
108 | const queries = [] | ||
109 | podIds.forEach(podId => { | ||
110 | queries.push({ type, videoId, podId }) | ||
111 | }) | ||
112 | |||
113 | return db.RequestVideoQadu.bulkCreate(queries, dbRequestOptions).asCallback(callback) | ||
114 | }) | ||
115 | } | ||
116 | } | ||
diff --git a/server/middlewares/validators/remote/videos.js b/server/middlewares/validators/remote/videos.js index cf9925b6c..ddc274c45 100644 --- a/server/middlewares/validators/remote/videos.js +++ b/server/middlewares/validators/remote/videos.js | |||
@@ -4,7 +4,8 @@ const checkErrors = require('../utils').checkErrors | |||
4 | const logger = require('../../../helpers/logger') | 4 | const logger = require('../../../helpers/logger') |
5 | 5 | ||
6 | const validatorsRemoteVideos = { | 6 | const validatorsRemoteVideos = { |
7 | remoteVideos | 7 | remoteVideos, |
8 | remoteQaduVideos | ||
8 | } | 9 | } |
9 | 10 | ||
10 | function remoteVideos (req, res, next) { | 11 | function remoteVideos (req, res, next) { |
@@ -15,6 +16,14 @@ function remoteVideos (req, res, next) { | |||
15 | checkErrors(req, res, next) | 16 | checkErrors(req, res, next) |
16 | } | 17 | } |
17 | 18 | ||
19 | function remoteQaduVideos (req, res, next) { | ||
20 | req.checkBody('data').isEachRemoteRequestVideosQaduValid() | ||
21 | |||
22 | logger.debug('Checking remoteVideosQadu parameters', { parameters: req.body }) | ||
23 | |||
24 | checkErrors(req, res, next) | ||
25 | } | ||
26 | |||
18 | // --------------------------------------------------------------------------- | 27 | // --------------------------------------------------------------------------- |
19 | 28 | ||
20 | module.exports = validatorsRemoteVideos | 29 | module.exports = validatorsRemoteVideos |
diff --git a/server/models/pod.js b/server/models/pod.js index 79afb737a..14814708e 100644 --- a/server/models/pod.js +++ b/server/models/pod.js | |||
@@ -1,8 +1,11 @@ | |||
1 | 'use strict' | 1 | 'use strict' |
2 | 2 | ||
3 | const each = require('async/each') | ||
3 | const map = require('lodash/map') | 4 | const map = require('lodash/map') |
5 | const waterfall = require('async/waterfall') | ||
4 | 6 | ||
5 | const constants = require('../initializers/constants') | 7 | const constants = require('../initializers/constants') |
8 | const logger = require('../helpers/logger') | ||
6 | const customPodsValidators = require('../helpers/custom-validators').pods | 9 | const customPodsValidators = require('../helpers/custom-validators').pods |
7 | 10 | ||
8 | // --------------------------------------------------------------------------- | 11 | // --------------------------------------------------------------------------- |
@@ -62,6 +65,7 @@ module.exports = function (sequelize, DataTypes) { | |||
62 | listBadPods, | 65 | listBadPods, |
63 | load, | 66 | load, |
64 | loadByHost, | 67 | loadByHost, |
68 | updatePodsScore, | ||
65 | removeAll | 69 | removeAll |
66 | }, | 70 | }, |
67 | instanceMethods: { | 71 | instanceMethods: { |
@@ -144,7 +148,7 @@ function listAllIds (transaction, callback) { | |||
144 | }) | 148 | }) |
145 | } | 149 | } |
146 | 150 | ||
147 | function listRandomPodIdsWithRequest (limit, callback) { | 151 | function listRandomPodIdsWithRequest (limit, tableRequestPod, callback) { |
148 | const self = this | 152 | const self = this |
149 | 153 | ||
150 | self.count().asCallback(function (err, count) { | 154 | self.count().asCallback(function (err, count) { |
@@ -166,7 +170,7 @@ function listRandomPodIdsWithRequest (limit, callback) { | |||
166 | where: { | 170 | where: { |
167 | id: { | 171 | id: { |
168 | $in: [ | 172 | $in: [ |
169 | this.sequelize.literal('SELECT "podId" FROM "RequestToPods"') | 173 | this.sequelize.literal('SELECT "podId" FROM "' + tableRequestPod + '"') |
170 | ] | 174 | ] |
171 | } | 175 | } |
172 | } | 176 | } |
@@ -207,3 +211,58 @@ function loadByHost (host, callback) { | |||
207 | function removeAll (callback) { | 211 | function removeAll (callback) { |
208 | return this.destroy().asCallback(callback) | 212 | return this.destroy().asCallback(callback) |
209 | } | 213 | } |
214 | |||
215 | function updatePodsScore (goodPods, badPods) { | ||
216 | const self = this | ||
217 | |||
218 | logger.info('Updating %d good pods and %d bad pods scores.', goodPods.length, badPods.length) | ||
219 | |||
220 | if (goodPods.length !== 0) { | ||
221 | this.incrementScores(goodPods, constants.PODS_SCORE.BONUS, function (err) { | ||
222 | if (err) logger.error('Cannot increment scores of good pods.', { error: err }) | ||
223 | }) | ||
224 | } | ||
225 | |||
226 | if (badPods.length !== 0) { | ||
227 | this.incrementScores(badPods, constants.PODS_SCORE.MALUS, function (err) { | ||
228 | if (err) logger.error('Cannot decrement scores of bad pods.', { error: err }) | ||
229 | removeBadPods.call(self) | ||
230 | }) | ||
231 | } | ||
232 | } | ||
233 | |||
234 | // --------------------------------------------------------------------------- | ||
235 | |||
236 | // Remove pods with a score of 0 (too many requests where they were unreachable) | ||
237 | function removeBadPods () { | ||
238 | const self = this | ||
239 | |||
240 | waterfall([ | ||
241 | function findBadPods (callback) { | ||
242 | self.sequelize.models.Pod.listBadPods(function (err, pods) { | ||
243 | if (err) { | ||
244 | logger.error('Cannot find bad pods.', { error: err }) | ||
245 | return callback(err) | ||
246 | } | ||
247 | |||
248 | return callback(null, pods) | ||
249 | }) | ||
250 | }, | ||
251 | |||
252 | function removeTheseBadPods (pods, callback) { | ||
253 | each(pods, function (pod, callbackEach) { | ||
254 | pod.destroy().asCallback(callbackEach) | ||
255 | }, function (err) { | ||
256 | return callback(err, pods.length) | ||
257 | }) | ||
258 | } | ||
259 | ], function (err, numberOfPodsRemoved) { | ||
260 | if (err) { | ||
261 | logger.error('Cannot remove bad pods.', { error: err }) | ||
262 | } else if (numberOfPodsRemoved) { | ||
263 | logger.info('Removed %d pods.', numberOfPodsRemoved) | ||
264 | } else { | ||
265 | logger.info('No need to remove bad pods.') | ||
266 | } | ||
267 | }) | ||
268 | } | ||
diff --git a/server/models/request-to-pod.js b/server/models/request-to-pod.js index f42a53458..0e01a842e 100644 --- a/server/models/request-to-pod.js +++ b/server/models/request-to-pod.js | |||
@@ -17,7 +17,7 @@ module.exports = function (sequelize, DataTypes) { | |||
17 | } | 17 | } |
18 | ], | 18 | ], |
19 | classMethods: { | 19 | classMethods: { |
20 | removePodOf | 20 | removeByRequestIdsAndPod |
21 | } | 21 | } |
22 | }) | 22 | }) |
23 | 23 | ||
@@ -26,7 +26,7 @@ module.exports = function (sequelize, DataTypes) { | |||
26 | 26 | ||
27 | // --------------------------------------------------------------------------- | 27 | // --------------------------------------------------------------------------- |
28 | 28 | ||
29 | function removePodOf (requestsIds, podId, callback) { | 29 | function removeByRequestIdsAndPod (requestsIds, podId, callback) { |
30 | if (!callback) callback = function () {} | 30 | if (!callback) callback = function () {} |
31 | 31 | ||
32 | const query = { | 32 | const query = { |
diff --git a/server/models/request-video-qadu.js b/server/models/request-video-qadu.js new file mode 100644 index 000000000..7010fc992 --- /dev/null +++ b/server/models/request-video-qadu.js | |||
@@ -0,0 +1,154 @@ | |||
1 | 'use strict' | ||
2 | |||
3 | /* | ||
4 | Request Video for Quick And Dirty Updates like: | ||
5 | - views | ||
6 | - likes | ||
7 | - dislikes | ||
8 | |||
9 | We can't put it in the same system than basic requests for efficiency. | ||
10 | Moreover we don't want to slow down the basic requests with a lot of views/likes/dislikes requests. | ||
11 | So we put it an independant request scheduler. | ||
12 | */ | ||
13 | |||
14 | const values = require('lodash/values') | ||
15 | |||
16 | const constants = require('../initializers/constants') | ||
17 | |||
18 | // --------------------------------------------------------------------------- | ||
19 | |||
20 | module.exports = function (sequelize, DataTypes) { | ||
21 | const RequestVideoQadu = sequelize.define('RequestVideoQadu', | ||
22 | { | ||
23 | type: { | ||
24 | type: DataTypes.ENUM(values(constants.REQUEST_VIDEO_QADU_TYPES)), | ||
25 | allowNull: false | ||
26 | } | ||
27 | }, | ||
28 | { | ||
29 | timestamps: false, | ||
30 | indexes: [ | ||
31 | { | ||
32 | fields: [ 'podId' ] | ||
33 | }, | ||
34 | { | ||
35 | fields: [ 'videoId' ] | ||
36 | } | ||
37 | ], | ||
38 | classMethods: { | ||
39 | associate, | ||
40 | |||
41 | listWithLimitAndRandom, | ||
42 | |||
43 | countTotalRequests, | ||
44 | removeAll, | ||
45 | removeByRequestIdsAndPod | ||
46 | } | ||
47 | } | ||
48 | ) | ||
49 | |||
50 | return RequestVideoQadu | ||
51 | } | ||
52 | |||
53 | // ------------------------------ STATICS ------------------------------ | ||
54 | |||
55 | function associate (models) { | ||
56 | this.belongsTo(models.Pod, { | ||
57 | foreignKey: { | ||
58 | name: 'podId', | ||
59 | allowNull: false | ||
60 | }, | ||
61 | onDelete: 'CASCADE' | ||
62 | }) | ||
63 | |||
64 | this.belongsTo(models.Video, { | ||
65 | foreignKey: { | ||
66 | name: 'videoId', | ||
67 | allowNull: false | ||
68 | }, | ||
69 | onDelete: 'CASCADE' | ||
70 | }) | ||
71 | } | ||
72 | |||
73 | function countTotalRequests (callback) { | ||
74 | const query = { | ||
75 | include: [ this.sequelize.models.Pod ] | ||
76 | } | ||
77 | |||
78 | return this.count(query).asCallback(callback) | ||
79 | } | ||
80 | |||
81 | function listWithLimitAndRandom (limitPods, limitRequestsPerPod, callback) { | ||
82 | const self = this | ||
83 | const Pod = this.sequelize.models.Pod | ||
84 | |||
85 | Pod.listRandomPodIdsWithRequest(limitPods, 'RequestVideoQadus', function (err, podIds) { | ||
86 | if (err) return callback(err) | ||
87 | |||
88 | // We don't have friends that have requests | ||
89 | if (podIds.length === 0) return callback(null, []) | ||
90 | |||
91 | const query = { | ||
92 | include: [ | ||
93 | { | ||
94 | model: self.sequelize.models.Pod, | ||
95 | where: { | ||
96 | id: { | ||
97 | $in: podIds | ||
98 | } | ||
99 | } | ||
100 | }, | ||
101 | { | ||
102 | model: self.sequelize.models.Video | ||
103 | } | ||
104 | ] | ||
105 | } | ||
106 | |||
107 | self.findAll(query).asCallback(function (err, requests) { | ||
108 | if (err) return callback(err) | ||
109 | |||
110 | const requestsGrouped = groupAndTruncateRequests(requests, limitRequestsPerPod) | ||
111 | return callback(err, requestsGrouped) | ||
112 | }) | ||
113 | }) | ||
114 | } | ||
115 | |||
116 | function removeByRequestIdsAndPod (ids, podId, callback) { | ||
117 | const query = { | ||
118 | where: { | ||
119 | id: { | ||
120 | $in: ids | ||
121 | }, | ||
122 | podId | ||
123 | } | ||
124 | } | ||
125 | |||
126 | this.destroy(query).asCallback(callback) | ||
127 | } | ||
128 | |||
129 | function removeAll (callback) { | ||
130 | // Delete all requests | ||
131 | this.truncate({ cascade: true }).asCallback(callback) | ||
132 | } | ||
133 | |||
134 | // --------------------------------------------------------------------------- | ||
135 | |||
136 | function groupAndTruncateRequests (requests, limitRequestsPerPod) { | ||
137 | const requestsGrouped = {} | ||
138 | |||
139 | requests.forEach(function (request) { | ||
140 | const pod = request.Pod | ||
141 | |||
142 | if (!requestsGrouped[pod.id]) requestsGrouped[pod.id] = [] | ||
143 | |||
144 | if (requestsGrouped[pod.id].length < limitRequestsPerPod) { | ||
145 | requestsGrouped[pod.id].push({ | ||
146 | request: request, | ||
147 | video: request.Video, | ||
148 | pod | ||
149 | }) | ||
150 | } | ||
151 | }) | ||
152 | |||
153 | return requestsGrouped | ||
154 | } | ||
diff --git a/server/models/request.js b/server/models/request.js index ca616d130..de73501fc 100644 --- a/server/models/request.js +++ b/server/models/request.js | |||
@@ -1,11 +1,8 @@ | |||
1 | 'use strict' | 1 | 'use strict' |
2 | 2 | ||
3 | const each = require('async/each') | ||
4 | const waterfall = require('async/waterfall') | ||
5 | const values = require('lodash/values') | 3 | const values = require('lodash/values') |
6 | 4 | ||
7 | const constants = require('../initializers/constants') | 5 | const constants = require('../initializers/constants') |
8 | const logger = require('../helpers/logger') | ||
9 | 6 | ||
10 | // --------------------------------------------------------------------------- | 7 | // --------------------------------------------------------------------------- |
11 | 8 | ||
@@ -28,8 +25,6 @@ module.exports = function (sequelize, DataTypes) { | |||
28 | listWithLimitAndRandom, | 25 | listWithLimitAndRandom, |
29 | 26 | ||
30 | countTotalRequests, | 27 | countTotalRequests, |
31 | removeBadPods, | ||
32 | updatePodsScore, | ||
33 | removeAll, | 28 | removeAll, |
34 | removeWithEmptyTo | 29 | removeWithEmptyTo |
35 | } | 30 | } |
@@ -60,71 +55,17 @@ function countTotalRequests (callback) { | |||
60 | return this.count(query).asCallback(callback) | 55 | return this.count(query).asCallback(callback) |
61 | } | 56 | } |
62 | 57 | ||
63 | // Remove pods with a score of 0 (too many requests where they were unreachable) | ||
64 | function removeBadPods () { | ||
65 | const self = this | ||
66 | |||
67 | waterfall([ | ||
68 | function findBadPods (callback) { | ||
69 | self.sequelize.models.Pod.listBadPods(function (err, pods) { | ||
70 | if (err) { | ||
71 | logger.error('Cannot find bad pods.', { error: err }) | ||
72 | return callback(err) | ||
73 | } | ||
74 | |||
75 | return callback(null, pods) | ||
76 | }) | ||
77 | }, | ||
78 | |||
79 | function removeTheseBadPods (pods, callback) { | ||
80 | each(pods, function (pod, callbackEach) { | ||
81 | pod.destroy().asCallback(callbackEach) | ||
82 | }, function (err) { | ||
83 | return callback(err, pods.length) | ||
84 | }) | ||
85 | } | ||
86 | ], function (err, numberOfPodsRemoved) { | ||
87 | if (err) { | ||
88 | logger.error('Cannot remove bad pods.', { error: err }) | ||
89 | } else if (numberOfPodsRemoved) { | ||
90 | logger.info('Removed %d pods.', numberOfPodsRemoved) | ||
91 | } else { | ||
92 | logger.info('No need to remove bad pods.') | ||
93 | } | ||
94 | }) | ||
95 | } | ||
96 | |||
97 | function updatePodsScore (goodPods, badPods) { | ||
98 | const self = this | ||
99 | const Pod = this.sequelize.models.Pod | ||
100 | |||
101 | logger.info('Updating %d good pods and %d bad pods scores.', goodPods.length, badPods.length) | ||
102 | |||
103 | if (goodPods.length !== 0) { | ||
104 | Pod.incrementScores(goodPods, constants.PODS_SCORE.BONUS, function (err) { | ||
105 | if (err) logger.error('Cannot increment scores of good pods.', { error: err }) | ||
106 | }) | ||
107 | } | ||
108 | |||
109 | if (badPods.length !== 0) { | ||
110 | Pod.incrementScores(badPods, constants.PODS_SCORE.MALUS, function (err) { | ||
111 | if (err) logger.error('Cannot decrement scores of bad pods.', { error: err }) | ||
112 | removeBadPods.call(self) | ||
113 | }) | ||
114 | } | ||
115 | } | ||
116 | |||
117 | function listWithLimitAndRandom (limitPods, limitRequestsPerPod, callback) { | 58 | function listWithLimitAndRandom (limitPods, limitRequestsPerPod, callback) { |
118 | const self = this | 59 | const self = this |
119 | const Pod = this.sequelize.models.Pod | 60 | const Pod = this.sequelize.models.Pod |
120 | 61 | ||
121 | Pod.listRandomPodIdsWithRequest(limitPods, function (err, podIds) { | 62 | Pod.listRandomPodIdsWithRequest(limitPods, 'RequestToPods', function (err, podIds) { |
122 | if (err) return callback(err) | 63 | if (err) return callback(err) |
123 | 64 | ||
124 | // We don't have friends that have requests | 65 | // We don't have friends that have requests |
125 | if (podIds.length === 0) return callback(null, []) | 66 | if (podIds.length === 0) return callback(null, []) |
126 | 67 | ||
127 | // The the first x requests of these pods | 68 | // The first x requests of these pods |
128 | // It is very important to sort by id ASC to keep the requests order! | 69 | // It is very important to sort by id ASC to keep the requests order! |
129 | const query = { | 70 | const query = { |
130 | order: [ | 71 | order: [ |
diff --git a/server/models/video.js b/server/models/video.js index d0fd61eb4..daa273845 100644 --- a/server/models/video.js +++ b/server/models/video.js | |||
@@ -80,6 +80,15 @@ module.exports = function (sequelize, DataTypes) { | |||
80 | if (res === false) throw new Error('Video duration is not valid.') | 80 | if (res === false) throw new Error('Video duration is not valid.') |
81 | } | 81 | } |
82 | } | 82 | } |
83 | }, | ||
84 | views: { | ||
85 | type: DataTypes.INTEGER, | ||
86 | allowNull: false, | ||
87 | defaultValue: 0, | ||
88 | validate: { | ||
89 | min: 0, | ||
90 | isInt: true | ||
91 | } | ||
83 | } | 92 | } |
84 | }, | 93 | }, |
85 | { | 94 | { |
@@ -101,6 +110,9 @@ module.exports = function (sequelize, DataTypes) { | |||
101 | }, | 110 | }, |
102 | { | 111 | { |
103 | fields: [ 'infoHash' ] | 112 | fields: [ 'infoHash' ] |
113 | }, | ||
114 | { | ||
115 | fields: [ 'views' ] | ||
104 | } | 116 | } |
105 | ], | 117 | ], |
106 | classMethods: { | 118 | classMethods: { |
@@ -336,6 +348,7 @@ function toFormatedJSON () { | |||
336 | magnetUri: this.generateMagnetUri(), | 348 | magnetUri: this.generateMagnetUri(), |
337 | author: this.Author.name, | 349 | author: this.Author.name, |
338 | duration: this.duration, | 350 | duration: this.duration, |
351 | views: this.views, | ||
339 | tags: map(this.Tags, 'name'), | 352 | tags: map(this.Tags, 'name'), |
340 | thumbnailPath: pathUtils.join(constants.STATIC_PATHS.THUMBNAILS, this.getThumbnailName()), | 353 | thumbnailPath: pathUtils.join(constants.STATIC_PATHS.THUMBNAILS, this.getThumbnailName()), |
341 | createdAt: this.createdAt, | 354 | createdAt: this.createdAt, |
diff --git a/server/tests/api/multiple-pods.js b/server/tests/api/multiple-pods.js index df12ba0e9..871db54be 100644 --- a/server/tests/api/multiple-pods.js +++ b/server/tests/api/multiple-pods.js | |||
@@ -3,6 +3,7 @@ | |||
3 | const chai = require('chai') | 3 | const chai = require('chai') |
4 | const each = require('async/each') | 4 | const each = require('async/each') |
5 | const expect = chai.expect | 5 | const expect = chai.expect |
6 | const parallel = require('async/parallel') | ||
6 | const series = require('async/series') | 7 | const series = require('async/series') |
7 | const WebTorrent = require('webtorrent') | 8 | const WebTorrent = require('webtorrent') |
8 | const webtorrent = new WebTorrent() | 9 | const webtorrent = new WebTorrent() |
@@ -375,6 +376,63 @@ describe('Test multiple pods', function () { | |||
375 | }) | 376 | }) |
376 | }) | 377 | }) |
377 | 378 | ||
379 | describe('Should update video views', function () { | ||
380 | let videoId1 | ||
381 | let videoId2 | ||
382 | |||
383 | before(function (done) { | ||
384 | videosUtils.getVideosList(servers[2].url, function (err, res) { | ||
385 | if (err) throw err | ||
386 | |||
387 | const videos = res.body.data.filter(video => video.isLocal === true) | ||
388 | videoId1 = videos[0].id | ||
389 | videoId2 = videos[1].id | ||
390 | |||
391 | done() | ||
392 | }) | ||
393 | }) | ||
394 | |||
395 | it('Should views multiple videos on owned servers', function (done) { | ||
396 | this.timeout(30000) | ||
397 | |||
398 | parallel([ | ||
399 | function (callback) { | ||
400 | videosUtils.getVideo(servers[2].url, videoId1, callback) | ||
401 | }, | ||
402 | |||
403 | function (callback) { | ||
404 | videosUtils.getVideo(servers[2].url, videoId1, callback) | ||
405 | }, | ||
406 | |||
407 | function (callback) { | ||
408 | videosUtils.getVideo(servers[2].url, videoId1, callback) | ||
409 | }, | ||
410 | |||
411 | function (callback) { | ||
412 | videosUtils.getVideo(servers[2].url, videoId2, callback) | ||
413 | } | ||
414 | ], function (err) { | ||
415 | if (err) throw err | ||
416 | |||
417 | setTimeout(done, 22000) | ||
418 | }) | ||
419 | }) | ||
420 | |||
421 | it('Should have views updated on each pod', function (done) { | ||
422 | each(servers, function (server, callback) { | ||
423 | videosUtils.getVideosList(server.url, function (err, res) { | ||
424 | if (err) throw err | ||
425 | |||
426 | const videos = res.body.data | ||
427 | expect(videos.find(video => video.views === 3)).to.be.exist | ||
428 | expect(videos.find(video => video.views === 1)).to.be.exist | ||
429 | |||
430 | callback() | ||
431 | }) | ||
432 | }, done) | ||
433 | }) | ||
434 | }) | ||
435 | /* | ||
378 | describe('Should manipulate these videos', function () { | 436 | describe('Should manipulate these videos', function () { |
379 | it('Should update the video 3 by asking pod 3', function (done) { | 437 | it('Should update the video 3 by asking pod 3', function (done) { |
380 | this.timeout(15000) | 438 | this.timeout(15000) |
@@ -462,7 +520,7 @@ describe('Test multiple pods', function () { | |||
462 | }, done) | 520 | }, done) |
463 | }) | 521 | }) |
464 | }) | 522 | }) |
465 | 523 | */ | |
466 | after(function (done) { | 524 | after(function (done) { |
467 | servers.forEach(function (server) { | 525 | servers.forEach(function (server) { |
468 | process.kill(-server.app.pid) | 526 | process.kill(-server.app.pid) |
diff --git a/server/tests/api/single-pod.js b/server/tests/api/single-pod.js index 83a2b4411..40c33686f 100644 --- a/server/tests/api/single-pod.js +++ b/server/tests/api/single-pod.js | |||
@@ -129,6 +129,17 @@ describe('Test a single pod', function () { | |||
129 | }) | 129 | }) |
130 | }) | 130 | }) |
131 | 131 | ||
132 | it('Should have the views updated', function (done) { | ||
133 | videosUtils.getVideo(server.url, videoId, function (err, res) { | ||
134 | if (err) throw err | ||
135 | |||
136 | const video = res.body | ||
137 | expect(video.views).to.equal(1) | ||
138 | |||
139 | done() | ||
140 | }) | ||
141 | }) | ||
142 | |||
132 | it('Should search the video by name by default', function (done) { | 143 | it('Should search the video by name by default', function (done) { |
133 | videosUtils.searchVideo(server.url, 'my', function (err, res) { | 144 | videosUtils.searchVideo(server.url, 'my', function (err, res) { |
134 | if (err) throw err | 145 | if (err) throw err |