aboutsummaryrefslogtreecommitdiffhomepage
path: root/server/models
diff options
context:
space:
mode:
authorChocobozzz <florian.bigard@gmail.com>2016-06-28 20:10:32 +0200
committerChocobozzz <florian.bigard@gmail.com>2016-06-28 20:10:32 +0200
commit00057e85a703713a8f0d96e01c49978be0987eb2 (patch)
tree54735435062d2ededa72448de9fcaa677cf2c79c /server/models
parentaaf61f3810e6d57c5130af959bd2860df32775e7 (diff)
downloadPeerTube-00057e85a703713a8f0d96e01c49978be0987eb2.tar.gz
PeerTube-00057e85a703713a8f0d96e01c49978be0987eb2.tar.zst
PeerTube-00057e85a703713a8f0d96e01c49978be0987eb2.zip
Request model refractoring -> use mongoose api
Diffstat (limited to 'server/models')
-rw-r--r--server/models/pods.js7
-rw-r--r--server/models/request.js280
-rw-r--r--server/models/requests.js73
3 files changed, 286 insertions, 74 deletions
diff --git a/server/models/pods.js b/server/models/pods.js
index daeadeb07..9502d92e4 100644
--- a/server/models/pods.js
+++ b/server/models/pods.js
@@ -1,6 +1,7 @@
1'use strict' 1'use strict'
2 2
3const mongoose = require('mongoose') 3const mongoose = require('mongoose')
4const map = require('lodash/map')
4 5
5const constants = require('../initializers/constants') 6const constants = require('../initializers/constants')
6const logger = require('../helpers/logger') 7const logger = require('../helpers/logger')
@@ -76,7 +77,11 @@ function list (callback) {
76} 77}
77 78
78function listAllIds (callback) { 79function listAllIds (callback) {
79 return PodsDB.find({}, { _id: 1 }, callback) 80 return PodsDB.find({}, { _id: 1 }, function (err, pods) {
81 if (err) return callback(err)
82
83 return callback(null, map(pods, '_id'))
84 })
80} 85}
81 86
82function listAllUrls (callback) { 87function listAllUrls (callback) {
diff --git a/server/models/request.js b/server/models/request.js
new file mode 100644
index 000000000..2a407388a
--- /dev/null
+++ b/server/models/request.js
@@ -0,0 +1,280 @@
1'use strict'
2
3const async = require('async')
4const map = require('lodash/map')
5const mongoose = require('mongoose')
6
7const constants = require('../initializers/constants')
8const logger = require('../helpers/logger')
9const Pods = require('../models/pods')
10const requests = require('../helpers/requests')
11
12const Video = mongoose.model('Video')
13
14let timer = null
15
16// ---------------------------------------------------------------------------
17
18const RequestSchema = mongoose.Schema({
19 request: mongoose.Schema.Types.Mixed,
20 to: [ { type: mongoose.Schema.Types.ObjectId, ref: 'users' } ]
21})
22
23RequestSchema.statics = {
24 activate,
25 deactivate,
26 flush,
27 forceSend
28}
29
30RequestSchema.pre('save', function (next) {
31 const self = this
32
33 if (self.to.length === 0) {
34 Pods.listAllIds(function (err, podIds) {
35 if (err) return next(err)
36
37 // No friends
38 if (podIds.length === 0) return
39
40 self.to = podIds
41 return next()
42 })
43 } else {
44 return next()
45 }
46})
47
48mongoose.model('Request', RequestSchema)
49
50// ------------------------------ STATICS ------------------------------
51
52function activate () {
53 logger.info('Requests scheduler activated.')
54 timer = setInterval(makeRequests.bind(this), constants.INTERVAL)
55}
56
57function deactivate () {
58 logger.info('Requests scheduler deactivated.')
59 clearInterval(timer)
60}
61
62function flush () {
63 removeAll.call(this, function (err) {
64 if (err) logger.error('Cannot flush the requests.', { error: err })
65 })
66}
67
68function forceSend () {
69 logger.info('Force requests scheduler sending.')
70 makeRequests.call(this)
71}
72
73// ---------------------------------------------------------------------------
74
75// Make a requests to friends of a certain type
76function makeRequest (toPod, requestsToMake, callback) {
77 if (!callback) callback = function () {}
78
79 const params = {
80 toPod: toPod,
81 encrypt: true, // Security
82 sign: true, // To prove our identity
83 method: 'POST',
84 path: '/api/' + constants.API_VERSION + '/remote/videos',
85 data: requestsToMake // Requests we need to make
86 }
87
88 // Make multiple retry requests to all of pods
89 // The function fire some useful callbacks
90 requests.makeSecureRequest(params, function (err, res) {
91 if (err || (res.statusCode !== 200 && res.statusCode !== 201 && res.statusCode !== 204)) {
92 logger.error('Error sending secure request to %s pod.', toPod.url, { error: err || new Error('Status code not 20x') })
93
94 return callback(false)
95 }
96
97 return callback(true)
98 })
99}
100
101// Make all the requests of the scheduler
102function makeRequests () {
103 const self = this
104
105 list.call(self, function (err, requests) {
106 if (err) {
107 logger.error('Cannot get the list of requests.', { err: err })
108 return // Abort
109 }
110
111 // If there are no requests, abort
112 if (requests.length === 0) {
113 logger.info('No requests to make.')
114 return
115 }
116
117 logger.info('Making requests to friends.')
118
119 // Requests by pods id
120 const requestsToMake = {}
121
122 requests.forEach(function (poolRequest) {
123 poolRequest.to.forEach(function (toPodId) {
124 if (!requestsToMake[toPodId]) {
125 requestsToMake[toPodId] = {
126 ids: [],
127 datas: []
128 }
129 }
130
131 requestsToMake[toPodId].ids.push(poolRequest._id)
132 requestsToMake[toPodId].datas.push(poolRequest.request)
133 })
134 })
135
136 const goodPods = []
137 const badPods = []
138
139 async.eachLimit(Object.keys(requestsToMake), constants.REQUESTS_IN_PARALLEL, function (toPodId, callbackEach) {
140 const requestToMake = requestsToMake[toPodId]
141
142 // FIXME: mongodb request inside a loop :/
143 Pods.findById(toPodId, function (err, toPod) {
144 if (err) {
145 logger.error('Error finding pod by id.', { err: err })
146 return callbackEach()
147 }
148
149 // Maybe the pod is not our friend anymore so simply remove them
150 if (!toPod) {
151 removePodOf.call(self, requestToMake.ids, toPodId)
152 return callbackEach()
153 }
154
155 makeRequest(toPod, requestToMake.datas, function (success) {
156 if (err) {
157 logger.error('Errors when sent request to %s.', toPod.url, { error: err })
158 // Do not stop the process just for one error
159 return callbackEach()
160 }
161
162 if (success === true) {
163 logger.debug('Removing requests for %s pod.', toPodId, { requestsIds: requestToMake.ids })
164
165 // Remove the pod id of these request ids
166 removePodOf.call(self, requestToMake.ids, toPodId)
167 goodPods.push(toPodId)
168 } else {
169 badPods.push(toPodId)
170 }
171
172 callbackEach()
173 })
174 })
175 }, function () {
176 // All the requests were made, we update the pods score
177 updatePodsScore(goodPods, badPods)
178 // Flush requests with no pod
179 removeWithEmptyTo.call(self)
180 })
181 })
182}
183
184// Remove pods with a score of 0 (too many requests where they were unreachable)
185function removeBadPods () {
186 async.waterfall([
187 function findBadPods (callback) {
188 Pods.findBadPods(function (err, pods) {
189 if (err) {
190 logger.error('Cannot find bad pods.', { error: err })
191 return callback(err)
192 }
193
194 return callback(null, pods)
195 })
196 },
197
198 function listVideosOfTheseBadPods (pods, callback) {
199 if (pods.length === 0) return callback(null)
200
201 const urls = map(pods, 'url')
202 const ids = map(pods, '_id')
203
204 Video.listByUrls(urls, function (err, videosList) {
205 if (err) {
206 logger.error('Cannot list videos urls.', { error: err, urls: urls })
207 return callback(null, ids, [])
208 }
209
210 return callback(null, ids, videosList)
211 })
212 },
213
214 function removeVideosOfTheseBadPods (podIds, videosList, callback) {
215 // We don't have to remove pods, skip
216 if (typeof podIds === 'function') return podIds(null)
217
218 async.each(videosList, function (video, callbackEach) {
219 video.remove(callbackEach)
220 }, function (err) {
221 if (err) {
222 // Don't stop the process
223 logger.error('Error while removing videos of bad pods.', { error: err })
224 return
225 }
226
227 return callback(null, podIds)
228 })
229 },
230
231 function removeBadPodsFromDB (podIds, callback) {
232 // We don't have to remove pods, skip
233 if (typeof podIds === 'function') return podIds(null)
234
235 Pods.removeAllByIds(podIds, callback)
236 }
237 ], function (err, removeResult) {
238 if (err) {
239 logger.error('Cannot remove bad pods.', { error: err })
240 } else if (removeResult) {
241 const podsRemoved = removeResult.result.n
242 logger.info('Removed %d pods.', podsRemoved)
243 } else {
244 logger.info('No need to remove bad pods.')
245 }
246 })
247}
248
249function updatePodsScore (goodPods, badPods) {
250 logger.info('Updating %d good pods and %d bad pods scores.', goodPods.length, badPods.length)
251
252 Pods.incrementScores(goodPods, constants.PODS_SCORE.BONUS, function (err) {
253 if (err) logger.error('Cannot increment scores of good pods.')
254 })
255
256 Pods.incrementScores(badPods, constants.PODS_SCORE.MALUS, function (err) {
257 if (err) logger.error('Cannot decrement scores of bad pods.')
258 removeBadPods()
259 })
260}
261
262function list (callback) {
263 this.find({ }, { _id: 1, request: 1, to: 1 }, callback)
264}
265
266function removeAll (callback) {
267 this.remove({ }, callback)
268}
269
270function removePodOf (requestsIds, podId, callback) {
271 if (!callback) callback = function () {}
272
273 this.update({ _id: { $in: requestsIds } }, { $pull: { to: podId } }, { multi: true }, callback)
274}
275
276function removeWithEmptyTo (callback) {
277 if (!callback) callback = function () {}
278
279 this.remove({ to: { $size: 0 } }, callback)
280}
diff --git a/server/models/requests.js b/server/models/requests.js
deleted file mode 100644
index e67ccad56..000000000
--- a/server/models/requests.js
+++ /dev/null
@@ -1,73 +0,0 @@
1'use strict'
2
3const mongoose = require('mongoose')
4
5const logger = require('../helpers/logger')
6
7// ---------------------------------------------------------------------------
8
9const requestsSchema = mongoose.Schema({
10 request: mongoose.Schema.Types.Mixed,
11 to: [ { type: mongoose.Schema.Types.ObjectId, ref: 'users' } ]
12})
13const RequestsDB = mongoose.model('requests', requestsSchema)
14
15// ---------------------------------------------------------------------------
16
17const Requests = {
18 create: create,
19 findById: findById,
20 list: list,
21 removeAll: removeAll,
22 removePodOf: removePodOf,
23 removeRequestById: removeRequestById,
24 removeRequests: removeRequests,
25 removeWithEmptyTo: removeWithEmptyTo
26}
27
28function create (request, to, callback) {
29 RequestsDB.create({ request: request, to: to }, callback)
30}
31
32function findById (id, callback) {
33 RequestsDB.findOne({ id: id }, callback)
34}
35
36function list (callback) {
37 RequestsDB.find({}, { _id: 1, request: 1, to: 1 }, callback)
38}
39
40function removeAll (callback) {
41 RequestsDB.remove({ }, callback)
42}
43
44function removePodOf (requestsIds, podId, callback) {
45 if (!callback) callback = function () {}
46
47 RequestsDB.update({ _id: { $in: requestsIds } }, { $pull: { to: podId } }, { multi: true }, callback)
48}
49
50function removeRequestById (id, callback) {
51 RequestsDB.remove({ id: id }, callback)
52}
53
54function removeRequests (ids) {
55 RequestsDB.remove({ _id: { $in: ids } }, function (err) {
56 if (err) {
57 logger.error('Cannot remove requests from the requests database.', { error: err })
58 return // Abort
59 }
60
61 logger.info('Pool requests flushed.')
62 })
63}
64
65function removeWithEmptyTo (callback) {
66 if (!callback) callback = function () {}
67
68 RequestsDB.remove({ to: { $size: 0 } }, callback)
69}
70
71// ---------------------------------------------------------------------------
72
73module.exports = Requests