From 3bcb78b3aff565996ee0e2aa96bce7f1bdd6d66a Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Tue, 24 Nov 2015 08:33:59 +0100 Subject: Make the network auto sufficient (eject bad pods with scores) --- src/database.js | 3 ++- src/pods.js | 64 +++++++++++++++++++++++++++++++++++++++++++++++---------- src/utils.js | 30 ++++++++++++++++++--------- src/videos.js | 21 +++++-------------- 4 files changed, 80 insertions(+), 38 deletions(-) (limited to 'src') diff --git a/src/database.js b/src/database.js index 020bfd961..740e89fa4 100644 --- a/src/database.js +++ b/src/database.js @@ -24,7 +24,8 @@ // ----------- Pods ----------- var podsSchema = mongoose.Schema({ url: String, - publicKey: String + publicKey: String, + score: { type: Number, max: global.FRIEND_BASE_SCORE } }) var PodsDB = mongoose.model('pods', podsSchema) diff --git a/src/pods.js b/src/pods.js index b4325ebcf..e26b3f0ae 100644 --- a/src/pods.js +++ b/src/pods.js @@ -16,6 +16,13 @@ var host = config.get('webserver.host') var port = config.get('webserver.port') + // ----------- Constants ----------- + + var PODS_SCORE = { + MALUS: -10, + BONUS: 10 + } + // ----------- Private functions ----------- function getForeignPodsList (url, callback) { @@ -27,6 +34,25 @@ }) } + function updatePodsScore (good_pods, bad_pods) { + logger.info('Updating %d good pods and %d bad pods scores.', good_pods.length, bad_pods.length) + + PodsDB.update({ _id: { $in: good_pods } }, { $inc: { score: PODS_SCORE.BONUS } }, { multi: true }).exec() + PodsDB.update({ _id: { $in: bad_pods } }, { $inc: { score: PODS_SCORE.MALUS } }, { multi: true }, function (err) { + if (err) throw err + removeBadPods() + }) + } + + function removeBadPods () { + PodsDB.remove({ score: 0 }, function (err, result) { + if (err) throw err + + var number_removed = result.result.n + if (number_removed !== 0) logger.info('Removed %d pod.', number_removed) + }) + } + // ----------- Public functions ----------- pods.list = function (callback) { @@ -46,7 +72,8 @@ var params = { url: data.url, - publicKey: data.publicKey + publicKey: data.publicKey, + score: global.FRIEND_BASE_SCORE } PodsDB.create(params, function (err, pod) { @@ -68,7 +95,9 @@ // { path, data } pods.makeSecureRequest = function (data, callback) { - PodsDB.find({}, { url: 1, publicKey: 1 }).exec(function (err, urls) { + if (callback === undefined) callback = function () {} + + PodsDB.find({}, { _id: 1, url: 1, publicKey: 1 }).exec(function (err, pods) { if (err) { logger.error('Cannot get the list of the pods.', { error: err }) return callback(err) @@ -84,15 +113,23 @@ data: data.data } + var bad_pods = [] + var good_pods = [] + utils.makeMultipleRetryRequest( params, - urls, + pods, - function callbackEachPodFinished (err, response, body, url) { + function callbackEachPodFinished (err, response, body, pod, callback_each_pod_finished) { if (err || response.statusCode !== 200) { - logger.error('Error sending secure request to %s/%s pod.', url, data.path, { error: err }) + bad_pods.push(pod._id) + logger.error('Error sending secure request to %s/%s pod.', pod.url, data.path, { error: err }) + } else { + good_pods.push(pod._id) } + + return callback_each_pod_finished() }, function callbackAllPodsFinished (err) { @@ -102,6 +139,8 @@ } logger.debug('Finished') + + updatePodsScore(good_pods, bad_pods) callback(null) } ) @@ -133,8 +172,8 @@ // ----------------------------------------------------------------------- function computeForeignPodsList (url, callback) { - // Always add a trust pod - pods_score[url] = Infinity + // Let's give 1 point to the pod we ask the friends list + pods_score[url] = 1 getForeignPodsList(url, function (foreign_pods_list) { if (foreign_pods_list.length === 0) return callback() @@ -175,16 +214,19 @@ pods_list, - function eachRequest (err, response, body, url) { + function eachRequest (err, response, body, pod, callback_each_request) { // We add the pod if it responded correctly with its public certificate if (!err && response.statusCode === 200) { - pods.add({ url: url, publicKey: body.cert }, function (err) { + pods.add({ url: pod.url, publicKey: body.cert, score: global.FRIEND_BASE_SCORE }, function (err) { if (err) { - logger.error('Error with adding %s pod.', url, { error: err }) + logger.error('Error with adding %s pod.', pod.url, { error: err }) } + + return callback_each_request() }) } else { - logger.error('Error with adding %s pod.', url, { error: err || new Error('Status not 200') }) + logger.error('Error with adding %s pod.', pod.url, { error: err || new Error('Status not 200') }) + return callback_each_request() } }, diff --git a/src/utils.js b/src/utils.js index d6b26db4b..dda6c7a0a 100644 --- a/src/utils.js +++ b/src/utils.js @@ -1,6 +1,7 @@ ;(function () { 'use strict' + var async = require('async') var config = require('config') var crypto = require('crypto') var fs = require('fs') @@ -30,14 +31,15 @@ } logger.debug('Sending informations to %s.', to_pod.url, { params: params }) + // Default 10 but in tests we want to be faster + var retries = utils.isTestInstance() ? 2 : 10 - // Replay 15 times, with factor 3 replay( request.post(params, function (err, response, body) { - callbackEach(err, response, body, to_pod.url) + callbackEach(err, response, body, to_pod) }), { - retries: 10, + retries: retries, factor: 3, maxTimeout: Infinity, errorCodes: [ 'EADDRINFO', 'ETIMEDOUT', 'ECONNRESET', 'ESOCKETTIMEDOUT', 'ENOTFOUND', 'ECONNREFUSED' ] @@ -68,7 +70,13 @@ } // Make a request for each pod - for (var pod of pods) { + async.each(pods, function (pod, callback_each_async) { + function callbackEachRetryRequest (err, response, body, pod) { + callbackEach(err, response, body, pod, function () { + callback_each_async() + }) + } + var params = { url: pod.url + all_data.path, method: all_data.method @@ -93,20 +101,18 @@ key: passwordEncrypted } - makeRetryRequest(copy_params, copy_url, copy_pod, copy_signature, callbackEach) + makeRetryRequest(copy_params, copy_url, copy_pod, copy_signature, callbackEachRetryRequest) }) })(crt, params, url, pod, signature) } else { params.json = { data: all_data.data } - makeRetryRequest(params, url, pod, signature, callbackEach) + makeRetryRequest(params, url, pod, signature, callbackEachRetryRequest) } } else { logger.debug('Make a GET/DELETE request') - makeRetryRequest(params, url, pod, signature, callbackEach) + makeRetryRequest(params, url, pod, signature, callbackEachRetryRequest) } - } - - return callback() + }, callback) } utils.certsExist = function (callback) { @@ -192,5 +198,9 @@ process.kill(-webtorrent_process.pid) } + utils.isTestInstance = function () { + return (process.env.NODE_ENV === 'test') + } + module.exports = utils })() diff --git a/src/videos.js b/src/videos.js index b95219c39..8c44cad95 100644 --- a/src/videos.js +++ b/src/videos.js @@ -78,14 +78,9 @@ data: params } - pods.makeSecureRequest(data, function (err) { - if (err) { - logger.error('Somes issues when sending this video to friends.', { error: err }) - return callback(err) - } - - return callback(null) - }) + // Do not wait the secure requests + pods.makeSecureRequest(data) + callback(null) }) }) } @@ -138,14 +133,8 @@ } // Yes this is a POST request because we add some informations in the body (signature, encrypt etc) - pods.makeSecureRequest(data, function (err) { - if (err) { - logger.error('Somes issues when sending we want to remove the video to friends.', { error: err }) - return callback(err) - } - - callback(null) - }) + pods.makeSecureRequest(data) + callback(null) }) }) }) -- cgit v1.2.3