Make the network auto sufficient (eject bad pods with scores)

4 files changed, 80 insertions, 38 deletions

diff --git a/src/database.js b/src/database.js
index 020bfd961..740e89fa4 100644
--- a/src/database.js
+++ b/src/database.js
@@ -24,7 +24,8 @@
   // ----------- Pods -----------
   var podsSchema = mongoose.Schema({
     url: String,
-    publicKey: String
+    publicKey: String,
+    score: { type: Number, max: global.FRIEND_BASE_SCORE }
   })
 
   var PodsDB = mongoose.model('pods', podsSchema)
diff --git a/src/pods.js b/src/pods.js
index b4325ebcf..e26b3f0ae 100644
--- a/src/pods.js
+++ b/src/pods.js
@@ -16,6 +16,13 @@
   var host = config.get('webserver.host')
   var port = config.get('webserver.port')
 
+  // ----------- Constants -----------
+
+  var PODS_SCORE = {
+    MALUS: -10,
+    BONUS: 10
+  }
+
   // ----------- Private functions -----------
 
   function getForeignPodsList (url, callback) {
@@ -27,6 +34,25 @@
     })
   }
 
+  function updatePodsScore (good_pods, bad_pods) {
+    logger.info('Updating %d good pods and %d bad pods scores.', good_pods.length, bad_pods.length)
+
+    PodsDB.update({ _id: { $in: good_pods } }, { $inc: { score: PODS_SCORE.BONUS } }, { multi: true }).exec()
+    PodsDB.update({ _id: { $in: bad_pods } }, { $inc: { score: PODS_SCORE.MALUS } }, { multi: true }, function (err) {
+      if (err) throw err
+      removeBadPods()
+    })
+  }
+
+  function removeBadPods () {
+    PodsDB.remove({ score: 0 }, function (err, result) {
+      if (err) throw err
+
+      var number_removed = result.result.n
+      if (number_removed !== 0) logger.info('Removed %d pod.', number_removed)
+    })
+  }
+
   // ----------- Public functions -----------
 
   pods.list = function (callback) {
@@ -46,7 +72,8 @@
 
     var params = {
       url: data.url,
-      publicKey: data.publicKey
+      publicKey: data.publicKey,
+      score: global.FRIEND_BASE_SCORE
     }
 
     PodsDB.create(params, function (err, pod) {
@@ -68,7 +95,9 @@
 
   // { path, data }
   pods.makeSecureRequest = function (data, callback) {
-    PodsDB.find({}, { url: 1, publicKey: 1 }).exec(function (err, urls) {
+    if (callback === undefined) callback = function () {}
+
+    PodsDB.find({}, { _id: 1, url: 1, publicKey: 1 }).exec(function (err, pods) {
       if (err) {
         logger.error('Cannot get the list of the pods.', { error: err })
         return callback(err)
@@ -84,15 +113,23 @@
         data: data.data
       }
 
+      var bad_pods = []
+      var good_pods = []
+
       utils.makeMultipleRetryRequest(
         params,
 
-        urls,
+        pods,
 
-        function callbackEachPodFinished (err, response, body, url) {
+        function callbackEachPodFinished (err, response, body, pod, callback_each_pod_finished) {
           if (err || response.statusCode !== 200) {
-            logger.error('Error sending secure request to %s/%s pod.', url, data.path, { error: err })
+            bad_pods.push(pod._id)
+            logger.error('Error sending secure request to %s/%s pod.', pod.url, data.path, { error: err })
+          } else {
+            good_pods.push(pod._id)
           }
+
+          return callback_each_pod_finished()
         },
 
         function callbackAllPodsFinished (err) {
@@ -102,6 +139,8 @@
           }
 
           logger.debug('Finished')
+
+          updatePodsScore(good_pods, bad_pods)
           callback(null)
         }
       )
@@ -133,8 +172,8 @@
     // -----------------------------------------------------------------------
 
     function computeForeignPodsList (url, callback) {
-      // Always add a trust pod
-      pods_score[url] = Infinity
+      // Let's give 1 point to the pod we ask the friends list
+      pods_score[url] = 1
 
       getForeignPodsList(url, function (foreign_pods_list) {
         if (foreign_pods_list.length === 0) return callback()
@@ -175,16 +214,19 @@
 
         pods_list,
 
-        function eachRequest (err, response, body, url) {
+        function eachRequest (err, response, body, pod, callback_each_request) {
           // We add the pod if it responded correctly with its public certificate
           if (!err && response.statusCode === 200) {
-            pods.add({ url: url, publicKey: body.cert }, function (err) {
+            pods.add({ url: pod.url, publicKey: body.cert, score: global.FRIEND_BASE_SCORE }, function (err) {
               if (err) {
-                logger.error('Error with adding %s pod.', url, { error: err })
+                logger.error('Error with adding %s pod.', pod.url, { error: err })
               }
+
+              return callback_each_request()
             })
           } else {
-            logger.error('Error with adding %s pod.', url, { error: err || new Error('Status not 200') })
+            logger.error('Error with adding %s pod.', pod.url, { error: err || new Error('Status not 200') })
+            return callback_each_request()
           }
         },
 
diff --git a/src/utils.js b/src/utils.js
index d6b26db4b..dda6c7a0a 100644
--- a/src/utils.js
+++ b/src/utils.js
@@ -1,6 +1,7 @@
 ;(function () {
   'use strict'
 
+  var async = require('async')
   var config = require('config')
   var crypto = require('crypto')
   var fs = require('fs')
@@ -30,14 +31,15 @@
     }
 
     logger.debug('Sending informations to %s.', to_pod.url, { params: params })
+    // Default 10 but in tests we want to be faster
+    var retries = utils.isTestInstance() ? 2 : 10
 
-    // Replay 15 times, with factor 3
     replay(
       request.post(params, function (err, response, body) {
-        callbackEach(err, response, body, to_pod.url)
+        callbackEach(err, response, body, to_pod)
       }),
       {
-        retries: 10,
+        retries: retries,
         factor: 3,
         maxTimeout: Infinity,
         errorCodes: [ 'EADDRINFO', 'ETIMEDOUT', 'ECONNRESET', 'ESOCKETTIMEDOUT', 'ENOTFOUND', 'ECONNREFUSED' ]
@@ -68,7 +70,13 @@
     }
 
     // Make a request for each pod
-    for (var pod of pods) {
+    async.each(pods, function (pod, callback_each_async) {
+      function callbackEachRetryRequest (err, response, body, pod) {
+        callbackEach(err, response, body, pod, function () {
+          callback_each_async()
+        })
+      }
+
       var params = {
         url: pod.url + all_data.path,
         method: all_data.method
@@ -93,20 +101,18 @@
                 key: passwordEncrypted
               }
 
-              makeRetryRequest(copy_params, copy_url, copy_pod, copy_signature, callbackEach)
+              makeRetryRequest(copy_params, copy_url, copy_pod, copy_signature, callbackEachRetryRequest)
             })
           })(crt, params, url, pod, signature)
         } else {
           params.json = { data: all_data.data }
-          makeRetryRequest(params, url, pod, signature, callbackEach)
+          makeRetryRequest(params, url, pod, signature, callbackEachRetryRequest)
         }
       } else {
         logger.debug('Make a GET/DELETE request')
-        makeRetryRequest(params, url, pod, signature, callbackEach)
+        makeRetryRequest(params, url, pod, signature, callbackEachRetryRequest)
       }
-    }
-
-    return callback()
+    }, callback)
   }
 
   utils.certsExist = function (callback) {
@@ -192,5 +198,9 @@
     process.kill(-webtorrent_process.pid)
   }
 
+  utils.isTestInstance = function () {
+    return (process.env.NODE_ENV === 'test')
+  }
+
   module.exports = utils
 })()
diff --git a/src/videos.js b/src/videos.js
index b95219c39..8c44cad95 100644
--- a/src/videos.js
+++ b/src/videos.js
@@ -78,14 +78,9 @@
           data: params
         }
 
-        pods.makeSecureRequest(data, function (err) {
-          if (err) {
-            logger.error('Somes issues when sending this video to friends.', { error: err })
-            return callback(err)
-          }
-
-          return callback(null)
-        })
+        // Do not wait the secure requests
+        pods.makeSecureRequest(data)
+        callback(null)
       })
     })
   }
@@ -138,14 +133,8 @@
             }
 
             // Yes this is a POST request because we add some informations in the body (signature, encrypt etc)
-            pods.makeSecureRequest(data, function (err) {
-              if (err) {
-                logger.error('Somes issues when sending we want to remove the video to friends.', { error: err })
-                return callback(err)
-              }
-
-              callback(null)
-            })
+            pods.makeSecureRequest(data)
+            callback(null)
           })
         })
       })