Make the network auto sufficient (eject bad pods with scores)

14 files changed, 310 insertions, 49 deletions

diff --git a/README.md b/README.md
index 1a738539a..9f028f40e 100644
--- a/README.md
+++ b/README.md
@@ -36,7 +36,7 @@ Thanks to [webtorrent](https://github.com/feross/webtorrent), we can make P2P (t
   - [ ] Inscription
   - [ ] Connection
   - [ ] Account rights (upload...)
-- [ ] Make the network auto sufficient (eject bad pods etc)
+- [X] Make the network auto sufficient (eject bad pods etc)
 - [ ] Manage API breaks
 - [ ] Add "DDOS" security (check if a pod don't send too many requests for example)
 
diff --git a/config/test-4.yaml b/config/test-4.yaml
new file mode 100644
index 000000000..6db8a5d03
--- /dev/null
+++ b/config/test-4.yaml
@@ -0,0 +1,19 @@
+listen:
+  port: 9004
+
+webserver:
+  host: 'localhost'
+  port: 9004
+
+database:
+  suffix: '-test4'
+
+# From the project root directory
+storage:
+  certs: 'test4/certs/'
+  uploads: 'test4/uploads/'
+  logs: 'test4/logs/'
+
+network:
+  friends:
+    - 'http://localhost:9002'
diff --git a/config/test-5.yaml b/config/test-5.yaml
new file mode 100644
index 000000000..7b3f18d35
--- /dev/null
+++ b/config/test-5.yaml
@@ -0,0 +1,20 @@
+listen:
+  port: 9005
+
+webserver:
+  host: 'localhost'
+  port: 9005
+
+database:
+  suffix: '-test5'
+
+# From the project root directory
+storage:
+  certs: 'test5/certs/'
+  uploads: 'test5/uploads/'
+  logs: 'test5/logs/'
+
+network:
+  friends:
+    - 'http://localhost:9001'
+    - 'http://localhost:9004'
diff --git a/config/test-6.yaml b/config/test-6.yaml
new file mode 100644
index 000000000..0c7675cc2
--- /dev/null
+++ b/config/test-6.yaml
@@ -0,0 +1,21 @@
+listen:
+  port: 9006
+
+webserver:
+  host: 'localhost'
+  port: 9006
+
+database:
+  suffix: '-test6'
+
+# From the project root directory
+storage:
+  certs: 'test6/certs/'
+  uploads: 'test6/uploads/'
+  logs: 'test6/logs/'
+
+network:
+  friends:
+    - 'http://localhost:9001'
+    - 'http://localhost:9002'
+    - 'http://localhost:9003'
diff --git a/package.json b/package.json
index 0c7af5690..86c832eb1 100644
--- a/package.json
+++ b/package.json
@@ -62,7 +62,9 @@
       "confirm",
       "it",
       "after",
+      "afterEach",
       "before",
+      "beforeEach",
       "describe"
     ]
   }
diff --git a/scripts/clean_test.sh b/scripts/clean_test.sh
index 8868cbddf..e46b5ecd5 100755
--- a/scripts/clean_test.sh
+++ b/scripts/clean_test.sh
@@ -1,5 +1,6 @@
 #!/bin/bash
 
-printf "use peertube-test1;\ndb.dropDatabase();\nuse peertube-test2;\ndb.dropDatabase();\nuse peertube-test3;\ndb.dropDatabase();" | mongo
-
-rm -rf ./test1 ./test2 ./test3
+for i in $(seq 1 6); do
+  printf "use peertube-test%s;\ndb.dropDatabase();" "$i" | mongo
+  rm -rf "./test$i"
+done
diff --git a/server.js b/server.js
index 715556414..3b899689c 100644
--- a/server.js
+++ b/server.js
@@ -1,9 +1,6 @@
 ;(function () {
   'use strict'
 
-  // ----------- Constants -----------
-  global.API_VERSION = 'v1'
-
   // ----------- Node modules -----------
   var bodyParser = require('body-parser')
   var express = require('express')
@@ -30,11 +27,16 @@
 
   checker.createDirectoriesIfNotExist()
 
+  // ----------- Constants -----------
+  var utils = require('./src/utils')
+
+  global.API_VERSION = 'v1'
+  global.FRIEND_BASE_SCORE = utils.isTestInstance() ? 20 : 100
+
   // ----------- PeerTube modules -----------
   var config = require('config')
   var logger = require('./src/logger')
   var routes = require('./routes')
-  var utils = require('./src/utils')
   var videos = require('./src/videos')
   var webtorrent = require('./src/webTorrentNode')
 
diff --git a/src/database.js b/src/database.js
index 020bfd961..740e89fa4 100644
--- a/src/database.js
+++ b/src/database.js
@@ -24,7 +24,8 @@
   // ----------- Pods -----------
   var podsSchema = mongoose.Schema({
     url: String,
-    publicKey: String
+    publicKey: String,
+    score: { type: Number, max: global.FRIEND_BASE_SCORE }
   })
 
   var PodsDB = mongoose.model('pods', podsSchema)
diff --git a/src/pods.js b/src/pods.js
index b4325ebcf..e26b3f0ae 100644
--- a/src/pods.js
+++ b/src/pods.js
@@ -16,6 +16,13 @@
   var host = config.get('webserver.host')
   var port = config.get('webserver.port')
 
+  // ----------- Constants -----------
+
+  var PODS_SCORE = {
+    MALUS: -10,
+    BONUS: 10
+  }
+
   // ----------- Private functions -----------
 
   function getForeignPodsList (url, callback) {
@@ -27,6 +34,25 @@
     })
   }
 
+  function updatePodsScore (good_pods, bad_pods) {
+    logger.info('Updating %d good pods and %d bad pods scores.', good_pods.length, bad_pods.length)
+
+    PodsDB.update({ _id: { $in: good_pods } }, { $inc: { score: PODS_SCORE.BONUS } }, { multi: true }).exec()
+    PodsDB.update({ _id: { $in: bad_pods } }, { $inc: { score: PODS_SCORE.MALUS } }, { multi: true }, function (err) {
+      if (err) throw err
+      removeBadPods()
+    })
+  }
+
+  function removeBadPods () {
+    PodsDB.remove({ score: 0 }, function (err, result) {
+      if (err) throw err
+
+      var number_removed = result.result.n
+      if (number_removed !== 0) logger.info('Removed %d pod.', number_removed)
+    })
+  }
+
   // ----------- Public functions -----------
 
   pods.list = function (callback) {
@@ -46,7 +72,8 @@
 
     var params = {
       url: data.url,
-      publicKey: data.publicKey
+      publicKey: data.publicKey,
+      score: global.FRIEND_BASE_SCORE
     }
 
     PodsDB.create(params, function (err, pod) {
@@ -68,7 +95,9 @@
 
   // { path, data }
   pods.makeSecureRequest = function (data, callback) {
-    PodsDB.find({}, { url: 1, publicKey: 1 }).exec(function (err, urls) {
+    if (callback === undefined) callback = function () {}
+
+    PodsDB.find({}, { _id: 1, url: 1, publicKey: 1 }).exec(function (err, pods) {
       if (err) {
         logger.error('Cannot get the list of the pods.', { error: err })
         return callback(err)
@@ -84,15 +113,23 @@
         data: data.data
       }
 
+      var bad_pods = []
+      var good_pods = []
+
       utils.makeMultipleRetryRequest(
         params,
 
-        urls,
+        pods,
 
-        function callbackEachPodFinished (err, response, body, url) {
+        function callbackEachPodFinished (err, response, body, pod, callback_each_pod_finished) {
           if (err || response.statusCode !== 200) {
-            logger.error('Error sending secure request to %s/%s pod.', url, data.path, { error: err })
+            bad_pods.push(pod._id)
+            logger.error('Error sending secure request to %s/%s pod.', pod.url, data.path, { error: err })
+          } else {
+            good_pods.push(pod._id)
           }
+
+          return callback_each_pod_finished()
         },
 
         function callbackAllPodsFinished (err) {
@@ -102,6 +139,8 @@
           }
 
           logger.debug('Finished')
+
+          updatePodsScore(good_pods, bad_pods)
           callback(null)
         }
       )
@@ -133,8 +172,8 @@
     // -----------------------------------------------------------------------
 
     function computeForeignPodsList (url, callback) {
-      // Always add a trust pod
-      pods_score[url] = Infinity
+      // Let's give 1 point to the pod we ask the friends list
+      pods_score[url] = 1
 
       getForeignPodsList(url, function (foreign_pods_list) {
         if (foreign_pods_list.length === 0) return callback()
@@ -175,16 +214,19 @@
 
         pods_list,
 
-        function eachRequest (err, response, body, url) {
+        function eachRequest (err, response, body, pod, callback_each_request) {
           // We add the pod if it responded correctly with its public certificate
           if (!err && response.statusCode === 200) {
-            pods.add({ url: url, publicKey: body.cert }, function (err) {
+            pods.add({ url: pod.url, publicKey: body.cert, score: global.FRIEND_BASE_SCORE }, function (err) {
               if (err) {
-                logger.error('Error with adding %s pod.', url, { error: err })
+                logger.error('Error with adding %s pod.', pod.url, { error: err })
               }
+
+              return callback_each_request()
             })
           } else {
-            logger.error('Error with adding %s pod.', url, { error: err || new Error('Status not 200') })
+            logger.error('Error with adding %s pod.', pod.url, { error: err || new Error('Status not 200') })
+            return callback_each_request()
           }
         },
 
diff --git a/src/utils.js b/src/utils.js
index d6b26db4b..dda6c7a0a 100644
--- a/src/utils.js
+++ b/src/utils.js
@@ -1,6 +1,7 @@
 ;(function () {
   'use strict'
 
+  var async = require('async')
   var config = require('config')
   var crypto = require('crypto')
   var fs = require('fs')
@@ -30,14 +31,15 @@
     }
 
     logger.debug('Sending informations to %s.', to_pod.url, { params: params })
+    // Default 10 but in tests we want to be faster
+    var retries = utils.isTestInstance() ? 2 : 10
 
-    // Replay 15 times, with factor 3
     replay(
       request.post(params, function (err, response, body) {
-        callbackEach(err, response, body, to_pod.url)
+        callbackEach(err, response, body, to_pod)
       }),
       {
-        retries: 10,
+        retries: retries,
         factor: 3,
         maxTimeout: Infinity,
         errorCodes: [ 'EADDRINFO', 'ETIMEDOUT', 'ECONNRESET', 'ESOCKETTIMEDOUT', 'ENOTFOUND', 'ECONNREFUSED' ]
@@ -68,7 +70,13 @@
     }
 
     // Make a request for each pod
-    for (var pod of pods) {
+    async.each(pods, function (pod, callback_each_async) {
+      function callbackEachRetryRequest (err, response, body, pod) {
+        callbackEach(err, response, body, pod, function () {
+          callback_each_async()
+        })
+      }
+
       var params = {
         url: pod.url + all_data.path,
         method: all_data.method
@@ -93,20 +101,18 @@
                 key: passwordEncrypted
               }
 
-              makeRetryRequest(copy_params, copy_url, copy_pod, copy_signature, callbackEach)
+              makeRetryRequest(copy_params, copy_url, copy_pod, copy_signature, callbackEachRetryRequest)
             })
           })(crt, params, url, pod, signature)
         } else {
           params.json = { data: all_data.data }
-          makeRetryRequest(params, url, pod, signature, callbackEach)
+          makeRetryRequest(params, url, pod, signature, callbackEachRetryRequest)
         }
       } else {
         logger.debug('Make a GET/DELETE request')
-        makeRetryRequest(params, url, pod, signature, callbackEach)
+        makeRetryRequest(params, url, pod, signature, callbackEachRetryRequest)
       }
-    }
-
-    return callback()
+    }, callback)
   }
 
   utils.certsExist = function (callback) {
@@ -192,5 +198,9 @@
     process.kill(-webtorrent_process.pid)
   }
 
+  utils.isTestInstance = function () {
+    return (process.env.NODE_ENV === 'test')
+  }
+
   module.exports = utils
 })()
diff --git a/src/videos.js b/src/videos.js
index b95219c39..8c44cad95 100644
--- a/src/videos.js
+++ b/src/videos.js
@@ -78,14 +78,9 @@
           data: params
         }
 
-        pods.makeSecureRequest(data, function (err) {
-          if (err) {
-            logger.error('Somes issues when sending this video to friends.', { error: err })
-            return callback(err)
-          }
-
-          return callback(null)
-        })
+        // Do not wait the secure requests
+        pods.makeSecureRequest(data)
+        callback(null)
       })
     })
   }
@@ -138,14 +133,8 @@
             }
 
             // Yes this is a POST request because we add some informations in the body (signature, encrypt etc)
-            pods.makeSecureRequest(data, function (err) {
-              if (err) {
-                logger.error('Somes issues when sending we want to remove the video to friends.', { error: err })
-                return callback(err)
-              }
-
-              callback(null)
-            })
+            pods.makeSecureRequest(data)
+            callback(null)
           })
         })
       })
diff --git a/test/api/friendsAdvanced.js b/test/api/friendsAdvanced.js
new file mode 100644
index 000000000..ccddac4dc
--- /dev/null
+++ b/test/api/friendsAdvanced.js
@@ -0,0 +1,154 @@
+;(function () {
+  'use strict'
+
+  var request = require('supertest')
+  var chai = require('chai')
+  var expect = chai.expect
+
+  var utils = require('../utils')
+
+  describe('Test advanced friends', function () {
+    var path = '/api/v1/pods/makefriends'
+    var apps = []
+    var urls = []
+
+    function makeFriend (pod_number, callback) {
+      // The first pod make friend with the third
+      request(urls[pod_number - 1])
+        .get(path)
+        .set('Accept', 'application/json')
+        .expect(204)
+        .end(function (err, res) {
+          if (err) throw err
+
+          // Wait for the request between pods
+          setTimeout(function () {
+            callback()
+          }, 1000)
+        })
+    }
+
+    function getFriendsList (pod_number, end) {
+      var path = '/api/v1/pods/'
+
+      request(urls[pod_number - 1])
+        .get(path)
+        .set('Accept', 'application/json')
+        .expect(200)
+        .expect('Content-Type', /json/)
+        .end(end)
+    }
+
+    function uploadVideo (pod_number, callback) {
+      var path = '/api/v1/videos'
+
+      request(urls[pod_number - 1])
+        .post(path)
+        .set('Accept', 'application/json')
+        .field('name', 'my super video')
+        .field('description', 'my super description')
+        .attach('input_video', __dirname + '/../fixtures/video_short.webm')
+        .expect(201)
+        .end(function (err) {
+          if (err) throw err
+
+          // Wait for the retry requests
+          setTimeout(callback, 10000)
+        })
+    }
+
+    beforeEach(function (done) {
+      this.timeout(30000)
+      utils.runMultipleServers(6, function (apps_run, urls_run) {
+        apps = apps_run
+        urls = urls_run
+        done()
+      })
+    })
+
+    afterEach(function (done) {
+      apps.forEach(function (app) {
+        process.kill(-app.pid)
+      })
+
+      if (this.ok) {
+        utils.flushTests(function () {
+          done()
+        })
+      } else {
+        done()
+      }
+    })
+
+    it('Should make friends with two pod each in a different group', function (done) {
+      this.timeout(10000)
+
+      // Pod 3 makes friend with the first one
+      makeFriend(3, function () {
+        // Pod 4 makes friend with the second one
+        makeFriend(4, function () {
+          // Now if the fifth wants to make friends with the third et the first
+          makeFriend(5, function () {
+            // It should have 0 friends
+            getFriendsList(5, function (err, res) {
+              if (err) throw err
+
+              expect(res.body.length).to.equal(0)
+
+              done()
+            })
+          })
+        })
+      })
+    })
+
+    it('Should make friends with the pods 1, 2, 3', function (done) {
+      this.timeout(100000)
+
+      // Pods 1, 2, 3 and 4 become friends
+      makeFriend(2, function () {
+        makeFriend(1, function () {
+          makeFriend(4, function () {
+            // Kill the server 4
+            apps[3].kill()
+
+            // Expulse pod 4 from pod 1 and 2
+            uploadVideo(1, function () {
+              uploadVideo(1, function () {
+                uploadVideo(2, function () {
+                  uploadVideo(2, function () {
+                    // Rerun server 4
+                    utils.runServer(4, function (app, url) {
+                      apps[3] = app
+                      getFriendsList(4, function (err, res) {
+                        if (err) throw err
+                        // Pod 4 didn't know pod 1 and 2 removed it
+                        expect(res.body.length).to.equal(3)
+
+                        // Pod 6 ask pod 1, 2 and 3
+                        makeFriend(6, function () {
+                          getFriendsList(6, function (err, res) {
+                            if (err) throw err
+
+                            // Pod 4 should not be our friend
+                            var result = res.body
+                            expect(result.length).to.equal(3)
+                            for (var pod of result) {
+                              expect(pod.url).not.equal(urls[3])
+                            }
+
+                            done()
+                          })
+                        })
+                      })
+                    })
+                  })
+                })
+              })
+            })
+          })
+        })
+      })
+    })
+  })
+})()
diff --git a/test/api/friends.js b/test/api/friendsBasic.js
index 845ccd1a8..40ed34199 100644
--- a/test/api/friends.js
+++ b/test/api/friendsBasic.js
@@ -19,7 +19,7 @@
       .end(end)
   }
 
-  describe('Test friends', function () {
+  describe('Test basic friends', function () {
     var apps = []
     var urls = []
 
diff --git a/test/utils.js b/test/utils.js
index 69f43d731..af3e8665d 100644
--- a/test/utils.js
+++ b/test/utils.js
@@ -74,8 +74,8 @@
   }
 
   module.exports = {
+    flushTests: flushTests,
     runMultipleServers: runMultipleServers,
-    runServer: runServer,
-    flushTests: flushTests
+    runServer: runServer
   }
 })()