aboutsummaryrefslogtreecommitdiffhomepage
diff options
context:
space:
mode:
authorChocobozzz <florian.bigard@gmail.com>2015-11-24 08:33:59 +0100
committerChocobozzz <florian.bigard@gmail.com>2015-11-24 08:33:59 +0100
commit3bcb78b3aff565996ee0e2aa96bce7f1bdd6d66a (patch)
treeb202172a35956b252282105f614fc9646ca64ebb
parent2e3b5b0db652ae4787e359aadbd4f52e473b6af7 (diff)
downloadPeerTube-3bcb78b3aff565996ee0e2aa96bce7f1bdd6d66a.tar.gz
PeerTube-3bcb78b3aff565996ee0e2aa96bce7f1bdd6d66a.tar.zst
PeerTube-3bcb78b3aff565996ee0e2aa96bce7f1bdd6d66a.zip
Make the network auto sufficient (eject bad pods with scores)
-rw-r--r--README.md2
-rw-r--r--config/test-4.yaml19
-rw-r--r--config/test-5.yaml20
-rw-r--r--config/test-6.yaml21
-rw-r--r--package.json2
-rwxr-xr-xscripts/clean_test.sh7
-rw-r--r--server.js10
-rw-r--r--src/database.js3
-rw-r--r--src/pods.js64
-rw-r--r--src/utils.js30
-rw-r--r--src/videos.js21
-rw-r--r--test/api/friendsAdvanced.js154
-rw-r--r--test/api/friendsBasic.js (renamed from test/api/friends.js)2
-rw-r--r--test/utils.js4
14 files changed, 310 insertions, 49 deletions
diff --git a/README.md b/README.md
index 1a738539a..9f028f40e 100644
--- a/README.md
+++ b/README.md
@@ -36,7 +36,7 @@ Thanks to [webtorrent](https://github.com/feross/webtorrent), we can make P2P (t
36 - [ ] Inscription 36 - [ ] Inscription
37 - [ ] Connection 37 - [ ] Connection
38 - [ ] Account rights (upload...) 38 - [ ] Account rights (upload...)
39- [ ] Make the network auto sufficient (eject bad pods etc) 39- [X] Make the network auto sufficient (eject bad pods etc)
40- [ ] Manage API breaks 40- [ ] Manage API breaks
41- [ ] Add "DDOS" security (check if a pod don't send too many requests for example) 41- [ ] Add "DDOS" security (check if a pod don't send too many requests for example)
42 42
diff --git a/config/test-4.yaml b/config/test-4.yaml
new file mode 100644
index 000000000..6db8a5d03
--- /dev/null
+++ b/config/test-4.yaml
@@ -0,0 +1,19 @@
1listen:
2 port: 9004
3
4webserver:
5 host: 'localhost'
6 port: 9004
7
8database:
9 suffix: '-test4'
10
11# From the project root directory
12storage:
13 certs: 'test4/certs/'
14 uploads: 'test4/uploads/'
15 logs: 'test4/logs/'
16
17network:
18 friends:
19 - 'http://localhost:9002'
diff --git a/config/test-5.yaml b/config/test-5.yaml
new file mode 100644
index 000000000..7b3f18d35
--- /dev/null
+++ b/config/test-5.yaml
@@ -0,0 +1,20 @@
1listen:
2 port: 9005
3
4webserver:
5 host: 'localhost'
6 port: 9005
7
8database:
9 suffix: '-test5'
10
11# From the project root directory
12storage:
13 certs: 'test5/certs/'
14 uploads: 'test5/uploads/'
15 logs: 'test5/logs/'
16
17network:
18 friends:
19 - 'http://localhost:9001'
20 - 'http://localhost:9004'
diff --git a/config/test-6.yaml b/config/test-6.yaml
new file mode 100644
index 000000000..0c7675cc2
--- /dev/null
+++ b/config/test-6.yaml
@@ -0,0 +1,21 @@
1listen:
2 port: 9006
3
4webserver:
5 host: 'localhost'
6 port: 9006
7
8database:
9 suffix: '-test6'
10
11# From the project root directory
12storage:
13 certs: 'test6/certs/'
14 uploads: 'test6/uploads/'
15 logs: 'test6/logs/'
16
17network:
18 friends:
19 - 'http://localhost:9001'
20 - 'http://localhost:9002'
21 - 'http://localhost:9003'
diff --git a/package.json b/package.json
index 0c7af5690..86c832eb1 100644
--- a/package.json
+++ b/package.json
@@ -62,7 +62,9 @@
62 "confirm", 62 "confirm",
63 "it", 63 "it",
64 "after", 64 "after",
65 "afterEach",
65 "before", 66 "before",
67 "beforeEach",
66 "describe" 68 "describe"
67 ] 69 ]
68 } 70 }
diff --git a/scripts/clean_test.sh b/scripts/clean_test.sh
index 8868cbddf..e46b5ecd5 100755
--- a/scripts/clean_test.sh
+++ b/scripts/clean_test.sh
@@ -1,5 +1,6 @@
1#!/bin/bash 1#!/bin/bash
2 2
3printf "use peertube-test1;\ndb.dropDatabase();\nuse peertube-test2;\ndb.dropDatabase();\nuse peertube-test3;\ndb.dropDatabase();" | mongo 3for i in $(seq 1 6); do
4 4 printf "use peertube-test%s;\ndb.dropDatabase();" "$i" | mongo
5rm -rf ./test1 ./test2 ./test3 5 rm -rf "./test$i"
6done
diff --git a/server.js b/server.js
index 715556414..3b899689c 100644
--- a/server.js
+++ b/server.js
@@ -1,9 +1,6 @@
1;(function () { 1;(function () {
2 'use strict' 2 'use strict'
3 3
4 // ----------- Constants -----------
5 global.API_VERSION = 'v1'
6
7 // ----------- Node modules ----------- 4 // ----------- Node modules -----------
8 var bodyParser = require('body-parser') 5 var bodyParser = require('body-parser')
9 var express = require('express') 6 var express = require('express')
@@ -30,11 +27,16 @@
30 27
31 checker.createDirectoriesIfNotExist() 28 checker.createDirectoriesIfNotExist()
32 29
30 // ----------- Constants -----------
31 var utils = require('./src/utils')
32
33 global.API_VERSION = 'v1'
34 global.FRIEND_BASE_SCORE = utils.isTestInstance() ? 20 : 100
35
33 // ----------- PeerTube modules ----------- 36 // ----------- PeerTube modules -----------
34 var config = require('config') 37 var config = require('config')
35 var logger = require('./src/logger') 38 var logger = require('./src/logger')
36 var routes = require('./routes') 39 var routes = require('./routes')
37 var utils = require('./src/utils')
38 var videos = require('./src/videos') 40 var videos = require('./src/videos')
39 var webtorrent = require('./src/webTorrentNode') 41 var webtorrent = require('./src/webTorrentNode')
40 42
diff --git a/src/database.js b/src/database.js
index 020bfd961..740e89fa4 100644
--- a/src/database.js
+++ b/src/database.js
@@ -24,7 +24,8 @@
24 // ----------- Pods ----------- 24 // ----------- Pods -----------
25 var podsSchema = mongoose.Schema({ 25 var podsSchema = mongoose.Schema({
26 url: String, 26 url: String,
27 publicKey: String 27 publicKey: String,
28 score: { type: Number, max: global.FRIEND_BASE_SCORE }
28 }) 29 })
29 30
30 var PodsDB = mongoose.model('pods', podsSchema) 31 var PodsDB = mongoose.model('pods', podsSchema)
diff --git a/src/pods.js b/src/pods.js
index b4325ebcf..e26b3f0ae 100644
--- a/src/pods.js
+++ b/src/pods.js
@@ -16,6 +16,13 @@
16 var host = config.get('webserver.host') 16 var host = config.get('webserver.host')
17 var port = config.get('webserver.port') 17 var port = config.get('webserver.port')
18 18
19 // ----------- Constants -----------
20
21 var PODS_SCORE = {
22 MALUS: -10,
23 BONUS: 10
24 }
25
19 // ----------- Private functions ----------- 26 // ----------- Private functions -----------
20 27
21 function getForeignPodsList (url, callback) { 28 function getForeignPodsList (url, callback) {
@@ -27,6 +34,25 @@
27 }) 34 })
28 } 35 }
29 36
37 function updatePodsScore (good_pods, bad_pods) {
38 logger.info('Updating %d good pods and %d bad pods scores.', good_pods.length, bad_pods.length)
39
40 PodsDB.update({ _id: { $in: good_pods } }, { $inc: { score: PODS_SCORE.BONUS } }, { multi: true }).exec()
41 PodsDB.update({ _id: { $in: bad_pods } }, { $inc: { score: PODS_SCORE.MALUS } }, { multi: true }, function (err) {
42 if (err) throw err
43 removeBadPods()
44 })
45 }
46
47 function removeBadPods () {
48 PodsDB.remove({ score: 0 }, function (err, result) {
49 if (err) throw err
50
51 var number_removed = result.result.n
52 if (number_removed !== 0) logger.info('Removed %d pod.', number_removed)
53 })
54 }
55
30 // ----------- Public functions ----------- 56 // ----------- Public functions -----------
31 57
32 pods.list = function (callback) { 58 pods.list = function (callback) {
@@ -46,7 +72,8 @@
46 72
47 var params = { 73 var params = {
48 url: data.url, 74 url: data.url,
49 publicKey: data.publicKey 75 publicKey: data.publicKey,
76 score: global.FRIEND_BASE_SCORE
50 } 77 }
51 78
52 PodsDB.create(params, function (err, pod) { 79 PodsDB.create(params, function (err, pod) {
@@ -68,7 +95,9 @@
68 95
69 // { path, data } 96 // { path, data }
70 pods.makeSecureRequest = function (data, callback) { 97 pods.makeSecureRequest = function (data, callback) {
71 PodsDB.find({}, { url: 1, publicKey: 1 }).exec(function (err, urls) { 98 if (callback === undefined) callback = function () {}
99
100 PodsDB.find({}, { _id: 1, url: 1, publicKey: 1 }).exec(function (err, pods) {
72 if (err) { 101 if (err) {
73 logger.error('Cannot get the list of the pods.', { error: err }) 102 logger.error('Cannot get the list of the pods.', { error: err })
74 return callback(err) 103 return callback(err)
@@ -84,15 +113,23 @@
84 data: data.data 113 data: data.data
85 } 114 }
86 115
116 var bad_pods = []
117 var good_pods = []
118
87 utils.makeMultipleRetryRequest( 119 utils.makeMultipleRetryRequest(
88 params, 120 params,
89 121
90 urls, 122 pods,
91 123
92 function callbackEachPodFinished (err, response, body, url) { 124 function callbackEachPodFinished (err, response, body, pod, callback_each_pod_finished) {
93 if (err || response.statusCode !== 200) { 125 if (err || response.statusCode !== 200) {
94 logger.error('Error sending secure request to %s/%s pod.', url, data.path, { error: err }) 126 bad_pods.push(pod._id)
127 logger.error('Error sending secure request to %s/%s pod.', pod.url, data.path, { error: err })
128 } else {
129 good_pods.push(pod._id)
95 } 130 }
131
132 return callback_each_pod_finished()
96 }, 133 },
97 134
98 function callbackAllPodsFinished (err) { 135 function callbackAllPodsFinished (err) {
@@ -102,6 +139,8 @@
102 } 139 }
103 140
104 logger.debug('Finished') 141 logger.debug('Finished')
142
143 updatePodsScore(good_pods, bad_pods)
105 callback(null) 144 callback(null)
106 } 145 }
107 ) 146 )
@@ -133,8 +172,8 @@
133 // ----------------------------------------------------------------------- 172 // -----------------------------------------------------------------------
134 173
135 function computeForeignPodsList (url, callback) { 174 function computeForeignPodsList (url, callback) {
136 // Always add a trust pod 175 // Let's give 1 point to the pod we ask the friends list
137 pods_score[url] = Infinity 176 pods_score[url] = 1
138 177
139 getForeignPodsList(url, function (foreign_pods_list) { 178 getForeignPodsList(url, function (foreign_pods_list) {
140 if (foreign_pods_list.length === 0) return callback() 179 if (foreign_pods_list.length === 0) return callback()
@@ -175,16 +214,19 @@
175 214
176 pods_list, 215 pods_list,
177 216
178 function eachRequest (err, response, body, url) { 217 function eachRequest (err, response, body, pod, callback_each_request) {
179 // We add the pod if it responded correctly with its public certificate 218 // We add the pod if it responded correctly with its public certificate
180 if (!err && response.statusCode === 200) { 219 if (!err && response.statusCode === 200) {
181 pods.add({ url: url, publicKey: body.cert }, function (err) { 220 pods.add({ url: pod.url, publicKey: body.cert, score: global.FRIEND_BASE_SCORE }, function (err) {
182 if (err) { 221 if (err) {
183 logger.error('Error with adding %s pod.', url, { error: err }) 222 logger.error('Error with adding %s pod.', pod.url, { error: err })
184 } 223 }
224
225 return callback_each_request()
185 }) 226 })
186 } else { 227 } else {
187 logger.error('Error with adding %s pod.', url, { error: err || new Error('Status not 200') }) 228 logger.error('Error with adding %s pod.', pod.url, { error: err || new Error('Status not 200') })
229 return callback_each_request()
188 } 230 }
189 }, 231 },
190 232
diff --git a/src/utils.js b/src/utils.js
index d6b26db4b..dda6c7a0a 100644
--- a/src/utils.js
+++ b/src/utils.js
@@ -1,6 +1,7 @@
1;(function () { 1;(function () {
2 'use strict' 2 'use strict'
3 3
4 var async = require('async')
4 var config = require('config') 5 var config = require('config')
5 var crypto = require('crypto') 6 var crypto = require('crypto')
6 var fs = require('fs') 7 var fs = require('fs')
@@ -30,14 +31,15 @@
30 } 31 }
31 32
32 logger.debug('Sending informations to %s.', to_pod.url, { params: params }) 33 logger.debug('Sending informations to %s.', to_pod.url, { params: params })
34 // Default 10 but in tests we want to be faster
35 var retries = utils.isTestInstance() ? 2 : 10
33 36
34 // Replay 15 times, with factor 3
35 replay( 37 replay(
36 request.post(params, function (err, response, body) { 38 request.post(params, function (err, response, body) {
37 callbackEach(err, response, body, to_pod.url) 39 callbackEach(err, response, body, to_pod)
38 }), 40 }),
39 { 41 {
40 retries: 10, 42 retries: retries,
41 factor: 3, 43 factor: 3,
42 maxTimeout: Infinity, 44 maxTimeout: Infinity,
43 errorCodes: [ 'EADDRINFO', 'ETIMEDOUT', 'ECONNRESET', 'ESOCKETTIMEDOUT', 'ENOTFOUND', 'ECONNREFUSED' ] 45 errorCodes: [ 'EADDRINFO', 'ETIMEDOUT', 'ECONNRESET', 'ESOCKETTIMEDOUT', 'ENOTFOUND', 'ECONNREFUSED' ]
@@ -68,7 +70,13 @@
68 } 70 }
69 71
70 // Make a request for each pod 72 // Make a request for each pod
71 for (var pod of pods) { 73 async.each(pods, function (pod, callback_each_async) {
74 function callbackEachRetryRequest (err, response, body, pod) {
75 callbackEach(err, response, body, pod, function () {
76 callback_each_async()
77 })
78 }
79
72 var params = { 80 var params = {
73 url: pod.url + all_data.path, 81 url: pod.url + all_data.path,
74 method: all_data.method 82 method: all_data.method
@@ -93,20 +101,18 @@
93 key: passwordEncrypted 101 key: passwordEncrypted
94 } 102 }
95 103
96 makeRetryRequest(copy_params, copy_url, copy_pod, copy_signature, callbackEach) 104 makeRetryRequest(copy_params, copy_url, copy_pod, copy_signature, callbackEachRetryRequest)
97 }) 105 })
98 })(crt, params, url, pod, signature) 106 })(crt, params, url, pod, signature)
99 } else { 107 } else {
100 params.json = { data: all_data.data } 108 params.json = { data: all_data.data }
101 makeRetryRequest(params, url, pod, signature, callbackEach) 109 makeRetryRequest(params, url, pod, signature, callbackEachRetryRequest)
102 } 110 }
103 } else { 111 } else {
104 logger.debug('Make a GET/DELETE request') 112 logger.debug('Make a GET/DELETE request')
105 makeRetryRequest(params, url, pod, signature, callbackEach) 113 makeRetryRequest(params, url, pod, signature, callbackEachRetryRequest)
106 } 114 }
107 } 115 }, callback)
108
109 return callback()
110 } 116 }
111 117
112 utils.certsExist = function (callback) { 118 utils.certsExist = function (callback) {
@@ -192,5 +198,9 @@
192 process.kill(-webtorrent_process.pid) 198 process.kill(-webtorrent_process.pid)
193 } 199 }
194 200
201 utils.isTestInstance = function () {
202 return (process.env.NODE_ENV === 'test')
203 }
204
195 module.exports = utils 205 module.exports = utils
196})() 206})()
diff --git a/src/videos.js b/src/videos.js
index b95219c39..8c44cad95 100644
--- a/src/videos.js
+++ b/src/videos.js
@@ -78,14 +78,9 @@
78 data: params 78 data: params
79 } 79 }
80 80
81 pods.makeSecureRequest(data, function (err) { 81 // Do not wait the secure requests
82 if (err) { 82 pods.makeSecureRequest(data)
83 logger.error('Somes issues when sending this video to friends.', { error: err }) 83 callback(null)
84 return callback(err)
85 }
86
87 return callback(null)
88 })
89 }) 84 })
90 }) 85 })
91 } 86 }
@@ -138,14 +133,8 @@
138 } 133 }
139 134
140 // Yes this is a POST request because we add some informations in the body (signature, encrypt etc) 135 // Yes this is a POST request because we add some informations in the body (signature, encrypt etc)
141 pods.makeSecureRequest(data, function (err) { 136 pods.makeSecureRequest(data)
142 if (err) { 137 callback(null)
143 logger.error('Somes issues when sending we want to remove the video to friends.', { error: err })
144 return callback(err)
145 }
146
147 callback(null)
148 })
149 }) 138 })
150 }) 139 })
151 }) 140 })
diff --git a/test/api/friendsAdvanced.js b/test/api/friendsAdvanced.js
new file mode 100644
index 000000000..ccddac4dc
--- /dev/null
+++ b/test/api/friendsAdvanced.js
@@ -0,0 +1,154 @@
1;(function () {
2 'use strict'
3
4 var request = require('supertest')
5 var chai = require('chai')
6 var expect = chai.expect
7
8 var utils = require('../utils')
9
10 describe('Test advanced friends', function () {
11 var path = '/api/v1/pods/makefriends'
12 var apps = []
13 var urls = []
14
15 function makeFriend (pod_number, callback) {
16 // The first pod make friend with the third
17 request(urls[pod_number - 1])
18 .get(path)
19 .set('Accept', 'application/json')
20 .expect(204)
21 .end(function (err, res) {
22 if (err) throw err
23
24 // Wait for the request between pods
25 setTimeout(function () {
26 callback()
27 }, 1000)
28 })
29 }
30
31 function getFriendsList (pod_number, end) {
32 var path = '/api/v1/pods/'
33
34 request(urls[pod_number - 1])
35 .get(path)
36 .set('Accept', 'application/json')
37 .expect(200)
38 .expect('Content-Type', /json/)
39 .end(end)
40 }
41
42 function uploadVideo (pod_number, callback) {
43 var path = '/api/v1/videos'
44
45 request(urls[pod_number - 1])
46 .post(path)
47 .set('Accept', 'application/json')
48 .field('name', 'my super video')
49 .field('description', 'my super description')
50 .attach('input_video', __dirname + '/../fixtures/video_short.webm')
51 .expect(201)
52 .end(function (err) {
53 if (err) throw err
54
55 // Wait for the retry requests
56 setTimeout(callback, 10000)
57 })
58 }
59
60 beforeEach(function (done) {
61 this.timeout(30000)
62 utils.runMultipleServers(6, function (apps_run, urls_run) {
63 apps = apps_run
64 urls = urls_run
65 done()
66 })
67 })
68
69 afterEach(function (done) {
70 apps.forEach(function (app) {
71 process.kill(-app.pid)
72 })
73
74 if (this.ok) {
75 utils.flushTests(function () {
76 done()
77 })
78 } else {
79 done()
80 }
81 })
82
83 it('Should make friends with two pod each in a different group', function (done) {
84 this.timeout(10000)
85
86 // Pod 3 makes friend with the first one
87 makeFriend(3, function () {
88 // Pod 4 makes friend with the second one
89 makeFriend(4, function () {
90 // Now if the fifth wants to make friends with the third et the first
91 makeFriend(5, function () {
92 // It should have 0 friends
93 getFriendsList(5, function (err, res) {
94 if (err) throw err
95
96 expect(res.body.length).to.equal(0)
97
98 done()
99 })
100 })
101 })
102 })
103 })
104
105 it('Should make friends with the pods 1, 2, 3', function (done) {
106 this.timeout(100000)
107
108 // Pods 1, 2, 3 and 4 become friends
109 makeFriend(2, function () {
110 makeFriend(1, function () {
111 makeFriend(4, function () {
112 // Kill the server 4
113 apps[3].kill()
114
115 // Expulse pod 4 from pod 1 and 2
116 uploadVideo(1, function () {
117 uploadVideo(1, function () {
118 uploadVideo(2, function () {
119 uploadVideo(2, function () {
120 // Rerun server 4
121 utils.runServer(4, function (app, url) {
122 apps[3] = app
123 getFriendsList(4, function (err, res) {
124 if (err) throw err
125 // Pod 4 didn't know pod 1 and 2 removed it
126 expect(res.body.length).to.equal(3)
127
128 // Pod 6 ask pod 1, 2 and 3
129 makeFriend(6, function () {
130 getFriendsList(6, function (err, res) {
131 if (err) throw err
132
133 // Pod 4 should not be our friend
134 var result = res.body
135 expect(result.length).to.equal(3)
136 for (var pod of result) {
137 expect(pod.url).not.equal(urls[3])
138 }
139
140 done()
141 })
142 })
143 })
144 })
145 })
146 })
147 })
148 })
149 })
150 })
151 })
152 })
153 })
154})()
diff --git a/test/api/friends.js b/test/api/friendsBasic.js
index 845ccd1a8..40ed34199 100644
--- a/test/api/friends.js
+++ b/test/api/friendsBasic.js
@@ -19,7 +19,7 @@
19 .end(end) 19 .end(end)
20 } 20 }
21 21
22 describe('Test friends', function () { 22 describe('Test basic friends', function () {
23 var apps = [] 23 var apps = []
24 var urls = [] 24 var urls = []
25 25
diff --git a/test/utils.js b/test/utils.js
index 69f43d731..af3e8665d 100644
--- a/test/utils.js
+++ b/test/utils.js
@@ -74,8 +74,8 @@
74 } 74 }
75 75
76 module.exports = { 76 module.exports = {
77 flushTests: flushTests,
77 runMultipleServers: runMultipleServers, 78 runMultipleServers: runMultipleServers,
78 runServer: runServer, 79 runServer: runServer
79 flushTests: flushTests
80 } 80 }
81})() 81})()