aboutsummaryrefslogtreecommitdiffhomepage
diff options
context:
space:
mode:
authorChocobozzz <florian.bigard@gmail.com>2015-12-04 16:13:32 +0100
committerChocobozzz <florian.bigard@gmail.com>2015-12-04 16:13:32 +0100
commit0b69752270f1ceea06a29872b3db23660a55d6d3 (patch)
tree42da726633f3e48f4fe592cfd2c1ca14346a159b
parentaf82cae07dc568e3cb10acd70113df56eb8b15a9 (diff)
downloadPeerTube-0b69752270f1ceea06a29872b3db23660a55d6d3.tar.gz
PeerTube-0b69752270f1ceea06a29872b3db23660a55d6d3.tar.zst
PeerTube-0b69752270f1ceea06a29872b3db23660a55d6d3.zip
Add a pool of requests instead of making a request at each action (add
video/remove video) for performance in big networks
-rw-r--r--middlewares/reqValidators/remote.js9
-rw-r--r--package.json2
-rw-r--r--routes/api/v1/remoteVideos.js8
-rw-r--r--server.js9
-rw-r--r--src/customValidators.js29
-rw-r--r--src/database.js12
-rw-r--r--src/pods.js89
-rw-r--r--src/poolRequests.js158
-rw-r--r--src/utils.js6
-rw-r--r--src/videos.js100
-rw-r--r--test/api/friendsAdvanced.js68
-rw-r--r--test/api/multiplePods.js131
12 files changed, 407 insertions, 214 deletions
diff --git a/middlewares/reqValidators/remote.js b/middlewares/reqValidators/remote.js
index e851b49a4..642dad1c7 100644
--- a/middlewares/reqValidators/remote.js
+++ b/middlewares/reqValidators/remote.js
@@ -18,10 +18,8 @@
18 } 18 }
19 19
20 remote.remoteVideosAdd = function (req, res, next) { 20 remote.remoteVideosAdd = function (req, res, next) {
21 req.checkBody('data.name', 'Should have a name').isLength(1, 50) 21 req.checkBody('data').isArray()
22 req.checkBody('data.description', 'Should have a description').isLength(1, 250) 22 req.checkBody('data').eachIsRemoteVideosAddValid()
23 req.checkBody('data.magnetUri', 'Should have a magnetUri').notEmpty()
24 req.checkBody('data.podUrl', 'Should have a podUrl').isURL()
25 23
26 logger.debug('Checking remoteVideosAdd parameters', { parameters: req.body }) 24 logger.debug('Checking remoteVideosAdd parameters', { parameters: req.body })
27 25
@@ -29,7 +27,8 @@
29 } 27 }
30 28
31 remote.remoteVideosRemove = function (req, res, next) { 29 remote.remoteVideosRemove = function (req, res, next) {
32 req.checkBody('data.magnetUri', 'Should have a magnetUri').notEmpty() 30 req.checkBody('data').isArray()
31 req.checkBody('data').eachIsRemoteVideosRemoveValid()
33 32
34 logger.debug('Checking remoteVideosRemove parameters', { parameters: req.body }) 33 logger.debug('Checking remoteVideosRemove parameters', { parameters: req.body })
35 34
diff --git a/package.json b/package.json
index 9b92e0152..8f2ed9e74 100644
--- a/package.json
+++ b/package.json
@@ -46,6 +46,7 @@
46 "jquery": "^2.1.4", 46 "jquery": "^2.1.4",
47 "js-yaml": "^3.3.1", 47 "js-yaml": "^3.3.1",
48 "load-grunt-tasks": "^3.3.0", 48 "load-grunt-tasks": "^3.3.0",
49 "lodash-node": "3.10.1",
49 "mkdirp": "^0.5.1", 50 "mkdirp": "^0.5.1",
50 "mongoose": "^4.0.5", 51 "mongoose": "^4.0.5",
51 "morgan": "^1.5.3", 52 "morgan": "^1.5.3",
@@ -57,6 +58,7 @@
57 "segfault-handler": "^0.2.4", 58 "segfault-handler": "^0.2.4",
58 "time-grunt": "^1.2.1", 59 "time-grunt": "^1.2.1",
59 "ursa": "^0.9.1", 60 "ursa": "^0.9.1",
61 "validator": "^4.3.0",
60 "webtorrent": "*", 62 "webtorrent": "*",
61 "winston": "^1.0.1", 63 "winston": "^1.0.1",
62 "ws": "^0.8.0" 64 "ws": "^0.8.0"
diff --git a/routes/api/v1/remoteVideos.js b/routes/api/v1/remoteVideos.js
index 10793b2b4..a104113b2 100644
--- a/routes/api/v1/remoteVideos.js
+++ b/routes/api/v1/remoteVideos.js
@@ -3,21 +3,23 @@
3 3
4 var express = require('express') 4 var express = require('express')
5 var router = express.Router() 5 var router = express.Router()
6 var pluck = require('lodash-node/compat/collection/pluck')
7
6 var middleware = require('../../../middlewares') 8 var middleware = require('../../../middlewares')
7 var miscMiddleware = middleware.misc 9 var miscMiddleware = middleware.misc
8 var reqValidator = middleware.reqValidators.remote 10 var reqValidator = middleware.reqValidators.remote
9 var videos = require('../../../src/videos') 11 var videos = require('../../../src/videos')
10 12
11 function addRemoteVideos (req, res, next) { 13 function addRemoteVideos (req, res, next) {
12 videos.addRemote(req.body.data, function (err, video) { 14 videos.addRemotes(req.body.data, function (err, videos) {
13 if (err) return next(err) 15 if (err) return next(err)
14 16
15 res.json(video) 17 res.json(videos)
16 }) 18 })
17 } 19 }
18 20
19 function removeRemoteVideo (req, res, next) { 21 function removeRemoteVideo (req, res, next) {
20 videos.removeRemote(req.body.signature.url, req.body.data.magnetUri, function (err) { 22 videos.removeRemotes(req.body.signature.url, pluck(req.body.data, 'magnetUri'), function (err) {
21 if (err) return next(err) 23 if (err) return next(err)
22 24
23 res.status(204) 25 res.status(204)
diff --git a/server.js b/server.js
index 3b899689c..11402ea78 100644
--- a/server.js
+++ b/server.js
@@ -35,7 +35,9 @@
35 35
36 // ----------- PeerTube modules ----------- 36 // ----------- PeerTube modules -----------
37 var config = require('config') 37 var config = require('config')
38 var customValidators = require('./src/customValidators')
38 var logger = require('./src/logger') 39 var logger = require('./src/logger')
40 var poolRequests = require('./src/poolRequests')
39 var routes = require('./routes') 41 var routes = require('./routes')
40 var videos = require('./src/videos') 42 var videos = require('./src/videos')
41 var webtorrent = require('./src/webTorrentNode') 43 var webtorrent = require('./src/webTorrentNode')
@@ -56,7 +58,9 @@
56 app.use(multer({ dest: uploads })) 58 app.use(multer({ dest: uploads }))
57 app.use(bodyParser.urlencoded({ extended: false })) 59 app.use(bodyParser.urlencoded({ extended: false }))
58 // Validate some params for the API 60 // Validate some params for the API
59 app.use(expressValidator()) 61 app.use(expressValidator({
62 customValidators: customValidators
63 }))
60 64
61 // ----------- Views, routes and static files ----------- 65 // ----------- Views, routes and static files -----------
62 66
@@ -154,6 +158,9 @@
154 158
155 // ----------- Make the server listening ----------- 159 // ----------- Make the server listening -----------
156 server.listen(port, function () { 160 server.listen(port, function () {
161 // Activate the pool requests
162 poolRequests.activate()
163
157 videos.seedAll(function () { 164 videos.seedAll(function () {
158 logger.info('Seeded all the videos') 165 logger.info('Seeded all the videos')
159 logger.info('Server listening on port %d', port) 166 logger.info('Server listening on port %d', port)
diff --git a/src/customValidators.js b/src/customValidators.js
new file mode 100644
index 000000000..73c2f8461
--- /dev/null
+++ b/src/customValidators.js
@@ -0,0 +1,29 @@
1;(function () {
2 'use strict'
3
4 var validator = require('validator')
5
6 var customValidators = {}
7
8 customValidators.eachIsRemoteVideosAddValid = function (values) {
9 return values.every(function (val) {
10 return validator.isLength(val.name, 1, 50) &&
11 validator.isLength(val.description, 1, 50) &&
12 validator.isLength(val.magnetUri, 10) &&
13 validator.isURL(val.podUrl)
14 })
15 }
16
17 customValidators.eachIsRemoteVideosRemoveValid = function (values) {
18 return values.every(function (val) {
19 return validator.isLength(val.magnetUri, 10)
20 })
21 }
22
23 customValidators.isArray = function (value) {
24 return Array.isArray(value)
25 }
26
27 // ----------- Export -----------
28 module.exports = customValidators
29})()
diff --git a/src/database.js b/src/database.js
index 740e89fa4..514a622dc 100644
--- a/src/database.js
+++ b/src/database.js
@@ -30,6 +30,15 @@
30 30
31 var PodsDB = mongoose.model('pods', podsSchema) 31 var PodsDB = mongoose.model('pods', podsSchema)
32 32
33 // ----------- PoolRequests -----------
34 var poolRequestsSchema = mongoose.Schema({
35 type: String,
36 id: String, // Special id to find duplicates (video created we want to remove...)
37 request: mongoose.Schema.Types.Mixed
38 })
39
40 var PoolRequestsDB = mongoose.model('poolRequests', poolRequestsSchema)
41
33 // ----------- Connection ----------- 42 // ----------- Connection -----------
34 43
35 mongoose.connect('mongodb://' + host + ':' + port + '/' + dbname) 44 mongoose.connect('mongodb://' + host + ':' + port + '/' + dbname)
@@ -45,6 +54,7 @@
45 // ----------- Export ----------- 54 // ----------- Export -----------
46 module.exports = { 55 module.exports = {
47 VideosDB: VideosDB, 56 VideosDB: VideosDB,
48 PodsDB: PodsDB 57 PodsDB: PodsDB,
58 PoolRequestsDB: PoolRequestsDB
49 } 59 }
50})() 60})()
diff --git a/src/pods.js b/src/pods.js
index e26b3f0ae..9afc6cc96 100644
--- a/src/pods.js
+++ b/src/pods.js
@@ -8,6 +8,7 @@
8 8
9 var logger = require('./logger') 9 var logger = require('./logger')
10 var PodsDB = require('./database').PodsDB 10 var PodsDB = require('./database').PodsDB
11 var poolRequests = require('./poolRequests')
11 var utils = require('./utils') 12 var utils = require('./utils')
12 13
13 var pods = {} 14 var pods = {}
@@ -16,13 +17,6 @@
16 var host = config.get('webserver.host') 17 var host = config.get('webserver.host')
17 var port = config.get('webserver.port') 18 var port = config.get('webserver.port')
18 19
19 // ----------- Constants -----------
20
21 var PODS_SCORE = {
22 MALUS: -10,
23 BONUS: 10
24 }
25
26 // ----------- Private functions ----------- 20 // ----------- Private functions -----------
27 21
28 function getForeignPodsList (url, callback) { 22 function getForeignPodsList (url, callback) {
@@ -34,25 +28,6 @@
34 }) 28 })
35 } 29 }
36 30
37 function updatePodsScore (good_pods, bad_pods) {
38 logger.info('Updating %d good pods and %d bad pods scores.', good_pods.length, bad_pods.length)
39
40 PodsDB.update({ _id: { $in: good_pods } }, { $inc: { score: PODS_SCORE.BONUS } }, { multi: true }).exec()
41 PodsDB.update({ _id: { $in: bad_pods } }, { $inc: { score: PODS_SCORE.MALUS } }, { multi: true }, function (err) {
42 if (err) throw err
43 removeBadPods()
44 })
45 }
46
47 function removeBadPods () {
48 PodsDB.remove({ score: 0 }, function (err, result) {
49 if (err) throw err
50
51 var number_removed = result.result.n
52 if (number_removed !== 0) logger.info('Removed %d pod.', number_removed)
53 })
54 }
55
56 // ----------- Public functions ----------- 31 // ----------- Public functions -----------
57 32
58 pods.list = function (callback) { 33 pods.list = function (callback) {
@@ -93,58 +68,16 @@
93 }) 68 })
94 } 69 }
95 70
96 // { path, data } 71 pods.addVideoToFriends = function (video) {
97 pods.makeSecureRequest = function (data, callback) { 72 // To avoid duplicates
98 if (callback === undefined) callback = function () {} 73 var id = video.name + video.magnetUri
99 74 poolRequests.addToPoolRequests(id, 'add', video)
100 PodsDB.find({}, { _id: 1, url: 1, publicKey: 1 }).exec(function (err, pods) { 75 }
101 if (err) {
102 logger.error('Cannot get the list of the pods.', { error: err })
103 return callback(err)
104 }
105
106 logger.debug('Make multiple requests.')
107
108 var params = {
109 encrypt: true,
110 sign: true,
111 method: data.method,
112 path: data.path,
113 data: data.data
114 }
115
116 var bad_pods = []
117 var good_pods = []
118
119 utils.makeMultipleRetryRequest(
120 params,
121
122 pods,
123
124 function callbackEachPodFinished (err, response, body, pod, callback_each_pod_finished) {
125 if (err || response.statusCode !== 200) {
126 bad_pods.push(pod._id)
127 logger.error('Error sending secure request to %s/%s pod.', pod.url, data.path, { error: err })
128 } else {
129 good_pods.push(pod._id)
130 }
131
132 return callback_each_pod_finished()
133 },
134
135 function callbackAllPodsFinished (err) {
136 if (err) {
137 logger.error('There was some errors when sending the video meta data.', { error: err })
138 return callback(err)
139 }
140
141 logger.debug('Finished')
142 76
143 updatePodsScore(good_pods, bad_pods) 77 pods.removeVideoToFriends = function (video) {
144 callback(null) 78 // To avoid duplicates
145 } 79 var id = video.name + video.magnetUri
146 ) 80 poolRequests.addToPoolRequests(id, 'remove', video)
147 })
148 } 81 }
149 82
150 pods.makeFriends = function (callback) { 83 pods.makeFriends = function (callback) {
@@ -214,7 +147,7 @@
214 147
215 pods_list, 148 pods_list,
216 149
217 function eachRequest (err, response, body, pod, callback_each_request) { 150 function eachRequest (err, response, body, url, pod, callback_each_request) {
218 // We add the pod if it responded correctly with its public certificate 151 // We add the pod if it responded correctly with its public certificate
219 if (!err && response.statusCode === 200) { 152 if (!err && response.statusCode === 200) {
220 pods.add({ url: pod.url, publicKey: body.cert, score: global.FRIEND_BASE_SCORE }, function (err) { 153 pods.add({ url: pod.url, publicKey: body.cert, score: global.FRIEND_BASE_SCORE }, function (err) {
diff --git a/src/poolRequests.js b/src/poolRequests.js
new file mode 100644
index 000000000..b117c9923
--- /dev/null
+++ b/src/poolRequests.js
@@ -0,0 +1,158 @@
1;(function () {
2 'use strict'
3
4 var async = require('async')
5
6 var logger = require('./logger')
7 var database = require('./database')
8 var PoolRequestsDB = database.PoolRequestsDB
9 var PodsDB = database.PodsDB
10 var utils = require('./utils')
11
12 var poolRequests = {}
13
14 // ----------- Constants -----------
15
16 // Time to wait between requests to the friends
17 var INTERVAL = utils.isTestInstance() ? 10000 : 60000
18 var PODS_SCORE = {
19 MALUS: -10,
20 BONUS: 10
21 }
22
23 // ----------- Private -----------
24 var timer = null
25
26 function makePoolRequests () {
27 logger.info('Making pool requests to friends.')
28
29 PoolRequestsDB.find({}, { type: 1, request: 1 }, function (err, pool_requests) {
30 if (err) throw err
31
32 var requests = {
33 add: [],
34 remove: []
35 }
36
37 async.each(pool_requests, function (pool_request, callback_each) {
38 if (pool_request.type === 'add') {
39 requests.add.push(pool_request.request)
40 } else if (pool_request.type === 'remove') {
41 requests.remove.push(pool_request.request)
42 } else {
43 throw new Error('Unkown pool request type.')
44 }
45
46 callback_each()
47 }, function () {
48 makePoolRequest('add', requests.add)
49 makePoolRequest('remove', requests.remove)
50 logger.info('Pool requests to friends sent.')
51 })
52 })
53 }
54
55 function updatePodsScore (good_pods, bad_pods) {
56 logger.info('Updating %d good pods and %d bad pods scores.', good_pods.length, bad_pods.length)
57
58 PodsDB.update({ _id: { $in: good_pods } }, { $inc: { score: PODS_SCORE.BONUS } }, { multi: true }).exec()
59 PodsDB.update({ _id: { $in: bad_pods } }, { $inc: { score: PODS_SCORE.MALUS } }, { multi: true }, function (err) {
60 if (err) throw err
61 removeBadPods()
62 })
63 }
64
65 function removeBadPods () {
66 PodsDB.remove({ score: 0 }, function (err, result) {
67 if (err) throw err
68
69 var number_removed = result.result.n
70 if (number_removed !== 0) logger.info('Removed %d pod.', number_removed)
71 })
72 }
73
74 function makePoolRequest (type, requests) {
75 logger.debug('Make pool requests scheduled.')
76 PodsDB.find({}, { _id: 1, url: 1, publicKey: 1 }).exec(function (err, pods) {
77 if (err) throw err
78
79 var params = {
80 encrypt: true,
81 sign: true,
82 method: 'POST',
83 path: null,
84 data: requests
85 }
86
87 if (type === 'add') {
88 params.path = '/api/' + global.API_VERSION + '/remotevideos/add'
89 } else if (type === 'remove') {
90 params.path = '/api/' + global.API_VERSION + '/remotevideos/remove'
91 } else {
92 throw new Error('Unkown pool request type.')
93 }
94
95 var bad_pods = []
96 var good_pods = []
97
98 utils.makeMultipleRetryRequest(params, pods, callbackEachPodFinished, callbackAllPodsFinished)
99
100 function callbackEachPodFinished (err, response, body, url, pod, callback_each_pod_finished) {
101 if (err || response.statusCode !== 200) {
102 bad_pods.push(pod._id)
103 logger.error('Error sending secure request to %s pod.', url, { error: err })
104 } else {
105 good_pods.push(pod._id)
106 }
107
108 return callback_each_pod_finished()
109 }
110
111 function callbackAllPodsFinished (err) {
112 if (err) {
113 logger.error('There was some errors when sending the video meta data.', { error: err })
114 }
115
116 updatePodsScore(good_pods, bad_pods)
117 PoolRequestsDB.remove().exec()
118 }
119 })
120 }
121
122 // ----------- Public -----------
123 poolRequests.activate = function () {
124 logger.info('Pool requests activated.')
125 timer = setInterval(makePoolRequests, INTERVAL)
126 }
127
128 poolRequests.addToPoolRequests = function (id, type, request) {
129 logger.debug('Add request to the pool requests.', { id: id, type: type, request: request })
130
131 PoolRequestsDB.findOne({ id: id }, function (err, entity) {
132 if (err) logger.error(err)
133
134 if (entity) {
135 if (entity.type === type) {
136 logger.error(new Error('Cannot insert two same requests.'))
137 return
138 }
139
140 // Remove the request of the other type
141 PoolRequestsDB.remove({ id: id }, function (err) {
142 if (err) logger.error(err)
143 })
144 } else {
145 PoolRequestsDB.create({ id: id, type: type, request: request }, function (err) {
146 if (err) logger.error(err)
147 })
148 }
149 })
150 }
151
152 poolRequests.deactivate = function () {
153 logger.info('Pool requests deactivated.')
154 clearInterval(timer)
155 }
156
157 module.exports = poolRequests
158})()
diff --git a/src/utils.js b/src/utils.js
index dda6c7a0a..4aa1fc55e 100644
--- a/src/utils.js
+++ b/src/utils.js
@@ -36,7 +36,7 @@
36 36
37 replay( 37 replay(
38 request.post(params, function (err, response, body) { 38 request.post(params, function (err, response, body) {
39 callbackEach(err, response, body, to_pod) 39 callbackEach(err, response, body, params.url, to_pod)
40 }), 40 }),
41 { 41 {
42 retries: retries, 42 retries: retries,
@@ -71,8 +71,8 @@
71 71
72 // Make a request for each pod 72 // Make a request for each pod
73 async.each(pods, function (pod, callback_each_async) { 73 async.each(pods, function (pod, callback_each_async) {
74 function callbackEachRetryRequest (err, response, body, pod) { 74 function callbackEachRetryRequest (err, response, body, url, pod) {
75 callbackEach(err, response, body, pod, function () { 75 callbackEach(err, response, body, url, pod, function () {
76 callback_each_async() 76 callback_each_async()
77 }) 77 })
78 } 78 }
diff --git a/src/videos.js b/src/videos.js
index 8c44cad95..e3a5b49f1 100644
--- a/src/videos.js
+++ b/src/videos.js
@@ -3,6 +3,7 @@
3 3
4 var async = require('async') 4 var async = require('async')
5 var config = require('config') 5 var config = require('config')
6 var dz = require('dezalgo')
6 var fs = require('fs') 7 var fs = require('fs')
7 var webtorrent = require('./webTorrentNode') 8 var webtorrent = require('./webTorrentNode')
8 9
@@ -67,19 +68,10 @@
67 return callback(err) 68 return callback(err)
68 } 69 }
69 70
70 // Now we'll send the video's meta data 71 // Now we'll add the video's meta data to our friends
71 params.namePath = null 72 params.namePath = null
72 73
73 logger.info('Sending %s video to friends.', video_file.path) 74 pods.addVideoToFriends(params)
74
75 var data = {
76 path: '/api/' + global.API_VERSION + '/remotevideos/add',
77 method: 'POST',
78 data: params
79 }
80
81 // Do not wait the secure requests
82 pods.makeSecureRequest(data)
83 callback(null) 75 callback(null)
84 }) 76 })
85 }) 77 })
@@ -124,16 +116,12 @@
124 return callback(err) 116 return callback(err)
125 } 117 }
126 118
127 var data = { 119 var params = {
128 path: '/api/' + global.API_VERSION + '/remotevideos/remove', 120 name: video.name,
129 method: 'POST', 121 magnetUri: video.magnetUri
130 data: {
131 magnetUri: video.magnetUri
132 }
133 } 122 }
134 123
135 // Yes this is a POST request because we add some informations in the body (signature, encrypt etc) 124 pods.removeVideoToFriends(params)
136 pods.makeSecureRequest(data)
137 callback(null) 125 callback(null)
138 }) 126 })
139 }) 127 })
@@ -142,49 +130,65 @@
142 } 130 }
143 131
144 // Use the magnet Uri because the _id field is not the same on different servers 132 // Use the magnet Uri because the _id field is not the same on different servers
145 videos.removeRemote = function (fromUrl, magnetUri, callback) { 133 videos.removeRemotes = function (fromUrl, magnetUris, callback) {
146 VideosDB.findOne({ magnetUri: magnetUri }, function (err, video) { 134 VideosDB.find({ magnetUri: { $in: magnetUris } }, function (err, videos) {
147 if (err || !video) { 135 if (err || !videos) {
148 logger.error('Cannot find the torrent URI of this remote video.') 136 logger.error('Cannot find the torrent URI of these remote videos.')
149 return callback(err) 137 return callback(err)
150 } 138 }
151 139
152 // TODO: move to reqValidators middleware ? 140 var to_remove = []
153 if (video.podUrl !== fromUrl) { 141 async.each(videos, function (video, callback_async) {
154 logger.error('The pod has not the rights on this video.') 142 callback_async = dz(callback_async)
155 return callback(err)
156 }
157 143
158 VideosDB.findByIdAndRemove(video._id, function (err) { 144 if (video.podUrl !== fromUrl) {
159 if (err) { 145 logger.error('The pod %s has not the rights on the video of %s.', fromUrl, video.podUrl)
160 logger.error('Cannot remove the remote video.') 146 } else {
161 return callback(err) 147 to_remove.push(video._id)
162 } 148 }
163 149
164 callback(null) 150 callback_async()
151 }, function () {
152 VideosDB.remove({ _id: { $in: to_remove } }, function (err) {
153 if (err) {
154 logger.error('Cannot remove the remote videos.')
155 return callback(err)
156 }
157
158 callback(null)
159 })
165 }) 160 })
166 }) 161 })
167 } 162 }
168 163
169 // { name, magnetUri, podUrl } 164 // { name, magnetUri, podUrl }
170 videos.addRemote = function (data, callback) { 165 videos.addRemotes = function (videos, callback) {
171 logger.debug('Add remote video from pod: %s', data.podUrl) 166 var to_add = []
172
173 var params = {
174 name: data.name,
175 namePath: null,
176 description: data.description,
177 magnetUri: data.magnetUri,
178 podUrl: data.podUrl
179 }
180 167
181 VideosDB.create(params, function (err, video) { 168 async.each(videos, function (video, callback_each) {
182 if (err) { 169 callback_each = dz(callback_each)
183 logger.error('Cannot insert this remote video.', { error: err }) 170 logger.debug('Add remote video from pod: %s', video.podUrl)
184 return callback(err) 171
172 var params = {
173 name: video.name,
174 namePath: null,
175 description: video.description,
176 magnetUri: video.magnetUri,
177 podUrl: video.podUrl
185 } 178 }
186 179
187 return callback(null, video) 180 to_add.push(params)
181
182 callback_each()
183 }, function () {
184 VideosDB.create(to_add, function (err, videos) {
185 if (err) {
186 logger.error('Cannot insert this remote video.', { error: err })
187 return callback(err)
188 }
189
190 return callback(null, videos)
191 })
188 }) 192 })
189 } 193 }
190 194
diff --git a/test/api/friendsAdvanced.js b/test/api/friendsAdvanced.js
index ccddac4dc..680d90aee 100644
--- a/test/api/friendsAdvanced.js
+++ b/test/api/friendsAdvanced.js
@@ -103,9 +103,9 @@
103 }) 103 })
104 104
105 it('Should make friends with the pods 1, 2, 3', function (done) { 105 it('Should make friends with the pods 1, 2, 3', function (done) {
106 this.timeout(100000) 106 this.timeout(150000)
107 107
108 // Pods 1, 2, 3 and 4 become friends 108 // Pods 1, 2, 3 and 4 become friends (yes this is beautiful)
109 makeFriend(2, function () { 109 makeFriend(2, function () {
110 makeFriend(1, function () { 110 makeFriend(1, function () {
111 makeFriend(4, function () { 111 makeFriend(4, function () {
@@ -114,37 +114,45 @@
114 114
115 // Expulse pod 4 from pod 1 and 2 115 // Expulse pod 4 from pod 1 and 2
116 uploadVideo(1, function () { 116 uploadVideo(1, function () {
117 uploadVideo(1, function () { 117 setTimeout(function () {
118 uploadVideo(2, function () { 118 uploadVideo(1, function () {
119 uploadVideo(2, function () { 119 setTimeout(function () {
120 // Rerun server 4 120 uploadVideo(2, function () {
121 utils.runServer(4, function (app, url) { 121 setTimeout(function () {
122 apps[3] = app 122 uploadVideo(2, function () {
123 getFriendsList(4, function (err, res) { 123 setTimeout(function () {
124 if (err) throw err 124 // Rerun server 4
125 // Pod 4 didn't know pod 1 and 2 removed it 125 utils.runServer(4, function (app, url) {
126 expect(res.body.length).to.equal(3) 126 apps[3] = app
127 127 getFriendsList(4, function (err, res) {
128 // Pod 6 ask pod 1, 2 and 3 128 if (err) throw err
129 makeFriend(6, function () { 129 // Pod 4 didn't know pod 1 and 2 removed it
130 getFriendsList(6, function (err, res) { 130 expect(res.body.length).to.equal(3)
131 if (err) throw err 131
132 132 // Pod 6 ask pod 1, 2 and 3
133 // Pod 4 should not be our friend 133 makeFriend(6, function () {
134 var result = res.body 134 getFriendsList(6, function (err, res) {
135 expect(result.length).to.equal(3) 135 if (err) throw err
136 for (var pod of result) { 136
137 expect(pod.url).not.equal(urls[3]) 137 // Pod 4 should not be our friend
138 } 138 var result = res.body
139 139 expect(result.length).to.equal(3)
140 done() 140 for (var pod of result) {
141 }) 141 expect(pod.url).not.equal(urls[3])
142 }
143
144 done()
145 })
146 })
147 })
148 })
149 }, 11000)
142 }) 150 })
143 }) 151 }, 11000)
144 }) 152 })
145 }) 153 }, 11000)
146 }) 154 })
147 }) 155 }, 11000)
148 }) 156 })
149 }) 157 })
150 }) 158 })
diff --git a/test/api/multiplePods.js b/test/api/multiplePods.js
index 1edfc1ce3..3ce57afa8 100644
--- a/test/api/multiplePods.js
+++ b/test/api/multiplePods.js
@@ -14,7 +14,7 @@
14 var path = '/api/v1/videos' 14 var path = '/api/v1/videos'
15 var apps = [] 15 var apps = []
16 var urls = [] 16 var urls = []
17 var video_id = -1 17 var to_remove = []
18 18
19 function getVideosList (url, end) { 19 function getVideosList (url, end) {
20 request(url) 20 request(url)
@@ -36,6 +36,14 @@
36 .end(end) 36 .end(end)
37 } 37 }
38 38
39 function removeVideo (url, id, end) {
40 request(url)
41 .delete(path + '/' + id)
42 .set('Accept', 'application/json')
43 .expect(204)
44 .end(end)
45 }
46
39 before(function (done) { 47 before(function (done) {
40 this.timeout(30000) 48 this.timeout(30000)
41 var path_friends = '/api/v1/pods/makefriends' 49 var path_friends = '/api/v1/pods/makefriends'
@@ -89,7 +97,7 @@
89 97
90 describe('Should upload the video and propagate on each pod', function () { 98 describe('Should upload the video and propagate on each pod', function () {
91 it('Should upload the video on pod 1 and propagate on each pod', function (done) { 99 it('Should upload the video on pod 1 and propagate on each pod', function (done) {
92 this.timeout(5000) 100 this.timeout(15000)
93 101
94 uploadVideo(urls[0], 'my super name for pod 1', 'my super description for pod 1', 'video_short1.webm', function (err) { 102 uploadVideo(urls[0], 'my super name for pod 1', 'my super description for pod 1', 'video_short1.webm', function (err) {
95 if (err) throw err 103 if (err) throw err
@@ -125,12 +133,12 @@
125 133
126 done() 134 done()
127 }) 135 })
128 }, 1000) 136 }, 11000)
129 }) 137 })
130 }) 138 })
131 139
132 it('Should upload the video on pod 2 and propagate on each pod', function (done) { 140 it('Should upload the video on pod 2 and propagate on each pod', function (done) {
133 this.timeout(5000) 141 this.timeout(15000)
134 142
135 uploadVideo(urls[1], 'my super name for pod 2', 'my super description for pod 2', 'video_short2.webm', function (err) { 143 uploadVideo(urls[1], 'my super name for pod 2', 'my super description for pod 2', 'video_short2.webm', function (err) {
136 if (err) throw err 144 if (err) throw err
@@ -166,47 +174,56 @@
166 174
167 done() 175 done()
168 }) 176 })
169 }, 1000) 177 }, 11000)
170 }) 178 })
171 }) 179 })
172 180
173 it('Should upload the video on pod 3 and propagate on each pod', function (done) { 181 it('Should upload two videos on pod 3 and propagate on each pod', function (done) {
174 this.timeout(5000) 182 this.timeout(15000)
175 183
176 uploadVideo(urls[2], 'my super name for pod 3', 'my super description for pod 3', 'video_short3.webm', function (err) { 184 uploadVideo(urls[2], 'my super name for pod 3', 'my super description for pod 3', 'video_short3.webm', function (err) {
177 if (err) throw err 185 if (err) throw err
186 uploadVideo(urls[2], 'my super name for pod 3-2', 'my super description for pod 3-2', 'video_short.webm', function (err) {
187 if (err) throw err
178 188
179 setTimeout(function () { 189 setTimeout(function () {
180 var base_magnet = null 190 var base_magnet = null
181 // All pods should have this video 191 // All pods should have this video
182 async.each(urls, function (url, callback) { 192 async.each(urls, function (url, callback) {
183 getVideosList(url, function (err, res) { 193 getVideosList(url, function (err, res) {
184 if (err) throw err 194 if (err) throw err
185
186 var videos = res.body
187 expect(videos).to.be.an('array')
188 expect(videos.length).to.equal(3)
189 var video = videos[2]
190 expect(video.name).to.equal('my super name for pod 3')
191 expect(video.description).to.equal('my super description for pod 3')
192 expect(video.podUrl).to.equal('http://localhost:9003')
193 expect(video.magnetUri).to.exist
194 195
195 // All pods should have the same magnet Uri 196 var videos = res.body
196 if (base_magnet === null) { 197 expect(videos).to.be.an('array')
197 base_magnet = video.magnetUri 198 expect(videos.length).to.equal(4)
198 } else { 199 var video = videos[2]
199 expect(video.magnetUri).to.equal.magnetUri 200 expect(video.name).to.equal('my super name for pod 3')
200 } 201 expect(video.description).to.equal('my super description for pod 3')
202 expect(video.podUrl).to.equal('http://localhost:9003')
203 expect(video.magnetUri).to.exist
204
205 video = videos[3]
206 expect(video.name).to.equal('my super name for pod 3-2')
207 expect(video.description).to.equal('my super description for pod 3-2')
208 expect(video.podUrl).to.equal('http://localhost:9003')
209 expect(video.magnetUri).to.exist
210
211 // All pods should have the same magnet Uri
212 if (base_magnet === null) {
213 base_magnet = video.magnetUri
214 } else {
215 expect(video.magnetUri).to.equal.magnetUri
216 }
217
218 callback()
219 })
220 }, function (err) {
221 if (err) throw err
201 222
202 callback() 223 done()
203 }) 224 })
204 }, function (err) { 225 }, 11000)
205 if (err) throw err 226 })
206
207 done()
208 })
209 }, 1000)
210 }) 227 })
211 }) 228 })
212 }) 229 })
@@ -220,6 +237,9 @@
220 if (err) throw err 237 if (err) throw err
221 238
222 var video = res.body[0] 239 var video = res.body[0]
240 to_remove.push(res.body[2]._id)
241 to_remove.push(res.body[3]._id)
242
223 webtorrent.add(video.magnetUri, function (torrent) { 243 webtorrent.add(video.magnetUri, function (torrent) {
224 expect(torrent.files).to.exist 244 expect(torrent.files).to.exist
225 expect(torrent.files.length).to.equal(1) 245 expect(torrent.files.length).to.equal(1)
@@ -257,7 +277,6 @@
257 if (err) throw err 277 if (err) throw err
258 278
259 var video = res.body[2] 279 var video = res.body[2]
260 video_id = res.body[1]._id
261 280
262 webtorrent.add(video.magnetUri, function (torrent) { 281 webtorrent.add(video.magnetUri, function (torrent) {
263 expect(torrent.files).to.exist 282 expect(torrent.files).to.exist
@@ -269,19 +288,39 @@
269 }) 288 })
270 }) 289 })
271 290
272 it('Should remove the file 2 by asking pod 2', function (done) { 291 it('Should add the file 3-2 by asking pod 1', function (done) {
273 request(urls[1]) 292 // Yes, this could be long
274 .delete(path + '/' + video_id) 293 this.timeout(200000)
275 .set('Accept', 'application/json') 294
276 .expect(204) 295 getVideosList(urls[0], function (err, res) {
277 .end(function (err, res) { 296 if (err) throw err
297
298 var video = res.body[3]
299
300 webtorrent.add(video.magnetUri, function (torrent) {
301 expect(torrent.files).to.exist
302 expect(torrent.files.length).to.equal(1)
303 expect(torrent.files[0].path).to.exist.and.to.not.equal('')
304
305 done()
306 })
307 })
308 })
309
310 it('Should remove the file 3 and 3-2 by asking pod 3', function (done) {
311 this.timeout(15000)
312
313 removeVideo(urls[2], to_remove[0], function (err) {
314 if (err) throw err
315 removeVideo(urls[2], to_remove[1], function (err) {
278 if (err) throw err 316 if (err) throw err
279 317
280 // Wait the propagation to the other pods 318 // Wait the propagation to the other pods
281 setTimeout(function () { 319 setTimeout(function () {
282 done() 320 done()
283 }, 1000) 321 }, 11000)
284 }) 322 })
323 })
285 }) 324 })
286 325
287 it('Should have videos 1 and 3 on each pod', function (done) { 326 it('Should have videos 1 and 3 on each pod', function (done) {
@@ -293,8 +332,10 @@
293 expect(videos).to.be.an('array') 332 expect(videos).to.be.an('array')
294 expect(videos.length).to.equal(2) 333 expect(videos.length).to.equal(2)
295 expect(videos[0]._id).not.to.equal(videos[1]._id) 334 expect(videos[0]._id).not.to.equal(videos[1]._id)
296 expect(videos[0]._id).not.to.equal(video_id) 335 expect(videos[0]._id).not.to.equal(to_remove[0])
297 expect(videos[1]._id).not.to.equal(video_id) 336 expect(videos[1]._id).not.to.equal(to_remove[0])
337 expect(videos[0]._id).not.to.equal(to_remove[1])
338 expect(videos[1]._id).not.to.equal(to_remove[1])
298 339
299 callback() 340 callback()
300 }) 341 })