diff options
-rw-r--r-- | middlewares/reqValidators/remote.js | 9 | ||||
-rw-r--r-- | package.json | 2 | ||||
-rw-r--r-- | routes/api/v1/remoteVideos.js | 8 | ||||
-rw-r--r-- | server.js | 9 | ||||
-rw-r--r-- | src/customValidators.js | 29 | ||||
-rw-r--r-- | src/database.js | 12 | ||||
-rw-r--r-- | src/pods.js | 89 | ||||
-rw-r--r-- | src/poolRequests.js | 158 | ||||
-rw-r--r-- | src/utils.js | 6 | ||||
-rw-r--r-- | src/videos.js | 100 | ||||
-rw-r--r-- | test/api/friendsAdvanced.js | 68 | ||||
-rw-r--r-- | test/api/multiplePods.js | 131 |
12 files changed, 407 insertions, 214 deletions
diff --git a/middlewares/reqValidators/remote.js b/middlewares/reqValidators/remote.js index e851b49a4..642dad1c7 100644 --- a/middlewares/reqValidators/remote.js +++ b/middlewares/reqValidators/remote.js | |||
@@ -18,10 +18,8 @@ | |||
18 | } | 18 | } |
19 | 19 | ||
20 | remote.remoteVideosAdd = function (req, res, next) { | 20 | remote.remoteVideosAdd = function (req, res, next) { |
21 | req.checkBody('data.name', 'Should have a name').isLength(1, 50) | 21 | req.checkBody('data').isArray() |
22 | req.checkBody('data.description', 'Should have a description').isLength(1, 250) | 22 | req.checkBody('data').eachIsRemoteVideosAddValid() |
23 | req.checkBody('data.magnetUri', 'Should have a magnetUri').notEmpty() | ||
24 | req.checkBody('data.podUrl', 'Should have a podUrl').isURL() | ||
25 | 23 | ||
26 | logger.debug('Checking remoteVideosAdd parameters', { parameters: req.body }) | 24 | logger.debug('Checking remoteVideosAdd parameters', { parameters: req.body }) |
27 | 25 | ||
@@ -29,7 +27,8 @@ | |||
29 | } | 27 | } |
30 | 28 | ||
31 | remote.remoteVideosRemove = function (req, res, next) { | 29 | remote.remoteVideosRemove = function (req, res, next) { |
32 | req.checkBody('data.magnetUri', 'Should have a magnetUri').notEmpty() | 30 | req.checkBody('data').isArray() |
31 | req.checkBody('data').eachIsRemoteVideosRemoveValid() | ||
33 | 32 | ||
34 | logger.debug('Checking remoteVideosRemove parameters', { parameters: req.body }) | 33 | logger.debug('Checking remoteVideosRemove parameters', { parameters: req.body }) |
35 | 34 | ||
diff --git a/package.json b/package.json index 9b92e0152..8f2ed9e74 100644 --- a/package.json +++ b/package.json | |||
@@ -46,6 +46,7 @@ | |||
46 | "jquery": "^2.1.4", | 46 | "jquery": "^2.1.4", |
47 | "js-yaml": "^3.3.1", | 47 | "js-yaml": "^3.3.1", |
48 | "load-grunt-tasks": "^3.3.0", | 48 | "load-grunt-tasks": "^3.3.0", |
49 | "lodash-node": "3.10.1", | ||
49 | "mkdirp": "^0.5.1", | 50 | "mkdirp": "^0.5.1", |
50 | "mongoose": "^4.0.5", | 51 | "mongoose": "^4.0.5", |
51 | "morgan": "^1.5.3", | 52 | "morgan": "^1.5.3", |
@@ -57,6 +58,7 @@ | |||
57 | "segfault-handler": "^0.2.4", | 58 | "segfault-handler": "^0.2.4", |
58 | "time-grunt": "^1.2.1", | 59 | "time-grunt": "^1.2.1", |
59 | "ursa": "^0.9.1", | 60 | "ursa": "^0.9.1", |
61 | "validator": "^4.3.0", | ||
60 | "webtorrent": "*", | 62 | "webtorrent": "*", |
61 | "winston": "^1.0.1", | 63 | "winston": "^1.0.1", |
62 | "ws": "^0.8.0" | 64 | "ws": "^0.8.0" |
diff --git a/routes/api/v1/remoteVideos.js b/routes/api/v1/remoteVideos.js index 10793b2b4..a104113b2 100644 --- a/routes/api/v1/remoteVideos.js +++ b/routes/api/v1/remoteVideos.js | |||
@@ -3,21 +3,23 @@ | |||
3 | 3 | ||
4 | var express = require('express') | 4 | var express = require('express') |
5 | var router = express.Router() | 5 | var router = express.Router() |
6 | var pluck = require('lodash-node/compat/collection/pluck') | ||
7 | |||
6 | var middleware = require('../../../middlewares') | 8 | var middleware = require('../../../middlewares') |
7 | var miscMiddleware = middleware.misc | 9 | var miscMiddleware = middleware.misc |
8 | var reqValidator = middleware.reqValidators.remote | 10 | var reqValidator = middleware.reqValidators.remote |
9 | var videos = require('../../../src/videos') | 11 | var videos = require('../../../src/videos') |
10 | 12 | ||
11 | function addRemoteVideos (req, res, next) { | 13 | function addRemoteVideos (req, res, next) { |
12 | videos.addRemote(req.body.data, function (err, video) { | 14 | videos.addRemotes(req.body.data, function (err, videos) { |
13 | if (err) return next(err) | 15 | if (err) return next(err) |
14 | 16 | ||
15 | res.json(video) | 17 | res.json(videos) |
16 | }) | 18 | }) |
17 | } | 19 | } |
18 | 20 | ||
19 | function removeRemoteVideo (req, res, next) { | 21 | function removeRemoteVideo (req, res, next) { |
20 | videos.removeRemote(req.body.signature.url, req.body.data.magnetUri, function (err) { | 22 | videos.removeRemotes(req.body.signature.url, pluck(req.body.data, 'magnetUri'), function (err) { |
21 | if (err) return next(err) | 23 | if (err) return next(err) |
22 | 24 | ||
23 | res.status(204) | 25 | res.status(204) |
@@ -35,7 +35,9 @@ | |||
35 | 35 | ||
36 | // ----------- PeerTube modules ----------- | 36 | // ----------- PeerTube modules ----------- |
37 | var config = require('config') | 37 | var config = require('config') |
38 | var customValidators = require('./src/customValidators') | ||
38 | var logger = require('./src/logger') | 39 | var logger = require('./src/logger') |
40 | var poolRequests = require('./src/poolRequests') | ||
39 | var routes = require('./routes') | 41 | var routes = require('./routes') |
40 | var videos = require('./src/videos') | 42 | var videos = require('./src/videos') |
41 | var webtorrent = require('./src/webTorrentNode') | 43 | var webtorrent = require('./src/webTorrentNode') |
@@ -56,7 +58,9 @@ | |||
56 | app.use(multer({ dest: uploads })) | 58 | app.use(multer({ dest: uploads })) |
57 | app.use(bodyParser.urlencoded({ extended: false })) | 59 | app.use(bodyParser.urlencoded({ extended: false })) |
58 | // Validate some params for the API | 60 | // Validate some params for the API |
59 | app.use(expressValidator()) | 61 | app.use(expressValidator({ |
62 | customValidators: customValidators | ||
63 | })) | ||
60 | 64 | ||
61 | // ----------- Views, routes and static files ----------- | 65 | // ----------- Views, routes and static files ----------- |
62 | 66 | ||
@@ -154,6 +158,9 @@ | |||
154 | 158 | ||
155 | // ----------- Make the server listening ----------- | 159 | // ----------- Make the server listening ----------- |
156 | server.listen(port, function () { | 160 | server.listen(port, function () { |
161 | // Activate the pool requests | ||
162 | poolRequests.activate() | ||
163 | |||
157 | videos.seedAll(function () { | 164 | videos.seedAll(function () { |
158 | logger.info('Seeded all the videos') | 165 | logger.info('Seeded all the videos') |
159 | logger.info('Server listening on port %d', port) | 166 | logger.info('Server listening on port %d', port) |
diff --git a/src/customValidators.js b/src/customValidators.js new file mode 100644 index 000000000..73c2f8461 --- /dev/null +++ b/src/customValidators.js | |||
@@ -0,0 +1,29 @@ | |||
1 | ;(function () { | ||
2 | 'use strict' | ||
3 | |||
4 | var validator = require('validator') | ||
5 | |||
6 | var customValidators = {} | ||
7 | |||
8 | customValidators.eachIsRemoteVideosAddValid = function (values) { | ||
9 | return values.every(function (val) { | ||
10 | return validator.isLength(val.name, 1, 50) && | ||
11 | validator.isLength(val.description, 1, 50) && | ||
12 | validator.isLength(val.magnetUri, 10) && | ||
13 | validator.isURL(val.podUrl) | ||
14 | }) | ||
15 | } | ||
16 | |||
17 | customValidators.eachIsRemoteVideosRemoveValid = function (values) { | ||
18 | return values.every(function (val) { | ||
19 | return validator.isLength(val.magnetUri, 10) | ||
20 | }) | ||
21 | } | ||
22 | |||
23 | customValidators.isArray = function (value) { | ||
24 | return Array.isArray(value) | ||
25 | } | ||
26 | |||
27 | // ----------- Export ----------- | ||
28 | module.exports = customValidators | ||
29 | })() | ||
diff --git a/src/database.js b/src/database.js index 740e89fa4..514a622dc 100644 --- a/src/database.js +++ b/src/database.js | |||
@@ -30,6 +30,15 @@ | |||
30 | 30 | ||
31 | var PodsDB = mongoose.model('pods', podsSchema) | 31 | var PodsDB = mongoose.model('pods', podsSchema) |
32 | 32 | ||
33 | // ----------- PoolRequests ----------- | ||
34 | var poolRequestsSchema = mongoose.Schema({ | ||
35 | type: String, | ||
36 | id: String, // Special id to find duplicates (video created we want to remove...) | ||
37 | request: mongoose.Schema.Types.Mixed | ||
38 | }) | ||
39 | |||
40 | var PoolRequestsDB = mongoose.model('poolRequests', poolRequestsSchema) | ||
41 | |||
33 | // ----------- Connection ----------- | 42 | // ----------- Connection ----------- |
34 | 43 | ||
35 | mongoose.connect('mongodb://' + host + ':' + port + '/' + dbname) | 44 | mongoose.connect('mongodb://' + host + ':' + port + '/' + dbname) |
@@ -45,6 +54,7 @@ | |||
45 | // ----------- Export ----------- | 54 | // ----------- Export ----------- |
46 | module.exports = { | 55 | module.exports = { |
47 | VideosDB: VideosDB, | 56 | VideosDB: VideosDB, |
48 | PodsDB: PodsDB | 57 | PodsDB: PodsDB, |
58 | PoolRequestsDB: PoolRequestsDB | ||
49 | } | 59 | } |
50 | })() | 60 | })() |
diff --git a/src/pods.js b/src/pods.js index e26b3f0ae..9afc6cc96 100644 --- a/src/pods.js +++ b/src/pods.js | |||
@@ -8,6 +8,7 @@ | |||
8 | 8 | ||
9 | var logger = require('./logger') | 9 | var logger = require('./logger') |
10 | var PodsDB = require('./database').PodsDB | 10 | var PodsDB = require('./database').PodsDB |
11 | var poolRequests = require('./poolRequests') | ||
11 | var utils = require('./utils') | 12 | var utils = require('./utils') |
12 | 13 | ||
13 | var pods = {} | 14 | var pods = {} |
@@ -16,13 +17,6 @@ | |||
16 | var host = config.get('webserver.host') | 17 | var host = config.get('webserver.host') |
17 | var port = config.get('webserver.port') | 18 | var port = config.get('webserver.port') |
18 | 19 | ||
19 | // ----------- Constants ----------- | ||
20 | |||
21 | var PODS_SCORE = { | ||
22 | MALUS: -10, | ||
23 | BONUS: 10 | ||
24 | } | ||
25 | |||
26 | // ----------- Private functions ----------- | 20 | // ----------- Private functions ----------- |
27 | 21 | ||
28 | function getForeignPodsList (url, callback) { | 22 | function getForeignPodsList (url, callback) { |
@@ -34,25 +28,6 @@ | |||
34 | }) | 28 | }) |
35 | } | 29 | } |
36 | 30 | ||
37 | function updatePodsScore (good_pods, bad_pods) { | ||
38 | logger.info('Updating %d good pods and %d bad pods scores.', good_pods.length, bad_pods.length) | ||
39 | |||
40 | PodsDB.update({ _id: { $in: good_pods } }, { $inc: { score: PODS_SCORE.BONUS } }, { multi: true }).exec() | ||
41 | PodsDB.update({ _id: { $in: bad_pods } }, { $inc: { score: PODS_SCORE.MALUS } }, { multi: true }, function (err) { | ||
42 | if (err) throw err | ||
43 | removeBadPods() | ||
44 | }) | ||
45 | } | ||
46 | |||
47 | function removeBadPods () { | ||
48 | PodsDB.remove({ score: 0 }, function (err, result) { | ||
49 | if (err) throw err | ||
50 | |||
51 | var number_removed = result.result.n | ||
52 | if (number_removed !== 0) logger.info('Removed %d pod.', number_removed) | ||
53 | }) | ||
54 | } | ||
55 | |||
56 | // ----------- Public functions ----------- | 31 | // ----------- Public functions ----------- |
57 | 32 | ||
58 | pods.list = function (callback) { | 33 | pods.list = function (callback) { |
@@ -93,58 +68,16 @@ | |||
93 | }) | 68 | }) |
94 | } | 69 | } |
95 | 70 | ||
96 | // { path, data } | 71 | pods.addVideoToFriends = function (video) { |
97 | pods.makeSecureRequest = function (data, callback) { | 72 | // To avoid duplicates |
98 | if (callback === undefined) callback = function () {} | 73 | var id = video.name + video.magnetUri |
99 | 74 | poolRequests.addToPoolRequests(id, 'add', video) | |
100 | PodsDB.find({}, { _id: 1, url: 1, publicKey: 1 }).exec(function (err, pods) { | 75 | } |
101 | if (err) { | ||
102 | logger.error('Cannot get the list of the pods.', { error: err }) | ||
103 | return callback(err) | ||
104 | } | ||
105 | |||
106 | logger.debug('Make multiple requests.') | ||
107 | |||
108 | var params = { | ||
109 | encrypt: true, | ||
110 | sign: true, | ||
111 | method: data.method, | ||
112 | path: data.path, | ||
113 | data: data.data | ||
114 | } | ||
115 | |||
116 | var bad_pods = [] | ||
117 | var good_pods = [] | ||
118 | |||
119 | utils.makeMultipleRetryRequest( | ||
120 | params, | ||
121 | |||
122 | pods, | ||
123 | |||
124 | function callbackEachPodFinished (err, response, body, pod, callback_each_pod_finished) { | ||
125 | if (err || response.statusCode !== 200) { | ||
126 | bad_pods.push(pod._id) | ||
127 | logger.error('Error sending secure request to %s/%s pod.', pod.url, data.path, { error: err }) | ||
128 | } else { | ||
129 | good_pods.push(pod._id) | ||
130 | } | ||
131 | |||
132 | return callback_each_pod_finished() | ||
133 | }, | ||
134 | |||
135 | function callbackAllPodsFinished (err) { | ||
136 | if (err) { | ||
137 | logger.error('There was some errors when sending the video meta data.', { error: err }) | ||
138 | return callback(err) | ||
139 | } | ||
140 | |||
141 | logger.debug('Finished') | ||
142 | 76 | ||
143 | updatePodsScore(good_pods, bad_pods) | 77 | pods.removeVideoToFriends = function (video) { |
144 | callback(null) | 78 | // To avoid duplicates |
145 | } | 79 | var id = video.name + video.magnetUri |
146 | ) | 80 | poolRequests.addToPoolRequests(id, 'remove', video) |
147 | }) | ||
148 | } | 81 | } |
149 | 82 | ||
150 | pods.makeFriends = function (callback) { | 83 | pods.makeFriends = function (callback) { |
@@ -214,7 +147,7 @@ | |||
214 | 147 | ||
215 | pods_list, | 148 | pods_list, |
216 | 149 | ||
217 | function eachRequest (err, response, body, pod, callback_each_request) { | 150 | function eachRequest (err, response, body, url, pod, callback_each_request) { |
218 | // We add the pod if it responded correctly with its public certificate | 151 | // We add the pod if it responded correctly with its public certificate |
219 | if (!err && response.statusCode === 200) { | 152 | if (!err && response.statusCode === 200) { |
220 | pods.add({ url: pod.url, publicKey: body.cert, score: global.FRIEND_BASE_SCORE }, function (err) { | 153 | pods.add({ url: pod.url, publicKey: body.cert, score: global.FRIEND_BASE_SCORE }, function (err) { |
diff --git a/src/poolRequests.js b/src/poolRequests.js new file mode 100644 index 000000000..b117c9923 --- /dev/null +++ b/src/poolRequests.js | |||
@@ -0,0 +1,158 @@ | |||
1 | ;(function () { | ||
2 | 'use strict' | ||
3 | |||
4 | var async = require('async') | ||
5 | |||
6 | var logger = require('./logger') | ||
7 | var database = require('./database') | ||
8 | var PoolRequestsDB = database.PoolRequestsDB | ||
9 | var PodsDB = database.PodsDB | ||
10 | var utils = require('./utils') | ||
11 | |||
12 | var poolRequests = {} | ||
13 | |||
14 | // ----------- Constants ----------- | ||
15 | |||
16 | // Time to wait between requests to the friends | ||
17 | var INTERVAL = utils.isTestInstance() ? 10000 : 60000 | ||
18 | var PODS_SCORE = { | ||
19 | MALUS: -10, | ||
20 | BONUS: 10 | ||
21 | } | ||
22 | |||
23 | // ----------- Private ----------- | ||
24 | var timer = null | ||
25 | |||
26 | function makePoolRequests () { | ||
27 | logger.info('Making pool requests to friends.') | ||
28 | |||
29 | PoolRequestsDB.find({}, { type: 1, request: 1 }, function (err, pool_requests) { | ||
30 | if (err) throw err | ||
31 | |||
32 | var requests = { | ||
33 | add: [], | ||
34 | remove: [] | ||
35 | } | ||
36 | |||
37 | async.each(pool_requests, function (pool_request, callback_each) { | ||
38 | if (pool_request.type === 'add') { | ||
39 | requests.add.push(pool_request.request) | ||
40 | } else if (pool_request.type === 'remove') { | ||
41 | requests.remove.push(pool_request.request) | ||
42 | } else { | ||
43 | throw new Error('Unkown pool request type.') | ||
44 | } | ||
45 | |||
46 | callback_each() | ||
47 | }, function () { | ||
48 | makePoolRequest('add', requests.add) | ||
49 | makePoolRequest('remove', requests.remove) | ||
50 | logger.info('Pool requests to friends sent.') | ||
51 | }) | ||
52 | }) | ||
53 | } | ||
54 | |||
55 | function updatePodsScore (good_pods, bad_pods) { | ||
56 | logger.info('Updating %d good pods and %d bad pods scores.', good_pods.length, bad_pods.length) | ||
57 | |||
58 | PodsDB.update({ _id: { $in: good_pods } }, { $inc: { score: PODS_SCORE.BONUS } }, { multi: true }).exec() | ||
59 | PodsDB.update({ _id: { $in: bad_pods } }, { $inc: { score: PODS_SCORE.MALUS } }, { multi: true }, function (err) { | ||
60 | if (err) throw err | ||
61 | removeBadPods() | ||
62 | }) | ||
63 | } | ||
64 | |||
65 | function removeBadPods () { | ||
66 | PodsDB.remove({ score: 0 }, function (err, result) { | ||
67 | if (err) throw err | ||
68 | |||
69 | var number_removed = result.result.n | ||
70 | if (number_removed !== 0) logger.info('Removed %d pod.', number_removed) | ||
71 | }) | ||
72 | } | ||
73 | |||
74 | function makePoolRequest (type, requests) { | ||
75 | logger.debug('Make pool requests scheduled.') | ||
76 | PodsDB.find({}, { _id: 1, url: 1, publicKey: 1 }).exec(function (err, pods) { | ||
77 | if (err) throw err | ||
78 | |||
79 | var params = { | ||
80 | encrypt: true, | ||
81 | sign: true, | ||
82 | method: 'POST', | ||
83 | path: null, | ||
84 | data: requests | ||
85 | } | ||
86 | |||
87 | if (type === 'add') { | ||
88 | params.path = '/api/' + global.API_VERSION + '/remotevideos/add' | ||
89 | } else if (type === 'remove') { | ||
90 | params.path = '/api/' + global.API_VERSION + '/remotevideos/remove' | ||
91 | } else { | ||
92 | throw new Error('Unkown pool request type.') | ||
93 | } | ||
94 | |||
95 | var bad_pods = [] | ||
96 | var good_pods = [] | ||
97 | |||
98 | utils.makeMultipleRetryRequest(params, pods, callbackEachPodFinished, callbackAllPodsFinished) | ||
99 | |||
100 | function callbackEachPodFinished (err, response, body, url, pod, callback_each_pod_finished) { | ||
101 | if (err || response.statusCode !== 200) { | ||
102 | bad_pods.push(pod._id) | ||
103 | logger.error('Error sending secure request to %s pod.', url, { error: err }) | ||
104 | } else { | ||
105 | good_pods.push(pod._id) | ||
106 | } | ||
107 | |||
108 | return callback_each_pod_finished() | ||
109 | } | ||
110 | |||
111 | function callbackAllPodsFinished (err) { | ||
112 | if (err) { | ||
113 | logger.error('There was some errors when sending the video meta data.', { error: err }) | ||
114 | } | ||
115 | |||
116 | updatePodsScore(good_pods, bad_pods) | ||
117 | PoolRequestsDB.remove().exec() | ||
118 | } | ||
119 | }) | ||
120 | } | ||
121 | |||
122 | // ----------- Public ----------- | ||
123 | poolRequests.activate = function () { | ||
124 | logger.info('Pool requests activated.') | ||
125 | timer = setInterval(makePoolRequests, INTERVAL) | ||
126 | } | ||
127 | |||
128 | poolRequests.addToPoolRequests = function (id, type, request) { | ||
129 | logger.debug('Add request to the pool requests.', { id: id, type: type, request: request }) | ||
130 | |||
131 | PoolRequestsDB.findOne({ id: id }, function (err, entity) { | ||
132 | if (err) logger.error(err) | ||
133 | |||
134 | if (entity) { | ||
135 | if (entity.type === type) { | ||
136 | logger.error(new Error('Cannot insert two same requests.')) | ||
137 | return | ||
138 | } | ||
139 | |||
140 | // Remove the request of the other type | ||
141 | PoolRequestsDB.remove({ id: id }, function (err) { | ||
142 | if (err) logger.error(err) | ||
143 | }) | ||
144 | } else { | ||
145 | PoolRequestsDB.create({ id: id, type: type, request: request }, function (err) { | ||
146 | if (err) logger.error(err) | ||
147 | }) | ||
148 | } | ||
149 | }) | ||
150 | } | ||
151 | |||
152 | poolRequests.deactivate = function () { | ||
153 | logger.info('Pool requests deactivated.') | ||
154 | clearInterval(timer) | ||
155 | } | ||
156 | |||
157 | module.exports = poolRequests | ||
158 | })() | ||
diff --git a/src/utils.js b/src/utils.js index dda6c7a0a..4aa1fc55e 100644 --- a/src/utils.js +++ b/src/utils.js | |||
@@ -36,7 +36,7 @@ | |||
36 | 36 | ||
37 | replay( | 37 | replay( |
38 | request.post(params, function (err, response, body) { | 38 | request.post(params, function (err, response, body) { |
39 | callbackEach(err, response, body, to_pod) | 39 | callbackEach(err, response, body, params.url, to_pod) |
40 | }), | 40 | }), |
41 | { | 41 | { |
42 | retries: retries, | 42 | retries: retries, |
@@ -71,8 +71,8 @@ | |||
71 | 71 | ||
72 | // Make a request for each pod | 72 | // Make a request for each pod |
73 | async.each(pods, function (pod, callback_each_async) { | 73 | async.each(pods, function (pod, callback_each_async) { |
74 | function callbackEachRetryRequest (err, response, body, pod) { | 74 | function callbackEachRetryRequest (err, response, body, url, pod) { |
75 | callbackEach(err, response, body, pod, function () { | 75 | callbackEach(err, response, body, url, pod, function () { |
76 | callback_each_async() | 76 | callback_each_async() |
77 | }) | 77 | }) |
78 | } | 78 | } |
diff --git a/src/videos.js b/src/videos.js index 8c44cad95..e3a5b49f1 100644 --- a/src/videos.js +++ b/src/videos.js | |||
@@ -3,6 +3,7 @@ | |||
3 | 3 | ||
4 | var async = require('async') | 4 | var async = require('async') |
5 | var config = require('config') | 5 | var config = require('config') |
6 | var dz = require('dezalgo') | ||
6 | var fs = require('fs') | 7 | var fs = require('fs') |
7 | var webtorrent = require('./webTorrentNode') | 8 | var webtorrent = require('./webTorrentNode') |
8 | 9 | ||
@@ -67,19 +68,10 @@ | |||
67 | return callback(err) | 68 | return callback(err) |
68 | } | 69 | } |
69 | 70 | ||
70 | // Now we'll send the video's meta data | 71 | // Now we'll add the video's meta data to our friends |
71 | params.namePath = null | 72 | params.namePath = null |
72 | 73 | ||
73 | logger.info('Sending %s video to friends.', video_file.path) | 74 | pods.addVideoToFriends(params) |
74 | |||
75 | var data = { | ||
76 | path: '/api/' + global.API_VERSION + '/remotevideos/add', | ||
77 | method: 'POST', | ||
78 | data: params | ||
79 | } | ||
80 | |||
81 | // Do not wait the secure requests | ||
82 | pods.makeSecureRequest(data) | ||
83 | callback(null) | 75 | callback(null) |
84 | }) | 76 | }) |
85 | }) | 77 | }) |
@@ -124,16 +116,12 @@ | |||
124 | return callback(err) | 116 | return callback(err) |
125 | } | 117 | } |
126 | 118 | ||
127 | var data = { | 119 | var params = { |
128 | path: '/api/' + global.API_VERSION + '/remotevideos/remove', | 120 | name: video.name, |
129 | method: 'POST', | 121 | magnetUri: video.magnetUri |
130 | data: { | ||
131 | magnetUri: video.magnetUri | ||
132 | } | ||
133 | } | 122 | } |
134 | 123 | ||
135 | // Yes this is a POST request because we add some informations in the body (signature, encrypt etc) | 124 | pods.removeVideoToFriends(params) |
136 | pods.makeSecureRequest(data) | ||
137 | callback(null) | 125 | callback(null) |
138 | }) | 126 | }) |
139 | }) | 127 | }) |
@@ -142,49 +130,65 @@ | |||
142 | } | 130 | } |
143 | 131 | ||
144 | // Use the magnet Uri because the _id field is not the same on different servers | 132 | // Use the magnet Uri because the _id field is not the same on different servers |
145 | videos.removeRemote = function (fromUrl, magnetUri, callback) { | 133 | videos.removeRemotes = function (fromUrl, magnetUris, callback) { |
146 | VideosDB.findOne({ magnetUri: magnetUri }, function (err, video) { | 134 | VideosDB.find({ magnetUri: { $in: magnetUris } }, function (err, videos) { |
147 | if (err || !video) { | 135 | if (err || !videos) { |
148 | logger.error('Cannot find the torrent URI of this remote video.') | 136 | logger.error('Cannot find the torrent URI of these remote videos.') |
149 | return callback(err) | 137 | return callback(err) |
150 | } | 138 | } |
151 | 139 | ||
152 | // TODO: move to reqValidators middleware ? | 140 | var to_remove = [] |
153 | if (video.podUrl !== fromUrl) { | 141 | async.each(videos, function (video, callback_async) { |
154 | logger.error('The pod has not the rights on this video.') | 142 | callback_async = dz(callback_async) |
155 | return callback(err) | ||
156 | } | ||
157 | 143 | ||
158 | VideosDB.findByIdAndRemove(video._id, function (err) { | 144 | if (video.podUrl !== fromUrl) { |
159 | if (err) { | 145 | logger.error('The pod %s has not the rights on the video of %s.', fromUrl, video.podUrl) |
160 | logger.error('Cannot remove the remote video.') | 146 | } else { |
161 | return callback(err) | 147 | to_remove.push(video._id) |
162 | } | 148 | } |
163 | 149 | ||
164 | callback(null) | 150 | callback_async() |
151 | }, function () { | ||
152 | VideosDB.remove({ _id: { $in: to_remove } }, function (err) { | ||
153 | if (err) { | ||
154 | logger.error('Cannot remove the remote videos.') | ||
155 | return callback(err) | ||
156 | } | ||
157 | |||
158 | callback(null) | ||
159 | }) | ||
165 | }) | 160 | }) |
166 | }) | 161 | }) |
167 | } | 162 | } |
168 | 163 | ||
169 | // { name, magnetUri, podUrl } | 164 | // { name, magnetUri, podUrl } |
170 | videos.addRemote = function (data, callback) { | 165 | videos.addRemotes = function (videos, callback) { |
171 | logger.debug('Add remote video from pod: %s', data.podUrl) | 166 | var to_add = [] |
172 | |||
173 | var params = { | ||
174 | name: data.name, | ||
175 | namePath: null, | ||
176 | description: data.description, | ||
177 | magnetUri: data.magnetUri, | ||
178 | podUrl: data.podUrl | ||
179 | } | ||
180 | 167 | ||
181 | VideosDB.create(params, function (err, video) { | 168 | async.each(videos, function (video, callback_each) { |
182 | if (err) { | 169 | callback_each = dz(callback_each) |
183 | logger.error('Cannot insert this remote video.', { error: err }) | 170 | logger.debug('Add remote video from pod: %s', video.podUrl) |
184 | return callback(err) | 171 | |
172 | var params = { | ||
173 | name: video.name, | ||
174 | namePath: null, | ||
175 | description: video.description, | ||
176 | magnetUri: video.magnetUri, | ||
177 | podUrl: video.podUrl | ||
185 | } | 178 | } |
186 | 179 | ||
187 | return callback(null, video) | 180 | to_add.push(params) |
181 | |||
182 | callback_each() | ||
183 | }, function () { | ||
184 | VideosDB.create(to_add, function (err, videos) { | ||
185 | if (err) { | ||
186 | logger.error('Cannot insert this remote video.', { error: err }) | ||
187 | return callback(err) | ||
188 | } | ||
189 | |||
190 | return callback(null, videos) | ||
191 | }) | ||
188 | }) | 192 | }) |
189 | } | 193 | } |
190 | 194 | ||
diff --git a/test/api/friendsAdvanced.js b/test/api/friendsAdvanced.js index ccddac4dc..680d90aee 100644 --- a/test/api/friendsAdvanced.js +++ b/test/api/friendsAdvanced.js | |||
@@ -103,9 +103,9 @@ | |||
103 | }) | 103 | }) |
104 | 104 | ||
105 | it('Should make friends with the pods 1, 2, 3', function (done) { | 105 | it('Should make friends with the pods 1, 2, 3', function (done) { |
106 | this.timeout(100000) | 106 | this.timeout(150000) |
107 | 107 | ||
108 | // Pods 1, 2, 3 and 4 become friends | 108 | // Pods 1, 2, 3 and 4 become friends (yes this is beautiful) |
109 | makeFriend(2, function () { | 109 | makeFriend(2, function () { |
110 | makeFriend(1, function () { | 110 | makeFriend(1, function () { |
111 | makeFriend(4, function () { | 111 | makeFriend(4, function () { |
@@ -114,37 +114,45 @@ | |||
114 | 114 | ||
115 | // Expulse pod 4 from pod 1 and 2 | 115 | // Expulse pod 4 from pod 1 and 2 |
116 | uploadVideo(1, function () { | 116 | uploadVideo(1, function () { |
117 | uploadVideo(1, function () { | 117 | setTimeout(function () { |
118 | uploadVideo(2, function () { | 118 | uploadVideo(1, function () { |
119 | uploadVideo(2, function () { | 119 | setTimeout(function () { |
120 | // Rerun server 4 | 120 | uploadVideo(2, function () { |
121 | utils.runServer(4, function (app, url) { | 121 | setTimeout(function () { |
122 | apps[3] = app | 122 | uploadVideo(2, function () { |
123 | getFriendsList(4, function (err, res) { | 123 | setTimeout(function () { |
124 | if (err) throw err | 124 | // Rerun server 4 |
125 | // Pod 4 didn't know pod 1 and 2 removed it | 125 | utils.runServer(4, function (app, url) { |
126 | expect(res.body.length).to.equal(3) | 126 | apps[3] = app |
127 | 127 | getFriendsList(4, function (err, res) { | |
128 | // Pod 6 ask pod 1, 2 and 3 | 128 | if (err) throw err |
129 | makeFriend(6, function () { | 129 | // Pod 4 didn't know pod 1 and 2 removed it |
130 | getFriendsList(6, function (err, res) { | 130 | expect(res.body.length).to.equal(3) |
131 | if (err) throw err | 131 | |
132 | 132 | // Pod 6 ask pod 1, 2 and 3 | |
133 | // Pod 4 should not be our friend | 133 | makeFriend(6, function () { |
134 | var result = res.body | 134 | getFriendsList(6, function (err, res) { |
135 | expect(result.length).to.equal(3) | 135 | if (err) throw err |
136 | for (var pod of result) { | 136 | |
137 | expect(pod.url).not.equal(urls[3]) | 137 | // Pod 4 should not be our friend |
138 | } | 138 | var result = res.body |
139 | 139 | expect(result.length).to.equal(3) | |
140 | done() | 140 | for (var pod of result) { |
141 | }) | 141 | expect(pod.url).not.equal(urls[3]) |
142 | } | ||
143 | |||
144 | done() | ||
145 | }) | ||
146 | }) | ||
147 | }) | ||
148 | }) | ||
149 | }, 11000) | ||
142 | }) | 150 | }) |
143 | }) | 151 | }, 11000) |
144 | }) | 152 | }) |
145 | }) | 153 | }, 11000) |
146 | }) | 154 | }) |
147 | }) | 155 | }, 11000) |
148 | }) | 156 | }) |
149 | }) | 157 | }) |
150 | }) | 158 | }) |
diff --git a/test/api/multiplePods.js b/test/api/multiplePods.js index 1edfc1ce3..3ce57afa8 100644 --- a/test/api/multiplePods.js +++ b/test/api/multiplePods.js | |||
@@ -14,7 +14,7 @@ | |||
14 | var path = '/api/v1/videos' | 14 | var path = '/api/v1/videos' |
15 | var apps = [] | 15 | var apps = [] |
16 | var urls = [] | 16 | var urls = [] |
17 | var video_id = -1 | 17 | var to_remove = [] |
18 | 18 | ||
19 | function getVideosList (url, end) { | 19 | function getVideosList (url, end) { |
20 | request(url) | 20 | request(url) |
@@ -36,6 +36,14 @@ | |||
36 | .end(end) | 36 | .end(end) |
37 | } | 37 | } |
38 | 38 | ||
39 | function removeVideo (url, id, end) { | ||
40 | request(url) | ||
41 | .delete(path + '/' + id) | ||
42 | .set('Accept', 'application/json') | ||
43 | .expect(204) | ||
44 | .end(end) | ||
45 | } | ||
46 | |||
39 | before(function (done) { | 47 | before(function (done) { |
40 | this.timeout(30000) | 48 | this.timeout(30000) |
41 | var path_friends = '/api/v1/pods/makefriends' | 49 | var path_friends = '/api/v1/pods/makefriends' |
@@ -89,7 +97,7 @@ | |||
89 | 97 | ||
90 | describe('Should upload the video and propagate on each pod', function () { | 98 | describe('Should upload the video and propagate on each pod', function () { |
91 | it('Should upload the video on pod 1 and propagate on each pod', function (done) { | 99 | it('Should upload the video on pod 1 and propagate on each pod', function (done) { |
92 | this.timeout(5000) | 100 | this.timeout(15000) |
93 | 101 | ||
94 | uploadVideo(urls[0], 'my super name for pod 1', 'my super description for pod 1', 'video_short1.webm', function (err) { | 102 | uploadVideo(urls[0], 'my super name for pod 1', 'my super description for pod 1', 'video_short1.webm', function (err) { |
95 | if (err) throw err | 103 | if (err) throw err |
@@ -125,12 +133,12 @@ | |||
125 | 133 | ||
126 | done() | 134 | done() |
127 | }) | 135 | }) |
128 | }, 1000) | 136 | }, 11000) |
129 | }) | 137 | }) |
130 | }) | 138 | }) |
131 | 139 | ||
132 | it('Should upload the video on pod 2 and propagate on each pod', function (done) { | 140 | it('Should upload the video on pod 2 and propagate on each pod', function (done) { |
133 | this.timeout(5000) | 141 | this.timeout(15000) |
134 | 142 | ||
135 | uploadVideo(urls[1], 'my super name for pod 2', 'my super description for pod 2', 'video_short2.webm', function (err) { | 143 | uploadVideo(urls[1], 'my super name for pod 2', 'my super description for pod 2', 'video_short2.webm', function (err) { |
136 | if (err) throw err | 144 | if (err) throw err |
@@ -166,47 +174,56 @@ | |||
166 | 174 | ||
167 | done() | 175 | done() |
168 | }) | 176 | }) |
169 | }, 1000) | 177 | }, 11000) |
170 | }) | 178 | }) |
171 | }) | 179 | }) |
172 | 180 | ||
173 | it('Should upload the video on pod 3 and propagate on each pod', function (done) { | 181 | it('Should upload two videos on pod 3 and propagate on each pod', function (done) { |
174 | this.timeout(5000) | 182 | this.timeout(15000) |
175 | 183 | ||
176 | uploadVideo(urls[2], 'my super name for pod 3', 'my super description for pod 3', 'video_short3.webm', function (err) { | 184 | uploadVideo(urls[2], 'my super name for pod 3', 'my super description for pod 3', 'video_short3.webm', function (err) { |
177 | if (err) throw err | 185 | if (err) throw err |
186 | uploadVideo(urls[2], 'my super name for pod 3-2', 'my super description for pod 3-2', 'video_short.webm', function (err) { | ||
187 | if (err) throw err | ||
178 | 188 | ||
179 | setTimeout(function () { | 189 | setTimeout(function () { |
180 | var base_magnet = null | 190 | var base_magnet = null |
181 | // All pods should have this video | 191 | // All pods should have this video |
182 | async.each(urls, function (url, callback) { | 192 | async.each(urls, function (url, callback) { |
183 | getVideosList(url, function (err, res) { | 193 | getVideosList(url, function (err, res) { |
184 | if (err) throw err | 194 | if (err) throw err |
185 | |||
186 | var videos = res.body | ||
187 | expect(videos).to.be.an('array') | ||
188 | expect(videos.length).to.equal(3) | ||
189 | var video = videos[2] | ||
190 | expect(video.name).to.equal('my super name for pod 3') | ||
191 | expect(video.description).to.equal('my super description for pod 3') | ||
192 | expect(video.podUrl).to.equal('http://localhost:9003') | ||
193 | expect(video.magnetUri).to.exist | ||
194 | 195 | ||
195 | // All pods should have the same magnet Uri | 196 | var videos = res.body |
196 | if (base_magnet === null) { | 197 | expect(videos).to.be.an('array') |
197 | base_magnet = video.magnetUri | 198 | expect(videos.length).to.equal(4) |
198 | } else { | 199 | var video = videos[2] |
199 | expect(video.magnetUri).to.equal.magnetUri | 200 | expect(video.name).to.equal('my super name for pod 3') |
200 | } | 201 | expect(video.description).to.equal('my super description for pod 3') |
202 | expect(video.podUrl).to.equal('http://localhost:9003') | ||
203 | expect(video.magnetUri).to.exist | ||
204 | |||
205 | video = videos[3] | ||
206 | expect(video.name).to.equal('my super name for pod 3-2') | ||
207 | expect(video.description).to.equal('my super description for pod 3-2') | ||
208 | expect(video.podUrl).to.equal('http://localhost:9003') | ||
209 | expect(video.magnetUri).to.exist | ||
210 | |||
211 | // All pods should have the same magnet Uri | ||
212 | if (base_magnet === null) { | ||
213 | base_magnet = video.magnetUri | ||
214 | } else { | ||
215 | expect(video.magnetUri).to.equal.magnetUri | ||
216 | } | ||
217 | |||
218 | callback() | ||
219 | }) | ||
220 | }, function (err) { | ||
221 | if (err) throw err | ||
201 | 222 | ||
202 | callback() | 223 | done() |
203 | }) | 224 | }) |
204 | }, function (err) { | 225 | }, 11000) |
205 | if (err) throw err | 226 | }) |
206 | |||
207 | done() | ||
208 | }) | ||
209 | }, 1000) | ||
210 | }) | 227 | }) |
211 | }) | 228 | }) |
212 | }) | 229 | }) |
@@ -220,6 +237,9 @@ | |||
220 | if (err) throw err | 237 | if (err) throw err |
221 | 238 | ||
222 | var video = res.body[0] | 239 | var video = res.body[0] |
240 | to_remove.push(res.body[2]._id) | ||
241 | to_remove.push(res.body[3]._id) | ||
242 | |||
223 | webtorrent.add(video.magnetUri, function (torrent) { | 243 | webtorrent.add(video.magnetUri, function (torrent) { |
224 | expect(torrent.files).to.exist | 244 | expect(torrent.files).to.exist |
225 | expect(torrent.files.length).to.equal(1) | 245 | expect(torrent.files.length).to.equal(1) |
@@ -257,7 +277,6 @@ | |||
257 | if (err) throw err | 277 | if (err) throw err |
258 | 278 | ||
259 | var video = res.body[2] | 279 | var video = res.body[2] |
260 | video_id = res.body[1]._id | ||
261 | 280 | ||
262 | webtorrent.add(video.magnetUri, function (torrent) { | 281 | webtorrent.add(video.magnetUri, function (torrent) { |
263 | expect(torrent.files).to.exist | 282 | expect(torrent.files).to.exist |
@@ -269,19 +288,39 @@ | |||
269 | }) | 288 | }) |
270 | }) | 289 | }) |
271 | 290 | ||
272 | it('Should remove the file 2 by asking pod 2', function (done) { | 291 | it('Should add the file 3-2 by asking pod 1', function (done) { |
273 | request(urls[1]) | 292 | // Yes, this could be long |
274 | .delete(path + '/' + video_id) | 293 | this.timeout(200000) |
275 | .set('Accept', 'application/json') | 294 | |
276 | .expect(204) | 295 | getVideosList(urls[0], function (err, res) { |
277 | .end(function (err, res) { | 296 | if (err) throw err |
297 | |||
298 | var video = res.body[3] | ||
299 | |||
300 | webtorrent.add(video.magnetUri, function (torrent) { | ||
301 | expect(torrent.files).to.exist | ||
302 | expect(torrent.files.length).to.equal(1) | ||
303 | expect(torrent.files[0].path).to.exist.and.to.not.equal('') | ||
304 | |||
305 | done() | ||
306 | }) | ||
307 | }) | ||
308 | }) | ||
309 | |||
310 | it('Should remove the file 3 and 3-2 by asking pod 3', function (done) { | ||
311 | this.timeout(15000) | ||
312 | |||
313 | removeVideo(urls[2], to_remove[0], function (err) { | ||
314 | if (err) throw err | ||
315 | removeVideo(urls[2], to_remove[1], function (err) { | ||
278 | if (err) throw err | 316 | if (err) throw err |
279 | 317 | ||
280 | // Wait the propagation to the other pods | 318 | // Wait the propagation to the other pods |
281 | setTimeout(function () { | 319 | setTimeout(function () { |
282 | done() | 320 | done() |
283 | }, 1000) | 321 | }, 11000) |
284 | }) | 322 | }) |
323 | }) | ||
285 | }) | 324 | }) |
286 | 325 | ||
287 | it('Should have videos 1 and 3 on each pod', function (done) { | 326 | it('Should have videos 1 and 3 on each pod', function (done) { |
@@ -293,8 +332,10 @@ | |||
293 | expect(videos).to.be.an('array') | 332 | expect(videos).to.be.an('array') |
294 | expect(videos.length).to.equal(2) | 333 | expect(videos.length).to.equal(2) |
295 | expect(videos[0]._id).not.to.equal(videos[1]._id) | 334 | expect(videos[0]._id).not.to.equal(videos[1]._id) |
296 | expect(videos[0]._id).not.to.equal(video_id) | 335 | expect(videos[0]._id).not.to.equal(to_remove[0]) |
297 | expect(videos[1]._id).not.to.equal(video_id) | 336 | expect(videos[1]._id).not.to.equal(to_remove[0]) |
337 | expect(videos[0]._id).not.to.equal(to_remove[1]) | ||
338 | expect(videos[1]._id).not.to.equal(to_remove[1]) | ||
298 | 339 | ||
299 | callback() | 340 | callback() |
300 | }) | 341 | }) |