aboutsummaryrefslogtreecommitdiffhomepage
path: root/server
diff options
context:
space:
mode:
authorChocobozzz <florian.bigard@gmail.com>2016-06-18 16:13:54 +0200
committerChocobozzz <florian.bigard@gmail.com>2016-06-18 16:13:54 +0200
commit528a9efa8272532bbd0dafc35c3e05e57c50f61e (patch)
tree62d4417df4ab9b2e53c44dc7271be81b88e4e0e5 /server
parentb2e4c0ba1a33b8a50491a1f8d111468a7da5640f (diff)
downloadPeerTube-528a9efa8272532bbd0dafc35c3e05e57c50f61e.tar.gz
PeerTube-528a9efa8272532bbd0dafc35c3e05e57c50f61e.tar.zst
PeerTube-528a9efa8272532bbd0dafc35c3e05e57c50f61e.zip
Try to make a better communication (between pods) module
Diffstat (limited to 'server')
-rw-r--r--server/controllers/api/v1/index.js4
-rw-r--r--server/controllers/api/v1/pods.js43
-rw-r--r--server/controllers/api/v1/remote.js80
-rw-r--r--server/controllers/api/v1/remoteVideos.js66
-rw-r--r--server/controllers/api/v1/videos.js30
-rw-r--r--server/helpers/customValidators.js48
-rw-r--r--server/helpers/requests.js114
-rw-r--r--server/initializers/constants.js13
-rw-r--r--server/lib/friends.js158
-rw-r--r--server/lib/requestsScheduler.js200
-rw-r--r--server/lib/videos.js27
-rw-r--r--server/middlewares/reqValidators/pods.js6
-rw-r--r--server/middlewares/reqValidators/remote.js26
-rw-r--r--server/models/pods.js15
-rw-r--r--server/models/requests.js32
-rw-r--r--server/tests/api/checkParams.js24
-rw-r--r--server/tests/api/friendsAdvanced.js19
-rw-r--r--server/tests/api/friendsBasic.js3
-rw-r--r--server/tests/api/multiplePods.js4
19 files changed, 483 insertions, 429 deletions
diff --git a/server/controllers/api/v1/index.js b/server/controllers/api/v1/index.js
index 7b3ec32c0..e0c29a8a2 100644
--- a/server/controllers/api/v1/index.js
+++ b/server/controllers/api/v1/index.js
@@ -5,12 +5,12 @@ const express = require('express')
5const router = express.Router() 5const router = express.Router()
6 6
7const podsController = require('./pods') 7const podsController = require('./pods')
8const remoteVideosController = require('./remoteVideos') 8const remoteController = require('./remote')
9const usersController = require('./users') 9const usersController = require('./users')
10const videosController = require('./videos') 10const videosController = require('./videos')
11 11
12router.use('/pods', podsController) 12router.use('/pods', podsController)
13router.use('/remotevideos', remoteVideosController) 13router.use('/remote', remoteController)
14router.use('/users', usersController) 14router.use('/users', usersController)
15router.use('/videos', videosController) 15router.use('/videos', videosController)
16router.use('/*', badRequest) 16router.use('/*', badRequest)
diff --git a/server/controllers/api/v1/pods.js b/server/controllers/api/v1/pods.js
index ecaeba666..881b2090d 100644
--- a/server/controllers/api/v1/pods.js
+++ b/server/controllers/api/v1/pods.js
@@ -9,19 +9,18 @@ const middlewares = require('../../../middlewares')
9const Pods = require('../../../models/pods') 9const Pods = require('../../../models/pods')
10const oAuth2 = middlewares.oauth2 10const oAuth2 = middlewares.oauth2
11const reqValidator = middlewares.reqValidators.pods 11const reqValidator = middlewares.reqValidators.pods
12const secureMiddleware = middlewares.secure 12const signatureValidator = middlewares.reqValidators.remote.signature
13const secureRequest = middlewares.reqValidators.remote.secureRequest
14const videos = require('../../../lib/videos') 13const videos = require('../../../lib/videos')
15const Videos = require('../../../models/videos') 14const Videos = require('../../../models/videos')
16 15
17const router = express.Router() 16const router = express.Router()
18 17
19router.get('/', listPods) 18router.get('/', listPodsUrl)
20router.post('/', reqValidator.podsAdd, addPods) 19router.post('/', reqValidator.podsAdd, addPods)
21router.get('/makefriends', oAuth2.authenticate, reqValidator.makeFriends, makeFriends) 20router.get('/makefriends', oAuth2.authenticate, reqValidator.makeFriends, makeFriends)
22router.get('/quitfriends', oAuth2.authenticate, quitFriends) 21router.get('/quitfriends', oAuth2.authenticate, quitFriends)
23// Post because this is a secured request 22// Post because this is a secured request
24router.post('/remove', secureRequest, secureMiddleware.decryptBody, removePods) 23router.post('/remove', signatureValidator, removePods)
25 24
26// --------------------------------------------------------------------------- 25// ---------------------------------------------------------------------------
27 26
@@ -30,22 +29,17 @@ module.exports = router
30// --------------------------------------------------------------------------- 29// ---------------------------------------------------------------------------
31 30
32function addPods (req, res, next) { 31function addPods (req, res, next) {
33 const informations = req.body.data 32 const informations = req.body
34 33
35 async.waterfall([ 34 async.waterfall([
36 function addPod (callback) { 35 function addPod (callback) {
37 Pods.add(informations, function (err) { 36 Pods.add(informations, callback)
38 return callback(err)
39 })
40 }, 37 },
41 38
42 function createVideosOfThisPod (callback) { 39 function sendMyVideos (podCreated, callback) {
43 // Create the remote videos from the new pod 40 friends.sendOwnedVideosToPod(podCreated._id)
44 videos.createRemoteVideos(informations.videos, function (err) {
45 if (err) logger.error('Cannot create remote videos.', { error: err })
46 41
47 return callback(err) 42 callback(null)
48 })
49 }, 43 },
50 44
51 function fetchMyCertificate (callback) { 45 function fetchMyCertificate (callback) {
@@ -57,30 +51,19 @@ function addPods (req, res, next) {
57 51
58 return callback(null, cert) 52 return callback(null, cert)
59 }) 53 })
60 },
61
62 function getListOfMyVideos (cert, callback) {
63 Videos.listOwned(function (err, videosList) {
64 if (err) {
65 logger.error('Cannot get the list of owned videos.')
66 return callback(err)
67 }
68
69 return callback(null, cert, videosList)
70 })
71 } 54 }
72 ], function (err, cert, videosList) { 55 ], function (err, cert) {
73 if (err) return next(err) 56 if (err) return next(err)
74 57
75 return res.json({ cert: cert, videos: videosList }) 58 return res.json({ cert: cert })
76 }) 59 })
77} 60}
78 61
79function listPods (req, res, next) { 62function listPodsUrl (req, res, next) {
80 Pods.list(function (err, podsList) { 63 Pods.listAllUrls(function (err, podsUrlList) {
81 if (err) return next(err) 64 if (err) return next(err)
82 65
83 res.json(podsList) 66 res.json(podsUrlList)
84 }) 67 })
85} 68}
86 69
diff --git a/server/controllers/api/v1/remote.js b/server/controllers/api/v1/remote.js
new file mode 100644
index 000000000..ced8470d7
--- /dev/null
+++ b/server/controllers/api/v1/remote.js
@@ -0,0 +1,80 @@
1'use strict'
2
3const async = require('async')
4const express = require('express')
5
6const middlewares = require('../../../middlewares')
7const secureMiddleware = middlewares.secure
8const reqValidator = middlewares.reqValidators.remote
9const logger = require('../../../helpers/logger')
10const Videos = require('../../../models/videos')
11const videos = require('../../../lib/videos')
12
13const router = express.Router()
14
15router.post('/videos',
16 reqValidator.signature,
17 reqValidator.dataToDecrypt,
18 secureMiddleware.decryptBody,
19 reqValidator.remoteVideos,
20 remoteVideos
21)
22
23// ---------------------------------------------------------------------------
24
25module.exports = router
26
27// ---------------------------------------------------------------------------
28
29function remoteVideos (req, res, next) {
30 const requests = req.body.data
31 const fromUrl = req.body.signature.url
32
33 // We need to process in the same order to keep consistency
34 // TODO: optimization
35 async.eachSeries(requests, function (request, callbackEach) {
36 const video = request.data
37
38 if (request.type === 'add') {
39 addRemoteVideo(video, callbackEach)
40 } else if (request.type === 'remove') {
41 removeRemoteVideo(video, fromUrl, callbackEach)
42 }
43 })
44
45 // We don't need to keep the other pod waiting
46 return res.type('json').status(204).end()
47}
48
49function addRemoteVideo (videoToCreate, callback) {
50 videos.createRemoteVideos([ videoToCreate ], function (err, remoteVideos) {
51 if (err) {
52 logger.error('Cannot create remote videos.', { error: err })
53 // Don't break the process
54 }
55
56 return callback()
57 })
58}
59
60function removeRemoteVideo (videoToRemove, fromUrl, callback) {
61 const magnetUris = [ videoToRemove.magnetUri ]
62
63 // We need the list because we have to remove some other stuffs (thumbnail etc)
64 Videos.listFromUrlAndMagnets(fromUrl, magnetUris, function (err, videosList) {
65 if (err) {
66 logger.error('Cannot list videos from url and magnets.', { error: err })
67 // Don't break the process
68 return callback()
69 }
70
71 videos.removeRemoteVideos(videosList, function (err) {
72 if (err) {
73 logger.error('Cannot remove remote videos.', { error: err })
74 // Don't break the process
75 }
76
77 return callback()
78 })
79 })
80}
diff --git a/server/controllers/api/v1/remoteVideos.js b/server/controllers/api/v1/remoteVideos.js
deleted file mode 100644
index 2f41c0411..000000000
--- a/server/controllers/api/v1/remoteVideos.js
+++ /dev/null
@@ -1,66 +0,0 @@
1'use strict'
2
3const express = require('express')
4const map = require('lodash/map')
5
6const middlewares = require('../../../middlewares')
7const secureMiddleware = middlewares.secure
8const reqValidator = middlewares.reqValidators.remote
9const logger = require('../../../helpers/logger')
10const Videos = require('../../../models/videos')
11const videos = require('../../../lib/videos')
12
13const router = express.Router()
14
15router.post('/add',
16 reqValidator.secureRequest,
17 secureMiddleware.decryptBody,
18 reqValidator.remoteVideosAdd,
19 addRemoteVideos
20)
21
22router.post('/remove',
23 reqValidator.secureRequest,
24 secureMiddleware.decryptBody,
25 reqValidator.remoteVideosRemove,
26 removeRemoteVideo
27)
28
29// ---------------------------------------------------------------------------
30
31module.exports = router
32
33// ---------------------------------------------------------------------------
34
35function addRemoteVideos (req, res, next) {
36 const videosToCreate = req.body.data
37 videos.createRemoteVideos(videosToCreate, function (err, remoteVideos) {
38 if (err) {
39 logger.error('Cannot create remote videos.', { error: err })
40 return next(err)
41 }
42
43 res.type('json').status(201).end()
44 })
45}
46
47function removeRemoteVideo (req, res, next) {
48 const fromUrl = req.body.signature.url
49 const magnetUris = map(req.body.data, 'magnetUri')
50
51 Videos.listFromUrlAndMagnets(fromUrl, magnetUris, function (err, videosList) {
52 if (err) {
53 logger.error('Cannot list videos from url and magnets.', { error: err })
54 return next(err)
55 }
56
57 videos.removeRemoteVideos(videosList, function (err) {
58 if (err) {
59 logger.error('Cannot remove remote videos.', { error: err })
60 return next(err)
61 }
62
63 res.type('json').status(204).end()
64 })
65 })
66}
diff --git a/server/controllers/api/v1/videos.js b/server/controllers/api/v1/videos.js
index 5449cbcfa..2edb31122 100644
--- a/server/controllers/api/v1/videos.js
+++ b/server/controllers/api/v1/videos.js
@@ -3,8 +3,6 @@
3const async = require('async') 3const async = require('async')
4const config = require('config') 4const config = require('config')
5const express = require('express') 5const express = require('express')
6const fs = require('fs')
7const path = require('path')
8const multer = require('multer') 6const multer = require('multer')
9 7
10const constants = require('../../../initializers/constants') 8const constants = require('../../../initializers/constants')
@@ -46,7 +44,6 @@ const storage = multer.diskStorage({
46}) 44})
47 45
48const reqFiles = multer({ storage: storage }).fields([{ name: 'videofile', maxCount: 1 }]) 46const reqFiles = multer({ storage: storage }).fields([{ name: 'videofile', maxCount: 1 }])
49const thumbnailsDir = path.join(__dirname, '..', '..', '..', '..', config.get('storage.thumbnails'))
50 47
51router.get('/', 48router.get('/',
52 reqValidatorPagination.pagination, 49 reqValidatorPagination.pagination,
@@ -127,34 +124,25 @@ function addVideo (req, res, next) {
127 return callback(err) 124 return callback(err)
128 } 125 }
129 126
130 return callback(null, torrent, thumbnailName, videoData, insertedVideo) 127 return callback(null, insertedVideo)
131 }) 128 })
132 }, 129 },
133 130
134 function getThumbnailBase64 (torrent, thumbnailName, videoData, insertedVideo, callback) { 131 function sendToFriends (insertedVideo, callback) {
135 videoData.createdDate = insertedVideo.createdDate 132 videos.convertVideoToRemote(insertedVideo, function (err, remoteVideo) {
136
137 fs.readFile(thumbnailsDir + thumbnailName, function (err, thumbnailData) {
138 if (err) { 133 if (err) {
139 // TODO unseed the video 134 // TODO unseed the video
140 // TODO remove thumbnail 135 // TODO remove thumbnail
141 // TODO: remove video 136 // TODO delete from DB
142 logger.error('Cannot read the thumbnail of the video') 137 logger.error('Cannot convert video to remote.')
143 return callback(err) 138 return callback(err)
144 } 139 }
145 140
146 return callback(null, videoData, thumbnailData) 141 // Now we'll add the video's meta data to our friends
147 }) 142 friends.addVideoToFriends(remoteVideo)
148 },
149
150 function sendToFriends (videoData, thumbnailData, callback) {
151 // Set the image in base64
152 videoData.thumbnailBase64 = new Buffer(thumbnailData).toString('base64')
153 143
154 // Now we'll add the video's meta data to our friends 144 return callback(null)
155 friends.addVideoToFriends(videoData) 145 })
156
157 return callback(null)
158 } 146 }
159 147
160 ], function andFinally (err) { 148 ], function andFinally (err) {
diff --git a/server/helpers/customValidators.js b/server/helpers/customValidators.js
index 9c3ff38ef..a6cf680e5 100644
--- a/server/helpers/customValidators.js
+++ b/server/helpers/customValidators.js
@@ -7,8 +7,7 @@ const VIDEOS_CONSTRAINTS_FIELDS = constants.VIDEOS_CONSTRAINTS_FIELDS
7 7
8const customValidators = { 8const customValidators = {
9 exists: exists, 9 exists: exists,
10 isEachAddRemoteVideosValid: isEachAddRemoteVideosValid, 10 isEachRemoteVideosValid: isEachRemoteVideosValid,
11 isEachRemoveRemoteVideosValid: isEachRemoveRemoteVideosValid,
12 isArray: isArray, 11 isArray: isArray,
13 isVideoAuthorValid: isVideoAuthorValid, 12 isVideoAuthorValid: isVideoAuthorValid,
14 isVideoDateValid: isVideoDateValid, 13 isVideoDateValid: isVideoDateValid,
@@ -25,23 +24,26 @@ function exists (value) {
25 return value !== undefined && value !== null 24 return value !== undefined && value !== null
26} 25}
27 26
28function isEachAddRemoteVideosValid (videos) { 27function isEachRemoteVideosValid (requests) {
29 return videos.every(function (video) { 28 return requests.every(function (request) {
30 return isVideoAuthorValid(video.author) && 29 const video = request.data
31 isVideoDateValid(video.createdDate) && 30 return (
32 isVideoDescriptionValid(video.description) && 31 isRequestTypeAddValid(request.type) &&
33 isVideoDurationValid(video.duration) && 32 isVideoAuthorValid(video.author) &&
34 isVideoMagnetUriValid(video.magnetUri) && 33 isVideoDateValid(video.createdDate) &&
35 isVideoNameValid(video.name) && 34 isVideoDescriptionValid(video.description) &&
36 isVideoPodUrlValid(video.podUrl) && 35 isVideoDurationValid(video.duration) &&
37 isVideoTagsValid(video.tags) && 36 isVideoMagnetUriValid(video.magnetUri) &&
38 isVideoThumbnailValid(video.thumbnailBase64) 37 isVideoNameValid(video.name) &&
39 }) 38 isVideoPodUrlValid(video.podUrl) &&
40} 39 isVideoTagsValid(video.tags) &&
41 40 isVideoThumbnailValid(video.thumbnailBase64)
42function isEachRemoveRemoteVideosValid (videos) { 41 ) ||
43 return videos.every(function (video) { 42 (
44 return isVideoMagnetUriValid(video.magnetUri) 43 isRequestTypeRemoveValid(request.type) &&
44 isVideoNameValid(video.name) &&
45 isVideoMagnetUriValid(video.magnetUri)
46 )
45 }) 47 })
46} 48}
47 49
@@ -49,6 +51,14 @@ function isArray (value) {
49 return Array.isArray(value) 51 return Array.isArray(value)
50} 52}
51 53
54function isRequestTypeAddValid (value) {
55 return value === 'add'
56}
57
58function isRequestTypeRemoveValid (value) {
59 return value === 'remove'
60}
61
52function isVideoAuthorValid (value) { 62function isVideoAuthorValid (value) {
53 return validator.isLength(value, VIDEOS_CONSTRAINTS_FIELDS.AUTHOR) 63 return validator.isLength(value, VIDEOS_CONSTRAINTS_FIELDS.AUTHOR)
54} 64}
diff --git a/server/helpers/requests.js b/server/helpers/requests.js
index 1e1bb4111..871342d60 100644
--- a/server/helpers/requests.js
+++ b/server/helpers/requests.js
@@ -1,12 +1,10 @@
1'use strict' 1'use strict'
2 2
3const async = require('async')
4const config = require('config') 3const config = require('config')
5const request = require('request')
6const replay = require('request-replay') 4const replay = require('request-replay')
5const request = require('request')
7 6
8const constants = require('../initializers/constants') 7const constants = require('../initializers/constants')
9const logger = require('./logger')
10const peertubeCrypto = require('./peertubeCrypto') 8const peertubeCrypto = require('./peertubeCrypto')
11 9
12const http = config.get('webserver.https') ? 'https' : 'http' 10const http = config.get('webserver.https') ? 'https' : 'http'
@@ -14,93 +12,67 @@ const host = config.get('webserver.host')
14const port = config.get('webserver.port') 12const port = config.get('webserver.port')
15 13
16const requests = { 14const requests = {
17 makeMultipleRetryRequest: makeMultipleRetryRequest 15 makeRetryRequest: makeRetryRequest,
16 makeSecureRequest: makeSecureRequest
18} 17}
19 18
20function makeMultipleRetryRequest (allData, pods, callbackEach, callback) { 19function makeRetryRequest (params, callback) {
21 if (!callback) { 20 replay(
22 callback = callbackEach 21 request(params, callback),
23 callbackEach = null 22 {
24 } 23 retries: constants.RETRY_REQUESTS,
24 factor: 3,
25 maxTimeout: Infinity,
26 errorCodes: [ 'EADDRINFO', 'ETIMEDOUT', 'ECONNRESET', 'ESOCKETTIMEDOUT', 'ENOTFOUND', 'ECONNREFUSED' ]
27 }
28 )
29}
25 30
26 const url = http + '://' + host + ':' + port 31function makeSecureRequest (params, callback) {
27 let signature 32 const myUrl = http + '://' + host + ':' + port
28 33
29 // Add signature if it is specified in the params 34 const requestParams = {
30 if (allData.method === 'POST' && allData.data && allData.sign === true) { 35 url: params.toPod.url + params.path
31 signature = peertubeCrypto.sign(url)
32 } 36 }
33 37
34 // Make a request for each pod 38 // Add data with POST requst ?
35 async.each(pods, function (pod, callbackEachAsync) { 39 if (params.method === 'POST') {
36 function callbackEachRetryRequest (err, response, body, url, pod) { 40 requestParams.json = {}
37 if (callbackEach !== null) {
38 callbackEach(err, response, body, url, pod, function () {
39 callbackEachAsync()
40 })
41 } else {
42 callbackEachAsync()
43 }
44 }
45 41
46 const params = { 42 // Add signature if it is specified in the params
47 url: pod.url + allData.path, 43 if (params.sign === true) {
48 method: allData.method 44 requestParams.json.signature = {
45 url: myUrl,
46 signature: peertubeCrypto.sign(myUrl)
47 }
49 } 48 }
50 49
51 // Add data with POST requst ? 50 // If there are data informations
52 if (allData.method === 'POST' && allData.data) { 51 if (params.data) {
53 // Encrypt data ? 52 // Encrypt data
54 if (allData.encrypt === true) { 53 if (params.encrypt === true) {
55 peertubeCrypto.encrypt(pod.publicKey, JSON.stringify(allData.data), function (err, encrypted) { 54 peertubeCrypto.encrypt(params.toPod.publicKey, JSON.stringify(params.data), function (err, encrypted) {
56 if (err) return callback(err) 55 if (err) return callback(err)
57 56
58 params.json = { 57 requestParams.json.data = encrypted.data
59 data: encrypted.data, 58 requestParams.json.key = encrypted.key
60 key: encrypted.key
61 }
62 59
63 makeRetryRequest(params, url, pod, signature, callbackEachRetryRequest) 60 request.post(requestParams, callback)
64 }) 61 })
65 } else { 62 } else {
66 params.json = { data: allData.data } 63 // No encryption
67 makeRetryRequest(params, url, pod, signature, callbackEachRetryRequest) 64 requestParams.json.data = params.data
65 request.post(requestParams, callback)
68 } 66 }
69 } else { 67 } else {
70 makeRetryRequest(params, url, pod, signature, callbackEachRetryRequest) 68 // No data
69 request.post(requestParams, callback)
71 } 70 }
72 }, callback) 71 } else {
72 request.get(requestParams, callback)
73 }
73} 74}
74 75
75// --------------------------------------------------------------------------- 76// ---------------------------------------------------------------------------
76 77
77module.exports = requests 78module.exports = requests
78
79// ---------------------------------------------------------------------------
80
81function makeRetryRequest (params, fromUrl, toPod, signature, callbackEach) {
82 // Append the signature
83 if (signature) {
84 params.json.signature = {
85 url: fromUrl,
86 signature: signature
87 }
88 }
89
90 logger.debug('Make retry requests to %s.', toPod.url)
91
92 replay(
93 request.post(params, function (err, response, body) {
94 callbackEach(err, response, body, params.url, toPod)
95 }),
96 {
97 retries: constants.REQUEST_RETRIES,
98 factor: 3,
99 maxTimeout: Infinity,
100 errorCodes: [ 'EADDRINFO', 'ETIMEDOUT', 'ECONNRESET', 'ESOCKETTIMEDOUT', 'ENOTFOUND', 'ECONNREFUSED' ]
101 }
102 ).on('replay', function (replay) {
103 logger.info('Replaying request to %s. Request failed: %d %s. Replay number: #%d. Will retry in: %d ms.',
104 params.url, replay.error.code, replay.error.message, replay.number, replay.delay)
105 })
106}
diff --git a/server/initializers/constants.js b/server/initializers/constants.js
index 22cbb1361..caeb340cf 100644
--- a/server/initializers/constants.js
+++ b/server/initializers/constants.js
@@ -18,11 +18,11 @@ const PODS_SCORE = {
18 BONUS: 10 18 BONUS: 10
19} 19}
20 20
21// Number of retries we make for the make retry requests (to friends...) 21// Number of requests in parallel we can make
22let REQUEST_RETRIES = 10 22const REQUESTS_IN_PARALLEL = 10
23 23
24// Different types or requests for the request scheduler module 24// Number of requests to retry for replay requests module
25const REQUEST_SCHEDULER_TYPE = [ 'add', 'remove' ] 25const RETRY_REQUESTS = 5
26 26
27// Sortable columns per schema 27// Sortable columns per schema
28const SEARCHABLE_COLUMNS = { 28const SEARCHABLE_COLUMNS = {
@@ -56,7 +56,6 @@ if (isTestInstance() === true) {
56 FRIEND_BASE_SCORE = 20 56 FRIEND_BASE_SCORE = 20
57 INTERVAL = 10000 57 INTERVAL = 10000
58 VIDEOS_CONSTRAINTS_FIELDS.DURATION.max = 14 58 VIDEOS_CONSTRAINTS_FIELDS.DURATION.max = 14
59 REQUEST_RETRIES = 2
60} 59}
61 60
62// --------------------------------------------------------------------------- 61// ---------------------------------------------------------------------------
@@ -67,8 +66,8 @@ module.exports = {
67 INTERVAL: INTERVAL, 66 INTERVAL: INTERVAL,
68 PAGINATION_COUNT_DEFAULT: PAGINATION_COUNT_DEFAULT, 67 PAGINATION_COUNT_DEFAULT: PAGINATION_COUNT_DEFAULT,
69 PODS_SCORE: PODS_SCORE, 68 PODS_SCORE: PODS_SCORE,
70 REQUEST_RETRIES: REQUEST_RETRIES, 69 REQUESTS_IN_PARALLEL: REQUESTS_IN_PARALLEL,
71 REQUEST_SCHEDULER_TYPE: REQUEST_SCHEDULER_TYPE, 70 RETRY_REQUESTS: RETRY_REQUESTS,
72 SEARCHABLE_COLUMNS: SEARCHABLE_COLUMNS, 71 SEARCHABLE_COLUMNS: SEARCHABLE_COLUMNS,
73 SORTABLE_COLUMNS: SORTABLE_COLUMNS, 72 SORTABLE_COLUMNS: SORTABLE_COLUMNS,
74 THUMBNAILS_SIZE: THUMBNAILS_SIZE, 73 THUMBNAILS_SIZE: THUMBNAILS_SIZE,
diff --git a/server/lib/friends.js b/server/lib/friends.js
index e986fa006..d81a603ad 100644
--- a/server/lib/friends.js
+++ b/server/lib/friends.js
@@ -24,15 +24,15 @@ const pods = {
24 getMyCertificate: getMyCertificate, 24 getMyCertificate: getMyCertificate,
25 makeFriends: makeFriends, 25 makeFriends: makeFriends,
26 quitFriends: quitFriends, 26 quitFriends: quitFriends,
27 removeVideoToFriends: removeVideoToFriends 27 removeVideoToFriends: removeVideoToFriends,
28 sendOwnedVideosToPod: sendOwnedVideosToPod
28} 29}
29 30
30function addVideoToFriends (video) { 31function addVideoToFriends (video) {
31 // To avoid duplicates
32 const id = video.name + video.magnetUri
33 // ensure namePath is null 32 // ensure namePath is null
34 video.namePath = null 33 video.namePath = null
35 requestsScheduler.addRequest(id, 'add', video) 34
35 requestsScheduler.addRequest('add', video)
36} 36}
37 37
38function hasFriends (callback) { 38function hasFriends (callback) {
@@ -60,7 +60,7 @@ function makeFriends (callback) {
60 60
61 const urls = config.get('network.friends') 61 const urls = config.get('network.friends')
62 62
63 async.each(urls, function (url, callbackEach) { 63 async.eachSeries(urls, function (url, callbackEach) {
64 computeForeignPodsList(url, podsScore, callbackEach) 64 computeForeignPodsList(url, podsScore, callbackEach)
65 }, function (err) { 65 }, function (err) {
66 if (err) return callback(err) 66 if (err) return callback(err)
@@ -78,7 +78,7 @@ function quitFriends (callback) {
78 // Stop pool requests 78 // Stop pool requests
79 requestsScheduler.deactivate() 79 requestsScheduler.deactivate()
80 // Flush pool requests 80 // Flush pool requests
81 requestsScheduler.forceSend() 81 requestsScheduler.flush()
82 82
83 async.waterfall([ 83 async.waterfall([
84 function getPodsList (callbackAsync) { 84 function getPodsList (callbackAsync) {
@@ -86,19 +86,25 @@ function quitFriends (callback) {
86 }, 86 },
87 87
88 function announceIQuitMyFriends (pods, callbackAsync) { 88 function announceIQuitMyFriends (pods, callbackAsync) {
89 const request = { 89 const requestParams = {
90 method: 'POST', 90 method: 'POST',
91 path: '/api/' + constants.API_VERSION + '/pods/remove', 91 path: '/api/' + constants.API_VERSION + '/pods/remove',
92 sign: true, 92 sign: true
93 encrypt: true,
94 data: {
95 url: 'me' // Fake data
96 }
97 } 93 }
98 94
99 // Announce we quit them 95 // Announce we quit them
100 requests.makeMultipleRetryRequest(request, pods, function (err) { 96 // We don't care if the request fails
101 return callbackAsync(err) 97 // The other pod will exclude us automatically after a while
98 async.eachLimit(pods, constants.REQUESTS_IN_PARALLEL, function (pod, callbackEach) {
99 requestParams.toPod = pod
100 requests.makeSecureRequest(requestParams, callbackEach)
101 }, function (err) {
102 if (err) {
103 logger.error('Some errors while quitting friends.', { err: err })
104 // Don't stop the process
105 }
106
107 return callbackAsync()
102 }) 108 })
103 }, 109 },
104 110
@@ -136,9 +142,28 @@ function quitFriends (callback) {
136} 142}
137 143
138function removeVideoToFriends (video) { 144function removeVideoToFriends (video) {
139 // To avoid duplicates 145 requestsScheduler.addRequest('remove', video)
140 const id = video.name + video.magnetUri 146}
141 requestsScheduler.addRequest(id, 'remove', video) 147
148function sendOwnedVideosToPod (podId) {
149 Videos.listOwned(function (err, videosList) {
150 if (err) {
151 logger.error('Cannot get the list of videos we own.')
152 return
153 }
154
155 videosList.forEach(function (video) {
156 videos.convertVideoToRemote(video, function (err, remoteVideo) {
157 if (err) {
158 logger.error('Cannot convert video to remote.', { error: err })
159 // Don't break the process
160 return
161 }
162
163 requestsScheduler.addRequestTo([ podId ], 'add', remoteVideo)
164 })
165 })
166 })
142} 167}
143 168
144// --------------------------------------------------------------------------- 169// ---------------------------------------------------------------------------
@@ -148,18 +173,19 @@ module.exports = pods
148// --------------------------------------------------------------------------- 173// ---------------------------------------------------------------------------
149 174
150function computeForeignPodsList (url, podsScore, callback) { 175function computeForeignPodsList (url, podsScore, callback) {
151 // Let's give 1 point to the pod we ask the friends list
152 podsScore[url] = 1
153
154 getForeignPodsList(url, function (err, foreignPodsList) { 176 getForeignPodsList(url, function (err, foreignPodsList) {
155 if (err) return callback(err) 177 if (err) return callback(err)
156 if (foreignPodsList.length === 0) return callback() 178
179 if (!foreignPodsList) foreignPodsList = []
180
181 // Let's give 1 point to the pod we ask the friends list
182 foreignPodsList.push({ url: url })
157 183
158 foreignPodsList.forEach(function (foreignPod) { 184 foreignPodsList.forEach(function (foreignPod) {
159 const foreignUrl = foreignPod.url 185 const foreignPodUrl = foreignPod.url
160 186
161 if (podsScore[foreignUrl]) podsScore[foreignUrl]++ 187 if (podsScore[foreignPodUrl]) podsScore[foreignPodUrl]++
162 else podsScore[foreignUrl] = 1 188 else podsScore[foreignPodUrl] = 1
163 }) 189 })
164 190
165 callback() 191 callback()
@@ -194,63 +220,43 @@ function makeRequestsToWinningPods (cert, podsList, callback) {
194 // Flush pool requests 220 // Flush pool requests
195 requestsScheduler.forceSend() 221 requestsScheduler.forceSend()
196 222
197 // Get the list of our videos to send to our new friends 223 async.eachLimit(podsList, constants.REQUESTS_IN_PARALLEL, function (pod, callbackEach) {
198 Videos.listOwned(function (err, videosList) { 224 const params = {
199 if (err) { 225 url: pod.url + '/api/' + constants.API_VERSION + '/pods/',
200 logger.error('Cannot get the list of videos we own.') 226 method: 'POST',
201 return callback(err) 227 json: {
202 } 228 url: http + '://' + host + ':' + port,
203 229 publicKey: cert
204 const data = { 230 }
205 url: http + '://' + host + ':' + port,
206 publicKey: cert,
207 videos: videosList
208 } 231 }
209 232
210 requests.makeMultipleRetryRequest( 233 requests.makeRetryRequest(params, function (err, res, body) {
211 { method: 'POST', path: '/api/' + constants.API_VERSION + '/pods/', data: data }, 234 if (err) {
212 235 logger.error('Error with adding %s pod.', pod.url, { error: err })
213 podsList, 236 // Don't break the process
214 237 return callbackEach()
215 // Callback called after each request 238 }
216 function eachRequest (err, response, body, url, pod, callbackEachRequest) {
217 // We add the pod if it responded correctly with its public certificate
218 if (!err && response.statusCode === 200) {
219 Pods.add({ url: pod.url, publicKey: body.cert, score: constants.FRIEND_BASE_SCORE }, function (err) {
220 if (err) {
221 logger.error('Error with adding %s pod.', pod.url, { error: err })
222 return callbackEachRequest()
223 }
224
225 videos.createRemoteVideos(body.videos, function (err) {
226 if (err) {
227 logger.error('Error with adding videos of pod.', pod.url, { error: err })
228 return callbackEachRequest()
229 }
230
231 logger.debug('Adding remote videos from %s.', pod.url, { videos: body.videos })
232 return callbackEachRequest()
233 })
234 })
235 } else {
236 logger.error('Error with adding %s pod.', pod.url, { error: err || new Error('Status not 200') })
237 return callbackEachRequest()
238 }
239 },
240 239
241 // Final callback, we've ended all the requests 240 if (res.statusCode === 200) {
242 function endRequests (err) { 241 Pods.add({ url: pod.url, publicKey: body.cert, score: constants.FRIEND_BASE_SCORE }, function (err, podCreated) {
243 // Now we made new friends, we can re activate the pool of requests 242 if (err) logger.error('Cannot add friend %s pod.', pod.url)
244 requestsScheduler.activate()
245 243
246 if (err) { 244 // Add our videos to the request scheduler
247 logger.error('There was some errors when we wanted to make friends.') 245 sendOwnedVideosToPod(podCreated._id)
248 return callback(err)
249 }
250 246
251 logger.debug('makeRequestsToWinningPods finished.') 247 return callbackEach()
252 return callback(null) 248 })
249 } else {
250 logger.error('Status not 200 for %s pod.', pod.url)
251 return callbackEach()
253 } 252 }
254 ) 253 })
254 }, function endRequests () {
255 // Final callback, we've ended all the requests
256 // Now we made new friends, we can re activate the pool of requests
257 requestsScheduler.activate()
258
259 logger.debug('makeRequestsToWinningPods finished.')
260 return callback()
255 }) 261 })
256} 262}
diff --git a/server/lib/requestsScheduler.js b/server/lib/requestsScheduler.js
index 78570209d..ac75e5b93 100644
--- a/server/lib/requestsScheduler.js
+++ b/server/lib/requestsScheduler.js
@@ -11,13 +11,14 @@ const requests = require('../helpers/requests')
11const videos = require('../lib/videos') 11const videos = require('../lib/videos')
12const Videos = require('../models/videos') 12const Videos = require('../models/videos')
13 13
14const REQUEST_SCHEDULER_TYPE = constants.REQUEST_SCHEDULER_TYPE
15let timer = null 14let timer = null
16 15
17const requestsScheduler = { 16const requestsScheduler = {
18 activate: activate, 17 activate: activate,
19 addRequest: addRequest, 18 addRequest: addRequest,
19 addRequestTo: addRequestTo,
20 deactivate: deactivate, 20 deactivate: deactivate,
21 flush: flush,
21 forceSend: forceSend 22 forceSend: forceSend
22} 23}
23 24
@@ -27,35 +28,37 @@ function activate () {
27} 28}
28 29
29// Add request to the scheduler 30// Add request to the scheduler
30function addRequest (id, type, request) { 31function addRequest (type, data) {
31 logger.debug('Add request to the requests scheduler.', { id: id, type: type, request: request }) 32 logger.debug('Add request of type %s to the requests scheduler.', type, { data: data })
32 33
33 Requests.findById(id, function (err, entity) { 34 const request = {
35 type: type,
36 data: data
37 }
38
39 Pods.listAllIds(function (err, podIds) {
34 if (err) { 40 if (err) {
35 logger.error('Error when trying to find a request.', { error: err }) 41 logger.debug('Cannot list pod ids.')
36 return // Abort 42 return
37 } 43 }
38 44
39 // If there were already a request with this id in the scheduler... 45 // No friends
40 if (entity) { 46 if (!podIds) return
41 if (entity.type === type) {
42 logger.error('Cannot insert two same requests.')
43 return // Abort
44 }
45 47
46 // Remove the request of the other type 48 Requests.create(request, podIds, function (err) {
47 Requests.removeRequestById(id, function (err) { 49 if (err) logger.error('Cannot create a request.', { error: err })
48 if (err) { 50 })
49 logger.error('Cannot remove a request.', { error: err }) 51 })
50 return // Abort 52}
51 } 53
52 }) 54function addRequestTo (podIds, type, data) {
53 } else { 55 const request = {
54 Requests.create(id, type, request, function (err) { 56 type: type,
55 if (err) logger.error('Cannot create a request.', { error: err }) 57 data: data
56 return // Abort 58 }
57 }) 59
58 } 60 Requests.create(request, podIds, function (err) {
61 if (err) logger.error('Cannot create a request.', { error: err })
59 }) 62 })
60} 63}
61 64
@@ -64,6 +67,14 @@ function deactivate () {
64 clearInterval(timer) 67 clearInterval(timer)
65} 68}
66 69
70function flush () {
71 Requests.removeAll(function (err) {
72 if (err) {
73 logger.error('Cannot flush the requests.', { error: err })
74 }
75 })
76}
77
67function forceSend () { 78function forceSend () {
68 logger.info('Force requests scheduler sending.') 79 logger.info('Force requests scheduler sending.')
69 makeRequests() 80 makeRequests()
@@ -76,54 +87,28 @@ module.exports = requestsScheduler
76// --------------------------------------------------------------------------- 87// ---------------------------------------------------------------------------
77 88
78// Make a requests to friends of a certain type 89// Make a requests to friends of a certain type
79function makeRequest (type, requestsToMake, callback) { 90function makeRequest (toPod, requestsToMake, callback) {
80 if (!callback) callback = function () {} 91 if (!callback) callback = function () {}
81 92
82 Pods.list(function (err, pods) { 93 const params = {
83 if (err) return callback(err) 94 toPod: toPod,
84 95 encrypt: true, // Security
85 const params = { 96 sign: true, // To prove our identity
86 encrypt: true, // Security 97 method: 'POST',
87 sign: true, // To prove our identity 98 path: '/api/' + constants.API_VERSION + '/remote/videos',
88 method: 'POST', 99 data: requestsToMake // Requests we need to make
89 path: null, // We build the path later 100 }
90 data: requestsToMake // Requests we need to make 101
91 } 102 // Make multiple retry requests to all of pods
92 103 // The function fire some useful callbacks
93 // If this is a valid type, we build the path 104 requests.makeSecureRequest(params, function (err, res) {
94 if (REQUEST_SCHEDULER_TYPE.indexOf(type) > -1) { 105 if (err || (res.statusCode !== 200 && res.statusCode !== 201 && res.statusCode !== 204)) {
95 params.path = '/api/' + constants.API_VERSION + '/remotevideos/' + type 106 logger.error('Error sending secure request to %s pod.', toPod.url, { error: err || new Error('Status code not 20x') })
96 } else { 107
97 return callback(new Error('Unkown pool request type.')) 108 return callback(false)
98 }
99
100 const badPods = []
101 const goodPods = []
102
103 // Make multiple retry requests to all of pods
104 // The function fire some useful callbacks
105 requests.makeMultipleRetryRequest(params, pods, callbackEachPodFinished, callbackAllPodsFinished)
106
107 function callbackEachPodFinished (err, response, body, url, pod, callbackEachPodFinished) {
108 // We failed the request, add the pod unreachable to the bad pods list
109 if (err || (response.statusCode !== 200 && response.statusCode !== 201 && response.statusCode !== 204)) {
110 badPods.push(pod._id)
111 logger.error('Error sending secure request to %s pod.', url, { error: err || new Error('Status code not 20x') })
112 } else {
113 // Request success
114 goodPods.push(pod._id)
115 }
116
117 return callbackEachPodFinished()
118 } 109 }
119 110
120 function callbackAllPodsFinished (err) { 111 return callback(true)
121 if (err) return callback(err)
122
123 // All the requests were made, we update the pods score
124 updatePodsScore(goodPods, badPods)
125 callback(null)
126 }
127 }) 112 })
128} 113}
129 114
@@ -143,38 +128,65 @@ function makeRequests () {
143 128
144 logger.info('Making requests to friends.') 129 logger.info('Making requests to friends.')
145 130
131 // Requests by pods id
146 const requestsToMake = {} 132 const requestsToMake = {}
147 for (const type of REQUEST_SCHEDULER_TYPE) {
148 requestsToMake[type] = {
149 ids: [],
150 requests: []
151 }
152 }
153 133
154 // For each requests to make, we add it to the correct request type
155 requests.forEach(function (poolRequest) { 134 requests.forEach(function (poolRequest) {
156 if (REQUEST_SCHEDULER_TYPE.indexOf(poolRequest.type) > -1) { 135 poolRequest.to.forEach(function (toPodId) {
157 const requestTypeToMake = requestsToMake[poolRequest.type] 136 if (!requestsToMake[toPodId]) {
158 requestTypeToMake.requests.push(poolRequest.request) 137 requestsToMake[toPodId] = {
159 requestTypeToMake.ids.push(poolRequest._id) 138 ids: [],
160 } else { 139 datas: []
161 logger.error('Unkown request type.', { request_type: poolRequest.type }) 140 }
162 return // abort 141 }
163 } 142
143 requestsToMake[toPodId].ids.push(poolRequest._id)
144 requestsToMake[toPodId].datas.push(poolRequest.request)
145 })
164 }) 146 })
165 147
166 for (let type of Object.keys(requestsToMake)) { 148 const goodPods = []
167 const requestTypeToMake = requestsToMake[type] 149 const badPods = []
168 // If there are requests for this type
169 if (requestTypeToMake.requests.length !== 0) {
170 makeRequest(type, requestTypeToMake.requests, function (err) {
171 if (err) logger.error('Errors when sent ' + type + ' requests.', { error: err })
172 150
173 // We made the requests, so we can remove them from the scheduler 151 async.eachLimit(Object.keys(requestsToMake), constants.REQUESTS_IN_PARALLEL, function (toPodId, callbackEach) {
174 Requests.removeRequests(requestTypeToMake.ids) 152 const requestToMake = requestsToMake[toPodId]
153
154 // FIXME: mongodb request inside a loop :/
155 Pods.findById(toPodId, function (err, toPod) {
156 if (err) return logger.error('Error finding pod by id.', { err: err })
157
158 // Maybe the pod is not our friend anymore so simply remove them
159 if (!toPod) {
160 Requests.removePodOf(requestToMake.ids, toPodId)
161 return callbackEach()
162 }
163
164 makeRequest(toPod, requestToMake.datas, function (success) {
165 if (err) {
166 logger.error('Errors when sent request to %s.', toPod.url, { error: err })
167 // Do not stop the process just for one error
168 return callbackEach()
169 }
170
171 if (success === true) {
172 logger.debug('Removing requests for %s pod.', toPodId, { requestsIds: requestToMake.ids })
173
174 // Remove the pod id of these request ids
175 Requests.removePodOf(requestToMake.ids, toPodId)
176 goodPods.push(toPodId)
177 } else {
178 badPods.push(toPodId)
179 }
180
181 callbackEach()
175 }) 182 })
176 } 183 })
177 } 184 }, function () {
185 // All the requests were made, we update the pods score
186 updatePodsScore(goodPods, badPods)
187 // Flush requests with no pod
188 Requests.removeWithEmptyTo()
189 })
178 }) 190 })
179} 191}
180 192
diff --git a/server/lib/videos.js b/server/lib/videos.js
index e0db0e1d5..a74c77dc4 100644
--- a/server/lib/videos.js
+++ b/server/lib/videos.js
@@ -17,6 +17,7 @@ const uploadDir = pathUtils.join(__dirname, '..', '..', config.get('storage.uplo
17const thumbnailsDir = pathUtils.join(__dirname, '..', '..', config.get('storage.thumbnails')) 17const thumbnailsDir = pathUtils.join(__dirname, '..', '..', config.get('storage.thumbnails'))
18 18
19const videos = { 19const videos = {
20 convertVideoToRemote: convertVideoToRemote,
20 createRemoteVideos: createRemoteVideos, 21 createRemoteVideos: createRemoteVideos,
21 getVideoDuration: getVideoDuration, 22 getVideoDuration: getVideoDuration,
22 getVideoState: getVideoState, 23 getVideoState: getVideoState,
@@ -27,6 +28,29 @@ const videos = {
27 seedAllExisting: seedAllExisting 28 seedAllExisting: seedAllExisting
28} 29}
29 30
31function convertVideoToRemote (video, callback) {
32 fs.readFile(thumbnailsDir + video.thumbnail, function (err, thumbnailData) {
33 if (err) {
34 logger.error('Cannot read the thumbnail of the video')
35 return callback(err)
36 }
37
38 const remoteVideo = {
39 name: video.name,
40 description: video.description,
41 magnetUri: video.magnetUri,
42 author: video.author,
43 duration: video.duration,
44 thumbnailBase64: new Buffer(thumbnailData).toString('base64'),
45 tags: video.tags,
46 createdDate: video.createdDate,
47 podUrl: video.podUrl
48 }
49
50 return callback(null, remoteVideo)
51 })
52}
53
30function createRemoteVideos (videos, callback) { 54function createRemoteVideos (videos, callback) {
31 // Create the remote videos from the new pod 55 // Create the remote videos from the new pod
32 createRemoteVideoObjects(videos, function (err, remoteVideos) { 56 createRemoteVideoObjects(videos, function (err, remoteVideos) {
@@ -154,7 +178,8 @@ function createRemoteVideoObjects (videos, callback) {
154 podUrl: video.podUrl, 178 podUrl: video.podUrl,
155 duration: video.duration, 179 duration: video.duration,
156 thumbnail: thumbnailName, 180 thumbnail: thumbnailName,
157 tags: video.tags 181 tags: video.tags,
182 author: video.author
158 } 183 }
159 remoteVideos.push(params) 184 remoteVideos.push(params)
160 185
diff --git a/server/middlewares/reqValidators/pods.js b/server/middlewares/reqValidators/pods.js
index 77449480c..78a4b76c1 100644
--- a/server/middlewares/reqValidators/pods.js
+++ b/server/middlewares/reqValidators/pods.js
@@ -26,8 +26,10 @@ function makeFriends (req, res, next) {
26} 26}
27 27
28function podsAdd (req, res, next) { 28function podsAdd (req, res, next) {
29 req.checkBody('data.url', 'Should have an url').notEmpty().isURL({ require_protocol: true }) 29 req.checkBody('url', 'Should have an url').notEmpty().isURL({ require_protocol: true })
30 req.checkBody('data.publicKey', 'Should have a public key').notEmpty() 30 req.checkBody('publicKey', 'Should have a public key').notEmpty()
31
32 // TODO: check we don't have it already
31 33
32 logger.debug('Checking podsAdd parameters', { parameters: req.body }) 34 logger.debug('Checking podsAdd parameters', { parameters: req.body })
33 35
diff --git a/server/middlewares/reqValidators/remote.js b/server/middlewares/reqValidators/remote.js
index b5f3118b0..a23673d89 100644
--- a/server/middlewares/reqValidators/remote.js
+++ b/server/middlewares/reqValidators/remote.js
@@ -4,36 +4,34 @@ const checkErrors = require('./utils').checkErrors
4const logger = require('../../helpers/logger') 4const logger = require('../../helpers/logger')
5 5
6const reqValidatorsRemote = { 6const reqValidatorsRemote = {
7 remoteVideosAdd: remoteVideosAdd, 7 dataToDecrypt: dataToDecrypt,
8 remoteVideosRemove: remoteVideosRemove, 8 remoteVideos: remoteVideos,
9 secureRequest: secureRequest 9 signature: signature
10} 10}
11 11
12function remoteVideosAdd (req, res, next) { 12function dataToDecrypt (req, res, next) {
13 req.checkBody('data').isArray() 13 req.checkBody('key', 'Should have a key').notEmpty()
14 req.checkBody('data').isEachAddRemoteVideosValid() 14 req.checkBody('data', 'Should have data').notEmpty()
15 15
16 logger.debug('Checking remoteVideosAdd parameters', { parameters: req.body }) 16 logger.debug('Checking dataToDecrypt parameters', { parameters: { keyLength: req.body.key.length, bodyLength: req.body.data.length } })
17 17
18 checkErrors(req, res, next) 18 checkErrors(req, res, next)
19} 19}
20 20
21function remoteVideosRemove (req, res, next) { 21function remoteVideos (req, res, next) {
22 req.checkBody('data').isArray() 22 req.checkBody('data').isArray()
23 req.checkBody('data').isEachRemoveRemoteVideosValid() 23 req.checkBody('data').isEachRemoteVideosValid()
24 24
25 logger.debug('Checking remoteVideosRemove parameters', { parameters: req.body }) 25 logger.debug('Checking remoteVideosAdd parameters', { parameters: req.body })
26 26
27 checkErrors(req, res, next) 27 checkErrors(req, res, next)
28} 28}
29 29
30function secureRequest (req, res, next) { 30function signature (req, res, next) {
31 req.checkBody('signature.url', 'Should have a signature url').isURL() 31 req.checkBody('signature.url', 'Should have a signature url').isURL()
32 req.checkBody('signature.signature', 'Should have a signature').notEmpty() 32 req.checkBody('signature.signature', 'Should have a signature').notEmpty()
33 req.checkBody('key', 'Should have a key').notEmpty()
34 req.checkBody('data', 'Should have data').notEmpty()
35 33
36 logger.debug('Checking secureRequest parameters', { parameters: { data: req.body.data, keyLength: req.body.key.length } }) 34 logger.debug('Checking signature parameters', { parameters: { signatureUrl: req.body.signature.url } })
37 35
38 checkErrors(req, res, next) 36 checkErrors(req, res, next)
39} 37}
diff --git a/server/models/pods.js b/server/models/pods.js
index 04cc2d6fc..daeadeb07 100644
--- a/server/models/pods.js
+++ b/server/models/pods.js
@@ -19,10 +19,13 @@ const PodsDB = mongoose.model('pods', podsSchema)
19const Pods = { 19const Pods = {
20 add: add, 20 add: add,
21 count: count, 21 count: count,
22 findById: findById,
22 findByUrl: findByUrl, 23 findByUrl: findByUrl,
23 findBadPods: findBadPods, 24 findBadPods: findBadPods,
24 incrementScores: incrementScores, 25 incrementScores: incrementScores,
25 list: list, 26 list: list,
27 listAllIds: listAllIds,
28 listAllUrls: listAllUrls,
26 remove: remove, 29 remove: remove,
27 removeAll: removeAll, 30 removeAll: removeAll,
28 removeAllByIds: removeAllByIds 31 removeAllByIds: removeAllByIds
@@ -48,6 +51,10 @@ function findBadPods (callback) {
48 PodsDB.find({ score: 0 }, callback) 51 PodsDB.find({ score: 0 }, callback)
49} 52}
50 53
54function findById (id, callback) {
55 PodsDB.findById(id, callback)
56}
57
51function findByUrl (url, callback) { 58function findByUrl (url, callback) {
52 PodsDB.findOne({ url: url }, callback) 59 PodsDB.findOne({ url: url }, callback)
53} 60}
@@ -68,6 +75,14 @@ function list (callback) {
68 }) 75 })
69} 76}
70 77
78function listAllIds (callback) {
79 return PodsDB.find({}, { _id: 1 }, callback)
80}
81
82function listAllUrls (callback) {
83 return PodsDB.find({}, { _id: 0, url: 1 }, callback)
84}
85
71function remove (url, callback) { 86function remove (url, callback) {
72 if (!callback) callback = function () {} 87 if (!callback) callback = function () {}
73 PodsDB.remove({ url: url }, callback) 88 PodsDB.remove({ url: url }, callback)
diff --git a/server/models/requests.js b/server/models/requests.js
index 2152ae0e9..e67ccad56 100644
--- a/server/models/requests.js
+++ b/server/models/requests.js
@@ -7,9 +7,8 @@ const logger = require('../helpers/logger')
7// --------------------------------------------------------------------------- 7// ---------------------------------------------------------------------------
8 8
9const requestsSchema = mongoose.Schema({ 9const requestsSchema = mongoose.Schema({
10 type: String, 10 request: mongoose.Schema.Types.Mixed,
11 id: String, // Special id to find duplicates (video created we want to remove...) 11 to: [ { type: mongoose.Schema.Types.ObjectId, ref: 'users' } ]
12 request: mongoose.Schema.Types.Mixed
13}) 12})
14const RequestsDB = mongoose.model('requests', requestsSchema) 13const RequestsDB = mongoose.model('requests', requestsSchema)
15 14
@@ -19,12 +18,15 @@ const Requests = {
19 create: create, 18 create: create,
20 findById: findById, 19 findById: findById,
21 list: list, 20 list: list,
21 removeAll: removeAll,
22 removePodOf: removePodOf,
22 removeRequestById: removeRequestById, 23 removeRequestById: removeRequestById,
23 removeRequests: removeRequests 24 removeRequests: removeRequests,
25 removeWithEmptyTo: removeWithEmptyTo
24} 26}
25 27
26function create (id, type, request, callback) { 28function create (request, to, callback) {
27 RequestsDB.create({ id: id, type: type, request: request }, callback) 29 RequestsDB.create({ request: request, to: to }, callback)
28} 30}
29 31
30function findById (id, callback) { 32function findById (id, callback) {
@@ -32,7 +34,17 @@ function findById (id, callback) {
32} 34}
33 35
34function list (callback) { 36function list (callback) {
35 RequestsDB.find({}, { _id: 1, type: 1, request: 1 }, callback) 37 RequestsDB.find({}, { _id: 1, request: 1, to: 1 }, callback)
38}
39
40function removeAll (callback) {
41 RequestsDB.remove({ }, callback)
42}
43
44function removePodOf (requestsIds, podId, callback) {
45 if (!callback) callback = function () {}
46
47 RequestsDB.update({ _id: { $in: requestsIds } }, { $pull: { to: podId } }, { multi: true }, callback)
36} 48}
37 49
38function removeRequestById (id, callback) { 50function removeRequestById (id, callback) {
@@ -50,6 +62,12 @@ function removeRequests (ids) {
50 }) 62 })
51} 63}
52 64
65function removeWithEmptyTo (callback) {
66 if (!callback) callback = function () {}
67
68 RequestsDB.remove({ to: { $size: 0 } }, callback)
69}
70
53// --------------------------------------------------------------------------- 71// ---------------------------------------------------------------------------
54 72
55module.exports = Requests 73module.exports = Requests
diff --git a/server/tests/api/checkParams.js b/server/tests/api/checkParams.js
index 95a7738f8..7f22a37cc 100644
--- a/server/tests/api/checkParams.js
+++ b/server/tests/api/checkParams.js
@@ -90,33 +90,27 @@ describe('Test parameters validator', function () {
90 90
91 it('Should fail without public key', function (done) { 91 it('Should fail without public key', function (done) {
92 const data = { 92 const data = {
93 data: { 93 url: 'http://coucou.com'
94 url: 'http://coucou.com'
95 }
96 } 94 }
97 makePostBodyRequest(path, data, done) 95 makePostBodyRequest(path, data, done)
98 }) 96 })
99 97
100 it('Should fail without an url', function (done) { 98 it('Should fail without an url', function (done) {
101 const data = { 99 const data = {
102 data: { 100 publicKey: 'mysuperpublickey'
103 publicKey: 'mysuperpublickey'
104 }
105 } 101 }
106 makePostBodyRequest(path, data, done) 102 makePostBodyRequest(path, data, done)
107 }) 103 })
108 104
109 it('Should fail with an incorrect url', function (done) { 105 it('Should fail with an incorrect url', function (done) {
110 const data = { 106 const data = {
111 data: { 107 url: 'coucou.com',
112 url: 'coucou.com', 108 publicKey: 'mysuperpublickey'
113 publicKey: 'mysuperpublickey'
114 }
115 } 109 }
116 makePostBodyRequest(path, data, function () { 110 makePostBodyRequest(path, data, function () {
117 data.data.url = 'http://coucou' 111 data.url = 'http://coucou'
118 makePostBodyRequest(path, data, function () { 112 makePostBodyRequest(path, data, function () {
119 data.data.url = 'coucou' 113 data.url = 'coucou'
120 makePostBodyRequest(path, data, done) 114 makePostBodyRequest(path, data, done)
121 }) 115 })
122 }) 116 })
@@ -124,10 +118,8 @@ describe('Test parameters validator', function () {
124 118
125 it('Should succeed with the correct parameters', function (done) { 119 it('Should succeed with the correct parameters', function (done) {
126 const data = { 120 const data = {
127 data: { 121 url: 'http://coucou.com',
128 url: 'http://coucou.com', 122 publicKey: 'mysuperpublickey'
129 publicKey: 'mysuperpublickey'
130 }
131 } 123 }
132 makePostBodyRequest(path, data, done, false) 124 makePostBodyRequest(path, data, done, false)
133 }) 125 })
diff --git a/server/tests/api/friendsAdvanced.js b/server/tests/api/friendsAdvanced.js
index 86620254e..b082270ff 100644
--- a/server/tests/api/friendsAdvanced.js
+++ b/server/tests/api/friendsAdvanced.js
@@ -130,6 +130,18 @@ describe('Test advanced friends', function () {
130 function (next) { 130 function (next) {
131 makeFriends(4, next) 131 makeFriends(4, next)
132 }, 132 },
133 // Check the pods 1, 2, 3 and 4 are friends
134 function (next) {
135 async.each([ 1, 2, 3, 4 ], function (i, callback) {
136 getFriendsList(i, function (err, res) {
137 if (err) throw err
138
139 expect(res.body.length).to.equal(3)
140
141 callback()
142 })
143 }, next)
144 },
133 // Kill pod 4 145 // Kill pod 4
134 function (next) { 146 function (next) {
135 servers[3].app.kill() 147 servers[3].app.kill()
@@ -152,7 +164,7 @@ describe('Test advanced friends', function () {
152 uploadVideo(2, next) 164 uploadVideo(2, next)
153 }, 165 },
154 function (next) { 166 function (next) {
155 setTimeout(next, 20000) 167 setTimeout(next, 11000)
156 }, 168 },
157 // Rerun server 4 169 // Rerun server 4
158 function (next) { 170 function (next) {
@@ -173,6 +185,9 @@ describe('Test advanced friends', function () {
173 // Pod 6 ask pod 1, 2 and 3 185 // Pod 6 ask pod 1, 2 and 3
174 function (next) { 186 function (next) {
175 makeFriends(6, next) 187 makeFriends(6, next)
188 },
189 function (next) {
190 setTimeout(next, 11000)
176 }], 191 }],
177 function (err) { 192 function (err) {
178 if (err) throw err 193 if (err) throw err
@@ -247,7 +262,7 @@ describe('Test advanced friends', function () {
247 262
248 done() 263 done()
249 }) 264 })
250 }, 5000) 265 }, 11000)
251 }) 266 })
252 }) 267 })
253 268
diff --git a/server/tests/api/friendsBasic.js b/server/tests/api/friendsBasic.js
index 68817e852..5b738ad39 100644
--- a/server/tests/api/friendsBasic.js
+++ b/server/tests/api/friendsBasic.js
@@ -25,9 +25,10 @@ describe('Test basic friends', function () {
25 if (err) throw err 25 if (err) throw err
26 26
27 const result = res.body 27 const result = res.body
28 const resultUrls = [ result[0].url, result[1].url ]
29 expect(result).to.be.an('array') 28 expect(result).to.be.an('array')
30 expect(result.length).to.equal(2) 29 expect(result.length).to.equal(2)
30
31 const resultUrls = [ result[0].url, result[1].url ]
31 expect(resultUrls[0]).to.not.equal(resultUrls[1]) 32 expect(resultUrls[0]).to.not.equal(resultUrls[1])
32 33
33 const errorString = 'Friends url do not correspond for ' + serverToTest.url 34 const errorString = 'Friends url do not correspond for ' + serverToTest.url
diff --git a/server/tests/api/multiplePods.js b/server/tests/api/multiplePods.js
index 40326c260..2a1bc64e6 100644
--- a/server/tests/api/multiplePods.js
+++ b/server/tests/api/multiplePods.js
@@ -105,6 +105,7 @@ describe('Test multiple pods', function () {
105 expect(video.duration).to.equal(10) 105 expect(video.duration).to.equal(10)
106 expect(video.tags).to.deep.equal([ 'tag1p1', 'tag2p1' ]) 106 expect(video.tags).to.deep.equal([ 'tag1p1', 'tag2p1' ])
107 expect(utils.dateIsValid(video.createdDate)).to.be.true 107 expect(utils.dateIsValid(video.createdDate)).to.be.true
108 expect(video.author).to.equal('root')
108 109
109 if (server.url !== 'http://localhost:9001') { 110 if (server.url !== 'http://localhost:9001') {
110 expect(video.isLocal).to.be.false 111 expect(video.isLocal).to.be.false
@@ -166,6 +167,7 @@ describe('Test multiple pods', function () {
166 expect(video.duration).to.equal(5) 167 expect(video.duration).to.equal(5)
167 expect(video.tags).to.deep.equal([ 'tag1p2', 'tag2p2', 'tag3p2' ]) 168 expect(video.tags).to.deep.equal([ 'tag1p2', 'tag2p2', 'tag3p2' ])
168 expect(utils.dateIsValid(video.createdDate)).to.be.true 169 expect(utils.dateIsValid(video.createdDate)).to.be.true
170 expect(video.author).to.equal('root')
169 171
170 if (server.url !== 'http://localhost:9002') { 172 if (server.url !== 'http://localhost:9002') {
171 expect(video.isLocal).to.be.false 173 expect(video.isLocal).to.be.false
@@ -243,6 +245,7 @@ describe('Test multiple pods', function () {
243 expect(video1.magnetUri).to.exist 245 expect(video1.magnetUri).to.exist
244 expect(video1.duration).to.equal(5) 246 expect(video1.duration).to.equal(5)
245 expect(video1.tags).to.deep.equal([ 'tag1p3' ]) 247 expect(video1.tags).to.deep.equal([ 'tag1p3' ])
248 expect(video1.author).to.equal('root')
246 expect(utils.dateIsValid(video1.createdDate)).to.be.true 249 expect(utils.dateIsValid(video1.createdDate)).to.be.true
247 250
248 expect(video2.name).to.equal('my super name for pod 3-2') 251 expect(video2.name).to.equal('my super name for pod 3-2')
@@ -251,6 +254,7 @@ describe('Test multiple pods', function () {
251 expect(video2.magnetUri).to.exist 254 expect(video2.magnetUri).to.exist
252 expect(video2.duration).to.equal(5) 255 expect(video2.duration).to.equal(5)
253 expect(video2.tags).to.deep.equal([ 'tag2p3', 'tag3p3', 'tag4p3' ]) 256 expect(video2.tags).to.deep.equal([ 'tag2p3', 'tag3p3', 'tag4p3' ])
257 expect(video2.author).to.equal('root')
254 expect(utils.dateIsValid(video2.createdDate)).to.be.true 258 expect(utils.dateIsValid(video2.createdDate)).to.be.true
255 259
256 if (server.url !== 'http://localhost:9003') { 260 if (server.url !== 'http://localhost:9003') {