aboutsummaryrefslogtreecommitdiffhomepage
path: root/models
diff options
context:
space:
mode:
Diffstat (limited to 'models')
-rw-r--r--models/pods.js276
-rw-r--r--models/poolRequests.js67
-rw-r--r--models/videos.js202
3 files changed, 185 insertions, 360 deletions
diff --git a/models/pods.js b/models/pods.js
index ed2f0d8ee..395b1e0b1 100644
--- a/models/pods.js
+++ b/models/pods.js
@@ -1,73 +1,61 @@
1;(function () { 1;(function () {
2 'use strict' 2 'use strict'
3 3
4 var async = require('async') 4 var mongoose = require('mongoose')
5 var config = require('config')
6 var fs = require('fs')
7 var request = require('request')
8 5
9 var constants = require('../initializers/constants') 6 var constants = require('../initializers/constants')
10 var logger = require('../helpers/logger') 7 var logger = require('../helpers/logger')
11 var PodsDB = require('../initializers/database').PodsDB
12 var poolRequests = require('../lib/poolRequests')
13 var utils = require('../helpers/utils')
14 8
15 var http = config.get('webserver.https') ? 'https' : 'http' 9 // ---------------------------------------------------------------------------
16 var host = config.get('webserver.host') 10
17 var port = config.get('webserver.port') 11 var podsSchema = mongoose.Schema({
12 url: String,
13 publicKey: String,
14 score: { type: Number, max: constants.FRIEND_BASE_SCORE }
15 })
16 var PodsDB = mongoose.model('pods', podsSchema)
18 17
19 var pods = { 18 // ---------------------------------------------------------------------------
19
20 var Pods = {
20 add: add, 21 add: add,
21 addVideoToFriends: addVideoToFriends, 22 count: count,
23 findByUrl: findByUrl,
24 findBadPods: findBadPods,
25 incrementScores: incrementScores,
22 list: list, 26 list: list,
23 hasFriends: hasFriends,
24 makeFriends: makeFriends,
25 quitFriends: quitFriends,
26 remove: remove, 27 remove: remove,
27 removeVideoToFriends 28 removeAll: removeAll,
29 removeAllByIds: removeAllByIds
28 } 30 }
29 31
30 // TODO: check if the pod is not already a friend 32 // TODO: check if the pod is not already a friend
31 function add (data, callback) { 33 function add (data, callback) {
32 var videos = require('./videos') 34 if (!callback) callback = function () {}
33 logger.info('Adding pod: %s', data.url)
34
35 var params = { 35 var params = {
36 url: data.url, 36 url: data.url,
37 publicKey: data.publicKey, 37 publicKey: data.publicKey,
38 score: constants.FRIEND_BASE_SCORE 38 score: constants.FRIEND_BASE_SCORE
39 } 39 }
40 40
41 PodsDB.create(params, function (err, pod) { 41 PodsDB.create(params, callback)
42 if (err) { 42 }
43 logger.error('Cannot insert the pod.', { error: err })
44 return callback(err)
45 }
46
47 videos.addRemotes(data.videos)
48 43
49 fs.readFile(utils.getCertDir() + 'peertube.pub', 'utf8', function (err, cert) { 44 function count (callback) {
50 if (err) { 45 return PodsDB.count(callback)
51 logger.error('Cannot read cert file.', { error: err }) 46 }
52 return callback(err)
53 }
54 47
55 videos.listOwned(function (err, videos_list) { 48 function findBadPods (callback) {
56 if (err) { 49 PodsDB.find({ score: 0 }, callback)
57 logger.error('Cannot get the list of owned videos.', { error: err }) 50 }
58 return callback(err)
59 }
60 51
61 return callback(null, { cert: cert, videos: videos_list }) 52 function findByUrl (url, callback) {
62 }) 53 PodsDB.findOne({ url: url }, callback)
63 })
64 })
65 } 54 }
66 55
67 function addVideoToFriends (video) { 56 function incrementScores (ids, value, callback) {
68 // To avoid duplicates 57 if (!callback) callback = function () {}
69 var id = video.name + video.magnetUri 58 PodsDB.update({ _id: { $in: ids } }, { $inc: { score: value } }, { multi: true }, callback)
70 poolRequests.addToPoolRequests(id, 'add', video)
71 } 59 }
72 60
73 function list (callback) { 61 function list (callback) {
@@ -81,202 +69,22 @@
81 }) 69 })
82 } 70 }
83 71
84 function hasFriends (callback) {
85 PodsDB.count(function (err, count) {
86 if (err) return callback(err)
87
88 var has_friends = (count !== 0)
89 callback(null, has_friends)
90 })
91 }
92
93 function makeFriends (callback) {
94 var videos = require('./videos')
95 var pods_score = {}
96
97 logger.info('Make friends!')
98 fs.readFile(utils.getCertDir() + 'peertube.pub', 'utf8', function (err, cert) {
99 if (err) {
100 logger.error('Cannot read public cert.', { error: err })
101 return callback(err)
102 }
103
104 var urls = config.get('network.friends')
105
106 async.each(urls, computeForeignPodsList, function () {
107 logger.debug('Pods scores computed.', { pods_score: pods_score })
108 var pods_list = computeWinningPods(urls, pods_score)
109 logger.debug('Pods that we keep computed.', { pods_to_keep: pods_list })
110
111 makeRequestsToWinningPods(cert, pods_list)
112 })
113 })
114
115 // -----------------------------------------------------------------------
116
117 function computeForeignPodsList (url, callback) {
118 // Let's give 1 point to the pod we ask the friends list
119 pods_score[url] = 1
120
121 getForeignPodsList(url, function (foreign_pods_list) {
122 if (foreign_pods_list.length === 0) return callback()
123
124 async.each(foreign_pods_list, function (foreign_pod, callback_each) {
125 var foreign_url = foreign_pod.url
126
127 if (pods_score[foreign_url]) pods_score[foreign_url]++
128 else pods_score[foreign_url] = 1
129
130 callback_each()
131 }, function () {
132 callback()
133 })
134 })
135 }
136
137 function computeWinningPods (urls, pods_score) {
138 // Build the list of pods to add
139 // Only add a pod if it exists in more than a half base pods
140 var pods_list = []
141 var base_score = urls.length / 2
142 Object.keys(pods_score).forEach(function (pod) {
143 if (pods_score[pod] > base_score) pods_list.push({ url: pod })
144 })
145
146 return pods_list
147 }
148
149 function makeRequestsToWinningPods (cert, pods_list) {
150 // Stop pool requests
151 poolRequests.deactivate()
152 // Flush pool requests
153 poolRequests.forceSend()
154
155 // Get the list of our videos to send to our new friends
156 videos.listOwned(function (err, videos_list) {
157 if (err) throw err
158
159 var data = {
160 url: http + '://' + host + ':' + port,
161 publicKey: cert,
162 videos: videos_list
163 }
164
165 utils.makeMultipleRetryRequest(
166 { method: 'POST', path: '/api/' + constants.API_VERSION + '/pods/', data: data },
167
168 pods_list,
169
170 function eachRequest (err, response, body, url, pod, callback_each_request) {
171 // We add the pod if it responded correctly with its public certificate
172 if (!err && response.statusCode === 200) {
173 add({ url: pod.url, publicKey: body.cert, score: constants.FRIEND_BASE_SCORE }, function (err) {
174 if (err) logger.error('Error with adding %s pod.', pod.url, { error: err })
175
176 videos.addRemotes(body.videos, function (err) {
177 if (err) logger.error('Error with adding videos of pod.', pod.url, { error: err })
178
179 logger.debug('Adding remote videos from %s.', pod.url, { videos: body.videos })
180 return callback_each_request()
181 })
182 })
183 } else {
184 logger.error('Error with adding %s pod.', pod.url, { error: err || new Error('Status not 200') })
185 return callback_each_request()
186 }
187 },
188
189 function endRequests (err) {
190 // Now we made new friends, we can re activate the pool of requests
191 poolRequests.activate()
192
193 if (err) {
194 logger.error('There was some errors when we wanted to make friends.', { error: err })
195 return callback(err)
196 }
197
198 logger.debug('makeRequestsToWinningPods finished.')
199 return callback(null)
200 }
201 )
202 })
203 }
204 }
205
206 function quitFriends (callback) {
207 // Stop pool requests
208 poolRequests.deactivate()
209 // Flush pool requests
210 poolRequests.forceSend()
211
212 PodsDB.find(function (err, pods) {
213 if (err) return callback(err)
214
215 var request = {
216 method: 'POST',
217 path: '/api/' + constants.API_VERSION + '/pods/remove',
218 sign: true,
219 encrypt: true,
220 data: {
221 url: 'me' // Fake data
222 }
223 }
224
225 // Announce we quit them
226 utils.makeMultipleRetryRequest(request, pods, function () {
227 PodsDB.remove(function (err) {
228 poolRequests.activate()
229
230 if (err) return callback(err)
231
232 logger.info('Broke friends, so sad :(')
233
234 var videos = require('./videos')
235 videos.removeAllRemotes(function (err) {
236 if (err) return callback(err)
237
238 logger.info('Removed all remote videos.')
239 callback(null)
240 })
241 })
242 })
243 })
244 }
245
246 function remove (url, callback) { 72 function remove (url, callback) {
247 var videos = require('./videos') 73 if (!callback) callback = function () {}
248 logger.info('Removing %s pod.', url) 74 PodsDB.remove({ url: url }, callback)
249
250 videos.removeAllRemotesOf(url, function (err) {
251 if (err) logger.error('Cannot remove all remote videos of %s.', url)
252
253 PodsDB.remove({ url: url }, function (err) {
254 if (err) return callback(err)
255
256 logger.info('%s pod removed.', url)
257 callback(null)
258 })
259 })
260 } 75 }
261 76
262 function removeVideoToFriends (video) { 77 function removeAll (callback) {
263 // To avoid duplicates 78 if (!callback) callback = function () {}
264 var id = video.name + video.magnetUri 79 PodsDB.remove(callback)
265 poolRequests.addToPoolRequests(id, 'remove', video)
266 } 80 }
267 81
268 // --------------------------------------------------------------------------- 82 function removeAllByIds (ids, callback) {
269 83 if (!callback) callback = function () {}
270 module.exports = pods 84 PodsDB.remove({ _id: { $in: ids } }, callback)
85 }
271 86
272 // --------------------------------------------------------------------------- 87 // ---------------------------------------------------------------------------
273 88
274 function getForeignPodsList (url, callback) { 89 module.exports = Pods
275 var path = '/api/' + constants.API_VERSION + '/pods'
276
277 request.get(url + path, function (err, response, body) {
278 if (err) throw err
279 callback(JSON.parse(body))
280 })
281 }
282})() 90})()
diff --git a/models/poolRequests.js b/models/poolRequests.js
new file mode 100644
index 000000000..0f488ef04
--- /dev/null
+++ b/models/poolRequests.js
@@ -0,0 +1,67 @@
1;(function () {
2 'use strict'
3
4 var mongoose = require('mongoose')
5
6 var logger = require('../helpers/logger')
7
8 // ---------------------------------------------------------------------------
9
10 var poolRequestsSchema = mongoose.Schema({
11 type: String,
12 id: String, // Special id to find duplicates (video created we want to remove...)
13 request: mongoose.Schema.Types.Mixed
14 })
15 var PoolRequestsDB = mongoose.model('poolRequests', poolRequestsSchema)
16
17 // ---------------------------------------------------------------------------
18
19 var PoolRequests = {
20 addRequest: addRequest,
21 list: list,
22 removeRequests: removeRequests
23 }
24
25 function addRequest (id, type, request) {
26 logger.debug('Add request to the pool requests.', { id: id, type: type, request: request })
27
28 PoolRequestsDB.findOne({ id: id }, function (err, entity) {
29 if (err) logger.error(err)
30
31 if (entity) {
32 if (entity.type === type) {
33 logger.error(new Error('Cannot insert two same requests.'))
34 return
35 }
36
37 // Remove the request of the other type
38 PoolRequestsDB.remove({ id: id }, function (err) {
39 if (err) logger.error(err)
40 })
41 } else {
42 PoolRequestsDB.create({ id: id, type: type, request: request }, function (err) {
43 if (err) logger.error(err)
44 })
45 }
46 })
47 }
48
49 function list (callback) {
50 PoolRequestsDB.find({}, { _id: 1, type: 1, request: 1 }, callback)
51 }
52
53 function removeRequests (ids) {
54 PoolRequestsDB.remove({ _id: { $in: ids } }, function (err) {
55 if (err) {
56 logger.error('Cannot remove requests from the pool requests database.', { error: err })
57 return
58 }
59
60 logger.info('Pool requests flushed.')
61 })
62 }
63
64 // ---------------------------------------------------------------------------
65
66 module.exports = PoolRequests
67})()
diff --git a/models/videos.js b/models/videos.js
index 5711c5657..10abee6e7 100644
--- a/models/videos.js
+++ b/models/videos.js
@@ -5,71 +5,62 @@
5 var config = require('config') 5 var config = require('config')
6 var dz = require('dezalgo') 6 var dz = require('dezalgo')
7 var fs = require('fs') 7 var fs = require('fs')
8 var webtorrent = require('../lib/webTorrentNode') 8 var mongoose = require('mongoose')
9 9
10 var logger = require('../helpers/logger') 10 var logger = require('../helpers/logger')
11 var pods = require('./pods')
12 var VideosDB = require('../initializers/database').VideosDB
13 11
14 var http = config.get('webserver.https') === true ? 'https' : 'http' 12 var http = config.get('webserver.https') === true ? 'https' : 'http'
15 var host = config.get('webserver.host') 13 var host = config.get('webserver.host')
16 var port = config.get('webserver.port') 14 var port = config.get('webserver.port')
15 var uploadDir = __dirname + '/../' + config.get('storage.uploads')
16
17 // ---------------------------------------------------------------------------
18
19 var videosSchema = mongoose.Schema({
20 name: String,
21 namePath: String,
22 description: String,
23 magnetUri: String,
24 podUrl: String
25 })
26 var VideosDB = mongoose.model('videos', videosSchema)
27
28 // ---------------------------------------------------------------------------
17 29
18 var videos = { 30 var Videos = {
19 add: add, 31 add: add,
20 addRemotes: addRemotes, 32 addRemotes: addRemotes,
21 get: get, 33 get: get,
34 getVideoState: getVideoState,
35 isOwned: isOwned,
22 list: list, 36 list: list,
23 listOwned: listOwned, 37 listOwned: listOwned,
24 remove: remove, 38 removeOwned: removeOwned,
25 removeAllRemotes: removeAllRemotes, 39 removeAllRemotes: removeAllRemotes,
26 removeAllRemotesOf: removeAllRemotesOf, 40 removeAllRemotesOf: removeAllRemotesOf,
27 removeRemotes: removeRemotes, 41 removeRemotesOfByMagnetUris: removeRemotesOfByMagnetUris,
28 search: search, 42 search: search
29 seedAll: seedAll,
30 uploadDir: uploadDir
31 } 43 }
32 44
33 // ----------- Public attributes ---------- 45 function add (video, callback) {
34 var uploadDir = __dirname + '/../' + config.get('storage.uploads') 46 logger.info('Adding %s video to database.', video.name)
35 47
36 function add (data, callback) { 48 var params = video
37 var video_file = data.video 49 params.podUrl = http + '://' + host + ':' + port
38 var video_data = data.data
39 50
40 logger.info('Adding %s video.', video_file.path) 51 VideosDB.create(params, function (err, video) {
41 seedVideo(video_file.path, function (err, torrent) {
42 if (err) { 52 if (err) {
43 logger.error('Cannot seed this video.', { error: err }) 53 logger.error('Cannot insert this video into database.', { error: err })
44 return callback(err) 54 return callback(err)
45 } 55 }
46 56
47 var params = { 57 callback(null)
48 name: video_data.name,
49 namePath: video_file.filename,
50 description: video_data.description,
51 magnetUri: torrent.magnetURI,
52 podUrl: http + '://' + host + ':' + port
53 }
54
55 VideosDB.create(params, function (err, video) {
56 if (err) {
57 logger.error('Cannot insert this video.', { error: err })
58 return callback(err)
59 }
60
61 // Now we'll add the video's meta data to our friends
62 params.namePath = null
63
64 pods.addVideoToFriends(params)
65 callback(null)
66 })
67 }) 58 })
68 } 59 }
69 60
70 // TODO: avoid doublons 61 // TODO: avoid doublons
71 function addRemotes (videos, callback) { 62 function addRemotes (videos, callback) {
72 if (callback === undefined) callback = function () {} 63 if (!callback) callback = function () {}
73 64
74 var to_add = [] 65 var to_add = []
75 66
@@ -111,6 +102,38 @@
111 }) 102 })
112 } 103 }
113 104
105 function getVideoState (id, callback) {
106 get(id, function (err, video) {
107 if (err) return callback(err)
108
109 var exist = (video !== null)
110 var owned = false
111 if (exist === true) {
112 owned = (video.namePath !== null)
113 }
114
115 return callback(null, { exist: exist, owned: owned })
116 })
117 }
118
119 function isOwned (id, callback) {
120 VideosDB.findById(id, function (err, video) {
121 if (err || !video) {
122 if (!err) err = new Error('Cannot find this video.')
123 logger.error('Cannot find this video.', { error: err })
124 return callback(err)
125 }
126
127 if (video.namePath === null) {
128 var error_string = 'Cannot remove the video of another pod.'
129 logger.error(error_string)
130 return callback(null, false, video)
131 }
132
133 callback(null, true, video)
134 })
135 }
136
114 function list (callback) { 137 function list (callback) {
115 VideosDB.find(function (err, videos_list) { 138 VideosDB.find(function (err, videos_list) {
116 if (err) { 139 if (err) {
@@ -134,76 +157,35 @@
134 }) 157 })
135 } 158 }
136 159
137 function remove (id, callback) { 160 function removeOwned (id, callback) {
138 // Maybe the torrent is not seeded, but we catch the error to don't stop the removing process 161 VideosDB.findByIdAndRemove(id, function (err, video) {
139 function removeTorrent (magnetUri, callback) { 162 if (err) {
140 try { 163 logger.error('Cannot remove the torrent.', { error: err })
141 webtorrent.remove(magnetUri, callback)
142 } catch (err) {
143 logger.warn('Cannot remove the torrent from WebTorrent', { err: err })
144 return callback(null)
145 }
146 }
147
148 VideosDB.findById(id, function (err, video) {
149 if (err || !video) {
150 if (!err) err = new Error('Cannot find this video.')
151 logger.error('Cannot find this video.', { error: err })
152 return callback(err) 164 return callback(err)
153 } 165 }
154 166
155 if (video.namePath === null) { 167 fs.unlink(uploadDir + video.namePath, function (err) {
156 var error_string = 'Cannot remove the video of another pod.' 168 if (err) {
157 logger.error(error_string) 169 logger.error('Cannot remove this video file.', { error: err })
158 return callback(new Error(error_string)) 170 return callback(err)
159 } 171 }
160
161 logger.info('Removing %s video', video.name)
162
163 removeTorrent(video.magnetUri, function () {
164 VideosDB.findByIdAndRemove(id, function (err) {
165 if (err) {
166 logger.error('Cannot remove the torrent.', { error: err })
167 return callback(err)
168 }
169
170 fs.unlink(uploadDir + video.namePath, function (err) {
171 if (err) {
172 logger.error('Cannot remove this video file.', { error: err })
173 return callback(err)
174 }
175
176 var params = {
177 name: video.name,
178 magnetUri: video.magnetUri
179 }
180 172
181 pods.removeVideoToFriends(params) 173 callback(null)
182 callback(null)
183 })
184 })
185 }) 174 })
186 }) 175 })
187 } 176 }
188 177
189 function removeAllRemotes (callback) { 178 function removeAllRemotes (callback) {
190 VideosDB.remove({ namePath: null }, function (err) { 179 VideosDB.remove({ namePath: null }, callback)
191 if (err) return callback(err)
192
193 callback(null)
194 })
195 } 180 }
196 181
197 function removeAllRemotesOf (fromUrl, callback) { 182 function removeAllRemotesOf (fromUrl, callback) {
198 VideosDB.remove({ podUrl: fromUrl }, function (err) { 183 // TODO { podUrl: { $in: urls } }
199 if (err) return callback(err) 184 VideosDB.remove({ podUrl: fromUrl }, callback)
200
201 callback(null)
202 })
203 } 185 }
204 186
205 // Use the magnet Uri because the _id field is not the same on different servers 187 // Use the magnet Uri because the _id field is not the same on different servers
206 function removeRemotes (fromUrl, magnetUris, callback) { 188 function removeRemotesOfByMagnetUris (fromUrl, magnetUris, callback) {
207 if (callback === undefined) callback = function () {} 189 if (callback === undefined) callback = function () {}
208 190
209 VideosDB.find({ magnetUri: { $in: magnetUris } }, function (err, videos) { 191 VideosDB.find({ magnetUri: { $in: magnetUris } }, function (err, videos) {
@@ -248,39 +230,7 @@
248 }) 230 })
249 } 231 }
250 232
251 function seedAll (callback) {
252 VideosDB.find({ namePath: { $ne: null } }, function (err, videos_list) {
253 if (err) {
254 logger.error('Cannot get list of the videos to seed.', { error: err })
255 return callback(err)
256 }
257
258 async.each(videos_list, function (video, each_callback) {
259 seedVideo(uploadDir + video.namePath, function (err) {
260 if (err) {
261 logger.error('Cannot seed this video.', { error: err })
262 return callback(err)
263 }
264
265 each_callback(null)
266 })
267 }, callback)
268 })
269 }
270
271 // --------------------------------------------------------------------------- 233 // ---------------------------------------------------------------------------
272 234
273 module.exports = videos 235 module.exports = Videos
274
275 // ---------------------------------------------------------------------------
276
277 function seedVideo (path, callback) {
278 logger.info('Seeding %s...', path)
279
280 webtorrent.seed(path, function (torrent) {
281 logger.info('%s seeded (%s).', path, torrent.magnetURI)
282
283 return callback(null, torrent)
284 })
285 }
286})() 236})()