From c173e56520b0fe4206b9ea8049b6add40bfeabcd Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Thu, 4 Feb 2016 21:10:33 +0100 Subject: Split models --- models/pods.js | 276 ++++++++----------------------------------------- models/poolRequests.js | 67 ++++++++++++ models/videos.js | 202 ++++++++++++++---------------------- 3 files changed, 185 insertions(+), 360 deletions(-) create mode 100644 models/poolRequests.js (limited to 'models') diff --git a/models/pods.js b/models/pods.js index ed2f0d8ee..395b1e0b1 100644 --- a/models/pods.js +++ b/models/pods.js @@ -1,73 +1,61 @@ ;(function () { 'use strict' - var async = require('async') - var config = require('config') - var fs = require('fs') - var request = require('request') + var mongoose = require('mongoose') var constants = require('../initializers/constants') var logger = require('../helpers/logger') - var PodsDB = require('../initializers/database').PodsDB - var poolRequests = require('../lib/poolRequests') - var utils = require('../helpers/utils') - var http = config.get('webserver.https') ? 'https' : 'http' - var host = config.get('webserver.host') - var port = config.get('webserver.port') + // --------------------------------------------------------------------------- + + var podsSchema = mongoose.Schema({ + url: String, + publicKey: String, + score: { type: Number, max: constants.FRIEND_BASE_SCORE } + }) + var PodsDB = mongoose.model('pods', podsSchema) - var pods = { + // --------------------------------------------------------------------------- + + var Pods = { add: add, - addVideoToFriends: addVideoToFriends, + count: count, + findByUrl: findByUrl, + findBadPods: findBadPods, + incrementScores: incrementScores, list: list, - hasFriends: hasFriends, - makeFriends: makeFriends, - quitFriends: quitFriends, remove: remove, - removeVideoToFriends + removeAll: removeAll, + removeAllByIds: removeAllByIds } // TODO: check if the pod is not already a friend function add (data, callback) { - var videos = require('./videos') - logger.info('Adding pod: %s', data.url) - + if (!callback) callback = function () {} var params = { url: data.url, publicKey: data.publicKey, score: constants.FRIEND_BASE_SCORE } - PodsDB.create(params, function (err, pod) { - if (err) { - logger.error('Cannot insert the pod.', { error: err }) - return callback(err) - } - - videos.addRemotes(data.videos) + PodsDB.create(params, callback) + } - fs.readFile(utils.getCertDir() + 'peertube.pub', 'utf8', function (err, cert) { - if (err) { - logger.error('Cannot read cert file.', { error: err }) - return callback(err) - } + function count (callback) { + return PodsDB.count(callback) + } - videos.listOwned(function (err, videos_list) { - if (err) { - logger.error('Cannot get the list of owned videos.', { error: err }) - return callback(err) - } + function findBadPods (callback) { + PodsDB.find({ score: 0 }, callback) + } - return callback(null, { cert: cert, videos: videos_list }) - }) - }) - }) + function findByUrl (url, callback) { + PodsDB.findOne({ url: url }, callback) } - function addVideoToFriends (video) { - // To avoid duplicates - var id = video.name + video.magnetUri - poolRequests.addToPoolRequests(id, 'add', video) + function incrementScores (ids, value, callback) { + if (!callback) callback = function () {} + PodsDB.update({ _id: { $in: ids } }, { $inc: { score: value } }, { multi: true }, callback) } function list (callback) { @@ -81,202 +69,22 @@ }) } - function hasFriends (callback) { - PodsDB.count(function (err, count) { - if (err) return callback(err) - - var has_friends = (count !== 0) - callback(null, has_friends) - }) - } - - function makeFriends (callback) { - var videos = require('./videos') - var pods_score = {} - - logger.info('Make friends!') - fs.readFile(utils.getCertDir() + 'peertube.pub', 'utf8', function (err, cert) { - if (err) { - logger.error('Cannot read public cert.', { error: err }) - return callback(err) - } - - var urls = config.get('network.friends') - - async.each(urls, computeForeignPodsList, function () { - logger.debug('Pods scores computed.', { pods_score: pods_score }) - var pods_list = computeWinningPods(urls, pods_score) - logger.debug('Pods that we keep computed.', { pods_to_keep: pods_list }) - - makeRequestsToWinningPods(cert, pods_list) - }) - }) - - // ----------------------------------------------------------------------- - - function computeForeignPodsList (url, callback) { - // Let's give 1 point to the pod we ask the friends list - pods_score[url] = 1 - - getForeignPodsList(url, function (foreign_pods_list) { - if (foreign_pods_list.length === 0) return callback() - - async.each(foreign_pods_list, function (foreign_pod, callback_each) { - var foreign_url = foreign_pod.url - - if (pods_score[foreign_url]) pods_score[foreign_url]++ - else pods_score[foreign_url] = 1 - - callback_each() - }, function () { - callback() - }) - }) - } - - function computeWinningPods (urls, pods_score) { - // Build the list of pods to add - // Only add a pod if it exists in more than a half base pods - var pods_list = [] - var base_score = urls.length / 2 - Object.keys(pods_score).forEach(function (pod) { - if (pods_score[pod] > base_score) pods_list.push({ url: pod }) - }) - - return pods_list - } - - function makeRequestsToWinningPods (cert, pods_list) { - // Stop pool requests - poolRequests.deactivate() - // Flush pool requests - poolRequests.forceSend() - - // Get the list of our videos to send to our new friends - videos.listOwned(function (err, videos_list) { - if (err) throw err - - var data = { - url: http + '://' + host + ':' + port, - publicKey: cert, - videos: videos_list - } - - utils.makeMultipleRetryRequest( - { method: 'POST', path: '/api/' + constants.API_VERSION + '/pods/', data: data }, - - pods_list, - - function eachRequest (err, response, body, url, pod, callback_each_request) { - // We add the pod if it responded correctly with its public certificate - if (!err && response.statusCode === 200) { - add({ url: pod.url, publicKey: body.cert, score: constants.FRIEND_BASE_SCORE }, function (err) { - if (err) logger.error('Error with adding %s pod.', pod.url, { error: err }) - - videos.addRemotes(body.videos, function (err) { - if (err) logger.error('Error with adding videos of pod.', pod.url, { error: err }) - - logger.debug('Adding remote videos from %s.', pod.url, { videos: body.videos }) - return callback_each_request() - }) - }) - } else { - logger.error('Error with adding %s pod.', pod.url, { error: err || new Error('Status not 200') }) - return callback_each_request() - } - }, - - function endRequests (err) { - // Now we made new friends, we can re activate the pool of requests - poolRequests.activate() - - if (err) { - logger.error('There was some errors when we wanted to make friends.', { error: err }) - return callback(err) - } - - logger.debug('makeRequestsToWinningPods finished.') - return callback(null) - } - ) - }) - } - } - - function quitFriends (callback) { - // Stop pool requests - poolRequests.deactivate() - // Flush pool requests - poolRequests.forceSend() - - PodsDB.find(function (err, pods) { - if (err) return callback(err) - - var request = { - method: 'POST', - path: '/api/' + constants.API_VERSION + '/pods/remove', - sign: true, - encrypt: true, - data: { - url: 'me' // Fake data - } - } - - // Announce we quit them - utils.makeMultipleRetryRequest(request, pods, function () { - PodsDB.remove(function (err) { - poolRequests.activate() - - if (err) return callback(err) - - logger.info('Broke friends, so sad :(') - - var videos = require('./videos') - videos.removeAllRemotes(function (err) { - if (err) return callback(err) - - logger.info('Removed all remote videos.') - callback(null) - }) - }) - }) - }) - } - function remove (url, callback) { - var videos = require('./videos') - logger.info('Removing %s pod.', url) - - videos.removeAllRemotesOf(url, function (err) { - if (err) logger.error('Cannot remove all remote videos of %s.', url) - - PodsDB.remove({ url: url }, function (err) { - if (err) return callback(err) - - logger.info('%s pod removed.', url) - callback(null) - }) - }) + if (!callback) callback = function () {} + PodsDB.remove({ url: url }, callback) } - function removeVideoToFriends (video) { - // To avoid duplicates - var id = video.name + video.magnetUri - poolRequests.addToPoolRequests(id, 'remove', video) + function removeAll (callback) { + if (!callback) callback = function () {} + PodsDB.remove(callback) } - // --------------------------------------------------------------------------- - - module.exports = pods + function removeAllByIds (ids, callback) { + if (!callback) callback = function () {} + PodsDB.remove({ _id: { $in: ids } }, callback) + } // --------------------------------------------------------------------------- - function getForeignPodsList (url, callback) { - var path = '/api/' + constants.API_VERSION + '/pods' - - request.get(url + path, function (err, response, body) { - if (err) throw err - callback(JSON.parse(body)) - }) - } + module.exports = Pods })() diff --git a/models/poolRequests.js b/models/poolRequests.js new file mode 100644 index 000000000..0f488ef04 --- /dev/null +++ b/models/poolRequests.js @@ -0,0 +1,67 @@ +;(function () { + 'use strict' + + var mongoose = require('mongoose') + + var logger = require('../helpers/logger') + + // --------------------------------------------------------------------------- + + var poolRequestsSchema = mongoose.Schema({ + type: String, + id: String, // Special id to find duplicates (video created we want to remove...) + request: mongoose.Schema.Types.Mixed + }) + var PoolRequestsDB = mongoose.model('poolRequests', poolRequestsSchema) + + // --------------------------------------------------------------------------- + + var PoolRequests = { + addRequest: addRequest, + list: list, + removeRequests: removeRequests + } + + function addRequest (id, type, request) { + logger.debug('Add request to the pool requests.', { id: id, type: type, request: request }) + + PoolRequestsDB.findOne({ id: id }, function (err, entity) { + if (err) logger.error(err) + + if (entity) { + if (entity.type === type) { + logger.error(new Error('Cannot insert two same requests.')) + return + } + + // Remove the request of the other type + PoolRequestsDB.remove({ id: id }, function (err) { + if (err) logger.error(err) + }) + } else { + PoolRequestsDB.create({ id: id, type: type, request: request }, function (err) { + if (err) logger.error(err) + }) + } + }) + } + + function list (callback) { + PoolRequestsDB.find({}, { _id: 1, type: 1, request: 1 }, callback) + } + + function removeRequests (ids) { + PoolRequestsDB.remove({ _id: { $in: ids } }, function (err) { + if (err) { + logger.error('Cannot remove requests from the pool requests database.', { error: err }) + return + } + + logger.info('Pool requests flushed.') + }) + } + + // --------------------------------------------------------------------------- + + module.exports = PoolRequests +})() diff --git a/models/videos.js b/models/videos.js index 5711c5657..10abee6e7 100644 --- a/models/videos.js +++ b/models/videos.js @@ -5,71 +5,62 @@ var config = require('config') var dz = require('dezalgo') var fs = require('fs') - var webtorrent = require('../lib/webTorrentNode') + var mongoose = require('mongoose') var logger = require('../helpers/logger') - var pods = require('./pods') - var VideosDB = require('../initializers/database').VideosDB var http = config.get('webserver.https') === true ? 'https' : 'http' var host = config.get('webserver.host') var port = config.get('webserver.port') + var uploadDir = __dirname + '/../' + config.get('storage.uploads') + + // --------------------------------------------------------------------------- + + var videosSchema = mongoose.Schema({ + name: String, + namePath: String, + description: String, + magnetUri: String, + podUrl: String + }) + var VideosDB = mongoose.model('videos', videosSchema) + + // --------------------------------------------------------------------------- - var videos = { + var Videos = { add: add, addRemotes: addRemotes, get: get, + getVideoState: getVideoState, + isOwned: isOwned, list: list, listOwned: listOwned, - remove: remove, + removeOwned: removeOwned, removeAllRemotes: removeAllRemotes, removeAllRemotesOf: removeAllRemotesOf, - removeRemotes: removeRemotes, - search: search, - seedAll: seedAll, - uploadDir: uploadDir + removeRemotesOfByMagnetUris: removeRemotesOfByMagnetUris, + search: search } - // ----------- Public attributes ---------- - var uploadDir = __dirname + '/../' + config.get('storage.uploads') + function add (video, callback) { + logger.info('Adding %s video to database.', video.name) - function add (data, callback) { - var video_file = data.video - var video_data = data.data + var params = video + params.podUrl = http + '://' + host + ':' + port - logger.info('Adding %s video.', video_file.path) - seedVideo(video_file.path, function (err, torrent) { + VideosDB.create(params, function (err, video) { if (err) { - logger.error('Cannot seed this video.', { error: err }) + logger.error('Cannot insert this video into database.', { error: err }) return callback(err) } - var params = { - name: video_data.name, - namePath: video_file.filename, - description: video_data.description, - magnetUri: torrent.magnetURI, - podUrl: http + '://' + host + ':' + port - } - - VideosDB.create(params, function (err, video) { - if (err) { - logger.error('Cannot insert this video.', { error: err }) - return callback(err) - } - - // Now we'll add the video's meta data to our friends - params.namePath = null - - pods.addVideoToFriends(params) - callback(null) - }) + callback(null) }) } // TODO: avoid doublons function addRemotes (videos, callback) { - if (callback === undefined) callback = function () {} + if (!callback) callback = function () {} var to_add = [] @@ -111,6 +102,38 @@ }) } + function getVideoState (id, callback) { + get(id, function (err, video) { + if (err) return callback(err) + + var exist = (video !== null) + var owned = false + if (exist === true) { + owned = (video.namePath !== null) + } + + return callback(null, { exist: exist, owned: owned }) + }) + } + + function isOwned (id, callback) { + VideosDB.findById(id, function (err, video) { + if (err || !video) { + if (!err) err = new Error('Cannot find this video.') + logger.error('Cannot find this video.', { error: err }) + return callback(err) + } + + if (video.namePath === null) { + var error_string = 'Cannot remove the video of another pod.' + logger.error(error_string) + return callback(null, false, video) + } + + callback(null, true, video) + }) + } + function list (callback) { VideosDB.find(function (err, videos_list) { if (err) { @@ -134,76 +157,35 @@ }) } - function remove (id, callback) { - // Maybe the torrent is not seeded, but we catch the error to don't stop the removing process - function removeTorrent (magnetUri, callback) { - try { - webtorrent.remove(magnetUri, callback) - } catch (err) { - logger.warn('Cannot remove the torrent from WebTorrent', { err: err }) - return callback(null) - } - } - - VideosDB.findById(id, function (err, video) { - if (err || !video) { - if (!err) err = new Error('Cannot find this video.') - logger.error('Cannot find this video.', { error: err }) + function removeOwned (id, callback) { + VideosDB.findByIdAndRemove(id, function (err, video) { + if (err) { + logger.error('Cannot remove the torrent.', { error: err }) return callback(err) } - if (video.namePath === null) { - var error_string = 'Cannot remove the video of another pod.' - logger.error(error_string) - return callback(new Error(error_string)) - } - - logger.info('Removing %s video', video.name) - - removeTorrent(video.magnetUri, function () { - VideosDB.findByIdAndRemove(id, function (err) { - if (err) { - logger.error('Cannot remove the torrent.', { error: err }) - return callback(err) - } - - fs.unlink(uploadDir + video.namePath, function (err) { - if (err) { - logger.error('Cannot remove this video file.', { error: err }) - return callback(err) - } - - var params = { - name: video.name, - magnetUri: video.magnetUri - } + fs.unlink(uploadDir + video.namePath, function (err) { + if (err) { + logger.error('Cannot remove this video file.', { error: err }) + return callback(err) + } - pods.removeVideoToFriends(params) - callback(null) - }) - }) + callback(null) }) }) } function removeAllRemotes (callback) { - VideosDB.remove({ namePath: null }, function (err) { - if (err) return callback(err) - - callback(null) - }) + VideosDB.remove({ namePath: null }, callback) } function removeAllRemotesOf (fromUrl, callback) { - VideosDB.remove({ podUrl: fromUrl }, function (err) { - if (err) return callback(err) - - callback(null) - }) + // TODO { podUrl: { $in: urls } } + VideosDB.remove({ podUrl: fromUrl }, callback) } // Use the magnet Uri because the _id field is not the same on different servers - function removeRemotes (fromUrl, magnetUris, callback) { + function removeRemotesOfByMagnetUris (fromUrl, magnetUris, callback) { if (callback === undefined) callback = function () {} VideosDB.find({ magnetUri: { $in: magnetUris } }, function (err, videos) { @@ -248,39 +230,7 @@ }) } - function seedAll (callback) { - VideosDB.find({ namePath: { $ne: null } }, function (err, videos_list) { - if (err) { - logger.error('Cannot get list of the videos to seed.', { error: err }) - return callback(err) - } - - async.each(videos_list, function (video, each_callback) { - seedVideo(uploadDir + video.namePath, function (err) { - if (err) { - logger.error('Cannot seed this video.', { error: err }) - return callback(err) - } - - each_callback(null) - }) - }, callback) - }) - } - // --------------------------------------------------------------------------- - module.exports = videos - - // --------------------------------------------------------------------------- - - function seedVideo (path, callback) { - logger.info('Seeding %s...', path) - - webtorrent.seed(path, function (torrent) { - logger.info('%s seeded (%s).', path, torrent.magnetURI) - - return callback(null, torrent) - }) - } + module.exports = Videos })() -- cgit v1.2.3