X-Git-Url: https://git.immae.eu/?a=blobdiff_plain;f=server%2Flib%2Ffriends.js;h=eafffaab01571676072e5eaa19a62c4a163654ba;hb=4b08096b2c58f801e5aad19e38d24ec59dd3e14e;hp=e052574a545ca432b887adf321aebdf4e866becf;hpb=d6ea0175cbfc9995c3b2ec23866f7abafa9e3904;p=github%2FChocobozzz%2FPeerTube.git diff --git a/server/lib/friends.js b/server/lib/friends.js index e052574a5..eafffaab0 100644 --- a/server/lib/friends.js +++ b/server/lib/friends.js @@ -1,42 +1,38 @@ 'use strict' -const async = require('async') -const config = require('config') +const each = require('async/each') +const eachLimit = require('async/eachLimit') +const eachSeries = require('async/eachSeries') const fs = require('fs') +const mongoose = require('mongoose') const request = require('request') +const urlUtil = require('url') +const waterfall = require('async/waterfall') const constants = require('../initializers/constants') const logger = require('../helpers/logger') -const peertubeCrypto = require('../helpers/peertubeCrypto') -const Pods = require('../models/pods') -const requestsScheduler = require('../lib/requestsScheduler') const requests = require('../helpers/requests') -const videos = require('../lib/videos') -const Videos = require('../models/videos') - -const http = config.get('webserver.https') ? 'https' : 'http' -const host = config.get('webserver.host') -const port = config.get('webserver.port') - -const pods = { - addVideoToFriends: addVideoToFriends, - hasFriends: hasFriends, - getMyCertificate: getMyCertificate, - makeFriends: makeFriends, - quitFriends: quitFriends, - removeVideoToFriends: removeVideoToFriends + +const Pod = mongoose.model('Pod') +const Request = mongoose.model('Request') +const Video = mongoose.model('Video') + +const friends = { + addVideoToFriends, + hasFriends, + getMyCertificate, + makeFriends, + quitFriends, + removeVideoToFriends, + sendOwnedVideosToPod } function addVideoToFriends (video) { - // To avoid duplicates - const id = video.name + video.magnetUri - // ensure namePath is null - video.namePath = null - requestsScheduler.addRequest(id, 'add', video) + createRequest('add', constants.REQUEST_ENDPOINTS.VIDEOS, video) } function hasFriends (callback) { - Pods.count(function (err, count) { + Pod.countAll(function (err, count) { if (err) return callback(err) const hasFriends = (count !== 0) @@ -45,10 +41,10 @@ function hasFriends (callback) { } function getMyCertificate (callback) { - fs.readFile(peertubeCrypto.getCertDir() + 'peertube.pub', 'utf8', callback) + fs.readFile(constants.CONFIG.STORAGE.CERT_DIR + 'peertube.pub', 'utf8', callback) } -function makeFriends (callback) { +function makeFriends (urls, callback) { const podsScore = {} logger.info('Make friends!') @@ -58,9 +54,7 @@ function makeFriends (callback) { return callback(err) } - const urls = config.get('network.friends') - - async.each(urls, function (url, callbackEach) { + eachSeries(urls, function (url, callbackEach) { computeForeignPodsList(url, podsScore, callbackEach) }, function (err) { if (err) return callback(err) @@ -76,75 +70,99 @@ function makeFriends (callback) { function quitFriends (callback) { // Stop pool requests - requestsScheduler.deactivate() + Request.deactivate() // Flush pool requests - requestsScheduler.forceSend() + Request.flush() + + waterfall([ + function getPodsList (callbackAsync) { + return Pod.list(callbackAsync) + }, + + function announceIQuitMyFriends (pods, callbackAsync) { + const requestParams = { + method: 'POST', + path: '/api/' + constants.API_VERSION + '/pods/remove', + sign: true + } - Pods.list(function (err, pods) { - if (err) return callback(err) + // Announce we quit them + // We don't care if the request fails + // The other pod will exclude us automatically after a while + eachLimit(pods, constants.REQUESTS_IN_PARALLEL, function (pod, callbackEach) { + requestParams.toPod = pod + requests.makeSecureRequest(requestParams, callbackEach) + }, function (err) { + if (err) { + logger.error('Some errors while quitting friends.', { err: err }) + // Don't stop the process + } - const request = { - method: 'POST', - path: '/api/' + constants.API_VERSION + '/pods/remove', - sign: true, - encrypt: true, - data: { - url: 'me' // Fake data - } + return callbackAsync(null, pods) + }) + }, + + function removePodsFromDB (pods, callbackAsync) { + each(pods, function (pod, callbackEach) { + pod.remove(callbackEach) + }, callbackAsync) } + ], function (err) { + // Don't forget to re activate the scheduler, even if there was an error + Request.activate() - // Announce we quit them - requests.makeMultipleRetryRequest(request, pods, function () { - Pods.removeAll(function (err) { - requestsScheduler.activate() + if (err) return callback(err) - if (err) return callback(err) + logger.info('Removed all remote videos.') + return callback(null) + }) +} - logger.info('Broke friends, so sad :(') +function removeVideoToFriends (videoParams) { + createRequest('remove', constants.REQUEST_ENDPOINTS.VIDEOS, videoParams) +} - Videos.listFromRemotes(function (err, videosList) { - if (err) return callback(err) +function sendOwnedVideosToPod (podId) { + Video.listOwned(function (err, videosList) { + if (err) { + logger.error('Cannot get the list of videos we own.') + return + } - videos.removeRemoteVideos(videosList, function (err) { - if (err) { - logger.error('Cannot remove remote videos.', { error: err }) - return callback(err) - } + videosList.forEach(function (video) { + video.toRemoteJSON(function (err, remoteVideo) { + if (err) { + logger.error('Cannot convert video to remote.', { error: err }) + // Don't break the process + return + } - logger.info('Removed all remote videos.') - callback(null) - }) - }) + createRequest('add', constants.REQUEST_ENDPOINTS.VIDEOS, remoteVideo, [ podId ]) }) }) }) } -function removeVideoToFriends (video) { - // To avoid duplicates - const id = video.name + video.magnetUri - requestsScheduler.addRequest(id, 'remove', video) -} - // --------------------------------------------------------------------------- -module.exports = pods +module.exports = friends // --------------------------------------------------------------------------- function computeForeignPodsList (url, podsScore, callback) { - // Let's give 1 point to the pod we ask the friends list - podsScore[url] = 1 - getForeignPodsList(url, function (err, foreignPodsList) { if (err) return callback(err) - if (foreignPodsList.length === 0) return callback() + + if (!foreignPodsList) foreignPodsList = [] + + // Let's give 1 point to the pod we ask the friends list + foreignPodsList.push({ url: url }) foreignPodsList.forEach(function (foreignPod) { - const foreignUrl = foreignPod.url + const foreignPodUrl = foreignPod.url - if (podsScore[foreignUrl]) podsScore[foreignUrl]++ - else podsScore[foreignUrl] = 1 + if (podsScore[foreignPodUrl]) podsScore[foreignPodUrl]++ + else podsScore[foreignPodUrl] = 1 }) callback() @@ -156,12 +174,13 @@ function computeWinningPods (urls, podsScore) { // Only add a pod if it exists in more than a half base pods const podsList = [] const baseScore = urls.length / 2 - Object.keys(podsScore).forEach(function (pod) { - if (podsScore[pod] > baseScore) podsList.push({ url: pod }) + Object.keys(podsScore).forEach(function (podUrl) { + // If the pod is not me and with a good score we add it + if (isMe(podUrl) === false && podsScore[podUrl] > baseScore) { + podsList.push({ url: podUrl }) + } }) - console.log(urls) - console.log(podsScore) - console.log(podsList) + return podsList } @@ -171,71 +190,92 @@ function getForeignPodsList (url, callback) { request.get(url + path, function (err, response, body) { if (err) return callback(err) - callback(null, JSON.parse(body)) + try { + const json = JSON.parse(body) + return callback(null, json) + } catch (err) { + return callback(err) + } }) } function makeRequestsToWinningPods (cert, podsList, callback) { // Stop pool requests - requestsScheduler.deactivate() + Request.deactivate() // Flush pool requests - requestsScheduler.forceSend() + Request.forceSend() - // Get the list of our videos to send to our new friends - Videos.listOwned(function (err, videosList) { - if (err) { - logger.error('Cannot get the list of videos we own.') - return callback(err) - } - - const data = { - url: http + '://' + host + ':' + port, - publicKey: cert, - videos: videosList + eachLimit(podsList, constants.REQUESTS_IN_PARALLEL, function (pod, callbackEach) { + const params = { + url: pod.url + '/api/' + constants.API_VERSION + '/pods/', + method: 'POST', + json: { + url: constants.CONFIG.WEBSERVER.URL, + publicKey: cert + } } - requests.makeMultipleRetryRequest( - { method: 'POST', path: '/api/' + constants.API_VERSION + '/pods/', data: data }, - - podsList, - - function eachRequest (err, response, body, url, pod, callbackEachRequest) { - // We add the pod if it responded correctly with its public certificate - if (!err && response.statusCode === 200) { - Pods.add({ url: pod.url, publicKey: body.cert, score: constants.FRIEND_BASE_SCORE }, function (err) { - if (err) { - logger.error('Error with adding %s pod.', pod.url, { error: err }) - return callbackEachRequest() - } - - videos.createRemoteVideos(body.videos, function (err) { - if (err) { - logger.error('Error with adding videos of pod.', pod.url, { error: err }) - return callbackEachRequest() - } - - logger.debug('Adding remote videos from %s.', pod.url, { videos: body.videos }) - return callbackEachRequest() - }) - }) - } else { - logger.error('Error with adding %s pod.', pod.url, { error: err || new Error('Status not 200') }) - return callbackEachRequest() - } - }, + requests.makeRetryRequest(params, function (err, res, body) { + if (err) { + logger.error('Error with adding %s pod.', pod.url, { error: err }) + // Don't break the process + return callbackEach() + } - function endRequests (err) { - // Now we made new friends, we can re activate the pool of requests - requestsScheduler.activate() + if (res.statusCode === 200) { + const podObj = new Pod({ url: pod.url, publicKey: body.cert }) + podObj.save(function (err, podCreated) { + if (err) { + logger.error('Cannot add friend %s pod.', pod.url, { error: err }) + return callbackEach() + } - if (err) { - logger.error('There was some errors when we wanted to make friends.') - return callback(err) - } + // Add our videos to the request scheduler + sendOwnedVideosToPod(podCreated._id) - logger.debug('makeRequestsToWinningPods finished.') - return callback(null) + return callbackEach() + }) + } else { + logger.error('Status not 200 for %s pod.', pod.url) + return callbackEach() } - ) + }) + }, function endRequests () { + // Final callback, we've ended all the requests + // Now we made new friends, we can re activate the pool of requests + Request.activate() + + logger.debug('makeRequestsToWinningPods finished.') + return callback() }) } + +function createRequest (type, endpoint, data, to) { + const req = new Request({ + endpoint, + request: { + type: type, + data: data + } + }) + + if (to) { + req.to = to + } + + req.save(function (err) { + if (err) logger.error('Cannot save the request.', { error: err }) + }) +} + +function isMe (url) { + const parsedUrl = urlUtil.parse(url) + + const hostname = parsedUrl.hostname + const port = parseInt(parsedUrl.port) + + const myHostname = constants.CONFIG.WEBSERVER.HOSTNAME + const myPort = constants.CONFIG.WEBSERVER.PORT + + return hostname === myHostname && port === myPort +}