X-Git-Url: https://git.immae.eu/?a=blobdiff_plain;f=server%2Flib%2Ffriends.js;h=424a30801f441f90960b220bbb6b1e656d355536;hb=9e167724f7e933f41d9ea2e1c31772bf4c560a28;hp=006a64404fd05405b9648fce4315d982d858cc48;hpb=b9a3e09ad5a7673f64556d1dba122ed4c4fac980;p=github%2FChocobozzz%2FPeerTube.git diff --git a/server/lib/friends.js b/server/lib/friends.js index 006a64404..424a30801 100644 --- a/server/lib/friends.js +++ b/server/lib/friends.js @@ -1,228 +1,339 @@ 'use strict' -var async = require('async') -var config = require('config') -var fs = require('fs') -var request = require('request') - -var constants = require('../initializers/constants') -var logger = require('../helpers/logger') -var peertubeCrypto = require('../helpers/peertubeCrypto') -var Pods = require('../models/pods') -var poolRequests = require('../lib/poolRequests') -var requests = require('../helpers/requests') -var Videos = require('../models/videos') - -var http = config.get('webserver.https') ? 'https' : 'http' -var host = config.get('webserver.host') -var port = config.get('webserver.port') - -var pods = { - addVideoToFriends: addVideoToFriends, - hasFriends: hasFriends, - makeFriends: makeFriends, - quitFriends: quitFriends, - removeVideoToFriends: removeVideoToFriends -} - -function addVideoToFriends (video) { - // To avoid duplicates - var id = video.name + video.magnetUri - // ensure namePath is null - video.namePath = null - poolRequests.addRequest(id, 'add', video) +const each = require('async/each') +const eachLimit = require('async/eachLimit') +const eachSeries = require('async/eachSeries') +const request = require('request') +const waterfall = require('async/waterfall') + +const constants = require('../initializers/constants') +const db = require('../initializers/database') +const logger = require('../helpers/logger') +const peertubeCrypto = require('../helpers/peertube-crypto') +const requests = require('../helpers/requests') +const RequestScheduler = require('./request-scheduler') +const RequestVideoQaduScheduler = require('./request-video-qadu-scheduler') + +const ENDPOINT_ACTIONS = constants.REQUEST_ENDPOINT_ACTIONS[constants.REQUEST_ENDPOINTS.VIDEOS] + +const requestScheduler = new RequestScheduler() +const requestSchedulerVideoQadu = new RequestVideoQaduScheduler() + +const friends = { + activate, + addVideoToFriends, + updateVideoToFriends, + reportAbuseVideoToFriend, + quickAndDirtyUpdateVideoToFriends, + hasFriends, + makeFriends, + quitFriends, + removeVideoToFriends, + sendOwnedVideosToPod +} + +function activate () { + requestScheduler.activate() + requestSchedulerVideoQadu.activate() +} + +function addVideoToFriends (videoData, transaction, callback) { + const options = { + type: ENDPOINT_ACTIONS.ADD, + endpoint: constants.REQUEST_ENDPOINTS.VIDEOS, + data: videoData, + transaction + } + createRequest(options, callback) +} + +function updateVideoToFriends (videoData, transaction, callback) { + const options = { + type: ENDPOINT_ACTIONS.UPDATE, + endpoint: constants.REQUEST_ENDPOINTS.VIDEOS, + data: videoData, + transaction + } + createRequest(options, callback) +} + +function removeVideoToFriends (videoParams) { + const options = { + type: ENDPOINT_ACTIONS.REMOVE, + endpoint: constants.REQUEST_ENDPOINTS.VIDEOS, + data: videoParams + } + createRequest(options) +} + +function reportAbuseVideoToFriend (reportData, video) { + const options = { + type: ENDPOINT_ACTIONS.REPORT_ABUSE, + endpoint: constants.REQUEST_ENDPOINTS.VIDEOS, + data: reportData, + toIds: [ video.Author.podId ] + } + createRequest(options) +} + +function quickAndDirtyUpdateVideoToFriends (videoId, type, transaction, callback) { + const options = { + videoId, + type, + transaction + } + return createVideoQaduRequest(options, callback) } function hasFriends (callback) { - Pods.count(function (err, count) { + db.Pod.countAll(function (err, count) { if (err) return callback(err) - var has_friends = (count !== 0) - callback(null, has_friends) + const hasFriends = (count !== 0) + callback(null, hasFriends) }) } -function makeFriends (callback) { - var pods_score = {} +function makeFriends (hosts, callback) { + const podsScore = {} logger.info('Make friends!') - fs.readFile(peertubeCrypto.getCertDir() + 'peertube.pub', 'utf8', function (err, cert) { + peertubeCrypto.getMyPublicCert(function (err, cert) { if (err) { logger.error('Cannot read public cert.') return callback(err) } - var urls = config.get('network.friends') - - async.each(urls, function (url, callback) { - computeForeignPodsList(url, pods_score, callback) + eachSeries(hosts, function (host, callbackEach) { + computeForeignPodsList(host, podsScore, callbackEach) }, function (err) { if (err) return callback(err) - logger.debug('Pods scores computed.', { pods_score: pods_score }) - var pods_list = computeWinningPods(urls, pods_score) - logger.debug('Pods that we keep computed.', { pods_to_keep: pods_list }) + logger.debug('Pods scores computed.', { podsScore: podsScore }) + const podsList = computeWinningPods(hosts, podsScore) + logger.debug('Pods that we keep.', { podsToKeep: podsList }) - makeRequestsToWinningPods(cert, pods_list, callback) + makeRequestsToWinningPods(cert, podsList, callback) }) }) } function quitFriends (callback) { // Stop pool requests - poolRequests.deactivate() - // Flush pool requests - poolRequests.forceSend() + requestScheduler.deactivate() + + waterfall([ + function flushRequests (callbackAsync) { + requestScheduler.flush(err => callbackAsync(err)) + }, + + function flushVideoQaduRequests (callbackAsync) { + requestSchedulerVideoQadu.flush(err => callbackAsync(err)) + }, + + function getPodsList (callbackAsync) { + return db.Pod.list(callbackAsync) + }, + + function announceIQuitMyFriends (pods, callbackAsync) { + const requestParams = { + method: 'POST', + path: '/api/' + constants.API_VERSION + '/pods/remove', + sign: true + } - Pods.list(function (err, pods) { - if (err) return callback(err) + // Announce we quit them + // We don't care if the request fails + // The other pod will exclude us automatically after a while + eachLimit(pods, constants.REQUESTS_IN_PARALLEL, function (pod, callbackEach) { + requestParams.toPod = pod + requests.makeSecureRequest(requestParams, callbackEach) + }, function (err) { + if (err) { + logger.error('Some errors while quitting friends.', { err: err }) + // Don't stop the process + } - var request = { - method: 'POST', - path: '/api/' + constants.API_VERSION + '/pods/remove', - sign: true, - encrypt: true, - data: { - url: 'me' // Fake data - } + return callbackAsync(null, pods) + }) + }, + + function removePodsFromDB (pods, callbackAsync) { + each(pods, function (pod, callbackEach) { + pod.destroy().asCallback(callbackEach) + }, callbackAsync) } + ], function (err) { + // Don't forget to re activate the scheduler, even if there was an error + requestScheduler.activate() - // Announce we quit them - requests.makeMultipleRetryRequest(request, pods, function () { - Pods.removeAll(function (err) { - poolRequests.activate() + if (err) return callback(err) - if (err) return callback(err) + logger.info('Removed all remote videos.') + return callback(null) + }) +} - logger.info('Broke friends, so sad :(') +function sendOwnedVideosToPod (podId) { + db.Video.listOwnedAndPopulateAuthorAndTags(function (err, videosList) { + if (err) { + logger.error('Cannot get the list of videos we own.') + return + } - Videos.removeAllRemotes(function (err) { - if (err) return callback(err) + videosList.forEach(function (video) { + video.toAddRemoteJSON(function (err, remoteVideo) { + if (err) { + logger.error('Cannot convert video to remote.', { error: err }) + // Don't break the process + return + } - logger.info('Removed all remote videos.') - callback(null) - }) + const options = { + type: 'add', + endpoint: constants.REQUEST_ENDPOINTS.VIDEOS, + data: remoteVideo, + toIds: [ podId ] + } + createRequest(options) }) }) }) } -function removeVideoToFriends (video) { - // To avoid duplicates - var id = video.name + video.magnetUri - poolRequests.addRequest(id, 'remove', video) -} - // --------------------------------------------------------------------------- -module.exports = pods +module.exports = friends // --------------------------------------------------------------------------- -function computeForeignPodsList (url, pods_score, callback) { - // Let's give 1 point to the pod we ask the friends list - pods_score[url] = 1 - - getForeignPodsList(url, function (err, foreign_pods_list) { +function computeForeignPodsList (host, podsScore, callback) { + getForeignPodsList(host, function (err, res) { if (err) return callback(err) - if (foreign_pods_list.length === 0) return callback() - async.each(foreign_pods_list, function (foreign_pod, callback_each) { - var foreign_url = foreign_pod.url + const foreignPodsList = res.data + + // Let's give 1 point to the pod we ask the friends list + foreignPodsList.push({ host }) - if (pods_score[foreign_url]) pods_score[foreign_url]++ - else pods_score[foreign_url] = 1 + foreignPodsList.forEach(function (foreignPod) { + const foreignPodHost = foreignPod.host - callback_each() - }, function () { - callback() + if (podsScore[foreignPodHost]) podsScore[foreignPodHost]++ + else podsScore[foreignPodHost] = 1 }) + + return callback() }) } -function computeWinningPods (urls, pods_score) { +function computeWinningPods (hosts, podsScore) { // Build the list of pods to add // Only add a pod if it exists in more than a half base pods - var pods_list = [] - var base_score = urls.length / 2 - Object.keys(pods_score).forEach(function (pod) { - if (pods_score[pod] > base_score) pods_list.push({ url: pod }) + const podsList = [] + const baseScore = hosts.length / 2 + + Object.keys(podsScore).forEach(function (podHost) { + // If the pod is not me and with a good score we add it + if (isMe(podHost) === false && podsScore[podHost] > baseScore) { + podsList.push({ host: podHost }) + } }) - return pods_list + return podsList } -function getForeignPodsList (url, callback) { - var path = '/api/' + constants.API_VERSION + '/pods' +function getForeignPodsList (host, callback) { + const path = '/api/' + constants.API_VERSION + '/pods' - request.get(url + path, function (err, response, body) { + request.get(constants.REMOTE_SCHEME.HTTP + '://' + host + path, function (err, response, body) { if (err) return callback(err) - callback(null, JSON.parse(body)) + try { + const json = JSON.parse(body) + return callback(null, json) + } catch (err) { + return callback(err) + } }) } -function makeRequestsToWinningPods (cert, pods_list, callback) { +function makeRequestsToWinningPods (cert, podsList, callback) { // Stop pool requests - poolRequests.deactivate() + requestScheduler.deactivate() // Flush pool requests - poolRequests.forceSend() + requestScheduler.forceSend() - // Get the list of our videos to send to our new friends - Videos.listOwned(function (err, videos_list) { - if (err) { - logger.error('Cannot get the list of videos we own.') - return callback(err) + eachLimit(podsList, constants.REQUESTS_IN_PARALLEL, function (pod, callbackEach) { + const params = { + url: constants.REMOTE_SCHEME.HTTP + '://' + pod.host + '/api/' + constants.API_VERSION + '/pods/', + method: 'POST', + json: { + host: constants.CONFIG.WEBSERVER.HOST, + email: constants.CONFIG.ADMIN.EMAIL, + publicKey: cert + } } - var data = { - url: http + '://' + host + ':' + port, - publicKey: cert, - videos: videos_list - } + requests.makeRetryRequest(params, function (err, res, body) { + if (err) { + logger.error('Error with adding %s pod.', pod.host, { error: err }) + // Don't break the process + return callbackEach() + } - requests.makeMultipleRetryRequest( - { method: 'POST', path: '/api/' + constants.API_VERSION + '/pods/', data: data }, - - pods_list, - - function eachRequest (err, response, body, url, pod, callback_each_request) { - // We add the pod if it responded correctly with its public certificate - if (!err && response.statusCode === 200) { - Pods.add({ url: pod.url, publicKey: body.cert, score: constants.FRIEND_BASE_SCORE }, function (err) { - if (err) { - logger.error('Error with adding %s pod.', pod.url, { error: err }) - return callback_each_request() - } - - Videos.addRemotes(body.videos, function (err) { - if (err) { - logger.error('Error with adding videos of pod.', pod.url, { error: err }) - return callback_each_request() - } - - logger.debug('Adding remote videos from %s.', pod.url, { videos: body.videos }) - return callback_each_request() - }) - }) - } else { - logger.error('Error with adding %s pod.', pod.url, { error: err || new Error('Status not 200') }) - return callback_each_request() - } - }, + if (res.statusCode === 200) { + const podObj = db.Pod.build({ host: pod.host, publicKey: body.cert, email: body.email }) + podObj.save().asCallback(function (err, podCreated) { + if (err) { + logger.error('Cannot add friend %s pod.', pod.host, { error: err }) + return callbackEach() + } - function endRequests (err) { - // Now we made new friends, we can re activate the pool of requests - poolRequests.activate() + // Add our videos to the request scheduler + sendOwnedVideosToPod(podCreated.id) - if (err) { - logger.error('There was some errors when we wanted to make friends.') - return callback(err) - } - - logger.debug('makeRequestsToWinningPods finished.') - return callback(null) + return callbackEach() + }) + } else { + logger.error('Status not 200 for %s pod.', pod.host) + return callbackEach() } - ) + }) + }, function endRequests () { + // Final callback, we've ended all the requests + // Now we made new friends, we can re activate the pool of requests + requestScheduler.activate() + + logger.debug('makeRequestsToWinningPods finished.') + return callback() + }) +} + +// Wrapper that populate "toIds" argument with all our friends if it is not specified +// { type, endpoint, data, toIds, transaction } +function createRequest (options, callback) { + if (!callback) callback = function () {} + if (options.toIds) return requestScheduler.createRequest(options, callback) + + // If the "toIds" pods is not specified, we send the request to all our friends + db.Pod.listAllIds(options.transaction, function (err, podIds) { + if (err) { + logger.error('Cannot get pod ids', { error: err }) + return + } + + const newOptions = Object.assign(options, { toIds: podIds }) + return requestScheduler.createRequest(newOptions, callback) }) } + +function createVideoQaduRequest (options, callback) { + if (!callback) callback = function () {} + + requestSchedulerVideoQadu.createRequest(options, callback) +} + +function isMe (host) { + return host === constants.CONFIG.WEBSERVER.HOST +}