'use strict'
-var async = require('async')
-var config = require('config')
-var fs = require('fs')
-var request = require('request')
-
-var constants = require('../initializers/constants')
-var logger = require('../helpers/logger')
-var peertubeCrypto = require('../helpers/peertubeCrypto')
-var Pods = require('../models/pods')
-var poolRequests = require('../lib/poolRequests')
-var requests = require('../helpers/requests')
-var Videos = require('../models/videos')
-
-var http = config.get('webserver.https') ? 'https' : 'http'
-var host = config.get('webserver.host')
-var port = config.get('webserver.port')
-
-var pods = {
- addVideoToFriends: addVideoToFriends,
- hasFriends: hasFriends,
- makeFriends: makeFriends,
- quitFriends: quitFriends,
- removeVideoToFriends: removeVideoToFriends
-}
-
-function addVideoToFriends (video) {
- // To avoid duplicates
- var id = video.name + video.magnetUri
- // ensure namePath is null
- video.namePath = null
- poolRequests.addRequest(id, 'add', video)
+const each = require('async/each')
+const eachLimit = require('async/eachLimit')
+const eachSeries = require('async/eachSeries')
+const request = require('request')
+const waterfall = require('async/waterfall')
+
+const constants = require('../initializers/constants')
+const db = require('../initializers/database')
+const logger = require('../helpers/logger')
+const peertubeCrypto = require('../helpers/peertube-crypto')
+const requests = require('../helpers/requests')
+const RequestScheduler = require('./request-scheduler')
+const RequestVideoQaduScheduler = require('./request-video-qadu-scheduler')
+
+const ENDPOINT_ACTIONS = constants.REQUEST_ENDPOINT_ACTIONS[constants.REQUEST_ENDPOINTS.VIDEOS]
+
+const requestScheduler = new RequestScheduler()
+const requestSchedulerVideoQadu = new RequestVideoQaduScheduler()
+
+const friends = {
+ activate,
+ addVideoToFriends,
+ updateVideoToFriends,
+ reportAbuseVideoToFriend,
+ quickAndDirtyUpdateVideoToFriends,
+ hasFriends,
+ makeFriends,
+ quitFriends,
+ removeVideoToFriends,
+ sendOwnedVideosToPod
+}
+
+function activate () {
+ requestScheduler.activate()
+ requestSchedulerVideoQadu.activate()
+}
+
+function addVideoToFriends (videoData, transaction, callback) {
+ const options = {
+ type: ENDPOINT_ACTIONS.ADD,
+ endpoint: constants.REQUEST_ENDPOINTS.VIDEOS,
+ data: videoData,
+ transaction
+ }
+ createRequest(options, callback)
+}
+
+function updateVideoToFriends (videoData, transaction, callback) {
+ const options = {
+ type: ENDPOINT_ACTIONS.UPDATE,
+ endpoint: constants.REQUEST_ENDPOINTS.VIDEOS,
+ data: videoData,
+ transaction
+ }
+ createRequest(options, callback)
+}
+
+function removeVideoToFriends (videoParams) {
+ const options = {
+ type: ENDPOINT_ACTIONS.REMOVE,
+ endpoint: constants.REQUEST_ENDPOINTS.VIDEOS,
+ data: videoParams
+ }
+ createRequest(options)
+}
+
+function reportAbuseVideoToFriend (reportData, video) {
+ const options = {
+ type: ENDPOINT_ACTIONS.REPORT_ABUSE,
+ endpoint: constants.REQUEST_ENDPOINTS.VIDEOS,
+ data: reportData,
+ toIds: [ video.Author.podId ]
+ }
+ createRequest(options)
+}
+
+function quickAndDirtyUpdateVideoToFriends (videoId, type, transaction, callback) {
+ const options = {
+ videoId,
+ type,
+ transaction
+ }
+ return createVideoQaduRequest(options, callback)
}
function hasFriends (callback) {
- Pods.count(function (err, count) {
+ db.Pod.countAll(function (err, count) {
if (err) return callback(err)
- var has_friends = (count !== 0)
- callback(null, has_friends)
+ const hasFriends = (count !== 0)
+ callback(null, hasFriends)
})
}
-function makeFriends (callback) {
- var pods_score = {}
+function makeFriends (hosts, callback) {
+ const podsScore = {}
logger.info('Make friends!')
- fs.readFile(peertubeCrypto.getCertDir() + 'peertube.pub', 'utf8', function (err, cert) {
+ peertubeCrypto.getMyPublicCert(function (err, cert) {
if (err) {
logger.error('Cannot read public cert.')
return callback(err)
}
- var urls = config.get('network.friends')
-
- async.each(urls, function (url, callback) {
- computeForeignPodsList(url, pods_score, callback)
+ eachSeries(hosts, function (host, callbackEach) {
+ computeForeignPodsList(host, podsScore, callbackEach)
}, function (err) {
if (err) return callback(err)
- logger.debug('Pods scores computed.', { pods_score: pods_score })
- var pods_list = computeWinningPods(urls, pods_score)
- logger.debug('Pods that we keep computed.', { pods_to_keep: pods_list })
+ logger.debug('Pods scores computed.', { podsScore: podsScore })
+ const podsList = computeWinningPods(hosts, podsScore)
+ logger.debug('Pods that we keep.', { podsToKeep: podsList })
- makeRequestsToWinningPods(cert, pods_list, callback)
+ makeRequestsToWinningPods(cert, podsList, callback)
})
})
}
function quitFriends (callback) {
// Stop pool requests
- poolRequests.deactivate()
- // Flush pool requests
- poolRequests.forceSend()
+ requestScheduler.deactivate()
+
+ waterfall([
+ function flushRequests (callbackAsync) {
+ requestScheduler.flush(err => callbackAsync(err))
+ },
+
+ function flushVideoQaduRequests (callbackAsync) {
+ requestSchedulerVideoQadu.flush(err => callbackAsync(err))
+ },
+
+ function getPodsList (callbackAsync) {
+ return db.Pod.list(callbackAsync)
+ },
+
+ function announceIQuitMyFriends (pods, callbackAsync) {
+ const requestParams = {
+ method: 'POST',
+ path: '/api/' + constants.API_VERSION + '/pods/remove',
+ sign: true
+ }
- Pods.list(function (err, pods) {
- if (err) return callback(err)
+ // Announce we quit them
+ // We don't care if the request fails
+ // The other pod will exclude us automatically after a while
+ eachLimit(pods, constants.REQUESTS_IN_PARALLEL, function (pod, callbackEach) {
+ requestParams.toPod = pod
+ requests.makeSecureRequest(requestParams, callbackEach)
+ }, function (err) {
+ if (err) {
+ logger.error('Some errors while quitting friends.', { err: err })
+ // Don't stop the process
+ }
- var request = {
- method: 'POST',
- path: '/api/' + constants.API_VERSION + '/pods/remove',
- sign: true,
- encrypt: true,
- data: {
- url: 'me' // Fake data
- }
+ return callbackAsync(null, pods)
+ })
+ },
+
+ function removePodsFromDB (pods, callbackAsync) {
+ each(pods, function (pod, callbackEach) {
+ pod.destroy().asCallback(callbackEach)
+ }, callbackAsync)
}
+ ], function (err) {
+ // Don't forget to re activate the scheduler, even if there was an error
+ requestScheduler.activate()
- // Announce we quit them
- requests.makeMultipleRetryRequest(request, pods, function () {
- Pods.removeAll(function (err) {
- poolRequests.activate()
+ if (err) return callback(err)
- if (err) return callback(err)
+ logger.info('Removed all remote videos.')
+ return callback(null)
+ })
+}
- logger.info('Broke friends, so sad :(')
+function sendOwnedVideosToPod (podId) {
+ db.Video.listOwnedAndPopulateAuthorAndTags(function (err, videosList) {
+ if (err) {
+ logger.error('Cannot get the list of videos we own.')
+ return
+ }
- Videos.removeAllRemotes(function (err) {
- if (err) return callback(err)
+ videosList.forEach(function (video) {
+ video.toAddRemoteJSON(function (err, remoteVideo) {
+ if (err) {
+ logger.error('Cannot convert video to remote.', { error: err })
+ // Don't break the process
+ return
+ }
- logger.info('Removed all remote videos.')
- callback(null)
- })
+ const options = {
+ type: 'add',
+ endpoint: constants.REQUEST_ENDPOINTS.VIDEOS,
+ data: remoteVideo,
+ toIds: [ podId ]
+ }
+ createRequest(options)
})
})
})
}
-function removeVideoToFriends (video) {
- // To avoid duplicates
- var id = video.name + video.magnetUri
- poolRequests.addRequest(id, 'remove', video)
-}
-
// ---------------------------------------------------------------------------
-module.exports = pods
+module.exports = friends
// ---------------------------------------------------------------------------
-function computeForeignPodsList (url, pods_score, callback) {
- // Let's give 1 point to the pod we ask the friends list
- pods_score[url] = 1
-
- getForeignPodsList(url, function (err, foreign_pods_list) {
+function computeForeignPodsList (host, podsScore, callback) {
+ getForeignPodsList(host, function (err, res) {
if (err) return callback(err)
- if (foreign_pods_list.length === 0) return callback()
- async.each(foreign_pods_list, function (foreign_pod, callback_each) {
- var foreign_url = foreign_pod.url
+ const foreignPodsList = res.data
+
+ // Let's give 1 point to the pod we ask the friends list
+ foreignPodsList.push({ host })
- if (pods_score[foreign_url]) pods_score[foreign_url]++
- else pods_score[foreign_url] = 1
+ foreignPodsList.forEach(function (foreignPod) {
+ const foreignPodHost = foreignPod.host
- callback_each()
- }, function () {
- callback()
+ if (podsScore[foreignPodHost]) podsScore[foreignPodHost]++
+ else podsScore[foreignPodHost] = 1
})
+
+ return callback()
})
}
-function computeWinningPods (urls, pods_score) {
+function computeWinningPods (hosts, podsScore) {
// Build the list of pods to add
// Only add a pod if it exists in more than a half base pods
- var pods_list = []
- var base_score = urls.length / 2
- Object.keys(pods_score).forEach(function (pod) {
- if (pods_score[pod] > base_score) pods_list.push({ url: pod })
+ const podsList = []
+ const baseScore = hosts.length / 2
+
+ Object.keys(podsScore).forEach(function (podHost) {
+ // If the pod is not me and with a good score we add it
+ if (isMe(podHost) === false && podsScore[podHost] > baseScore) {
+ podsList.push({ host: podHost })
+ }
})
- return pods_list
+ return podsList
}
-function getForeignPodsList (url, callback) {
- var path = '/api/' + constants.API_VERSION + '/pods'
+function getForeignPodsList (host, callback) {
+ const path = '/api/' + constants.API_VERSION + '/pods'
- request.get(url + path, function (err, response, body) {
+ request.get(constants.REMOTE_SCHEME.HTTP + '://' + host + path, function (err, response, body) {
if (err) return callback(err)
- callback(null, JSON.parse(body))
+ try {
+ const json = JSON.parse(body)
+ return callback(null, json)
+ } catch (err) {
+ return callback(err)
+ }
})
}
-function makeRequestsToWinningPods (cert, pods_list, callback) {
+function makeRequestsToWinningPods (cert, podsList, callback) {
// Stop pool requests
- poolRequests.deactivate()
+ requestScheduler.deactivate()
// Flush pool requests
- poolRequests.forceSend()
+ requestScheduler.forceSend()
- // Get the list of our videos to send to our new friends
- Videos.listOwned(function (err, videos_list) {
- if (err) {
- logger.error('Cannot get the list of videos we own.')
- return callback(err)
+ eachLimit(podsList, constants.REQUESTS_IN_PARALLEL, function (pod, callbackEach) {
+ const params = {
+ url: constants.REMOTE_SCHEME.HTTP + '://' + pod.host + '/api/' + constants.API_VERSION + '/pods/',
+ method: 'POST',
+ json: {
+ host: constants.CONFIG.WEBSERVER.HOST,
+ email: constants.CONFIG.ADMIN.EMAIL,
+ publicKey: cert
+ }
}
- var data = {
- url: http + '://' + host + ':' + port,
- publicKey: cert,
- videos: videos_list
- }
+ requests.makeRetryRequest(params, function (err, res, body) {
+ if (err) {
+ logger.error('Error with adding %s pod.', pod.host, { error: err })
+ // Don't break the process
+ return callbackEach()
+ }
- requests.makeMultipleRetryRequest(
- { method: 'POST', path: '/api/' + constants.API_VERSION + '/pods/', data: data },
-
- pods_list,
-
- function eachRequest (err, response, body, url, pod, callback_each_request) {
- // We add the pod if it responded correctly with its public certificate
- if (!err && response.statusCode === 200) {
- Pods.add({ url: pod.url, publicKey: body.cert, score: constants.FRIEND_BASE_SCORE }, function (err) {
- if (err) {
- logger.error('Error with adding %s pod.', pod.url, { error: err })
- return callback_each_request()
- }
-
- Videos.addRemotes(body.videos, function (err) {
- if (err) {
- logger.error('Error with adding videos of pod.', pod.url, { error: err })
- return callback_each_request()
- }
-
- logger.debug('Adding remote videos from %s.', pod.url, { videos: body.videos })
- return callback_each_request()
- })
- })
- } else {
- logger.error('Error with adding %s pod.', pod.url, { error: err || new Error('Status not 200') })
- return callback_each_request()
- }
- },
+ if (res.statusCode === 200) {
+ const podObj = db.Pod.build({ host: pod.host, publicKey: body.cert, email: body.email })
+ podObj.save().asCallback(function (err, podCreated) {
+ if (err) {
+ logger.error('Cannot add friend %s pod.', pod.host, { error: err })
+ return callbackEach()
+ }
- function endRequests (err) {
- // Now we made new friends, we can re activate the pool of requests
- poolRequests.activate()
+ // Add our videos to the request scheduler
+ sendOwnedVideosToPod(podCreated.id)
- if (err) {
- logger.error('There was some errors when we wanted to make friends.')
- return callback(err)
- }
-
- logger.debug('makeRequestsToWinningPods finished.')
- return callback(null)
+ return callbackEach()
+ })
+ } else {
+ logger.error('Status not 200 for %s pod.', pod.host)
+ return callbackEach()
}
- )
+ })
+ }, function endRequests () {
+ // Final callback, we've ended all the requests
+ // Now we made new friends, we can re activate the pool of requests
+ requestScheduler.activate()
+
+ logger.debug('makeRequestsToWinningPods finished.')
+ return callback()
+ })
+}
+
+// Wrapper that populate "toIds" argument with all our friends if it is not specified
+// { type, endpoint, data, toIds, transaction }
+function createRequest (options, callback) {
+ if (!callback) callback = function () {}
+ if (options.toIds) return requestScheduler.createRequest(options, callback)
+
+ // If the "toIds" pods is not specified, we send the request to all our friends
+ db.Pod.listAllIds(options.transaction, function (err, podIds) {
+ if (err) {
+ logger.error('Cannot get pod ids', { error: err })
+ return
+ }
+
+ const newOptions = Object.assign(options, { toIds: podIds })
+ return requestScheduler.createRequest(newOptions, callback)
})
}
+
+function createVideoQaduRequest (options, callback) {
+ if (!callback) callback = function () {}
+
+ requestSchedulerVideoQadu.createRequest(options, callback)
+}
+
+function isMe (host) {
+ return host === constants.CONFIG.WEBSERVER.HOST
+}
projects / github / Chocobozzz / PeerTube.git / blobdiff