server/lib/friends.js

   1 'use strict'
   2
   3 const each = require('async/each')
   4 const eachLimit = require('async/eachLimit')
   5 const eachSeries = require('async/eachSeries')
   6 const fs = require('fs')
   7 const request = require('request')
   8 const waterfall = require('async/waterfall')
   9
  10 const constants = require('../initializers/constants')
  11 const db = require('../initializers/database')
  12 const logger = require('../helpers/logger')
  13 const requests = require('../helpers/requests')
  14
  15 const friends = {
  16   addVideoToFriends,
  17   updateVideoToFriends,
  18   reportAbuseVideoToFriend,
  19   hasFriends,
  20   getMyCertificate,
  21   makeFriends,
  22   quitFriends,
  23   removeVideoToFriends,
  24   sendOwnedVideosToPod
  25 }
  26
  27 function addVideoToFriends (videoData) {
  28   createRequest('add', constants.REQUEST_ENDPOINTS.VIDEOS, videoData)
  29 }
  30
  31 function updateVideoToFriends (videoData) {
  32   createRequest('update', constants.REQUEST_ENDPOINTS.VIDEOS, videoData)
  33 }
  34
  35 function removeVideoToFriends (videoParams) {
  36   createRequest('remove', constants.REQUEST_ENDPOINTS.VIDEOS, videoParams)
  37 }
  38
  39 function reportAbuseVideoToFriend (reportData, video) {
  40   createRequest('report-abuse', constants.REQUEST_ENDPOINTS.VIDEOS, reportData, [ video.Author.podId ])
  41 }
  42
  43 function hasFriends (callback) {
  44   db.Pod.countAll(function (err, count) {
  45     if (err) return callback(err)
  46
  47     const hasFriends = (count !== 0)
  48     callback(null, hasFriends)
  49   })
  50 }
  51
  52 function getMyCertificate (callback) {
  53   fs.readFile(constants.CONFIG.STORAGE.CERT_DIR + 'peertube.pub', 'utf8', callback)
  54 }
  55
  56 function makeFriends (hosts, callback) {
  57   const podsScore = {}
  58
  59   logger.info('Make friends!')
  60   getMyCertificate(function (err, cert) {
  61     if (err) {
  62       logger.error('Cannot read public cert.')
  63       return callback(err)
  64     }
  65
  66     eachSeries(hosts, function (host, callbackEach) {
  67       computeForeignPodsList(host, podsScore, callbackEach)
  68     }, function (err) {
  69       if (err) return callback(err)
  70
  71       logger.debug('Pods scores computed.', { podsScore: podsScore })
  72       const podsList = computeWinningPods(hosts, podsScore)
  73       logger.debug('Pods that we keep.', { podsToKeep: podsList })
  74
  75       makeRequestsToWinningPods(cert, podsList, callback)
  76     })
  77   })
  78 }
  79
  80 function quitFriends (callback) {
  81   // Stop pool requests
  82   db.Request.deactivate()
  83
  84   waterfall([
  85     function flushRequests (callbackAsync) {
  86       db.Request.flush(callbackAsync)
  87     },
  88
  89     function getPodsList (callbackAsync) {
  90       return db.Pod.list(callbackAsync)
  91     },
  92
  93     function announceIQuitMyFriends (pods, callbackAsync) {
  94       const requestParams = {
  95         method: 'POST',
  96         path: '/api/' + constants.API_VERSION + '/pods/remove',
  97         sign: true
  98       }
  99
 100       // Announce we quit them
 101       // We don't care if the request fails
 102       // The other pod will exclude us automatically after a while
 103       eachLimit(pods, constants.REQUESTS_IN_PARALLEL, function (pod, callbackEach) {
 104         requestParams.toPod = pod
 105         requests.makeSecureRequest(requestParams, callbackEach)
 106       }, function (err) {
 107         if (err) {
 108           logger.error('Some errors while quitting friends.', { err: err })
 109           // Don't stop the process
 110         }
 111
 112         return callbackAsync(null, pods)
 113       })
 114     },
 115
 116     function removePodsFromDB (pods, callbackAsync) {
 117       each(pods, function (pod, callbackEach) {
 118         pod.destroy().asCallback(callbackEach)
 119       }, callbackAsync)
 120     }
 121   ], function (err) {
 122     // Don't forget to re activate the scheduler, even if there was an error
 123     db.Request.activate()
 124
 125     if (err) return callback(err)
 126
 127     logger.info('Removed all remote videos.')
 128     return callback(null)
 129   })
 130 }
 131
 132 function sendOwnedVideosToPod (podId) {
 133   db.Video.listOwnedAndPopulateAuthorAndTags(function (err, videosList) {
 134     if (err) {
 135       logger.error('Cannot get the list of videos we own.')
 136       return
 137     }
 138
 139     videosList.forEach(function (video) {
 140       video.toAddRemoteJSON(function (err, remoteVideo) {
 141         if (err) {
 142           logger.error('Cannot convert video to remote.', { error: err })
 143           // Don't break the process
 144           return
 145         }
 146
 147         createRequest('add', constants.REQUEST_ENDPOINTS.VIDEOS, remoteVideo, [ podId ])
 148       })
 149     })
 150   })
 151 }
 152
 153 // ---------------------------------------------------------------------------
 154
 155 module.exports = friends
 156
 157 // ---------------------------------------------------------------------------
 158
 159 function computeForeignPodsList (host, podsScore, callback) {
 160   getForeignPodsList(host, function (err, res) {
 161     if (err) return callback(err)
 162
 163     const foreignPodsList = res.data
 164
 165     // Let's give 1 point to the pod we ask the friends list
 166     foreignPodsList.push({ host })
 167
 168     foreignPodsList.forEach(function (foreignPod) {
 169       const foreignPodHost = foreignPod.host
 170
 171       if (podsScore[foreignPodHost]) podsScore[foreignPodHost]++
 172       else podsScore[foreignPodHost] = 1
 173     })
 174
 175     callback()
 176   })
 177 }
 178
 179 function computeWinningPods (hosts, podsScore) {
 180   // Build the list of pods to add
 181   // Only add a pod if it exists in more than a half base pods
 182   const podsList = []
 183   const baseScore = hosts.length / 2
 184   Object.keys(podsScore).forEach(function (podHost) {
 185     // If the pod is not me and with a good score we add it
 186     if (isMe(podHost) === false && podsScore[podHost] > baseScore) {
 187       podsList.push({ host: podHost })
 188     }
 189   })
 190
 191   return podsList
 192 }
 193
 194 function getForeignPodsList (host, callback) {
 195   const path = '/api/' + constants.API_VERSION + '/pods'
 196
 197   request.get(constants.REMOTE_SCHEME.HTTP + '://' + host + path, function (err, response, body) {
 198     if (err) return callback(err)
 199
 200     try {
 201       const json = JSON.parse(body)
 202       return callback(null, json)
 203     } catch (err) {
 204       return callback(err)
 205     }
 206   })
 207 }
 208
 209 function makeRequestsToWinningPods (cert, podsList, callback) {
 210   // Stop pool requests
 211   db.Request.deactivate()
 212   // Flush pool requests
 213   db.Request.forceSend()
 214
 215   eachLimit(podsList, constants.REQUESTS_IN_PARALLEL, function (pod, callbackEach) {
 216     const params = {
 217       url: constants.REMOTE_SCHEME.HTTP + '://' + pod.host + '/api/' + constants.API_VERSION + '/pods/',
 218       method: 'POST',
 219       json: {
 220         host: constants.CONFIG.WEBSERVER.HOST,
 221         publicKey: cert
 222       }
 223     }
 224
 225     requests.makeRetryRequest(params, function (err, res, body) {
 226       if (err) {
 227         logger.error('Error with adding %s pod.', pod.host, { error: err })
 228         // Don't break the process
 229         return callbackEach()
 230       }
 231
 232       if (res.statusCode === 200) {
 233         const podObj = db.Pod.build({ host: pod.host, publicKey: body.cert })
 234         podObj.save().asCallback(function (err, podCreated) {
 235           if (err) {
 236             logger.error('Cannot add friend %s pod.', pod.host, { error: err })
 237             return callbackEach()
 238           }
 239
 240           // Add our videos to the request scheduler
 241           sendOwnedVideosToPod(podCreated.id)
 242
 243           return callbackEach()
 244         })
 245       } else {
 246         logger.error('Status not 200 for %s pod.', pod.host)
 247         return callbackEach()
 248       }
 249     })
 250   }, function endRequests () {
 251     // Final callback, we've ended all the requests
 252     // Now we made new friends, we can re activate the pool of requests
 253     db.Request.activate()
 254
 255     logger.debug('makeRequestsToWinningPods finished.')
 256     return callback()
 257   })
 258 }
 259
 260 // Wrapper that populate "toIds" argument with all our friends if it is not specified
 261 function createRequest (type, endpoint, data, toIds) {
 262   if (toIds) return _createRequest(type, endpoint, data, toIds)
 263
 264   // If the "toIds" pods is not specified, we send the request to all our friends
 265   db.Pod.listAllIds(function (err, podIds) {
 266     if (err) {
 267       logger.error('Cannot get pod ids', { error: err })
 268       return
 269     }
 270
 271     return _createRequest(type, endpoint, data, podIds)
 272   })
 273 }
 274
 275 function _createRequest (type, endpoint, data, toIds) {
 276   const pods = []
 277
 278   // If there are no destination pods abort
 279   if (toIds.length === 0) return
 280
 281   toIds.forEach(function (toPod) {
 282     pods.push(db.Pod.build({ id: toPod }))
 283   })
 284
 285   const createQuery = {
 286     endpoint,
 287     request: {
 288       type: type,
 289       data: data
 290     }
 291   }
 292
 293   // We run in transaction to keep coherency between Request and RequestToPod tables
 294   db.sequelize.transaction(function (t) {
 295     const dbRequestOptions = {
 296       transaction: t
 297     }
 298
 299     return db.Request.create(createQuery, dbRequestOptions).then(function (request) {
 300       return request.setPods(pods, dbRequestOptions)
 301     })
 302   }).asCallback(function (err) {
 303     if (err) logger.error('Error in createRequest transaction.', { error: err })
 304   })
 305 }
 306
 307 function isMe (host) {
 308   return host === constants.CONFIG.WEBSERVER.HOST
 309 }