Server: throttle "seedAll" when starting the server

[github/Chocobozzz/PeerTube.git] / server / models / request.js

diff --git a/server/models/request.js b/server/models/request.js
index 2a407388a7f4358d139e953764d192f2983284bb..4d521919a11185fd4cb3c522bd1f99be1e6061f3 100644 (file)
--- a/server/models/request.js
+++ b/server/models/request.js
@@ -1,14 +1,16 @@
 'use strict'
 
-const async = require('async')
+const each = require('async/each')
+const eachLimit = require('async/eachLimit')
 const map = require('lodash/map')
 const mongoose = require('mongoose')
+const waterfall = require('async/waterfall')
 
 const constants = require('../initializers/constants')
 const logger = require('../helpers/logger')
-const Pods = require('../models/pods')
 const requests = require('../helpers/requests')
 
+const Pod = mongoose.model('Pod')
 const Video = mongoose.model('Video')
 
 let timer = null
@@ -31,7 +33,7 @@ RequestSchema.pre('save', function (next) {
   const self = this
 
   if (self.to.length === 0) {
-    Pods.listAllIds(function (err, podIds) {
+    Pod.listAllIds(function (err, podIds) {
       if (err) return next(err)
 
       // No friends
@@ -102,7 +104,7 @@ function makeRequest (toPod, requestsToMake, callback) {
 function makeRequests () {
   const self = this
 
-  list.call(self, function (err, requests) {
+  listWithLimit.call(self, constants.REQUESTS_LIMIT, function (err, requests) {
     if (err) {
       logger.error('Cannot get the list of requests.', { err: err })
       return // Abort
@@ -136,11 +138,11 @@ function makeRequests () {
     const goodPods = []
     const badPods = []
 
-    async.eachLimit(Object.keys(requestsToMake), constants.REQUESTS_IN_PARALLEL, function (toPodId, callbackEach) {
+    eachLimit(Object.keys(requestsToMake), constants.REQUESTS_IN_PARALLEL, function (toPodId, callbackEach) {
       const requestToMake = requestsToMake[toPodId]
 
       // FIXME: mongodb request inside a loop :/
-      Pods.findById(toPodId, function (err, toPod) {
+      Pod.load(toPodId, function (err, toPod) {
         if (err) {
           logger.error('Error finding pod by id.', { err: err })
           return callbackEach()
@@ -183,9 +185,9 @@ function makeRequests () {
 
 // Remove pods with a score of 0 (too many requests where they were unreachable)
 function removeBadPods () {
-  async.waterfall([
+  waterfall([
     function findBadPods (callback) {
-      Pods.findBadPods(function (err, pods) {
+      Pod.listBadPods(function (err, pods) {
         if (err) {
           logger.error('Cannot find bad pods.', { error: err })
           return callback(err)
@@ -199,23 +201,25 @@ function removeBadPods () {
       if (pods.length === 0) return callback(null)
 
       const urls = map(pods, 'url')
-      const ids = map(pods, '_id')
 
       Video.listByUrls(urls, function (err, videosList) {
         if (err) {
           logger.error('Cannot list videos urls.', { error: err, urls: urls })
-          return callback(null, ids, [])
+          return callback(null, pods, [])
         }
 
-        return callback(null, ids, videosList)
+        return callback(null, pods, videosList)
       })
     },
 
-    function removeVideosOfTheseBadPods (podIds, videosList, callback) {
+    function removeVideosOfTheseBadPods (pods, videosList, callback) {
       // We don't have to remove pods, skip
-      if (typeof podIds === 'function') return podIds(null)
+      if (typeof pods === 'function') {
+        callback = pods
+        return callback(null)
+      }
 
-      async.each(videosList, function (video, callbackEach) {
+      each(videosList, function (video, callbackEach) {
         video.remove(callbackEach)
       }, function (err) {
         if (err) {
@@ -224,22 +228,30 @@ function removeBadPods () {
           return
         }
 
-        return callback(null, podIds)
+        return callback(null, pods)
       })
     },
 
-    function removeBadPodsFromDB (podIds, callback) {
+    function removeBadPodsFromDB (pods, callback) {
       // We don't have to remove pods, skip
-      if (typeof podIds === 'function') return podIds(null)
+      if (typeof pods === 'function') {
+        callback = pods
+        return callback(null)
+      }
+
+      each(pods, function (pod, callbackEach) {
+        pod.remove(callbackEach)
+      }, function (err) {
+        if (err) return callback(err)
 
-      Pods.removeAllByIds(podIds, callback)
+        return callback(null, pods.length)
+      })
     }
-  ], function (err, removeResult) {
+  ], function (err, numberOfPodsRemoved) {
     if (err) {
       logger.error('Cannot remove bad pods.', { error: err })
-    } else if (removeResult) {
-      const podsRemoved = removeResult.result.n
-      logger.info('Removed %d pods.', podsRemoved)
+    } else if (numberOfPodsRemoved) {
+      logger.info('Removed %d pods.', numberOfPodsRemoved)
     } else {
       logger.info('No need to remove bad pods.')
     }
@@ -249,18 +261,18 @@ function removeBadPods () {
 function updatePodsScore (goodPods, badPods) {
   logger.info('Updating %d good pods and %d bad pods scores.', goodPods.length, badPods.length)
 
-  Pods.incrementScores(goodPods, constants.PODS_SCORE.BONUS, function (err) {
+  Pod.incrementScores(goodPods, constants.PODS_SCORE.BONUS, function (err) {
     if (err) logger.error('Cannot increment scores of good pods.')
   })
 
-  Pods.incrementScores(badPods, constants.PODS_SCORE.MALUS, function (err) {
+  Pod.incrementScores(badPods, constants.PODS_SCORE.MALUS, function (err) {
     if (err) logger.error('Cannot decrement scores of bad pods.')
     removeBadPods()
   })
 }
 
-function list (callback) {
-  this.find({ }, { _id: 1, request: 1, to: 1 }, callback)
+function listWithLimit (limit, callback) {
+  this.find({ }, { _id: 1, request: 1, to: 1 }).sort({ _id: 1 }).limit(limit).exec(callback)
 }
 
 function removeAll (callback) {