Server: make a basic "quick and dirty update" for videos

This system will be useful to to update some int video attributes
(likes, dislikes, views...)

The classic system is not used because we need some optimization for
scaling

5 files changed, 232 insertions, 65 deletions

diff --git a/server/models/pod.js b/server/models/pod.js
index 79afb737a..14814708e 100644
--- a/server/models/pod.js
+++ b/server/models/pod.js
@@ -1,8 +1,11 @@
 'use strict'
 
+const each = require('async/each')
 const map = require('lodash/map')
+const waterfall = require('async/waterfall')
 
 const constants = require('../initializers/constants')
+const logger = require('../helpers/logger')
 const customPodsValidators = require('../helpers/custom-validators').pods
 
 // ---------------------------------------------------------------------------
@@ -62,6 +65,7 @@ module.exports = function (sequelize, DataTypes) {
         listBadPods,
         load,
         loadByHost,
+        updatePodsScore,
         removeAll
       },
       instanceMethods: {
@@ -144,7 +148,7 @@ function listAllIds (transaction, callback) {
   })
 }
 
-function listRandomPodIdsWithRequest (limit, callback) {
+function listRandomPodIdsWithRequest (limit, tableRequestPod, callback) {
   const self = this
 
   self.count().asCallback(function (err, count) {
@@ -166,7 +170,7 @@ function listRandomPodIdsWithRequest (limit, callback) {
       where: {
         id: {
           $in: [
-            this.sequelize.literal('SELECT "podId" FROM "RequestToPods"')
+            this.sequelize.literal('SELECT "podId" FROM "' + tableRequestPod + '"')
           ]
         }
       }
@@ -207,3 +211,58 @@ function loadByHost (host, callback) {
 function removeAll (callback) {
   return this.destroy().asCallback(callback)
 }
+
+function updatePodsScore (goodPods, badPods) {
+  const self = this
+
+  logger.info('Updating %d good pods and %d bad pods scores.', goodPods.length, badPods.length)
+
+  if (goodPods.length !== 0) {
+    this.incrementScores(goodPods, constants.PODS_SCORE.BONUS, function (err) {
+      if (err) logger.error('Cannot increment scores of good pods.', { error: err })
+    })
+  }
+
+  if (badPods.length !== 0) {
+    this.incrementScores(badPods, constants.PODS_SCORE.MALUS, function (err) {
+      if (err) logger.error('Cannot decrement scores of bad pods.', { error: err })
+      removeBadPods.call(self)
+    })
+  }
+}
+
+// ---------------------------------------------------------------------------
+
+// Remove pods with a score of 0 (too many requests where they were unreachable)
+function removeBadPods () {
+  const self = this
+
+  waterfall([
+    function findBadPods (callback) {
+      self.sequelize.models.Pod.listBadPods(function (err, pods) {
+        if (err) {
+          logger.error('Cannot find bad pods.', { error: err })
+          return callback(err)
+        }
+
+        return callback(null, pods)
+      })
+    },
+
+    function removeTheseBadPods (pods, callback) {
+      each(pods, function (pod, callbackEach) {
+        pod.destroy().asCallback(callbackEach)
+      }, function (err) {
+        return callback(err, pods.length)
+      })
+    }
+  ], function (err, numberOfPodsRemoved) {
+    if (err) {
+      logger.error('Cannot remove bad pods.', { error: err })
+    } else if (numberOfPodsRemoved) {
+      logger.info('Removed %d pods.', numberOfPodsRemoved)
+    } else {
+      logger.info('No need to remove bad pods.')
+    }
+  })
+}
diff --git a/server/models/request-to-pod.js b/server/models/request-to-pod.js
index f42a53458..0e01a842e 100644
--- a/server/models/request-to-pod.js
+++ b/server/models/request-to-pod.js
@@ -17,7 +17,7 @@ module.exports = function (sequelize, DataTypes) {
       }
     ],
     classMethods: {
-      removePodOf
+      removeByRequestIdsAndPod
     }
   })
 
@@ -26,7 +26,7 @@ module.exports = function (sequelize, DataTypes) {
 
 // ---------------------------------------------------------------------------
 
-function removePodOf (requestsIds, podId, callback) {
+function removeByRequestIdsAndPod (requestsIds, podId, callback) {
   if (!callback) callback = function () {}
 
   const query = {
diff --git a/server/models/request-video-qadu.js b/server/models/request-video-qadu.js
new file mode 100644
index 000000000..7010fc992
--- /dev/null
+++ b/server/models/request-video-qadu.js
@@ -0,0 +1,154 @@
+'use strict'
+
+/*
+  Request Video for Quick And Dirty Updates like:
+   - views
+   - likes
+   - dislikes
+
+  We can't put it in the same system than basic requests for efficiency.
+  Moreover we don't want to slow down the basic requests with a lot of views/likes/dislikes requests.
+  So we put it an independant request scheduler.
+*/
+
+const values = require('lodash/values')
+
+const constants = require('../initializers/constants')
+
+// ---------------------------------------------------------------------------
+
+module.exports = function (sequelize, DataTypes) {
+  const RequestVideoQadu = sequelize.define('RequestVideoQadu',
+    {
+      type: {
+        type: DataTypes.ENUM(values(constants.REQUEST_VIDEO_QADU_TYPES)),
+        allowNull: false
+      }
+    },
+    {
+      timestamps: false,
+      indexes: [
+        {
+          fields: [ 'podId' ]
+        },
+        {
+          fields: [ 'videoId' ]
+        }
+      ],
+      classMethods: {
+        associate,
+
+        listWithLimitAndRandom,
+
+        countTotalRequests,
+        removeAll,
+        removeByRequestIdsAndPod
+      }
+    }
+  )
+
+  return RequestVideoQadu
+}
+
+// ------------------------------ STATICS ------------------------------
+
+function associate (models) {
+  this.belongsTo(models.Pod, {
+    foreignKey: {
+      name: 'podId',
+      allowNull: false
+    },
+    onDelete: 'CASCADE'
+  })
+
+  this.belongsTo(models.Video, {
+    foreignKey: {
+      name: 'videoId',
+      allowNull: false
+    },
+    onDelete: 'CASCADE'
+  })
+}
+
+function countTotalRequests (callback) {
+  const query = {
+    include: [ this.sequelize.models.Pod ]
+  }
+
+  return this.count(query).asCallback(callback)
+}
+
+function listWithLimitAndRandom (limitPods, limitRequestsPerPod, callback) {
+  const self = this
+  const Pod = this.sequelize.models.Pod
+
+  Pod.listRandomPodIdsWithRequest(limitPods, 'RequestVideoQadus', function (err, podIds) {
+    if (err) return callback(err)
+
+    // We don't have friends that have requests
+    if (podIds.length === 0) return callback(null, [])
+
+    const query = {
+      include: [
+        {
+          model: self.sequelize.models.Pod,
+          where: {
+            id: {
+              $in: podIds
+            }
+          }
+        },
+        {
+          model: self.sequelize.models.Video
+        }
+      ]
+    }
+
+    self.findAll(query).asCallback(function (err, requests) {
+      if (err) return callback(err)
+
+      const requestsGrouped = groupAndTruncateRequests(requests, limitRequestsPerPod)
+      return callback(err, requestsGrouped)
+    })
+  })
+}
+
+function removeByRequestIdsAndPod (ids, podId, callback) {
+  const query = {
+    where: {
+      id: {
+        $in: ids
+      },
+      podId
+    }
+  }
+
+  this.destroy(query).asCallback(callback)
+}
+
+function removeAll (callback) {
+  // Delete all requests
+  this.truncate({ cascade: true }).asCallback(callback)
+}
+
+// ---------------------------------------------------------------------------
+
+function groupAndTruncateRequests (requests, limitRequestsPerPod) {
+  const requestsGrouped = {}
+
+  requests.forEach(function (request) {
+    const pod = request.Pod
+
+    if (!requestsGrouped[pod.id]) requestsGrouped[pod.id] = []
+
+    if (requestsGrouped[pod.id].length < limitRequestsPerPod) {
+      requestsGrouped[pod.id].push({
+        request: request,
+        video: request.Video,
+        pod
+      })
+    }
+  })
+
+  return requestsGrouped
+}
diff --git a/server/models/request.js b/server/models/request.js
index ca616d130..de73501fc 100644
--- a/server/models/request.js
+++ b/server/models/request.js
@@ -1,11 +1,8 @@
 'use strict'
 
-const each = require('async/each')
-const waterfall = require('async/waterfall')
 const values = require('lodash/values')
 
 const constants = require('../initializers/constants')
-const logger = require('../helpers/logger')
 
 // ---------------------------------------------------------------------------
 
@@ -28,8 +25,6 @@ module.exports = function (sequelize, DataTypes) {
         listWithLimitAndRandom,
 
         countTotalRequests,
-        removeBadPods,
-        updatePodsScore,
         removeAll,
         removeWithEmptyTo
       }
@@ -60,71 +55,17 @@ function countTotalRequests (callback) {
   return this.count(query).asCallback(callback)
 }
 
-// Remove pods with a score of 0 (too many requests where they were unreachable)
-function removeBadPods () {
-  const self = this
-
-  waterfall([
-    function findBadPods (callback) {
-      self.sequelize.models.Pod.listBadPods(function (err, pods) {
-        if (err) {
-          logger.error('Cannot find bad pods.', { error: err })
-          return callback(err)
-        }
-
-        return callback(null, pods)
-      })
-    },
-
-    function removeTheseBadPods (pods, callback) {
-      each(pods, function (pod, callbackEach) {
-        pod.destroy().asCallback(callbackEach)
-      }, function (err) {
-        return callback(err, pods.length)
-      })
-    }
-  ], function (err, numberOfPodsRemoved) {
-    if (err) {
-      logger.error('Cannot remove bad pods.', { error: err })
-    } else if (numberOfPodsRemoved) {
-      logger.info('Removed %d pods.', numberOfPodsRemoved)
-    } else {
-      logger.info('No need to remove bad pods.')
-    }
-  })
-}
-
-function updatePodsScore (goodPods, badPods) {
-  const self = this
-  const Pod = this.sequelize.models.Pod
-
-  logger.info('Updating %d good pods and %d bad pods scores.', goodPods.length, badPods.length)
-
-  if (goodPods.length !== 0) {
-    Pod.incrementScores(goodPods, constants.PODS_SCORE.BONUS, function (err) {
-      if (err) logger.error('Cannot increment scores of good pods.', { error: err })
-    })
-  }
-
-  if (badPods.length !== 0) {
-    Pod.incrementScores(badPods, constants.PODS_SCORE.MALUS, function (err) {
-      if (err) logger.error('Cannot decrement scores of bad pods.', { error: err })
-      removeBadPods.call(self)
-    })
-  }
-}
-
 function listWithLimitAndRandom (limitPods, limitRequestsPerPod, callback) {
   const self = this
   const Pod = this.sequelize.models.Pod
 
-  Pod.listRandomPodIdsWithRequest(limitPods, function (err, podIds) {
+  Pod.listRandomPodIdsWithRequest(limitPods, 'RequestToPods', function (err, podIds) {
     if (err) return callback(err)
 
     // We don't have friends that have requests
     if (podIds.length === 0) return callback(null, [])
 
-    // The the first x requests of these pods
+    // The first x requests of these pods
     // It is very important to sort by id ASC to keep the requests order!
     const query = {
       order: [
diff --git a/server/models/video.js b/server/models/video.js
index d0fd61eb4..daa273845 100644
--- a/server/models/video.js
+++ b/server/models/video.js
@@ -80,6 +80,15 @@ module.exports = function (sequelize, DataTypes) {
             if (res === false) throw new Error('Video duration is not valid.')
           }
         }
+      },
+      views: {
+        type: DataTypes.INTEGER,
+        allowNull: false,
+        defaultValue: 0,
+        validate: {
+          min: 0,
+          isInt: true
+        }
       }
     },
     {
@@ -101,6 +110,9 @@ module.exports = function (sequelize, DataTypes) {
         },
         {
           fields: [ 'infoHash' ]
+        },
+        {
+          fields: [ 'views' ]
         }
       ],
       classMethods: {
@@ -336,6 +348,7 @@ function toFormatedJSON () {
     magnetUri: this.generateMagnetUri(),
     author: this.Author.name,
     duration: this.duration,
+    views: this.views,
     tags: map(this.Tags, 'name'),
     thumbnailPath: pathUtils.join(constants.STATIC_PATHS.THUMBNAILS, this.getThumbnailName()),
     createdAt: this.createdAt,