From 9e167724f7e933f41d9ea2e1c31772bf4c560a28 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Tue, 21 Feb 2017 21:35:59 +0100 Subject: Server: make a basic "quick and dirty update" for videos This system will be useful to to update some int video attributes (likes, dislikes, views...) The classic system is not used because we need some optimization for scaling --- server/models/pod.js | 63 ++++++++++++++- server/models/request-to-pod.js | 4 +- server/models/request-video-qadu.js | 154 ++++++++++++++++++++++++++++++++++++ server/models/request.js | 63 +-------------- server/models/video.js | 13 +++ 5 files changed, 232 insertions(+), 65 deletions(-) create mode 100644 server/models/request-video-qadu.js (limited to 'server/models') diff --git a/server/models/pod.js b/server/models/pod.js index 79afb737a..14814708e 100644 --- a/server/models/pod.js +++ b/server/models/pod.js @@ -1,8 +1,11 @@ 'use strict' +const each = require('async/each') const map = require('lodash/map') +const waterfall = require('async/waterfall') const constants = require('../initializers/constants') +const logger = require('../helpers/logger') const customPodsValidators = require('../helpers/custom-validators').pods // --------------------------------------------------------------------------- @@ -62,6 +65,7 @@ module.exports = function (sequelize, DataTypes) { listBadPods, load, loadByHost, + updatePodsScore, removeAll }, instanceMethods: { @@ -144,7 +148,7 @@ function listAllIds (transaction, callback) { }) } -function listRandomPodIdsWithRequest (limit, callback) { +function listRandomPodIdsWithRequest (limit, tableRequestPod, callback) { const self = this self.count().asCallback(function (err, count) { @@ -166,7 +170,7 @@ function listRandomPodIdsWithRequest (limit, callback) { where: { id: { $in: [ - this.sequelize.literal('SELECT "podId" FROM "RequestToPods"') + this.sequelize.literal('SELECT "podId" FROM "' + tableRequestPod + '"') ] } } @@ -207,3 +211,58 @@ function loadByHost (host, callback) { function removeAll (callback) { return this.destroy().asCallback(callback) } + +function updatePodsScore (goodPods, badPods) { + const self = this + + logger.info('Updating %d good pods and %d bad pods scores.', goodPods.length, badPods.length) + + if (goodPods.length !== 0) { + this.incrementScores(goodPods, constants.PODS_SCORE.BONUS, function (err) { + if (err) logger.error('Cannot increment scores of good pods.', { error: err }) + }) + } + + if (badPods.length !== 0) { + this.incrementScores(badPods, constants.PODS_SCORE.MALUS, function (err) { + if (err) logger.error('Cannot decrement scores of bad pods.', { error: err }) + removeBadPods.call(self) + }) + } +} + +// --------------------------------------------------------------------------- + +// Remove pods with a score of 0 (too many requests where they were unreachable) +function removeBadPods () { + const self = this + + waterfall([ + function findBadPods (callback) { + self.sequelize.models.Pod.listBadPods(function (err, pods) { + if (err) { + logger.error('Cannot find bad pods.', { error: err }) + return callback(err) + } + + return callback(null, pods) + }) + }, + + function removeTheseBadPods (pods, callback) { + each(pods, function (pod, callbackEach) { + pod.destroy().asCallback(callbackEach) + }, function (err) { + return callback(err, pods.length) + }) + } + ], function (err, numberOfPodsRemoved) { + if (err) { + logger.error('Cannot remove bad pods.', { error: err }) + } else if (numberOfPodsRemoved) { + logger.info('Removed %d pods.', numberOfPodsRemoved) + } else { + logger.info('No need to remove bad pods.') + } + }) +} diff --git a/server/models/request-to-pod.js b/server/models/request-to-pod.js index f42a53458..0e01a842e 100644 --- a/server/models/request-to-pod.js +++ b/server/models/request-to-pod.js @@ -17,7 +17,7 @@ module.exports = function (sequelize, DataTypes) { } ], classMethods: { - removePodOf + removeByRequestIdsAndPod } }) @@ -26,7 +26,7 @@ module.exports = function (sequelize, DataTypes) { // --------------------------------------------------------------------------- -function removePodOf (requestsIds, podId, callback) { +function removeByRequestIdsAndPod (requestsIds, podId, callback) { if (!callback) callback = function () {} const query = { diff --git a/server/models/request-video-qadu.js b/server/models/request-video-qadu.js new file mode 100644 index 000000000..7010fc992 --- /dev/null +++ b/server/models/request-video-qadu.js @@ -0,0 +1,154 @@ +'use strict' + +/* + Request Video for Quick And Dirty Updates like: + - views + - likes + - dislikes + + We can't put it in the same system than basic requests for efficiency. + Moreover we don't want to slow down the basic requests with a lot of views/likes/dislikes requests. + So we put it an independant request scheduler. +*/ + +const values = require('lodash/values') + +const constants = require('../initializers/constants') + +// --------------------------------------------------------------------------- + +module.exports = function (sequelize, DataTypes) { + const RequestVideoQadu = sequelize.define('RequestVideoQadu', + { + type: { + type: DataTypes.ENUM(values(constants.REQUEST_VIDEO_QADU_TYPES)), + allowNull: false + } + }, + { + timestamps: false, + indexes: [ + { + fields: [ 'podId' ] + }, + { + fields: [ 'videoId' ] + } + ], + classMethods: { + associate, + + listWithLimitAndRandom, + + countTotalRequests, + removeAll, + removeByRequestIdsAndPod + } + } + ) + + return RequestVideoQadu +} + +// ------------------------------ STATICS ------------------------------ + +function associate (models) { + this.belongsTo(models.Pod, { + foreignKey: { + name: 'podId', + allowNull: false + }, + onDelete: 'CASCADE' + }) + + this.belongsTo(models.Video, { + foreignKey: { + name: 'videoId', + allowNull: false + }, + onDelete: 'CASCADE' + }) +} + +function countTotalRequests (callback) { + const query = { + include: [ this.sequelize.models.Pod ] + } + + return this.count(query).asCallback(callback) +} + +function listWithLimitAndRandom (limitPods, limitRequestsPerPod, callback) { + const self = this + const Pod = this.sequelize.models.Pod + + Pod.listRandomPodIdsWithRequest(limitPods, 'RequestVideoQadus', function (err, podIds) { + if (err) return callback(err) + + // We don't have friends that have requests + if (podIds.length === 0) return callback(null, []) + + const query = { + include: [ + { + model: self.sequelize.models.Pod, + where: { + id: { + $in: podIds + } + } + }, + { + model: self.sequelize.models.Video + } + ] + } + + self.findAll(query).asCallback(function (err, requests) { + if (err) return callback(err) + + const requestsGrouped = groupAndTruncateRequests(requests, limitRequestsPerPod) + return callback(err, requestsGrouped) + }) + }) +} + +function removeByRequestIdsAndPod (ids, podId, callback) { + const query = { + where: { + id: { + $in: ids + }, + podId + } + } + + this.destroy(query).asCallback(callback) +} + +function removeAll (callback) { + // Delete all requests + this.truncate({ cascade: true }).asCallback(callback) +} + +// --------------------------------------------------------------------------- + +function groupAndTruncateRequests (requests, limitRequestsPerPod) { + const requestsGrouped = {} + + requests.forEach(function (request) { + const pod = request.Pod + + if (!requestsGrouped[pod.id]) requestsGrouped[pod.id] = [] + + if (requestsGrouped[pod.id].length < limitRequestsPerPod) { + requestsGrouped[pod.id].push({ + request: request, + video: request.Video, + pod + }) + } + }) + + return requestsGrouped +} diff --git a/server/models/request.js b/server/models/request.js index ca616d130..de73501fc 100644 --- a/server/models/request.js +++ b/server/models/request.js @@ -1,11 +1,8 @@ 'use strict' -const each = require('async/each') -const waterfall = require('async/waterfall') const values = require('lodash/values') const constants = require('../initializers/constants') -const logger = require('../helpers/logger') // --------------------------------------------------------------------------- @@ -28,8 +25,6 @@ module.exports = function (sequelize, DataTypes) { listWithLimitAndRandom, countTotalRequests, - removeBadPods, - updatePodsScore, removeAll, removeWithEmptyTo } @@ -60,71 +55,17 @@ function countTotalRequests (callback) { return this.count(query).asCallback(callback) } -// Remove pods with a score of 0 (too many requests where they were unreachable) -function removeBadPods () { - const self = this - - waterfall([ - function findBadPods (callback) { - self.sequelize.models.Pod.listBadPods(function (err, pods) { - if (err) { - logger.error('Cannot find bad pods.', { error: err }) - return callback(err) - } - - return callback(null, pods) - }) - }, - - function removeTheseBadPods (pods, callback) { - each(pods, function (pod, callbackEach) { - pod.destroy().asCallback(callbackEach) - }, function (err) { - return callback(err, pods.length) - }) - } - ], function (err, numberOfPodsRemoved) { - if (err) { - logger.error('Cannot remove bad pods.', { error: err }) - } else if (numberOfPodsRemoved) { - logger.info('Removed %d pods.', numberOfPodsRemoved) - } else { - logger.info('No need to remove bad pods.') - } - }) -} - -function updatePodsScore (goodPods, badPods) { - const self = this - const Pod = this.sequelize.models.Pod - - logger.info('Updating %d good pods and %d bad pods scores.', goodPods.length, badPods.length) - - if (goodPods.length !== 0) { - Pod.incrementScores(goodPods, constants.PODS_SCORE.BONUS, function (err) { - if (err) logger.error('Cannot increment scores of good pods.', { error: err }) - }) - } - - if (badPods.length !== 0) { - Pod.incrementScores(badPods, constants.PODS_SCORE.MALUS, function (err) { - if (err) logger.error('Cannot decrement scores of bad pods.', { error: err }) - removeBadPods.call(self) - }) - } -} - function listWithLimitAndRandom (limitPods, limitRequestsPerPod, callback) { const self = this const Pod = this.sequelize.models.Pod - Pod.listRandomPodIdsWithRequest(limitPods, function (err, podIds) { + Pod.listRandomPodIdsWithRequest(limitPods, 'RequestToPods', function (err, podIds) { if (err) return callback(err) // We don't have friends that have requests if (podIds.length === 0) return callback(null, []) - // The the first x requests of these pods + // The first x requests of these pods // It is very important to sort by id ASC to keep the requests order! const query = { order: [ diff --git a/server/models/video.js b/server/models/video.js index d0fd61eb4..daa273845 100644 --- a/server/models/video.js +++ b/server/models/video.js @@ -80,6 +80,15 @@ module.exports = function (sequelize, DataTypes) { if (res === false) throw new Error('Video duration is not valid.') } } + }, + views: { + type: DataTypes.INTEGER, + allowNull: false, + defaultValue: 0, + validate: { + min: 0, + isInt: true + } } }, { @@ -101,6 +110,9 @@ module.exports = function (sequelize, DataTypes) { }, { fields: [ 'infoHash' ] + }, + { + fields: [ 'views' ] } ], classMethods: { @@ -336,6 +348,7 @@ function toFormatedJSON () { magnetUri: this.generateMagnetUri(), author: this.Author.name, duration: this.duration, + views: this.views, tags: map(this.Tags, 'name'), thumbnailPath: pathUtils.join(constants.STATIC_PATHS.THUMBNAILS, this.getThumbnailName()), createdAt: this.createdAt, -- cgit v1.2.3