From e4c87ec26962e359d1c70b03ed188a3f19d6a25b Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Sun, 26 Feb 2017 18:57:33 +0100 Subject: [PATCH] Server: implement video views --- server/controllers/api/remote/videos.js | 108 ++++++++++- server/controllers/api/videos.js | 3 + .../custom-validators/remote/videos.js | 24 ++- server/helpers/custom-validators/videos.js | 14 +- server/helpers/utils.js | 7 + server/initializers/constants.js | 24 ++- server/lib/friends.js | 22 ++- server/lib/request-video-event-scheduler.js | 109 +++++++++++ server/lib/request-video-qadu-scheduler.js | 2 +- .../middlewares/validators/remote/videos.js | 12 +- server/models/pod.js | 9 +- server/models/request-video-event.js | 169 ++++++++++++++++++ server/models/request-video-qadu.js | 5 +- server/models/request.js | 2 + server/tests/api/multiple-pods.js | 150 +++++++++++++--- 15 files changed, 612 insertions(+), 48 deletions(-) create mode 100644 server/lib/request-video-event-scheduler.js create mode 100644 server/models/request-video-event.js diff --git a/server/controllers/api/remote/videos.js b/server/controllers/api/remote/videos.js index 79b503d4d..39c9579c1 100644 --- a/server/controllers/api/remote/videos.js +++ b/server/controllers/api/remote/videos.js @@ -38,6 +38,13 @@ router.post('/qadu', remoteVideosQadu ) +router.post('/events', + signatureValidators.signature, + secureMiddleware.checkSignature, + videosValidators.remoteEventsVideos, + remoteVideosEvents +) + // --------------------------------------------------------------------------- module.exports = router @@ -84,6 +91,84 @@ function remoteVideosQadu (req, res, next) { return res.type('json').status(204).end() } +function remoteVideosEvents (req, res, next) { + const requests = req.body.data + const fromPod = res.locals.secure.pod + + eachSeries(requests, function (request, callbackEach) { + const eventData = request.data + + processVideosEventsRetryWrapper(eventData, fromPod, callbackEach) + }, function (err) { + if (err) logger.error('Error managing remote videos.', { error: err }) + }) + + return res.type('json').status(204).end() +} + +function processVideosEventsRetryWrapper (eventData, fromPod, finalCallback) { + const options = { + arguments: [ eventData, fromPod ], + errorMessage: 'Cannot process videos events with many retries.' + } + + databaseUtils.retryTransactionWrapper(processVideosEvents, options, finalCallback) +} + +function processVideosEvents (eventData, fromPod, finalCallback) { + waterfall([ + databaseUtils.startSerializableTransaction, + + function findVideo (t, callback) { + fetchOwnedVideo(eventData.remoteId, function (err, videoInstance) { + return callback(err, t, videoInstance) + }) + }, + + function updateVideoIntoDB (t, videoInstance, callback) { + const options = { transaction: t } + + let columnToUpdate + + switch (eventData.eventType) { + case constants.REQUEST_VIDEO_EVENT_TYPES.VIEWS: + columnToUpdate = 'views' + break + + case constants.REQUEST_VIDEO_EVENT_TYPES.LIKES: + columnToUpdate = 'likes' + break + + case constants.REQUEST_VIDEO_EVENT_TYPES.DISLIKES: + columnToUpdate = 'dislikes' + break + + default: + return callback(new Error('Unknown video event type.')) + } + + const query = {} + query[columnToUpdate] = eventData.count + + videoInstance.increment(query, options).asCallback(function (err) { + return callback(err, t) + }) + }, + + databaseUtils.commitTransaction + + ], function (err, t) { + if (err) { + console.log(err) + logger.debug('Cannot process a video event.', { error: err }) + return databaseUtils.rollbackTransaction(err, t, finalCallback) + } + + logger.info('Remote video event processed for video %s.', eventData.remoteId) + return finalCallback(null) + }) +} + function quickAndDirtyUpdateVideoRetryWrapper (videoData, fromPod, finalCallback) { const options = { arguments: [ videoData, fromPod ], @@ -98,7 +183,7 @@ function quickAndDirtyUpdateVideo (videoData, fromPod, finalCallback) { databaseUtils.startSerializableTransaction, function findVideo (t, callback) { - fetchVideo(fromPod.host, videoData.remoteId, function (err, videoInstance) { + fetchRemoteVideo(fromPod.host, videoData.remoteId, function (err, videoInstance) { return callback(err, t, videoInstance) }) }, @@ -264,7 +349,7 @@ function updateRemoteVideo (videoAttributesToUpdate, fromPod, finalCallback) { databaseUtils.startSerializableTransaction, function findVideo (t, callback) { - fetchVideo(fromPod.host, videoAttributesToUpdate.remoteId, function (err, videoInstance) { + fetchRemoteVideo(fromPod.host, videoAttributesToUpdate.remoteId, function (err, videoInstance) { return callback(err, t, videoInstance) }) }, @@ -317,7 +402,7 @@ function updateRemoteVideo (videoAttributesToUpdate, fromPod, finalCallback) { function removeRemoteVideo (videoToRemoveData, fromPod, callback) { // We need the instance because we have to remove some other stuffs (thumbnail etc) - fetchVideo(fromPod.host, videoToRemoveData.remoteId, function (err, video) { + fetchRemoteVideo(fromPod.host, videoToRemoveData.remoteId, function (err, video) { // Do not return the error, continue the process if (err) return callback(null) @@ -334,7 +419,7 @@ function removeRemoteVideo (videoToRemoveData, fromPod, callback) { } function reportAbuseRemoteVideo (reportData, fromPod, callback) { - db.Video.load(reportData.videoRemoteId, function (err, video) { + fetchOwnedVideo(reportData.videoRemoteId, function (err, video) { if (err || !video) { if (!err) err = new Error('video not found') @@ -362,7 +447,20 @@ function reportAbuseRemoteVideo (reportData, fromPod, callback) { }) } -function fetchVideo (podHost, remoteId, callback) { +function fetchOwnedVideo (id, callback) { + db.Video.load(id, function (err, video) { + if (err || !video) { + if (!err) err = new Error('video not found') + + logger.error('Cannot load owned video from id.', { error: err, id }) + return callback(err) + } + + return callback(null, video) + }) +} + +function fetchRemoteVideo (podHost, remoteId, callback) { db.Video.loadByHostAndRemoteId(podHost, remoteId, function (err, video) { if (err || !video) { if (!err) err = new Error('video not found') diff --git a/server/controllers/api/videos.js b/server/controllers/api/videos.js index 9f4bbb7b7..d64ed4e4e 100644 --- a/server/controllers/api/videos.js +++ b/server/controllers/api/videos.js @@ -333,6 +333,9 @@ function getVideo (req, res, next) { // For example, only add a view when a user watch a video during 30s etc friends.quickAndDirtyUpdateVideoToFriends(videoInstance.id, constants.REQUEST_VIDEO_QADU_TYPES.VIEWS) }) + } else { + // Just send the event to our friends + friends.addEventToRemoteVideo(videoInstance.id, constants.REQUEST_VIDEO_EVENT_TYPES.VIEWS) } // Do not wait the view system diff --git a/server/helpers/custom-validators/remote/videos.js b/server/helpers/custom-validators/remote/videos.js index 2e9cf822e..c1786014d 100644 --- a/server/helpers/custom-validators/remote/videos.js +++ b/server/helpers/custom-validators/remote/videos.js @@ -1,6 +1,7 @@ 'use strict' const has = require('lodash/has') +const values = require('lodash/values') const constants = require('../../../initializers/constants') const videosValidators = require('../videos') @@ -10,13 +11,17 @@ const ENDPOINT_ACTIONS = constants.REQUEST_ENDPOINT_ACTIONS[constants.REQUEST_EN const remoteVideosValidators = { isEachRemoteRequestVideosValid, - isEachRemoteRequestVideosQaduValid + isEachRemoteRequestVideosQaduValid, + isEachRemoteRequestVideosEventsValid } function isEachRemoteRequestVideosValid (requests) { return miscValidators.isArray(requests) && requests.every(function (request) { const video = request.data + + if (!video) return false + return ( isRequestTypeAddValid(request.type) && isCommonVideoAttributesValid(video) && @@ -45,6 +50,8 @@ function isEachRemoteRequestVideosQaduValid (requests) { requests.every(function (request) { const video = request.data + if (!video) return false + return ( videosValidators.isVideoRemoteIdValid(video.remoteId) && (has(video, 'views') === false || videosValidators.isVideoViewsValid) && @@ -54,6 +61,21 @@ function isEachRemoteRequestVideosQaduValid (requests) { }) } +function isEachRemoteRequestVideosEventsValid (requests) { + return miscValidators.isArray(requests) && + requests.every(function (request) { + const eventData = request.data + + if (!eventData) return false + + return ( + videosValidators.isVideoRemoteIdValid(eventData.remoteId) && + values(constants.REQUEST_VIDEO_EVENT_TYPES).indexOf(eventData.eventType) !== -1 && + videosValidators.isVideoEventCountValid(eventData.count) + ) + }) +} + // --------------------------------------------------------------------------- module.exports = remoteVideosValidators diff --git a/server/helpers/custom-validators/videos.js b/server/helpers/custom-validators/videos.js index 1d844118b..c5a1f3cb5 100644 --- a/server/helpers/custom-validators/videos.js +++ b/server/helpers/custom-validators/videos.js @@ -7,6 +7,7 @@ const usersValidators = require('./users') const miscValidators = require('./misc') const VIDEOS_CONSTRAINTS_FIELDS = constants.CONSTRAINTS_FIELDS.VIDEOS const VIDEO_ABUSES_CONSTRAINTS_FIELDS = constants.CONSTRAINTS_FIELDS.VIDEO_ABUSES +const VIDEO_EVENTS_CONSTRAINTS_FIELDS = constants.CONSTRAINTS_FIELDS.VIDEO_EVENTS const videosValidators = { isVideoAuthorValid, @@ -25,7 +26,8 @@ const videosValidators = { isVideoFile, isVideoViewsValid, isVideoLikesValid, - isVideoDislikesValid + isVideoDislikesValid, + isVideoEventCountValid } function isVideoAuthorValid (value) { @@ -86,15 +88,19 @@ function isVideoAbuseReporterUsernameValid (value) { } function isVideoViewsValid (value) { - return validator.isInt(value, { min: 0 }) + return validator.isInt(value + '', VIDEOS_CONSTRAINTS_FIELDS.VIEWS) } function isVideoLikesValid (value) { - return validator.isInt(value, { min: 0 }) + return validator.isInt(value + '', VIDEOS_CONSTRAINTS_FIELDS.LIKES) } function isVideoDislikesValid (value) { - return validator.isInt(value, { min: 0 }) + return validator.isInt(value + '', VIDEOS_CONSTRAINTS_FIELDS.DISLIKES) +} + +function isVideoEventCountValid (value) { + return validator.isInt(value + '', VIDEO_EVENTS_CONSTRAINTS_FIELDS.COUNT) } function isVideoFile (value, files) { diff --git a/server/helpers/utils.js b/server/helpers/utils.js index 9f4b14582..6d40e8f3f 100644 --- a/server/helpers/utils.js +++ b/server/helpers/utils.js @@ -6,6 +6,7 @@ const logger = require('./logger') const utils = { badRequest, + createEmptyCallback, cleanForExit, generateRandomString, isTestInstance, @@ -29,6 +30,12 @@ function cleanForExit (webtorrentProcess) { process.kill(-webtorrentProcess.pid) } +function createEmptyCallback () { + return function (err) { + if (err) logger.error('Error in empty callback.', { error: err }) + } +} + function isTestInstance () { return (process.env.NODE_ENV === 'test') } diff --git a/server/initializers/constants.js b/server/initializers/constants.js index 668bfe56c..b99186e13 100644 --- a/server/initializers/constants.js +++ b/server/initializers/constants.js @@ -85,7 +85,13 @@ const CONSTRAINTS_FIELDS = { TAGS: { min: 1, max: 3 }, // Number of total tags TAG: { min: 2, max: 10 }, // Length THUMBNAIL: { min: 2, max: 30 }, - THUMBNAIL_DATA: { min: 0, max: 20000 } // Bytes + THUMBNAIL_DATA: { min: 0, max: 20000 }, // Bytes + VIEWS: { min: 0 }, + LIKES: { min: 0 }, + DISLIKES: { min: 0 } + }, + VIDEO_EVENTS: { + COUNT: { min: 0 } } } @@ -120,12 +126,17 @@ const REQUESTS_VIDEO_QADU_LIMIT_PODS = 10 // The QADU requests are not big const REQUESTS_VIDEO_QADU_LIMIT_PER_POD = 50 +const REQUESTS_VIDEO_EVENT_LIMIT_PODS = 10 +// The EVENTS requests are not big +const REQUESTS_VIDEO_EVENT_LIMIT_PER_POD = 50 + // Number of requests to retry for replay requests module const RETRY_REQUESTS = 5 const REQUEST_ENDPOINTS = { VIDEOS: 'videos', - QADU: 'videos/qadu' + QADU: 'videos/qadu', + EVENT: 'videos/events' } const REQUEST_ENDPOINT_ACTIONS = {} REQUEST_ENDPOINT_ACTIONS[REQUEST_ENDPOINTS.VIDEOS] = { @@ -141,6 +152,12 @@ const REQUEST_VIDEO_QADU_TYPES = { VIEWS: 'views' } +const REQUEST_VIDEO_EVENT_TYPES = { + LIKES: 'likes', + DISLIKES: 'dislikes', + VIEWS: 'views' +} + const REMOTE_SCHEME = { HTTP: 'https', WS: 'wss' @@ -210,6 +227,7 @@ module.exports = { REMOTE_SCHEME, REQUEST_ENDPOINT_ACTIONS, REQUEST_ENDPOINTS, + REQUEST_VIDEO_EVENT_TYPES, REQUEST_VIDEO_QADU_TYPES, REQUESTS_IN_PARALLEL, REQUESTS_INTERVAL, @@ -217,6 +235,8 @@ module.exports = { REQUESTS_LIMIT_PODS, REQUESTS_VIDEO_QADU_LIMIT_PER_POD, REQUESTS_VIDEO_QADU_LIMIT_PODS, + REQUESTS_VIDEO_EVENT_LIMIT_PER_POD, + REQUESTS_VIDEO_EVENT_LIMIT_PODS, RETRY_REQUESTS, SEARCHABLE_COLUMNS, SIGNATURE_ALGORITHM, diff --git a/server/lib/friends.js b/server/lib/friends.js index 424a30801..203f0e52c 100644 --- a/server/lib/friends.js +++ b/server/lib/friends.js @@ -11,13 +11,16 @@ const db = require('../initializers/database') const logger = require('../helpers/logger') const peertubeCrypto = require('../helpers/peertube-crypto') const requests = require('../helpers/requests') +const utils = require('../helpers/utils') const RequestScheduler = require('./request-scheduler') const RequestVideoQaduScheduler = require('./request-video-qadu-scheduler') +const RequestVideoEventScheduler = require('./request-video-event-scheduler') const ENDPOINT_ACTIONS = constants.REQUEST_ENDPOINT_ACTIONS[constants.REQUEST_ENDPOINTS.VIDEOS] const requestScheduler = new RequestScheduler() const requestSchedulerVideoQadu = new RequestVideoQaduScheduler() +const requestSchedulerVideoEvent = new RequestVideoEventScheduler() const friends = { activate, @@ -25,6 +28,7 @@ const friends = { updateVideoToFriends, reportAbuseVideoToFriend, quickAndDirtyUpdateVideoToFriends, + addEventToRemoteVideo, hasFriends, makeFriends, quitFriends, @@ -35,6 +39,7 @@ const friends = { function activate () { requestScheduler.activate() requestSchedulerVideoQadu.activate() + requestSchedulerVideoEvent.activate() } function addVideoToFriends (videoData, transaction, callback) { @@ -85,6 +90,15 @@ function quickAndDirtyUpdateVideoToFriends (videoId, type, transaction, callback return createVideoQaduRequest(options, callback) } +function addEventToRemoteVideo (videoId, type, transaction, callback) { + const options = { + videoId, + type, + transaction + } + createVideoEventRequest(options, callback) +} + function hasFriends (callback) { db.Pod.countAll(function (err, count) { if (err) return callback(err) @@ -329,11 +343,17 @@ function createRequest (options, callback) { } function createVideoQaduRequest (options, callback) { - if (!callback) callback = function () {} + if (!callback) callback = utils.createEmptyCallback() requestSchedulerVideoQadu.createRequest(options, callback) } +function createVideoEventRequest (options, callback) { + if (!callback) callback = utils.createEmptyCallback() + + requestSchedulerVideoEvent.createRequest(options, callback) +} + function isMe (host) { return host === constants.CONFIG.WEBSERVER.HOST } diff --git a/server/lib/request-video-event-scheduler.js b/server/lib/request-video-event-scheduler.js new file mode 100644 index 000000000..5ea5631b0 --- /dev/null +++ b/server/lib/request-video-event-scheduler.js @@ -0,0 +1,109 @@ +'use strict' + +const BaseRequestScheduler = require('./base-request-scheduler') +const constants = require('../initializers/constants') +const db = require('../initializers/database') + +module.exports = class RequestVideoEventScheduler extends BaseRequestScheduler { + + constructor () { + super() + + // We limit the size of the requests + this.limitPods = constants.REQUESTS_VIDEO_EVENT_LIMIT_PODS + this.limitPerPod = constants.REQUESTS_VIDEO_EVENT_LIMIT_PER_POD + + this.description = 'video event requests' + } + + getRequestModel () { + return db.RequestVideoEvent + } + + getRequestToPodModel () { + return db.RequestVideoEvent + } + + buildRequestObjects (eventsToProcess) { + const requestsToMakeGrouped = {} + + /* Example: + { + pod1: { + video1: { views: 4, likes: 5 }, + video2: { likes: 5 } + } + } + */ + const eventsPerVideoPerPod = {} + + // We group video events per video and per pod + // We add the counts of the same event types + Object.keys(eventsToProcess).forEach(toPodId => { + eventsToProcess[toPodId].forEach(eventToProcess => { + if (!eventsPerVideoPerPod[toPodId]) eventsPerVideoPerPod[toPodId] = {} + + if (!requestsToMakeGrouped[toPodId]) { + requestsToMakeGrouped[toPodId] = { + toPod: eventToProcess.pod, + endpoint: constants.REQUEST_ENDPOINTS.EVENT, + ids: [], // request ids, to delete them from the DB in the future + datas: [] // requests data + } + } + requestsToMakeGrouped[toPodId].ids.push(eventToProcess.id) + + const eventsPerVideo = eventsPerVideoPerPod[toPodId] + const remoteId = eventToProcess.video.remoteId + if (!eventsPerVideo[remoteId]) eventsPerVideo[remoteId] = {} + + const events = eventsPerVideo[remoteId] + if (!events[eventToProcess.type]) events[eventToProcess.type] = 0 + + events[eventToProcess.type] += eventToProcess.count + }) + }) + + // Now we build our requests array per pod + Object.keys(eventsPerVideoPerPod).forEach(toPodId => { + const eventsForPod = eventsPerVideoPerPod[toPodId] + + Object.keys(eventsForPod).forEach(remoteId => { + const eventsForVideo = eventsForPod[remoteId] + + Object.keys(eventsForVideo).forEach(eventType => { + requestsToMakeGrouped[toPodId].datas.push({ + data: { + remoteId, + eventType, + count: eventsForVideo[eventType] + } + }) + }) + }) + }) + + return requestsToMakeGrouped + } + + // { type, videoId, count?, transaction? } + createRequest (options, callback) { + const type = options.type + const videoId = options.videoId + const transaction = options.transaction + let count = options.count + + if (count === undefined) count = 1 + + const dbRequestOptions = {} + if (transaction) dbRequestOptions.transaction = transaction + + const createQuery = { + type, + count, + videoId + } + + return db.RequestVideoEvent.create(createQuery, dbRequestOptions).asCallback(callback) + } +} diff --git a/server/lib/request-video-qadu-scheduler.js b/server/lib/request-video-qadu-scheduler.js index 401b2fb44..29e44a6c4 100644 --- a/server/lib/request-video-qadu-scheduler.js +++ b/server/lib/request-video-qadu-scheduler.js @@ -12,7 +12,7 @@ module.exports = class RequestVideoQaduScheduler extends BaseRequestScheduler { // We limit the size of the requests this.limitPods = constants.REQUESTS_VIDEO_QADU_LIMIT_PODS - this.limitPerPod = constants.REQUESTS_VIDEO_QADU_LIMIT_PODS + this.limitPerPod = constants.REQUESTS_VIDEO_QADU_LIMIT_PER_POD this.description = 'video QADU requests' } diff --git a/server/middlewares/validators/remote/videos.js b/server/middlewares/validators/remote/videos.js index ddc274c45..f2c6cba5e 100644 --- a/server/middlewares/validators/remote/videos.js +++ b/server/middlewares/validators/remote/videos.js @@ -5,7 +5,8 @@ const logger = require('../../../helpers/logger') const validatorsRemoteVideos = { remoteVideos, - remoteQaduVideos + remoteQaduVideos, + remoteEventsVideos } function remoteVideos (req, res, next) { @@ -19,11 +20,18 @@ function remoteVideos (req, res, next) { function remoteQaduVideos (req, res, next) { req.checkBody('data').isEachRemoteRequestVideosQaduValid() - logger.debug('Checking remoteVideosQadu parameters', { parameters: req.body }) + logger.debug('Checking remoteQaduVideos parameters', { parameters: req.body }) checkErrors(req, res, next) } +function remoteEventsVideos (req, res, next) { + req.checkBody('data').isEachRemoteRequestVideosEventsValid() + + logger.debug('Checking remoteEventsVideos parameters', { parameters: req.body }) + + checkErrors(req, res, next) +} // --------------------------------------------------------------------------- module.exports = validatorsRemoteVideos diff --git a/server/models/pod.js b/server/models/pod.js index 14814708e..8e2d488e1 100644 --- a/server/models/pod.js +++ b/server/models/pod.js @@ -148,7 +148,12 @@ function listAllIds (transaction, callback) { }) } -function listRandomPodIdsWithRequest (limit, tableRequestPod, callback) { +function listRandomPodIdsWithRequest (limit, tableWithPods, tableWithPodsJoins, callback) { + if (!callback) { + callback = tableWithPodsJoins + tableWithPodsJoins = '' + } + const self = this self.count().asCallback(function (err, count) { @@ -170,7 +175,7 @@ function listRandomPodIdsWithRequest (limit, tableRequestPod, callback) { where: { id: { $in: [ - this.sequelize.literal('SELECT "podId" FROM "' + tableRequestPod + '"') + this.sequelize.literal(`SELECT DISTINCT "${tableWithPods}"."podId" FROM "${tableWithPods}" ${tableWithPodsJoins}`) ] } } diff --git a/server/models/request-video-event.js b/server/models/request-video-event.js new file mode 100644 index 000000000..ef3ebcb3a --- /dev/null +++ b/server/models/request-video-event.js @@ -0,0 +1,169 @@ +'use strict' + +/* + Request Video events (likes, dislikes, views...) +*/ + +const values = require('lodash/values') + +const constants = require('../initializers/constants') +const customVideosValidators = require('../helpers/custom-validators').videos + +// --------------------------------------------------------------------------- + +module.exports = function (sequelize, DataTypes) { + const RequestVideoEvent = sequelize.define('RequestVideoEvent', + { + type: { + type: DataTypes.ENUM(values(constants.REQUEST_VIDEO_EVENT_TYPES)), + allowNull: false + }, + count: { + type: DataTypes.INTEGER, + allowNull: false, + validate: { + countValid: function (value) { + const res = customVideosValidators.isVideoEventCountValid(value) + if (res === false) throw new Error('Video event count is not valid.') + } + } + } + }, + { + updatedAt: false, + indexes: [ + { + fields: [ 'videoId' ] + } + ], + classMethods: { + associate, + + listWithLimitAndRandom, + + countTotalRequests, + removeAll, + removeByRequestIdsAndPod + } + } + ) + + return RequestVideoEvent +} + +// ------------------------------ STATICS ------------------------------ + +function associate (models) { + this.belongsTo(models.Video, { + foreignKey: { + name: 'videoId', + allowNull: false + }, + onDelete: 'CASCADE' + }) +} + +function countTotalRequests (callback) { + const query = {} + return this.count(query).asCallback(callback) +} + +function listWithLimitAndRandom (limitPods, limitRequestsPerPod, callback) { + const self = this + const Pod = this.sequelize.models.Pod + + // We make a join between videos and authors to find the podId of our video event requests + const podJoins = 'INNER JOIN "Videos" ON "Videos"."authorId" = "Authors"."id" ' + + 'INNER JOIN "RequestVideoEvents" ON "RequestVideoEvents"."videoId" = "Videos"."id"' + + Pod.listRandomPodIdsWithRequest(limitPods, 'Authors', podJoins, function (err, podIds) { + if (err) return callback(err) + + // We don't have friends that have requests + if (podIds.length === 0) return callback(null, []) + + const query = { + include: [ + { + model: self.sequelize.models.Video, + include: [ + { + model: self.sequelize.models.Author, + include: [ + { + model: self.sequelize.models.Pod, + where: { + id: { + $in: podIds + } + } + } + ] + } + ] + } + ] + } + + self.findAll(query).asCallback(function (err, requests) { + if (err) return callback(err) + + const requestsGrouped = groupAndTruncateRequests(requests, limitRequestsPerPod) + return callback(err, requestsGrouped) + }) + }) +} + +function removeByRequestIdsAndPod (ids, podId, callback) { + const query = { + where: { + id: { + $in: ids + } + }, + include: [ + { + model: this.sequelize.models.Video, + include: [ + { + model: this.sequelize.models.Author, + where: { + podId + } + } + ] + } + ] + } + + this.destroy(query).asCallback(callback) +} + +function removeAll (callback) { + // Delete all requests + this.truncate({ cascade: true }).asCallback(callback) +} + +// --------------------------------------------------------------------------- + +function groupAndTruncateRequests (events, limitRequestsPerPod) { + const eventsGrouped = {} + + events.forEach(function (event) { + const pod = event.Video.Author.Pod + + if (!eventsGrouped[pod.id]) eventsGrouped[pod.id] = [] + + if (eventsGrouped[pod.id].length < limitRequestsPerPod) { + eventsGrouped[pod.id].push({ + id: event.id, + type: event.type, + count: event.count, + video: event.Video, + pod + }) + } + }) + + return eventsGrouped +} diff --git a/server/models/request-video-qadu.js b/server/models/request-video-qadu.js index 7010fc992..5d88738aa 100644 --- a/server/models/request-video-qadu.js +++ b/server/models/request-video-qadu.js @@ -71,10 +71,7 @@ function associate (models) { } function countTotalRequests (callback) { - const query = { - include: [ this.sequelize.models.Pod ] - } - + const query = {} return this.count(query).asCallback(callback) } diff --git a/server/models/request.js b/server/models/request.js index de73501fc..3a047f7ee 100644 --- a/server/models/request.js +++ b/server/models/request.js @@ -48,6 +48,8 @@ function associate (models) { } function countTotalRequests (callback) { + // We need to include Pod because there are no cascade delete when a pod is removed + // So we could count requests that do not have existing pod anymore const query = { include: [ this.sequelize.models.Pod ] } diff --git a/server/tests/api/multiple-pods.js b/server/tests/api/multiple-pods.js index 871db54be..94b62e27a 100644 --- a/server/tests/api/multiple-pods.js +++ b/server/tests/api/multiple-pods.js @@ -377,19 +377,44 @@ describe('Test multiple pods', function () { }) describe('Should update video views', function () { - let videoId1 - let videoId2 + let localVideosPod3 = [] + let remoteVideosPod1 = [] + let remoteVideosPod2 = [] + let remoteVideosPod3 = [] before(function (done) { - videosUtils.getVideosList(servers[2].url, function (err, res) { - if (err) throw err + parallel([ + function (callback) { + videosUtils.getVideosList(servers[0].url, function (err, res) { + if (err) throw err - const videos = res.body.data.filter(video => video.isLocal === true) - videoId1 = videos[0].id - videoId2 = videos[1].id + remoteVideosPod1 = res.body.data.filter(video => video.isLocal === false).map(video => video.id) - done() - }) + callback() + }) + }, + + function (callback) { + videosUtils.getVideosList(servers[1].url, function (err, res) { + if (err) throw err + + remoteVideosPod2 = res.body.data.filter(video => video.isLocal === false).map(video => video.id) + + callback() + }) + }, + + function (callback) { + videosUtils.getVideosList(servers[2].url, function (err, res) { + if (err) throw err + + localVideosPod3 = res.body.data.filter(video => video.isLocal === true).map(video => video.id) + remoteVideosPod3 = res.body.data.filter(video => video.isLocal === false).map(video => video.id) + + callback() + }) + } + ], done) }) it('Should views multiple videos on owned servers', function (done) { @@ -397,42 +422,115 @@ describe('Test multiple pods', function () { parallel([ function (callback) { - videosUtils.getVideo(servers[2].url, videoId1, callback) + videosUtils.getVideo(servers[2].url, localVideosPod3[0], callback) }, function (callback) { - videosUtils.getVideo(servers[2].url, videoId1, callback) + videosUtils.getVideo(servers[2].url, localVideosPod3[0], callback) }, function (callback) { - videosUtils.getVideo(servers[2].url, videoId1, callback) + videosUtils.getVideo(servers[2].url, localVideosPod3[0], callback) }, function (callback) { - videosUtils.getVideo(servers[2].url, videoId2, callback) + videosUtils.getVideo(servers[2].url, localVideosPod3[1], callback) + }, + + function (callback) { + setTimeout(done, 22000) } ], function (err) { if (err) throw err - setTimeout(done, 22000) + each(servers, function (server, callback) { + videosUtils.getVideosList(server.url, function (err, res) { + if (err) throw err + + const videos = res.body.data + expect(videos.find(video => video.views === 3)).to.be.exist + expect(videos.find(video => video.views === 1)).to.be.exist + + callback() + }) + }, done) }) }) - it('Should have views updated on each pod', function (done) { - each(servers, function (server, callback) { - videosUtils.getVideosList(server.url, function (err, res) { - if (err) throw err + it('Should views multiple videos on each servers', function (done) { + this.timeout(30000) - const videos = res.body.data - expect(videos.find(video => video.views === 3)).to.be.exist - expect(videos.find(video => video.views === 1)).to.be.exist + parallel([ + function (callback) { + videosUtils.getVideo(servers[0].url, remoteVideosPod1[0], callback) + }, - callback() - }) - }, done) + function (callback) { + videosUtils.getVideo(servers[1].url, remoteVideosPod2[0], callback) + }, + + function (callback) { + videosUtils.getVideo(servers[1].url, remoteVideosPod2[0], callback) + }, + + function (callback) { + videosUtils.getVideo(servers[2].url, remoteVideosPod3[0], callback) + }, + + function (callback) { + videosUtils.getVideo(servers[2].url, remoteVideosPod3[1], callback) + }, + + function (callback) { + videosUtils.getVideo(servers[2].url, remoteVideosPod3[1], callback) + }, + + function (callback) { + videosUtils.getVideo(servers[2].url, remoteVideosPod3[1], callback) + }, + + function (callback) { + videosUtils.getVideo(servers[2].url, localVideosPod3[1], callback) + }, + + function (callback) { + videosUtils.getVideo(servers[2].url, localVideosPod3[1], callback) + }, + + function (callback) { + videosUtils.getVideo(servers[2].url, localVideosPod3[1], callback) + }, + + function (callback) { + setTimeout(done, 22000) + } + ], function (err) { + if (err) throw err + + let baseVideos = null + each(servers, function (server, callback) { + videosUtils.getVideosList(server.url, function (err, res) { + if (err) throw err + + const videos = res.body + + // Initialize base videos for future comparisons + if (baseVideos === null) { + baseVideos = videos + return callback() + } + + for (let i = 0; i < baseVideos.length; i++) { + expect(baseVideos[i].views).to.equal(videos[i].views) + } + + callback() + }) + }, done) + }) }) }) -/* + describe('Should manipulate these videos', function () { it('Should update the video 3 by asking pod 3', function (done) { this.timeout(15000) @@ -520,7 +618,7 @@ describe('Test multiple pods', function () { }, done) }) }) -*/ + after(function (done) { servers.forEach(function (server) { process.kill(-server.app.pid) -- 2.41.0