From a6fd2b30bf717eec14972a2175354781f5f43e77 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Fri, 30 Dec 2016 12:53:41 +0100 Subject: Server: move remote routes in their own directory --- server/controllers/api/remote/index.js | 16 +++ server/controllers/api/remote/videos.js | 237 ++++++++++++++++++++++++++++++++ 2 files changed, 253 insertions(+) create mode 100644 server/controllers/api/remote/index.js create mode 100644 server/controllers/api/remote/videos.js (limited to 'server/controllers/api/remote') diff --git a/server/controllers/api/remote/index.js b/server/controllers/api/remote/index.js new file mode 100644 index 000000000..2947632d5 --- /dev/null +++ b/server/controllers/api/remote/index.js @@ -0,0 +1,16 @@ +'use strict' + +const express = require('express') + +const utils = require('../../../helpers/utils') + +const router = express.Router() + +const videosRemoteController = require('./videos') + +router.use('/videos', videosRemoteController) +router.use('/*', utils.badRequest) + +// --------------------------------------------------------------------------- + +module.exports = router diff --git a/server/controllers/api/remote/videos.js b/server/controllers/api/remote/videos.js new file mode 100644 index 000000000..87c49bff9 --- /dev/null +++ b/server/controllers/api/remote/videos.js @@ -0,0 +1,237 @@ +'use strict' + +const eachSeries = require('async/eachSeries') +const express = require('express') +const waterfall = require('async/waterfall') + +const db = require('../../../initializers/database') +const middlewares = require('../../../middlewares') +const secureMiddleware = middlewares.secure +const validators = middlewares.validators.remote +const logger = require('../../../helpers/logger') + +const router = express.Router() + +router.post('/', + validators.signature, + secureMiddleware.checkSignature, + validators.remoteVideos, + remoteVideos +) + +// --------------------------------------------------------------------------- + +module.exports = router + +// --------------------------------------------------------------------------- + +function remoteVideos (req, res, next) { + const requests = req.body.data + const fromPod = res.locals.secure.pod + + // We need to process in the same order to keep consistency + // TODO: optimization + eachSeries(requests, function (request, callbackEach) { + const videoData = request.data + + switch (request.type) { + case 'add': + addRemoteVideo(videoData, fromPod, callbackEach) + break + + case 'update': + updateRemoteVideo(videoData, fromPod, callbackEach) + break + + case 'remove': + removeRemoteVideo(videoData, fromPod, callbackEach) + break + + default: + logger.error('Unkown remote request type %s.', request.type) + } + }, function (err) { + if (err) logger.error('Error managing remote videos.', { error: err }) + }) + + // We don't need to keep the other pod waiting + return res.type('json').status(204).end() +} + +function addRemoteVideo (videoToCreateData, fromPod, finalCallback) { + logger.debug('Adding remote video "%s".', videoToCreateData.name) + + waterfall([ + + function startTransaction (callback) { + db.sequelize.transaction().asCallback(function (err, t) { + return callback(err, t) + }) + }, + + function findOrCreateAuthor (t, callback) { + const name = videoToCreateData.author + const podId = fromPod.id + // This author is from another pod so we do not associate a user + const userId = null + + db.Author.findOrCreateAuthor(name, podId, userId, t, function (err, authorInstance) { + return callback(err, t, authorInstance) + }) + }, + + function findOrCreateTags (t, author, callback) { + const tags = videoToCreateData.tags + + db.Tag.findOrCreateTags(tags, t, function (err, tagInstances) { + return callback(err, t, author, tagInstances) + }) + }, + + function createVideoObject (t, author, tagInstances, callback) { + const videoData = { + name: videoToCreateData.name, + remoteId: videoToCreateData.remoteId, + extname: videoToCreateData.extname, + infoHash: videoToCreateData.infoHash, + description: videoToCreateData.description, + authorId: author.id, + duration: videoToCreateData.duration, + createdAt: videoToCreateData.createdAt, + updatedAt: videoToCreateData.updatedAt + } + + const video = db.Video.build(videoData) + + return callback(null, t, tagInstances, video) + }, + + function generateThumbnail (t, tagInstances, video, callback) { + db.Video.generateThumbnailFromData(video, videoToCreateData.thumbnailData, function (err) { + if (err) { + logger.error('Cannot generate thumbnail from data.', { error: err }) + return callback(err) + } + + return callback(err, t, tagInstances, video) + }) + }, + + function insertVideoIntoDB (t, tagInstances, video, callback) { + const options = { + transaction: t + } + + video.save(options).asCallback(function (err, videoCreated) { + return callback(err, t, tagInstances, videoCreated) + }) + }, + + function associateTagsToVideo (t, tagInstances, video, callback) { + const options = { transaction: t } + + video.setTags(tagInstances, options).asCallback(function (err) { + return callback(err, t) + }) + } + + ], function (err, t) { + if (err) { + logger.error('Cannot insert the remote video.') + + // Abort transaction? + if (t) t.rollback() + + return finalCallback(err) + } + + // Commit transaction + t.commit() + + return finalCallback() + }) +} + +function updateRemoteVideo (videoAttributesToUpdate, fromPod, finalCallback) { + logger.debug('Updating remote video "%s".', videoAttributesToUpdate.name) + + waterfall([ + + function startTransaction (callback) { + db.sequelize.transaction().asCallback(function (err, t) { + return callback(err, t) + }) + }, + + function findVideo (t, callback) { + db.Video.loadByHostAndRemoteId(fromPod.host, videoAttributesToUpdate.remoteId, function (err, videoInstance) { + if (err || !videoInstance) { + logger.error('Cannot load video from host and remote id.', { error: err.message }) + return callback(err) + } + + return callback(null, t, videoInstance) + }) + }, + + function findOrCreateTags (t, videoInstance, callback) { + const tags = videoAttributesToUpdate.tags + + db.Tag.findOrCreateTags(tags, t, function (err, tagInstances) { + return callback(err, t, videoInstance, tagInstances) + }) + }, + + function updateVideoIntoDB (t, videoInstance, tagInstances, callback) { + const options = { transaction: t } + + videoInstance.set('name', videoAttributesToUpdate.name) + videoInstance.set('description', videoAttributesToUpdate.description) + videoInstance.set('infoHash', videoAttributesToUpdate.infoHash) + videoInstance.set('duration', videoAttributesToUpdate.duration) + videoInstance.set('createdAt', videoAttributesToUpdate.createdAt) + videoInstance.set('updatedAt', videoAttributesToUpdate.updatedAt) + videoInstance.set('extname', videoAttributesToUpdate.extname) + + videoInstance.save(options).asCallback(function (err) { + return callback(err, t, videoInstance, tagInstances) + }) + }, + + function associateTagsToVideo (t, videoInstance, tagInstances, callback) { + const options = { transaction: t } + + videoInstance.setTags(tagInstances, options).asCallback(function (err) { + return callback(err, t) + }) + } + + ], function (err, t) { + if (err) { + logger.error('Cannot update the remote video.') + + // Abort transaction? + if (t) t.rollback() + + return finalCallback(err) + } + + // Commit transaction + t.commit() + + return finalCallback() + }) +} + +function removeRemoteVideo (videoToRemoveData, fromPod, callback) { + // We need the instance because we have to remove some other stuffs (thumbnail etc) + db.Video.loadByHostAndRemoteId(fromPod.host, videoToRemoveData.remoteId, function (err, video) { + if (err || !video) { + logger.error('Cannot load video from host and remote id.', { error: err.message }) + return callback(err) + } + + logger.debug('Removing remote video %s.', video.remoteId) + video.destroy().asCallback(callback) + }) +} -- cgit v1.2.3 From 55fa55a9be566cca2ba95322f2ae23b434aed62a Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Wed, 4 Jan 2017 20:59:23 +0100 Subject: Server: add video abuse support --- server/controllers/api/remote/videos.js | 68 ++++++++++++++++++++++++--------- 1 file changed, 50 insertions(+), 18 deletions(-) (limited to 'server/controllers/api/remote') diff --git a/server/controllers/api/remote/videos.js b/server/controllers/api/remote/videos.js index 87c49bff9..d02da4463 100644 --- a/server/controllers/api/remote/videos.js +++ b/server/controllers/api/remote/videos.js @@ -7,15 +7,16 @@ const waterfall = require('async/waterfall') const db = require('../../../initializers/database') const middlewares = require('../../../middlewares') const secureMiddleware = middlewares.secure -const validators = middlewares.validators.remote +const videosValidators = middlewares.validators.remote.videos +const signatureValidators = middlewares.validators.remote.signature const logger = require('../../../helpers/logger') const router = express.Router() router.post('/', - validators.signature, + signatureValidators.signature, secureMiddleware.checkSignature, - validators.remoteVideos, + videosValidators.remoteVideos, remoteVideos ) @@ -32,19 +33,23 @@ function remoteVideos (req, res, next) { // We need to process in the same order to keep consistency // TODO: optimization eachSeries(requests, function (request, callbackEach) { - const videoData = request.data + const data = request.data switch (request.type) { case 'add': - addRemoteVideo(videoData, fromPod, callbackEach) + addRemoteVideo(data, fromPod, callbackEach) break case 'update': - updateRemoteVideo(videoData, fromPod, callbackEach) + updateRemoteVideo(data, fromPod, callbackEach) break case 'remove': - removeRemoteVideo(videoData, fromPod, callbackEach) + removeRemoteVideo(data, fromPod, callbackEach) + break + + case 'report-abuse': + reportAbuseRemoteVideo(data, fromPod, callbackEach) break default: @@ -164,13 +169,8 @@ function updateRemoteVideo (videoAttributesToUpdate, fromPod, finalCallback) { }, function findVideo (t, callback) { - db.Video.loadByHostAndRemoteId(fromPod.host, videoAttributesToUpdate.remoteId, function (err, videoInstance) { - if (err || !videoInstance) { - logger.error('Cannot load video from host and remote id.', { error: err.message }) - return callback(err) - } - - return callback(null, t, videoInstance) + fetchVideo(fromPod.host, videoAttributesToUpdate.remoteId, function (err, videoInstance) { + return callback(err, t, videoInstance) }) }, @@ -225,13 +225,45 @@ function updateRemoteVideo (videoAttributesToUpdate, fromPod, finalCallback) { function removeRemoteVideo (videoToRemoveData, fromPod, callback) { // We need the instance because we have to remove some other stuffs (thumbnail etc) - db.Video.loadByHostAndRemoteId(fromPod.host, videoToRemoveData.remoteId, function (err, video) { + fetchVideo(fromPod.host, videoToRemoveData.remoteId, function (err, video) { + if (err) return callback(err) + + logger.debug('Removing remote video %s.', video.remoteId) + video.destroy().asCallback(callback) + }) +} + +function reportAbuseRemoteVideo (reportData, fromPod, callback) { + db.Video.load(reportData.videoRemoteId, function (err, video) { if (err || !video) { - logger.error('Cannot load video from host and remote id.', { error: err.message }) + if (!err) err = new Error('video not found') + + logger.error('Cannot load video from host and remote id.', { error: err }) return callback(err) } - logger.debug('Removing remote video %s.', video.remoteId) - video.destroy().asCallback(callback) + logger.debug('Reporting remote abuse for video %s.', video.id) + + const videoAbuseData = { + reporterUsername: reportData.reporterUsername, + reason: reportData.reportReason, + reporterPodId: fromPod.id, + videoId: video.id + } + + db.VideoAbuse.create(videoAbuseData).asCallback(callback) + }) +} + +function fetchVideo (podHost, remoteId, callback) { + db.Video.loadByHostAndRemoteId(podHost, remoteId, function (err, video) { + if (err || !video) { + if (!err) err = new Error('video not found') + + logger.error('Cannot load video from host and remote id.', { error: err }) + return callback(err) + } + + return callback(null, video) }) } -- cgit v1.2.3 From ed04d94f6d7132055f97a2f757b85c03c5f2a0b6 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Fri, 6 Jan 2017 23:24:47 +0100 Subject: Server: try to have a better video integrity --- server/controllers/api/remote/videos.js | 56 +++++++++++++++++++++++++++------ 1 file changed, 47 insertions(+), 9 deletions(-) (limited to 'server/controllers/api/remote') diff --git a/server/controllers/api/remote/videos.js b/server/controllers/api/remote/videos.js index d02da4463..6d768eae8 100644 --- a/server/controllers/api/remote/videos.js +++ b/server/controllers/api/remote/videos.js @@ -10,6 +10,7 @@ const secureMiddleware = middlewares.secure const videosValidators = middlewares.validators.remote.videos const signatureValidators = middlewares.validators.remote.signature const logger = require('../../../helpers/logger') +const utils = require('../../../helpers/utils') const router = express.Router() @@ -37,11 +38,11 @@ function remoteVideos (req, res, next) { switch (request.type) { case 'add': - addRemoteVideo(data, fromPod, callbackEach) + addRemoteVideoRetryWrapper(data, fromPod, callbackEach) break case 'update': - updateRemoteVideo(data, fromPod, callbackEach) + updateRemoteVideoRetryWrapper(data, fromPod, callbackEach) break case 'remove': @@ -63,13 +64,30 @@ function remoteVideos (req, res, next) { return res.type('json').status(204).end() } +// Handle retries on fail +function addRemoteVideoRetryWrapper (videoToCreateData, fromPod, finalCallback) { + utils.transactionRetryer( + function (callback) { + return addRemoteVideo(videoToCreateData, fromPod, callback) + }, + function (err) { + if (err) { + logger.error('Cannot insert the remote video with many retries.', { error: err }) + return finalCallback(err) + } + + return finalCallback() + } + ) +} + function addRemoteVideo (videoToCreateData, fromPod, finalCallback) { - logger.debug('Adding remote video "%s".', videoToCreateData.name) + logger.debug('Adding remote video "%s".', videoToCreateData.remoteId) waterfall([ function startTransaction (callback) { - db.sequelize.transaction().asCallback(function (err, t) { + db.sequelize.transaction({ isolationLevel: 'SERIALIZABLE' }).asCallback(function (err, t) { return callback(err, t) }) }, @@ -103,6 +121,7 @@ function addRemoteVideo (videoToCreateData, fromPod, finalCallback) { authorId: author.id, duration: videoToCreateData.duration, createdAt: videoToCreateData.createdAt, + // FIXME: updatedAt does not seems to be considered by Sequelize updatedAt: videoToCreateData.updatedAt } @@ -142,7 +161,8 @@ function addRemoteVideo (videoToCreateData, fromPod, finalCallback) { ], function (err, t) { if (err) { - logger.error('Cannot insert the remote video.') + // This is just a debug because we will retry the insert + logger.debug('Cannot insert the remote video.', { error: err }) // Abort transaction? if (t) t.rollback() @@ -157,8 +177,25 @@ function addRemoteVideo (videoToCreateData, fromPod, finalCallback) { }) } +// Handle retries on fail +function updateRemoteVideoRetryWrapper (videoAttributesToUpdate, fromPod, finalCallback) { + utils.transactionRetryer( + function (callback) { + return updateRemoteVideo(videoAttributesToUpdate, fromPod, callback) + }, + function (err) { + if (err) { + logger.error('Cannot update the remote video with many retries.', { error: err }) + return finalCallback(err) + } + + return finalCallback() + } + ) +} + function updateRemoteVideo (videoAttributesToUpdate, fromPod, finalCallback) { - logger.debug('Updating remote video "%s".', videoAttributesToUpdate.name) + logger.debug('Updating remote video "%s".', videoAttributesToUpdate.remoteId) waterfall([ @@ -208,7 +245,8 @@ function updateRemoteVideo (videoAttributesToUpdate, fromPod, finalCallback) { ], function (err, t) { if (err) { - logger.error('Cannot update the remote video.') + // This is just a debug because we will retry the insert + logger.debug('Cannot update the remote video.', { error: err }) // Abort transaction? if (t) t.rollback() @@ -238,7 +276,7 @@ function reportAbuseRemoteVideo (reportData, fromPod, callback) { if (err || !video) { if (!err) err = new Error('video not found') - logger.error('Cannot load video from host and remote id.', { error: err }) + logger.error('Cannot load video from id.', { error: err, id: reportData.videoRemoteId }) return callback(err) } @@ -260,7 +298,7 @@ function fetchVideo (podHost, remoteId, callback) { if (err || !video) { if (!err) err = new Error('video not found') - logger.error('Cannot load video from host and remote id.', { error: err }) + logger.error('Cannot load video from host and remote id.', { error: err, podHost, remoteId }) return callback(err) } -- cgit v1.2.3 From dea32aacde362a5fbd62a88cd32487768b788468 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Wed, 11 Jan 2017 16:22:50 +0100 Subject: Server: always check commit result --- server/controllers/api/remote/videos.js | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) (limited to 'server/controllers/api/remote') diff --git a/server/controllers/api/remote/videos.js b/server/controllers/api/remote/videos.js index 6d768eae8..17bdce019 100644 --- a/server/controllers/api/remote/videos.js +++ b/server/controllers/api/remote/videos.js @@ -171,9 +171,12 @@ function addRemoteVideo (videoToCreateData, fromPod, finalCallback) { } // Commit transaction - t.commit() + t.commit().asCallback(function (err) { + if (err) return finalCallback(err) - return finalCallback() + logger.info('Remote video %s inserted.', videoToCreateData.videoToCreateData.name) + return finalCallback(null) + }) }) } @@ -254,10 +257,13 @@ function updateRemoteVideo (videoAttributesToUpdate, fromPod, finalCallback) { return finalCallback(err) } - // Commit transaction - t.commit() + // Commit transaction + t.commit().asCallback(function (err) { + if (err) return finalCallback(err) - return finalCallback() + logger.info('Remote video %s updated', videoAttributesToUpdate.name) + return finalCallback(null) + }) }) } -- cgit v1.2.3 From d8cc063e9775688a1631eda9203411a2dba0333c Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Wed, 11 Jan 2017 18:06:51 +0100 Subject: Server: do not break remote videos processing on error --- server/controllers/api/remote/videos.js | 33 ++++++++++++++++++++++++--------- 1 file changed, 24 insertions(+), 9 deletions(-) (limited to 'server/controllers/api/remote') diff --git a/server/controllers/api/remote/videos.js b/server/controllers/api/remote/videos.js index 17bdce019..b9494f602 100644 --- a/server/controllers/api/remote/videos.js +++ b/server/controllers/api/remote/videos.js @@ -73,10 +73,10 @@ function addRemoteVideoRetryWrapper (videoToCreateData, fromPod, finalCallback) function (err) { if (err) { logger.error('Cannot insert the remote video with many retries.', { error: err }) - return finalCallback(err) } - return finalCallback() + // Do not return the error, continue the process + return finalCallback(null) } ) } @@ -174,7 +174,7 @@ function addRemoteVideo (videoToCreateData, fromPod, finalCallback) { t.commit().asCallback(function (err) { if (err) return finalCallback(err) - logger.info('Remote video %s inserted.', videoToCreateData.videoToCreateData.name) + logger.info('Remote video %s inserted.', videoToCreateData.name) return finalCallback(null) }) }) @@ -189,10 +189,10 @@ function updateRemoteVideoRetryWrapper (videoAttributesToUpdate, fromPod, finalC function (err) { if (err) { logger.error('Cannot update the remote video with many retries.', { error: err }) - return finalCallback(err) } - return finalCallback() + // Do not return the error, continue the process + return finalCallback(null) } ) } @@ -270,10 +270,18 @@ function updateRemoteVideo (videoAttributesToUpdate, fromPod, finalCallback) { function removeRemoteVideo (videoToRemoveData, fromPod, callback) { // We need the instance because we have to remove some other stuffs (thumbnail etc) fetchVideo(fromPod.host, videoToRemoveData.remoteId, function (err, video) { - if (err) return callback(err) + // Do not return the error, continue the process + if (err) return callback(null) logger.debug('Removing remote video %s.', video.remoteId) - video.destroy().asCallback(callback) + video.destroy().asCallback(function (err) { + // Do not return the error, continue the process + if (err) { + logger.error('Cannot remove remote video with id %s.', videoToRemoveData.remoteId, { error: err }) + } + + return callback(null) + }) }) } @@ -283,7 +291,8 @@ function reportAbuseRemoteVideo (reportData, fromPod, callback) { if (!err) err = new Error('video not found') logger.error('Cannot load video from id.', { error: err, id: reportData.videoRemoteId }) - return callback(err) + // Do not return the error, continue the process + return callback(null) } logger.debug('Reporting remote abuse for video %s.', video.id) @@ -295,7 +304,13 @@ function reportAbuseRemoteVideo (reportData, fromPod, callback) { videoId: video.id } - db.VideoAbuse.create(videoAbuseData).asCallback(callback) + db.VideoAbuse.create(videoAbuseData).asCallback(function (err) { + if (err) { + logger.error('Cannot create remote abuse video.', { error: err }) + } + + return callback(null) + }) }) } -- cgit v1.2.3 From edc5e86006bf5e4a2819c380bb65734fe9caa87e Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Wed, 11 Jan 2017 18:41:40 +0100 Subject: Server: transaction serializable for videos --- server/controllers/api/remote/videos.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'server/controllers/api/remote') diff --git a/server/controllers/api/remote/videos.js b/server/controllers/api/remote/videos.js index b9494f602..c45a86dbb 100644 --- a/server/controllers/api/remote/videos.js +++ b/server/controllers/api/remote/videos.js @@ -203,7 +203,7 @@ function updateRemoteVideo (videoAttributesToUpdate, fromPod, finalCallback) { waterfall([ function startTransaction (callback) { - db.sequelize.transaction().asCallback(function (err, t) { + db.sequelize.transaction({ isolationLevel: 'SERIALIZABLE' }).asCallback(function (err, t) { return callback(err, t) }) }, -- cgit v1.2.3