From feb4bdfd9b46e87aadfa7c0d5338cde887d1f58c Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Sun, 11 Dec 2016 21:50:51 +0100 Subject: First version with PostgreSQL --- server/controllers/api/clients.js | 8 +- server/controllers/api/pods.js | 15 +- server/controllers/api/remote.js | 95 +++++-- server/controllers/api/requests.js | 10 +- server/controllers/api/users.js | 28 +-- server/controllers/api/videos.js | 63 +++-- server/controllers/client.js | 11 +- server/helpers/custom-validators/videos.js | 19 +- server/helpers/logger.js | 6 +- server/initializers/checker.js | 8 +- server/initializers/constants.js | 19 +- server/initializers/database.js | 58 +++-- server/initializers/installer.js | 29 ++- server/initializers/migrator.js | 12 +- server/lib/friends.js | 73 ++++-- server/lib/oauth-model.js | 32 +-- server/middlewares/pods.js | 1 - server/middlewares/secure.js | 6 +- server/middlewares/sort.js | 4 +- server/middlewares/validators/users.js | 13 +- server/middlewares/validators/videos.js | 17 +- server/models/application.js | 45 ++-- server/models/author.js | 28 +++ server/models/oauth-client.js | 72 ++++-- server/models/oauth-token.js | 109 ++++++-- server/models/pods.js | 156 +++++++----- server/models/request.js | 187 ++++++++------ server/models/requestToPod.js | 30 +++ server/models/user.js | 132 ++++++---- server/models/utils.js | 31 +-- server/models/video.js | 388 ++++++++++++++++++++--------- server/tests/api/check-params.js | 6 +- server/tests/api/friends-basic.js | 4 +- server/tests/api/multiple-pods.js | 8 +- server/tests/api/requests.js | 29 +-- server/tests/api/single-pod.js | 16 +- server/tests/api/users.js | 8 +- server/tests/utils/servers.js | 4 +- server/tests/utils/videos.js | 2 +- 39 files changed, 1113 insertions(+), 669 deletions(-) create mode 100644 server/models/author.js create mode 100644 server/models/requestToPod.js (limited to 'server') diff --git a/server/controllers/api/clients.js b/server/controllers/api/clients.js index 7755f6c2b..cf83cb835 100644 --- a/server/controllers/api/clients.js +++ b/server/controllers/api/clients.js @@ -1,13 +1,11 @@ 'use strict' const express = require('express') -const mongoose = require('mongoose') const constants = require('../../initializers/constants') +const db = require('../../initializers/database') const logger = require('../../helpers/logger') -const Client = mongoose.model('OAuthClient') - const router = express.Router() router.get('/local', getLocalClient) @@ -27,12 +25,12 @@ function getLocalClient (req, res, next) { return res.type('json').status(403).end() } - Client.loadFirstClient(function (err, client) { + db.OAuthClient.loadFirstClient(function (err, client) { if (err) return next(err) if (!client) return next(new Error('No client available.')) res.json({ - client_id: client._id, + client_id: client.clientId, client_secret: client.clientSecret }) }) diff --git a/server/controllers/api/pods.js b/server/controllers/api/pods.js index 7857fcee0..79f3f9d8d 100644 --- a/server/controllers/api/pods.js +++ b/server/controllers/api/pods.js @@ -1,9 +1,9 @@ 'use strict' const express = require('express') -const mongoose = require('mongoose') const waterfall = require('async/waterfall') +const db = require('../../initializers/database') const logger = require('../../helpers/logger') const friends = require('../../lib/friends') const middlewares = require('../../middlewares') @@ -15,7 +15,6 @@ const validators = middlewares.validators.pods const signatureValidator = middlewares.validators.remote.signature const router = express.Router() -const Pod = mongoose.model('Pod') router.get('/', listPods) router.post('/', @@ -53,15 +52,15 @@ function addPods (req, res, next) { waterfall([ function addPod (callback) { - const pod = new Pod(informations) - pod.save(function (err, podCreated) { + const pod = db.Pod.build(informations) + pod.save().asCallback(function (err, podCreated) { // Be sure about the number of parameters for the callback return callback(err, podCreated) }) }, function sendMyVideos (podCreated, callback) { - friends.sendOwnedVideosToPod(podCreated._id) + friends.sendOwnedVideosToPod(podCreated.id) callback(null) }, @@ -84,7 +83,7 @@ function addPods (req, res, next) { } function listPods (req, res, next) { - Pod.list(function (err, podsList) { + db.Pod.list(function (err, podsList) { if (err) return next(err) res.json(getFormatedPods(podsList)) @@ -111,11 +110,11 @@ function removePods (req, res, next) { waterfall([ function loadPod (callback) { - Pod.loadByHost(host, callback) + db.Pod.loadByHost(host, callback) }, function removePod (pod, callback) { - pod.remove(callback) + pod.destroy().asCallback(callback) } ], function (err) { if (err) return next(err) diff --git a/server/controllers/api/remote.js b/server/controllers/api/remote.js index f1046c534..d856576a9 100644 --- a/server/controllers/api/remote.js +++ b/server/controllers/api/remote.js @@ -3,15 +3,15 @@ const each = require('async/each') const eachSeries = require('async/eachSeries') const express = require('express') -const mongoose = require('mongoose') +const waterfall = require('async/waterfall') +const db = require('../../initializers/database') const middlewares = require('../../middlewares') const secureMiddleware = middlewares.secure const validators = middlewares.validators.remote const logger = require('../../helpers/logger') const router = express.Router() -const Video = mongoose.model('Video') router.post('/videos', validators.signature, @@ -53,34 +53,99 @@ function remoteVideos (req, res, next) { function addRemoteVideo (videoToCreateData, fromHost, callback) { logger.debug('Adding remote video "%s".', videoToCreateData.name) - const video = new Video(videoToCreateData) - video.podHost = fromHost - Video.generateThumbnailFromBase64(video, videoToCreateData.thumbnailBase64, function (err) { - if (err) { - logger.error('Cannot generate thumbnail from base 64 data.', { error: err }) - return callback(err) + waterfall([ + + function findOrCreatePod (callback) { + fromHost + + const query = { + where: { + host: fromHost + }, + defaults: { + host: fromHost + } + } + + db.Pod.findOrCreate(query).asCallback(function (err, result) { + // [ instance, wasCreated ] + return callback(err, result[0]) + }) + }, + + function findOrCreateAuthor (pod, callback) { + const username = videoToCreateData.author + + const query = { + where: { + name: username, + podId: pod.id + }, + defaults: { + name: username, + podId: pod.id + } + } + + db.Author.findOrCreate(query).asCallback(function (err, result) { + // [ instance, wasCreated ] + return callback(err, result[0]) + }) + }, + + function createVideoObject (author, callback) { + const videoData = { + name: videoToCreateData.name, + remoteId: videoToCreateData.remoteId, + extname: videoToCreateData.extname, + infoHash: videoToCreateData.infoHash, + description: videoToCreateData.description, + authorId: author.id, + duration: videoToCreateData.duration, + tags: videoToCreateData.tags + } + + const video = db.Video.build(videoData) + + return callback(null, video) + }, + + function generateThumbnail (video, callback) { + db.Video.generateThumbnailFromBase64(video, videoToCreateData.thumbnailBase64, function (err) { + if (err) { + logger.error('Cannot generate thumbnail from base 64 data.', { error: err }) + return callback(err) + } + + video.save().asCallback(callback) + }) + }, + + function insertIntoDB (video, callback) { + video.save().asCallback(callback) } - video.save(callback) - }) + ], callback) } function removeRemoteVideo (videoToRemoveData, fromHost, callback) { + // TODO: use bulkDestroy? + // We need the list because we have to remove some other stuffs (thumbnail etc) - Video.listByHostAndRemoteId(fromHost, videoToRemoveData.remoteId, function (err, videosList) { + db.Video.listByHostAndRemoteId(fromHost, videoToRemoveData.remoteId, function (err, videosList) { if (err) { - logger.error('Cannot list videos from host and magnets.', { error: err }) + logger.error('Cannot list videos from host and remote id.', { error: err.message }) return callback(err) } if (videosList.length === 0) { - logger.error('No remote video was found for this pod.', { magnetUri: videoToRemoveData.magnetUri, podHost: fromHost }) + logger.error('No remote video was found for this pod.', { remoteId: videoToRemoveData.remoteId, podHost: fromHost }) } each(videosList, function (video, callbackEach) { - logger.debug('Removing remote video %s.', video.magnetUri) + logger.debug('Removing remote video %s.', video.remoteId) - video.remove(callbackEach) + video.destroy().asCallback(callbackEach) }, callback) }) } diff --git a/server/controllers/api/requests.js b/server/controllers/api/requests.js index 52aad6997..1f9193fc8 100644 --- a/server/controllers/api/requests.js +++ b/server/controllers/api/requests.js @@ -1,15 +1,13 @@ 'use strict' const express = require('express') -const mongoose = require('mongoose') const constants = require('../../initializers/constants') +const db = require('../../initializers/database') const middlewares = require('../../middlewares') const admin = middlewares.admin const oAuth = middlewares.oauth -const Request = mongoose.model('Request') - const router = express.Router() router.get('/stats', @@ -25,13 +23,13 @@ module.exports = router // --------------------------------------------------------------------------- function getStatsRequests (req, res, next) { - Request.list(function (err, requests) { + db.Request.countTotalRequests(function (err, totalRequests) { if (err) return next(err) return res.json({ - requests: requests, + totalRequests: totalRequests, maxRequestsInParallel: constants.REQUESTS_IN_PARALLEL, - remainingMilliSeconds: Request.remainingMilliSeconds(), + remainingMilliSeconds: db.Request.remainingMilliSeconds(), milliSecondsInterval: constants.REQUESTS_INTERVAL }) }) diff --git a/server/controllers/api/users.js b/server/controllers/api/users.js index b4d687312..890028b36 100644 --- a/server/controllers/api/users.js +++ b/server/controllers/api/users.js @@ -2,10 +2,10 @@ const each = require('async/each') const express = require('express') -const mongoose = require('mongoose') const waterfall = require('async/waterfall') const constants = require('../../initializers/constants') +const db = require('../../initializers/database') const friends = require('../../lib/friends') const logger = require('../../helpers/logger') const middlewares = require('../../middlewares') @@ -17,9 +17,6 @@ const validatorsPagination = middlewares.validators.pagination const validatorsSort = middlewares.validators.sort const validatorsUsers = middlewares.validators.users -const User = mongoose.model('User') -const Video = mongoose.model('Video') - const router = express.Router() router.get('/me', oAuth.authenticate, getUserInformation) @@ -62,13 +59,13 @@ module.exports = router // --------------------------------------------------------------------------- function createUser (req, res, next) { - const user = new User({ + const user = db.User.build({ username: req.body.username, password: req.body.password, role: constants.USER_ROLES.USER }) - user.save(function (err, createdUser) { + user.save().asCallback(function (err, createdUser) { if (err) return next(err) return res.type('json').status(204).end() @@ -76,7 +73,7 @@ function createUser (req, res, next) { } function getUserInformation (req, res, next) { - User.loadByUsername(res.locals.oauth.token.user.username, function (err, user) { + db.User.loadByUsername(res.locals.oauth.token.user.username, function (err, user) { if (err) return next(err) return res.json(user.toFormatedJSON()) @@ -84,7 +81,7 @@ function getUserInformation (req, res, next) { } function listUsers (req, res, next) { - User.listForApi(req.query.start, req.query.count, req.query.sort, function (err, usersList, usersTotal) { + db.User.listForApi(req.query.start, req.query.count, req.query.sort, function (err, usersList, usersTotal) { if (err) return next(err) res.json(getFormatedUsers(usersList, usersTotal)) @@ -94,18 +91,19 @@ function listUsers (req, res, next) { function removeUser (req, res, next) { waterfall([ function getUser (callback) { - User.loadById(req.params.id, callback) + db.User.loadById(req.params.id, callback) }, + // TODO: use foreignkey? function getVideos (user, callback) { - Video.listOwnedByAuthor(user.username, function (err, videos) { + db.Video.listOwnedByAuthor(user.username, function (err, videos) { return callback(err, user, videos) }) }, function removeVideosFromDB (user, videos, callback) { each(videos, function (video, callbackEach) { - video.remove(callbackEach) + video.destroy().asCallback(callbackEach) }, function (err) { return callback(err, user, videos) }) @@ -115,7 +113,7 @@ function removeUser (req, res, next) { videos.forEach(function (video) { const params = { name: video.name, - magnetUri: video.magnetUri + remoteId: video.id } friends.removeVideoToFriends(params) @@ -125,7 +123,7 @@ function removeUser (req, res, next) { }, function removeUserFromDB (user, callback) { - user.remove(callback) + user.destroy().asCallback(callback) } ], function andFinally (err) { if (err) { @@ -138,11 +136,11 @@ function removeUser (req, res, next) { } function updateUser (req, res, next) { - User.loadByUsername(res.locals.oauth.token.user.username, function (err, user) { + db.User.loadByUsername(res.locals.oauth.token.user.username, function (err, user) { if (err) return next(err) user.password = req.body.password - user.save(function (err) { + user.save().asCallback(function (err) { if (err) return next(err) return res.sendStatus(204) diff --git a/server/controllers/api/videos.js b/server/controllers/api/videos.js index daf452573..a61f2b2c9 100644 --- a/server/controllers/api/videos.js +++ b/server/controllers/api/videos.js @@ -2,12 +2,12 @@ const express = require('express') const fs = require('fs') -const mongoose = require('mongoose') const multer = require('multer') const path = require('path') const waterfall = require('async/waterfall') const constants = require('../../initializers/constants') +const db = require('../../initializers/database') const logger = require('../../helpers/logger') const friends = require('../../lib/friends') const middlewares = require('../../middlewares') @@ -22,7 +22,6 @@ const sort = middlewares.sort const utils = require('../../helpers/utils') const router = express.Router() -const Video = mongoose.model('Video') // multer configuration const storage = multer.diskStorage({ @@ -87,40 +86,60 @@ function addVideo (req, res, next) { const videoInfos = req.body waterfall([ - function createVideoObject (callback) { - const id = mongoose.Types.ObjectId() + function findOrCreateAuthor (callback) { + const username = res.locals.oauth.token.user.username + + const query = { + where: { + name: username, + podId: null + }, + defaults: { + name: username, + podId: null // null because it is OUR pod + } + } + + db.Author.findOrCreate(query).asCallback(function (err, result) { + // [ instance, wasCreated ] + return callback(err, result[0]) + }) + }, + + function createVideoObject (author, callback) { const videoData = { - _id: id, name: videoInfos.name, remoteId: null, extname: path.extname(videoFile.filename), description: videoInfos.description, - author: res.locals.oauth.token.user.username, duration: videoFile.duration, - tags: videoInfos.tags + tags: videoInfos.tags, + authorId: author.id } - const video = new Video(videoData) + const video = db.Video.build(videoData) - return callback(null, video) + return callback(null, author, video) }, - // Set the videoname the same as the MongoDB id - function renameVideoFile (video, callback) { + // Set the videoname the same as the id + function renameVideoFile (author, video, callback) { const videoDir = constants.CONFIG.STORAGE.VIDEOS_DIR const source = path.join(videoDir, videoFile.filename) const destination = path.join(videoDir, video.getVideoFilename()) fs.rename(source, destination, function (err) { - return callback(err, video) + return callback(err, author, video) }) }, - function insertIntoDB (video, callback) { - video.save(function (err, video) { - // Assert there are only one argument sent to the next function (video) - return callback(err, video) + function insertIntoDB (author, video, callback) { + video.save().asCallback(function (err, videoCreated) { + // Do not forget to add Author informations to the created video + videoCreated.Author = author + + return callback(err, videoCreated) }) }, @@ -147,7 +166,7 @@ function addVideo (req, res, next) { } function getVideo (req, res, next) { - Video.load(req.params.id, function (err, video) { + db.Video.loadAndPopulateAuthorAndPod(req.params.id, function (err, video) { if (err) return next(err) if (!video) { @@ -159,7 +178,7 @@ function getVideo (req, res, next) { } function listVideos (req, res, next) { - Video.listForApi(req.query.start, req.query.count, req.query.sort, function (err, videosList, videosTotal) { + db.Video.listForApi(req.query.start, req.query.count, req.query.sort, function (err, videosList, videosTotal) { if (err) return next(err) res.json(getFormatedVideos(videosList, videosTotal)) @@ -171,11 +190,11 @@ function removeVideo (req, res, next) { waterfall([ function getVideo (callback) { - Video.load(videoId, callback) + db.Video.load(videoId, callback) }, function removeFromDB (video, callback) { - video.remove(function (err) { + video.destroy().asCallback(function (err) { if (err) return callback(err) return callback(null, video) @@ -185,7 +204,7 @@ function removeVideo (req, res, next) { function sendInformationToFriends (video, callback) { const params = { name: video.name, - remoteId: video._id + remoteId: video.id } friends.removeVideoToFriends(params) @@ -203,7 +222,7 @@ function removeVideo (req, res, next) { } function searchVideos (req, res, next) { - Video.search(req.params.value, req.query.field, req.query.start, req.query.count, req.query.sort, + db.Video.searchAndPopulateAuthorAndPod(req.params.value, req.query.field, req.query.start, req.query.count, req.query.sort, function (err, videosList, videosTotal) { if (err) return next(err) diff --git a/server/controllers/client.js b/server/controllers/client.js index 572db6133..a5fac5626 100644 --- a/server/controllers/client.js +++ b/server/controllers/client.js @@ -3,13 +3,12 @@ const parallel = require('async/parallel') const express = require('express') const fs = require('fs') -const mongoose = require('mongoose') const path = require('path') const validator = require('express-validator').validator const constants = require('../initializers/constants') +const db = require('../initializers/database') -const Video = mongoose.model('Video') const router = express.Router() const opengraphComment = '' @@ -45,14 +44,14 @@ function addOpenGraphTags (htmlStringPage, video) { if (video.isOwned()) { basePreviewUrlHttp = constants.CONFIG.WEBSERVER.URL } else { - basePreviewUrlHttp = constants.REMOTE_SCHEME.HTTP + '://' + video.podHost + basePreviewUrlHttp = constants.REMOTE_SCHEME.HTTP + '://' + video.Author.Pod.host } // We fetch the remote preview (bigger than the thumbnail) // This should not overhead the remote server since social websites put in a cache the OpenGraph tags // We can't use the thumbnail because these social websites want bigger images (> 200x200 for Facebook for example) const previewUrl = basePreviewUrlHttp + constants.STATIC_PATHS.PREVIEWS + video.getPreviewName() - const videoUrl = constants.CONFIG.WEBSERVER.URL + '/videos/watch/' + video._id + const videoUrl = constants.CONFIG.WEBSERVER.URL + '/videos/watch/' + video.id const metaTags = { 'og:type': 'video', @@ -86,7 +85,7 @@ function generateWatchHtmlPage (req, res, next) { const videoId = req.params.id // Let Angular application handle errors - if (!validator.isMongoId(videoId)) return res.sendFile(indexPath) + if (!validator.isUUID(videoId, 4)) return res.sendFile(indexPath) parallel({ file: function (callback) { @@ -94,7 +93,7 @@ function generateWatchHtmlPage (req, res, next) { }, video: function (callback) { - Video.load(videoId, callback) + db.Video.loadAndPopulateAuthorAndPod(videoId, callback) } }, function (err, results) { if (err) return next(err) diff --git a/server/helpers/custom-validators/videos.js b/server/helpers/custom-validators/videos.js index 1a7753265..be8256a80 100644 --- a/server/helpers/custom-validators/videos.js +++ b/server/helpers/custom-validators/videos.js @@ -13,7 +13,7 @@ const videosValidators = { isVideoDateValid, isVideoDescriptionValid, isVideoDurationValid, - isVideoMagnetValid, + isVideoInfoHashValid, isVideoNameValid, isVideoPodHostValid, isVideoTagsValid, @@ -28,14 +28,15 @@ function isEachRemoteVideosValid (requests) { return ( isRequestTypeAddValid(request.type) && isVideoAuthorValid(video.author) && - isVideoDateValid(video.createdDate) && + isVideoDateValid(video.createdAt) && isVideoDescriptionValid(video.description) && isVideoDurationValid(video.duration) && - isVideoMagnetValid(video.magnet) && + isVideoInfoHashValid(video.infoHash) && isVideoNameValid(video.name) && isVideoTagsValid(video.tags) && isVideoThumbnail64Valid(video.thumbnailBase64) && - isVideoRemoteIdValid(video.remoteId) + isVideoRemoteIdValid(video.remoteId) && + isVideoExtnameValid(video.extname) ) || ( isRequestTypeRemoveValid(request.type) && @@ -61,8 +62,12 @@ function isVideoDurationValid (value) { return validator.isInt(value + '', VIDEOS_CONSTRAINTS_FIELDS.DURATION) } -function isVideoMagnetValid (value) { - return validator.isLength(value.infoHash, VIDEOS_CONSTRAINTS_FIELDS.MAGNET.INFO_HASH) +function isVideoExtnameValid (value) { + return VIDEOS_CONSTRAINTS_FIELDS.EXTNAME.indexOf(value) !== -1 +} + +function isVideoInfoHashValid (value) { + return validator.isLength(value, VIDEOS_CONSTRAINTS_FIELDS.INFO_HASH) } function isVideoNameValid (value) { @@ -93,7 +98,7 @@ function isVideoThumbnail64Valid (value) { } function isVideoRemoteIdValid (value) { - return validator.isMongoId(value) + return validator.isUUID(value, 4) } // --------------------------------------------------------------------------- diff --git a/server/helpers/logger.js b/server/helpers/logger.js index fcc1789fd..281acedb8 100644 --- a/server/helpers/logger.js +++ b/server/helpers/logger.js @@ -22,7 +22,8 @@ const logger = new winston.Logger({ json: true, maxsize: 5242880, maxFiles: 5, - colorize: false + colorize: false, + prettyPrint: true }), new winston.transports.Console({ level: 'debug', @@ -30,7 +31,8 @@ const logger = new winston.Logger({ handleExceptions: true, humanReadableUnhandledException: true, json: false, - colorize: true + colorize: true, + prettyPrint: true }) ], exitOnError: true diff --git a/server/initializers/checker.js b/server/initializers/checker.js index aea013fa9..7b402de82 100644 --- a/server/initializers/checker.js +++ b/server/initializers/checker.js @@ -1,10 +1,8 @@ 'use strict' const config = require('config') -const mongoose = require('mongoose') -const Client = mongoose.model('OAuthClient') -const User = mongoose.model('User') +const db = require('./database') const checker = { checkConfig, @@ -44,7 +42,7 @@ function checkMissedConfig () { } function clientsExist (callback) { - Client.list(function (err, clients) { + db.OAuthClient.list(function (err, clients) { if (err) return callback(err) return callback(null, clients.length !== 0) @@ -52,7 +50,7 @@ function clientsExist (callback) { } function usersExist (callback) { - User.countTotal(function (err, totalUsers) { + db.User.countTotal(function (err, totalUsers) { if (err) return callback(err) return callback(null, totalUsers !== 0) diff --git a/server/initializers/constants.js b/server/initializers/constants.js index 3ddf87454..1ad0c82a0 100644 --- a/server/initializers/constants.js +++ b/server/initializers/constants.js @@ -14,13 +14,13 @@ const PAGINATION_COUNT_DEFAULT = 15 // Sortable columns per schema const SEARCHABLE_COLUMNS = { - VIDEOS: [ 'name', 'magnetUri', 'podHost', 'author', 'tags' ] + VIDEOS: [ 'name', 'magnetUri', 'host', 'author', 'tags' ] } // Sortable columns per schema const SORTABLE_COLUMNS = { - USERS: [ 'username', '-username', 'createdDate', '-createdDate' ], - VIDEOS: [ 'name', '-name', 'duration', '-duration', 'createdDate', '-createdDate' ] + USERS: [ 'username', '-username', 'createdAt', '-createdAt' ], + VIDEOS: [ 'name', '-name', 'duration', '-duration', 'createdAt', '-createdAt' ] } const OAUTH_LIFETIME = { @@ -67,9 +67,8 @@ const CONSTRAINTS_FIELDS = { VIDEOS: { NAME: { min: 3, max: 50 }, // Length DESCRIPTION: { min: 3, max: 250 }, // Length - MAGNET: { - INFO_HASH: { min: 10, max: 50 } // Length - }, + EXTNAME: [ '.mp4', '.ogv', '.webm' ], + INFO_HASH: { min: 10, max: 50 }, // Length DURATION: { min: 1, max: 7200 }, // Number TAGS: { min: 1, max: 3 }, // Number of total tags TAG: { min: 2, max: 10 }, // Length @@ -88,7 +87,7 @@ const FRIEND_SCORE = { // --------------------------------------------------------------------------- -const MONGO_MIGRATION_SCRIPTS = [ +const MIGRATION_SCRIPTS = [ { script: '0005-create-application', version: 5 @@ -122,7 +121,7 @@ const MONGO_MIGRATION_SCRIPTS = [ version: 40 } ] -const LAST_MONGO_SCHEMA_VERSION = (maxBy(MONGO_MIGRATION_SCRIPTS, 'version'))['version'] +const LAST_SQL_SCHEMA_VERSION = (maxBy(MIGRATION_SCRIPTS, 'version'))['version'] // --------------------------------------------------------------------------- @@ -198,8 +197,8 @@ module.exports = { CONFIG, CONSTRAINTS_FIELDS, FRIEND_SCORE, - LAST_MONGO_SCHEMA_VERSION, - MONGO_MIGRATION_SCRIPTS, + LAST_SQL_SCHEMA_VERSION, + MIGRATION_SCRIPTS, OAUTH_LIFETIME, PAGINATION_COUNT_DEFAULT, PODS_SCORE, diff --git a/server/initializers/database.js b/server/initializers/database.js index 0564e4e77..cc6f59b63 100644 --- a/server/initializers/database.js +++ b/server/initializers/database.js @@ -1,36 +1,46 @@ 'use strict' -const mongoose = require('mongoose') +const fs = require('fs') +const path = require('path') +const Sequelize = require('sequelize') const constants = require('../initializers/constants') const logger = require('../helpers/logger') -// Bootstrap models -require('../models/application') -require('../models/oauth-token') -require('../models/user') -require('../models/oauth-client') -require('../models/video') -// Request model needs Video model -require('../models/pods') -// Request model needs Pod model -require('../models/request') - -const database = { - connect: connect -} - -function connect () { - mongoose.Promise = global.Promise - mongoose.connect('mongodb://' + constants.CONFIG.DATABASE.HOSTNAME + ':' + constants.CONFIG.DATABASE.PORT + '/' + constants.CONFIG.DATABASE.DBNAME) - mongoose.connection.on('error', function () { - throw new Error('Mongodb connection error.') +const database = {} + +const sequelize = new Sequelize(constants.CONFIG.DATABASE.DBNAME, 'peertube', 'peertube', { + dialect: 'postgres', + host: constants.CONFIG.DATABASE.HOSTNAME, + port: constants.CONFIG.DATABASE.PORT +}) + +const modelDirectory = path.join(__dirname, '..', 'models') +fs.readdir(modelDirectory, function (err, files) { + if (err) throw err + + files.filter(function (file) { + if (file === 'utils.js') return false + + return true }) + .forEach(function (file) { + const model = sequelize.import(path.join(modelDirectory, file)) - mongoose.connection.on('open', function () { - logger.info('Connected to mongodb.') + database[model.name] = model }) -} + + Object.keys(database).forEach(function (modelName) { + if ('associate' in database[modelName]) { + database[modelName].associate(database) + } + }) + + logger.info('Database is ready.') +}) + +database.sequelize = sequelize +database.Sequelize = Sequelize // --------------------------------------------------------------------------- diff --git a/server/initializers/installer.js b/server/initializers/installer.js index 1df300ba8..4823bc8c8 100644 --- a/server/initializers/installer.js +++ b/server/initializers/installer.js @@ -3,26 +3,27 @@ const config = require('config') const each = require('async/each') const mkdirp = require('mkdirp') -const mongoose = require('mongoose') const passwordGenerator = require('password-generator') const path = require('path') const series = require('async/series') const checker = require('./checker') const constants = require('./constants') +const db = require('./database') const logger = require('../helpers/logger') const peertubeCrypto = require('../helpers/peertube-crypto') -const Application = mongoose.model('Application') -const Client = mongoose.model('OAuthClient') -const User = mongoose.model('User') - const installer = { installApplication } function installApplication (callback) { series([ + function createDatabase (callbackAsync) { + db.sequelize.sync().asCallback(callbackAsync) + // db.sequelize.sync({ force: true }).asCallback(callbackAsync) + }, + function createDirectories (callbackAsync) { createDirectoriesIfNotExist(callbackAsync) }, @@ -65,16 +66,18 @@ function createOAuthClientIfNotExist (callback) { logger.info('Creating a default OAuth Client.') - const secret = passwordGenerator(32, false) - const client = new Client({ + const id = passwordGenerator(32, false, /[a-z0-9]/) + const secret = passwordGenerator(32, false, /[a-zA-Z0-9]/) + const client = db.OAuthClient.build({ + clientId: id, clientSecret: secret, grants: [ 'password', 'refresh_token' ] }) - client.save(function (err, createdClient) { + client.save().asCallback(function (err, createdClient) { if (err) return callback(err) - logger.info('Client id: ' + createdClient._id) + logger.info('Client id: ' + createdClient.clientId) logger.info('Client secret: ' + createdClient.clientSecret) return callback(null) @@ -106,21 +109,21 @@ function createOAuthAdminIfNotExist (callback) { password = passwordGenerator(8, true) } - const user = new User({ + const user = db.User.build({ username, password, role }) - user.save(function (err, createdUser) { + user.save().asCallback(function (err, createdUser) { if (err) return callback(err) logger.info('Username: ' + username) logger.info('User password: ' + password) logger.info('Creating Application collection.') - const application = new Application({ mongoSchemaVersion: constants.LAST_MONGO_SCHEMA_VERSION }) - application.save(callback) + const application = db.Application.build({ sqlSchemaVersion: constants.LAST_SQL_SCHEMA_VERSION }) + application.save().asCallback(callback) }) }) } diff --git a/server/initializers/migrator.js b/server/initializers/migrator.js index 6b31d994f..9e5350e60 100644 --- a/server/initializers/migrator.js +++ b/server/initializers/migrator.js @@ -1,24 +1,22 @@ 'use strict' const eachSeries = require('async/eachSeries') -const mongoose = require('mongoose') const path = require('path') const constants = require('./constants') +const db = require('./database') const logger = require('../helpers/logger') -const Application = mongoose.model('Application') - const migrator = { migrate: migrate } function migrate (callback) { - Application.loadMongoSchemaVersion(function (err, actualVersion) { + db.Application.loadSqlSchemaVersion(function (err, actualVersion) { if (err) return callback(err) // If there are a new mongo schemas - if (!actualVersion || actualVersion < constants.LAST_MONGO_SCHEMA_VERSION) { + if (!actualVersion || actualVersion < constants.LAST_SQL_SCHEMA_VERSION) { logger.info('Begin migrations.') eachSeries(constants.MONGO_MIGRATION_SCRIPTS, function (entity, callbackEach) { @@ -36,12 +34,12 @@ function migrate (callback) { if (err) return callbackEach(err) // Update the new mongo version schema - Application.updateMongoSchemaVersion(versionScript, callbackEach) + db.Application.updateSqlSchemaVersion(versionScript, callbackEach) }) }, function (err) { if (err) return callback(err) - logger.info('Migrations finished. New mongo version schema: %s', constants.LAST_MONGO_SCHEMA_VERSION) + logger.info('Migrations finished. New SQL version schema: %s', constants.LAST_SQL_SCHEMA_VERSION) return callback(null) }) } else { diff --git a/server/lib/friends.js b/server/lib/friends.js index eaea040ca..3ed29f651 100644 --- a/server/lib/friends.js +++ b/server/lib/friends.js @@ -4,18 +4,14 @@ const each = require('async/each') const eachLimit = require('async/eachLimit') const eachSeries = require('async/eachSeries') const fs = require('fs') -const mongoose = require('mongoose') const request = require('request') const waterfall = require('async/waterfall') const constants = require('../initializers/constants') +const db = require('../initializers/database') const logger = require('../helpers/logger') const requests = require('../helpers/requests') -const Pod = mongoose.model('Pod') -const Request = mongoose.model('Request') -const Video = mongoose.model('Video') - const friends = { addVideoToFriends, hasFriends, @@ -31,7 +27,7 @@ function addVideoToFriends (video) { } function hasFriends (callback) { - Pod.countAll(function (err, count) { + db.Pod.countAll(function (err, count) { if (err) return callback(err) const hasFriends = (count !== 0) @@ -69,13 +65,13 @@ function makeFriends (hosts, callback) { function quitFriends (callback) { // Stop pool requests - Request.deactivate() + db.Request.deactivate() // Flush pool requests - Request.flush() + db.Request.flush() waterfall([ function getPodsList (callbackAsync) { - return Pod.list(callbackAsync) + return db.Pod.list(callbackAsync) }, function announceIQuitMyFriends (pods, callbackAsync) { @@ -103,12 +99,12 @@ function quitFriends (callback) { function removePodsFromDB (pods, callbackAsync) { each(pods, function (pod, callbackEach) { - pod.remove(callbackEach) + pod.destroy().asCallback(callbackEach) }, callbackAsync) } ], function (err) { // Don't forget to re activate the scheduler, even if there was an error - Request.activate() + db.Request.activate() if (err) return callback(err) @@ -122,7 +118,7 @@ function removeVideoToFriends (videoParams) { } function sendOwnedVideosToPod (podId) { - Video.listOwned(function (err, videosList) { + db.Video.listOwnedAndPopulateAuthor(function (err, videosList) { if (err) { logger.error('Cannot get the list of videos we own.') return @@ -200,9 +196,9 @@ function getForeignPodsList (host, callback) { function makeRequestsToWinningPods (cert, podsList, callback) { // Stop pool requests - Request.deactivate() + db.Request.deactivate() // Flush pool requests - Request.forceSend() + db.Request.forceSend() eachLimit(podsList, constants.REQUESTS_IN_PARALLEL, function (pod, callbackEach) { const params = { @@ -222,8 +218,8 @@ function makeRequestsToWinningPods (cert, podsList, callback) { } if (res.statusCode === 200) { - const podObj = new Pod({ host: pod.host, publicKey: body.cert }) - podObj.save(function (err, podCreated) { + const podObj = db.Pod.build({ host: pod.host, publicKey: body.cert }) + podObj.save().asCallback(function (err, podCreated) { if (err) { logger.error('Cannot add friend %s pod.', pod.host, { error: err }) return callbackEach() @@ -242,28 +238,57 @@ function makeRequestsToWinningPods (cert, podsList, callback) { }, function endRequests () { // Final callback, we've ended all the requests // Now we made new friends, we can re activate the pool of requests - Request.activate() + db.Request.activate() logger.debug('makeRequestsToWinningPods finished.') return callback() }) } +// Wrapper that populate "to" argument with all our friends if it is not specified function createRequest (type, endpoint, data, to) { - const req = new Request({ + if (to) return _createRequest(type, endpoint, data, to) + + // If the "to" pods is not specified, we send the request to all our friends + db.Pod.listAllIds(function (err, podIds) { + if (err) { + logger.error('Cannot get pod ids', { error: err }) + return + } + + return _createRequest(type, endpoint, data, podIds) + }) +} + +function _createRequest (type, endpoint, data, to) { + const pods = [] + + // If there are no destination pods abort + if (to.length === 0) return + + to.forEach(function (toPod) { + pods.push(db.Pod.build({ id: toPod })) + }) + + const createQuery = { endpoint, request: { type: type, data: data } - }) - - if (to) { - req.to = to } - req.save(function (err) { - if (err) logger.error('Cannot save the request.', { error: err }) + // We run in transaction to keep coherency between Request and RequestToPod tables + db.sequelize.transaction(function (t) { + const dbRequestOptions = { + transaction: t + } + + return db.Request.create(createQuery, dbRequestOptions).then(function (request) { + return request.setPods(pods, dbRequestOptions) + }) + }).asCallback(function (err) { + if (err) logger.error('Error in createRequest transaction.', { error: err }) }) } diff --git a/server/lib/oauth-model.js b/server/lib/oauth-model.js index d011c4b72..1c12f1b14 100644 --- a/server/lib/oauth-model.js +++ b/server/lib/oauth-model.js @@ -1,11 +1,6 @@ -const mongoose = require('mongoose') - +const db = require('../initializers/database') const logger = require('../helpers/logger') -const OAuthClient = mongoose.model('OAuthClient') -const OAuthToken = mongoose.model('OAuthToken') -const User = mongoose.model('User') - // See https://github.com/oauthjs/node-oauth2-server/wiki/Model-specification for the model specifications const OAuthModel = { getAccessToken, @@ -21,27 +16,25 @@ const OAuthModel = { function getAccessToken (bearerToken) { logger.debug('Getting access token (bearerToken: ' + bearerToken + ').') - return OAuthToken.getByTokenAndPopulateUser(bearerToken) + return db.OAuthToken.getByTokenAndPopulateUser(bearerToken) } function getClient (clientId, clientSecret) { logger.debug('Getting Client (clientId: ' + clientId + ', clientSecret: ' + clientSecret + ').') - // TODO req validator - const mongoId = new mongoose.mongo.ObjectID(clientId) - return OAuthClient.getByIdAndSecret(mongoId, clientSecret) + return db.OAuthClient.getByIdAndSecret(clientId, clientSecret) } function getRefreshToken (refreshToken) { logger.debug('Getting RefreshToken (refreshToken: ' + refreshToken + ').') - return OAuthToken.getByRefreshTokenAndPopulateClient(refreshToken) + return db.OAuthToken.getByRefreshTokenAndPopulateClient(refreshToken) } function getUser (username, password) { logger.debug('Getting User (username: ' + username + ', password: ' + password + ').') - return User.getByUsername(username).then(function (user) { + return db.User.getByUsername(username).then(function (user) { if (!user) return null // We need to return a promise @@ -60,8 +53,8 @@ function getUser (username, password) { } function revokeToken (token) { - return OAuthToken.getByRefreshTokenAndPopulateUser(token.refreshToken).then(function (tokenDB) { - if (tokenDB) tokenDB.remove() + return db.OAuthToken.getByRefreshTokenAndPopulateUser(token.refreshToken).then(function (tokenDB) { + if (tokenDB) tokenDB.destroy() /* * Thanks to https://github.com/manjeshpv/node-oauth2-server-implementation/blob/master/components/oauth/mongo-models.js @@ -80,18 +73,19 @@ function revokeToken (token) { function saveToken (token, client, user) { logger.debug('Saving token ' + token.accessToken + ' for client ' + client.id + ' and user ' + user.id + '.') - const tokenObj = new OAuthToken({ + const tokenToCreate = { accessToken: token.accessToken, accessTokenExpiresAt: token.accessTokenExpiresAt, - client: client.id, refreshToken: token.refreshToken, refreshTokenExpiresAt: token.refreshTokenExpiresAt, - user: user.id - }) + oAuthClientId: client.id, + userId: user.id + } - return tokenObj.save().then(function (tokenCreated) { + return db.OAuthToken.create(tokenToCreate).then(function (tokenCreated) { tokenCreated.client = client tokenCreated.user = user + return tokenCreated }).catch(function (err) { throw err diff --git a/server/middlewares/pods.js b/server/middlewares/pods.js index 487ea1259..e38fb341d 100644 --- a/server/middlewares/pods.js +++ b/server/middlewares/pods.js @@ -44,7 +44,6 @@ module.exports = podsMiddleware function getHostWithPort (host) { const splitted = host.split(':') - console.log(splitted) // The port was not specified if (splitted.length === 1) { if (constants.REMOTE_SCHEME.HTTP === 'https') return host + ':443' diff --git a/server/middlewares/secure.js b/server/middlewares/secure.js index ee836beed..b7b4cdfb4 100644 --- a/server/middlewares/secure.js +++ b/server/middlewares/secure.js @@ -1,18 +1,16 @@ 'use strict' +const db = require('../initializers/database') const logger = require('../helpers/logger') -const mongoose = require('mongoose') const peertubeCrypto = require('../helpers/peertube-crypto') -const Pod = mongoose.model('Pod') - const secureMiddleware = { checkSignature } function checkSignature (req, res, next) { const host = req.body.signature.host - Pod.loadByHost(host, function (err, pod) { + db.Pod.loadByHost(host, function (err, pod) { if (err) { logger.error('Cannot get signed host in body.', { error: err }) return res.sendStatus(500) diff --git a/server/middlewares/sort.js b/server/middlewares/sort.js index f0b7274eb..477e10571 100644 --- a/server/middlewares/sort.js +++ b/server/middlewares/sort.js @@ -6,13 +6,13 @@ const sortMiddleware = { } function setUsersSort (req, res, next) { - if (!req.query.sort) req.query.sort = '-createdDate' + if (!req.query.sort) req.query.sort = '-createdAt' return next() } function setVideosSort (req, res, next) { - if (!req.query.sort) req.query.sort = '-createdDate' + if (!req.query.sort) req.query.sort = '-createdAt' return next() } diff --git a/server/middlewares/validators/users.js b/server/middlewares/validators/users.js index 02e4f34cb..0629550bc 100644 --- a/server/middlewares/validators/users.js +++ b/server/middlewares/validators/users.js @@ -1,12 +1,9 @@ 'use strict' -const mongoose = require('mongoose') - const checkErrors = require('./utils').checkErrors +const db = require('../../initializers/database') const logger = require('../../helpers/logger') -const User = mongoose.model('User') - const validatorsUsers = { usersAdd, usersRemove, @@ -20,7 +17,7 @@ function usersAdd (req, res, next) { logger.debug('Checking usersAdd parameters', { parameters: req.body }) checkErrors(req, res, function () { - User.loadByUsername(req.body.username, function (err, user) { + db.User.loadByUsername(req.body.username, function (err, user) { if (err) { logger.error('Error in usersAdd request validator.', { error: err }) return res.sendStatus(500) @@ -34,12 +31,12 @@ function usersAdd (req, res, next) { } function usersRemove (req, res, next) { - req.checkParams('id', 'Should have a valid id').notEmpty().isMongoId() + req.checkParams('id', 'Should have a valid id').notEmpty().isInt() logger.debug('Checking usersRemove parameters', { parameters: req.params }) checkErrors(req, res, function () { - User.loadById(req.params.id, function (err, user) { + db.User.loadById(req.params.id, function (err, user) { if (err) { logger.error('Error in usersRemove request validator.', { error: err }) return res.sendStatus(500) @@ -55,7 +52,7 @@ function usersRemove (req, res, next) { } function usersUpdate (req, res, next) { - req.checkParams('id', 'Should have a valid id').notEmpty().isMongoId() + req.checkParams('id', 'Should have a valid id').notEmpty().isInt() // Add old password verification req.checkBody('password', 'Should have a valid password').isUserPasswordValid() diff --git a/server/middlewares/validators/videos.js b/server/middlewares/validators/videos.js index 76e943e77..7e90ca047 100644 --- a/server/middlewares/validators/videos.js +++ b/server/middlewares/validators/videos.js @@ -1,14 +1,11 @@ 'use strict' -const mongoose = require('mongoose') - const checkErrors = require('./utils').checkErrors const constants = require('../../initializers/constants') const customVideosValidators = require('../../helpers/custom-validators').videos +const db = require('../../initializers/database') const logger = require('../../helpers/logger') -const Video = mongoose.model('Video') - const validatorsVideos = { videosAdd, videosGet, @@ -29,7 +26,7 @@ function videosAdd (req, res, next) { checkErrors(req, res, function () { const videoFile = req.files.videofile[0] - Video.getDurationFromFile(videoFile.path, function (err, duration) { + db.Video.getDurationFromFile(videoFile.path, function (err, duration) { if (err) { return res.status(400).send('Cannot retrieve metadata of the file.') } @@ -45,12 +42,12 @@ function videosAdd (req, res, next) { } function videosGet (req, res, next) { - req.checkParams('id', 'Should have a valid id').notEmpty().isMongoId() + req.checkParams('id', 'Should have a valid id').notEmpty().isUUID(4) logger.debug('Checking videosGet parameters', { parameters: req.params }) checkErrors(req, res, function () { - Video.load(req.params.id, function (err, video) { + db.Video.load(req.params.id, function (err, video) { if (err) { logger.error('Error in videosGet request validator.', { error: err }) return res.sendStatus(500) @@ -64,12 +61,12 @@ function videosGet (req, res, next) { } function videosRemove (req, res, next) { - req.checkParams('id', 'Should have a valid id').notEmpty().isMongoId() + req.checkParams('id', 'Should have a valid id').notEmpty().isUUID(4) logger.debug('Checking videosRemove parameters', { parameters: req.params }) checkErrors(req, res, function () { - Video.load(req.params.id, function (err, video) { + db.Video.loadAndPopulateAuthor(req.params.id, function (err, video) { if (err) { logger.error('Error in videosRemove request validator.', { error: err }) return res.sendStatus(500) @@ -77,7 +74,7 @@ function videosRemove (req, res, next) { if (!video) return res.status(404).send('Video not found') else if (video.isOwned() === false) return res.status(403).send('Cannot remove video of another pod') - else if (video.author !== res.locals.oauth.token.user.username) return res.status(403).send('Cannot remove video of another user') + else if (video.Author.name !== res.locals.oauth.token.user.username) return res.status(403).send('Cannot remove video of another user') next() }) diff --git a/server/models/application.js b/server/models/application.js index 452ac4283..ec1d7b122 100644 --- a/server/models/application.js +++ b/server/models/application.js @@ -1,31 +1,36 @@ -const mongoose = require('mongoose') +module.exports = function (sequelize, DataTypes) { + const Application = sequelize.define('Application', + { + sqlSchemaVersion: { + type: DataTypes.INTEGER, + defaultValue: 0 + } + }, + { + classMethods: { + loadSqlSchemaVersion, + updateSqlSchemaVersion + } + } + ) + + return Application +} // --------------------------------------------------------------------------- -const ApplicationSchema = mongoose.Schema({ - mongoSchemaVersion: { - type: Number, - default: 0 +function loadSqlSchemaVersion (callback) { + const query = { + attributes: [ 'sqlSchemaVersion' ] } -}) - -ApplicationSchema.statics = { - loadMongoSchemaVersion, - updateMongoSchemaVersion -} - -mongoose.model('Application', ApplicationSchema) - -// --------------------------------------------------------------------------- -function loadMongoSchemaVersion (callback) { - return this.findOne({}, { mongoSchemaVersion: 1 }, function (err, data) { - const version = data ? data.mongoSchemaVersion : 0 + return this.findOne(query).asCallback(function (err, data) { + const version = data ? data.sqlSchemaVersion : 0 return callback(err, version) }) } -function updateMongoSchemaVersion (newVersion, callback) { - return this.update({}, { mongoSchemaVersion: newVersion }, callback) +function updateSqlSchemaVersion (newVersion, callback) { + return this.update({ sqlSchemaVersion: newVersion }).asCallback(callback) } diff --git a/server/models/author.js b/server/models/author.js new file mode 100644 index 000000000..493c2ca11 --- /dev/null +++ b/server/models/author.js @@ -0,0 +1,28 @@ +module.exports = function (sequelize, DataTypes) { + const Author = sequelize.define('Author', + { + name: { + type: DataTypes.STRING + } + }, + { + classMethods: { + associate + } + } + ) + + return Author +} + +// --------------------------------------------------------------------------- + +function associate (models) { + this.belongsTo(models.Pod, { + foreignKey: { + name: 'podId', + allowNull: true + }, + onDelete: 'cascade' + }) +} diff --git a/server/models/oauth-client.js b/server/models/oauth-client.js index a1aefa985..15118591a 100644 --- a/server/models/oauth-client.js +++ b/server/models/oauth-client.js @@ -1,33 +1,63 @@ -const mongoose = require('mongoose') - -// --------------------------------------------------------------------------- - -const OAuthClientSchema = mongoose.Schema({ - clientSecret: String, - grants: Array, - redirectUris: Array -}) - -OAuthClientSchema.path('clientSecret').required(true) - -OAuthClientSchema.statics = { - getByIdAndSecret, - list, - loadFirstClient +module.exports = function (sequelize, DataTypes) { + const OAuthClient = sequelize.define('OAuthClient', + { + clientId: { + type: DataTypes.STRING + }, + clientSecret: { + type: DataTypes.STRING + }, + grants: { + type: DataTypes.ARRAY(DataTypes.STRING) + }, + redirectUris: { + type: DataTypes.ARRAY(DataTypes.STRING) + } + }, + { + classMethods: { + associate, + + getByIdAndSecret, + list, + loadFirstClient + } + } + ) + + return OAuthClient } -mongoose.model('OAuthClient', OAuthClientSchema) +// TODO: validation +// OAuthClientSchema.path('clientSecret').required(true) // --------------------------------------------------------------------------- +function associate (models) { + this.hasMany(models.OAuthToken, { + foreignKey: { + name: 'oAuthClientId', + allowNull: false + }, + onDelete: 'cascade' + }) +} + function list (callback) { - return this.find(callback) + return this.findAll().asCallback(callback) } function loadFirstClient (callback) { - return this.findOne({}, callback) + return this.findOne().asCallback(callback) } -function getByIdAndSecret (id, clientSecret) { - return this.findOne({ _id: id, clientSecret: clientSecret }).exec() +function getByIdAndSecret (clientId, clientSecret) { + const query = { + where: { + clientId: clientId, + clientSecret: clientSecret + } + } + + return this.findOne(query) } diff --git a/server/models/oauth-token.js b/server/models/oauth-token.js index aff73bfb1..c9108bf95 100644 --- a/server/models/oauth-token.js +++ b/server/models/oauth-token.js @@ -1,42 +1,71 @@ -const mongoose = require('mongoose') - const logger = require('../helpers/logger') // --------------------------------------------------------------------------- -const OAuthTokenSchema = mongoose.Schema({ - accessToken: String, - accessTokenExpiresAt: Date, - client: { type: mongoose.Schema.Types.ObjectId, ref: 'OAuthClient' }, - refreshToken: String, - refreshTokenExpiresAt: Date, - user: { type: mongoose.Schema.Types.ObjectId, ref: 'User' } -}) - -OAuthTokenSchema.path('accessToken').required(true) -OAuthTokenSchema.path('client').required(true) -OAuthTokenSchema.path('user').required(true) - -OAuthTokenSchema.statics = { - getByRefreshTokenAndPopulateClient, - getByTokenAndPopulateUser, - getByRefreshTokenAndPopulateUser, - removeByUserId +module.exports = function (sequelize, DataTypes) { + const OAuthToken = sequelize.define('OAuthToken', + { + accessToken: { + type: DataTypes.STRING + }, + accessTokenExpiresAt: { + type: DataTypes.DATE + }, + refreshToken: { + type: DataTypes.STRING + }, + refreshTokenExpiresAt: { + type: DataTypes.DATE + } + }, + { + classMethods: { + associate, + + getByRefreshTokenAndPopulateClient, + getByTokenAndPopulateUser, + getByRefreshTokenAndPopulateUser, + removeByUserId + } + } + ) + + return OAuthToken } -mongoose.model('OAuthToken', OAuthTokenSchema) +// TODO: validation +// OAuthTokenSchema.path('accessToken').required(true) +// OAuthTokenSchema.path('client').required(true) +// OAuthTokenSchema.path('user').required(true) // --------------------------------------------------------------------------- +function associate (models) { + this.belongsTo(models.User, { + foreignKey: { + name: 'userId', + allowNull: false + }, + onDelete: 'cascade' + }) +} + function getByRefreshTokenAndPopulateClient (refreshToken) { - return this.findOne({ refreshToken: refreshToken }).populate('client').exec().then(function (token) { + const query = { + where: { + refreshToken: refreshToken + }, + include: [ this.associations.OAuthClient ] + } + + return this.findOne(query).then(function (token) { if (!token) return token const tokenInfos = { refreshToken: token.refreshToken, refreshTokenExpiresAt: token.refreshTokenExpiresAt, client: { - id: token.client._id.toString() + id: token.client.id }, user: { id: token.user @@ -50,13 +79,41 @@ function getByRefreshTokenAndPopulateClient (refreshToken) { } function getByTokenAndPopulateUser (bearerToken) { - return this.findOne({ accessToken: bearerToken }).populate('user').exec() + const query = { + where: { + accessToken: bearerToken + }, + include: [ this.sequelize.models.User ] + } + + return this.findOne(query).then(function (token) { + if (token) token.user = token.User + + return token + }) } function getByRefreshTokenAndPopulateUser (refreshToken) { - return this.findOne({ refreshToken: refreshToken }).populate('user').exec() + const query = { + where: { + refreshToken: refreshToken + }, + include: [ this.sequelize.models.User ] + } + + return this.findOne(query).then(function (token) { + token.user = token.User + + return token + }) } function removeByUserId (userId, callback) { - return this.remove({ user: userId }, callback) + const query = { + where: { + userId: userId + } + } + + return this.destroy(query).asCallback(callback) } diff --git a/server/models/pods.js b/server/models/pods.js index 49c73472a..2c1f56203 100644 --- a/server/models/pods.js +++ b/server/models/pods.js @@ -1,79 +1,62 @@ 'use strict' -const each = require('async/each') -const mongoose = require('mongoose') const map = require('lodash/map') -const validator = require('express-validator').validator const constants = require('../initializers/constants') -const Video = mongoose.model('Video') - // --------------------------------------------------------------------------- -const PodSchema = mongoose.Schema({ - host: String, - publicKey: String, - score: { type: Number, max: constants.FRIEND_SCORE.MAX }, - createdDate: { - type: Date, - default: Date.now - } -}) - -PodSchema.path('host').validate(validator.isURL) -PodSchema.path('publicKey').required(true) -PodSchema.path('score').validate(function (value) { return !isNaN(value) }) - -PodSchema.methods = { - toFormatedJSON +module.exports = function (sequelize, DataTypes) { + const Pod = sequelize.define('Pod', + { + host: { + type: DataTypes.STRING + }, + publicKey: { + type: DataTypes.STRING(5000) + }, + score: { + type: DataTypes.INTEGER, + defaultValue: constants.FRIEND_SCORE.BASE + } + // Check createdAt + }, + { + classMethods: { + associate, + + countAll, + incrementScores, + list, + listAllIds, + listBadPods, + load, + loadByHost, + removeAll + }, + instanceMethods: { + toFormatedJSON + } + } + ) + + return Pod } -PodSchema.statics = { - countAll, - incrementScores, - list, - listAllIds, - listBadPods, - load, - loadByHost, - removeAll -} - -PodSchema.pre('save', function (next) { - const self = this - - Pod.loadByHost(this.host, function (err, pod) { - if (err) return next(err) - - if (pod) return next(new Error('Pod already exists.')) - - self.score = constants.FRIEND_SCORE.BASE - return next() - }) -}) - -PodSchema.pre('remove', function (next) { - // Remove the videos owned by this pod too - Video.listByHost(this.host, function (err, videos) { - if (err) return next(err) - - each(videos, function (video, callbackEach) { - video.remove(callbackEach) - }, next) - }) -}) - -const Pod = mongoose.model('Pod', PodSchema) +// TODO: max score -> constants.FRIENDS_SCORE.MAX +// TODO: validation +// PodSchema.path('host').validate(validator.isURL) +// PodSchema.path('publicKey').required(true) +// PodSchema.path('score').validate(function (value) { return !isNaN(value) }) // ------------------------------ METHODS ------------------------------ function toFormatedJSON () { const json = { - id: this._id, + id: this.id, host: this.host, score: this.score, - createdDate: this.createdDate + createdAt: this.createdAt } return json @@ -81,39 +64,76 @@ function toFormatedJSON () { // ------------------------------ Statics ------------------------------ +function associate (models) { + this.belongsToMany(models.Request, { + foreignKey: 'podId', + through: models.RequestToPod, + onDelete: 'CASCADE' + }) +} + function countAll (callback) { - return this.count(callback) + return this.count().asCallback(callback) } function incrementScores (ids, value, callback) { if (!callback) callback = function () {} - return this.update({ _id: { $in: ids } }, { $inc: { score: value } }, { multi: true }, callback) + + const update = { + score: this.sequelize.literal('score +' + value) + } + + const query = { + where: { + id: { + $in: ids + } + } + } + + return this.update(update, query).asCallback(callback) } function list (callback) { - return this.find(callback) + return this.findAll().asCallback(callback) } function listAllIds (callback) { - return this.find({}, { _id: 1 }, function (err, pods) { + const query = { + attributes: [ 'id' ] + } + + return this.findAll(query).asCallback(function (err, pods) { if (err) return callback(err) - return callback(null, map(pods, '_id')) + return callback(null, map(pods, 'id')) }) } function listBadPods (callback) { - return this.find({ score: 0 }, callback) + const query = { + where: { + score: { $lte: 0 } + } + } + + return this.findAll(query).asCallback(callback) } function load (id, callback) { - return this.findById(id, callback) + return this.findById(id).asCallback(callback) } function loadByHost (host, callback) { - return this.findOne({ host }, callback) + const query = { + where: { + host: host + } + } + + return this.findOne(query).asCallback(callback) } function removeAll (callback) { - return this.remove({}, callback) + return this.destroy().asCallback(callback) } diff --git a/server/models/request.js b/server/models/request.js index c2cfe83ce..882f747b7 100644 --- a/server/models/request.js +++ b/server/models/request.js @@ -2,66 +2,58 @@ const each = require('async/each') const eachLimit = require('async/eachLimit') -const values = require('lodash/values') -const mongoose = require('mongoose') const waterfall = require('async/waterfall') const constants = require('../initializers/constants') const logger = require('../helpers/logger') const requests = require('../helpers/requests') -const Pod = mongoose.model('Pod') - let timer = null let lastRequestTimestamp = 0 // --------------------------------------------------------------------------- -const RequestSchema = mongoose.Schema({ - request: mongoose.Schema.Types.Mixed, - endpoint: { - type: String, - enum: [ values(constants.REQUEST_ENDPOINTS) ] - }, - to: [ +module.exports = function (sequelize, DataTypes) { + const Request = sequelize.define('Request', + { + request: { + type: DataTypes.JSON + }, + endpoint: { + // TODO: enum? + type: DataTypes.STRING + } + }, { - type: mongoose.Schema.Types.ObjectId, - ref: 'Pod' + classMethods: { + associate, + + activate, + countTotalRequests, + deactivate, + flush, + forceSend, + remainingMilliSeconds + } } - ] -}) - -RequestSchema.statics = { - activate, - deactivate, - flush, - forceSend, - list, - remainingMilliSeconds -} - -RequestSchema.pre('save', function (next) { - const self = this - - if (self.to.length === 0) { - Pod.listAllIds(function (err, podIds) { - if (err) return next(err) - - // No friends - if (podIds.length === 0) return - - self.to = podIds - return next() - }) - } else { - return next() - } -}) + ) -mongoose.model('Request', RequestSchema) + return Request +} // ------------------------------ STATICS ------------------------------ +function associate (models) { + this.belongsToMany(models.Pod, { + foreignKey: { + name: 'requestId', + allowNull: false + }, + through: models.RequestToPod, + onDelete: 'CASCADE' + }) +} + function activate () { logger.info('Requests scheduler activated.') lastRequestTimestamp = Date.now() @@ -73,6 +65,14 @@ function activate () { }, constants.REQUESTS_INTERVAL) } +function countTotalRequests (callback) { + const query = { + include: [ this.sequelize.models.Pod ] + } + + return this.count(query).asCallback(callback) +} + function deactivate () { logger.info('Requests scheduler deactivated.') clearInterval(timer) @@ -90,10 +90,6 @@ function forceSend () { makeRequests.call(this) } -function list (callback) { - this.find({ }, callback) -} - function remainingMilliSeconds () { if (timer === null) return -1 @@ -136,6 +132,7 @@ function makeRequest (toPod, requestEndpoint, requestsToMake, callback) { // Make all the requests of the scheduler function makeRequests () { const self = this + const RequestToPod = this.sequelize.models.RequestToPod // We limit the size of the requests (REQUESTS_LIMIT) // We don't want to stuck with the same failing requests so we get a random list @@ -156,20 +153,20 @@ function makeRequests () { // We want to group requests by destinations pod and endpoint const requestsToMakeGrouped = {} - requests.forEach(function (poolRequest) { - poolRequest.to.forEach(function (toPodId) { - const hashKey = toPodId + poolRequest.endpoint + requests.forEach(function (request) { + request.Pods.forEach(function (toPod) { + const hashKey = toPod.id + request.endpoint if (!requestsToMakeGrouped[hashKey]) { requestsToMakeGrouped[hashKey] = { - toPodId, - endpoint: poolRequest.endpoint, - ids: [], // pool request ids, to delete them from the DB in the future + toPodId: toPod.id, + endpoint: request.endpoint, + ids: [], // request ids, to delete them from the DB in the future datas: [] // requests data, } } - requestsToMakeGrouped[hashKey].ids.push(poolRequest._id) - requestsToMakeGrouped[hashKey].datas.push(poolRequest.request) + requestsToMakeGrouped[hashKey].ids.push(request.id) + requestsToMakeGrouped[hashKey].datas.push(request.request) }) }) @@ -179,8 +176,8 @@ function makeRequests () { eachLimit(Object.keys(requestsToMakeGrouped), constants.REQUESTS_IN_PARALLEL, function (hashKey, callbackEach) { const requestToMake = requestsToMakeGrouped[hashKey] - // FIXME: mongodb request inside a loop :/ - Pod.load(requestToMake.toPodId, function (err, toPod) { + // FIXME: SQL request inside a loop :/ + self.sequelize.models.Pod.load(requestToMake.toPodId, function (err, toPod) { if (err) { logger.error('Error finding pod by id.', { err: err }) return callbackEach() @@ -191,7 +188,7 @@ function makeRequests () { const requestIdsToDelete = requestToMake.ids logger.info('Removing %d requests of unexisting pod %s.', requestIdsToDelete.length, requestToMake.toPodId) - removePodOf.call(self, requestIdsToDelete, requestToMake.toPodId) + RequestToPod.removePodOf.call(self, requestIdsToDelete, requestToMake.toPodId) return callbackEach() } @@ -202,7 +199,7 @@ function makeRequests () { goodPods.push(requestToMake.toPodId) // Remove the pod id of these request ids - removePodOf.call(self, requestToMake.ids, requestToMake.toPodId, callbackEach) + RequestToPod.removePodOf(requestToMake.ids, requestToMake.toPodId, callbackEach) } else { badPods.push(requestToMake.toPodId) callbackEach() @@ -211,18 +208,22 @@ function makeRequests () { }) }, function () { // All the requests were made, we update the pods score - updatePodsScore(goodPods, badPods) + updatePodsScore.call(self, goodPods, badPods) // Flush requests with no pod - removeWithEmptyTo.call(self) + removeWithEmptyTo.call(self, function (err) { + if (err) logger.error('Error when removing requests with no pods.', { error: err }) + }) }) }) } // Remove pods with a score of 0 (too many requests where they were unreachable) function removeBadPods () { + const self = this + waterfall([ function findBadPods (callback) { - Pod.listBadPods(function (err, pods) { + self.sequelize.models.Pod.listBadPods(function (err, pods) { if (err) { logger.error('Cannot find bad pods.', { error: err }) return callback(err) @@ -233,10 +234,8 @@ function removeBadPods () { }, function removeTheseBadPods (pods, callback) { - if (pods.length === 0) return callback(null, 0) - each(pods, function (pod, callbackEach) { - pod.remove(callbackEach) + pod.destroy().asCallback(callbackEach) }, function (err) { return callback(err, pods.length) }) @@ -253,43 +252,67 @@ function removeBadPods () { } function updatePodsScore (goodPods, badPods) { + const self = this + const Pod = this.sequelize.models.Pod + logger.info('Updating %d good pods and %d bad pods scores.', goodPods.length, badPods.length) - Pod.incrementScores(goodPods, constants.PODS_SCORE.BONUS, function (err) { - if (err) logger.error('Cannot increment scores of good pods.') - }) + if (goodPods.length !== 0) { + Pod.incrementScores(goodPods, constants.PODS_SCORE.BONUS, function (err) { + if (err) logger.error('Cannot increment scores of good pods.') + }) + } - Pod.incrementScores(badPods, constants.PODS_SCORE.MALUS, function (err) { - if (err) logger.error('Cannot decrement scores of bad pods.') - removeBadPods() - }) + if (badPods.length !== 0) { + Pod.incrementScores(badPods, constants.PODS_SCORE.MALUS, function (err) { + if (err) logger.error('Cannot decrement scores of bad pods.') + removeBadPods.call(self) + }) + } } function listWithLimitAndRandom (limit, callback) { const self = this - self.count(function (err, count) { + self.count().asCallback(function (err, count) { if (err) return callback(err) + // Optimization... + if (count === 0) return callback(null, []) + let start = Math.floor(Math.random() * count) - limit if (start < 0) start = 0 - self.find().sort({ _id: 1 }).skip(start).limit(limit).exec(callback) + const query = { + order: [ + [ 'id', 'ASC' ] + ], + offset: start, + limit: limit, + include: [ this.sequelize.models.Pod ] + } + + self.findAll(query).asCallback(callback) }) } function removeAll (callback) { - this.remove({ }, callback) -} - -function removePodOf (requestsIds, podId, callback) { - if (!callback) callback = function () {} - - this.update({ _id: { $in: requestsIds } }, { $pull: { to: podId } }, { multi: true }, callback) + // Delete all requests + this.destroy({ truncate: true }).asCallback(callback) } function removeWithEmptyTo (callback) { if (!callback) callback = function () {} - this.remove({ to: { $size: 0 } }, callback) + const query = { + where: { + id: { + $notIn: [ + this.sequelize.literal('SELECT "requestId" FROM "RequestToPods"') + ] + } + } + } + + this.destroy(query).asCallback(callback) } diff --git a/server/models/requestToPod.js b/server/models/requestToPod.js new file mode 100644 index 000000000..378c2bdcf --- /dev/null +++ b/server/models/requestToPod.js @@ -0,0 +1,30 @@ +'use strict' + +// --------------------------------------------------------------------------- + +module.exports = function (sequelize, DataTypes) { + const RequestToPod = sequelize.define('RequestToPod', {}, { + classMethods: { + removePodOf + } + }) + + return RequestToPod +} + +// --------------------------------------------------------------------------- + +function removePodOf (requestsIds, podId, callback) { + if (!callback) callback = function () {} + + const query = { + where: { + requestId: { + $in: requestsIds + }, + podId: podId + } + } + + this.destroy(query).asCallback(callback) +} diff --git a/server/models/user.js b/server/models/user.js index a19de7072..e50eb96ea 100644 --- a/server/models/user.js +++ b/server/models/user.js @@ -1,60 +1,60 @@ -const mongoose = require('mongoose') - -const customUsersValidators = require('../helpers/custom-validators').users const modelUtils = require('./utils') const peertubeCrypto = require('../helpers/peertube-crypto') -const OAuthToken = mongoose.model('OAuthToken') - // --------------------------------------------------------------------------- -const UserSchema = mongoose.Schema({ - createdDate: { - type: Date, - default: Date.now - }, - password: String, - username: String, - role: String -}) - -UserSchema.path('password').required(customUsersValidators.isUserPasswordValid) -UserSchema.path('username').required(customUsersValidators.isUserUsernameValid) -UserSchema.path('role').validate(customUsersValidators.isUserRoleValid) - -UserSchema.methods = { - isPasswordMatch, - toFormatedJSON +module.exports = function (sequelize, DataTypes) { + const User = sequelize.define('User', + { + password: { + type: DataTypes.STRING + }, + username: { + type: DataTypes.STRING + }, + role: { + type: DataTypes.STRING + } + }, + { + classMethods: { + associate, + + countTotal, + getByUsername, + list, + listForApi, + loadById, + loadByUsername + }, + instanceMethods: { + isPasswordMatch, + toFormatedJSON + }, + hooks: { + beforeCreate: beforeCreateOrUpdate, + beforeUpdate: beforeCreateOrUpdate + } + } + ) + + return User } -UserSchema.statics = { - countTotal, - getByUsername, - list, - listForApi, - loadById, - loadByUsername -} +// TODO: Validation +// UserSchema.path('password').required(customUsersValidators.isUserPasswordValid) +// UserSchema.path('username').required(customUsersValidators.isUserUsernameValid) +// UserSchema.path('role').validate(customUsersValidators.isUserRoleValid) -UserSchema.pre('save', function (next) { - const user = this - - peertubeCrypto.cryptPassword(this.password, function (err, hash) { +function beforeCreateOrUpdate (user, options, next) { + peertubeCrypto.cryptPassword(user.password, function (err, hash) { if (err) return next(err) user.password = hash return next() }) -}) - -UserSchema.pre('remove', function (next) { - const user = this - - OAuthToken.removeByUserId(user._id, next) -}) - -mongoose.model('User', UserSchema) +} // ------------------------------ METHODS ------------------------------ @@ -64,35 +64,63 @@ function isPasswordMatch (password, callback) { function toFormatedJSON () { return { - id: this._id, + id: this.id, username: this.username, role: this.role, - createdDate: this.createdDate + createdAt: this.createdAt } } // ------------------------------ STATICS ------------------------------ +function associate (models) { + this.hasMany(models.OAuthToken, { + foreignKey: 'userId', + onDelete: 'cascade' + }) +} + function countTotal (callback) { - return this.count(callback) + return this.count().asCallback(callback) } function getByUsername (username) { - return this.findOne({ username: username }) + const query = { + where: { + username: username + } + } + + return this.findOne(query) } function list (callback) { - return this.find(callback) + return this.find().asCallback(callback) } function listForApi (start, count, sort, callback) { - const query = {} - return modelUtils.listForApiWithCount.call(this, query, start, count, sort, callback) + const query = { + offset: start, + limit: count, + order: [ modelUtils.getSort(sort) ] + } + + return this.findAndCountAll(query).asCallback(function (err, result) { + if (err) return callback(err) + + return callback(null, result.rows, result.count) + }) } function loadById (id, callback) { - return this.findById(id, callback) + return this.findById(id).asCallback(callback) } function loadByUsername (username, callback) { - return this.findOne({ username: username }, callback) + const query = { + where: { + username: username + } + } + + return this.findOne(query).asCallback(callback) } diff --git a/server/models/utils.js b/server/models/utils.js index e798aabe6..49636b3d8 100644 --- a/server/models/utils.js +++ b/server/models/utils.js @@ -1,28 +1,23 @@ 'use strict' -const parallel = require('async/parallel') - const utils = { - listForApiWithCount + getSort } -function listForApiWithCount (query, start, count, sort, callback) { - const self = this +// Translate for example "-name" to [ 'name', 'DESC' ] +function getSort (value) { + let field + let direction - parallel([ - function (asyncCallback) { - self.find(query).skip(start).limit(count).sort(sort).exec(asyncCallback) - }, - function (asyncCallback) { - self.count(query, asyncCallback) - } - ], function (err, results) { - if (err) return callback(err) + if (value.substring(0, 1) === '-') { + direction = 'DESC' + field = value.substring(1) + } else { + direction = 'ASC' + field = value + } - const data = results[0] - const total = results[1] - return callback(null, data, total) - }) + return [ field, direction ] } // --------------------------------------------------------------------------- diff --git a/server/models/video.js b/server/models/video.js index 330067cdf..8ef07c9e6 100644 --- a/server/models/video.js +++ b/server/models/video.js @@ -7,102 +7,93 @@ const magnetUtil = require('magnet-uri') const parallel = require('async/parallel') const parseTorrent = require('parse-torrent') const pathUtils = require('path') -const mongoose = require('mongoose') const constants = require('../initializers/constants') -const customVideosValidators = require('../helpers/custom-validators').videos const logger = require('../helpers/logger') const modelUtils = require('./utils') // --------------------------------------------------------------------------- +module.exports = function (sequelize, DataTypes) { // TODO: add indexes on searchable columns -const VideoSchema = mongoose.Schema({ - name: String, - extname: { - type: String, - enum: [ '.mp4', '.webm', '.ogv' ] - }, - remoteId: mongoose.Schema.Types.ObjectId, - description: String, - magnet: { - infoHash: String - }, - podHost: String, - author: String, - duration: Number, - tags: [ String ], - createdDate: { - type: Date, - default: Date.now - } -}) - -VideoSchema.path('name').validate(customVideosValidators.isVideoNameValid) -VideoSchema.path('description').validate(customVideosValidators.isVideoDescriptionValid) -VideoSchema.path('podHost').validate(customVideosValidators.isVideoPodHostValid) -VideoSchema.path('author').validate(customVideosValidators.isVideoAuthorValid) -VideoSchema.path('duration').validate(customVideosValidators.isVideoDurationValid) -VideoSchema.path('tags').validate(customVideosValidators.isVideoTagsValid) - -VideoSchema.methods = { - generateMagnetUri, - getVideoFilename, - getThumbnailName, - getPreviewName, - getTorrentName, - isOwned, - toFormatedJSON, - toRemoteJSON -} - -VideoSchema.statics = { - generateThumbnailFromBase64, - getDurationFromFile, - listForApi, - listByHostAndRemoteId, - listByHost, - listOwned, - listOwnedByAuthor, - listRemotes, - load, - search -} - -VideoSchema.pre('remove', function (next) { - const video = this - const tasks = [] - - tasks.push( - function (callback) { - removeThumbnail(video, callback) - } - ) - - if (video.isOwned()) { - tasks.push( - function (callback) { - removeFile(video, callback) + const Video = sequelize.define('Video', + { + id: { + type: DataTypes.UUID, + defaultValue: DataTypes.UUIDV4, + primaryKey: true }, - function (callback) { - removeTorrent(video, callback) + name: { + type: DataTypes.STRING }, - function (callback) { - removePreview(video, callback) + extname: { + // TODO: enum? + type: DataTypes.STRING + }, + remoteId: { + type: DataTypes.UUID + }, + description: { + type: DataTypes.STRING + }, + infoHash: { + type: DataTypes.STRING + }, + duration: { + type: DataTypes.INTEGER + }, + tags: { + type: DataTypes.ARRAY(DataTypes.STRING) } - ) - } + }, + { + classMethods: { + associate, + + generateThumbnailFromBase64, + getDurationFromFile, + listForApi, + listByHostAndRemoteId, + listOwnedAndPopulateAuthor, + listOwnedByAuthor, + load, + loadAndPopulateAuthor, + loadAndPopulateAuthorAndPod, + searchAndPopulateAuthorAndPod + }, + instanceMethods: { + generateMagnetUri, + getVideoFilename, + getThumbnailName, + getPreviewName, + getTorrentName, + isOwned, + toFormatedJSON, + toRemoteJSON + }, + hooks: { + beforeCreate, + afterDestroy + } + } + ) - parallel(tasks, next) -}) + return Video +} -VideoSchema.pre('save', function (next) { - const video = this +// TODO: Validation +// VideoSchema.path('name').validate(customVideosValidators.isVideoNameValid) +// VideoSchema.path('description').validate(customVideosValidators.isVideoDescriptionValid) +// VideoSchema.path('podHost').validate(customVideosValidators.isVideoPodHostValid) +// VideoSchema.path('author').validate(customVideosValidators.isVideoAuthorValid) +// VideoSchema.path('duration').validate(customVideosValidators.isVideoDurationValid) +// VideoSchema.path('tags').validate(customVideosValidators.isVideoTagsValid) + +function beforeCreate (video, options, next) { const tasks = [] if (video.isOwned()) { const videoPath = pathUtils.join(constants.CONFIG.STORAGE.VIDEOS_DIR, video.getVideoFilename()) - this.podHost = constants.CONFIG.WEBSERVER.HOST tasks.push( // TODO: refractoring @@ -123,7 +114,7 @@ VideoSchema.pre('save', function (next) { if (err) return callback(err) const parsedTorrent = parseTorrent(torrent) - video.magnet.infoHash = parsedTorrent.infoHash + video.infoHash = parsedTorrent.infoHash callback(null) }) @@ -141,12 +132,46 @@ VideoSchema.pre('save', function (next) { } return next() -}) +} + +function afterDestroy (video, options, next) { + const tasks = [] -mongoose.model('Video', VideoSchema) + tasks.push( + function (callback) { + removeThumbnail(video, callback) + } + ) + + if (video.isOwned()) { + tasks.push( + function (callback) { + removeFile(video, callback) + }, + function (callback) { + removeTorrent(video, callback) + }, + function (callback) { + removePreview(video, callback) + } + ) + } + + parallel(tasks, next) +} // ------------------------------ METHODS ------------------------------ +function associate (models) { + this.belongsTo(models.Author, { + foreignKey: { + name: 'authorId', + allowNull: false + }, + onDelete: 'cascade' + }) +} + function generateMagnetUri () { let baseUrlHttp, baseUrlWs @@ -154,8 +179,8 @@ function generateMagnetUri () { baseUrlHttp = constants.CONFIG.WEBSERVER.URL baseUrlWs = constants.CONFIG.WEBSERVER.WS + '://' + constants.CONFIG.WEBSERVER.HOSTNAME + ':' + constants.CONFIG.WEBSERVER.PORT } else { - baseUrlHttp = constants.REMOTE_SCHEME.HTTP + '://' + this.podHost - baseUrlWs = constants.REMOTE_SCHEME.WS + '://' + this.podHost + baseUrlHttp = constants.REMOTE_SCHEME.HTTP + '://' + this.Author.Pod.host + baseUrlWs = constants.REMOTE_SCHEME.WS + '://' + this.Author.Pod.host } const xs = baseUrlHttp + constants.STATIC_PATHS.TORRENTS + this.getTorrentName() @@ -166,7 +191,7 @@ function generateMagnetUri () { xs, announce, urlList, - infoHash: this.magnet.infoHash, + infoHash: this.infoHash, name: this.name } @@ -174,20 +199,20 @@ function generateMagnetUri () { } function getVideoFilename () { - if (this.isOwned()) return this._id + this.extname + if (this.isOwned()) return this.id + this.extname return this.remoteId + this.extname } function getThumbnailName () { // We always have a copy of the thumbnail - return this._id + '.jpg' + return this.id + '.jpg' } function getPreviewName () { const extension = '.jpg' - if (this.isOwned()) return this._id + extension + if (this.isOwned()) return this.id + extension return this.remoteId + extension } @@ -195,7 +220,7 @@ function getPreviewName () { function getTorrentName () { const extension = '.torrent' - if (this.isOwned()) return this._id + extension + if (this.isOwned()) return this.id + extension return this.remoteId + extension } @@ -205,18 +230,27 @@ function isOwned () { } function toFormatedJSON () { + let podHost + + if (this.Author.Pod) { + podHost = this.Author.Pod.host + } else { + // It means it's our video + podHost = constants.CONFIG.WEBSERVER.HOST + } + const json = { - id: this._id, + id: this.id, name: this.name, description: this.description, - podHost: this.podHost, + podHost, isLocal: this.isOwned(), magnetUri: this.generateMagnetUri(), - author: this.author, + author: this.Author.name, duration: this.duration, tags: this.tags, thumbnailPath: constants.STATIC_PATHS.THUMBNAILS + '/' + this.getThumbnailName(), - createdDate: this.createdDate + createdAt: this.createdAt } return json @@ -236,13 +270,13 @@ function toRemoteJSON (callback) { const remoteVideo = { name: self.name, description: self.description, - magnet: self.magnet, - remoteId: self._id, - author: self.author, + infoHash: self.infoHash, + remoteId: self.id, + author: self.Author.name, duration: self.duration, thumbnailBase64: new Buffer(thumbnailData).toString('base64'), tags: self.tags, - createdDate: self.createdDate, + createdAt: self.createdAt, extname: self.extname } @@ -273,50 +307,168 @@ function getDurationFromFile (videoPath, callback) { } function listForApi (start, count, sort, callback) { - const query = {} - return modelUtils.listForApiWithCount.call(this, query, start, count, sort, callback) + const query = { + offset: start, + limit: count, + order: [ modelUtils.getSort(sort) ], + include: [ + { + model: this.sequelize.models.Author, + include: [ this.sequelize.models.Pod ] + } + ] + } + + return this.findAndCountAll(query).asCallback(function (err, result) { + if (err) return callback(err) + + return callback(null, result.rows, result.count) + }) } function listByHostAndRemoteId (fromHost, remoteId, callback) { - this.find({ podHost: fromHost, remoteId: remoteId }, callback) -} + const query = { + where: { + remoteId: remoteId + }, + include: [ + { + model: this.sequelize.models.Author, + include: [ + { + model: this.sequelize.models.Pod, + where: { + host: fromHost + } + } + ] + } + ] + } -function listByHost (fromHost, callback) { - this.find({ podHost: fromHost }, callback) + return this.findAll(query).asCallback(callback) } -function listOwned (callback) { +function listOwnedAndPopulateAuthor (callback) { // If remoteId is null this is *our* video - this.find({ remoteId: null }, callback) + const query = { + where: { + remoteId: null + }, + include: [ this.sequelize.models.Author ] + } + + return this.findAll(query).asCallback(callback) } function listOwnedByAuthor (author, callback) { - this.find({ remoteId: null, author: author }, callback) -} + const query = { + where: { + remoteId: null + }, + include: [ + { + model: this.sequelize.models.Author, + where: { + name: author + } + } + ] + } -function listRemotes (callback) { - this.find({ remoteId: { $ne: null } }, callback) + return this.findAll(query).asCallback(callback) } function load (id, callback) { - this.findById(id, callback) + return this.findById(id).asCallback(callback) } -function search (value, field, start, count, sort, callback) { - const query = {} +function loadAndPopulateAuthor (id, callback) { + const options = { + include: [ this.sequelize.models.Author ] + } + + return this.findById(id, options).asCallback(callback) +} + +function loadAndPopulateAuthorAndPod (id, callback) { + const options = { + include: [ + { + model: this.sequelize.models.Author, + include: [ this.sequelize.models.Pod ] + } + ] + } + + return this.findById(id, options).asCallback(callback) +} + +function searchAndPopulateAuthorAndPod (value, field, start, count, sort, callback) { + const podInclude = { + model: this.sequelize.models.Pod + } + const authorInclude = { + model: this.sequelize.models.Author, + include: [ + podInclude + ] + } + + const query = { + where: {}, + include: [ + authorInclude + ], + offset: start, + limit: count, + order: [ modelUtils.getSort(sort) ] + } + + // TODO: include our pod for podHost searches (we are not stored in the database) // Make an exact search with the magnet if (field === 'magnetUri') { const infoHash = magnetUtil.decode(value).infoHash - query.magnet = { - infoHash - } + query.where.infoHash = infoHash } else if (field === 'tags') { - query[field] = value + query.where[field] = value + } else if (field === 'host') { + const whereQuery = { + '$Author.Pod.host$': { + $like: '%' + value + '%' + } + } + + // Include our pod? (not stored in the database) + if (constants.CONFIG.WEBSERVER.HOST.indexOf(value) !== -1) { + query.where = { + $or: [ + whereQuery, + { + remoteId: null + } + ] + } + } else { + query.where = whereQuery + } + } else if (field === 'author') { + query.where = { + '$Author.name$': { + $like: '%' + value + '%' + } + } } else { - query[field] = new RegExp(value, 'i') + query.where[field] = { + $like: '%' + value + '%' + } } - modelUtils.listForApiWithCount.call(this, query, start, count, sort, callback) + return this.findAndCountAll(query).asCallback(function (err, result) { + if (err) return callback(err) + + return callback(null, result.rows, result.count) + }) } // --------------------------------------------------------------------------- diff --git a/server/tests/api/check-params.js b/server/tests/api/check-params.js index 444c2fc55..d9e51770c 100644 --- a/server/tests/api/check-params.js +++ b/server/tests/api/check-params.js @@ -465,7 +465,7 @@ describe('Test parameters validator', function () { it('Should return 404 with an incorrect video', function (done) { request(server.url) - .get(path + '123456789012345678901234') + .get(path + '4da6fde3-88f7-4d16-b119-108df5630b06') .set('Accept', 'application/json') .expect(404, done) }) @@ -490,7 +490,7 @@ describe('Test parameters validator', function () { it('Should fail with a video which does not exist', function (done) { request(server.url) - .delete(path + '123456789012345678901234') + .delete(path + '4da6fde3-88f7-4d16-b119-108df5630b06') .set('Authorization', 'Bearer ' + server.accessToken) .expect(404, done) }) @@ -711,7 +711,7 @@ describe('Test parameters validator', function () { it('Should return 404 with a non existing id', function (done) { request(server.url) - .delete(path + '579f982228c99c221d8092b8') + .delete(path + '45') .set('Authorization', 'Bearer ' + server.accessToken) .expect(404, done) }) diff --git a/server/tests/api/friends-basic.js b/server/tests/api/friends-basic.js index a871f9838..3a904dbd7 100644 --- a/server/tests/api/friends-basic.js +++ b/server/tests/api/friends-basic.js @@ -97,7 +97,7 @@ describe('Test basic friends', function () { const pod = result[0] expect(pod.host).to.equal(servers[2].host) expect(pod.score).to.equal(20) - expect(miscsUtils.dateIsValid(pod.createdDate)).to.be.true + expect(miscsUtils.dateIsValid(pod.createdAt)).to.be.true next() }) @@ -114,7 +114,7 @@ describe('Test basic friends', function () { const pod = result[0] expect(pod.host).to.equal(servers[1].host) expect(pod.score).to.equal(20) - expect(miscsUtils.dateIsValid(pod.createdDate)).to.be.true + expect(miscsUtils.dateIsValid(pod.createdAt)).to.be.true next() }) diff --git a/server/tests/api/multiple-pods.js b/server/tests/api/multiple-pods.js index be278d7c5..f0fe59c5f 100644 --- a/server/tests/api/multiple-pods.js +++ b/server/tests/api/multiple-pods.js @@ -104,7 +104,7 @@ describe('Test multiple pods', function () { expect(video.magnetUri).to.exist expect(video.duration).to.equal(10) expect(video.tags).to.deep.equal([ 'tag1p1', 'tag2p1' ]) - expect(miscsUtils.dateIsValid(video.createdDate)).to.be.true + expect(miscsUtils.dateIsValid(video.createdAt)).to.be.true expect(video.author).to.equal('root') if (server.url !== 'http://localhost:9001') { @@ -166,7 +166,7 @@ describe('Test multiple pods', function () { expect(video.magnetUri).to.exist expect(video.duration).to.equal(5) expect(video.tags).to.deep.equal([ 'tag1p2', 'tag2p2', 'tag3p2' ]) - expect(miscsUtils.dateIsValid(video.createdDate)).to.be.true + expect(miscsUtils.dateIsValid(video.createdAt)).to.be.true expect(video.author).to.equal('root') if (server.url !== 'http://localhost:9002') { @@ -246,7 +246,7 @@ describe('Test multiple pods', function () { expect(video1.duration).to.equal(5) expect(video1.tags).to.deep.equal([ 'tag1p3' ]) expect(video1.author).to.equal('root') - expect(miscsUtils.dateIsValid(video1.createdDate)).to.be.true + expect(miscsUtils.dateIsValid(video1.createdAt)).to.be.true expect(video2.name).to.equal('my super name for pod 3-2') expect(video2.description).to.equal('my super description for pod 3-2') @@ -255,7 +255,7 @@ describe('Test multiple pods', function () { expect(video2.duration).to.equal(5) expect(video2.tags).to.deep.equal([ 'tag2p3', 'tag3p3', 'tag4p3' ]) expect(video2.author).to.equal('root') - expect(miscsUtils.dateIsValid(video2.createdDate)).to.be.true + expect(miscsUtils.dateIsValid(video2.createdAt)).to.be.true if (server.url !== 'http://localhost:9003') { expect(video1.isLocal).to.be.false diff --git a/server/tests/api/requests.js b/server/tests/api/requests.js index af36f6e34..7e790b54b 100644 --- a/server/tests/api/requests.js +++ b/server/tests/api/requests.js @@ -69,7 +69,7 @@ describe('Test requests stats', function () { }) }) - it('Should have the correct request', function (done) { + it('Should have the correct total request', function (done) { this.timeout(15000) const server = servers[0] @@ -83,11 +83,7 @@ describe('Test requests stats', function () { if (err) throw err const body = res.body - expect(body.requests).to.have.lengthOf(1) - - const request = body.requests[0] - expect(request.to).to.have.lengthOf(1) - expect(request.request.type).to.equal('add') + expect(body.totalRequests).to.equal(1) // Wait one cycle setTimeout(done, 10000) @@ -95,27 +91,6 @@ describe('Test requests stats', function () { }) }) - it('Should have the correct requests', function (done) { - const server = servers[0] - - uploadVideo(server, function (err) { - if (err) throw err - - getRequestsStats(server, function (err, res) { - if (err) throw err - - const body = res.body - expect(body.requests).to.have.lengthOf(2) - - const request = body.requests[1] - expect(request.to).to.have.lengthOf(1) - expect(request.request.type).to.equal('add') - - done() - }) - }) - }) - after(function (done) { process.kill(-servers[0].app.pid) diff --git a/server/tests/api/single-pod.js b/server/tests/api/single-pod.js index 65d1a7a65..aedecacf3 100644 --- a/server/tests/api/single-pod.js +++ b/server/tests/api/single-pod.js @@ -82,7 +82,7 @@ describe('Test a single pod', function () { expect(video.author).to.equal('root') expect(video.isLocal).to.be.true expect(video.tags).to.deep.equal([ 'tag1', 'tag2', 'tag3' ]) - expect(miscsUtils.dateIsValid(video.createdDate)).to.be.true + expect(miscsUtils.dateIsValid(video.createdAt)).to.be.true videosUtils.testVideoImage(server.url, 'video_short.webm', video.thumbnailPath, function (err, test) { if (err) throw err @@ -116,7 +116,7 @@ describe('Test a single pod', function () { expect(video.author).to.equal('root') expect(video.isLocal).to.be.true expect(video.tags).to.deep.equal([ 'tag1', 'tag2', 'tag3' ]) - expect(miscsUtils.dateIsValid(video.createdDate)).to.be.true + expect(miscsUtils.dateIsValid(video.createdAt)).to.be.true videosUtils.testVideoImage(server.url, 'video_short.webm', video.thumbnailPath, function (err, test) { if (err) throw err @@ -142,7 +142,7 @@ describe('Test a single pod', function () { expect(video.author).to.equal('root') expect(video.isLocal).to.be.true expect(video.tags).to.deep.equal([ 'tag1', 'tag2', 'tag3' ]) - expect(miscsUtils.dateIsValid(video.createdDate)).to.be.true + expect(miscsUtils.dateIsValid(video.createdAt)).to.be.true videosUtils.testVideoImage(server.url, 'video_short.webm', video.thumbnailPath, function (err, test) { if (err) throw err @@ -154,7 +154,7 @@ describe('Test a single pod', function () { }) it('Should search the video by podHost', function (done) { - videosUtils.searchVideo(server.url, '9001', 'podHost', function (err, res) { + videosUtils.searchVideo(server.url, '9001', 'host', function (err, res) { if (err) throw err expect(res.body.total).to.equal(1) @@ -168,7 +168,7 @@ describe('Test a single pod', function () { expect(video.author).to.equal('root') expect(video.isLocal).to.be.true expect(video.tags).to.deep.equal([ 'tag1', 'tag2', 'tag3' ]) - expect(miscsUtils.dateIsValid(video.createdDate)).to.be.true + expect(miscsUtils.dateIsValid(video.createdAt)).to.be.true videosUtils.testVideoImage(server.url, 'video_short.webm', video.thumbnailPath, function (err, test) { if (err) throw err @@ -194,7 +194,7 @@ describe('Test a single pod', function () { expect(video.author).to.equal('root') expect(video.isLocal).to.be.true expect(video.tags).to.deep.equal([ 'tag1', 'tag2', 'tag3' ]) - expect(miscsUtils.dateIsValid(video.createdDate)).to.be.true + expect(miscsUtils.dateIsValid(video.createdAt)).to.be.true videosUtils.testVideoImage(server.url, 'video_short.webm', video.thumbnailPath, function (err, test) { if (err) throw err @@ -425,7 +425,7 @@ describe('Test a single pod', function () { }) it('Should search all the 9001 port videos', function (done) { - videosUtils.searchVideoWithPagination(server.url, '9001', 'podHost', 0, 15, function (err, res) { + videosUtils.searchVideoWithPagination(server.url, '9001', 'host', 0, 15, function (err, res) { if (err) throw err const videos = res.body.data @@ -437,7 +437,7 @@ describe('Test a single pod', function () { }) it('Should search all the localhost videos', function (done) { - videosUtils.searchVideoWithPagination(server.url, 'localhost', 'podHost', 0, 15, function (err, res) { + videosUtils.searchVideoWithPagination(server.url, 'localhost', 'host', 0, 15, function (err, res) { if (err) throw err const videos = res.body.data diff --git a/server/tests/api/users.js b/server/tests/api/users.js index 94267f104..e6d937eb0 100644 --- a/server/tests/api/users.js +++ b/server/tests/api/users.js @@ -261,8 +261,8 @@ describe('Test users', function () { }) }) - it('Should list only the second user by createdDate desc', function (done) { - usersUtils.getUsersListPaginationAndSort(server.url, 0, 1, '-createdDate', function (err, res) { + it('Should list only the second user by createdAt desc', function (done) { + usersUtils.getUsersListPaginationAndSort(server.url, 0, 1, '-createdAt', function (err, res) { if (err) throw err const result = res.body @@ -279,8 +279,8 @@ describe('Test users', function () { }) }) - it('Should list all the users by createdDate asc', function (done) { - usersUtils.getUsersListPaginationAndSort(server.url, 0, 2, 'createdDate', function (err, res) { + it('Should list all the users by createdAt asc', function (done) { + usersUtils.getUsersListPaginationAndSort(server.url, 0, 2, 'createdAt', function (err, res) { if (err) throw err const result = res.body diff --git a/server/tests/utils/servers.js b/server/tests/utils/servers.js index 01c9a2f39..4e55f8f5c 100644 --- a/server/tests/utils/servers.js +++ b/server/tests/utils/servers.js @@ -60,12 +60,12 @@ function runServer (number, callback) { // These actions are async so we need to be sure that they have both been done const serverRunString = { - 'Connected to mongodb': false, + 'Database is ready': false, 'Server listening on port': false } const regexps = { - client_id: 'Client id: ([a-f0-9]+)', + client_id: 'Client id: (.+)', client_secret: 'Client secret: (.+)', user_username: 'Username: (.+)', user_password: 'User password: (.+)' diff --git a/server/tests/utils/videos.js b/server/tests/utils/videos.js index 536093db1..5c120597f 100644 --- a/server/tests/utils/videos.js +++ b/server/tests/utils/videos.js @@ -25,7 +25,7 @@ function getAllVideosListBy (url, end) { request(url) .get(path) - .query({ sort: 'createdDate' }) + .query({ sort: 'createdAt' }) .query({ start: 0 }) .query({ count: 10000 }) .set('Accept', 'application/json') -- cgit v1.2.3 From 3897209f46f4c4581be2b8963bf9acc28ca5032b Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Mon, 19 Dec 2016 21:50:20 +0100 Subject: Server: rename Pods -> Pod --- server/controllers/api/remote.js | 2 - server/models/pod.js | 139 +++++++++++++++++++++++++++++++++++++++ server/models/pods.js | 139 --------------------------------------- 3 files changed, 139 insertions(+), 141 deletions(-) create mode 100644 server/models/pod.js delete mode 100644 server/models/pods.js (limited to 'server') diff --git a/server/controllers/api/remote.js b/server/controllers/api/remote.js index d856576a9..a6753a2b0 100644 --- a/server/controllers/api/remote.js +++ b/server/controllers/api/remote.js @@ -56,8 +56,6 @@ function addRemoteVideo (videoToCreateData, fromHost, callback) { waterfall([ function findOrCreatePod (callback) { - fromHost - const query = { where: { host: fromHost diff --git a/server/models/pod.js b/server/models/pod.js new file mode 100644 index 000000000..2c1f56203 --- /dev/null +++ b/server/models/pod.js @@ -0,0 +1,139 @@ +'use strict' + +const map = require('lodash/map') + +const constants = require('../initializers/constants') + +// --------------------------------------------------------------------------- + +module.exports = function (sequelize, DataTypes) { + const Pod = sequelize.define('Pod', + { + host: { + type: DataTypes.STRING + }, + publicKey: { + type: DataTypes.STRING(5000) + }, + score: { + type: DataTypes.INTEGER, + defaultValue: constants.FRIEND_SCORE.BASE + } + // Check createdAt + }, + { + classMethods: { + associate, + + countAll, + incrementScores, + list, + listAllIds, + listBadPods, + load, + loadByHost, + removeAll + }, + instanceMethods: { + toFormatedJSON + } + } + ) + + return Pod +} + +// TODO: max score -> constants.FRIENDS_SCORE.MAX +// TODO: validation +// PodSchema.path('host').validate(validator.isURL) +// PodSchema.path('publicKey').required(true) +// PodSchema.path('score').validate(function (value) { return !isNaN(value) }) + +// ------------------------------ METHODS ------------------------------ + +function toFormatedJSON () { + const json = { + id: this.id, + host: this.host, + score: this.score, + createdAt: this.createdAt + } + + return json +} + +// ------------------------------ Statics ------------------------------ + +function associate (models) { + this.belongsToMany(models.Request, { + foreignKey: 'podId', + through: models.RequestToPod, + onDelete: 'CASCADE' + }) +} + +function countAll (callback) { + return this.count().asCallback(callback) +} + +function incrementScores (ids, value, callback) { + if (!callback) callback = function () {} + + const update = { + score: this.sequelize.literal('score +' + value) + } + + const query = { + where: { + id: { + $in: ids + } + } + } + + return this.update(update, query).asCallback(callback) +} + +function list (callback) { + return this.findAll().asCallback(callback) +} + +function listAllIds (callback) { + const query = { + attributes: [ 'id' ] + } + + return this.findAll(query).asCallback(function (err, pods) { + if (err) return callback(err) + + return callback(null, map(pods, 'id')) + }) +} + +function listBadPods (callback) { + const query = { + where: { + score: { $lte: 0 } + } + } + + return this.findAll(query).asCallback(callback) +} + +function load (id, callback) { + return this.findById(id).asCallback(callback) +} + +function loadByHost (host, callback) { + const query = { + where: { + host: host + } + } + + return this.findOne(query).asCallback(callback) +} + +function removeAll (callback) { + return this.destroy().asCallback(callback) +} diff --git a/server/models/pods.js b/server/models/pods.js deleted file mode 100644 index 2c1f56203..000000000 --- a/server/models/pods.js +++ /dev/null @@ -1,139 +0,0 @@ -'use strict' - -const map = require('lodash/map') - -const constants = require('../initializers/constants') - -// --------------------------------------------------------------------------- - -module.exports = function (sequelize, DataTypes) { - const Pod = sequelize.define('Pod', - { - host: { - type: DataTypes.STRING - }, - publicKey: { - type: DataTypes.STRING(5000) - }, - score: { - type: DataTypes.INTEGER, - defaultValue: constants.FRIEND_SCORE.BASE - } - // Check createdAt - }, - { - classMethods: { - associate, - - countAll, - incrementScores, - list, - listAllIds, - listBadPods, - load, - loadByHost, - removeAll - }, - instanceMethods: { - toFormatedJSON - } - } - ) - - return Pod -} - -// TODO: max score -> constants.FRIENDS_SCORE.MAX -// TODO: validation -// PodSchema.path('host').validate(validator.isURL) -// PodSchema.path('publicKey').required(true) -// PodSchema.path('score').validate(function (value) { return !isNaN(value) }) - -// ------------------------------ METHODS ------------------------------ - -function toFormatedJSON () { - const json = { - id: this.id, - host: this.host, - score: this.score, - createdAt: this.createdAt - } - - return json -} - -// ------------------------------ Statics ------------------------------ - -function associate (models) { - this.belongsToMany(models.Request, { - foreignKey: 'podId', - through: models.RequestToPod, - onDelete: 'CASCADE' - }) -} - -function countAll (callback) { - return this.count().asCallback(callback) -} - -function incrementScores (ids, value, callback) { - if (!callback) callback = function () {} - - const update = { - score: this.sequelize.literal('score +' + value) - } - - const query = { - where: { - id: { - $in: ids - } - } - } - - return this.update(update, query).asCallback(callback) -} - -function list (callback) { - return this.findAll().asCallback(callback) -} - -function listAllIds (callback) { - const query = { - attributes: [ 'id' ] - } - - return this.findAll(query).asCallback(function (err, pods) { - if (err) return callback(err) - - return callback(null, map(pods, 'id')) - }) -} - -function listBadPods (callback) { - const query = { - where: { - score: { $lte: 0 } - } - } - - return this.findAll(query).asCallback(callback) -} - -function load (id, callback) { - return this.findById(id).asCallback(callback) -} - -function loadByHost (host, callback) { - const query = { - where: { - host: host - } - } - - return this.findOne(query).asCallback(callback) -} - -function removeAll (callback) { - return this.destroy().asCallback(callback) -} -- cgit v1.2.3 From 7920c273a204e2469416a30b752b12ccd3160102 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Sat, 24 Dec 2016 16:59:17 +0100 Subject: Move tags in another table --- server/controllers/api/remote.js | 95 +++++++++++++++++++++++++++++++------- server/controllers/api/videos.js | 99 ++++++++++++++++++++++++++++++++-------- server/controllers/client.js | 2 +- server/initializers/database.js | 13 +++++- server/lib/friends.js | 10 ++-- server/models/pod.js | 3 +- server/models/request.js | 6 ++- server/models/tag.js | 30 ++++++++++++ server/models/video.js | 97 +++++++++++++++++++++++---------------- server/models/videoTag.js | 9 ++++ server/tests/api/check-params.js | 4 +- server/tests/api/requests.js | 15 +++--- server/tests/api/single-pod.js | 90 ++++++++++++++++++------------------ 13 files changed, 334 insertions(+), 139 deletions(-) create mode 100644 server/models/tag.js create mode 100644 server/models/videoTag.js (limited to 'server') diff --git a/server/controllers/api/remote.js b/server/controllers/api/remote.js index a6753a2b0..c7a5658e8 100644 --- a/server/controllers/api/remote.js +++ b/server/controllers/api/remote.js @@ -50,28 +50,35 @@ function remoteVideos (req, res, next) { return res.type('json').status(204).end() } -function addRemoteVideo (videoToCreateData, fromHost, callback) { +function addRemoteVideo (videoToCreateData, fromHost, finalCallback) { logger.debug('Adding remote video "%s".', videoToCreateData.name) waterfall([ - function findOrCreatePod (callback) { + function startTransaction (callback) { + db.sequelize.transaction().asCallback(function (err, t) { + return callback(err, t) + }) + }, + + function findOrCreatePod (t, callback) { const query = { where: { host: fromHost }, defaults: { host: fromHost - } + }, + transaction: t } db.Pod.findOrCreate(query).asCallback(function (err, result) { // [ instance, wasCreated ] - return callback(err, result[0]) + return callback(err, t, result[0]) }) }, - function findOrCreateAuthor (pod, callback) { + function findOrCreateAuthor (t, pod, callback) { const username = videoToCreateData.author const query = { @@ -82,16 +89,45 @@ function addRemoteVideo (videoToCreateData, fromHost, callback) { defaults: { name: username, podId: pod.id - } + }, + transaction: t } db.Author.findOrCreate(query).asCallback(function (err, result) { // [ instance, wasCreated ] - return callback(err, result[0]) + return callback(err, t, result[0]) }) }, - function createVideoObject (author, callback) { + function findOrCreateTags (t, author, callback) { + const tags = videoToCreateData.tags + const tagInstances = [] + + each(tags, function (tag, callbackEach) { + const query = { + where: { + name: tag + }, + defaults: { + name: tag + }, + transaction: t + } + + db.Tag.findOrCreate(query).asCallback(function (err, res) { + if (err) return callbackEach(err) + + // res = [ tag, isCreated ] + const tag = res[0] + tagInstances.push(tag) + return callbackEach() + }) + }, function (err) { + return callback(err, t, author, tagInstances) + }) + }, + + function createVideoObject (t, author, tagInstances, callback) { const videoData = { name: videoToCreateData.name, remoteId: videoToCreateData.remoteId, @@ -99,31 +135,58 @@ function addRemoteVideo (videoToCreateData, fromHost, callback) { infoHash: videoToCreateData.infoHash, description: videoToCreateData.description, authorId: author.id, - duration: videoToCreateData.duration, - tags: videoToCreateData.tags + duration: videoToCreateData.duration } const video = db.Video.build(videoData) - return callback(null, video) + return callback(null, t, tagInstances, video) }, - function generateThumbnail (video, callback) { + function generateThumbnail (t, tagInstances, video, callback) { db.Video.generateThumbnailFromBase64(video, videoToCreateData.thumbnailBase64, function (err) { if (err) { logger.error('Cannot generate thumbnail from base 64 data.', { error: err }) return callback(err) } - video.save().asCallback(callback) + return callback(err, t, tagInstances, video) }) }, - function insertIntoDB (video, callback) { - video.save().asCallback(callback) + function insertVideoIntoDB (t, tagInstances, video, callback) { + const options = { + transaction: t + } + + video.save(options).asCallback(function (err, videoCreated) { + return callback(err, t, tagInstances, videoCreated) + }) + }, + + function associateTagsToVideo (t, tagInstances, video, callback) { + const options = { transaction: t } + + video.setTags(tagInstances, options).asCallback(function (err) { + return callback(err, t) + }) } - ], callback) + ], function (err, t) { + if (err) { + logger.error('Cannot insert the remote video.') + + // Abort transaction? + if (t) t.rollback() + + return finalCallback(err) + } + + // Commit transaction + t.commit() + + return finalCallback() + }) } function removeRemoteVideo (videoToRemoveData, fromHost, callback) { diff --git a/server/controllers/api/videos.js b/server/controllers/api/videos.js index a61f2b2c9..992f03db0 100644 --- a/server/controllers/api/videos.js +++ b/server/controllers/api/videos.js @@ -1,5 +1,6 @@ 'use strict' +const each = require('async/each') const express = require('express') const fs = require('fs') const multer = require('multer') @@ -87,7 +88,13 @@ function addVideo (req, res, next) { waterfall([ - function findOrCreateAuthor (callback) { + function startTransaction (callback) { + db.sequelize.transaction().asCallback(function (err, t) { + return callback(err, t) + }) + }, + + function findOrCreateAuthor (t, callback) { const username = res.locals.oauth.token.user.username const query = { @@ -98,75 +105,125 @@ function addVideo (req, res, next) { defaults: { name: username, podId: null // null because it is OUR pod - } + }, + transaction: t } db.Author.findOrCreate(query).asCallback(function (err, result) { // [ instance, wasCreated ] - return callback(err, result[0]) + return callback(err, t, result[0]) + }) + }, + + function findOrCreateTags (t, author, callback) { + const tags = videoInfos.tags + const tagInstances = [] + + each(tags, function (tag, callbackEach) { + const query = { + where: { + name: tag + }, + defaults: { + name: tag + }, + transaction: t + } + + db.Tag.findOrCreate(query).asCallback(function (err, res) { + if (err) return callbackEach(err) + + // res = [ tag, isCreated ] + const tag = res[0] + tagInstances.push(tag) + return callbackEach() + }) + }, function (err) { + return callback(err, t, author, tagInstances) }) }, - function createVideoObject (author, callback) { + function createVideoObject (t, author, tagInstances, callback) { const videoData = { name: videoInfos.name, remoteId: null, extname: path.extname(videoFile.filename), description: videoInfos.description, duration: videoFile.duration, - tags: videoInfos.tags, authorId: author.id } const video = db.Video.build(videoData) - return callback(null, author, video) + return callback(null, t, author, tagInstances, video) }, // Set the videoname the same as the id - function renameVideoFile (author, video, callback) { + function renameVideoFile (t, author, tagInstances, video, callback) { const videoDir = constants.CONFIG.STORAGE.VIDEOS_DIR const source = path.join(videoDir, videoFile.filename) const destination = path.join(videoDir, video.getVideoFilename()) fs.rename(source, destination, function (err) { - return callback(err, author, video) + return callback(err, t, author, tagInstances, video) }) }, - function insertIntoDB (author, video, callback) { - video.save().asCallback(function (err, videoCreated) { + function insertVideoIntoDB (t, author, tagInstances, video, callback) { + const options = { transaction: t } + + // Add tags association + video.save(options).asCallback(function (err, videoCreated) { + if (err) return callback(err) + // Do not forget to add Author informations to the created video videoCreated.Author = author - return callback(err, videoCreated) + return callback(err, t, tagInstances, videoCreated) }) }, - function sendToFriends (video, callback) { + function associateTagsToVideo (t, tagInstances, video, callback) { + const options = { transaction: t } + + video.setTags(tagInstances, options).asCallback(function (err) { + video.Tags = tagInstances + + return callback(err, t, video) + }) + }, + + function sendToFriends (t, video, callback) { video.toRemoteJSON(function (err, remoteVideo) { if (err) return callback(err) // Now we'll add the video's meta data to our friends friends.addVideoToFriends(remoteVideo) - return callback(null) + return callback(null, t) }) } - ], function andFinally (err) { + ], function andFinally (err, t) { if (err) { logger.error('Cannot insert the video.') + + // Abort transaction? + if (t) t.rollback() + return next(err) } + // Commit transaction + t.commit() + // TODO : include Location of the new video -> 201 return res.type('json').status(204).end() }) } function getVideo (req, res, next) { - db.Video.loadAndPopulateAuthorAndPod(req.params.id, function (err, video) { + db.Video.loadAndPopulateAuthorAndPodAndTags(req.params.id, function (err, video) { if (err) return next(err) if (!video) { @@ -222,12 +279,14 @@ function removeVideo (req, res, next) { } function searchVideos (req, res, next) { - db.Video.searchAndPopulateAuthorAndPod(req.params.value, req.query.field, req.query.start, req.query.count, req.query.sort, - function (err, videosList, videosTotal) { - if (err) return next(err) + db.Video.searchAndPopulateAuthorAndPodAndTags( + req.params.value, req.query.field, req.query.start, req.query.count, req.query.sort, + function (err, videosList, videosTotal) { + if (err) return next(err) - res.json(getFormatedVideos(videosList, videosTotal)) - }) + res.json(getFormatedVideos(videosList, videosTotal)) + } + ) } // --------------------------------------------------------------------------- diff --git a/server/controllers/client.js b/server/controllers/client.js index a5fac5626..8c242af07 100644 --- a/server/controllers/client.js +++ b/server/controllers/client.js @@ -93,7 +93,7 @@ function generateWatchHtmlPage (req, res, next) { }, video: function (callback) { - db.Video.loadAndPopulateAuthorAndPod(videoId, callback) + db.Video.loadAndPopulateAuthorAndPodAndTags(videoId, callback) } }, function (err, results) { if (err) return next(err) diff --git a/server/initializers/database.js b/server/initializers/database.js index cc6f59b63..9642231b9 100644 --- a/server/initializers/database.js +++ b/server/initializers/database.js @@ -6,13 +6,24 @@ const Sequelize = require('sequelize') const constants = require('../initializers/constants') const logger = require('../helpers/logger') +const utils = require('../helpers/utils') const database = {} const sequelize = new Sequelize(constants.CONFIG.DATABASE.DBNAME, 'peertube', 'peertube', { dialect: 'postgres', host: constants.CONFIG.DATABASE.HOSTNAME, - port: constants.CONFIG.DATABASE.PORT + port: constants.CONFIG.DATABASE.PORT, + benchmark: utils.isTestInstance(), + + logging: function (message, benchmark) { + let newMessage = message + if (benchmark !== undefined) { + newMessage += ' | ' + benchmark + 'ms' + } + + logger.debug(newMessage) + } }) const modelDirectory = path.join(__dirname, '..', 'models') diff --git a/server/lib/friends.js b/server/lib/friends.js index 3ed29f651..ad9e4fdae 100644 --- a/server/lib/friends.js +++ b/server/lib/friends.js @@ -66,10 +66,12 @@ function makeFriends (hosts, callback) { function quitFriends (callback) { // Stop pool requests db.Request.deactivate() - // Flush pool requests - db.Request.flush() waterfall([ + function flushRequests (callbackAsync) { + db.Request.flush(callbackAsync) + }, + function getPodsList (callbackAsync) { return db.Pod.list(callbackAsync) }, @@ -118,7 +120,7 @@ function removeVideoToFriends (videoParams) { } function sendOwnedVideosToPod (podId) { - db.Video.listOwnedAndPopulateAuthor(function (err, videosList) { + db.Video.listOwnedAndPopulateAuthorAndTags(function (err, videosList) { if (err) { logger.error('Cannot get the list of videos we own.') return @@ -226,7 +228,7 @@ function makeRequestsToWinningPods (cert, podsList, callback) { } // Add our videos to the request scheduler - sendOwnedVideosToPod(podCreated._id) + sendOwnedVideosToPod(podCreated.id) return callbackEach() }) diff --git a/server/models/pod.js b/server/models/pod.js index 2c1f56203..fff6970a7 100644 --- a/server/models/pod.js +++ b/server/models/pod.js @@ -19,7 +19,6 @@ module.exports = function (sequelize, DataTypes) { type: DataTypes.INTEGER, defaultValue: constants.FRIEND_SCORE.BASE } - // Check createdAt }, { classMethods: { @@ -68,7 +67,7 @@ function associate (models) { this.belongsToMany(models.Request, { foreignKey: 'podId', through: models.RequestToPod, - onDelete: 'CASCADE' + onDelete: 'cascade' }) } diff --git a/server/models/request.js b/server/models/request.js index 882f747b7..70aa32610 100644 --- a/server/models/request.js +++ b/server/models/request.js @@ -79,9 +79,11 @@ function deactivate () { timer = null } -function flush () { +function flush (callback) { removeAll.call(this, function (err) { if (err) logger.error('Cannot flush the requests.', { error: err }) + + return callback(err) }) } @@ -298,7 +300,7 @@ function listWithLimitAndRandom (limit, callback) { function removeAll (callback) { // Delete all requests - this.destroy({ truncate: true }).asCallback(callback) + this.truncate({ cascade: true }).asCallback(callback) } function removeWithEmptyTo (callback) { diff --git a/server/models/tag.js b/server/models/tag.js new file mode 100644 index 000000000..874e88842 --- /dev/null +++ b/server/models/tag.js @@ -0,0 +1,30 @@ +'use strict' + +// --------------------------------------------------------------------------- + +module.exports = function (sequelize, DataTypes) { + const Tag = sequelize.define('Tag', + { + name: { + type: DataTypes.STRING + } + }, + { + classMethods: { + associate + } + } + ) + + return Tag +} + +// --------------------------------------------------------------------------- + +function associate (models) { + this.belongsToMany(models.Video, { + foreignKey: 'tagId', + through: models.VideoTag, + onDelete: 'cascade' + }) +} diff --git a/server/models/video.js b/server/models/video.js index 8ef07c9e6..0023a24e1 100644 --- a/server/models/video.js +++ b/server/models/video.js @@ -4,6 +4,7 @@ const createTorrent = require('create-torrent') const ffmpeg = require('fluent-ffmpeg') const fs = require('fs') const magnetUtil = require('magnet-uri') +const map = require('lodash/map') const parallel = require('async/parallel') const parseTorrent = require('parse-torrent') const pathUtils = require('path') @@ -41,9 +42,6 @@ module.exports = function (sequelize, DataTypes) { }, duration: { type: DataTypes.INTEGER - }, - tags: { - type: DataTypes.ARRAY(DataTypes.STRING) } }, { @@ -54,12 +52,12 @@ module.exports = function (sequelize, DataTypes) { getDurationFromFile, listForApi, listByHostAndRemoteId, - listOwnedAndPopulateAuthor, + listOwnedAndPopulateAuthorAndTags, listOwnedByAuthor, load, loadAndPopulateAuthor, - loadAndPopulateAuthorAndPod, - searchAndPopulateAuthorAndPod + loadAndPopulateAuthorAndPodAndTags, + searchAndPopulateAuthorAndPodAndTags }, instanceMethods: { generateMagnetUri, @@ -170,6 +168,12 @@ function associate (models) { }, onDelete: 'cascade' }) + + this.belongsToMany(models.Tag, { + foreignKey: 'videoId', + through: models.VideoTag, + onDelete: 'cascade' + }) } function generateMagnetUri () { @@ -248,7 +252,7 @@ function toFormatedJSON () { magnetUri: this.generateMagnetUri(), author: this.Author.name, duration: this.duration, - tags: this.tags, + tags: map(this.Tags, 'name'), thumbnailPath: constants.STATIC_PATHS.THUMBNAILS + '/' + this.getThumbnailName(), createdAt: this.createdAt } @@ -275,7 +279,7 @@ function toRemoteJSON (callback) { author: self.Author.name, duration: self.duration, thumbnailBase64: new Buffer(thumbnailData).toString('base64'), - tags: self.tags, + tags: map(self.Tags, 'name'), createdAt: self.createdAt, extname: self.extname } @@ -310,12 +314,15 @@ function listForApi (start, count, sort, callback) { const query = { offset: start, limit: count, + distinct: true, // For the count, a video can have many tags order: [ modelUtils.getSort(sort) ], include: [ { model: this.sequelize.models.Author, - include: [ this.sequelize.models.Pod ] - } + include: [ { model: this.sequelize.models.Pod, required: false } ] + }, + + this.sequelize.models.Tag ] } @@ -337,6 +344,7 @@ function listByHostAndRemoteId (fromHost, remoteId, callback) { include: [ { model: this.sequelize.models.Pod, + required: true, where: { host: fromHost } @@ -349,13 +357,13 @@ function listByHostAndRemoteId (fromHost, remoteId, callback) { return this.findAll(query).asCallback(callback) } -function listOwnedAndPopulateAuthor (callback) { +function listOwnedAndPopulateAuthorAndTags (callback) { // If remoteId is null this is *our* video const query = { where: { remoteId: null }, - include: [ this.sequelize.models.Author ] + include: [ this.sequelize.models.Author, this.sequelize.models.Tag ] } return this.findAll(query).asCallback(callback) @@ -391,23 +399,26 @@ function loadAndPopulateAuthor (id, callback) { return this.findById(id, options).asCallback(callback) } -function loadAndPopulateAuthorAndPod (id, callback) { +function loadAndPopulateAuthorAndPodAndTags (id, callback) { const options = { include: [ { model: this.sequelize.models.Author, - include: [ this.sequelize.models.Pod ] - } + include: [ { model: this.sequelize.models.Pod, required: false } ] + }, + this.sequelize.models.Tag ] } return this.findById(id, options).asCallback(callback) } -function searchAndPopulateAuthorAndPod (value, field, start, count, sort, callback) { +function searchAndPopulateAuthorAndPodAndTags (value, field, start, count, sort, callback) { const podInclude = { - model: this.sequelize.models.Pod + model: this.sequelize.models.Pod, + required: false } + const authorInclude = { model: this.sequelize.models.Author, include: [ @@ -415,55 +426,61 @@ function searchAndPopulateAuthorAndPod (value, field, start, count, sort, callba ] } + const tagInclude = { + model: this.sequelize.models.Tag + } + const query = { where: {}, - include: [ - authorInclude - ], offset: start, limit: count, + distinct: true, // For the count, a video can have many tags order: [ modelUtils.getSort(sort) ] } - // TODO: include our pod for podHost searches (we are not stored in the database) // Make an exact search with the magnet if (field === 'magnetUri') { const infoHash = magnetUtil.decode(value).infoHash query.where.infoHash = infoHash } else if (field === 'tags') { - query.where[field] = value - } else if (field === 'host') { - const whereQuery = { - '$Author.Pod.host$': { - $like: '%' + value + '%' + const escapedValue = this.sequelize.escape('%' + value + '%') + query.where = { + id: { + $in: this.sequelize.literal( + '(SELECT "VideoTags"."videoId" FROM "Tags" INNER JOIN "VideoTags" ON "Tags"."id" = "VideoTags"."tagId" WHERE name LIKE ' + escapedValue + ')' + ) } } - - // Include our pod? (not stored in the database) - if (constants.CONFIG.WEBSERVER.HOST.indexOf(value) !== -1) { - query.where = { - $or: [ - whereQuery, - { - remoteId: null - } - ] + } else if (field === 'host') { + // FIXME: Include our pod? (not stored in the database) + podInclude.where = { + host: { + $like: '%' + value + '%' } - } else { - query.where = whereQuery } + podInclude.required = true } else if (field === 'author') { - query.where = { - '$Author.name$': { + authorInclude.where = { + name: { $like: '%' + value + '%' } } + + // authorInclude.or = true } else { query.where[field] = { $like: '%' + value + '%' } } + query.include = [ + authorInclude, tagInclude + ] + + if (tagInclude.where) { + // query.include.push([ this.sequelize.models.Tag ]) + } + return this.findAndCountAll(query).asCallback(function (err, result) { if (err) return callback(err) diff --git a/server/models/videoTag.js b/server/models/videoTag.js new file mode 100644 index 000000000..0f2b20838 --- /dev/null +++ b/server/models/videoTag.js @@ -0,0 +1,9 @@ +'use strict' + +// --------------------------------------------------------------------------- + +module.exports = function (sequelize, DataTypes) { + const VideoTag = sequelize.define('VideoTag', {}, {}) + + return VideoTag +} diff --git a/server/tests/api/check-params.js b/server/tests/api/check-params.js index d9e51770c..9aecc3720 100644 --- a/server/tests/api/check-params.js +++ b/server/tests/api/check-params.js @@ -456,7 +456,7 @@ describe('Test parameters validator', function () { }) }) - it('Should fail without a mongodb id', function (done) { + it('Should fail without a correct uuid', function (done) { request(server.url) .get(path + 'coucou') .set('Accept', 'application/json') @@ -481,7 +481,7 @@ describe('Test parameters validator', function () { .expect(400, done) }) - it('Should fail without a mongodb id', function (done) { + it('Should fail without a correct uuid', function (done) { request(server.url) .delete(path + 'hello') .set('Authorization', 'Bearer ' + server.accessToken) diff --git a/server/tests/api/requests.js b/server/tests/api/requests.js index 7e790b54b..933ed29b4 100644 --- a/server/tests/api/requests.js +++ b/server/tests/api/requests.js @@ -79,15 +79,16 @@ describe('Test requests stats', function () { uploadVideo(server, function (err) { if (err) throw err - getRequestsStats(server, function (err, res) { - if (err) throw err + setTimeout(function () { + getRequestsStats(server, function (err, res) { + if (err) throw err - const body = res.body - expect(body.totalRequests).to.equal(1) + const body = res.body + expect(body.totalRequests).to.equal(1) - // Wait one cycle - setTimeout(done, 10000) - }) + done() + }) + }, 1000) }) }) diff --git a/server/tests/api/single-pod.js b/server/tests/api/single-pod.js index aedecacf3..66b762f82 100644 --- a/server/tests/api/single-pod.js +++ b/server/tests/api/single-pod.js @@ -153,31 +153,32 @@ describe('Test a single pod', function () { }) }) - it('Should search the video by podHost', function (done) { - videosUtils.searchVideo(server.url, '9001', 'host', function (err, res) { - if (err) throw err - - expect(res.body.total).to.equal(1) - expect(res.body.data).to.be.an('array') - expect(res.body.data.length).to.equal(1) - - const video = res.body.data[0] - expect(video.name).to.equal('my super name') - expect(video.description).to.equal('my super description') - expect(video.podHost).to.equal('localhost:9001') - expect(video.author).to.equal('root') - expect(video.isLocal).to.be.true - expect(video.tags).to.deep.equal([ 'tag1', 'tag2', 'tag3' ]) - expect(miscsUtils.dateIsValid(video.createdAt)).to.be.true - - videosUtils.testVideoImage(server.url, 'video_short.webm', video.thumbnailPath, function (err, test) { - if (err) throw err - expect(test).to.equal(true) - - done() - }) - }) - }) + // Not implemented yet + // it('Should search the video by podHost', function (done) { + // videosUtils.searchVideo(server.url, '9001', 'host', function (err, res) { + // if (err) throw err + + // expect(res.body.total).to.equal(1) + // expect(res.body.data).to.be.an('array') + // expect(res.body.data.length).to.equal(1) + + // const video = res.body.data[0] + // expect(video.name).to.equal('my super name') + // expect(video.description).to.equal('my super description') + // expect(video.podHost).to.equal('localhost:9001') + // expect(video.author).to.equal('root') + // expect(video.isLocal).to.be.true + // expect(video.tags).to.deep.equal([ 'tag1', 'tag2', 'tag3' ]) + // expect(miscsUtils.dateIsValid(video.createdAt)).to.be.true + + // videosUtils.testVideoImage(server.url, 'video_short.webm', video.thumbnailPath, function (err, test) { + // if (err) throw err + // expect(test).to.equal(true) + + // done() + // }) + // }) + // }) it('Should search the video by tag', function (done) { videosUtils.searchVideo(server.url, 'tag1', 'tags', function (err, res) { @@ -230,7 +231,7 @@ describe('Test a single pod', function () { }) it('Should not find a search by tag', function (done) { - videosUtils.searchVideo(server.url, 'tag', 'tags', function (err, res) { + videosUtils.searchVideo(server.url, 'hello', 'tags', function (err, res) { if (err) throw err expect(res.body.total).to.equal(0) @@ -424,29 +425,30 @@ describe('Test a single pod', function () { }) }) - it('Should search all the 9001 port videos', function (done) { - videosUtils.searchVideoWithPagination(server.url, '9001', 'host', 0, 15, function (err, res) { - if (err) throw err + // Not implemented yet + // it('Should search all the 9001 port videos', function (done) { + // videosUtils.searchVideoWithPagination(server.url, '9001', 'host', 0, 15, function (err, res) { + // if (err) throw err - const videos = res.body.data - expect(res.body.total).to.equal(6) - expect(videos.length).to.equal(6) + // const videos = res.body.data + // expect(res.body.total).to.equal(6) + // expect(videos.length).to.equal(6) - done() - }) - }) + // done() + // }) + // }) - it('Should search all the localhost videos', function (done) { - videosUtils.searchVideoWithPagination(server.url, 'localhost', 'host', 0, 15, function (err, res) { - if (err) throw err + // it('Should search all the localhost videos', function (done) { + // videosUtils.searchVideoWithPagination(server.url, 'localhost', 'host', 0, 15, function (err, res) { + // if (err) throw err - const videos = res.body.data - expect(res.body.total).to.equal(6) - expect(videos.length).to.equal(6) + // const videos = res.body.data + // expect(res.body.total).to.equal(6) + // expect(videos.length).to.equal(6) - done() - }) - }) + // done() + // }) + // }) it('Should search the good magnetUri video', function (done) { const video = videosListBase[0] -- cgit v1.2.3 From b769007f733769d3afe2d29a3eb23e2e7693f301 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Sun, 25 Dec 2016 09:44:57 +0100 Subject: Update migrations code --- server/initializers/checker.js | 2 +- server/initializers/constants.js | 43 ++--------- server/initializers/database.js | 65 ++++++++++------ server/initializers/installer.js | 5 +- .../migrations/0005-create-application.js | 17 ----- server/initializers/migrations/0005-example.js | 14 ++++ .../initializers/migrations/0010-users-password.js | 22 ------ server/initializers/migrations/0015-admin-role.js | 16 ---- .../migrations/0020-requests-endpoint.js | 15 ---- .../migrations/0025-video-filenames.js | 57 -------------- .../initializers/migrations/0030-video-magnet.js | 32 -------- server/initializers/migrations/0035-url-to-host.js | 30 -------- .../migrations/0040-video-remote-id.js | 59 --------------- server/initializers/migrator.js | 88 ++++++++++++++++------ server/models/application.js | 26 +++++-- server/models/video.js | 7 +- server/tests/utils/servers.js | 2 +- 17 files changed, 155 insertions(+), 345 deletions(-) delete mode 100644 server/initializers/migrations/0005-create-application.js create mode 100644 server/initializers/migrations/0005-example.js delete mode 100644 server/initializers/migrations/0010-users-password.js delete mode 100644 server/initializers/migrations/0015-admin-role.js delete mode 100644 server/initializers/migrations/0020-requests-endpoint.js delete mode 100644 server/initializers/migrations/0025-video-filenames.js delete mode 100644 server/initializers/migrations/0030-video-magnet.js delete mode 100644 server/initializers/migrations/0035-url-to-host.js delete mode 100644 server/initializers/migrations/0040-video-remote-id.js (limited to 'server') diff --git a/server/initializers/checker.js b/server/initializers/checker.js index 7b402de82..2753604dc 100644 --- a/server/initializers/checker.js +++ b/server/initializers/checker.js @@ -27,7 +27,7 @@ function checkConfig () { function checkMissedConfig () { const required = [ 'listen.port', 'webserver.https', 'webserver.hostname', 'webserver.port', - 'database.hostname', 'database.port', 'database.suffix', + 'database.hostname', 'database.port', 'database.suffix', 'database.username', 'database.password', 'storage.certs', 'storage.videos', 'storage.logs', 'storage.thumbnails', 'storage.previews' ] const miss = [] diff --git a/server/initializers/constants.js b/server/initializers/constants.js index 1ad0c82a0..6f39b65da 100644 --- a/server/initializers/constants.js +++ b/server/initializers/constants.js @@ -37,7 +37,9 @@ const CONFIG = { DATABASE: { DBNAME: 'peertube' + config.get('database.suffix'), HOSTNAME: config.get('database.hostname'), - PORT: config.get('database.port') + PORT: config.get('database.port'), + USERNAME: config.get('database.username'), + PASSWORD: config.get('database.password') }, STORAGE: { CERT_DIR: path.join(__dirname, '..', '..', config.get('storage.certs')), @@ -87,41 +89,7 @@ const FRIEND_SCORE = { // --------------------------------------------------------------------------- -const MIGRATION_SCRIPTS = [ - { - script: '0005-create-application', - version: 5 - }, - { - script: '0010-users-password', - version: 10 - }, - { - script: '0015-admin-role', - version: 15 - }, - { - script: '0020-requests-endpoint', - version: 20 - }, - { - script: '0025-video-filenames', - version: 25 - }, - { - script: '0030-video-magnet', - version: 30 - }, - { - script: '0035-url-to-host', - version: 35 - }, - { - script: '0040-video-remote-id', - version: 40 - } -] -const LAST_SQL_SCHEMA_VERSION = (maxBy(MIGRATION_SCRIPTS, 'version'))['version'] +const LAST_MIGRATION_VERSION = 0 // --------------------------------------------------------------------------- @@ -197,8 +165,7 @@ module.exports = { CONFIG, CONSTRAINTS_FIELDS, FRIEND_SCORE, - LAST_SQL_SCHEMA_VERSION, - MIGRATION_SCRIPTS, + LAST_MIGRATION_VERSION, OAUTH_LIFETIME, PAGINATION_COUNT_DEFAULT, PODS_SCORE, diff --git a/server/initializers/database.js b/server/initializers/database.js index 9642231b9..f8f68adeb 100644 --- a/server/initializers/database.js +++ b/server/initializers/database.js @@ -10,7 +10,11 @@ const utils = require('../helpers/utils') const database = {} -const sequelize = new Sequelize(constants.CONFIG.DATABASE.DBNAME, 'peertube', 'peertube', { +const dbname = constants.CONFIG.DATABASE.DBNAME +const username = constants.CONFIG.DATABASE.USERNAME +const password = constants.CONFIG.DATABASE.PASSWORD + +const sequelize = new Sequelize(dbname, username, password, { dialect: 'postgres', host: constants.CONFIG.DATABASE.HOSTNAME, port: constants.CONFIG.DATABASE.PORT, @@ -26,33 +30,48 @@ const sequelize = new Sequelize(constants.CONFIG.DATABASE.DBNAME, 'peertube', 'p } }) -const modelDirectory = path.join(__dirname, '..', 'models') -fs.readdir(modelDirectory, function (err, files) { - if (err) throw err +database.sequelize = sequelize +database.Sequelize = Sequelize +database.init = init - files.filter(function (file) { - if (file === 'utils.js') return false +// --------------------------------------------------------------------------- - return true - }) - .forEach(function (file) { - const model = sequelize.import(path.join(modelDirectory, file)) +module.exports = database - database[model.name] = model - }) +// --------------------------------------------------------------------------- - Object.keys(database).forEach(function (modelName) { - if ('associate' in database[modelName]) { - database[modelName].associate(database) - } - }) +function init (silent, callback) { + if (!callback) { + callback = silent + silent = false + } - logger.info('Database is ready.') -}) + if (!callback) callback = function () {} -database.sequelize = sequelize -database.Sequelize = Sequelize + const modelDirectory = path.join(__dirname, '..', 'models') + fs.readdir(modelDirectory, function (err, files) { + if (err) throw err -// --------------------------------------------------------------------------- + files.filter(function (file) { + // For all models but not utils.js + if (file === 'utils.js') return false -module.exports = database + return true + }) + .forEach(function (file) { + const model = sequelize.import(path.join(modelDirectory, file)) + + database[model.name] = model + }) + + Object.keys(database).forEach(function (modelName) { + if ('associate' in database[modelName]) { + database[modelName].associate(database) + } + }) + + if (!silent) logger.info('Database is ready.') + + return callback(null) + }) +} diff --git a/server/initializers/installer.js b/server/initializers/installer.js index 4823bc8c8..d5382364e 100644 --- a/server/initializers/installer.js +++ b/server/initializers/installer.js @@ -121,9 +121,8 @@ function createOAuthAdminIfNotExist (callback) { logger.info('Username: ' + username) logger.info('User password: ' + password) - logger.info('Creating Application collection.') - const application = db.Application.build({ sqlSchemaVersion: constants.LAST_SQL_SCHEMA_VERSION }) - application.save().asCallback(callback) + logger.info('Creating Application table.') + db.Application.create({ migrationVersion: constants.LAST_MIGRATION_VERSION }).asCallback(callback) }) }) } diff --git a/server/initializers/migrations/0005-create-application.js b/server/initializers/migrations/0005-create-application.js deleted file mode 100644 index e99dec019..000000000 --- a/server/initializers/migrations/0005-create-application.js +++ /dev/null @@ -1,17 +0,0 @@ -/* - Create the application collection in MongoDB. - Used to store the actual MongoDB scheme version. -*/ - -const mongoose = require('mongoose') - -const Application = mongoose.model('Application') - -exports.up = function (callback) { - const application = new Application() - application.save(callback) -} - -exports.down = function (callback) { - throw new Error('Not implemented.') -} diff --git a/server/initializers/migrations/0005-example.js b/server/initializers/migrations/0005-example.js new file mode 100644 index 000000000..481c2c4dd --- /dev/null +++ b/server/initializers/migrations/0005-example.js @@ -0,0 +1,14 @@ +/* + This is just an example. +*/ + +const db = require('../database') + +// options contains the transaction +exports.up = function (options, callback) { + // db.Application.create({ migrationVersion: 42 }, { transaction: options.transaction }).asCallback(callback) +} + +exports.down = function (options, callback) { + throw new Error('Not implemented.') +} diff --git a/server/initializers/migrations/0010-users-password.js b/server/initializers/migrations/0010-users-password.js deleted file mode 100644 index a0616a269..000000000 --- a/server/initializers/migrations/0010-users-password.js +++ /dev/null @@ -1,22 +0,0 @@ -/* - Convert plain user password to encrypted user password. -*/ - -const eachSeries = require('async/eachSeries') -const mongoose = require('mongoose') - -const User = mongoose.model('User') - -exports.up = function (callback) { - User.list(function (err, users) { - if (err) return callback(err) - - eachSeries(users, function (user, callbackEach) { - user.save(callbackEach) - }, callback) - }) -} - -exports.down = function (callback) { - throw new Error('Not implemented.') -} diff --git a/server/initializers/migrations/0015-admin-role.js b/server/initializers/migrations/0015-admin-role.js deleted file mode 100644 index af06dca9e..000000000 --- a/server/initializers/migrations/0015-admin-role.js +++ /dev/null @@ -1,16 +0,0 @@ -/* - Set the admin role to the root user. -*/ - -const constants = require('../constants') -const mongoose = require('mongoose') - -const User = mongoose.model('User') - -exports.up = function (callback) { - User.update({ username: 'root' }, { role: constants.USER_ROLES.ADMIN }, callback) -} - -exports.down = function (callback) { - throw new Error('Not implemented.') -} diff --git a/server/initializers/migrations/0020-requests-endpoint.js b/server/initializers/migrations/0020-requests-endpoint.js deleted file mode 100644 index 55feec571..000000000 --- a/server/initializers/migrations/0020-requests-endpoint.js +++ /dev/null @@ -1,15 +0,0 @@ -/* - Set the endpoint videos for requests. -*/ - -const mongoose = require('mongoose') - -const Request = mongoose.model('Request') - -exports.up = function (callback) { - Request.update({ }, { endpoint: 'videos' }, callback) -} - -exports.down = function (callback) { - throw new Error('Not implemented.') -} diff --git a/server/initializers/migrations/0025-video-filenames.js b/server/initializers/migrations/0025-video-filenames.js deleted file mode 100644 index df21494d7..000000000 --- a/server/initializers/migrations/0025-video-filenames.js +++ /dev/null @@ -1,57 +0,0 @@ -/* - Rename thumbnails and video filenames to _id.extension -*/ - -const each = require('async/each') -const fs = require('fs') -const path = require('path') -const mongoose = require('mongoose') - -const constants = require('../constants') -const logger = require('../../helpers/logger') - -const Video = mongoose.model('Video') - -exports.up = function (callback) { - // Use of lean because the new Video scheme does not have filename field - Video.find({ filename: { $ne: null } }).lean().exec(function (err, videos) { - if (err) throw err - - each(videos, function (video, callbackEach) { - const torrentName = video.filename + '.torrent' - const thumbnailName = video.thumbnail - const thumbnailExtension = path.extname(thumbnailName) - const videoName = video.filename - const videoExtension = path.extname(videoName) - - const newTorrentName = video._id + '.torrent' - const newThumbnailName = video._id + thumbnailExtension - const newVideoName = video._id + videoExtension - - const torrentsDir = constants.CONFIG.STORAGE.TORRENTS_DIR - const thumbnailsDir = constants.CONFIG.STORAGE.THUMBNAILS_DIR - const videosDir = constants.CONFIG.STORAGE.VIDEOS_DIR - - logger.info('Renaming %s to %s.', torrentsDir + torrentName, torrentsDir + newTorrentName) - fs.renameSync(torrentsDir + torrentName, torrentsDir + newTorrentName) - - logger.info('Renaming %s to %s.', thumbnailsDir + thumbnailName, thumbnailsDir + newThumbnailName) - fs.renameSync(thumbnailsDir + thumbnailName, thumbnailsDir + newThumbnailName) - - logger.info('Renaming %s to %s.', videosDir + videoName, videosDir + newVideoName) - fs.renameSync(videosDir + videoName, videosDir + newVideoName) - - Video.load(video._id, function (err, videoObj) { - if (err) return callbackEach(err) - - videoObj.extname = videoExtension - videoObj.remoteId = null - videoObj.save(callbackEach) - }) - }, callback) - }) -} - -exports.down = function (callback) { - throw new Error('Not implemented.') -} diff --git a/server/initializers/migrations/0030-video-magnet.js b/server/initializers/migrations/0030-video-magnet.js deleted file mode 100644 index b9119d61c..000000000 --- a/server/initializers/migrations/0030-video-magnet.js +++ /dev/null @@ -1,32 +0,0 @@ -/* - Change video magnet structures -*/ - -const each = require('async/each') -const magnet = require('magnet-uri') -const mongoose = require('mongoose') - -const Video = mongoose.model('Video') - -exports.up = function (callback) { - // Use of lean because the new Video scheme does not have magnetUri field - Video.find({ }).lean().exec(function (err, videos) { - if (err) throw err - - each(videos, function (video, callbackEach) { - const parsed = magnet.decode(video.magnetUri) - const infoHash = parsed.infoHash - - Video.load(video._id, function (err, videoObj) { - if (err) return callbackEach(err) - - videoObj.magnet.infoHash = infoHash - videoObj.save(callbackEach) - }) - }, callback) - }) -} - -exports.down = function (callback) { - throw new Error('Not implemented.') -} diff --git a/server/initializers/migrations/0035-url-to-host.js b/server/initializers/migrations/0035-url-to-host.js deleted file mode 100644 index 6243304d5..000000000 --- a/server/initializers/migrations/0035-url-to-host.js +++ /dev/null @@ -1,30 +0,0 @@ -/* - Change video magnet structures -*/ - -const each = require('async/each') -const mongoose = require('mongoose') - -const Video = mongoose.model('Video') - -exports.up = function (callback) { - // Use of lean because the new Video scheme does not have podUrl field - Video.find({ }).lean().exec(function (err, videos) { - if (err) throw err - - each(videos, function (video, callbackEach) { - Video.load(video._id, function (err, videoObj) { - if (err) return callbackEach(err) - - const host = video.podUrl.split('://')[1] - - videoObj.podHost = host - videoObj.save(callbackEach) - }) - }, callback) - }) -} - -exports.down = function (callback) { - throw new Error('Not implemented.') -} diff --git a/server/initializers/migrations/0040-video-remote-id.js b/server/initializers/migrations/0040-video-remote-id.js deleted file mode 100644 index 46a14a689..000000000 --- a/server/initializers/migrations/0040-video-remote-id.js +++ /dev/null @@ -1,59 +0,0 @@ -/* - Use remote id as identifier -*/ - -const map = require('lodash/map') -const mongoose = require('mongoose') -const readline = require('readline') - -const rl = readline.createInterface({ - input: process.stdin, - output: process.stdout -}) - -const logger = require('../../helpers/logger') -const friends = require('../../lib/friends') - -const Pod = mongoose.model('Pod') -const Video = mongoose.model('Video') - -exports.up = function (callback) { - Pod.find({}).lean().exec(function (err, pods) { - if (err) return callback(err) - - // We need to quit friends first - if (pods.length === 0) { - return setVideosRemoteId(callback) - } - - const timeout = setTimeout(function () { - throw new Error('You need to enter a value!') - }, 10000) - - rl.question('I am sorry but I need to quit friends for upgrading. Do you want to continue? (yes/*)', function (answer) { - if (answer !== 'yes') throw new Error('I cannot continue.') - - clearTimeout(timeout) - rl.close() - - const urls = map(pods, 'url') - logger.info('Saying goodbye to: ' + urls.join(', ')) - - setVideosRemoteId(function () { - friends.quitFriends(callback) - }) - }) - }) -} - -exports.down = function (callback) { - throw new Error('Not implemented.') -} - -function setVideosRemoteId (callback) { - Video.update({ filename: { $ne: null } }, { remoteId: null }, function (err) { - if (err) throw err - - Video.update({ filename: null }, { remoteId: mongoose.Types.ObjectId() }, callback) - }) -} diff --git a/server/initializers/migrator.js b/server/initializers/migrator.js index 9e5350e60..eaecb4936 100644 --- a/server/initializers/migrator.js +++ b/server/initializers/migrator.js @@ -1,6 +1,7 @@ 'use strict' const eachSeries = require('async/eachSeries') +const fs = require('fs') const path = require('path') const constants = require('./constants') @@ -12,35 +13,24 @@ const migrator = { } function migrate (callback) { - db.Application.loadSqlSchemaVersion(function (err, actualVersion) { + db.Application.loadMigrationVersion(function (err, actualVersion) { if (err) return callback(err) - // If there are a new mongo schemas - if (!actualVersion || actualVersion < constants.LAST_SQL_SCHEMA_VERSION) { + // If there are a new migration scripts + if (actualVersion < constants.LAST_MIGRATION_VERSION) { logger.info('Begin migrations.') - eachSeries(constants.MONGO_MIGRATION_SCRIPTS, function (entity, callbackEach) { - const versionScript = entity.version - - // Do not execute old migration scripts - if (versionScript <= actualVersion) return callbackEach(null) - - // Load the migration module and run it - const migrationScriptName = entity.script - logger.info('Executing %s migration script.', migrationScriptName) + getMigrationScripts(function (err, migrationScripts) { + if (err) return callback(err) - const migrationScript = require(path.join(__dirname, 'migrations', migrationScriptName)) - migrationScript.up(function (err) { - if (err) return callbackEach(err) + eachSeries(migrationScripts, function (entity, callbackEach) { + executeMigration(actualVersion, entity, callbackEach) + }, function (err) { + if (err) return callback(err) - // Update the new mongo version schema - db.Application.updateSqlSchemaVersion(versionScript, callbackEach) + logger.info('Migrations finished. New migration version schema: %s', constants.LAST_MIGRATION_VERSION) + return callback(null) }) - }, function (err) { - if (err) return callback(err) - - logger.info('Migrations finished. New SQL version schema: %s', constants.LAST_SQL_SCHEMA_VERSION) - return callback(null) }) } else { return callback(null) @@ -52,3 +42,57 @@ function migrate (callback) { module.exports = migrator +// --------------------------------------------------------------------------- + +function getMigrationScripts (callback) { + fs.readdir(path.join(__dirname, 'migrations'), function (err, files) { + if (err) return callback(err) + + const filesToMigrate = [] + + files.forEach(function (file) { + // Filename is something like 'version-blabla.js' + const version = file.split('-')[0] + filesToMigrate.push({ + version, + script: file + }) + }) + + return callback(err, filesToMigrate) + }) +} + +function executeMigration (actualVersion, entity, callback) { + const versionScript = entity.version + + // Do not execute old migration scripts + if (versionScript <= actualVersion) return callback(null) + + // Load the migration module and run it + const migrationScriptName = entity.script + logger.info('Executing %s migration script.', migrationScriptName) + + const migrationScript = require(path.join(__dirname, 'migrations', migrationScriptName)) + + db.sequelize.transaction().asCallback(function (err, t) { + if (err) return callback(err) + + migrationScript.up({ transaction: t }, function (err) { + if (err) { + t.rollback() + return callback(err) + } + + // Update the new migration version + db.Application.updateMigrationVersion(versionScript, t, function (err) { + if (err) { + t.rollback() + return callback(err) + } + + t.commit() + }) + }) + }) +} diff --git a/server/models/application.js b/server/models/application.js index ec1d7b122..4114ed76d 100644 --- a/server/models/application.js +++ b/server/models/application.js @@ -1,15 +1,15 @@ module.exports = function (sequelize, DataTypes) { const Application = sequelize.define('Application', { - sqlSchemaVersion: { + migrationVersion: { type: DataTypes.INTEGER, defaultValue: 0 } }, { classMethods: { - loadSqlSchemaVersion, - updateSqlSchemaVersion + loadMigrationVersion, + updateMigrationVersion } } ) @@ -19,18 +19,28 @@ module.exports = function (sequelize, DataTypes) { // --------------------------------------------------------------------------- -function loadSqlSchemaVersion (callback) { +function loadMigrationVersion (callback) { const query = { - attributes: [ 'sqlSchemaVersion' ] + attributes: [ 'migrationVersion' ] } return this.findOne(query).asCallback(function (err, data) { - const version = data ? data.sqlSchemaVersion : 0 + const version = data ? data.migrationVersion : 0 return callback(err, version) }) } -function updateSqlSchemaVersion (newVersion, callback) { - return this.update({ sqlSchemaVersion: newVersion }).asCallback(callback) +function updateMigrationVersion (newVersion, transaction, callback) { + const options = { + where: {} + } + + if (!callback) { + transaction = callback + } else { + options.transaction = transaction + } + + return this.update({ migrationVersion: newVersion }, options).asCallback(callback) } diff --git a/server/models/video.js b/server/models/video.js index 0023a24e1..af05a861f 100644 --- a/server/models/video.js +++ b/server/models/video.js @@ -16,7 +16,7 @@ const modelUtils = require('./utils') // --------------------------------------------------------------------------- module.exports = function (sequelize, DataTypes) { -// TODO: add indexes on searchable columns + // TODO: add indexes on searchable columns const Video = sequelize.define('Video', { id: { @@ -50,6 +50,7 @@ module.exports = function (sequelize, DataTypes) { generateThumbnailFromBase64, getDurationFromFile, + list, listForApi, listByHostAndRemoteId, listOwnedAndPopulateAuthorAndTags, @@ -310,6 +311,10 @@ function getDurationFromFile (videoPath, callback) { }) } +function list (callback) { + return this.find().asCallback() +} + function listForApi (start, count, sort, callback) { const query = { offset: start, diff --git a/server/tests/utils/servers.js b/server/tests/utils/servers.js index 4e55f8f5c..e7c756499 100644 --- a/server/tests/utils/servers.js +++ b/server/tests/utils/servers.js @@ -103,7 +103,7 @@ function runServer (number, callback) { if (serverRunString[key] === false) dontContinue = true } - // If no, there is maybe one thing not already initialized (mongodb...) + // If no, there is maybe one thing not already initialized (client/user credentials generation...) if (dontContinue === true) return server.app.stdout.removeListener('data', onStdout) -- cgit v1.2.3 From f83e27958109b829ba6326efda0679cc032003e5 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Sun, 25 Dec 2016 09:47:49 +0100 Subject: Fix standard lint --- server/helpers/peertube-crypto.js | 10 ---------- server/initializers/constants.js | 1 - server/initializers/migrations/0005-example.js | 22 +++++++++++----------- 3 files changed, 11 insertions(+), 22 deletions(-) (limited to 'server') diff --git a/server/helpers/peertube-crypto.js b/server/helpers/peertube-crypto.js index 2e07df00e..302ddca58 100644 --- a/server/helpers/peertube-crypto.js +++ b/server/helpers/peertube-crypto.js @@ -9,8 +9,6 @@ const ursa = require('ursa') const constants = require('../initializers/constants') const logger = require('./logger') -const algorithm = 'aes-256-ctr' - const peertubeCrypto = { checkSignature, comparePassword, @@ -113,11 +111,3 @@ function createCerts (callback) { }) }) } - -function generatePassword (callback) { - crypto.randomBytes(32, function (err, buf) { - if (err) return callback(err) - - callback(null, buf.toString('utf8')) - }) -} diff --git a/server/initializers/constants.js b/server/initializers/constants.js index 6f39b65da..fc501845a 100644 --- a/server/initializers/constants.js +++ b/server/initializers/constants.js @@ -1,7 +1,6 @@ 'use strict' const config = require('config') -const maxBy = require('lodash/maxBy') const path = require('path') // --------------------------------------------------------------------------- diff --git a/server/initializers/migrations/0005-example.js b/server/initializers/migrations/0005-example.js index 481c2c4dd..cedc42919 100644 --- a/server/initializers/migrations/0005-example.js +++ b/server/initializers/migrations/0005-example.js @@ -1,14 +1,14 @@ -/* - This is just an example. -*/ +// /* +// This is just an example. +// */ -const db = require('../database') +// const db = require('../database') -// options contains the transaction -exports.up = function (options, callback) { - // db.Application.create({ migrationVersion: 42 }, { transaction: options.transaction }).asCallback(callback) -} +// // options contains the transaction +// exports.up = function (options, callback) { +// db.Application.create({ migrationVersion: 42 }, { transaction: options.transaction }).asCallback(callback) +// } -exports.down = function (options, callback) { - throw new Error('Not implemented.') -} +// exports.down = function (options, callback) { +// throw new Error('Not implemented.') +// } -- cgit v1.2.3 From 56ac84d0a32844c7b7df4c584dccb6e6c17e35de Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Sun, 25 Dec 2016 12:05:47 +0100 Subject: Fix real world script --- server/tests/real-world/real-world.js | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) (limited to 'server') diff --git a/server/tests/real-world/real-world.js b/server/tests/real-world/real-world.js index dba1970c5..2ae3dc15b 100644 --- a/server/tests/real-world/real-world.js +++ b/server/tests/real-world/real-world.js @@ -30,9 +30,9 @@ let integrityInterval = parseInt(program.integrity) || 60000 const numberOfPods = 6 // Wait requests between pods -const requestsMaxPerInterval = constants.INTERVAL / actionInterval +const requestsMaxPerInterval = constants.REQUESTS_INTERVAL / actionInterval const intervalsToMakeAllRequests = Math.ceil(requestsMaxPerInterval / constants.REQUESTS_LIMIT) -const waitForBeforeIntegrityCheck = (intervalsToMakeAllRequests * constants.INTERVAL) + 1000 +const waitForBeforeIntegrityCheck = (intervalsToMakeAllRequests * constants.REQUESTS_INTERVAL) + 1000 integrityInterval += waitForBeforeIntegrityCheck @@ -160,9 +160,9 @@ function exitServers (servers, callback) { function upload (servers, numServer, callback) { if (!callback) callback = function () {} - const name = 'my super name for pod 1' - const description = 'my super description for pod 1' - const tags = [ 'tag1p1', 'tag2p1' ] + const name = Date.now() + ' name' + const description = Date.now() + ' description' + const tags = [ Date.now().toString().substring(0, 5) + 't1', Date.now().toString().substring(0, 5) + 't2' ] const file = 'video_short1.webm' console.log('Upload video to server ' + numServer) @@ -205,6 +205,7 @@ function checkIntegrity (servers, callback) { for (const video of videos) { if (!isEqual(video, videos[0])) { console.error('Integrity not ok!') + process.exit(-1) } } -- cgit v1.2.3 From 124648d7fcfc3c9a91fe702cbc40c317429c05bd Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Sun, 25 Dec 2016 12:06:08 +0100 Subject: Server: add createdAt from remote video in database --- server/controllers/api/remote.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'server') diff --git a/server/controllers/api/remote.js b/server/controllers/api/remote.js index c7a5658e8..2cf916ff3 100644 --- a/server/controllers/api/remote.js +++ b/server/controllers/api/remote.js @@ -135,7 +135,8 @@ function addRemoteVideo (videoToCreateData, fromHost, finalCallback) { infoHash: videoToCreateData.infoHash, description: videoToCreateData.description, authorId: author.id, - duration: videoToCreateData.duration + duration: videoToCreateData.duration, + createdAt: videoToCreateData.createdAt } const video = db.Video.build(videoData) -- cgit v1.2.3 From 178edb20259f90b1c59f40728aaf8073f097f1f5 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Mon, 26 Dec 2016 17:44:25 +0100 Subject: Server: correctly sort tags by name asc --- server/models/video.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'server') diff --git a/server/models/video.js b/server/models/video.js index af05a861f..04478c8d7 100644 --- a/server/models/video.js +++ b/server/models/video.js @@ -320,7 +320,7 @@ function listForApi (start, count, sort, callback) { offset: start, limit: count, distinct: true, // For the count, a video can have many tags - order: [ modelUtils.getSort(sort) ], + order: [ modelUtils.getSort(sort), [ this.sequelize.models.Tag, 'name', 'ASC' ] ], include: [ { model: this.sequelize.models.Author, @@ -440,7 +440,7 @@ function searchAndPopulateAuthorAndPodAndTags (value, field, start, count, sort, offset: start, limit: count, distinct: true, // For the count, a video can have many tags - order: [ modelUtils.getSort(sort) ] + order: [ modelUtils.getSort(sort), [ this.sequelize.models.Tag, 'name', 'ASC' ] ] } // Make an exact search with the magnet -- cgit v1.2.3 From 00d6a41e46e4c4948b0d5b4cf21433150a57c067 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Tue, 27 Dec 2016 18:33:38 +0100 Subject: Add script to migrate from mongodb to postgresql Usage: NODE_ENV=production ./scripts/mongo-to-postgre.js --mongo-database peertube-prod --- server/helpers/peertube-crypto.js | 1 - 1 file changed, 1 deletion(-) (limited to 'server') diff --git a/server/helpers/peertube-crypto.js b/server/helpers/peertube-crypto.js index 302ddca58..610cb16cd 100644 --- a/server/helpers/peertube-crypto.js +++ b/server/helpers/peertube-crypto.js @@ -1,7 +1,6 @@ 'use strict' const bcrypt = require('bcrypt') -const crypto = require('crypto') const fs = require('fs') const openssl = require('openssl-wrapper') const ursa = require('ursa') -- cgit v1.2.3 From 67bf9b96bbcd92b069fe86d9223fe0f8b9c6e677 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Wed, 28 Dec 2016 15:49:23 +0100 Subject: Server: add database field validations --- server/helpers/custom-validators/pods.js | 9 +++- server/helpers/custom-validators/videos.js | 6 --- server/initializers/constants.js | 2 +- server/initializers/installer.js | 10 ++-- server/models/application.js | 8 +++- server/models/author.js | 13 +++++- server/models/oauth-client.js | 11 +++-- server/models/oauth-token.js | 19 ++++---- server/models/pod.js | 34 +++++++++----- server/models/request.js | 14 +++--- server/models/tag.js | 3 +- server/models/user.js | 32 +++++++++---- server/models/video.js | 74 ++++++++++++++++++++++-------- 13 files changed, 162 insertions(+), 73 deletions(-) (limited to 'server') diff --git a/server/helpers/custom-validators/pods.js b/server/helpers/custom-validators/pods.js index 0154a2424..8bb3733ff 100644 --- a/server/helpers/custom-validators/pods.js +++ b/server/helpers/custom-validators/pods.js @@ -5,14 +5,19 @@ const validator = require('express-validator').validator const miscValidators = require('./misc') const podsValidators = { - isEachUniqueHostValid + isEachUniqueHostValid, + isHostValid +} + +function isHostValid (host) { + return validator.isURL(host) && host.split('://').length === 1 } function isEachUniqueHostValid (hosts) { return miscValidators.isArray(hosts) && hosts.length !== 0 && hosts.every(function (host) { - return validator.isURL(host) && host.split('://').length === 1 && hosts.indexOf(host) === hosts.lastIndexOf(host) + return isHostValid(host) && hosts.indexOf(host) === hosts.lastIndexOf(host) }) } diff --git a/server/helpers/custom-validators/videos.js b/server/helpers/custom-validators/videos.js index be8256a80..da857ba5f 100644 --- a/server/helpers/custom-validators/videos.js +++ b/server/helpers/custom-validators/videos.js @@ -15,7 +15,6 @@ const videosValidators = { isVideoDurationValid, isVideoInfoHashValid, isVideoNameValid, - isVideoPodHostValid, isVideoTagsValid, isVideoThumbnailValid, isVideoThumbnail64Valid @@ -74,11 +73,6 @@ function isVideoNameValid (value) { return validator.isLength(value, VIDEOS_CONSTRAINTS_FIELDS.NAME) } -function isVideoPodHostValid (value) { - // TODO: set options (TLD...) - return validator.isURL(value) -} - function isVideoTagsValid (tags) { return miscValidators.isArray(tags) && validator.isInt(tags.length, VIDEOS_CONSTRAINTS_FIELDS.TAGS) && diff --git a/server/initializers/constants.js b/server/initializers/constants.js index fc501845a..0af7aca3c 100644 --- a/server/initializers/constants.js +++ b/server/initializers/constants.js @@ -69,7 +69,7 @@ const CONSTRAINTS_FIELDS = { NAME: { min: 3, max: 50 }, // Length DESCRIPTION: { min: 3, max: 250 }, // Length EXTNAME: [ '.mp4', '.ogv', '.webm' ], - INFO_HASH: { min: 10, max: 50 }, // Length + INFO_HASH: { min: 40, max: 40 }, // Length, infohash is 20 bytes length but we represent it in hexa so 20 * 2 DURATION: { min: 1, max: 7200 }, // Number TAGS: { min: 1, max: 3 }, // Number of total tags TAG: { min: 2, max: 10 }, // Length diff --git a/server/initializers/installer.js b/server/initializers/installer.js index d5382364e..fb63b81ac 100644 --- a/server/initializers/installer.js +++ b/server/initializers/installer.js @@ -96,6 +96,7 @@ function createOAuthAdminIfNotExist (callback) { const username = 'root' const role = constants.USER_ROLES.ADMIN + const createOptions = {} let password = '' // Do not generate a random password for tests @@ -105,17 +106,20 @@ function createOAuthAdminIfNotExist (callback) { if (process.env.NODE_APP_INSTANCE) { password += process.env.NODE_APP_INSTANCE } + + // Our password is weak so do not validate it + createOptions.validate = false } else { password = passwordGenerator(8, true) } - const user = db.User.build({ + const userData = { username, password, role - }) + } - user.save().asCallback(function (err, createdUser) { + db.User.create(userData, createOptions).asCallback(function (err, createdUser) { if (err) return callback(err) logger.info('Username: ' + username) diff --git a/server/models/application.js b/server/models/application.js index 4114ed76d..46dcfde33 100644 --- a/server/models/application.js +++ b/server/models/application.js @@ -1,9 +1,15 @@ +'use strict' + module.exports = function (sequelize, DataTypes) { const Application = sequelize.define('Application', { migrationVersion: { type: DataTypes.INTEGER, - defaultValue: 0 + defaultValue: 0, + allowNull: false, + validate: { + isInt: true + } } }, { diff --git a/server/models/author.js b/server/models/author.js index 493c2ca11..e0ac868ea 100644 --- a/server/models/author.js +++ b/server/models/author.js @@ -1,8 +1,19 @@ +'use strict' + +const customUsersValidators = require('../helpers/custom-validators').users + module.exports = function (sequelize, DataTypes) { const Author = sequelize.define('Author', { name: { - type: DataTypes.STRING + type: DataTypes.STRING, + allowNull: false, + validate: { + usernameValid: function (value) { + const res = customUsersValidators.isUserUsernameValid(value) + if (res === false) throw new Error('Username is not valid.') + } + } } }, { diff --git a/server/models/oauth-client.js b/server/models/oauth-client.js index 15118591a..b56838d4c 100644 --- a/server/models/oauth-client.js +++ b/server/models/oauth-client.js @@ -1,11 +1,15 @@ +'use strict' + module.exports = function (sequelize, DataTypes) { const OAuthClient = sequelize.define('OAuthClient', { clientId: { - type: DataTypes.STRING + type: DataTypes.STRING, + allowNull: false }, clientSecret: { - type: DataTypes.STRING + type: DataTypes.STRING, + allowNull: false }, grants: { type: DataTypes.ARRAY(DataTypes.STRING) @@ -28,9 +32,6 @@ module.exports = function (sequelize, DataTypes) { return OAuthClient } -// TODO: validation -// OAuthClientSchema.path('clientSecret').required(true) - // --------------------------------------------------------------------------- function associate (models) { diff --git a/server/models/oauth-token.js b/server/models/oauth-token.js index c9108bf95..f8de4e916 100644 --- a/server/models/oauth-token.js +++ b/server/models/oauth-token.js @@ -1,3 +1,5 @@ +'use strict' + const logger = require('../helpers/logger') // --------------------------------------------------------------------------- @@ -6,16 +8,20 @@ module.exports = function (sequelize, DataTypes) { const OAuthToken = sequelize.define('OAuthToken', { accessToken: { - type: DataTypes.STRING + type: DataTypes.STRING, + allowNull: false }, accessTokenExpiresAt: { - type: DataTypes.DATE + type: DataTypes.DATE, + allowNull: false }, refreshToken: { - type: DataTypes.STRING + type: DataTypes.STRING, + allowNull: false }, refreshTokenExpiresAt: { - type: DataTypes.DATE + type: DataTypes.DATE, + allowNull: false } }, { @@ -33,11 +39,6 @@ module.exports = function (sequelize, DataTypes) { return OAuthToken } -// TODO: validation -// OAuthTokenSchema.path('accessToken').required(true) -// OAuthTokenSchema.path('client').required(true) -// OAuthTokenSchema.path('user').required(true) - // --------------------------------------------------------------------------- function associate (models) { diff --git a/server/models/pod.js b/server/models/pod.js index fff6970a7..84f78f200 100644 --- a/server/models/pod.js +++ b/server/models/pod.js @@ -3,6 +3,7 @@ const map = require('lodash/map') const constants = require('../initializers/constants') +const customPodsValidators = require('../helpers/custom-validators').pods // --------------------------------------------------------------------------- @@ -10,14 +11,27 @@ module.exports = function (sequelize, DataTypes) { const Pod = sequelize.define('Pod', { host: { - type: DataTypes.STRING + type: DataTypes.STRING, + allowNull: false, + validate: { + isHost: function (value) { + const res = customPodsValidators.isHostValid(value) + if (res === false) throw new Error('Host not valid.') + } + } }, publicKey: { - type: DataTypes.STRING(5000) + type: DataTypes.STRING(5000), + allowNull: false }, score: { type: DataTypes.INTEGER, - defaultValue: constants.FRIEND_SCORE.BASE + defaultValue: constants.FRIEND_SCORE.BASE, + allowNull: false, + validate: { + isInt: true, + max: constants.FRIEND_SCORE.MAX + } } }, { @@ -42,12 +56,6 @@ module.exports = function (sequelize, DataTypes) { return Pod } -// TODO: max score -> constants.FRIENDS_SCORE.MAX -// TODO: validation -// PodSchema.path('host').validate(validator.isURL) -// PodSchema.path('publicKey').required(true) -// PodSchema.path('score').validate(function (value) { return !isNaN(value) }) - // ------------------------------ METHODS ------------------------------ function toFormatedJSON () { @@ -82,15 +90,17 @@ function incrementScores (ids, value, callback) { score: this.sequelize.literal('score +' + value) } - const query = { + const options = { where: { id: { $in: ids } - } + }, + // In this case score is a literal and not an integer so we do not validate it + validate: false } - return this.update(update, query).asCallback(callback) + return this.update(update, options).asCallback(callback) } function list (callback) { diff --git a/server/models/request.js b/server/models/request.js index 70aa32610..e18f8fe3d 100644 --- a/server/models/request.js +++ b/server/models/request.js @@ -3,6 +3,7 @@ const each = require('async/each') const eachLimit = require('async/eachLimit') const waterfall = require('async/waterfall') +const values = require('lodash/values') const constants = require('../initializers/constants') const logger = require('../helpers/logger') @@ -17,11 +18,12 @@ module.exports = function (sequelize, DataTypes) { const Request = sequelize.define('Request', { request: { - type: DataTypes.JSON + type: DataTypes.JSON, + allowNull: false }, endpoint: { - // TODO: enum? - type: DataTypes.STRING + type: DataTypes.ENUM(values(constants.REQUEST_ENDPOINTS)), + allowNull: false } }, { @@ -196,7 +198,7 @@ function makeRequests () { makeRequest(toPod, requestToMake.endpoint, requestToMake.datas, function (success) { if (success === true) { - logger.debug('Removing requests for %s pod.', requestToMake.toPodId, { requestsIds: requestToMake.ids }) + logger.debug('Removing requests for pod %s.', requestToMake.toPodId, { requestsIds: requestToMake.ids }) goodPods.push(requestToMake.toPodId) @@ -261,13 +263,13 @@ function updatePodsScore (goodPods, badPods) { if (goodPods.length !== 0) { Pod.incrementScores(goodPods, constants.PODS_SCORE.BONUS, function (err) { - if (err) logger.error('Cannot increment scores of good pods.') + if (err) logger.error('Cannot increment scores of good pods.', { error: err }) }) } if (badPods.length !== 0) { Pod.incrementScores(badPods, constants.PODS_SCORE.MALUS, function (err) { - if (err) logger.error('Cannot decrement scores of bad pods.') + if (err) logger.error('Cannot decrement scores of bad pods.', { error: err }) removeBadPods.call(self) }) } diff --git a/server/models/tag.js b/server/models/tag.js index 874e88842..d6c2d3bb1 100644 --- a/server/models/tag.js +++ b/server/models/tag.js @@ -6,7 +6,8 @@ module.exports = function (sequelize, DataTypes) { const Tag = sequelize.define('Tag', { name: { - type: DataTypes.STRING + type: DataTypes.STRING, + allowNull: false } }, { diff --git a/server/models/user.js b/server/models/user.js index e50eb96ea..944986a44 100644 --- a/server/models/user.js +++ b/server/models/user.js @@ -1,5 +1,11 @@ +'use strict' + +const values = require('lodash/values') + const modelUtils = require('./utils') +const constants = require('../initializers/constants') const peertubeCrypto = require('../helpers/peertube-crypto') +const customUsersValidators = require('../helpers/custom-validators').users // --------------------------------------------------------------------------- @@ -7,13 +13,28 @@ module.exports = function (sequelize, DataTypes) { const User = sequelize.define('User', { password: { - type: DataTypes.STRING + type: DataTypes.STRING, + allowNull: false, + validate: { + passwordValid: function (value) { + const res = customUsersValidators.isUserPasswordValid(value) + if (res === false) throw new Error('Password not valid.') + } + } }, username: { - type: DataTypes.STRING + type: DataTypes.STRING, + allowNull: false, + validate: { + usernameValid: function (value) { + const res = customUsersValidators.isUserUsernameValid(value) + if (res === false) throw new Error('Username not valid.') + } + } }, role: { - type: DataTypes.STRING + type: DataTypes.ENUM(values(constants.USER_ROLES)), + allowNull: false } }, { @@ -41,11 +62,6 @@ module.exports = function (sequelize, DataTypes) { return User } -// TODO: Validation -// UserSchema.path('password').required(customUsersValidators.isUserPasswordValid) -// UserSchema.path('username').required(customUsersValidators.isUserUsernameValid) -// UserSchema.path('role').validate(customUsersValidators.isUserRoleValid) - function beforeCreateOrUpdate (user, options, next) { peertubeCrypto.cryptPassword(user.password, function (err, hash) { if (err) return next(err) diff --git a/server/models/video.js b/server/models/video.js index 04478c8d7..3ebc48ad4 100644 --- a/server/models/video.js +++ b/server/models/video.js @@ -8,10 +8,12 @@ const map = require('lodash/map') const parallel = require('async/parallel') const parseTorrent = require('parse-torrent') const pathUtils = require('path') +const values = require('lodash/values') const constants = require('../initializers/constants') const logger = require('../helpers/logger') const modelUtils = require('./utils') +const customVideosValidators = require('../helpers/custom-validators').videos // --------------------------------------------------------------------------- @@ -22,26 +24,61 @@ module.exports = function (sequelize, DataTypes) { id: { type: DataTypes.UUID, defaultValue: DataTypes.UUIDV4, - primaryKey: true + primaryKey: true, + validate: { + isUUID: 4 + } }, name: { - type: DataTypes.STRING + type: DataTypes.STRING, + allowNull: false, + validate: { + nameValid: function (value) { + const res = customVideosValidators.isVideoNameValid(value) + if (res === false) throw new Error('Video name is not valid.') + } + } }, extname: { - // TODO: enum? - type: DataTypes.STRING + type: DataTypes.ENUM(values(constants.CONSTRAINTS_FIELDS.VIDEOS.EXTNAME)), + allowNull: false }, remoteId: { - type: DataTypes.UUID + type: DataTypes.UUID, + allowNull: true, + validate: { + isUUID: 4 + } }, description: { - type: DataTypes.STRING + type: DataTypes.STRING, + allowNull: false, + validate: { + descriptionValid: function (value) { + const res = customVideosValidators.isVideoDescriptionValid(value) + if (res === false) throw new Error('Video description is not valid.') + } + } }, infoHash: { - type: DataTypes.STRING + type: DataTypes.STRING, + allowNull: false, + validate: { + infoHashValid: function (value) { + const res = customVideosValidators.isVideoInfoHashValid(value) + if (res === false) throw new Error('Video info hash is not valid.') + } + } }, duration: { - type: DataTypes.INTEGER + type: DataTypes.INTEGER, + allowNull: false, + validate: { + durationValid: function (value) { + const res = customVideosValidators.isVideoDurationValid(value) + if (res === false) throw new Error('Video duration is not valid.') + } + } } }, { @@ -71,6 +108,7 @@ module.exports = function (sequelize, DataTypes) { toRemoteJSON }, hooks: { + beforeValidate, beforeCreate, afterDestroy } @@ -80,13 +118,14 @@ module.exports = function (sequelize, DataTypes) { return Video } -// TODO: Validation -// VideoSchema.path('name').validate(customVideosValidators.isVideoNameValid) -// VideoSchema.path('description').validate(customVideosValidators.isVideoDescriptionValid) -// VideoSchema.path('podHost').validate(customVideosValidators.isVideoPodHostValid) -// VideoSchema.path('author').validate(customVideosValidators.isVideoAuthorValid) -// VideoSchema.path('duration').validate(customVideosValidators.isVideoDurationValid) -// VideoSchema.path('tags').validate(customVideosValidators.isVideoTagsValid) +function beforeValidate (video, options, next) { + if (video.isOwned()) { + // 40 hexa length + video.infoHash = '0123456789abcdef0123456789abcdef01234567' + } + + return next(null) +} function beforeCreate (video, options, next) { const tasks = [] @@ -113,9 +152,8 @@ function beforeCreate (video, options, next) { if (err) return callback(err) const parsedTorrent = parseTorrent(torrent) - video.infoHash = parsedTorrent.infoHash - - callback(null) + video.set('infoHash', parsedTorrent.infoHash) + video.validate().asCallback(callback) }) }) }, -- cgit v1.2.3 From 319d072e8eb7266cd8d33e0bb2fb5ebe76c487d1 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Thu, 29 Dec 2016 09:33:28 +0100 Subject: Server: Add postgresql indexes --- server/models/author.js | 8 ++++++++ server/models/oauth-client.js | 22 ++++++++++------------ server/models/oauth-token.js | 24 ++++++++++++++++++++++++ server/models/pod.js | 8 ++++++++ server/models/requestToPod.js | 12 ++++++++++++ server/models/tag.js | 7 +++++++ server/models/user.js | 5 +++++ server/models/video.js | 20 ++++++++++++++++++++ server/models/videoTag.js | 11 ++++++++++- 9 files changed, 104 insertions(+), 13 deletions(-) (limited to 'server') diff --git a/server/models/author.js b/server/models/author.js index e0ac868ea..8f5b598c8 100644 --- a/server/models/author.js +++ b/server/models/author.js @@ -17,6 +17,14 @@ module.exports = function (sequelize, DataTypes) { } }, { + indexes: [ + { + fields: [ 'name' ] + }, + { + fields: [ 'podId' ] + } + ], classMethods: { associate } diff --git a/server/models/oauth-client.js b/server/models/oauth-client.js index b56838d4c..758c4cf2f 100644 --- a/server/models/oauth-client.js +++ b/server/models/oauth-client.js @@ -19,9 +19,17 @@ module.exports = function (sequelize, DataTypes) { } }, { + indexes: [ + { + fields: [ 'clientId' ], + unique: true + }, + { + fields: [ 'clientId', 'clientSecret' ], + unique: true + } + ], classMethods: { - associate, - getByIdAndSecret, list, loadFirstClient @@ -34,16 +42,6 @@ module.exports = function (sequelize, DataTypes) { // --------------------------------------------------------------------------- -function associate (models) { - this.hasMany(models.OAuthToken, { - foreignKey: { - name: 'oAuthClientId', - allowNull: false - }, - onDelete: 'cascade' - }) -} - function list (callback) { return this.findAll().asCallback(callback) } diff --git a/server/models/oauth-token.js b/server/models/oauth-token.js index f8de4e916..68e7c9ff7 100644 --- a/server/models/oauth-token.js +++ b/server/models/oauth-token.js @@ -25,6 +25,22 @@ module.exports = function (sequelize, DataTypes) { } }, { + indexes: [ + { + fields: [ 'refreshToken' ], + unique: true + }, + { + fields: [ 'accessToken' ], + unique: true + }, + { + fields: [ 'userId' ] + }, + { + fields: [ 'oAuthClientId' ] + } + ], classMethods: { associate, @@ -49,6 +65,14 @@ function associate (models) { }, onDelete: 'cascade' }) + + this.belongsTo(models.OAuthClient, { + foreignKey: { + name: 'oAuthClientId', + allowNull: false + }, + onDelete: 'cascade' + }) } function getByRefreshTokenAndPopulateClient (refreshToken) { diff --git a/server/models/pod.js b/server/models/pod.js index 84f78f200..83ecd732e 100644 --- a/server/models/pod.js +++ b/server/models/pod.js @@ -35,6 +35,14 @@ module.exports = function (sequelize, DataTypes) { } }, { + indexes: [ + { + fields: [ 'host' ] + }, + { + fields: [ 'score' ] + } + ], classMethods: { associate, diff --git a/server/models/requestToPod.js b/server/models/requestToPod.js index 378c2bdcf..f42a53458 100644 --- a/server/models/requestToPod.js +++ b/server/models/requestToPod.js @@ -4,6 +4,18 @@ module.exports = function (sequelize, DataTypes) { const RequestToPod = sequelize.define('RequestToPod', {}, { + indexes: [ + { + fields: [ 'requestId' ] + }, + { + fields: [ 'podId' ] + }, + { + fields: [ 'requestId', 'podId' ], + unique: true + } + ], classMethods: { removePodOf } diff --git a/server/models/tag.js b/server/models/tag.js index d6c2d3bb1..27eecdc84 100644 --- a/server/models/tag.js +++ b/server/models/tag.js @@ -11,6 +11,13 @@ module.exports = function (sequelize, DataTypes) { } }, { + timestamps: false, + indexes: [ + { + fields: [ 'name' ], + unique: true + } + ], classMethods: { associate } diff --git a/server/models/user.js b/server/models/user.js index 944986a44..631cd96c9 100644 --- a/server/models/user.js +++ b/server/models/user.js @@ -38,6 +38,11 @@ module.exports = function (sequelize, DataTypes) { } }, { + indexes: [ + { + fields: [ 'username' ] + } + ], classMethods: { associate, diff --git a/server/models/video.js b/server/models/video.js index 3ebc48ad4..d1595ce51 100644 --- a/server/models/video.js +++ b/server/models/video.js @@ -82,6 +82,26 @@ module.exports = function (sequelize, DataTypes) { } }, { + indexes: [ + { + fields: [ 'authorId' ] + }, + { + fields: [ 'remoteId' ] + }, + { + fields: [ 'name' ] + }, + { + fields: [ 'createdAt' ] + }, + { + fields: [ 'duration' ] + }, + { + fields: [ 'infoHash' ] + } + ], classMethods: { associate, diff --git a/server/models/videoTag.js b/server/models/videoTag.js index 0f2b20838..cd9277a6e 100644 --- a/server/models/videoTag.js +++ b/server/models/videoTag.js @@ -3,7 +3,16 @@ // --------------------------------------------------------------------------- module.exports = function (sequelize, DataTypes) { - const VideoTag = sequelize.define('VideoTag', {}, {}) + const VideoTag = sequelize.define('VideoTag', {}, { + indexes: [ + { + fields: [ 'videoId' ] + }, + { + fields: [ 'tagId' ] + } + ] + }) return VideoTag } -- cgit v1.2.3 From 4712081f2a5f48749cf125d729e78b926ab28d6d Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Thu, 29 Dec 2016 10:33:36 +0100 Subject: Server: add association between author and user --- server/controllers/api/remote.js | 6 ++++-- server/controllers/api/videos.js | 17 ++++++++++------- server/initializers/checker.js | 4 ++-- server/models/author.js | 11 +++++++++++ server/models/oauth-client.js | 6 +++--- server/models/user.js | 5 +++++ 6 files changed, 35 insertions(+), 14 deletions(-) (limited to 'server') diff --git a/server/controllers/api/remote.js b/server/controllers/api/remote.js index 2cf916ff3..94d6e740e 100644 --- a/server/controllers/api/remote.js +++ b/server/controllers/api/remote.js @@ -84,11 +84,13 @@ function addRemoteVideo (videoToCreateData, fromHost, finalCallback) { const query = { where: { name: username, - podId: pod.id + podId: pod.id, + userId: null }, defaults: { name: username, - podId: pod.id + podId: pod.id, + userId: null }, transaction: t } diff --git a/server/controllers/api/videos.js b/server/controllers/api/videos.js index 992f03db0..170224634 100644 --- a/server/controllers/api/videos.js +++ b/server/controllers/api/videos.js @@ -95,23 +95,26 @@ function addVideo (req, res, next) { }, function findOrCreateAuthor (t, callback) { - const username = res.locals.oauth.token.user.username + const user = res.locals.oauth.token.User const query = { where: { - name: username, - podId: null + name: user.username, + podId: null, + userId: user.id }, defaults: { - name: username, - podId: null // null because it is OUR pod + name: user.username, + podId: null, // null because it is OUR pod + userId: user.id }, transaction: t } db.Author.findOrCreate(query).asCallback(function (err, result) { - // [ instance, wasCreated ] - return callback(err, t, result[0]) + const authorInstance = result[0] + + return callback(err, t, authorInstance) }) }, diff --git a/server/initializers/checker.js b/server/initializers/checker.js index 2753604dc..6471bb4f1 100644 --- a/server/initializers/checker.js +++ b/server/initializers/checker.js @@ -42,10 +42,10 @@ function checkMissedConfig () { } function clientsExist (callback) { - db.OAuthClient.list(function (err, clients) { + db.OAuthClient.countTotal(function (err, totalClients) { if (err) return callback(err) - return callback(null, clients.length !== 0) + return callback(null, totalClients !== 0) }) } diff --git a/server/models/author.js b/server/models/author.js index 8f5b598c8..5835ada99 100644 --- a/server/models/author.js +++ b/server/models/author.js @@ -23,6 +23,9 @@ module.exports = function (sequelize, DataTypes) { }, { fields: [ 'podId' ] + }, + { + fields: [ 'userId' ] } ], classMethods: { @@ -44,4 +47,12 @@ function associate (models) { }, onDelete: 'cascade' }) + + this.belongsTo(models.User, { + foreignKey: { + name: 'userId', + allowNull: true + }, + onDelete: 'cascade' + }) } diff --git a/server/models/oauth-client.js b/server/models/oauth-client.js index 758c4cf2f..021a34007 100644 --- a/server/models/oauth-client.js +++ b/server/models/oauth-client.js @@ -30,8 +30,8 @@ module.exports = function (sequelize, DataTypes) { } ], classMethods: { + countTotal, getByIdAndSecret, - list, loadFirstClient } } @@ -42,8 +42,8 @@ module.exports = function (sequelize, DataTypes) { // --------------------------------------------------------------------------- -function list (callback) { - return this.findAll().asCallback(callback) +function countTotal (callback) { + return this.count().asCallback(callback) } function loadFirstClient (callback) { diff --git a/server/models/user.js b/server/models/user.js index 631cd96c9..36ed723cc 100644 --- a/server/models/user.js +++ b/server/models/user.js @@ -94,6 +94,11 @@ function toFormatedJSON () { // ------------------------------ STATICS ------------------------------ function associate (models) { + this.hasOne(models.Author, { + foreignKey: 'userId', + onDelete: 'cascade' + }) + this.hasMany(models.OAuthToken, { foreignKey: 'userId', onDelete: 'cascade' -- cgit v1.2.3 From 98ac898a03ed7bbb4edec74fe823b3f2d6d4904a Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Thu, 29 Dec 2016 11:17:11 +0100 Subject: Server: use video hook to send information to other pods when a video is deleted --- server/controllers/api/pods.js | 2 +- server/controllers/api/users.js | 32 ++------------------------------ server/controllers/api/videos.js | 24 ++++++------------------ server/models/video.js | 14 ++++++++++++++ 4 files changed, 23 insertions(+), 49 deletions(-) (limited to 'server') diff --git a/server/controllers/api/pods.js b/server/controllers/api/pods.js index 79f3f9d8d..d9279f1d9 100644 --- a/server/controllers/api/pods.js +++ b/server/controllers/api/pods.js @@ -113,7 +113,7 @@ function removePods (req, res, next) { db.Pod.loadByHost(host, callback) }, - function removePod (pod, callback) { + function deletePod (pod, callback) { pod.destroy().asCallback(callback) } ], function (err) { diff --git a/server/controllers/api/users.js b/server/controllers/api/users.js index 890028b36..e4423680c 100644 --- a/server/controllers/api/users.js +++ b/server/controllers/api/users.js @@ -90,39 +90,11 @@ function listUsers (req, res, next) { function removeUser (req, res, next) { waterfall([ - function getUser (callback) { + function loadUser (callback) { db.User.loadById(req.params.id, callback) }, - // TODO: use foreignkey? - function getVideos (user, callback) { - db.Video.listOwnedByAuthor(user.username, function (err, videos) { - return callback(err, user, videos) - }) - }, - - function removeVideosFromDB (user, videos, callback) { - each(videos, function (video, callbackEach) { - video.destroy().asCallback(callbackEach) - }, function (err) { - return callback(err, user, videos) - }) - }, - - function sendInformationToFriends (user, videos, callback) { - videos.forEach(function (video) { - const params = { - name: video.name, - remoteId: video.id - } - - friends.removeVideoToFriends(params) - }) - - return callback(null, user) - }, - - function removeUserFromDB (user, callback) { + function deleteUser (user, callback) { user.destroy().asCallback(callback) } ], function andFinally (err) { diff --git a/server/controllers/api/videos.js b/server/controllers/api/videos.js index 170224634..ddf85d77d 100644 --- a/server/controllers/api/videos.js +++ b/server/controllers/api/videos.js @@ -249,27 +249,15 @@ function removeVideo (req, res, next) { const videoId = req.params.id waterfall([ - function getVideo (callback) { - db.Video.load(videoId, callback) - }, - - function removeFromDB (video, callback) { - video.destroy().asCallback(function (err) { - if (err) return callback(err) - - return callback(null, video) + function loadVideo (callback) { + db.Video.load(videoId, function (err, video) { + return callback(err, video) }) }, - function sendInformationToFriends (video, callback) { - const params = { - name: video.name, - remoteId: video.id - } - - friends.removeVideoToFriends(params) - - return callback(null) + function deleteVideo (video, callback) { + // Informations to other pods will be sent by the afterDestroy video hook + video.destroy().asCallback(callback) } ], function andFinally (err) { if (err) { diff --git a/server/models/video.js b/server/models/video.js index d1595ce51..564e362fd 100644 --- a/server/models/video.js +++ b/server/models/video.js @@ -12,6 +12,7 @@ const values = require('lodash/values') const constants = require('../initializers/constants') const logger = require('../helpers/logger') +const friends = require('../lib/friends') const modelUtils = require('./utils') const customVideosValidators = require('../helpers/custom-validators').videos @@ -205,11 +206,24 @@ function afterDestroy (video, options, next) { function (callback) { removeFile(video, callback) }, + function (callback) { removeTorrent(video, callback) }, + function (callback) { removePreview(video, callback) + }, + + function (callback) { + const params = { + name: video.name, + remoteId: video.id + } + + friends.removeVideoToFriends(params) + + return callback() } ) } -- cgit v1.2.3 From 4d32448895ad29ef694bcf790d59253249ad5939 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Thu, 29 Dec 2016 12:13:19 +0100 Subject: Server: use binary data instead of base64 to send thumbnails --- server/controllers/api/remote.js | 4 ++-- server/controllers/api/users.js | 2 -- server/helpers/custom-validators/videos.js | 9 ++++----- server/initializers/constants.js | 2 +- server/models/video.js | 11 ++++++----- 5 files changed, 13 insertions(+), 15 deletions(-) (limited to 'server') diff --git a/server/controllers/api/remote.js b/server/controllers/api/remote.js index 94d6e740e..ac850c2d2 100644 --- a/server/controllers/api/remote.js +++ b/server/controllers/api/remote.js @@ -147,9 +147,9 @@ function addRemoteVideo (videoToCreateData, fromHost, finalCallback) { }, function generateThumbnail (t, tagInstances, video, callback) { - db.Video.generateThumbnailFromBase64(video, videoToCreateData.thumbnailBase64, function (err) { + db.Video.generateThumbnailFromData(video, videoToCreateData.thumbnailData, function (err) { if (err) { - logger.error('Cannot generate thumbnail from base 64 data.', { error: err }) + logger.error('Cannot generate thumbnail from data.', { error: err }) return callback(err) } diff --git a/server/controllers/api/users.js b/server/controllers/api/users.js index e4423680c..53bf56790 100644 --- a/server/controllers/api/users.js +++ b/server/controllers/api/users.js @@ -1,12 +1,10 @@ 'use strict' -const each = require('async/each') const express = require('express') const waterfall = require('async/waterfall') const constants = require('../../initializers/constants') const db = require('../../initializers/database') -const friends = require('../../lib/friends') const logger = require('../../helpers/logger') const middlewares = require('../../middlewares') const admin = middlewares.admin diff --git a/server/helpers/custom-validators/videos.js b/server/helpers/custom-validators/videos.js index da857ba5f..4aaa6aaa9 100644 --- a/server/helpers/custom-validators/videos.js +++ b/server/helpers/custom-validators/videos.js @@ -17,7 +17,7 @@ const videosValidators = { isVideoNameValid, isVideoTagsValid, isVideoThumbnailValid, - isVideoThumbnail64Valid + isVideoThumbnailDataValid } function isEachRemoteVideosValid (requests) { @@ -33,7 +33,7 @@ function isEachRemoteVideosValid (requests) { isVideoInfoHashValid(video.infoHash) && isVideoNameValid(video.name) && isVideoTagsValid(video.tags) && - isVideoThumbnail64Valid(video.thumbnailBase64) && + isVideoThumbnailDataValid(video.thumbnailData) && isVideoRemoteIdValid(video.remoteId) && isVideoExtnameValid(video.extname) ) || @@ -86,9 +86,8 @@ function isVideoThumbnailValid (value) { return validator.isLength(value, VIDEOS_CONSTRAINTS_FIELDS.THUMBNAIL) } -function isVideoThumbnail64Valid (value) { - return validator.isBase64(value) && - validator.isByteLength(value, VIDEOS_CONSTRAINTS_FIELDS.THUMBNAIL64) +function isVideoThumbnailDataValid (value) { + return validator.isByteLength(value, VIDEOS_CONSTRAINTS_FIELDS.THUMBNAIL_DATA) } function isVideoRemoteIdValid (value) { diff --git a/server/initializers/constants.js b/server/initializers/constants.js index 0af7aca3c..474a37277 100644 --- a/server/initializers/constants.js +++ b/server/initializers/constants.js @@ -74,7 +74,7 @@ const CONSTRAINTS_FIELDS = { TAGS: { min: 1, max: 3 }, // Number of total tags TAG: { min: 2, max: 10 }, // Length THUMBNAIL: { min: 2, max: 30 }, - THUMBNAIL64: { min: 0, max: 20000 } // Bytes + THUMBNAIL_DATA: { min: 0, max: 20000 } // Bytes } } diff --git a/server/models/video.js b/server/models/video.js index 564e362fd..0e84e8986 100644 --- a/server/models/video.js +++ b/server/models/video.js @@ -1,5 +1,6 @@ 'use strict' +const Buffer = require('safe-buffer').Buffer const createTorrent = require('create-torrent') const ffmpeg = require('fluent-ffmpeg') const fs = require('fs') @@ -106,7 +107,7 @@ module.exports = function (sequelize, DataTypes) { classMethods: { associate, - generateThumbnailFromBase64, + generateThumbnailFromData, getDurationFromFile, list, listForApi, @@ -336,7 +337,7 @@ function toFormatedJSON () { function toRemoteJSON (callback) { const self = this - // Convert thumbnail to base64 + // Get thumbnail data to send to the other pod const thumbnailPath = pathUtils.join(constants.CONFIG.STORAGE.THUMBNAILS_DIR, this.getThumbnailName()) fs.readFile(thumbnailPath, function (err, thumbnailData) { if (err) { @@ -351,7 +352,7 @@ function toRemoteJSON (callback) { remoteId: self.id, author: self.Author.name, duration: self.duration, - thumbnailBase64: new Buffer(thumbnailData).toString('base64'), + thumbnailData: thumbnailData.toString('binary'), tags: map(self.Tags, 'name'), createdAt: self.createdAt, extname: self.extname @@ -363,12 +364,12 @@ function toRemoteJSON (callback) { // ------------------------------ STATICS ------------------------------ -function generateThumbnailFromBase64 (video, thumbnailData, callback) { +function generateThumbnailFromData (video, thumbnailData, callback) { // Creating the thumbnail for a remote video const thumbnailName = video.getThumbnailName() const thumbnailPath = constants.CONFIG.STORAGE.THUMBNAILS_DIR + thumbnailName - fs.writeFile(thumbnailPath, thumbnailData, { encoding: 'base64' }, function (err) { + fs.writeFile(thumbnailPath, Buffer.from(thumbnailData, 'binary'), function (err) { if (err) return callback(err) return callback(null, thumbnailName) -- cgit v1.2.3 From 4ff0d86208dafbdd07beb6286fd93c795db8a95f Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Thu, 29 Dec 2016 18:02:03 +0100 Subject: Server: little refractoring --- server/controllers/api/remote.js | 79 ++++++++-------------------------------- server/controllers/api/videos.js | 46 ++++------------------- server/middlewares/secure.js | 4 ++ server/models/author.js | 29 ++++++++++++++- server/models/tag.js | 40 +++++++++++++++++++- 5 files changed, 93 insertions(+), 105 deletions(-) (limited to 'server') diff --git a/server/controllers/api/remote.js b/server/controllers/api/remote.js index ac850c2d2..929ade555 100644 --- a/server/controllers/api/remote.js +++ b/server/controllers/api/remote.js @@ -28,7 +28,7 @@ module.exports = router function remoteVideos (req, res, next) { const requests = req.body.data - const fromHost = req.body.signature.host + const fromPod = res.locals.secure.pod // We need to process in the same order to keep consistency // TODO: optimization @@ -36,9 +36,9 @@ function remoteVideos (req, res, next) { const videoData = request.data if (request.type === 'add') { - addRemoteVideo(videoData, fromHost, callbackEach) + addRemoteVideo(videoData, fromPod, callbackEach) } else if (request.type === 'remove') { - removeRemoteVideo(videoData, fromHost, callbackEach) + removeRemoteVideo(videoData, fromPod, callbackEach) } else { logger.error('Unkown remote request type %s.', request.type) } @@ -50,7 +50,7 @@ function remoteVideos (req, res, next) { return res.type('json').status(204).end() } -function addRemoteVideo (videoToCreateData, fromHost, finalCallback) { +function addRemoteVideo (videoToCreateData, fromPod, finalCallback) { logger.debug('Adding remote video "%s".', videoToCreateData.name) waterfall([ @@ -61,70 +61,21 @@ function addRemoteVideo (videoToCreateData, fromHost, finalCallback) { }) }, - function findOrCreatePod (t, callback) { - const query = { - where: { - host: fromHost - }, - defaults: { - host: fromHost - }, - transaction: t - } - - db.Pod.findOrCreate(query).asCallback(function (err, result) { - // [ instance, wasCreated ] - return callback(err, t, result[0]) - }) - }, + function findOrCreateAuthor (t, callback) { + const name = videoToCreateData.author + const podId = fromPod.id + // This author is from another pod so we do not associate a user + const userId = null - function findOrCreateAuthor (t, pod, callback) { - const username = videoToCreateData.author - - const query = { - where: { - name: username, - podId: pod.id, - userId: null - }, - defaults: { - name: username, - podId: pod.id, - userId: null - }, - transaction: t - } - - db.Author.findOrCreate(query).asCallback(function (err, result) { - // [ instance, wasCreated ] - return callback(err, t, result[0]) + db.Author.findOrCreateAuthor(name, podId, userId, t, function (err, authorInstance) { + return callback(err, t, authorInstance) }) }, function findOrCreateTags (t, author, callback) { const tags = videoToCreateData.tags - const tagInstances = [] - - each(tags, function (tag, callbackEach) { - const query = { - where: { - name: tag - }, - defaults: { - name: tag - }, - transaction: t - } - - db.Tag.findOrCreate(query).asCallback(function (err, res) { - if (err) return callbackEach(err) - // res = [ tag, isCreated ] - const tag = res[0] - tagInstances.push(tag) - return callbackEach() - }) - }, function (err) { + db.Tag.findOrCreateTags(tags, t, function (err, tagInstances) { return callback(err, t, author, tagInstances) }) }, @@ -192,18 +143,18 @@ function addRemoteVideo (videoToCreateData, fromHost, finalCallback) { }) } -function removeRemoteVideo (videoToRemoveData, fromHost, callback) { +function removeRemoteVideo (videoToRemoveData, fromPod, callback) { // TODO: use bulkDestroy? // We need the list because we have to remove some other stuffs (thumbnail etc) - db.Video.listByHostAndRemoteId(fromHost, videoToRemoveData.remoteId, function (err, videosList) { + db.Video.listByHostAndRemoteId(fromPod.host, videoToRemoveData.remoteId, function (err, videosList) { if (err) { logger.error('Cannot list videos from host and remote id.', { error: err.message }) return callback(err) } if (videosList.length === 0) { - logger.error('No remote video was found for this pod.', { remoteId: videoToRemoveData.remoteId, podHost: fromHost }) + logger.error('No remote video was found for this pod.', { remoteId: videoToRemoveData.remoteId, podHost: fromPod.host }) } each(videosList, function (video, callbackEach) { diff --git a/server/controllers/api/videos.js b/server/controllers/api/videos.js index ddf85d77d..f29edac74 100644 --- a/server/controllers/api/videos.js +++ b/server/controllers/api/videos.js @@ -1,6 +1,5 @@ 'use strict' -const each = require('async/each') const express = require('express') const fs = require('fs') const multer = require('multer') @@ -97,51 +96,20 @@ function addVideo (req, res, next) { function findOrCreateAuthor (t, callback) { const user = res.locals.oauth.token.User - const query = { - where: { - name: user.username, - podId: null, - userId: user.id - }, - defaults: { - name: user.username, - podId: null, // null because it is OUR pod - userId: user.id - }, - transaction: t - } - - db.Author.findOrCreate(query).asCallback(function (err, result) { - const authorInstance = result[0] + const name = user.username + // null because it is OUR pod + const podId = null + const userId = user.id + db.Author.findOrCreateAuthor(name, podId, userId, t, function (err, authorInstance) { return callback(err, t, authorInstance) }) }, function findOrCreateTags (t, author, callback) { const tags = videoInfos.tags - const tagInstances = [] - - each(tags, function (tag, callbackEach) { - const query = { - where: { - name: tag - }, - defaults: { - name: tag - }, - transaction: t - } - - db.Tag.findOrCreate(query).asCallback(function (err, res) { - if (err) return callbackEach(err) - - // res = [ tag, isCreated ] - const tag = res[0] - tagInstances.push(tag) - return callbackEach() - }) - }, function (err) { + + db.Tag.findOrCreateTags(tags, t, function (err, tagInstances) { return callback(err, t, author, tagInstances) }) }, diff --git a/server/middlewares/secure.js b/server/middlewares/secure.js index b7b4cdfb4..2aae715c4 100644 --- a/server/middlewares/secure.js +++ b/server/middlewares/secure.js @@ -26,6 +26,10 @@ function checkSignature (req, res, next) { const signatureOk = peertubeCrypto.checkSignature(pod.publicKey, host, req.body.signature.signature) if (signatureOk === true) { + res.locals.secure = { + pod + } + return next() } diff --git a/server/models/author.js b/server/models/author.js index 5835ada99..7d15fb6ec 100644 --- a/server/models/author.js +++ b/server/models/author.js @@ -29,7 +29,9 @@ module.exports = function (sequelize, DataTypes) { } ], classMethods: { - associate + associate, + + findOrCreateAuthor } } ) @@ -56,3 +58,28 @@ function associate (models) { onDelete: 'cascade' }) } + +function findOrCreateAuthor (name, podId, userId, transaction, callback) { + if (!callback) { + callback = transaction + transaction = null + } + + const author = { + name, + podId, + userId + } + + const query = { + where: author, + defaults: author + } + + if (transaction) query.transaction = transaction + + this.findOrCreate(query).asCallback(function (err, result) { + // [ instance, wasCreated ] + return callback(err, result[0]) + }) +} diff --git a/server/models/tag.js b/server/models/tag.js index 27eecdc84..145e090c1 100644 --- a/server/models/tag.js +++ b/server/models/tag.js @@ -1,5 +1,7 @@ 'use strict' +const each = require('async/each') + // --------------------------------------------------------------------------- module.exports = function (sequelize, DataTypes) { @@ -19,7 +21,9 @@ module.exports = function (sequelize, DataTypes) { } ], classMethods: { - associate + associate, + + findOrCreateTags } } ) @@ -36,3 +40,37 @@ function associate (models) { onDelete: 'cascade' }) } + +function findOrCreateTags (tags, transaction, callback) { + if (!callback) { + callback = transaction + transaction = null + } + + const self = this + const tagInstances = [] + + each(tags, function (tag, callbackEach) { + const query = { + where: { + name: tag + }, + defaults: { + name: tag + } + } + + if (transaction) query.transaction = transaction + + self.findOrCreate(query).asCallback(function (err, res) { + if (err) return callbackEach(err) + + // res = [ tag, isCreated ] + const tag = res[0] + tagInstances.push(tag) + return callbackEach() + }) + }, function (err) { + return callback(err, tagInstances) + }) +} -- cgit v1.2.3 From 7b1f49de22c40ae121ddb3c399b2540ba56fd414 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Thu, 29 Dec 2016 19:07:05 +0100 Subject: Server: add ability to update a video --- server/controllers/api/videos.js | 85 ++++++++++++++++++++++++++- server/lib/friends.js | 7 ++- server/middlewares/validators/videos.js | 41 +++++++++---- server/models/video.js | 21 ++++++- server/tests/api/check-params.js | 101 ++++++++++++++++++++++++++++++++ server/tests/api/single-pod.js | 76 ++++++++++++++++++++++++ server/tests/utils/videos.js | 28 ++++++++- 7 files changed, 344 insertions(+), 15 deletions(-) (limited to 'server') diff --git a/server/controllers/api/videos.js b/server/controllers/api/videos.js index f29edac74..1b306d1cf 100644 --- a/server/controllers/api/videos.js +++ b/server/controllers/api/videos.js @@ -50,6 +50,12 @@ router.get('/', pagination.setPagination, listVideos ) +router.put('/:id', + oAuth.authenticate, + reqFiles, + validatorsVideos.videosUpdate, + updateVideo +) router.post('/', oAuth.authenticate, reqFiles, @@ -165,7 +171,7 @@ function addVideo (req, res, next) { }, function sendToFriends (t, video, callback) { - video.toRemoteJSON(function (err, remoteVideo) { + video.toAddRemoteJSON(function (err, remoteVideo) { if (err) return callback(err) // Now we'll add the video's meta data to our friends @@ -193,6 +199,83 @@ function addVideo (req, res, next) { }) } +function updateVideo (req, res, next) { + let videoInstance = res.locals.video + const videoInfosToUpdate = req.body + + waterfall([ + + function startTransaction (callback) { + db.sequelize.transaction().asCallback(function (err, t) { + return callback(err, t) + }) + }, + + function findOrCreateTags (t, callback) { + if (videoInfosToUpdate.tags) { + db.Tag.findOrCreateTags(videoInfosToUpdate.tags, t, function (err, tagInstances) { + return callback(err, t, tagInstances) + }) + } else { + return callback(null, t, null) + } + }, + + function updateVideoIntoDB (t, tagInstances, callback) { + const options = { transaction: t } + + if (videoInfosToUpdate.name) videoInstance.set('name', videoInfosToUpdate.name) + if (videoInfosToUpdate.description) videoInstance.set('description', videoInfosToUpdate.description) + + // Add tags association + videoInstance.save(options).asCallback(function (err) { + if (err) return callback(err) + + return callback(err, t, tagInstances) + }) + }, + + function associateTagsToVideo (t, tagInstances, callback) { + if (tagInstances) { + const options = { transaction: t } + + videoInstance.setTags(tagInstances, options).asCallback(function (err) { + videoInstance.Tags = tagInstances + + return callback(err, t) + }) + } else { + return callback(null, t) + } + }, + + function sendToFriends (t, callback) { + const json = videoInstance.toUpdateRemoteJSON() + + // Now we'll update the video's meta data to our friends + friends.updateVideoToFriends(json) + + return callback(null, t) + } + + ], function andFinally (err, t) { + if (err) { + logger.error('Cannot insert the video.') + + // Abort transaction? + if (t) t.rollback() + + return next(err) + } + + // Commit transaction + t.commit() + + // TODO : include Location of the new video -> 201 + return res.type('json').status(204).end() + }) +} + function getVideo (req, res, next) { db.Video.loadAndPopulateAuthorAndPodAndTags(req.params.id, function (err, video) { if (err) return next(err) diff --git a/server/lib/friends.js b/server/lib/friends.js index ad9e4fdae..589b79660 100644 --- a/server/lib/friends.js +++ b/server/lib/friends.js @@ -14,6 +14,7 @@ const requests = require('../helpers/requests') const friends = { addVideoToFriends, + updateVideoToFriends, hasFriends, getMyCertificate, makeFriends, @@ -26,6 +27,10 @@ function addVideoToFriends (video) { createRequest('add', constants.REQUEST_ENDPOINTS.VIDEOS, video) } +function updateVideoToFriends (video) { + createRequest('update', constants.REQUEST_ENDPOINTS.VIDEOS, video) +} + function hasFriends (callback) { db.Pod.countAll(function (err, count) { if (err) return callback(err) @@ -127,7 +132,7 @@ function sendOwnedVideosToPod (podId) { } videosList.forEach(function (video) { - video.toRemoteJSON(function (err, remoteVideo) { + video.toAddRemoteJSON(function (err, remoteVideo) { if (err) { logger.error('Cannot convert video to remote.', { error: err }) // Don't break the process diff --git a/server/middlewares/validators/videos.js b/server/middlewares/validators/videos.js index 7e90ca047..09a188c76 100644 --- a/server/middlewares/validators/videos.js +++ b/server/middlewares/validators/videos.js @@ -8,6 +8,7 @@ const logger = require('../../helpers/logger') const validatorsVideos = { videosAdd, + videosUpdate, videosGet, videosRemove, videosSearch @@ -41,22 +42,26 @@ function videosAdd (req, res, next) { }) } -function videosGet (req, res, next) { +function videosUpdate (req, res, next) { req.checkParams('id', 'Should have a valid id').notEmpty().isUUID(4) + req.checkBody('name', 'Should have a valid name').optional().isVideoNameValid() + req.checkBody('description', 'Should have a valid description').optional().isVideoDescriptionValid() + req.checkBody('tags', 'Should have correct tags').optional().isVideoTagsValid() - logger.debug('Checking videosGet parameters', { parameters: req.params }) + logger.debug('Checking videosUpdate parameters', { parameters: req.body }) checkErrors(req, res, function () { - db.Video.load(req.params.id, function (err, video) { - if (err) { - logger.error('Error in videosGet request validator.', { error: err }) - return res.sendStatus(500) - } + checkVideoExists(req.params.id, res, next) + }) +} - if (!video) return res.status(404).send('Video not found') +function videosGet (req, res, next) { + req.checkParams('id', 'Should have a valid id').notEmpty().isUUID(4) - next() - }) + logger.debug('Checking videosGet parameters', { parameters: req.params }) + + checkErrors(req, res, function () { + checkVideoExists(req.params.id, res, next) }) } @@ -94,3 +99,19 @@ function videosSearch (req, res, next) { // --------------------------------------------------------------------------- module.exports = validatorsVideos + +// --------------------------------------------------------------------------- + +function checkVideoExists (id, res, callback) { + db.Video.loadAndPopulateAuthorAndPodAndTags(id, function (err, video) { + if (err) { + logger.error('Error in video request validator.', { error: err }) + return res.sendStatus(500) + } + + if (!video) return res.status(404).send('Video not found') + + res.locals.video = video + callback() + }) +} diff --git a/server/models/video.js b/server/models/video.js index 0e84e8986..14fbe2f71 100644 --- a/server/models/video.js +++ b/server/models/video.js @@ -127,7 +127,8 @@ module.exports = function (sequelize, DataTypes) { getTorrentName, isOwned, toFormatedJSON, - toRemoteJSON + toAddRemoteJSON, + toUpdateRemoteJSON }, hooks: { beforeValidate, @@ -334,7 +335,7 @@ function toFormatedJSON () { return json } -function toRemoteJSON (callback) { +function toAddRemoteJSON (callback) { const self = this // Get thumbnail data to send to the other pod @@ -362,6 +363,22 @@ function toRemoteJSON (callback) { }) } +function toUpdateRemoteJSON (callback) { + const json = { + name: this.name, + description: this.description, + infoHash: this.infoHash, + remoteId: this.id, + author: this.Author.name, + duration: this.duration, + tags: map(this.Tags, 'name'), + createdAt: this.createdAt, + extname: this.extname + } + + return json +} + // ------------------------------ STATICS ------------------------------ function generateThumbnailFromData (video, thumbnailData, callback) { diff --git a/server/tests/api/check-params.js b/server/tests/api/check-params.js index 9aecc3720..e8f2aa821 100644 --- a/server/tests/api/check-params.js +++ b/server/tests/api/check-params.js @@ -10,6 +10,7 @@ const loginUtils = require('../utils/login') const requestsUtils = require('../utils/requests') const serversUtils = require('../utils/servers') const usersUtils = require('../utils/users') +const videosUtils = require('../utils/videos') describe('Test parameters validator', function () { let server = null @@ -439,6 +440,106 @@ describe('Test parameters validator', function () { }) }) + describe('When updating a video', function () { + let videoId + + before(function (done) { + videosUtils.getVideosList(server.url, function (err, res) { + if (err) throw err + + videoId = res.body.data[0].id + + return done() + }) + }) + + it('Should fail with nothing', function (done) { + const data = {} + requestsUtils.makePutBodyRequest(server.url, path + videoId, server.accessToken, data, done) + }) + + it('Should fail without a valid uuid', function (done) { + const data = { + description: 'my super description', + tags: [ 'tag1', 'tag2' ] + } + requestsUtils.makePutBodyRequest(server.url, path + 'blabla', server.accessToken, data, done) + }) + + it('Should fail with an unknown id', function (done) { + const data = { + description: 'my super description', + tags: [ 'tag1', 'tag2' ] + } + requestsUtils.makePutBodyRequest(server.url, path + '4da6fde3-88f7-4d16-b119-108df5630b06', server.accessToken, data, done) + }) + + it('Should fail with a long name', function (done) { + const data = { + name: 'My very very very very very very very very very very very very very very very very long name', + description: 'my super description', + tags: [ 'tag1', 'tag2' ] + } + requestsUtils.makePutBodyRequest(server.url, path + videoId, server.accessToken, data, done) + }) + + it('Should fail with a long description', function (done) { + const data = { + name: 'my super name', + description: 'my super description which is very very very very very very very very very very very very very very' + + 'very very very very very very very very very very very very very very very very very very very very very' + + 'very very very very very very very very very very very very very very very long', + tags: [ 'tag1', 'tag2' ] + } + requestsUtils.makePutBodyRequest(server.url, path + videoId, server.accessToken, data, done) + }) + + it('Should fail with too many tags', function (done) { + const data = { + name: 'my super name', + description: 'my super description', + tags: [ 'tag1', 'tag2', 'tag3', 'tag4' ] + } + requestsUtils.makePutBodyRequest(server.url, path + videoId, server.accessToken, data, done) + }) + + it('Should fail with not enough tags', function (done) { + const data = { + name: 'my super name', + description: 'my super description', + tags: [ ] + } + requestsUtils.makePutBodyRequest(server.url, path + videoId, server.accessToken, data, done) + }) + + it('Should fail with a tag length too low', function (done) { + const data = { + name: 'my super name', + description: 'my super description', + tags: [ 'tag1', 't' ] + } + requestsUtils.makePutBodyRequest(server.url, path + videoId, server.accessToken, data, done) + }) + + it('Should fail with a tag length too big', function (done) { + const data = { + name: 'my super name', + description: 'my super description', + tags: [ 'mysupertagtoolong', 'tag1' ] + } + requestsUtils.makePutBodyRequest(server.url, path + videoId, server.accessToken, data, done) + }) + + it('Should fail with malformed tags', function (done) { + const data = { + name: 'my super name', + description: 'my super description', + tags: [ 'my tag' ] + } + requestsUtils.makePutBodyRequest(server.url, path + videoId, server.accessToken, data, done) + }) + }) + describe('When getting a video', function () { it('Should return the list of the videos with nothing', function (done) { request(server.url) diff --git a/server/tests/api/single-pod.js b/server/tests/api/single-pod.js index 66b762f82..57146900d 100644 --- a/server/tests/api/single-pod.js +++ b/server/tests/api/single-pod.js @@ -495,10 +495,86 @@ describe('Test a single pod', function () { expect(videos[2].name === 'video_short2.webm name') expect(videos[3].name === 'video_short3.webm name') + videoId = videos[3].id + done() }) }) + it('Should update a video', function (done) { + const name = 'my super video updated' + const description = 'my super description updated' + const tags = [ 'tagup1', 'tagup2' ] + + videosUtils.updateVideo(server.url, server.accessToken, videoId, name, description, tags, done) + }) + + it('Should have the video updated', function (done) { + videosUtils.getVideo(server.url, videoId, function (err, res) { + if (err) throw err + + const video = res.body + + expect(video.name).to.equal('my super video updated') + expect(video.description).to.equal('my super description updated') + expect(video.podHost).to.equal('localhost:9001') + expect(video.author).to.equal('root') + expect(video.isLocal).to.be.true + expect(video.tags).to.deep.equal([ 'tagup1', 'tagup2' ]) + expect(miscsUtils.dateIsValid(video.createdAt)).to.be.true + + done() + }) + }) + + it('Should update only the tags of a video', function (done) { + const tags = [ 'tag1', 'tag2', 'supertag' ] + + videosUtils.updateVideo(server.url, server.accessToken, videoId, null, null, tags, function (err) { + if (err) throw err + + videosUtils.getVideo(server.url, videoId, function (err, res) { + if (err) throw err + + const video = res.body + + expect(video.name).to.equal('my super video updated') + expect(video.description).to.equal('my super description updated') + expect(video.podHost).to.equal('localhost:9001') + expect(video.author).to.equal('root') + expect(video.isLocal).to.be.true + expect(video.tags).to.deep.equal([ 'tag1', 'tag2', 'supertag' ]) + expect(miscsUtils.dateIsValid(video.createdAt)).to.be.true + + done() + }) + }) + }) + + it('Should update only the description of a video', function (done) { + const description = 'hello everybody' + + videosUtils.updateVideo(server.url, server.accessToken, videoId, null, description, null, function (err) { + if (err) throw err + + videosUtils.getVideo(server.url, videoId, function (err, res) { + if (err) throw err + + const video = res.body + + expect(video.name).to.equal('my super video updated') + expect(video.description).to.equal('hello everybody') + expect(video.podHost).to.equal('localhost:9001') + expect(video.author).to.equal('root') + expect(video.isLocal).to.be.true + expect(video.tags).to.deep.equal([ 'tag1', 'tag2', 'supertag' ]) + expect(miscsUtils.dateIsValid(video.createdAt)).to.be.true + + done() + }) + }) + }) + after(function (done) { process.kill(-server.app.pid) diff --git a/server/tests/utils/videos.js b/server/tests/utils/videos.js index 5c120597f..beafd3cf5 100644 --- a/server/tests/utils/videos.js +++ b/server/tests/utils/videos.js @@ -15,7 +15,8 @@ const videosUtils = { searchVideoWithPagination, searchVideoWithSort, testVideoImage, - uploadVideo + uploadVideo, + updateVideo } // ---------------------- Export functions -------------------- @@ -194,6 +195,31 @@ function uploadVideo (url, accessToken, name, description, tags, fixture, specia .end(end) } +function updateVideo (url, accessToken, id, name, description, tags, specialStatus, end) { + if (!end) { + end = specialStatus + specialStatus = 204 + } + + const path = '/api/v1/videos/' + id + + const req = request(url) + .put(path) + .set('Accept', 'application/json') + .set('Authorization', 'Bearer ' + accessToken) + + if (name) req.field('name', name) + if (description) req.field('description', description) + + if (tags) { + for (let i = 0; i < tags.length; i++) { + req.field('tags[' + i + ']', tags[i]) + } + } + + req.expect(specialStatus).end(end) +} + // --------------------------------------------------------------------------- module.exports = videosUtils -- cgit v1.2.3 From 3d118fb501f576a298f6bb059167e4c7f4dd8dcc Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Fri, 30 Dec 2016 11:27:42 +0100 Subject: Server: propagate video update to other pods --- server/controllers/api/remote.js | 108 ++++++++++++++++++++++++----- server/controllers/api/videos.js | 2 - server/helpers/custom-validators/videos.js | 15 ++++ server/models/video.js | 6 +- server/tests/api/multiple-pods.js | 55 ++++++++++++--- 5 files changed, 152 insertions(+), 34 deletions(-) (limited to 'server') diff --git a/server/controllers/api/remote.js b/server/controllers/api/remote.js index 929ade555..254ae56d5 100644 --- a/server/controllers/api/remote.js +++ b/server/controllers/api/remote.js @@ -35,12 +35,21 @@ function remoteVideos (req, res, next) { eachSeries(requests, function (request, callbackEach) { const videoData = request.data - if (request.type === 'add') { - addRemoteVideo(videoData, fromPod, callbackEach) - } else if (request.type === 'remove') { - removeRemoteVideo(videoData, fromPod, callbackEach) - } else { - logger.error('Unkown remote request type %s.', request.type) + switch (request.type) { + case 'add': + addRemoteVideo(videoData, fromPod, callbackEach) + break + + case 'update': + updateRemoteVideo(videoData, fromPod, callbackEach) + break + + case 'remove': + removeRemoteVideo(videoData, fromPod, callbackEach) + break + + default: + logger.error('Unkown remote request type %s.', request.type) } }, function (err) { if (err) logger.error('Error managing remote videos.', { error: err }) @@ -143,24 +152,85 @@ function addRemoteVideo (videoToCreateData, fromPod, finalCallback) { }) } -function removeRemoteVideo (videoToRemoveData, fromPod, callback) { - // TODO: use bulkDestroy? +function updateRemoteVideo (videoAttributesToUpdate, fromPod, finalCallback) { + logger.debug('Updating remote video "%s".', videoAttributesToUpdate.name) - // We need the list because we have to remove some other stuffs (thumbnail etc) - db.Video.listByHostAndRemoteId(fromPod.host, videoToRemoveData.remoteId, function (err, videosList) { - if (err) { - logger.error('Cannot list videos from host and remote id.', { error: err.message }) - return callback(err) + waterfall([ + + function startTransaction (callback) { + db.sequelize.transaction().asCallback(function (err, t) { + return callback(err, t) + }) + }, + + function findVideo (t, callback) { + db.Video.loadByHostAndRemoteId(fromPod.host, videoAttributesToUpdate.remoteId, function (err, videoInstance) { + if (err || !videoInstance) { + logger.error('Cannot load video from host and remote id.', { error: err.message }) + return callback(err) + } + + return callback(null, t, videoInstance) + }) + }, + + function findOrCreateTags (t, videoInstance, callback) { + const tags = videoAttributesToUpdate.tags + + db.Tag.findOrCreateTags(tags, t, function (err, tagInstances) { + return callback(err, t, videoInstance, tagInstances) + }) + }, + + function updateVideoIntoDB (t, videoInstance, tagInstances, callback) { + const options = { transaction: t } + + videoInstance.set('name', videoAttributesToUpdate.name) + videoInstance.set('description', videoAttributesToUpdate.description) + videoInstance.set('infoHash', videoAttributesToUpdate.infoHash) + videoInstance.set('duration', videoAttributesToUpdate.duration) + videoInstance.set('createdAt', videoAttributesToUpdate.createdAt) + videoInstance.set('extname', videoAttributesToUpdate.extname) + + videoInstance.save(options).asCallback(function (err) { + return callback(err, t, videoInstance, tagInstances) + }) + }, + + function associateTagsToVideo (t, videoInstance, tagInstances, callback) { + const options = { transaction: t } + + videoInstance.setTags(tagInstances, options).asCallback(function (err) { + return callback(err, t) + }) } - if (videosList.length === 0) { - logger.error('No remote video was found for this pod.', { remoteId: videoToRemoveData.remoteId, podHost: fromPod.host }) + ], function (err, t) { + if (err) { + logger.error('Cannot update the remote video.') + + // Abort transaction? + if (t) t.rollback() + + return finalCallback(err) } - each(videosList, function (video, callbackEach) { - logger.debug('Removing remote video %s.', video.remoteId) + // Commit transaction + t.commit() + + return finalCallback() + }) +} + +function removeRemoteVideo (videoToRemoveData, fromPod, callback) { + // We need the instance because we have to remove some other stuffs (thumbnail etc) + db.Video.loadByHostAndRemoteId(fromPod.host, videoToRemoveData.remoteId, function (err, video) { + if (err || !video) { + logger.error('Cannot load video from host and remote id.', { error: err.message }) + return callback(err) + } - video.destroy().asCallback(callbackEach) - }, callback) + logger.debug('Removing remote video %s.', video.remoteId) + video.destroy().asCallback(callback) }) } diff --git a/server/controllers/api/videos.js b/server/controllers/api/videos.js index 1b306d1cf..e5c52a87b 100644 --- a/server/controllers/api/videos.js +++ b/server/controllers/api/videos.js @@ -229,8 +229,6 @@ function updateVideo (req, res, next) { // Add tags association videoInstance.save(options).asCallback(function (err) { - if (err) return callback(err) - return callback(err, t, tagInstances) }) }, diff --git a/server/helpers/custom-validators/videos.js b/server/helpers/custom-validators/videos.js index 4aaa6aaa9..b76eec1b5 100644 --- a/server/helpers/custom-validators/videos.js +++ b/server/helpers/custom-validators/videos.js @@ -37,6 +37,17 @@ function isEachRemoteVideosValid (requests) { isVideoRemoteIdValid(video.remoteId) && isVideoExtnameValid(video.extname) ) || + ( + isRequestTypeUpdateValid(request.type) && + isVideoDateValid(video.createdAt) && + isVideoDescriptionValid(video.description) && + isVideoDurationValid(video.duration) && + isVideoInfoHashValid(video.infoHash) && + isVideoNameValid(video.name) && + isVideoTagsValid(video.tags) && + isVideoRemoteIdValid(video.remoteId) && + isVideoExtnameValid(video.extname) + ) || ( isRequestTypeRemoveValid(request.type) && isVideoNameValid(video.name) && @@ -104,6 +115,10 @@ function isRequestTypeAddValid (value) { return value === 'add' } +function isRequestTypeUpdateValid (value) { + return value === 'update' +} + function isRequestTypeRemoveValid (value) { return value === 'remove' } diff --git a/server/models/video.js b/server/models/video.js index 14fbe2f71..f51d08f06 100644 --- a/server/models/video.js +++ b/server/models/video.js @@ -111,10 +111,10 @@ module.exports = function (sequelize, DataTypes) { getDurationFromFile, list, listForApi, - listByHostAndRemoteId, listOwnedAndPopulateAuthorAndTags, listOwnedByAuthor, load, + loadByHostAndRemoteId, loadAndPopulateAuthor, loadAndPopulateAuthorAndPodAndTags, searchAndPopulateAuthorAndPodAndTags @@ -428,7 +428,7 @@ function listForApi (start, count, sort, callback) { }) } -function listByHostAndRemoteId (fromHost, remoteId, callback) { +function loadByHostAndRemoteId (fromHost, remoteId, callback) { const query = { where: { remoteId: remoteId @@ -449,7 +449,7 @@ function listByHostAndRemoteId (fromHost, remoteId, callback) { ] } - return this.findAll(query).asCallback(callback) + return this.findOne(query).asCallback(callback) } function listOwnedAndPopulateAuthorAndTags (callback) { diff --git a/server/tests/api/multiple-pods.js b/server/tests/api/multiple-pods.js index f0fe59c5f..672187068 100644 --- a/server/tests/api/multiple-pods.js +++ b/server/tests/api/multiple-pods.js @@ -299,8 +299,8 @@ describe('Test multiple pods', function () { if (err) throw err const video = res.body.data[0] - toRemove.push(res.body.data[2].id) - toRemove.push(res.body.data[3].id) + toRemove.push(res.body.data[2]) + toRemove.push(res.body.data[3]) webtorrent.add(video.magnetUri, function (torrent) { expect(torrent.files).to.exist @@ -368,16 +368,51 @@ describe('Test multiple pods', function () { }) }) }) + }) + + describe('Should manipulate these videos', function () { + it('Should update the video 3 by asking pod 3', function (done) { + this.timeout(15000) + + const name = 'my super video updated' + const description = 'my super description updated' + const tags = [ 'tagup1', 'tagup2' ] + + videosUtils.updateVideo(servers[2].url, servers[2].accessToken, toRemove[0].id, name, description, tags, function (err) { + if (err) throw err + + setTimeout(done, 11000) + }) + }) + + it('Should have the video 3 updated on each pod', function (done) { + each(servers, function (server, callback) { + videosUtils.getVideosList(server.url, function (err, res) { + if (err) throw err + + const videos = res.body.data + const videoUpdated = videos.find(function (video) { + return video.name === 'my super video updated' + }) + + expect(!!videoUpdated).to.be.true + expect(videoUpdated.description).to.equal('my super description updated') + expect(videoUpdated.tags).to.deep.equal([ 'tagup1', 'tagup2' ]) + + callback() + }) + }, done) + }) - it('Should remove the file 3 and 3-2 by asking pod 3', function (done) { + it('Should remove the videos 3 and 3-2 by asking pod 3', function (done) { this.timeout(15000) series([ function (next) { - videosUtils.removeVideo(servers[2].url, servers[2].accessToken, toRemove[0], next) + videosUtils.removeVideo(servers[2].url, servers[2].accessToken, toRemove[0].id, next) }, function (next) { - videosUtils.removeVideo(servers[2].url, servers[2].accessToken, toRemove[1], next) + videosUtils.removeVideo(servers[2].url, servers[2].accessToken, toRemove[1].id, next) }], function (err) { if (err) throw err @@ -394,11 +429,11 @@ describe('Test multiple pods', function () { const videos = res.body.data expect(videos).to.be.an('array') expect(videos.length).to.equal(2) - expect(videos[0].id).not.to.equal(videos[1].id) - expect(videos[0].id).not.to.equal(toRemove[0]) - expect(videos[1].id).not.to.equal(toRemove[0]) - expect(videos[0].id).not.to.equal(toRemove[1]) - expect(videos[1].id).not.to.equal(toRemove[1]) + expect(videos[0].name).not.to.equal(videos[1].name) + expect(videos[0].name).not.to.equal(toRemove[0].name) + expect(videos[1].name).not.to.equal(toRemove[0].name) + expect(videos[0].name).not.to.equal(toRemove[1].name) + expect(videos[1].name).not.to.equal(toRemove[1].name) callback() }) -- cgit v1.2.3 From 79066fdf33f79d2d41394f10881e2c226ca26b49 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Fri, 30 Dec 2016 11:45:00 +0100 Subject: Server: add updatedAt attribute to videos --- server/controllers/api/remote.js | 4 +++- server/helpers/custom-validators/videos.js | 2 ++ server/models/video.js | 6 ++++-- server/tests/api/multiple-pods.js | 5 +++++ server/tests/api/single-pod.js | 8 ++++++++ server/tests/utils/miscs.js | 6 ++++-- 6 files changed, 26 insertions(+), 5 deletions(-) (limited to 'server') diff --git a/server/controllers/api/remote.js b/server/controllers/api/remote.js index 254ae56d5..a36c31c38 100644 --- a/server/controllers/api/remote.js +++ b/server/controllers/api/remote.js @@ -98,7 +98,8 @@ function addRemoteVideo (videoToCreateData, fromPod, finalCallback) { description: videoToCreateData.description, authorId: author.id, duration: videoToCreateData.duration, - createdAt: videoToCreateData.createdAt + createdAt: videoToCreateData.createdAt, + updatedAt: videoToCreateData.updatedAt } const video = db.Video.build(videoData) @@ -190,6 +191,7 @@ function updateRemoteVideo (videoAttributesToUpdate, fromPod, finalCallback) { videoInstance.set('infoHash', videoAttributesToUpdate.infoHash) videoInstance.set('duration', videoAttributesToUpdate.duration) videoInstance.set('createdAt', videoAttributesToUpdate.createdAt) + videoInstance.set('updatedAt', videoAttributesToUpdate.updatedAt) videoInstance.set('extname', videoAttributesToUpdate.extname) videoInstance.save(options).asCallback(function (err) { diff --git a/server/helpers/custom-validators/videos.js b/server/helpers/custom-validators/videos.js index b76eec1b5..8448386d9 100644 --- a/server/helpers/custom-validators/videos.js +++ b/server/helpers/custom-validators/videos.js @@ -28,6 +28,7 @@ function isEachRemoteVideosValid (requests) { isRequestTypeAddValid(request.type) && isVideoAuthorValid(video.author) && isVideoDateValid(video.createdAt) && + isVideoDateValid(video.updatedAt) && isVideoDescriptionValid(video.description) && isVideoDurationValid(video.duration) && isVideoInfoHashValid(video.infoHash) && @@ -40,6 +41,7 @@ function isEachRemoteVideosValid (requests) { ( isRequestTypeUpdateValid(request.type) && isVideoDateValid(video.createdAt) && + isVideoDateValid(video.updatedAt) && isVideoDescriptionValid(video.description) && isVideoDurationValid(video.duration) && isVideoInfoHashValid(video.infoHash) && diff --git a/server/models/video.js b/server/models/video.js index f51d08f06..3fe8368c7 100644 --- a/server/models/video.js +++ b/server/models/video.js @@ -20,7 +20,6 @@ const customVideosValidators = require('../helpers/custom-validators').videos // --------------------------------------------------------------------------- module.exports = function (sequelize, DataTypes) { - // TODO: add indexes on searchable columns const Video = sequelize.define('Video', { id: { @@ -329,7 +328,8 @@ function toFormatedJSON () { duration: this.duration, tags: map(this.Tags, 'name'), thumbnailPath: constants.STATIC_PATHS.THUMBNAILS + '/' + this.getThumbnailName(), - createdAt: this.createdAt + createdAt: this.createdAt, + updatedAt: this.updatedAt } return json @@ -356,6 +356,7 @@ function toAddRemoteJSON (callback) { thumbnailData: thumbnailData.toString('binary'), tags: map(self.Tags, 'name'), createdAt: self.createdAt, + updatedAt: self.updatedAt, extname: self.extname } @@ -373,6 +374,7 @@ function toUpdateRemoteJSON (callback) { duration: this.duration, tags: map(this.Tags, 'name'), createdAt: this.createdAt, + updatedAt: this.updatedAt, extname: this.extname } diff --git a/server/tests/api/multiple-pods.js b/server/tests/api/multiple-pods.js index 672187068..4442a7ff7 100644 --- a/server/tests/api/multiple-pods.js +++ b/server/tests/api/multiple-pods.js @@ -105,6 +105,7 @@ describe('Test multiple pods', function () { expect(video.duration).to.equal(10) expect(video.tags).to.deep.equal([ 'tag1p1', 'tag2p1' ]) expect(miscsUtils.dateIsValid(video.createdAt)).to.be.true + expect(miscsUtils.dateIsValid(video.updatedAt)).to.be.true expect(video.author).to.equal('root') if (server.url !== 'http://localhost:9001') { @@ -167,6 +168,7 @@ describe('Test multiple pods', function () { expect(video.duration).to.equal(5) expect(video.tags).to.deep.equal([ 'tag1p2', 'tag2p2', 'tag3p2' ]) expect(miscsUtils.dateIsValid(video.createdAt)).to.be.true + expect(miscsUtils.dateIsValid(video.updatedAt)).to.be.true expect(video.author).to.equal('root') if (server.url !== 'http://localhost:9002') { @@ -247,6 +249,7 @@ describe('Test multiple pods', function () { expect(video1.tags).to.deep.equal([ 'tag1p3' ]) expect(video1.author).to.equal('root') expect(miscsUtils.dateIsValid(video1.createdAt)).to.be.true + expect(miscsUtils.dateIsValid(video1.updatedAt)).to.be.true expect(video2.name).to.equal('my super name for pod 3-2') expect(video2.description).to.equal('my super description for pod 3-2') @@ -256,6 +259,7 @@ describe('Test multiple pods', function () { expect(video2.tags).to.deep.equal([ 'tag2p3', 'tag3p3', 'tag4p3' ]) expect(video2.author).to.equal('root') expect(miscsUtils.dateIsValid(video2.createdAt)).to.be.true + expect(miscsUtils.dateIsValid(video2.updatedAt)).to.be.true if (server.url !== 'http://localhost:9003') { expect(video1.isLocal).to.be.false @@ -398,6 +402,7 @@ describe('Test multiple pods', function () { expect(!!videoUpdated).to.be.true expect(videoUpdated.description).to.equal('my super description updated') expect(videoUpdated.tags).to.deep.equal([ 'tagup1', 'tagup2' ]) + expect(miscsUtils.dateIsValid(videoUpdated.updatedAt, 20000)).to.be.true callback() }) diff --git a/server/tests/api/single-pod.js b/server/tests/api/single-pod.js index 57146900d..29512dfc6 100644 --- a/server/tests/api/single-pod.js +++ b/server/tests/api/single-pod.js @@ -83,6 +83,7 @@ describe('Test a single pod', function () { expect(video.isLocal).to.be.true expect(video.tags).to.deep.equal([ 'tag1', 'tag2', 'tag3' ]) expect(miscsUtils.dateIsValid(video.createdAt)).to.be.true + expect(miscsUtils.dateIsValid(video.updatedAt)).to.be.true videosUtils.testVideoImage(server.url, 'video_short.webm', video.thumbnailPath, function (err, test) { if (err) throw err @@ -117,6 +118,7 @@ describe('Test a single pod', function () { expect(video.isLocal).to.be.true expect(video.tags).to.deep.equal([ 'tag1', 'tag2', 'tag3' ]) expect(miscsUtils.dateIsValid(video.createdAt)).to.be.true + expect(miscsUtils.dateIsValid(video.updatedAt)).to.be.true videosUtils.testVideoImage(server.url, 'video_short.webm', video.thumbnailPath, function (err, test) { if (err) throw err @@ -143,6 +145,7 @@ describe('Test a single pod', function () { expect(video.isLocal).to.be.true expect(video.tags).to.deep.equal([ 'tag1', 'tag2', 'tag3' ]) expect(miscsUtils.dateIsValid(video.createdAt)).to.be.true + expect(miscsUtils.dateIsValid(video.updatedAt)).to.be.true videosUtils.testVideoImage(server.url, 'video_short.webm', video.thumbnailPath, function (err, test) { if (err) throw err @@ -170,6 +173,7 @@ describe('Test a single pod', function () { // expect(video.isLocal).to.be.true // expect(video.tags).to.deep.equal([ 'tag1', 'tag2', 'tag3' ]) // expect(miscsUtils.dateIsValid(video.createdAt)).to.be.true + // expect(miscsUtils.dateIsValid(video.updatedAt)).to.be.true // videosUtils.testVideoImage(server.url, 'video_short.webm', video.thumbnailPath, function (err, test) { // if (err) throw err @@ -196,6 +200,7 @@ describe('Test a single pod', function () { expect(video.isLocal).to.be.true expect(video.tags).to.deep.equal([ 'tag1', 'tag2', 'tag3' ]) expect(miscsUtils.dateIsValid(video.createdAt)).to.be.true + expect(miscsUtils.dateIsValid(video.updatedAt)).to.be.true videosUtils.testVideoImage(server.url, 'video_short.webm', video.thumbnailPath, function (err, test) { if (err) throw err @@ -522,6 +527,7 @@ describe('Test a single pod', function () { expect(video.isLocal).to.be.true expect(video.tags).to.deep.equal([ 'tagup1', 'tagup2' ]) expect(miscsUtils.dateIsValid(video.createdAt)).to.be.true + expect(miscsUtils.dateIsValid(video.updatedAt)).to.be.true done() }) @@ -545,6 +551,7 @@ describe('Test a single pod', function () { expect(video.isLocal).to.be.true expect(video.tags).to.deep.equal([ 'tag1', 'tag2', 'supertag' ]) expect(miscsUtils.dateIsValid(video.createdAt)).to.be.true + expect(miscsUtils.dateIsValid(video.updatedAt)).to.be.true done() }) @@ -569,6 +576,7 @@ describe('Test a single pod', function () { expect(video.isLocal).to.be.true expect(video.tags).to.deep.equal([ 'tag1', 'tag2', 'supertag' ]) expect(miscsUtils.dateIsValid(video.createdAt)).to.be.true + expect(miscsUtils.dateIsValid(video.updatedAt)).to.be.true done() }) diff --git a/server/tests/utils/miscs.js b/server/tests/utils/miscs.js index 4ceff65df..c4b661496 100644 --- a/server/tests/utils/miscs.js +++ b/server/tests/utils/miscs.js @@ -6,12 +6,14 @@ const miscsUtils = { // ---------------------- Export functions -------------------- -function dateIsValid (dateString) { +function dateIsValid (dateString, interval) { const dateToCheck = new Date(dateString) const now = new Date() // Check if the interval is more than 2 minutes - if (now - dateToCheck > 120000) return false + if (!interval) interval = 120000 + + if (now - dateToCheck > interval) return false return true } -- cgit v1.2.3 From 818f7987eba27c59793e2103168b26129c9404f2 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Fri, 30 Dec 2016 11:51:08 +0100 Subject: Server: optimization for videoGet and videoRemove --- server/controllers/api/videos.js | 28 +++++----------------------- server/middlewares/validators/videos.js | 15 ++++++++------- 2 files changed, 13 insertions(+), 30 deletions(-) (limited to 'server') diff --git a/server/controllers/api/videos.js b/server/controllers/api/videos.js index e5c52a87b..35d6979e5 100644 --- a/server/controllers/api/videos.js +++ b/server/controllers/api/videos.js @@ -200,7 +200,7 @@ function addVideo (req, res, next) { } function updateVideo (req, res, next) { - let videoInstance = res.locals.video + const videoInstance = res.locals.video const videoInfosToUpdate = req.body waterfall([ @@ -275,15 +275,8 @@ function updateVideo (req, res, next) { } function getVideo (req, res, next) { - db.Video.loadAndPopulateAuthorAndPodAndTags(req.params.id, function (err, video) { - if (err) return next(err) - - if (!video) { - return res.type('json').status(204).end() - } - - res.json(video.toFormatedJSON()) - }) + const videoInstance = res.locals.video + res.json(videoInstance.toFormatedJSON()) } function listVideos (req, res, next) { @@ -295,20 +288,9 @@ function listVideos (req, res, next) { } function removeVideo (req, res, next) { - const videoId = req.params.id + const videoInstance = res.locals.video - waterfall([ - function loadVideo (callback) { - db.Video.load(videoId, function (err, video) { - return callback(err, video) - }) - }, - - function deleteVideo (video, callback) { - // Informations to other pods will be sent by the afterDestroy video hook - video.destroy().asCallback(callback) - } - ], function andFinally (err) { + videoInstance.destroy().asCallback(function (err) { if (err) { logger.error('Errors when removed the video.', { error: err }) return next(err) diff --git a/server/middlewares/validators/videos.js b/server/middlewares/validators/videos.js index 09a188c76..1b6dbccf0 100644 --- a/server/middlewares/validators/videos.js +++ b/server/middlewares/validators/videos.js @@ -71,15 +71,16 @@ function videosRemove (req, res, next) { logger.debug('Checking videosRemove parameters', { parameters: req.params }) checkErrors(req, res, function () { - db.Video.loadAndPopulateAuthor(req.params.id, function (err, video) { - if (err) { - logger.error('Error in videosRemove request validator.', { error: err }) - return res.sendStatus(500) + checkVideoExists(req.params.id, res, function () { + // We need to make additional checks + + if (res.locals.video.isOwned() === false) { + return res.status(403).send('Cannot remove video of another pod') } - if (!video) return res.status(404).send('Video not found') - else if (video.isOwned() === false) return res.status(403).send('Cannot remove video of another pod') - else if (video.Author.name !== res.locals.oauth.token.user.username) return res.status(403).send('Cannot remove video of another user') + if (res.locals.video.authorId !== res.locals.oauth.token.User.id) { + return res.status(403).send('Cannot remove video of another user') + } next() }) -- cgit v1.2.3 From efe923bcdaf15b47593ad8583df09a92c715ac6c Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Fri, 30 Dec 2016 12:23:53 +0100 Subject: Server: split check params tests --- server/controllers/api/remote.js | 1 - server/tests/api/check-params.js | 865 ------------------------------ server/tests/api/check-params/index.js | 8 + server/tests/api/check-params/pods.js | 204 +++++++ server/tests/api/check-params/remotes.js | 60 +++ server/tests/api/check-params/requests.js | 87 +++ server/tests/api/check-params/users.js | 284 ++++++++++ server/tests/api/check-params/videos.js | 456 ++++++++++++++++ server/tests/utils/login.js | 11 +- server/tests/utils/servers.js | 2 +- 10 files changed, 1110 insertions(+), 868 deletions(-) delete mode 100644 server/tests/api/check-params.js create mode 100644 server/tests/api/check-params/index.js create mode 100644 server/tests/api/check-params/pods.js create mode 100644 server/tests/api/check-params/remotes.js create mode 100644 server/tests/api/check-params/requests.js create mode 100644 server/tests/api/check-params/users.js create mode 100644 server/tests/api/check-params/videos.js (limited to 'server') diff --git a/server/controllers/api/remote.js b/server/controllers/api/remote.js index a36c31c38..be5e6dc98 100644 --- a/server/controllers/api/remote.js +++ b/server/controllers/api/remote.js @@ -1,6 +1,5 @@ 'use strict' -const each = require('async/each') const eachSeries = require('async/eachSeries') const express = require('express') const waterfall = require('async/waterfall') diff --git a/server/tests/api/check-params.js b/server/tests/api/check-params.js deleted file mode 100644 index e8f2aa821..000000000 --- a/server/tests/api/check-params.js +++ /dev/null @@ -1,865 +0,0 @@ -'use strict' - -const chai = require('chai') -const expect = chai.expect -const pathUtils = require('path') -const request = require('supertest') -const series = require('async/series') - -const loginUtils = require('../utils/login') -const requestsUtils = require('../utils/requests') -const serversUtils = require('../utils/servers') -const usersUtils = require('../utils/users') -const videosUtils = require('../utils/videos') - -describe('Test parameters validator', function () { - let server = null - let userAccessToken = null - - // --------------------------------------------------------------- - - before(function (done) { - this.timeout(20000) - - series([ - function (next) { - serversUtils.flushTests(next) - }, - function (next) { - serversUtils.runServer(1, function (server1) { - server = server1 - - next() - }) - }, - function (next) { - loginUtils.loginAndGetAccessToken(server, function (err, token) { - if (err) throw err - server.accessToken = token - - next() - }) - } - ], done) - }) - - describe('Of the pods API', function () { - const path = '/api/v1/pods/' - - describe('When making friends', function () { - let userAccessToken = null - - before(function (done) { - usersUtils.createUser(server.url, server.accessToken, 'user1', 'password', function () { - server.user = { - username: 'user1', - password: 'password' - } - - loginUtils.loginAndGetAccessToken(server, function (err, accessToken) { - if (err) throw err - - userAccessToken = accessToken - - done() - }) - }) - }) - - describe('When making friends', function () { - const body = { - hosts: [ 'localhost:9002' ] - } - - it('Should fail without hosts', function (done) { - request(server.url) - .post(path + '/makefriends') - .set('Authorization', 'Bearer ' + server.accessToken) - .set('Accept', 'application/json') - .expect(400, done) - }) - - it('Should fail if hosts is not an array', function (done) { - request(server.url) - .post(path + '/makefriends') - .send({ hosts: 'localhost:9002' }) - .set('Authorization', 'Bearer ' + server.accessToken) - .set('Accept', 'application/json') - .expect(400, done) - }) - - it('Should fail if the array is not composed by hosts', function (done) { - request(server.url) - .post(path + '/makefriends') - .send({ hosts: [ 'localhost:9002', 'localhost:coucou' ] }) - .set('Authorization', 'Bearer ' + server.accessToken) - .set('Accept', 'application/json') - .expect(400, done) - }) - - it('Should fail if the array is composed with http schemes', function (done) { - request(server.url) - .post(path + '/makefriends') - .send({ hosts: [ 'localhost:9002', 'http://localhost:9003' ] }) - .set('Authorization', 'Bearer ' + server.accessToken) - .set('Accept', 'application/json') - .expect(400, done) - }) - - it('Should fail if hosts are not unique', function (done) { - request(server.url) - .post(path + '/makefriends') - .send({ urls: [ 'localhost:9002', 'localhost:9002' ] }) - .set('Authorization', 'Bearer ' + server.accessToken) - .set('Accept', 'application/json') - .expect(400, done) - }) - - it('Should fail with a invalid token', function (done) { - request(server.url) - .post(path + '/makefriends') - .send(body) - .set('Authorization', 'Bearer faketoken') - .set('Accept', 'application/json') - .expect(401, done) - }) - - it('Should fail if the user is not an administrator', function (done) { - request(server.url) - .post(path + '/makefriends') - .send(body) - .set('Authorization', 'Bearer ' + userAccessToken) - .set('Accept', 'application/json') - .expect(403, done) - }) - }) - - describe('When quitting friends', function () { - it('Should fail with a invalid token', function (done) { - request(server.url) - .get(path + '/quitfriends') - .query({ start: 'hello' }) - .set('Authorization', 'Bearer faketoken') - .set('Accept', 'application/json') - .expect(401, done) - }) - - it('Should fail if the user is not an administrator', function (done) { - request(server.url) - .get(path + '/quitfriends') - .query({ start: 'hello' }) - .set('Authorization', 'Bearer ' + userAccessToken) - .set('Accept', 'application/json') - .expect(403, done) - }) - }) - }) - - describe('When adding a pod', function () { - it('Should fail with nothing', function (done) { - const data = {} - requestsUtils.makePostBodyRequest(server.url, path, null, data, done) - }) - - it('Should fail without public key', function (done) { - const data = { - host: 'coucou.com' - } - requestsUtils.makePostBodyRequest(server.url, path, null, data, done) - }) - - it('Should fail without an host', function (done) { - const data = { - publicKey: 'mysuperpublickey' - } - requestsUtils.makePostBodyRequest(server.url, path, null, data, done) - }) - - it('Should fail with an incorrect host', function (done) { - const data = { - host: 'http://coucou.com', - publicKey: 'mysuperpublickey' - } - requestsUtils.makePostBodyRequest(server.url, path, null, data, function () { - data.host = 'http://coucou' - requestsUtils.makePostBodyRequest(server.url, path, null, data, function () { - data.host = 'coucou' - requestsUtils.makePostBodyRequest(server.url, path, null, data, done) - }) - }) - }) - - it('Should succeed with the correct parameters', function (done) { - const data = { - host: 'coucou.com', - publicKey: 'mysuperpublickey' - } - requestsUtils.makePostBodyRequest(server.url, path, null, data, done, 200) - }) - }) - }) - - describe('Of the videos API', function () { - const path = '/api/v1/videos/' - - describe('When listing a video', function () { - it('Should fail with a bad start pagination', function (done) { - request(server.url) - .get(path) - .query({ start: 'hello' }) - .set('Accept', 'application/json') - .expect(400, done) - }) - - it('Should fail with a bad count pagination', function (done) { - request(server.url) - .get(path) - .query({ count: 'hello' }) - .set('Accept', 'application/json') - .expect(400, done) - }) - - it('Should fail with an incorrect sort', function (done) { - request(server.url) - .get(path) - .query({ sort: 'hello' }) - .set('Accept', 'application/json') - .expect(400, done) - }) - }) - - describe('When searching a video', function () { - it('Should fail with nothing', function (done) { - request(server.url) - .get(pathUtils.join(path, 'search')) - .set('Accept', 'application/json') - .expect(400, done) - }) - - it('Should fail with a bad start pagination', function (done) { - request(server.url) - .get(pathUtils.join(path, 'search', 'test')) - .query({ start: 'hello' }) - .set('Accept', 'application/json') - .expect(400, done) - }) - - it('Should fail with a bad count pagination', function (done) { - request(server.url) - .get(pathUtils.join(path, 'search', 'test')) - .query({ count: 'hello' }) - .set('Accept', 'application/json') - .expect(400, done) - }) - - it('Should fail with an incorrect sort', function (done) { - request(server.url) - .get(pathUtils.join(path, 'search', 'test')) - .query({ sort: 'hello' }) - .set('Accept', 'application/json') - .expect(400, done) - }) - }) - - describe('When adding a video', function () { - it('Should fail with nothing', function (done) { - const data = {} - const attach = {} - requestsUtils.makePostUploadRequest(server.url, path, server.accessToken, data, attach, done) - }) - - it('Should fail without name', function (done) { - const data = { - description: 'my super description', - tags: [ 'tag1', 'tag2' ] - } - const attach = { - 'videofile': pathUtils.join(__dirname, 'fixtures', 'video_short.webm') - } - requestsUtils.makePostUploadRequest(server.url, path, server.accessToken, data, attach, done) - }) - - it('Should fail with a long name', function (done) { - const data = { - name: 'My very very very very very very very very very very very very very very very very long name', - description: 'my super description', - tags: [ 'tag1', 'tag2' ] - } - const attach = { - 'videofile': pathUtils.join(__dirname, 'fixtures', 'video_short.webm') - } - requestsUtils.makePostUploadRequest(server.url, path, server.accessToken, data, attach, done) - }) - - it('Should fail without description', function (done) { - const data = { - name: 'my super name', - tags: [ 'tag1', 'tag2' ] - } - const attach = { - 'videofile': pathUtils.join(__dirname, 'fixtures', 'video_short.webm') - } - requestsUtils.makePostUploadRequest(server.url, path, server.accessToken, data, attach, done) - }) - - it('Should fail with a long description', function (done) { - const data = { - name: 'my super name', - description: 'my super description which is very very very very very very very very very very very very very very' + - 'very very very very very very very very very very very very very very very very very very very very very' + - 'very very very very very very very very very very very very very very very long', - tags: [ 'tag1', 'tag2' ] - } - const attach = { - 'videofile': pathUtils.join(__dirname, 'fixtures', 'video_short.webm') - } - requestsUtils.makePostUploadRequest(server.url, path, server.accessToken, data, attach, done) - }) - - it('Should fail without tags', function (done) { - const data = { - name: 'my super name', - description: 'my super description' - } - const attach = { - 'videofile': pathUtils.join(__dirname, 'fixtures', 'video_short.webm') - } - requestsUtils.makePostUploadRequest(server.url, path, server.accessToken, data, attach, done) - }) - - it('Should fail with too many tags', function (done) { - const data = { - name: 'my super name', - description: 'my super description', - tags: [ 'tag1', 'tag2', 'tag3', 'tag4' ] - } - const attach = { - 'videofile': pathUtils.join(__dirname, 'fixtures', 'video_short.webm') - } - requestsUtils.makePostUploadRequest(server.url, path, server.accessToken, data, attach, done) - }) - - it('Should fail with not enough tags', function (done) { - const data = { - name: 'my super name', - description: 'my super description', - tags: [ ] - } - const attach = { - 'videofile': pathUtils.join(__dirname, 'fixtures', 'video_short.webm') - } - requestsUtils.makePostUploadRequest(server.url, path, server.accessToken, data, attach, done) - }) - - it('Should fail with a tag length too low', function (done) { - const data = { - name: 'my super name', - description: 'my super description', - tags: [ 'tag1', 't' ] - } - const attach = { - 'videofile': pathUtils.join(__dirname, 'fixtures', 'video_short.webm') - } - requestsUtils.makePostUploadRequest(server.url, path, server.accessToken, data, attach, done) - }) - - it('Should fail with a tag length too big', function (done) { - const data = { - name: 'my super name', - description: 'my super description', - tags: [ 'mysupertagtoolong', 'tag1' ] - } - const attach = { - 'videofile': pathUtils.join(__dirname, 'fixtures', 'video_short.webm') - } - requestsUtils.makePostUploadRequest(server.url, path, server.accessToken, data, attach, done) - }) - - it('Should fail with malformed tags', function (done) { - const data = { - name: 'my super name', - description: 'my super description', - tags: [ 'my tag' ] - } - const attach = { - 'videofile': pathUtils.join(__dirname, 'fixtures', 'video_short.webm') - } - requestsUtils.makePostUploadRequest(server.url, path, server.accessToken, data, attach, done) - }) - - it('Should fail without an input file', function (done) { - const data = { - name: 'my super name', - description: 'my super description', - tags: [ 'tag1', 'tag2' ] - } - const attach = {} - requestsUtils.makePostUploadRequest(server.url, path, server.accessToken, data, attach, done) - }) - - it('Should fail without an incorrect input file', function (done) { - const data = { - name: 'my super name', - description: 'my super description', - tags: [ 'tag1', 'tag2' ] - } - const attach = { - 'videofile': pathUtils.join(__dirname, 'fixtures', 'video_short_fake.webm') - } - requestsUtils.makePostUploadRequest(server.url, path, server.accessToken, data, attach, done) - }) - - it('Should fail with a too big duration', function (done) { - const data = { - name: 'my super name', - description: 'my super description', - tags: [ 'tag1', 'tag2' ] - } - const attach = { - 'videofile': pathUtils.join(__dirname, 'fixtures', 'video_too_long.webm') - } - requestsUtils.makePostUploadRequest(server.url, path, server.accessToken, data, attach, done) - }) - - it('Should succeed with the correct parameters', function (done) { - const data = { - name: 'my super name', - description: 'my super description', - tags: [ 'tag1', 'tag2' ] - } - const attach = { - 'videofile': pathUtils.join(__dirname, 'fixtures', 'video_short.webm') - } - requestsUtils.makePostUploadRequest(server.url, path, server.accessToken, data, attach, function () { - attach.videofile = pathUtils.join(__dirname, 'fixtures', 'video_short.mp4') - requestsUtils.makePostUploadRequest(server.url, path, server.accessToken, data, attach, function () { - attach.videofile = pathUtils.join(__dirname, 'fixtures', 'video_short.ogv') - requestsUtils.makePostUploadRequest(server.url, path, server.accessToken, data, attach, done, 204) - }, false) - }, false) - }) - }) - - describe('When updating a video', function () { - let videoId - - before(function (done) { - videosUtils.getVideosList(server.url, function (err, res) { - if (err) throw err - - videoId = res.body.data[0].id - - return done() - }) - }) - - it('Should fail with nothing', function (done) { - const data = {} - requestsUtils.makePutBodyRequest(server.url, path + videoId, server.accessToken, data, done) - }) - - it('Should fail without a valid uuid', function (done) { - const data = { - description: 'my super description', - tags: [ 'tag1', 'tag2' ] - } - requestsUtils.makePutBodyRequest(server.url, path + 'blabla', server.accessToken, data, done) - }) - - it('Should fail with an unknown id', function (done) { - const data = { - description: 'my super description', - tags: [ 'tag1', 'tag2' ] - } - requestsUtils.makePutBodyRequest(server.url, path + '4da6fde3-88f7-4d16-b119-108df5630b06', server.accessToken, data, done) - }) - - it('Should fail with a long name', function (done) { - const data = { - name: 'My very very very very very very very very very very very very very very very very long name', - description: 'my super description', - tags: [ 'tag1', 'tag2' ] - } - requestsUtils.makePutBodyRequest(server.url, path + videoId, server.accessToken, data, done) - }) - - it('Should fail with a long description', function (done) { - const data = { - name: 'my super name', - description: 'my super description which is very very very very very very very very very very very very very very' + - 'very very very very very very very very very very very very very very very very very very very very very' + - 'very very very very very very very very very very very very very very very long', - tags: [ 'tag1', 'tag2' ] - } - requestsUtils.makePutBodyRequest(server.url, path + videoId, server.accessToken, data, done) - }) - - it('Should fail with too many tags', function (done) { - const data = { - name: 'my super name', - description: 'my super description', - tags: [ 'tag1', 'tag2', 'tag3', 'tag4' ] - } - requestsUtils.makePutBodyRequest(server.url, path + videoId, server.accessToken, data, done) - }) - - it('Should fail with not enough tags', function (done) { - const data = { - name: 'my super name', - description: 'my super description', - tags: [ ] - } - requestsUtils.makePutBodyRequest(server.url, path + videoId, server.accessToken, data, done) - }) - - it('Should fail with a tag length too low', function (done) { - const data = { - name: 'my super name', - description: 'my super description', - tags: [ 'tag1', 't' ] - } - requestsUtils.makePutBodyRequest(server.url, path + videoId, server.accessToken, data, done) - }) - - it('Should fail with a tag length too big', function (done) { - const data = { - name: 'my super name', - description: 'my super description', - tags: [ 'mysupertagtoolong', 'tag1' ] - } - requestsUtils.makePutBodyRequest(server.url, path + videoId, server.accessToken, data, done) - }) - - it('Should fail with malformed tags', function (done) { - const data = { - name: 'my super name', - description: 'my super description', - tags: [ 'my tag' ] - } - requestsUtils.makePutBodyRequest(server.url, path + videoId, server.accessToken, data, done) - }) - }) - - describe('When getting a video', function () { - it('Should return the list of the videos with nothing', function (done) { - request(server.url) - .get(path) - .set('Accept', 'application/json') - .expect(200) - .expect('Content-Type', /json/) - .end(function (err, res) { - if (err) throw err - - expect(res.body.data).to.be.an('array') - expect(res.body.data.length).to.equal(3) - - done() - }) - }) - - it('Should fail without a correct uuid', function (done) { - request(server.url) - .get(path + 'coucou') - .set('Accept', 'application/json') - .expect(400, done) - }) - - it('Should return 404 with an incorrect video', function (done) { - request(server.url) - .get(path + '4da6fde3-88f7-4d16-b119-108df5630b06') - .set('Accept', 'application/json') - .expect(404, done) - }) - - it('Should succeed with the correct parameters') - }) - - describe('When removing a video', function () { - it('Should have 404 with nothing', function (done) { - request(server.url) - .delete(path) - .set('Authorization', 'Bearer ' + server.accessToken) - .expect(400, done) - }) - - it('Should fail without a correct uuid', function (done) { - request(server.url) - .delete(path + 'hello') - .set('Authorization', 'Bearer ' + server.accessToken) - .expect(400, done) - }) - - it('Should fail with a video which does not exist', function (done) { - request(server.url) - .delete(path + '4da6fde3-88f7-4d16-b119-108df5630b06') - .set('Authorization', 'Bearer ' + server.accessToken) - .expect(404, done) - }) - - it('Should fail with a video of another user') - - it('Should fail with a video of another pod') - - it('Should succeed with the correct parameters') - }) - }) - - describe('Of the users API', function () { - const path = '/api/v1/users/' - let userId = null - let rootId = null - - describe('When listing users', function () { - it('Should fail with a bad start pagination', function (done) { - request(server.url) - .get(path) - .query({ start: 'hello' }) - .set('Accept', 'application/json') - .expect(400, done) - }) - - it('Should fail with a bad count pagination', function (done) { - request(server.url) - .get(path) - .query({ count: 'hello' }) - .set('Accept', 'application/json') - .expect(400, done) - }) - - it('Should fail with an incorrect sort', function (done) { - request(server.url) - .get(path) - .query({ sort: 'hello' }) - .set('Accept', 'application/json') - .expect(400, done) - }) - }) - - describe('When adding a new user', function () { - it('Should fail with a too small username', function (done) { - const data = { - username: 'ji', - password: 'mysuperpassword' - } - - requestsUtils.makePostBodyRequest(server.url, path, server.accessToken, data, done) - }) - - it('Should fail with a too long username', function (done) { - const data = { - username: 'mysuperusernamewhichisverylong', - password: 'mysuperpassword' - } - - requestsUtils.makePostBodyRequest(server.url, path, server.accessToken, data, done) - }) - - it('Should fail with an incorrect username', function (done) { - const data = { - username: 'my username', - password: 'mysuperpassword' - } - - requestsUtils.makePostBodyRequest(server.url, path, server.accessToken, data, done) - }) - - it('Should fail with a too small password', function (done) { - const data = { - username: 'myusername', - password: 'bla' - } - - requestsUtils.makePostBodyRequest(server.url, path, server.accessToken, data, done) - }) - - it('Should fail with a too long password', function (done) { - const data = { - username: 'myusername', - password: 'my super long password which is very very very very very very very very very very very very very very' + - 'very very very very very very very very very very very very very very very veryv very very very very' + - 'very very very very very very very very very very very very very very very very very very very very long' - } - - requestsUtils.makePostBodyRequest(server.url, path, server.accessToken, data, done) - }) - - it('Should fail with an non authenticated user', function (done) { - const data = { - username: 'myusername', - password: 'my super password' - } - - requestsUtils.makePostBodyRequest(server.url, path, 'super token', data, done, 401) - }) - - it('Should fail if we add a user with the same username', function (done) { - const data = { - username: 'user1', - password: 'my super password' - } - - requestsUtils.makePostBodyRequest(server.url, path, server.accessToken, data, done, 409) - }) - - it('Should succeed with the correct params', function (done) { - const data = { - username: 'user2', - password: 'my super password' - } - - requestsUtils.makePostBodyRequest(server.url, path, server.accessToken, data, done, 204) - }) - - it('Should fail with a non admin user', function (done) { - server.user = { - username: 'user1', - password: 'password' - } - - loginUtils.loginAndGetAccessToken(server, function (err, accessToken) { - if (err) throw err - - userAccessToken = accessToken - - const data = { - username: 'user3', - password: 'my super password' - } - - requestsUtils.makePostBodyRequest(server.url, path, userAccessToken, data, done, 403) - }) - }) - }) - - describe('When updating a user', function () { - before(function (done) { - usersUtils.getUsersList(server.url, function (err, res) { - if (err) throw err - - userId = res.body.data[1].id - rootId = res.body.data[2].id - done() - }) - }) - - it('Should fail with a too small password', function (done) { - const data = { - password: 'bla' - } - - requestsUtils.makePutBodyRequest(server.url, path + userId, userAccessToken, data, done) - }) - - it('Should fail with a too long password', function (done) { - const data = { - password: 'my super long password which is very very very very very very very very very very very very very very' + - 'very very very very very very very very very very very very very very very veryv very very very very' + - 'very very very very very very very very very very very very very very very very very very very very long' - } - - requestsUtils.makePutBodyRequest(server.url, path + userId, userAccessToken, data, done) - }) - - it('Should fail with an non authenticated user', function (done) { - const data = { - password: 'my super password' - } - - requestsUtils.makePutBodyRequest(server.url, path + userId, 'super token', data, done, 401) - }) - - it('Should succeed with the correct params', function (done) { - const data = { - password: 'my super password' - } - - requestsUtils.makePutBodyRequest(server.url, path + userId, userAccessToken, data, done, 204) - }) - }) - - describe('When getting my information', function () { - it('Should fail with a non authenticated user', function (done) { - request(server.url) - .get(path + 'me') - .set('Authorization', 'Bearer faketoken') - .set('Accept', 'application/json') - .expect(401, done) - }) - - it('Should success with the correct parameters', function (done) { - request(server.url) - .get(path + 'me') - .set('Authorization', 'Bearer ' + userAccessToken) - .set('Accept', 'application/json') - .expect(200, done) - }) - }) - - describe('When removing an user', function () { - it('Should fail with an incorrect id', function (done) { - request(server.url) - .delete(path + 'bla-bla') - .set('Authorization', 'Bearer ' + server.accessToken) - .expect(400, done) - }) - - it('Should fail with the root user', function (done) { - request(server.url) - .delete(path + rootId) - .set('Authorization', 'Bearer ' + server.accessToken) - .expect(400, done) - }) - - it('Should return 404 with a non existing id', function (done) { - request(server.url) - .delete(path + '45') - .set('Authorization', 'Bearer ' + server.accessToken) - .expect(404, done) - }) - }) - }) - - describe('Of the remote videos API', function () { - describe('When making a secure request', function () { - it('Should check a secure request') - }) - - describe('When adding a video', function () { - it('Should check when adding a video') - }) - - describe('When removing a video', function () { - it('Should check when removing a video') - }) - }) - - describe('Of the requests API', function () { - const path = '/api/v1/requests/stats' - - it('Should fail with an non authenticated user', function (done) { - request(server.url) - .get(path) - .set('Accept', 'application/json') - .expect(401, done) - }) - - it('Should fail with a non admin user', function (done) { - request(server.url) - .get(path) - .set('Authorization', 'Bearer ' + userAccessToken) - .set('Accept', 'application/json') - .expect(403, done) - }) - }) - - after(function (done) { - process.kill(-server.app.pid) - - // Keep the logs if the test failed - if (this.ok) { - serversUtils.flushTests(done) - } else { - done() - } - }) -}) diff --git a/server/tests/api/check-params/index.js b/server/tests/api/check-params/index.js new file mode 100644 index 000000000..3d6f09267 --- /dev/null +++ b/server/tests/api/check-params/index.js @@ -0,0 +1,8 @@ +'use strict' + +// Order of the tests we want to execute +require('./pods') +require('./remotes') +require('./users') +require('./requests') +require('./videos') diff --git a/server/tests/api/check-params/pods.js b/server/tests/api/check-params/pods.js new file mode 100644 index 000000000..2f85af644 --- /dev/null +++ b/server/tests/api/check-params/pods.js @@ -0,0 +1,204 @@ +'use strict' + +const request = require('supertest') +const series = require('async/series') + +const loginUtils = require('../../utils/login') +const requestsUtils = require('../../utils/requests') +const serversUtils = require('../../utils/servers') +const usersUtils = require('../../utils/users') + +describe('Test pods API validators', function () { + const path = '/api/v1/pods/' + let server = null + + // --------------------------------------------------------------- + + before(function (done) { + this.timeout(20000) + + series([ + function (next) { + serversUtils.flushTests(next) + }, + function (next) { + serversUtils.runServer(1, function (server1) { + server = server1 + + next() + }) + }, + function (next) { + loginUtils.loginAndGetAccessToken(server, function (err, token) { + if (err) throw err + server.accessToken = token + + next() + }) + } + ], done) + }) + + describe('When making friends', function () { + let userAccessToken = null + + before(function (done) { + usersUtils.createUser(server.url, server.accessToken, 'user1', 'password', function () { + server.user = { + username: 'user1', + password: 'password' + } + + loginUtils.loginAndGetAccessToken(server, function (err, accessToken) { + if (err) throw err + + userAccessToken = accessToken + + done() + }) + }) + }) + + describe('When making friends', function () { + const body = { + hosts: [ 'localhost:9002' ] + } + + it('Should fail without hosts', function (done) { + request(server.url) + .post(path + '/makefriends') + .set('Authorization', 'Bearer ' + server.accessToken) + .set('Accept', 'application/json') + .expect(400, done) + }) + + it('Should fail if hosts is not an array', function (done) { + request(server.url) + .post(path + '/makefriends') + .send({ hosts: 'localhost:9002' }) + .set('Authorization', 'Bearer ' + server.accessToken) + .set('Accept', 'application/json') + .expect(400, done) + }) + + it('Should fail if the array is not composed by hosts', function (done) { + request(server.url) + .post(path + '/makefriends') + .send({ hosts: [ 'localhost:9002', 'localhost:coucou' ] }) + .set('Authorization', 'Bearer ' + server.accessToken) + .set('Accept', 'application/json') + .expect(400, done) + }) + + it('Should fail if the array is composed with http schemes', function (done) { + request(server.url) + .post(path + '/makefriends') + .send({ hosts: [ 'localhost:9002', 'http://localhost:9003' ] }) + .set('Authorization', 'Bearer ' + server.accessToken) + .set('Accept', 'application/json') + .expect(400, done) + }) + + it('Should fail if hosts are not unique', function (done) { + request(server.url) + .post(path + '/makefriends') + .send({ urls: [ 'localhost:9002', 'localhost:9002' ] }) + .set('Authorization', 'Bearer ' + server.accessToken) + .set('Accept', 'application/json') + .expect(400, done) + }) + + it('Should fail with a invalid token', function (done) { + request(server.url) + .post(path + '/makefriends') + .send(body) + .set('Authorization', 'Bearer faketoken') + .set('Accept', 'application/json') + .expect(401, done) + }) + + it('Should fail if the user is not an administrator', function (done) { + request(server.url) + .post(path + '/makefriends') + .send(body) + .set('Authorization', 'Bearer ' + userAccessToken) + .set('Accept', 'application/json') + .expect(403, done) + }) + }) + + describe('When quitting friends', function () { + it('Should fail with a invalid token', function (done) { + request(server.url) + .get(path + '/quitfriends') + .query({ start: 'hello' }) + .set('Authorization', 'Bearer faketoken') + .set('Accept', 'application/json') + .expect(401, done) + }) + + it('Should fail if the user is not an administrator', function (done) { + request(server.url) + .get(path + '/quitfriends') + .query({ start: 'hello' }) + .set('Authorization', 'Bearer ' + userAccessToken) + .set('Accept', 'application/json') + .expect(403, done) + }) + }) + }) + + describe('When adding a pod', function () { + it('Should fail with nothing', function (done) { + const data = {} + requestsUtils.makePostBodyRequest(server.url, path, null, data, done) + }) + + it('Should fail without public key', function (done) { + const data = { + host: 'coucou.com' + } + requestsUtils.makePostBodyRequest(server.url, path, null, data, done) + }) + + it('Should fail without an host', function (done) { + const data = { + publicKey: 'mysuperpublickey' + } + requestsUtils.makePostBodyRequest(server.url, path, null, data, done) + }) + + it('Should fail with an incorrect host', function (done) { + const data = { + host: 'http://coucou.com', + publicKey: 'mysuperpublickey' + } + requestsUtils.makePostBodyRequest(server.url, path, null, data, function () { + data.host = 'http://coucou' + requestsUtils.makePostBodyRequest(server.url, path, null, data, function () { + data.host = 'coucou' + requestsUtils.makePostBodyRequest(server.url, path, null, data, done) + }) + }) + }) + + it('Should succeed with the correct parameters', function (done) { + const data = { + host: 'coucou.com', + publicKey: 'mysuperpublickey' + } + requestsUtils.makePostBodyRequest(server.url, path, null, data, done, 200) + }) + }) + + after(function (done) { + process.kill(-server.app.pid) + + // Keep the logs if the test failed + if (this.ok) { + serversUtils.flushTests(done) + } else { + done() + } + }) +}) diff --git a/server/tests/api/check-params/remotes.js b/server/tests/api/check-params/remotes.js new file mode 100644 index 000000000..30ba3b697 --- /dev/null +++ b/server/tests/api/check-params/remotes.js @@ -0,0 +1,60 @@ +'use strict' + +const series = require('async/series') + +const loginUtils = require('../../utils/login') +const serversUtils = require('../../utils/servers') + +describe('Test remote videos API validators', function () { + let server = null + + // --------------------------------------------------------------- + + before(function (done) { + this.timeout(20000) + + series([ + function (next) { + serversUtils.flushTests(next) + }, + function (next) { + serversUtils.runServer(1, function (server1) { + server = server1 + + next() + }) + }, + function (next) { + loginUtils.loginAndGetAccessToken(server, function (err, token) { + if (err) throw err + server.accessToken = token + + next() + }) + } + ], done) + }) + + describe('When making a secure request', function () { + it('Should check a secure request') + }) + + describe('When adding a video', function () { + it('Should check when adding a video') + }) + + describe('When removing a video', function () { + it('Should check when removing a video') + }) + + after(function (done) { + process.kill(-server.app.pid) + + // Keep the logs if the test failed + if (this.ok) { + serversUtils.flushTests(done) + } else { + done() + } + }) +}) diff --git a/server/tests/api/check-params/requests.js b/server/tests/api/check-params/requests.js new file mode 100644 index 000000000..08f58db43 --- /dev/null +++ b/server/tests/api/check-params/requests.js @@ -0,0 +1,87 @@ +'use strict' + +const request = require('supertest') +const series = require('async/series') + +const loginUtils = require('../../utils/login') +const usersUtils = require('../../utils/users') +const serversUtils = require('../../utils/servers') + +describe('Test requests API validators', function () { + const path = '/api/v1/requests/stats' + let server = null + let userAccessToken = null + + // --------------------------------------------------------------- + + before(function (done) { + this.timeout(20000) + + series([ + function (next) { + serversUtils.flushTests(next) + }, + function (next) { + serversUtils.runServer(1, function (server1) { + server = server1 + + next() + }) + }, + function (next) { + loginUtils.loginAndGetAccessToken(server, function (err, token) { + if (err) throw err + server.accessToken = token + + next() + }) + }, + function (next) { + const username = 'user' + const password = 'my super password' + + usersUtils.createUser(server.url, server.accessToken, username, password, next) + }, + function (next) { + const user = { + username: 'user', + password: 'my super password' + } + + loginUtils.getUserAccessToken(server, user, function (err, accessToken) { + if (err) throw err + + userAccessToken = accessToken + + next() + }) + } + ], done) + }) + + it('Should fail with an non authenticated user', function (done) { + request(server.url) + .get(path) + .set('Accept', 'application/json') + .expect(401, done) + }) + + it('Should fail with a non admin user', function (done) { + request(server.url) + .get(path) + .set('Authorization', 'Bearer ' + userAccessToken) + .set('Accept', 'application/json') + .expect(403, done) + }) + + after(function (done) { + process.kill(-server.app.pid) + + // Keep the logs if the test failed + if (this.ok) { + serversUtils.flushTests(done) + } else { + done() + } + }) +}) diff --git a/server/tests/api/check-params/users.js b/server/tests/api/check-params/users.js new file mode 100644 index 000000000..c1fcf34a4 --- /dev/null +++ b/server/tests/api/check-params/users.js @@ -0,0 +1,284 @@ +'use strict' + +const request = require('supertest') +const series = require('async/series') + +const loginUtils = require('../../utils/login') +const requestsUtils = require('../../utils/requests') +const serversUtils = require('../../utils/servers') +const usersUtils = require('../../utils/users') + +describe('Test users API validators', function () { + const path = '/api/v1/users/' + let userId = null + let rootId = null + let server = null + let userAccessToken = null + + // --------------------------------------------------------------- + + before(function (done) { + this.timeout(20000) + + series([ + function (next) { + serversUtils.flushTests(next) + }, + function (next) { + serversUtils.runServer(1, function (server1) { + server = server1 + + next() + }) + }, + function (next) { + loginUtils.loginAndGetAccessToken(server, function (err, token) { + if (err) throw err + server.accessToken = token + + next() + }) + }, + function (next) { + const username = 'user1' + const password = 'my super password' + + usersUtils.createUser(server.url, server.accessToken, username, password, next) + }, + function (next) { + const user = { + username: 'user1', + password: 'my super password' + } + + loginUtils.getUserAccessToken(server, user, function (err, accessToken) { + if (err) throw err + + userAccessToken = accessToken + + next() + }) + } + ], done) + }) + + describe('When listing users', function () { + it('Should fail with a bad start pagination', function (done) { + request(server.url) + .get(path) + .query({ start: 'hello' }) + .set('Accept', 'application/json') + .expect(400, done) + }) + + it('Should fail with a bad count pagination', function (done) { + request(server.url) + .get(path) + .query({ count: 'hello' }) + .set('Accept', 'application/json') + .expect(400, done) + }) + + it('Should fail with an incorrect sort', function (done) { + request(server.url) + .get(path) + .query({ sort: 'hello' }) + .set('Accept', 'application/json') + .expect(400, done) + }) + }) + + describe('When adding a new user', function () { + it('Should fail with a too small username', function (done) { + const data = { + username: 'ji', + password: 'mysuperpassword' + } + + requestsUtils.makePostBodyRequest(server.url, path, server.accessToken, data, done) + }) + + it('Should fail with a too long username', function (done) { + const data = { + username: 'mysuperusernamewhichisverylong', + password: 'mysuperpassword' + } + + requestsUtils.makePostBodyRequest(server.url, path, server.accessToken, data, done) + }) + + it('Should fail with an incorrect username', function (done) { + const data = { + username: 'my username', + password: 'mysuperpassword' + } + + requestsUtils.makePostBodyRequest(server.url, path, server.accessToken, data, done) + }) + + it('Should fail with a too small password', function (done) { + const data = { + username: 'myusername', + password: 'bla' + } + + requestsUtils.makePostBodyRequest(server.url, path, server.accessToken, data, done) + }) + + it('Should fail with a too long password', function (done) { + const data = { + username: 'myusername', + password: 'my super long password which is very very very very very very very very very very very very very very' + + 'very very very very very very very very very very very very very very very veryv very very very very' + + 'very very very very very very very very very very very very very very very very very very very very long' + } + + requestsUtils.makePostBodyRequest(server.url, path, server.accessToken, data, done) + }) + + it('Should fail with an non authenticated user', function (done) { + const data = { + username: 'myusername', + password: 'my super password' + } + + requestsUtils.makePostBodyRequest(server.url, path, 'super token', data, done, 401) + }) + + it('Should fail if we add a user with the same username', function (done) { + const data = { + username: 'user1', + password: 'my super password' + } + + requestsUtils.makePostBodyRequest(server.url, path, server.accessToken, data, done, 409) + }) + + it('Should succeed with the correct params', function (done) { + const data = { + username: 'user2', + password: 'my super password' + } + + requestsUtils.makePostBodyRequest(server.url, path, server.accessToken, data, done, 204) + }) + + it('Should fail with a non admin user', function (done) { + server.user = { + username: 'user1', + password: 'my super password' + } + + loginUtils.loginAndGetAccessToken(server, function (err, accessToken) { + if (err) throw err + + userAccessToken = accessToken + + const data = { + username: 'user3', + password: 'my super password' + } + + requestsUtils.makePostBodyRequest(server.url, path, userAccessToken, data, done, 403) + }) + }) + }) + + describe('When updating a user', function () { + before(function (done) { + usersUtils.getUsersList(server.url, function (err, res) { + if (err) throw err + + userId = res.body.data[1].id + rootId = res.body.data[2].id + done() + }) + }) + + it('Should fail with a too small password', function (done) { + const data = { + password: 'bla' + } + + requestsUtils.makePutBodyRequest(server.url, path + userId, userAccessToken, data, done) + }) + + it('Should fail with a too long password', function (done) { + const data = { + password: 'my super long password which is very very very very very very very very very very very very very very' + + 'very very very very very very very very very very very very very very very veryv very very very very' + + 'very very very very very very very very very very very very very very very very very very very very long' + } + + requestsUtils.makePutBodyRequest(server.url, path + userId, userAccessToken, data, done) + }) + + it('Should fail with an non authenticated user', function (done) { + const data = { + password: 'my super password' + } + + requestsUtils.makePutBodyRequest(server.url, path + userId, 'super token', data, done, 401) + }) + + it('Should succeed with the correct params', function (done) { + const data = { + password: 'my super password' + } + + requestsUtils.makePutBodyRequest(server.url, path + userId, userAccessToken, data, done, 204) + }) + }) + + describe('When getting my information', function () { + it('Should fail with a non authenticated user', function (done) { + request(server.url) + .get(path + 'me') + .set('Authorization', 'Bearer faketoken') + .set('Accept', 'application/json') + .expect(401, done) + }) + + it('Should success with the correct parameters', function (done) { + request(server.url) + .get(path + 'me') + .set('Authorization', 'Bearer ' + userAccessToken) + .set('Accept', 'application/json') + .expect(200, done) + }) + }) + + describe('When removing an user', function () { + it('Should fail with an incorrect id', function (done) { + request(server.url) + .delete(path + 'bla-bla') + .set('Authorization', 'Bearer ' + server.accessToken) + .expect(400, done) + }) + + it('Should fail with the root user', function (done) { + request(server.url) + .delete(path + rootId) + .set('Authorization', 'Bearer ' + server.accessToken) + .expect(400, done) + }) + + it('Should return 404 with a non existing id', function (done) { + request(server.url) + .delete(path + '45') + .set('Authorization', 'Bearer ' + server.accessToken) + .expect(404, done) + }) + }) + + after(function (done) { + process.kill(-server.app.pid) + + // Keep the logs if the test failed + if (this.ok) { + serversUtils.flushTests(done) + } else { + done() + } + }) +}) diff --git a/server/tests/api/check-params/videos.js b/server/tests/api/check-params/videos.js new file mode 100644 index 000000000..d18305291 --- /dev/null +++ b/server/tests/api/check-params/videos.js @@ -0,0 +1,456 @@ +'use strict' + +const chai = require('chai') +const expect = chai.expect +const pathUtils = require('path') +const request = require('supertest') +const series = require('async/series') + +const loginUtils = require('../../utils/login') +const requestsUtils = require('../../utils/requests') +const serversUtils = require('../../utils/servers') +const videosUtils = require('../../utils/videos') + +describe('Test videos API validator', function () { + const path = '/api/v1/videos/' + let server = null + + // --------------------------------------------------------------- + + before(function (done) { + this.timeout(20000) + + series([ + function (next) { + serversUtils.flushTests(next) + }, + function (next) { + serversUtils.runServer(1, function (server1) { + server = server1 + + next() + }) + }, + function (next) { + loginUtils.loginAndGetAccessToken(server, function (err, token) { + if (err) throw err + server.accessToken = token + + next() + }) + } + ], done) + }) + + describe('When listing a video', function () { + it('Should fail with a bad start pagination', function (done) { + request(server.url) + .get(path) + .query({ start: 'hello' }) + .set('Accept', 'application/json') + .expect(400, done) + }) + + it('Should fail with a bad count pagination', function (done) { + request(server.url) + .get(path) + .query({ count: 'hello' }) + .set('Accept', 'application/json') + .expect(400, done) + }) + + it('Should fail with an incorrect sort', function (done) { + request(server.url) + .get(path) + .query({ sort: 'hello' }) + .set('Accept', 'application/json') + .expect(400, done) + }) + }) + + describe('When searching a video', function () { + it('Should fail with nothing', function (done) { + request(server.url) + .get(pathUtils.join(path, 'search')) + .set('Accept', 'application/json') + .expect(400, done) + }) + + it('Should fail with a bad start pagination', function (done) { + request(server.url) + .get(pathUtils.join(path, 'search', 'test')) + .query({ start: 'hello' }) + .set('Accept', 'application/json') + .expect(400, done) + }) + + it('Should fail with a bad count pagination', function (done) { + request(server.url) + .get(pathUtils.join(path, 'search', 'test')) + .query({ count: 'hello' }) + .set('Accept', 'application/json') + .expect(400, done) + }) + + it('Should fail with an incorrect sort', function (done) { + request(server.url) + .get(pathUtils.join(path, 'search', 'test')) + .query({ sort: 'hello' }) + .set('Accept', 'application/json') + .expect(400, done) + }) + }) + + describe('When adding a video', function () { + it('Should fail with nothing', function (done) { + const data = {} + const attach = {} + requestsUtils.makePostUploadRequest(server.url, path, server.accessToken, data, attach, done) + }) + + it('Should fail without name', function (done) { + const data = { + description: 'my super description', + tags: [ 'tag1', 'tag2' ] + } + const attach = { + 'videofile': pathUtils.join(__dirname, '..', 'fixtures', 'video_short.webm') + } + requestsUtils.makePostUploadRequest(server.url, path, server.accessToken, data, attach, done) + }) + + it('Should fail with a long name', function (done) { + const data = { + name: 'My very very very very very very very very very very very very very very very very long name', + description: 'my super description', + tags: [ 'tag1', 'tag2' ] + } + const attach = { + 'videofile': pathUtils.join(__dirname, '..', 'fixtures', 'video_short.webm') + } + requestsUtils.makePostUploadRequest(server.url, path, server.accessToken, data, attach, done) + }) + + it('Should fail without description', function (done) { + const data = { + name: 'my super name', + tags: [ 'tag1', 'tag2' ] + } + const attach = { + 'videofile': pathUtils.join(__dirname, '..', 'fixtures', 'video_short.webm') + } + requestsUtils.makePostUploadRequest(server.url, path, server.accessToken, data, attach, done) + }) + + it('Should fail with a long description', function (done) { + const data = { + name: 'my super name', + description: 'my super description which is very very very very very very very very very very very very very very' + + 'very very very very very very very very very very very very very very very very very very very very very' + + 'very very very very very very very very very very very very very very very long', + tags: [ 'tag1', 'tag2' ] + } + const attach = { + 'videofile': pathUtils.join(__dirname, '..', 'fixtures', 'video_short.webm') + } + requestsUtils.makePostUploadRequest(server.url, path, server.accessToken, data, attach, done) + }) + + it('Should fail without tags', function (done) { + const data = { + name: 'my super name', + description: 'my super description' + } + const attach = { + 'videofile': pathUtils.join(__dirname, '..', 'fixtures', 'video_short.webm') + } + requestsUtils.makePostUploadRequest(server.url, path, server.accessToken, data, attach, done) + }) + + it('Should fail with too many tags', function (done) { + const data = { + name: 'my super name', + description: 'my super description', + tags: [ 'tag1', 'tag2', 'tag3', 'tag4' ] + } + const attach = { + 'videofile': pathUtils.join(__dirname, '..', 'fixtures', 'video_short.webm') + } + requestsUtils.makePostUploadRequest(server.url, path, server.accessToken, data, attach, done) + }) + + it('Should fail with not enough tags', function (done) { + const data = { + name: 'my super name', + description: 'my super description', + tags: [ ] + } + const attach = { + 'videofile': pathUtils.join(__dirname, '..', 'fixtures', 'video_short.webm') + } + requestsUtils.makePostUploadRequest(server.url, path, server.accessToken, data, attach, done) + }) + + it('Should fail with a tag length too low', function (done) { + const data = { + name: 'my super name', + description: 'my super description', + tags: [ 'tag1', 't' ] + } + const attach = { + 'videofile': pathUtils.join(__dirname, '..', 'fixtures', 'video_short.webm') + } + requestsUtils.makePostUploadRequest(server.url, path, server.accessToken, data, attach, done) + }) + + it('Should fail with a tag length too big', function (done) { + const data = { + name: 'my super name', + description: 'my super description', + tags: [ 'mysupertagtoolong', 'tag1' ] + } + const attach = { + 'videofile': pathUtils.join(__dirname, '..', 'fixtures', 'video_short.webm') + } + requestsUtils.makePostUploadRequest(server.url, path, server.accessToken, data, attach, done) + }) + + it('Should fail with malformed tags', function (done) { + const data = { + name: 'my super name', + description: 'my super description', + tags: [ 'my tag' ] + } + const attach = { + 'videofile': pathUtils.join(__dirname, '..', 'fixtures', 'video_short.webm') + } + requestsUtils.makePostUploadRequest(server.url, path, server.accessToken, data, attach, done) + }) + + it('Should fail without an input file', function (done) { + const data = { + name: 'my super name', + description: 'my super description', + tags: [ 'tag1', 'tag2' ] + } + const attach = {} + requestsUtils.makePostUploadRequest(server.url, path, server.accessToken, data, attach, done) + }) + + it('Should fail without an incorrect input file', function (done) { + const data = { + name: 'my super name', + description: 'my super description', + tags: [ 'tag1', 'tag2' ] + } + const attach = { + 'videofile': pathUtils.join(__dirname, '..', 'fixtures', 'video_short_fake.webm') + } + requestsUtils.makePostUploadRequest(server.url, path, server.accessToken, data, attach, done) + }) + + it('Should fail with a too big duration', function (done) { + const data = { + name: 'my super name', + description: 'my super description', + tags: [ 'tag1', 'tag2' ] + } + const attach = { + 'videofile': pathUtils.join(__dirname, '..', 'fixtures', 'video_too_long.webm') + } + requestsUtils.makePostUploadRequest(server.url, path, server.accessToken, data, attach, done) + }) + + it('Should succeed with the correct parameters', function (done) { + const data = { + name: 'my super name', + description: 'my super description', + tags: [ 'tag1', 'tag2' ] + } + const attach = { + 'videofile': pathUtils.join(__dirname, '..', 'fixtures', 'video_short.webm') + } + requestsUtils.makePostUploadRequest(server.url, path, server.accessToken, data, attach, function () { + attach.videofile = pathUtils.join(__dirname, '..', 'fixtures', 'video_short.mp4') + requestsUtils.makePostUploadRequest(server.url, path, server.accessToken, data, attach, function () { + attach.videofile = pathUtils.join(__dirname, '..', 'fixtures', 'video_short.ogv') + requestsUtils.makePostUploadRequest(server.url, path, server.accessToken, data, attach, done, 204) + }, false) + }, false) + }) + }) + + describe('When updating a video', function () { + let videoId + + before(function (done) { + videosUtils.getVideosList(server.url, function (err, res) { + if (err) throw err + + videoId = res.body.data[0].id + + return done() + }) + }) + + it('Should fail with nothing', function (done) { + const data = {} + requestsUtils.makePutBodyRequest(server.url, path, server.accessToken, data, done) + }) + + it('Should fail without a valid uuid', function (done) { + const data = { + description: 'my super description', + tags: [ 'tag1', 'tag2' ] + } + requestsUtils.makePutBodyRequest(server.url, path + 'blabla', server.accessToken, data, done) + }) + + it('Should fail with an unknown id', function (done) { + const data = { + description: 'my super description', + tags: [ 'tag1', 'tag2' ] + } + requestsUtils.makePutBodyRequest(server.url, path + '4da6fde3-88f7-4d16-b119-108df5630b06', server.accessToken, data, done, 404) + }) + + it('Should fail with a long name', function (done) { + const data = { + name: 'My very very very very very very very very very very very very very very very very long name', + description: 'my super description', + tags: [ 'tag1', 'tag2' ] + } + requestsUtils.makePutBodyRequest(server.url, path + videoId, server.accessToken, data, done) + }) + + it('Should fail with a long description', function (done) { + const data = { + name: 'my super name', + description: 'my super description which is very very very very very very very very very very very very very very' + + 'very very very very very very very very very very very very very very very very very very very very very' + + 'very very very very very very very very very very very very very very very long', + tags: [ 'tag1', 'tag2' ] + } + requestsUtils.makePutBodyRequest(server.url, path + videoId, server.accessToken, data, done) + }) + + it('Should fail with too many tags', function (done) { + const data = { + name: 'my super name', + description: 'my super description', + tags: [ 'tag1', 'tag2', 'tag3', 'tag4' ] + } + requestsUtils.makePutBodyRequest(server.url, path + videoId, server.accessToken, data, done) + }) + + it('Should fail with not enough tags', function (done) { + const data = { + name: 'my super name', + description: 'my super description', + tags: [ ] + } + requestsUtils.makePutBodyRequest(server.url, path + videoId, server.accessToken, data, done) + }) + + it('Should fail with a tag length too low', function (done) { + const data = { + name: 'my super name', + description: 'my super description', + tags: [ 'tag1', 't' ] + } + requestsUtils.makePutBodyRequest(server.url, path + videoId, server.accessToken, data, done) + }) + + it('Should fail with a tag length too big', function (done) { + const data = { + name: 'my super name', + description: 'my super description', + tags: [ 'mysupertagtoolong', 'tag1' ] + } + requestsUtils.makePutBodyRequest(server.url, path + videoId, server.accessToken, data, done) + }) + + it('Should fail with malformed tags', function (done) { + const data = { + name: 'my super name', + description: 'my super description', + tags: [ 'my tag' ] + } + requestsUtils.makePutBodyRequest(server.url, path + videoId, server.accessToken, data, done) + }) + }) + + describe('When getting a video', function () { + it('Should return the list of the videos with nothing', function (done) { + request(server.url) + .get(path) + .set('Accept', 'application/json') + .expect(200) + .expect('Content-Type', /json/) + .end(function (err, res) { + if (err) throw err + + expect(res.body.data).to.be.an('array') + expect(res.body.data.length).to.equal(3) + + done() + }) + }) + + it('Should fail without a correct uuid', function (done) { + request(server.url) + .get(path + 'coucou') + .set('Accept', 'application/json') + .expect(400, done) + }) + + it('Should return 404 with an incorrect video', function (done) { + request(server.url) + .get(path + '4da6fde3-88f7-4d16-b119-108df5630b06') + .set('Accept', 'application/json') + .expect(404, done) + }) + + it('Should succeed with the correct parameters') + }) + + describe('When removing a video', function () { + it('Should have 404 with nothing', function (done) { + request(server.url) + .delete(path) + .set('Authorization', 'Bearer ' + server.accessToken) + .expect(400, done) + }) + + it('Should fail without a correct uuid', function (done) { + request(server.url) + .delete(path + 'hello') + .set('Authorization', 'Bearer ' + server.accessToken) + .expect(400, done) + }) + + it('Should fail with a video which does not exist', function (done) { + request(server.url) + .delete(path + '4da6fde3-88f7-4d16-b119-108df5630b06') + .set('Authorization', 'Bearer ' + server.accessToken) + .expect(404, done) + }) + + it('Should fail with a video of another user') + + it('Should fail with a video of another pod') + + it('Should succeed with the correct parameters') + }) + + after(function (done) { + process.kill(-server.app.pid) + + // Keep the logs if the test failed + if (this.ok) { + serversUtils.flushTests(done) + } else { + done() + } + }) +}) diff --git a/server/tests/utils/login.js b/server/tests/utils/login.js index 465564e14..c984c0baf 100644 --- a/server/tests/utils/login.js +++ b/server/tests/utils/login.js @@ -4,7 +4,8 @@ const request = require('supertest') const loginUtils = { login, - loginAndGetAccessToken + loginAndGetAccessToken, + getUserAccessToken } // ---------------------- Export functions -------------------- @@ -43,6 +44,14 @@ function loginAndGetAccessToken (server, callback) { }) } +function getUserAccessToken (server, user, callback) { + login(server.url, server.client, user, 200, function (err, res) { + if (err) return callback(err) + + return callback(null, res.body.access_token) + }) +} + // --------------------------------------------------------------------------- module.exports = loginUtils diff --git a/server/tests/utils/servers.js b/server/tests/utils/servers.js index e7c756499..1946ef49a 100644 --- a/server/tests/utils/servers.js +++ b/server/tests/utils/servers.js @@ -34,7 +34,7 @@ function flushAndRunMultipleServers (totalServers, serversRun) { runServer(j, function (app, url) { anotherServerDone(j, app, url) }) - }, 1000 * j) + }, 1000 * (j - 1)) } }) } -- cgit v1.2.3 From 8fd66b75bfbd8fd4945f1944411461b05eb74795 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Fri, 30 Dec 2016 12:39:49 +0100 Subject: Server: fix video remoe validation --- server/middlewares/validators/videos.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'server') diff --git a/server/middlewares/validators/videos.js b/server/middlewares/validators/videos.js index 1b6dbccf0..295ed05fa 100644 --- a/server/middlewares/validators/videos.js +++ b/server/middlewares/validators/videos.js @@ -78,7 +78,7 @@ function videosRemove (req, res, next) { return res.status(403).send('Cannot remove video of another pod') } - if (res.locals.video.authorId !== res.locals.oauth.token.User.id) { + if (res.locals.video.Author.userId !== res.locals.oauth.token.User.id) { return res.status(403).send('Cannot remove video of another user') } -- cgit v1.2.3 From a6fd2b30bf717eec14972a2175354781f5f43e77 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Fri, 30 Dec 2016 12:53:41 +0100 Subject: Server: move remote routes in their own directory --- server/controllers/api/index.js | 8 +- server/controllers/api/remote.js | 237 -------------------------------- server/controllers/api/remote/index.js | 16 +++ server/controllers/api/remote/videos.js | 237 ++++++++++++++++++++++++++++++++ server/helpers/utils.js | 5 + 5 files changed, 261 insertions(+), 242 deletions(-) delete mode 100644 server/controllers/api/remote.js create mode 100644 server/controllers/api/remote/index.js create mode 100644 server/controllers/api/remote/videos.js (limited to 'server') diff --git a/server/controllers/api/index.js b/server/controllers/api/index.js index 4cb65ed55..f13ff922c 100644 --- a/server/controllers/api/index.js +++ b/server/controllers/api/index.js @@ -2,6 +2,8 @@ const express = require('express') +const utils = require('../../helpers/utils') + const router = express.Router() const clientsController = require('./clients') @@ -18,7 +20,7 @@ router.use('/requests', requestsController) router.use('/users', usersController) router.use('/videos', videosController) router.use('/ping', pong) -router.use('/*', badRequest) +router.use('/*', utils.badRequest) // --------------------------------------------------------------------------- @@ -29,7 +31,3 @@ module.exports = router function pong (req, res, next) { return res.send('pong').status(200).end() } - -function badRequest (req, res, next) { - res.type('json').status(400).end() -} diff --git a/server/controllers/api/remote.js b/server/controllers/api/remote.js deleted file mode 100644 index be5e6dc98..000000000 --- a/server/controllers/api/remote.js +++ /dev/null @@ -1,237 +0,0 @@ -'use strict' - -const eachSeries = require('async/eachSeries') -const express = require('express') -const waterfall = require('async/waterfall') - -const db = require('../../initializers/database') -const middlewares = require('../../middlewares') -const secureMiddleware = middlewares.secure -const validators = middlewares.validators.remote -const logger = require('../../helpers/logger') - -const router = express.Router() - -router.post('/videos', - validators.signature, - secureMiddleware.checkSignature, - validators.remoteVideos, - remoteVideos -) - -// --------------------------------------------------------------------------- - -module.exports = router - -// --------------------------------------------------------------------------- - -function remoteVideos (req, res, next) { - const requests = req.body.data - const fromPod = res.locals.secure.pod - - // We need to process in the same order to keep consistency - // TODO: optimization - eachSeries(requests, function (request, callbackEach) { - const videoData = request.data - - switch (request.type) { - case 'add': - addRemoteVideo(videoData, fromPod, callbackEach) - break - - case 'update': - updateRemoteVideo(videoData, fromPod, callbackEach) - break - - case 'remove': - removeRemoteVideo(videoData, fromPod, callbackEach) - break - - default: - logger.error('Unkown remote request type %s.', request.type) - } - }, function (err) { - if (err) logger.error('Error managing remote videos.', { error: err }) - }) - - // We don't need to keep the other pod waiting - return res.type('json').status(204).end() -} - -function addRemoteVideo (videoToCreateData, fromPod, finalCallback) { - logger.debug('Adding remote video "%s".', videoToCreateData.name) - - waterfall([ - - function startTransaction (callback) { - db.sequelize.transaction().asCallback(function (err, t) { - return callback(err, t) - }) - }, - - function findOrCreateAuthor (t, callback) { - const name = videoToCreateData.author - const podId = fromPod.id - // This author is from another pod so we do not associate a user - const userId = null - - db.Author.findOrCreateAuthor(name, podId, userId, t, function (err, authorInstance) { - return callback(err, t, authorInstance) - }) - }, - - function findOrCreateTags (t, author, callback) { - const tags = videoToCreateData.tags - - db.Tag.findOrCreateTags(tags, t, function (err, tagInstances) { - return callback(err, t, author, tagInstances) - }) - }, - - function createVideoObject (t, author, tagInstances, callback) { - const videoData = { - name: videoToCreateData.name, - remoteId: videoToCreateData.remoteId, - extname: videoToCreateData.extname, - infoHash: videoToCreateData.infoHash, - description: videoToCreateData.description, - authorId: author.id, - duration: videoToCreateData.duration, - createdAt: videoToCreateData.createdAt, - updatedAt: videoToCreateData.updatedAt - } - - const video = db.Video.build(videoData) - - return callback(null, t, tagInstances, video) - }, - - function generateThumbnail (t, tagInstances, video, callback) { - db.Video.generateThumbnailFromData(video, videoToCreateData.thumbnailData, function (err) { - if (err) { - logger.error('Cannot generate thumbnail from data.', { error: err }) - return callback(err) - } - - return callback(err, t, tagInstances, video) - }) - }, - - function insertVideoIntoDB (t, tagInstances, video, callback) { - const options = { - transaction: t - } - - video.save(options).asCallback(function (err, videoCreated) { - return callback(err, t, tagInstances, videoCreated) - }) - }, - - function associateTagsToVideo (t, tagInstances, video, callback) { - const options = { transaction: t } - - video.setTags(tagInstances, options).asCallback(function (err) { - return callback(err, t) - }) - } - - ], function (err, t) { - if (err) { - logger.error('Cannot insert the remote video.') - - // Abort transaction? - if (t) t.rollback() - - return finalCallback(err) - } - - // Commit transaction - t.commit() - - return finalCallback() - }) -} - -function updateRemoteVideo (videoAttributesToUpdate, fromPod, finalCallback) { - logger.debug('Updating remote video "%s".', videoAttributesToUpdate.name) - - waterfall([ - - function startTransaction (callback) { - db.sequelize.transaction().asCallback(function (err, t) { - return callback(err, t) - }) - }, - - function findVideo (t, callback) { - db.Video.loadByHostAndRemoteId(fromPod.host, videoAttributesToUpdate.remoteId, function (err, videoInstance) { - if (err || !videoInstance) { - logger.error('Cannot load video from host and remote id.', { error: err.message }) - return callback(err) - } - - return callback(null, t, videoInstance) - }) - }, - - function findOrCreateTags (t, videoInstance, callback) { - const tags = videoAttributesToUpdate.tags - - db.Tag.findOrCreateTags(tags, t, function (err, tagInstances) { - return callback(err, t, videoInstance, tagInstances) - }) - }, - - function updateVideoIntoDB (t, videoInstance, tagInstances, callback) { - const options = { transaction: t } - - videoInstance.set('name', videoAttributesToUpdate.name) - videoInstance.set('description', videoAttributesToUpdate.description) - videoInstance.set('infoHash', videoAttributesToUpdate.infoHash) - videoInstance.set('duration', videoAttributesToUpdate.duration) - videoInstance.set('createdAt', videoAttributesToUpdate.createdAt) - videoInstance.set('updatedAt', videoAttributesToUpdate.updatedAt) - videoInstance.set('extname', videoAttributesToUpdate.extname) - - videoInstance.save(options).asCallback(function (err) { - return callback(err, t, videoInstance, tagInstances) - }) - }, - - function associateTagsToVideo (t, videoInstance, tagInstances, callback) { - const options = { transaction: t } - - videoInstance.setTags(tagInstances, options).asCallback(function (err) { - return callback(err, t) - }) - } - - ], function (err, t) { - if (err) { - logger.error('Cannot update the remote video.') - - // Abort transaction? - if (t) t.rollback() - - return finalCallback(err) - } - - // Commit transaction - t.commit() - - return finalCallback() - }) -} - -function removeRemoteVideo (videoToRemoveData, fromPod, callback) { - // We need the instance because we have to remove some other stuffs (thumbnail etc) - db.Video.loadByHostAndRemoteId(fromPod.host, videoToRemoveData.remoteId, function (err, video) { - if (err || !video) { - logger.error('Cannot load video from host and remote id.', { error: err.message }) - return callback(err) - } - - logger.debug('Removing remote video %s.', video.remoteId) - video.destroy().asCallback(callback) - }) -} diff --git a/server/controllers/api/remote/index.js b/server/controllers/api/remote/index.js new file mode 100644 index 000000000..2947632d5 --- /dev/null +++ b/server/controllers/api/remote/index.js @@ -0,0 +1,16 @@ +'use strict' + +const express = require('express') + +const utils = require('../../../helpers/utils') + +const router = express.Router() + +const videosRemoteController = require('./videos') + +router.use('/videos', videosRemoteController) +router.use('/*', utils.badRequest) + +// --------------------------------------------------------------------------- + +module.exports = router diff --git a/server/controllers/api/remote/videos.js b/server/controllers/api/remote/videos.js new file mode 100644 index 000000000..87c49bff9 --- /dev/null +++ b/server/controllers/api/remote/videos.js @@ -0,0 +1,237 @@ +'use strict' + +const eachSeries = require('async/eachSeries') +const express = require('express') +const waterfall = require('async/waterfall') + +const db = require('../../../initializers/database') +const middlewares = require('../../../middlewares') +const secureMiddleware = middlewares.secure +const validators = middlewares.validators.remote +const logger = require('../../../helpers/logger') + +const router = express.Router() + +router.post('/', + validators.signature, + secureMiddleware.checkSignature, + validators.remoteVideos, + remoteVideos +) + +// --------------------------------------------------------------------------- + +module.exports = router + +// --------------------------------------------------------------------------- + +function remoteVideos (req, res, next) { + const requests = req.body.data + const fromPod = res.locals.secure.pod + + // We need to process in the same order to keep consistency + // TODO: optimization + eachSeries(requests, function (request, callbackEach) { + const videoData = request.data + + switch (request.type) { + case 'add': + addRemoteVideo(videoData, fromPod, callbackEach) + break + + case 'update': + updateRemoteVideo(videoData, fromPod, callbackEach) + break + + case 'remove': + removeRemoteVideo(videoData, fromPod, callbackEach) + break + + default: + logger.error('Unkown remote request type %s.', request.type) + } + }, function (err) { + if (err) logger.error('Error managing remote videos.', { error: err }) + }) + + // We don't need to keep the other pod waiting + return res.type('json').status(204).end() +} + +function addRemoteVideo (videoToCreateData, fromPod, finalCallback) { + logger.debug('Adding remote video "%s".', videoToCreateData.name) + + waterfall([ + + function startTransaction (callback) { + db.sequelize.transaction().asCallback(function (err, t) { + return callback(err, t) + }) + }, + + function findOrCreateAuthor (t, callback) { + const name = videoToCreateData.author + const podId = fromPod.id + // This author is from another pod so we do not associate a user + const userId = null + + db.Author.findOrCreateAuthor(name, podId, userId, t, function (err, authorInstance) { + return callback(err, t, authorInstance) + }) + }, + + function findOrCreateTags (t, author, callback) { + const tags = videoToCreateData.tags + + db.Tag.findOrCreateTags(tags, t, function (err, tagInstances) { + return callback(err, t, author, tagInstances) + }) + }, + + function createVideoObject (t, author, tagInstances, callback) { + const videoData = { + name: videoToCreateData.name, + remoteId: videoToCreateData.remoteId, + extname: videoToCreateData.extname, + infoHash: videoToCreateData.infoHash, + description: videoToCreateData.description, + authorId: author.id, + duration: videoToCreateData.duration, + createdAt: videoToCreateData.createdAt, + updatedAt: videoToCreateData.updatedAt + } + + const video = db.Video.build(videoData) + + return callback(null, t, tagInstances, video) + }, + + function generateThumbnail (t, tagInstances, video, callback) { + db.Video.generateThumbnailFromData(video, videoToCreateData.thumbnailData, function (err) { + if (err) { + logger.error('Cannot generate thumbnail from data.', { error: err }) + return callback(err) + } + + return callback(err, t, tagInstances, video) + }) + }, + + function insertVideoIntoDB (t, tagInstances, video, callback) { + const options = { + transaction: t + } + + video.save(options).asCallback(function (err, videoCreated) { + return callback(err, t, tagInstances, videoCreated) + }) + }, + + function associateTagsToVideo (t, tagInstances, video, callback) { + const options = { transaction: t } + + video.setTags(tagInstances, options).asCallback(function (err) { + return callback(err, t) + }) + } + + ], function (err, t) { + if (err) { + logger.error('Cannot insert the remote video.') + + // Abort transaction? + if (t) t.rollback() + + return finalCallback(err) + } + + // Commit transaction + t.commit() + + return finalCallback() + }) +} + +function updateRemoteVideo (videoAttributesToUpdate, fromPod, finalCallback) { + logger.debug('Updating remote video "%s".', videoAttributesToUpdate.name) + + waterfall([ + + function startTransaction (callback) { + db.sequelize.transaction().asCallback(function (err, t) { + return callback(err, t) + }) + }, + + function findVideo (t, callback) { + db.Video.loadByHostAndRemoteId(fromPod.host, videoAttributesToUpdate.remoteId, function (err, videoInstance) { + if (err || !videoInstance) { + logger.error('Cannot load video from host and remote id.', { error: err.message }) + return callback(err) + } + + return callback(null, t, videoInstance) + }) + }, + + function findOrCreateTags (t, videoInstance, callback) { + const tags = videoAttributesToUpdate.tags + + db.Tag.findOrCreateTags(tags, t, function (err, tagInstances) { + return callback(err, t, videoInstance, tagInstances) + }) + }, + + function updateVideoIntoDB (t, videoInstance, tagInstances, callback) { + const options = { transaction: t } + + videoInstance.set('name', videoAttributesToUpdate.name) + videoInstance.set('description', videoAttributesToUpdate.description) + videoInstance.set('infoHash', videoAttributesToUpdate.infoHash) + videoInstance.set('duration', videoAttributesToUpdate.duration) + videoInstance.set('createdAt', videoAttributesToUpdate.createdAt) + videoInstance.set('updatedAt', videoAttributesToUpdate.updatedAt) + videoInstance.set('extname', videoAttributesToUpdate.extname) + + videoInstance.save(options).asCallback(function (err) { + return callback(err, t, videoInstance, tagInstances) + }) + }, + + function associateTagsToVideo (t, videoInstance, tagInstances, callback) { + const options = { transaction: t } + + videoInstance.setTags(tagInstances, options).asCallback(function (err) { + return callback(err, t) + }) + } + + ], function (err, t) { + if (err) { + logger.error('Cannot update the remote video.') + + // Abort transaction? + if (t) t.rollback() + + return finalCallback(err) + } + + // Commit transaction + t.commit() + + return finalCallback() + }) +} + +function removeRemoteVideo (videoToRemoveData, fromPod, callback) { + // We need the instance because we have to remove some other stuffs (thumbnail etc) + db.Video.loadByHostAndRemoteId(fromPod.host, videoToRemoveData.remoteId, function (err, video) { + if (err || !video) { + logger.error('Cannot load video from host and remote id.', { error: err.message }) + return callback(err) + } + + logger.debug('Removing remote video %s.', video.remoteId) + video.destroy().asCallback(callback) + }) +} diff --git a/server/helpers/utils.js b/server/helpers/utils.js index 9f27671b6..7e0c9823c 100644 --- a/server/helpers/utils.js +++ b/server/helpers/utils.js @@ -5,11 +5,16 @@ const crypto = require('crypto') const logger = require('./logger') const utils = { + badRequest, cleanForExit, generateRandomString, isTestInstance } +function badRequest (req, res, next) { + res.type('json').status(400).end() +} + function generateRandomString (size, callback) { crypto.pseudoRandomBytes(size, function (err, raw) { if (err) return callback(err) -- cgit v1.2.3 From 55fa55a9be566cca2ba95322f2ae23b434aed62a Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Wed, 4 Jan 2017 20:59:23 +0100 Subject: Server: add video abuse support --- server/controllers/api/pods.js | 17 +- server/controllers/api/remote/videos.js | 68 ++++++-- server/controllers/api/users.js | 18 +- server/controllers/api/videos.js | 61 +++++-- server/helpers/custom-validators/index.js | 2 + server/helpers/custom-validators/remote/index.js | 11 ++ server/helpers/custom-validators/remote/videos.js | 74 +++++++++ server/helpers/custom-validators/videos.js | 66 ++------ server/helpers/utils.js | 16 +- server/initializers/constants.js | 4 + server/lib/friends.js | 39 +++-- server/middlewares/sort.js | 7 + server/middlewares/validators/remote.js | 30 ---- server/middlewares/validators/remote/index.js | 13 ++ server/middlewares/validators/remote/signature.js | 21 +++ server/middlewares/validators/remote/videos.js | 20 +++ server/middlewares/validators/sort.js | 23 ++- server/middlewares/validators/videos.js | 15 +- server/models/request-to-pod.js | 42 +++++ server/models/requestToPod.js | 42 ----- server/models/video-abuse.js | 113 +++++++++++++ server/models/video-tag.js | 18 ++ server/models/video.js | 8 + server/models/videoTag.js | 18 -- server/tests/api/check-params/index.js | 1 + server/tests/api/check-params/remotes.js | 4 + server/tests/api/check-params/video-abuses.js | 180 ++++++++++++++++++++ server/tests/api/friends-advanced.js | 10 +- server/tests/api/friends-basic.js | 12 +- server/tests/api/video-abuse.js | 191 ++++++++++++++++++++++ server/tests/utils/video-abuses.js | 73 +++++++++ 31 files changed, 978 insertions(+), 239 deletions(-) create mode 100644 server/helpers/custom-validators/remote/index.js create mode 100644 server/helpers/custom-validators/remote/videos.js delete mode 100644 server/middlewares/validators/remote.js create mode 100644 server/middlewares/validators/remote/index.js create mode 100644 server/middlewares/validators/remote/signature.js create mode 100644 server/middlewares/validators/remote/videos.js create mode 100644 server/models/request-to-pod.js delete mode 100644 server/models/requestToPod.js create mode 100644 server/models/video-abuse.js create mode 100644 server/models/video-tag.js delete mode 100644 server/models/videoTag.js create mode 100644 server/tests/api/check-params/video-abuses.js create mode 100644 server/tests/api/video-abuse.js create mode 100644 server/tests/utils/video-abuses.js (limited to 'server') diff --git a/server/controllers/api/pods.js b/server/controllers/api/pods.js index d9279f1d9..38702face 100644 --- a/server/controllers/api/pods.js +++ b/server/controllers/api/pods.js @@ -5,6 +5,7 @@ const waterfall = require('async/waterfall') const db = require('../../initializers/database') const logger = require('../../helpers/logger') +const utils = require('../../helpers/utils') const friends = require('../../lib/friends') const middlewares = require('../../middlewares') const admin = middlewares.admin @@ -36,7 +37,7 @@ router.get('/quitfriends', ) // Post because this is a secured request router.post('/remove', - signatureValidator, + signatureValidator.signature, checkSignature, removePods ) @@ -86,7 +87,7 @@ function listPods (req, res, next) { db.Pod.list(function (err, podsList) { if (err) return next(err) - res.json(getFormatedPods(podsList)) + res.json(utils.getFormatedObjects(podsList, podsList.length)) }) } @@ -130,15 +131,3 @@ function quitFriends (req, res, next) { res.type('json').status(204).end() }) } - -// --------------------------------------------------------------------------- - -function getFormatedPods (pods) { - const formatedPods = [] - - pods.forEach(function (pod) { - formatedPods.push(pod.toFormatedJSON()) - }) - - return formatedPods -} diff --git a/server/controllers/api/remote/videos.js b/server/controllers/api/remote/videos.js index 87c49bff9..d02da4463 100644 --- a/server/controllers/api/remote/videos.js +++ b/server/controllers/api/remote/videos.js @@ -7,15 +7,16 @@ const waterfall = require('async/waterfall') const db = require('../../../initializers/database') const middlewares = require('../../../middlewares') const secureMiddleware = middlewares.secure -const validators = middlewares.validators.remote +const videosValidators = middlewares.validators.remote.videos +const signatureValidators = middlewares.validators.remote.signature const logger = require('../../../helpers/logger') const router = express.Router() router.post('/', - validators.signature, + signatureValidators.signature, secureMiddleware.checkSignature, - validators.remoteVideos, + videosValidators.remoteVideos, remoteVideos ) @@ -32,19 +33,23 @@ function remoteVideos (req, res, next) { // We need to process in the same order to keep consistency // TODO: optimization eachSeries(requests, function (request, callbackEach) { - const videoData = request.data + const data = request.data switch (request.type) { case 'add': - addRemoteVideo(videoData, fromPod, callbackEach) + addRemoteVideo(data, fromPod, callbackEach) break case 'update': - updateRemoteVideo(videoData, fromPod, callbackEach) + updateRemoteVideo(data, fromPod, callbackEach) break case 'remove': - removeRemoteVideo(videoData, fromPod, callbackEach) + removeRemoteVideo(data, fromPod, callbackEach) + break + + case 'report-abuse': + reportAbuseRemoteVideo(data, fromPod, callbackEach) break default: @@ -164,13 +169,8 @@ function updateRemoteVideo (videoAttributesToUpdate, fromPod, finalCallback) { }, function findVideo (t, callback) { - db.Video.loadByHostAndRemoteId(fromPod.host, videoAttributesToUpdate.remoteId, function (err, videoInstance) { - if (err || !videoInstance) { - logger.error('Cannot load video from host and remote id.', { error: err.message }) - return callback(err) - } - - return callback(null, t, videoInstance) + fetchVideo(fromPod.host, videoAttributesToUpdate.remoteId, function (err, videoInstance) { + return callback(err, t, videoInstance) }) }, @@ -225,13 +225,45 @@ function updateRemoteVideo (videoAttributesToUpdate, fromPod, finalCallback) { function removeRemoteVideo (videoToRemoveData, fromPod, callback) { // We need the instance because we have to remove some other stuffs (thumbnail etc) - db.Video.loadByHostAndRemoteId(fromPod.host, videoToRemoveData.remoteId, function (err, video) { + fetchVideo(fromPod.host, videoToRemoveData.remoteId, function (err, video) { + if (err) return callback(err) + + logger.debug('Removing remote video %s.', video.remoteId) + video.destroy().asCallback(callback) + }) +} + +function reportAbuseRemoteVideo (reportData, fromPod, callback) { + db.Video.load(reportData.videoRemoteId, function (err, video) { if (err || !video) { - logger.error('Cannot load video from host and remote id.', { error: err.message }) + if (!err) err = new Error('video not found') + + logger.error('Cannot load video from host and remote id.', { error: err }) return callback(err) } - logger.debug('Removing remote video %s.', video.remoteId) - video.destroy().asCallback(callback) + logger.debug('Reporting remote abuse for video %s.', video.id) + + const videoAbuseData = { + reporterUsername: reportData.reporterUsername, + reason: reportData.reportReason, + reporterPodId: fromPod.id, + videoId: video.id + } + + db.VideoAbuse.create(videoAbuseData).asCallback(callback) + }) +} + +function fetchVideo (podHost, remoteId, callback) { + db.Video.loadByHostAndRemoteId(podHost, remoteId, function (err, video) { + if (err || !video) { + if (!err) err = new Error('video not found') + + logger.error('Cannot load video from host and remote id.', { error: err }) + return callback(err) + } + + return callback(null, video) }) } diff --git a/server/controllers/api/users.js b/server/controllers/api/users.js index 53bf56790..6cd0e84f7 100644 --- a/server/controllers/api/users.js +++ b/server/controllers/api/users.js @@ -6,6 +6,7 @@ const waterfall = require('async/waterfall') const constants = require('../../initializers/constants') const db = require('../../initializers/database') const logger = require('../../helpers/logger') +const utils = require('../../helpers/utils') const middlewares = require('../../middlewares') const admin = middlewares.admin const oAuth = middlewares.oauth @@ -82,7 +83,7 @@ function listUsers (req, res, next) { db.User.listForApi(req.query.start, req.query.count, req.query.sort, function (err, usersList, usersTotal) { if (err) return next(err) - res.json(getFormatedUsers(usersList, usersTotal)) + res.json(utils.getFormatedObjects(usersList, usersTotal)) }) } @@ -121,18 +122,3 @@ function updateUser (req, res, next) { function success (req, res, next) { res.end() } - -// --------------------------------------------------------------------------- - -function getFormatedUsers (users, usersTotal) { - const formatedUsers = [] - - users.forEach(function (user) { - formatedUsers.push(user.toFormatedJSON()) - }) - - return { - total: usersTotal, - data: formatedUsers - } -} diff --git a/server/controllers/api/videos.js b/server/controllers/api/videos.js index 35d6979e5..6829804ec 100644 --- a/server/controllers/api/videos.js +++ b/server/controllers/api/videos.js @@ -11,6 +11,7 @@ const db = require('../../initializers/database') const logger = require('../../helpers/logger') const friends = require('../../lib/friends') const middlewares = require('../../middlewares') +const admin = middlewares.admin const oAuth = middlewares.oauth const pagination = middlewares.pagination const validators = middlewares.validators @@ -43,6 +44,21 @@ const storage = multer.diskStorage({ const reqFiles = multer({ storage: storage }).fields([{ name: 'videofile', maxCount: 1 }]) +router.get('/abuse', + oAuth.authenticate, + admin.ensureIsAdmin, + validatorsPagination.pagination, + validatorsSort.videoAbusesSort, + sort.setVideoAbusesSort, + pagination.setPagination, + listVideoAbuses +) +router.post('/:id/abuse', + oAuth.authenticate, + validatorsVideos.videoAbuseReport, + reportVideoAbuse +) + router.get('/', validatorsPagination.pagination, validatorsSort.videosSort, @@ -283,7 +299,7 @@ function listVideos (req, res, next) { db.Video.listForApi(req.query.start, req.query.count, req.query.sort, function (err, videosList, videosTotal) { if (err) return next(err) - res.json(getFormatedVideos(videosList, videosTotal)) + res.json(utils.getFormatedObjects(videosList, videosTotal)) }) } @@ -306,22 +322,45 @@ function searchVideos (req, res, next) { function (err, videosList, videosTotal) { if (err) return next(err) - res.json(getFormatedVideos(videosList, videosTotal)) + res.json(utils.getFormatedObjects(videosList, videosTotal)) } ) } -// --------------------------------------------------------------------------- - -function getFormatedVideos (videos, videosTotal) { - const formatedVideos = [] +function listVideoAbuses (req, res, next) { + db.VideoAbuse.listForApi(req.query.start, req.query.count, req.query.sort, function (err, abusesList, abusesTotal) { + if (err) return next(err) - videos.forEach(function (video) { - formatedVideos.push(video.toFormatedJSON()) + res.json(utils.getFormatedObjects(abusesList, abusesTotal)) }) +} - return { - total: videosTotal, - data: formatedVideos +function reportVideoAbuse (req, res, next) { + const videoInstance = res.locals.video + const reporterUsername = res.locals.oauth.token.User.username + + const abuse = { + reporterUsername, + reason: req.body.reason, + videoId: videoInstance.id, + reporterPodId: null // This is our pod that reported this abuse } + + db.VideoAbuse.create(abuse).asCallback(function (err) { + if (err) return next(err) + + // We send the information to the destination pod + if (videoInstance.isOwned() === false) { + const reportData = { + reporterUsername, + reportReason: abuse.reason, + videoRemoteId: videoInstance.remoteId + } + + friends.reportAbuseVideoToFriend(reportData, videoInstance) + } + + return res.type('json').status(204).end() + }) } + diff --git a/server/helpers/custom-validators/index.js b/server/helpers/custom-validators/index.js index 96b5b20b9..9383e0304 100644 --- a/server/helpers/custom-validators/index.js +++ b/server/helpers/custom-validators/index.js @@ -2,12 +2,14 @@ const miscValidators = require('./misc') const podsValidators = require('./pods') +const remoteValidators = require('./remote') const usersValidators = require('./users') const videosValidators = require('./videos') const validators = { misc: miscValidators, pods: podsValidators, + remote: remoteValidators, users: usersValidators, videos: videosValidators } diff --git a/server/helpers/custom-validators/remote/index.js b/server/helpers/custom-validators/remote/index.js new file mode 100644 index 000000000..1939a95f4 --- /dev/null +++ b/server/helpers/custom-validators/remote/index.js @@ -0,0 +1,11 @@ +'use strict' + +const remoteVideosValidators = require('./videos') + +const validators = { + videos: remoteVideosValidators +} + +// --------------------------------------------------------------------------- + +module.exports = validators diff --git a/server/helpers/custom-validators/remote/videos.js b/server/helpers/custom-validators/remote/videos.js new file mode 100644 index 000000000..c3ca00e1c --- /dev/null +++ b/server/helpers/custom-validators/remote/videos.js @@ -0,0 +1,74 @@ +'use strict' + +const videosValidators = require('../videos') +const miscValidators = require('../misc') + +const remoteVideosValidators = { + isEachRemoteRequestVideosValid +} + +function isEachRemoteRequestVideosValid (requests) { + return miscValidators.isArray(requests) && + requests.every(function (request) { + const video = request.data + return ( + isRequestTypeAddValid(request.type) && + videosValidators.isVideoAuthorValid(video.author) && + videosValidators.isVideoDateValid(video.createdAt) && + videosValidators.isVideoDateValid(video.updatedAt) && + videosValidators.isVideoDescriptionValid(video.description) && + videosValidators.isVideoDurationValid(video.duration) && + videosValidators.isVideoInfoHashValid(video.infoHash) && + videosValidators.isVideoNameValid(video.name) && + videosValidators.isVideoTagsValid(video.tags) && + videosValidators.isVideoThumbnailDataValid(video.thumbnailData) && + videosValidators.isVideoRemoteIdValid(video.remoteId) && + videosValidators.isVideoExtnameValid(video.extname) + ) || + ( + isRequestTypeUpdateValid(request.type) && + videosValidators.isVideoDateValid(video.createdAt) && + videosValidators.isVideoDateValid(video.updatedAt) && + videosValidators.isVideoDescriptionValid(video.description) && + videosValidators.isVideoDurationValid(video.duration) && + videosValidators.isVideoInfoHashValid(video.infoHash) && + videosValidators.isVideoNameValid(video.name) && + videosValidators.isVideoTagsValid(video.tags) && + videosValidators.isVideoRemoteIdValid(video.remoteId) && + videosValidators.isVideoExtnameValid(video.extname) + ) || + ( + isRequestTypeRemoveValid(request.type) && + videosValidators.isVideoNameValid(video.name) && + videosValidators.isVideoRemoteIdValid(video.remoteId) + ) || + ( + isRequestTypeReportAbuseValid(request.type) && + videosValidators.isVideoRemoteIdValid(request.data.videoRemoteId) && + videosValidators.isVideoAbuseReasonValid(request.data.reportReason) && + videosValidators.isVideoAbuseReporterUsernameValid(request.data.reporterUsername) + ) + }) +} + +// --------------------------------------------------------------------------- + +module.exports = remoteVideosValidators + +// --------------------------------------------------------------------------- + +function isRequestTypeAddValid (value) { + return value === 'add' +} + +function isRequestTypeUpdateValid (value) { + return value === 'update' +} + +function isRequestTypeRemoveValid (value) { + return value === 'remove' +} + +function isRequestTypeReportAbuseValid (value) { + return value === 'report-abuse' +} diff --git a/server/helpers/custom-validators/videos.js b/server/helpers/custom-validators/videos.js index 8448386d9..7f727854d 100644 --- a/server/helpers/custom-validators/videos.js +++ b/server/helpers/custom-validators/videos.js @@ -6,9 +6,9 @@ const constants = require('../../initializers/constants') const usersValidators = require('./users') const miscValidators = require('./misc') const VIDEOS_CONSTRAINTS_FIELDS = constants.CONSTRAINTS_FIELDS.VIDEOS +const VIDEO_ABUSES_CONSTRAINTS_FIELDS = constants.CONSTRAINTS_FIELDS.VIDEO_ABUSES const videosValidators = { - isEachRemoteVideosValid, isVideoAuthorValid, isVideoDateValid, isVideoDescriptionValid, @@ -17,45 +17,11 @@ const videosValidators = { isVideoNameValid, isVideoTagsValid, isVideoThumbnailValid, - isVideoThumbnailDataValid -} - -function isEachRemoteVideosValid (requests) { - return miscValidators.isArray(requests) && - requests.every(function (request) { - const video = request.data - return ( - isRequestTypeAddValid(request.type) && - isVideoAuthorValid(video.author) && - isVideoDateValid(video.createdAt) && - isVideoDateValid(video.updatedAt) && - isVideoDescriptionValid(video.description) && - isVideoDurationValid(video.duration) && - isVideoInfoHashValid(video.infoHash) && - isVideoNameValid(video.name) && - isVideoTagsValid(video.tags) && - isVideoThumbnailDataValid(video.thumbnailData) && - isVideoRemoteIdValid(video.remoteId) && - isVideoExtnameValid(video.extname) - ) || - ( - isRequestTypeUpdateValid(request.type) && - isVideoDateValid(video.createdAt) && - isVideoDateValid(video.updatedAt) && - isVideoDescriptionValid(video.description) && - isVideoDurationValid(video.duration) && - isVideoInfoHashValid(video.infoHash) && - isVideoNameValid(video.name) && - isVideoTagsValid(video.tags) && - isVideoRemoteIdValid(video.remoteId) && - isVideoExtnameValid(video.extname) - ) || - ( - isRequestTypeRemoveValid(request.type) && - isVideoNameValid(video.name) && - isVideoRemoteIdValid(video.remoteId) - ) - }) + isVideoThumbnailDataValid, + isVideoExtnameValid, + isVideoRemoteIdValid, + isVideoAbuseReasonValid, + isVideoAbuseReporterUsernameValid } function isVideoAuthorValid (value) { @@ -107,20 +73,14 @@ function isVideoRemoteIdValid (value) { return validator.isUUID(value, 4) } -// --------------------------------------------------------------------------- - -module.exports = videosValidators - -// --------------------------------------------------------------------------- - -function isRequestTypeAddValid (value) { - return value === 'add' +function isVideoAbuseReasonValid (value) { + return validator.isLength(value, VIDEO_ABUSES_CONSTRAINTS_FIELDS.REASON) } -function isRequestTypeUpdateValid (value) { - return value === 'update' +function isVideoAbuseReporterUsernameValid (value) { + return usersValidators.isUserUsernameValid(value) } -function isRequestTypeRemoveValid (value) { - return value === 'remove' -} +// --------------------------------------------------------------------------- + +module.exports = videosValidators diff --git a/server/helpers/utils.js b/server/helpers/utils.js index 7e0c9823c..9f4b14582 100644 --- a/server/helpers/utils.js +++ b/server/helpers/utils.js @@ -8,7 +8,8 @@ const utils = { badRequest, cleanForExit, generateRandomString, - isTestInstance + isTestInstance, + getFormatedObjects } function badRequest (req, res, next) { @@ -32,6 +33,19 @@ function isTestInstance () { return (process.env.NODE_ENV === 'test') } +function getFormatedObjects (objects, objectsTotal) { + const formatedObjects = [] + + objects.forEach(function (object) { + formatedObjects.push(object.toFormatedJSON()) + }) + + return { + total: objectsTotal, + data: formatedObjects + } +} + // --------------------------------------------------------------------------- module.exports = utils diff --git a/server/initializers/constants.js b/server/initializers/constants.js index 474a37277..6ba8a9da0 100644 --- a/server/initializers/constants.js +++ b/server/initializers/constants.js @@ -19,6 +19,7 @@ const SEARCHABLE_COLUMNS = { // Sortable columns per schema const SORTABLE_COLUMNS = { USERS: [ 'username', '-username', 'createdAt', '-createdAt' ], + VIDEO_ABUSES: [ 'createdAt', '-createdAt' ], VIDEOS: [ 'name', '-name', 'duration', '-duration', 'createdAt', '-createdAt' ] } @@ -65,6 +66,9 @@ const CONSTRAINTS_FIELDS = { USERNAME: { min: 3, max: 20 }, // Length PASSWORD: { min: 6, max: 255 } // Length }, + VIDEO_ABUSES: { + REASON: { min: 2, max: 300 } // Length + }, VIDEOS: { NAME: { min: 3, max: 50 }, // Length DESCRIPTION: { min: 3, max: 250 }, // Length diff --git a/server/lib/friends.js b/server/lib/friends.js index 589b79660..4afb91b8b 100644 --- a/server/lib/friends.js +++ b/server/lib/friends.js @@ -15,6 +15,7 @@ const requests = require('../helpers/requests') const friends = { addVideoToFriends, updateVideoToFriends, + reportAbuseVideoToFriend, hasFriends, getMyCertificate, makeFriends, @@ -23,12 +24,20 @@ const friends = { sendOwnedVideosToPod } -function addVideoToFriends (video) { - createRequest('add', constants.REQUEST_ENDPOINTS.VIDEOS, video) +function addVideoToFriends (videoData) { + createRequest('add', constants.REQUEST_ENDPOINTS.VIDEOS, videoData) } -function updateVideoToFriends (video) { - createRequest('update', constants.REQUEST_ENDPOINTS.VIDEOS, video) +function updateVideoToFriends (videoData) { + createRequest('update', constants.REQUEST_ENDPOINTS.VIDEOS, videoData) +} + +function removeVideoToFriends (videoParams) { + createRequest('remove', constants.REQUEST_ENDPOINTS.VIDEOS, videoParams) +} + +function reportAbuseVideoToFriend (reportData, video) { + createRequest('report-abuse', constants.REQUEST_ENDPOINTS.VIDEOS, reportData, [ video.Author.podId ]) } function hasFriends (callback) { @@ -120,10 +129,6 @@ function quitFriends (callback) { }) } -function removeVideoToFriends (videoParams) { - createRequest('remove', constants.REQUEST_ENDPOINTS.VIDEOS, videoParams) -} - function sendOwnedVideosToPod (podId) { db.Video.listOwnedAndPopulateAuthorAndTags(function (err, videosList) { if (err) { @@ -152,10 +157,10 @@ module.exports = friends // --------------------------------------------------------------------------- function computeForeignPodsList (host, podsScore, callback) { - getForeignPodsList(host, function (err, foreignPodsList) { + getForeignPodsList(host, function (err, res) { if (err) return callback(err) - if (!foreignPodsList) foreignPodsList = [] + const foreignPodsList = res.data // Let's give 1 point to the pod we ask the friends list foreignPodsList.push({ host }) @@ -252,11 +257,11 @@ function makeRequestsToWinningPods (cert, podsList, callback) { }) } -// Wrapper that populate "to" argument with all our friends if it is not specified -function createRequest (type, endpoint, data, to) { - if (to) return _createRequest(type, endpoint, data, to) +// Wrapper that populate "toIds" argument with all our friends if it is not specified +function createRequest (type, endpoint, data, toIds) { + if (toIds) return _createRequest(type, endpoint, data, toIds) - // If the "to" pods is not specified, we send the request to all our friends + // If the "toIds" pods is not specified, we send the request to all our friends db.Pod.listAllIds(function (err, podIds) { if (err) { logger.error('Cannot get pod ids', { error: err }) @@ -267,13 +272,13 @@ function createRequest (type, endpoint, data, to) { }) } -function _createRequest (type, endpoint, data, to) { +function _createRequest (type, endpoint, data, toIds) { const pods = [] // If there are no destination pods abort - if (to.length === 0) return + if (toIds.length === 0) return - to.forEach(function (toPod) { + toIds.forEach(function (toPod) { pods.push(db.Pod.build({ id: toPod })) }) diff --git a/server/middlewares/sort.js b/server/middlewares/sort.js index 477e10571..39e167265 100644 --- a/server/middlewares/sort.js +++ b/server/middlewares/sort.js @@ -2,6 +2,7 @@ const sortMiddleware = { setUsersSort, + setVideoAbusesSort, setVideosSort } @@ -11,6 +12,12 @@ function setUsersSort (req, res, next) { return next() } +function setVideoAbusesSort (req, res, next) { + if (!req.query.sort) req.query.sort = '-createdAt' + + return next() +} + function setVideosSort (req, res, next) { if (!req.query.sort) req.query.sort = '-createdAt' diff --git a/server/middlewares/validators/remote.js b/server/middlewares/validators/remote.js deleted file mode 100644 index 858d193cc..000000000 --- a/server/middlewares/validators/remote.js +++ /dev/null @@ -1,30 +0,0 @@ -'use strict' - -const checkErrors = require('./utils').checkErrors -const logger = require('../../helpers/logger') - -const validatorsRemote = { - remoteVideos, - signature -} - -function remoteVideos (req, res, next) { - req.checkBody('data').isEachRemoteVideosValid() - - logger.debug('Checking remoteVideos parameters', { parameters: req.body }) - - checkErrors(req, res, next) -} - -function signature (req, res, next) { - req.checkBody('signature.host', 'Should have a signature host').isURL() - req.checkBody('signature.signature', 'Should have a signature').notEmpty() - - logger.debug('Checking signature parameters', { parameters: { signatureHost: req.body.signature.host } }) - - checkErrors(req, res, next) -} - -// --------------------------------------------------------------------------- - -module.exports = validatorsRemote diff --git a/server/middlewares/validators/remote/index.js b/server/middlewares/validators/remote/index.js new file mode 100644 index 000000000..022a2fe50 --- /dev/null +++ b/server/middlewares/validators/remote/index.js @@ -0,0 +1,13 @@ +'use strict' + +const remoteSignatureValidators = require('./signature') +const remoteVideosValidators = require('./videos') + +const validators = { + signature: remoteSignatureValidators, + videos: remoteVideosValidators +} + +// --------------------------------------------------------------------------- + +module.exports = validators diff --git a/server/middlewares/validators/remote/signature.js b/server/middlewares/validators/remote/signature.js new file mode 100644 index 000000000..5880a2c2c --- /dev/null +++ b/server/middlewares/validators/remote/signature.js @@ -0,0 +1,21 @@ +'use strict' + +const checkErrors = require('../utils').checkErrors +const logger = require('../../../helpers/logger') + +const validatorsRemoteSignature = { + signature +} + +function signature (req, res, next) { + req.checkBody('signature.host', 'Should have a signature host').isURL() + req.checkBody('signature.signature', 'Should have a signature').notEmpty() + + logger.debug('Checking signature parameters', { parameters: { signatureHost: req.body.signature.host } }) + + checkErrors(req, res, next) +} + +// --------------------------------------------------------------------------- + +module.exports = validatorsRemoteSignature diff --git a/server/middlewares/validators/remote/videos.js b/server/middlewares/validators/remote/videos.js new file mode 100644 index 000000000..cf9925b6c --- /dev/null +++ b/server/middlewares/validators/remote/videos.js @@ -0,0 +1,20 @@ +'use strict' + +const checkErrors = require('../utils').checkErrors +const logger = require('../../../helpers/logger') + +const validatorsRemoteVideos = { + remoteVideos +} + +function remoteVideos (req, res, next) { + req.checkBody('data').isEachRemoteRequestVideosValid() + + logger.debug('Checking remoteVideos parameters', { parameters: req.body }) + + checkErrors(req, res, next) +} + +// --------------------------------------------------------------------------- + +module.exports = validatorsRemoteVideos diff --git a/server/middlewares/validators/sort.js b/server/middlewares/validators/sort.js index 431d3fffd..b7eec0316 100644 --- a/server/middlewares/validators/sort.js +++ b/server/middlewares/validators/sort.js @@ -6,29 +6,38 @@ const logger = require('../../helpers/logger') const validatorsSort = { usersSort, + videoAbusesSort, videosSort } function usersSort (req, res, next) { const sortableColumns = constants.SORTABLE_COLUMNS.USERS - req.checkQuery('sort', 'Should have correct sortable column').optional().isIn(sortableColumns) + checkSort(req, res, next, sortableColumns) +} - logger.debug('Checking sort parameters', { parameters: req.query }) +function videoAbusesSort (req, res, next) { + const sortableColumns = constants.SORTABLE_COLUMNS.VIDEO_ABUSES - checkErrors(req, res, next) + checkSort(req, res, next, sortableColumns) } function videosSort (req, res, next) { const sortableColumns = constants.SORTABLE_COLUMNS.VIDEOS + checkSort(req, res, next, sortableColumns) +} + +// --------------------------------------------------------------------------- + +module.exports = validatorsSort + +// --------------------------------------------------------------------------- + +function checkSort (req, res, next, sortableColumns) { req.checkQuery('sort', 'Should have correct sortable column').optional().isIn(sortableColumns) logger.debug('Checking sort parameters', { parameters: req.query }) checkErrors(req, res, next) } - -// --------------------------------------------------------------------------- - -module.exports = validatorsSort diff --git a/server/middlewares/validators/videos.js b/server/middlewares/validators/videos.js index 295ed05fa..ff18a99c2 100644 --- a/server/middlewares/validators/videos.js +++ b/server/middlewares/validators/videos.js @@ -11,7 +11,9 @@ const validatorsVideos = { videosUpdate, videosGet, videosRemove, - videosSearch + videosSearch, + + videoAbuseReport } function videosAdd (req, res, next) { @@ -97,6 +99,17 @@ function videosSearch (req, res, next) { checkErrors(req, res, next) } +function videoAbuseReport (req, res, next) { + req.checkParams('id', 'Should have a valid id').notEmpty().isUUID(4) + req.checkBody('reason', 'Should have a valid reason').isVideoAbuseReasonValid() + + logger.debug('Checking videoAbuseReport parameters', { parameters: req.body }) + + checkErrors(req, res, function () { + checkVideoExists(req.params.id, res, next) + }) +} + // --------------------------------------------------------------------------- module.exports = validatorsVideos diff --git a/server/models/request-to-pod.js b/server/models/request-to-pod.js new file mode 100644 index 000000000..f42a53458 --- /dev/null +++ b/server/models/request-to-pod.js @@ -0,0 +1,42 @@ +'use strict' + +// --------------------------------------------------------------------------- + +module.exports = function (sequelize, DataTypes) { + const RequestToPod = sequelize.define('RequestToPod', {}, { + indexes: [ + { + fields: [ 'requestId' ] + }, + { + fields: [ 'podId' ] + }, + { + fields: [ 'requestId', 'podId' ], + unique: true + } + ], + classMethods: { + removePodOf + } + }) + + return RequestToPod +} + +// --------------------------------------------------------------------------- + +function removePodOf (requestsIds, podId, callback) { + if (!callback) callback = function () {} + + const query = { + where: { + requestId: { + $in: requestsIds + }, + podId: podId + } + } + + this.destroy(query).asCallback(callback) +} diff --git a/server/models/requestToPod.js b/server/models/requestToPod.js deleted file mode 100644 index f42a53458..000000000 --- a/server/models/requestToPod.js +++ /dev/null @@ -1,42 +0,0 @@ -'use strict' - -// --------------------------------------------------------------------------- - -module.exports = function (sequelize, DataTypes) { - const RequestToPod = sequelize.define('RequestToPod', {}, { - indexes: [ - { - fields: [ 'requestId' ] - }, - { - fields: [ 'podId' ] - }, - { - fields: [ 'requestId', 'podId' ], - unique: true - } - ], - classMethods: { - removePodOf - } - }) - - return RequestToPod -} - -// --------------------------------------------------------------------------- - -function removePodOf (requestsIds, podId, callback) { - if (!callback) callback = function () {} - - const query = { - where: { - requestId: { - $in: requestsIds - }, - podId: podId - } - } - - this.destroy(query).asCallback(callback) -} diff --git a/server/models/video-abuse.js b/server/models/video-abuse.js new file mode 100644 index 000000000..766a7568d --- /dev/null +++ b/server/models/video-abuse.js @@ -0,0 +1,113 @@ +'use strict' + +const constants = require('../initializers/constants') +const modelUtils = require('./utils') +const customVideosValidators = require('../helpers/custom-validators').videos + +module.exports = function (sequelize, DataTypes) { + const VideoAbuse = sequelize.define('VideoAbuse', + { + reporterUsername: { + type: DataTypes.STRING, + allowNull: false, + validate: { + reporterUsernameValid: function (value) { + const res = customVideosValidators.isVideoAbuseReporterUsernameValid(value) + if (res === false) throw new Error('Video abuse reporter username is not valid.') + } + } + }, + reason: { + type: DataTypes.STRING, + allowNull: false, + validate: { + reasonValid: function (value) { + const res = customVideosValidators.isVideoAbuseReasonValid(value) + if (res === false) throw new Error('Video abuse reason is not valid.') + } + } + } + }, + { + indexes: [ + { + fields: [ 'videoId' ] + }, + { + fields: [ 'reporterPodId' ] + } + ], + classMethods: { + associate, + + listForApi + }, + instanceMethods: { + toFormatedJSON + } + } + ) + + return VideoAbuse +} + +// --------------------------------------------------------------------------- + +function associate (models) { + this.belongsTo(models.Pod, { + foreignKey: { + name: 'reporterPodId', + allowNull: true + }, + onDelete: 'cascade' + }) + + this.belongsTo(models.Video, { + foreignKey: { + name: 'videoId', + allowNull: false + }, + onDelete: 'cascade' + }) +} + +function listForApi (start, count, sort, callback) { + const query = { + offset: start, + limit: count, + order: [ modelUtils.getSort(sort) ], + include: [ + { + model: this.sequelize.models.Pod, + required: false + } + ] + } + + return this.findAndCountAll(query).asCallback(function (err, result) { + if (err) return callback(err) + + return callback(null, result.rows, result.count) + }) +} + +function toFormatedJSON () { + let reporterPodHost + + if (this.Pod) { + reporterPodHost = this.Pod.host + } else { + // It means it's our video + reporterPodHost = constants.CONFIG.WEBSERVER.HOST + } + + const json = { + id: this.id, + reporterPodHost, + reason: this.reason, + reporterUsername: this.reporterUsername, + videoId: this.videoId + } + + return json +} diff --git a/server/models/video-tag.js b/server/models/video-tag.js new file mode 100644 index 000000000..cd9277a6e --- /dev/null +++ b/server/models/video-tag.js @@ -0,0 +1,18 @@ +'use strict' + +// --------------------------------------------------------------------------- + +module.exports = function (sequelize, DataTypes) { + const VideoTag = sequelize.define('VideoTag', {}, { + indexes: [ + { + fields: [ 'videoId' ] + }, + { + fields: [ 'tagId' ] + } + ] + }) + + return VideoTag +} diff --git a/server/models/video.js b/server/models/video.js index 3fe8368c7..4c197a835 100644 --- a/server/models/video.js +++ b/server/models/video.js @@ -248,6 +248,14 @@ function associate (models) { through: models.VideoTag, onDelete: 'cascade' }) + + this.hasMany(models.VideoAbuse, { + foreignKey: { + name: 'videoId', + allowNull: false + }, + onDelete: 'cascade' + }) } function generateMagnetUri () { diff --git a/server/models/videoTag.js b/server/models/videoTag.js deleted file mode 100644 index cd9277a6e..000000000 --- a/server/models/videoTag.js +++ /dev/null @@ -1,18 +0,0 @@ -'use strict' - -// --------------------------------------------------------------------------- - -module.exports = function (sequelize, DataTypes) { - const VideoTag = sequelize.define('VideoTag', {}, { - indexes: [ - { - fields: [ 'videoId' ] - }, - { - fields: [ 'tagId' ] - } - ] - }) - - return VideoTag -} diff --git a/server/tests/api/check-params/index.js b/server/tests/api/check-params/index.js index 3d6f09267..d0824f08a 100644 --- a/server/tests/api/check-params/index.js +++ b/server/tests/api/check-params/index.js @@ -6,3 +6,4 @@ require('./remotes') require('./users') require('./requests') require('./videos') +require('./video-abuses') diff --git a/server/tests/api/check-params/remotes.js b/server/tests/api/check-params/remotes.js index 30ba3b697..c1ab9fb2b 100644 --- a/server/tests/api/check-params/remotes.js +++ b/server/tests/api/check-params/remotes.js @@ -47,6 +47,10 @@ describe('Test remote videos API validators', function () { it('Should check when removing a video') }) + describe('When reporting abuse on a video', function () { + it('Should check when reporting a video abuse') + }) + after(function (done) { process.kill(-server.app.pid) diff --git a/server/tests/api/check-params/video-abuses.js b/server/tests/api/check-params/video-abuses.js new file mode 100644 index 000000000..8cb4ccdc1 --- /dev/null +++ b/server/tests/api/check-params/video-abuses.js @@ -0,0 +1,180 @@ +'use strict' + +const request = require('supertest') +const series = require('async/series') + +const loginUtils = require('../../utils/login') +const requestsUtils = require('../../utils/requests') +const serversUtils = require('../../utils/servers') +const usersUtils = require('../../utils/users') +const videosUtils = require('../../utils/videos') + +describe('Test video abuses API validators', function () { + let server = null + let userAccessToken = null + + // --------------------------------------------------------------- + + before(function (done) { + this.timeout(20000) + + series([ + function (next) { + serversUtils.flushTests(next) + }, + function (next) { + serversUtils.runServer(1, function (server1) { + server = server1 + + next() + }) + }, + function (next) { + loginUtils.loginAndGetAccessToken(server, function (err, token) { + if (err) throw err + server.accessToken = token + + next() + }) + }, + function (next) { + const username = 'user1' + const password = 'my super password' + + usersUtils.createUser(server.url, server.accessToken, username, password, next) + }, + function (next) { + const user = { + username: 'user1', + password: 'my super password' + } + + loginUtils.getUserAccessToken(server, user, function (err, accessToken) { + if (err) throw err + + userAccessToken = accessToken + + next() + }) + }, + // Upload some videos on each pods + function (next) { + const name = 'my super name for pod' + const description = 'my super description for pod' + const tags = [ 'tag' ] + const file = 'video_short2.webm' + videosUtils.uploadVideo(server.url, server.accessToken, name, description, tags, file, next) + }, + function (next) { + videosUtils.getVideosList(server.url, function (err, res) { + if (err) throw err + + const videos = res.body.data + server.video = videos[0] + + next() + }) + } + ], done) + }) + + describe('When listing video abuses', function () { + const path = '/api/v1/videos/abuse' + + it('Should fail with a bad start pagination', function (done) { + request(server.url) + .get(path) + .query({ start: 'hello' }) + .set('Authorization', 'Bearer ' + server.accessToken) + .set('Accept', 'application/json') + .expect(400, done) + }) + + it('Should fail with a bad count pagination', function (done) { + request(server.url) + .get(path) + .query({ count: 'hello' }) + .set('Accept', 'application/json') + .set('Authorization', 'Bearer ' + server.accessToken) + .expect(400, done) + }) + + it('Should fail with an incorrect sort', function (done) { + request(server.url) + .get(path) + .query({ sort: 'hello' }) + .set('Accept', 'application/json') + .set('Authorization', 'Bearer ' + server.accessToken) + .expect(400, done) + }) + + it('Should fail with a non authenticated user', function (done) { + request(server.url) + .get(path) + .query({ sort: 'hello' }) + .set('Accept', 'application/json') + .expect(401, done) + }) + + it('Should fail with a non admin user', function (done) { + request(server.url) + .get(path) + .query({ sort: 'hello' }) + .set('Accept', 'application/json') + .set('Authorization', 'Bearer ' + userAccessToken) + .expect(403, done) + }) + }) + + describe('When reporting a video abuse', function () { + const basePath = '/api/v1/videos/' + + it('Should fail with nothing', function (done) { + const path = basePath + server.video + '/abuse' + const data = {} + requestsUtils.makePostBodyRequest(server.url, path, server.accessToken, data, done) + }) + + it('Should fail with a wrong video', function (done) { + const wrongPath = '/api/v1/videos/blabla/abuse' + const data = {} + requestsUtils.makePostBodyRequest(server.url, wrongPath, server.accessToken, data, done) + }) + + it('Should fail with a non authenticated user', function (done) { + const data = {} + const path = basePath + server.video + '/abuse' + requestsUtils.makePostBodyRequest(server.url, path, 'hello', data, done, 401) + }) + + it('Should fail with a reason too short', function (done) { + const data = { + reason: 'h' + } + const path = basePath + server.video + '/abuse' + requestsUtils.makePostBodyRequest(server.url, path, server.accessToken, data, done) + }) + + it('Should fail with a reason too big', function (done) { + const data = { + reason: '0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef' + + '0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef' + + '0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef' + + '0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef' + } + const path = basePath + server.video + '/abuse' + requestsUtils.makePostBodyRequest(server.url, path, server.accessToken, data, done) + }) + }) + + after(function (done) { + process.kill(-server.app.pid) + + // Keep the logs if the test failed + if (this.ok) { + serversUtils.flushTests(done) + } else { + done() + } + }) +}) diff --git a/server/tests/api/friends-advanced.js b/server/tests/api/friends-advanced.js index 0a2d58d82..708138bc9 100644 --- a/server/tests/api/friends-advanced.js +++ b/server/tests/api/friends-advanced.js @@ -86,7 +86,7 @@ describe('Test advanced friends', function () { getFriendsList(5, function (err, res) { if (err) throw err - expect(res.body.length).to.equal(0) + expect(res.body.data.length).to.equal(0) done() }) @@ -111,7 +111,7 @@ describe('Test advanced friends', function () { getFriendsList(i, function (err, res) { if (err) throw err - expect(res.body.length).to.equal(0) + expect(res.body.data.length).to.equal(0) callback() }) @@ -140,7 +140,7 @@ describe('Test advanced friends', function () { getFriendsList(i, function (err, res) { if (err) throw err - expect(res.body.length).to.equal(3) + expect(res.body.data.length).to.equal(3) callback() }) @@ -182,7 +182,7 @@ describe('Test advanced friends', function () { if (err) throw err // Pod 4 didn't know pod 1 and 2 removed it - expect(res.body.length).to.equal(3) + expect(res.body.data.length).to.equal(3) next() }) }, @@ -200,7 +200,7 @@ describe('Test advanced friends', function () { if (err) throw err // Pod 4 should not be our friend - const result = res.body + const result = res.body.data expect(result.length).to.equal(3) for (const pod of result) { expect(pod.host).not.equal(servers[3].host) diff --git a/server/tests/api/friends-basic.js b/server/tests/api/friends-basic.js index 3a904dbd7..6f37ff291 100644 --- a/server/tests/api/friends-basic.js +++ b/server/tests/api/friends-basic.js @@ -28,7 +28,7 @@ describe('Test basic friends', function () { podsUtils.getFriendsList(serverToTest.url, function (err, res) { if (err) throw err - const result = res.body + const result = res.body.data expect(result).to.be.an('array') expect(result.length).to.equal(2) @@ -65,7 +65,7 @@ describe('Test basic friends', function () { podsUtils.getFriendsList(server.url, function (err, res) { if (err) throw err - const result = res.body + const result = res.body.data expect(result).to.be.an('array') expect(result.length).to.equal(0) callback() @@ -90,7 +90,7 @@ describe('Test basic friends', function () { podsUtils.getFriendsList(servers[1].url, function (err, res) { if (err) throw err - const result = res.body + const result = res.body.data expect(result).to.be.an('array') expect(result.length).to.equal(1) @@ -107,7 +107,7 @@ describe('Test basic friends', function () { podsUtils.getFriendsList(servers[2].url, function (err, res) { if (err) throw err - const result = res.body + const result = res.body.data expect(result).to.be.an('array') expect(result.length).to.equal(1) @@ -154,7 +154,7 @@ describe('Test basic friends', function () { podsUtils.getFriendsList(servers[1].url, function (err, res) { if (err) throw err - const result = res.body + const result = res.body.data expect(result).to.be.an('array') expect(result.length).to.equal(0) @@ -167,7 +167,7 @@ describe('Test basic friends', function () { podsUtils.getFriendsList(url, function (err, res) { if (err) throw err - const result = res.body + const result = res.body.data expect(result).to.be.an('array') expect(result.length).to.equal(1) expect(result[0].host).not.to.be.equal(servers[1].host) diff --git a/server/tests/api/video-abuse.js b/server/tests/api/video-abuse.js new file mode 100644 index 000000000..58db17c42 --- /dev/null +++ b/server/tests/api/video-abuse.js @@ -0,0 +1,191 @@ +'use strict' + +const chai = require('chai') +const each = require('async/each') +const expect = chai.expect +const series = require('async/series') + +const loginUtils = require('../utils/login') +const podsUtils = require('../utils/pods') +const serversUtils = require('../utils/servers') +const videosUtils = require('../utils/videos') +const videoAbusesUtils = require('../utils/video-abuses') + +describe('Test video abuses', function () { + let servers = [] + + before(function (done) { + this.timeout(30000) + + series([ + // Run servers + function (next) { + serversUtils.flushAndRunMultipleServers(2, function (serversRun) { + servers = serversRun + next() + }) + }, + // Get the access tokens + function (next) { + each(servers, function (server, callbackEach) { + loginUtils.loginAndGetAccessToken(server, function (err, accessToken) { + if (err) return callbackEach(err) + + server.accessToken = accessToken + callbackEach() + }) + }, next) + }, + // Pod 1 make friends too + function (next) { + const server = servers[0] + podsUtils.makeFriends(server.url, server.accessToken, next) + }, + // Upload some videos on each pods + function (next) { + const name = 'my super name for pod 1' + const description = 'my super description for pod 1' + const tags = [ 'tag' ] + const file = 'video_short2.webm' + videosUtils.uploadVideo(servers[0].url, servers[0].accessToken, name, description, tags, file, next) + }, + function (next) { + const name = 'my super name for pod 2' + const description = 'my super description for pod 2' + const tags = [ 'tag' ] + const file = 'video_short2.webm' + videosUtils.uploadVideo(servers[1].url, servers[1].accessToken, name, description, tags, file, next) + }, + // Wait videos propagation + function (next) { + setTimeout(next, 11000) + }, + function (next) { + videosUtils.getVideosList(servers[0].url, function (err, res) { + if (err) throw err + + const videos = res.body.data + + expect(videos.length).to.equal(2) + + servers[0].video = videos.find(function (video) { return video.name === 'my super name for pod 1' }) + servers[1].video = videos.find(function (video) { return video.name === 'my super name for pod 2' }) + + next() + }) + } + ], done) + }) + + it('Should not have video abuses', function (done) { + videoAbusesUtils.getVideoAbusesList(servers[0].url, servers[0].accessToken, function (err, res) { + if (err) throw err + + expect(res.body.total).to.equal(0) + expect(res.body.data).to.be.an('array') + expect(res.body.data.length).to.equal(0) + + done() + }) + }) + + it('Should report abuse on a local video', function (done) { + this.timeout(15000) + + const reason = 'my super bad reason' + videoAbusesUtils.reportVideoAbuse(servers[0].url, servers[0].accessToken, servers[0].video.id, reason, function (err) { + if (err) throw err + + // We wait requests propagation, even if the pod 1 is not supposed to make a request to pod 2 + setTimeout(done, 11000) + }) + }) + + it('Should have 1 video abuses on pod 1 and 0 on pod 2', function (done) { + videoAbusesUtils.getVideoAbusesList(servers[0].url, servers[0].accessToken, function (err, res) { + if (err) throw err + + expect(res.body.total).to.equal(1) + expect(res.body.data).to.be.an('array') + expect(res.body.data.length).to.equal(1) + + const abuse = res.body.data[0] + expect(abuse.reason).to.equal('my super bad reason') + expect(abuse.reporterUsername).to.equal('root') + expect(abuse.reporterPodHost).to.equal('localhost:9001') + expect(abuse.videoId).to.equal(servers[0].video.id) + + videoAbusesUtils.getVideoAbusesList(servers[1].url, servers[1].accessToken, function (err, res) { + if (err) throw err + + expect(res.body.total).to.equal(0) + expect(res.body.data).to.be.an('array') + expect(res.body.data.length).to.equal(0) + + done() + }) + }) + }) + + it('Should report abuse on a remote video', function (done) { + this.timeout(15000) + + const reason = 'my super bad reason 2' + videoAbusesUtils.reportVideoAbuse(servers[0].url, servers[0].accessToken, servers[1].video.id, reason, function (err) { + if (err) throw err + + // We wait requests propagation + setTimeout(done, 11000) + }) + }) + + it('Should have 2 video abuse on pod 1 and 1 on pod 2', function (done) { + videoAbusesUtils.getVideoAbusesList(servers[0].url, servers[0].accessToken, function (err, res) { + if (err) throw err + + expect(res.body.total).to.equal(2) + expect(res.body.data).to.be.an('array') + expect(res.body.data.length).to.equal(2) + + let abuse = res.body.data[0] + expect(abuse.reason).to.equal('my super bad reason') + expect(abuse.reporterUsername).to.equal('root') + expect(abuse.reporterPodHost).to.equal('localhost:9001') + expect(abuse.videoId).to.equal(servers[0].video.id) + + abuse = res.body.data[1] + expect(abuse.reason).to.equal('my super bad reason 2') + expect(abuse.reporterUsername).to.equal('root') + expect(abuse.reporterPodHost).to.equal('localhost:9001') + expect(abuse.videoId).to.equal(servers[1].video.id) + + videoAbusesUtils.getVideoAbusesList(servers[1].url, servers[1].accessToken, function (err, res) { + if (err) throw err + + expect(res.body.total).to.equal(1) + expect(res.body.data).to.be.an('array') + expect(res.body.data.length).to.equal(1) + + let abuse = res.body.data[0] + expect(abuse.reason).to.equal('my super bad reason 2') + expect(abuse.reporterUsername).to.equal('root') + expect(abuse.reporterPodHost).to.equal('localhost:9001') + + done() + }) + }) + }) + + after(function (done) { + servers.forEach(function (server) { + process.kill(-server.app.pid) + }) + + // Keep the logs if the test failed + if (this.ok) { + serversUtils.flushTests(done) + } else { + done() + } + }) +}) diff --git a/server/tests/utils/video-abuses.js b/server/tests/utils/video-abuses.js new file mode 100644 index 000000000..596c824b3 --- /dev/null +++ b/server/tests/utils/video-abuses.js @@ -0,0 +1,73 @@ +'use strict' + +const request = require('supertest') + +const videosUtils = { + getVideoAbusesList, + getVideoAbusesListPagination, + getVideoAbusesListSort, + reportVideoAbuse +} + +// ---------------------- Export functions -------------------- + +function reportVideoAbuse (url, token, videoId, reason, specialStatus, end) { + if (!end) { + end = specialStatus + specialStatus = 204 + } + + const path = '/api/v1/videos/' + videoId + '/abuse' + + request(url) + .post(path) + .set('Accept', 'application/json') + .set('Authorization', 'Bearer ' + token) + .send({ reason }) + .expect(specialStatus) + .end(end) +} + +function getVideoAbusesList (url, token, end) { + const path = '/api/v1/videos/abuse' + + request(url) + .get(path) + .query({ sort: 'createdAt' }) + .set('Accept', 'application/json') + .set('Authorization', 'Bearer ' + token) + .expect(200) + .expect('Content-Type', /json/) + .end(end) +} + +function getVideoAbusesListPagination (url, token, start, count, end) { + const path = '/api/v1/videos/abuse' + + request(url) + .get(path) + .query({ start: start }) + .query({ count: count }) + .set('Accept', 'application/json') + .set('Authorization', 'Bearer ' + token) + .expect(200) + .expect('Content-Type', /json/) + .end(end) +} + +function getVideoAbusesListSort (url, token, sort, end) { + const path = '/api/v1/videos/abuse' + + request(url) + .get(path) + .query({ sort: sort }) + .set('Accept', 'application/json') + .set('Authorization', 'Bearer ' + token) + .expect(200) + .expect('Content-Type', /json/) + .end(end) +} + +// --------------------------------------------------------------------------- + +module.exports = videosUtils -- cgit v1.2.3 From b981a525c37d226b3fa59287a6ce338f54583d0c Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Wed, 4 Jan 2017 21:15:57 +0100 Subject: Server: we don't need the video name when removing a remote video --- server/helpers/custom-validators/remote/videos.js | 1 - server/models/video.js | 1 - 2 files changed, 2 deletions(-) (limited to 'server') diff --git a/server/helpers/custom-validators/remote/videos.js b/server/helpers/custom-validators/remote/videos.js index c3ca00e1c..7c27b9dbb 100644 --- a/server/helpers/custom-validators/remote/videos.js +++ b/server/helpers/custom-validators/remote/videos.js @@ -39,7 +39,6 @@ function isEachRemoteRequestVideosValid (requests) { ) || ( isRequestTypeRemoveValid(request.type) && - videosValidators.isVideoNameValid(video.name) && videosValidators.isVideoRemoteIdValid(video.remoteId) ) || ( diff --git a/server/models/video.js b/server/models/video.js index 4c197a835..b3060705d 100644 --- a/server/models/video.js +++ b/server/models/video.js @@ -218,7 +218,6 @@ function afterDestroy (video, options, next) { function (callback) { const params = { - name: video.name, remoteId: video.id } -- cgit v1.2.3 From bdfbd4f162d66c3a6bd7c312a99e0b692e830792 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Wed, 4 Jan 2017 22:23:07 +0100 Subject: Server: use crypto instead of ursa for pod signature --- server/helpers/peertube-crypto.js | 54 ++++++++++++++++++----- server/helpers/requests.js | 46 ++++++++++--------- server/initializers/constants.js | 13 +++++- server/middlewares/secure.js | 9 +++- server/middlewares/validators/remote/signature.js | 2 +- server/models/request.js | 2 +- 6 files changed, 90 insertions(+), 36 deletions(-) (limited to 'server') diff --git a/server/helpers/peertube-crypto.js b/server/helpers/peertube-crypto.js index 610cb16cd..0f1e02ad6 100644 --- a/server/helpers/peertube-crypto.js +++ b/server/helpers/peertube-crypto.js @@ -1,9 +1,9 @@ 'use strict' +const crypto = require('crypto') const bcrypt = require('bcrypt') const fs = require('fs') const openssl = require('openssl-wrapper') -const ursa = require('ursa') const constants = require('../initializers/constants') const logger = require('./logger') @@ -16,12 +16,51 @@ const peertubeCrypto = { sign } -function checkSignature (publicKey, rawData, hexSignature) { - const crt = ursa.createPublicKey(publicKey) - const isValid = crt.hashAndVerify('sha256', new Buffer(rawData).toString('hex'), hexSignature, 'hex') +function checkSignature (publicKey, data, hexSignature) { + const verify = crypto.createVerify(constants.SIGNATURE_ALGORITHM) + + let dataString + if (typeof data === 'string') { + dataString = data + } else { + try { + dataString = JSON.stringify(data) + } catch (err) { + logger.error('Cannot check signature.', { error: err }) + return false + } + } + + verify.update(dataString, 'utf8') + + const isValid = verify.verify(publicKey, hexSignature, constants.SIGNATURE_ENCODING) return isValid } +function sign (data) { + const sign = crypto.createSign(constants.SIGNATURE_ALGORITHM) + + let dataString + if (typeof data === 'string') { + dataString = data + } else { + try { + dataString = JSON.stringify(data) + } catch (err) { + logger.error('Cannot sign data.', { error: err }) + return '' + } + } + + sign.update(dataString, 'utf8') + + // TODO: make async + const myKey = fs.readFileSync(constants.CONFIG.STORAGE.CERT_DIR + 'peertube.key.pem') + const signature = sign.sign(myKey, constants.SIGNATURE_ENCODING) + + return signature +} + function comparePassword (plainPassword, hashPassword, callback) { bcrypt.compare(plainPassword, hashPassword, function (err, isPasswordMatch) { if (err) return callback(err) @@ -52,13 +91,6 @@ function cryptPassword (password, callback) { }) } -function sign (data) { - const myKey = ursa.createPrivateKey(fs.readFileSync(constants.CONFIG.STORAGE.CERT_DIR + 'peertube.key.pem')) - const signature = myKey.hashAndSign('sha256', data, 'utf8', 'hex') - - return signature -} - // --------------------------------------------------------------------------- module.exports = peertubeCrypto diff --git a/server/helpers/requests.js b/server/helpers/requests.js index b0cda09fe..095b95e1c 100644 --- a/server/helpers/requests.js +++ b/server/helpers/requests.js @@ -28,31 +28,37 @@ function makeSecureRequest (params, callback) { url: constants.REMOTE_SCHEME.HTTP + '://' + params.toPod.host + params.path } - // Add data with POST requst ? - if (params.method === 'POST') { - requestParams.json = {} - - // Add signature if it is specified in the params - if (params.sign === true) { - const host = constants.CONFIG.WEBSERVER.HOST - - requestParams.json.signature = { - host, - signature: peertubeCrypto.sign(host) - } - } + if (params.method !== 'POST') { + return callback(new Error('Cannot make a secure request with a non POST method.')) + } + + requestParams.json = {} - // If there are data informations + // Add signature if it is specified in the params + if (params.sign === true) { + const host = constants.CONFIG.WEBSERVER.HOST + + let dataToSign if (params.data) { - requestParams.json.data = params.data - request.post(requestParams, callback) + dataToSign = dataToSign = params.data } else { - // No data - request.post(requestParams, callback) + // We do not have data to sign so we just take our host + // It is not ideal but the connection should be in HTTPS + dataToSign = host } - } else { - request.get(requestParams, callback) + + requestParams.json.signature = { + host, // Which host we pretend to be + signature: peertubeCrypto.sign(dataToSign) + } + } + + // If there are data informations + if (params.data) { + requestParams.json.data = params.data } + + request.post(requestParams, callback) } // --------------------------------------------------------------------------- diff --git a/server/initializers/constants.js b/server/initializers/constants.js index 6ba8a9da0..a6adb75bf 100644 --- a/server/initializers/constants.js +++ b/server/initializers/constants.js @@ -118,16 +118,21 @@ const REQUEST_ENDPOINTS = { VIDEOS: 'videos' } -// --------------------------------------------------------------------------- - const REMOTE_SCHEME = { HTTP: 'https', WS: 'wss' } +// --------------------------------------------------------------------------- + +const SIGNATURE_ALGORITHM = 'RSA-SHA256' +const SIGNATURE_ENCODING = 'hex' + // Password encryption const BCRYPT_SALT_SIZE = 10 +// --------------------------------------------------------------------------- + // Express static paths (router) const STATIC_PATHS = { PREVIEWS: '/static/previews/', @@ -143,6 +148,8 @@ let STATIC_MAX_AGE = '30d' const THUMBNAILS_SIZE = '200x110' const PREVIEWS_SIZE = '640x480' +// --------------------------------------------------------------------------- + const USER_ROLES = { ADMIN: 'admin', USER: 'user' @@ -180,6 +187,8 @@ module.exports = { REQUESTS_LIMIT, RETRY_REQUESTS, SEARCHABLE_COLUMNS, + SIGNATURE_ALGORITHM, + SIGNATURE_ENCODING, SORTABLE_COLUMNS, STATIC_MAX_AGE, STATIC_PATHS, diff --git a/server/middlewares/secure.js b/server/middlewares/secure.js index 2aae715c4..b6e6d818b 100644 --- a/server/middlewares/secure.js +++ b/server/middlewares/secure.js @@ -23,7 +23,14 @@ function checkSignature (req, res, next) { logger.debug('Checking signature from %s.', host) - const signatureOk = peertubeCrypto.checkSignature(pod.publicKey, host, req.body.signature.signature) + let signatureShouldBe + if (req.body.data) { + signatureShouldBe = req.body.data + } else { + signatureShouldBe = host + } + + const signatureOk = peertubeCrypto.checkSignature(pod.publicKey, signatureShouldBe, req.body.signature.signature) if (signatureOk === true) { res.locals.secure = { diff --git a/server/middlewares/validators/remote/signature.js b/server/middlewares/validators/remote/signature.js index 5880a2c2c..002232c05 100644 --- a/server/middlewares/validators/remote/signature.js +++ b/server/middlewares/validators/remote/signature.js @@ -11,7 +11,7 @@ function signature (req, res, next) { req.checkBody('signature.host', 'Should have a signature host').isURL() req.checkBody('signature.signature', 'Should have a signature').notEmpty() - logger.debug('Checking signature parameters', { parameters: { signatureHost: req.body.signature.host } }) + logger.debug('Checking signature parameters', { parameters: { signature: req.body.signature } }) checkErrors(req, res, next) } diff --git a/server/models/request.js b/server/models/request.js index e18f8fe3d..bae227c05 100644 --- a/server/models/request.js +++ b/server/models/request.js @@ -122,7 +122,7 @@ function makeRequest (toPod, requestEndpoint, requestsToMake, callback) { 'Error sending secure request to %s pod.', toPod.host, { - error: err || new Error('Status code not 20x : ' + res.statusCode) + error: err ? err.message : 'Status code not 20x : ' + res.statusCode } ) -- cgit v1.2.3 From bb0b243c92577872a5f4d98f707e078082af4d2a Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Fri, 6 Jan 2017 23:24:20 +0100 Subject: Server: improve real world script --- server/tests/real-world/real-world.js | 29 ++++++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) (limited to 'server') diff --git a/server/tests/real-world/real-world.js b/server/tests/real-world/real-world.js index 2ae3dc15b..751d3923f 100644 --- a/server/tests/real-world/real-world.js +++ b/server/tests/real-world/real-world.js @@ -2,6 +2,7 @@ const each = require('async/each') const isEqual = require('lodash/isEqual') +const differenceWith = require('lodash/differenceWith') const program = require('commander') const series = require('async/series') @@ -20,28 +21,35 @@ program .option('-a, --action [interval]', 'Interval in ms for an action') .option('-i, --integrity [interval]', 'Interval in ms for an integrity check') .option('-f, --flush', 'Flush datas on exit') + .option('-d, --difference', 'Display difference if integrity is not okay') .parse(process.argv) -const createWeight = parseInt(program.create) || 5 -const removeWeight = parseInt(program.remove) || 4 +const createWeight = program.create !== undefined ? parseInt(program.create) : 5 +const removeWeight = program.remove !== undefined ? parseInt(program.remove) : 4 const flushAtExit = program.flush || false -const actionInterval = parseInt(program.action) || 500 -let integrityInterval = parseInt(program.integrity) || 60000 +const actionInterval = program.action !== undefined ? parseInt(program.action) : 500 +let integrityInterval = program.integrity !== undefined ? parseInt(program.integrity) : 60000 +const displayDiffOnFail = program.integrity || false const numberOfPods = 6 + // Wait requests between pods -const requestsMaxPerInterval = constants.REQUESTS_INTERVAL / actionInterval +const baseRequestInterval = integrityInterval < constants.REQUESTS_INTERVAL ? integrityInterval : constants.REQUESTS_INTERVAL +const requestsMaxPerInterval = baseRequestInterval / actionInterval const intervalsToMakeAllRequests = Math.ceil(requestsMaxPerInterval / constants.REQUESTS_LIMIT) const waitForBeforeIntegrityCheck = (intervalsToMakeAllRequests * constants.REQUESTS_INTERVAL) + 1000 -integrityInterval += waitForBeforeIntegrityCheck - console.log('Create weight: %d, remove weight: %d.', createWeight, removeWeight) if (flushAtExit) { console.log('Program will flush data on exit.') } else { console.log('Program will not flush data on exit.') } +if (displayDiffOnFail) { + console.log('Program will display diff on failure.') +} else { + console.log('Program will not display diff on failure') +} console.log('Interval in ms for each action: %d.', actionInterval) console.log('Interval in ms for each integrity check: %d.', integrityInterval) console.log('Will wait %d ms before an integrity check.', waitForBeforeIntegrityCheck) @@ -73,6 +81,8 @@ runServers(numberOfPods, function (err, servers) { }, actionInterval) setInterval(function () { + if (checking === true) return + console.log('Checking integrity...') checking = true @@ -196,6 +206,7 @@ function checkIntegrity (servers, callback) { delete serverVideo.id delete serverVideo.isLocal delete serverVideo.thumbnailPath + delete serverVideo.updatedAt } videos.push(serverVideos) @@ -206,6 +217,10 @@ function checkIntegrity (servers, callback) { if (!isEqual(video, videos[0])) { console.error('Integrity not ok!') + if (displayDiffOnFail) { + console.log(differenceWith(videos[0], video, isEqual)) + } + process.exit(-1) } } -- cgit v1.2.3 From ed04d94f6d7132055f97a2f757b85c03c5f2a0b6 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Fri, 6 Jan 2017 23:24:47 +0100 Subject: Server: try to have a better video integrity --- server/controllers/api/remote/videos.js | 56 +++++++++++++--- server/controllers/api/videos.js | 114 +++++++++++++++++++++----------- server/helpers/utils.js | 16 ++++- server/lib/friends.js | 66 ++++++++++++------ server/models/pod.js | 9 ++- server/models/request.js | 4 +- 6 files changed, 194 insertions(+), 71 deletions(-) (limited to 'server') diff --git a/server/controllers/api/remote/videos.js b/server/controllers/api/remote/videos.js index d02da4463..6d768eae8 100644 --- a/server/controllers/api/remote/videos.js +++ b/server/controllers/api/remote/videos.js @@ -10,6 +10,7 @@ const secureMiddleware = middlewares.secure const videosValidators = middlewares.validators.remote.videos const signatureValidators = middlewares.validators.remote.signature const logger = require('../../../helpers/logger') +const utils = require('../../../helpers/utils') const router = express.Router() @@ -37,11 +38,11 @@ function remoteVideos (req, res, next) { switch (request.type) { case 'add': - addRemoteVideo(data, fromPod, callbackEach) + addRemoteVideoRetryWrapper(data, fromPod, callbackEach) break case 'update': - updateRemoteVideo(data, fromPod, callbackEach) + updateRemoteVideoRetryWrapper(data, fromPod, callbackEach) break case 'remove': @@ -63,13 +64,30 @@ function remoteVideos (req, res, next) { return res.type('json').status(204).end() } +// Handle retries on fail +function addRemoteVideoRetryWrapper (videoToCreateData, fromPod, finalCallback) { + utils.transactionRetryer( + function (callback) { + return addRemoteVideo(videoToCreateData, fromPod, callback) + }, + function (err) { + if (err) { + logger.error('Cannot insert the remote video with many retries.', { error: err }) + return finalCallback(err) + } + + return finalCallback() + } + ) +} + function addRemoteVideo (videoToCreateData, fromPod, finalCallback) { - logger.debug('Adding remote video "%s".', videoToCreateData.name) + logger.debug('Adding remote video "%s".', videoToCreateData.remoteId) waterfall([ function startTransaction (callback) { - db.sequelize.transaction().asCallback(function (err, t) { + db.sequelize.transaction({ isolationLevel: 'SERIALIZABLE' }).asCallback(function (err, t) { return callback(err, t) }) }, @@ -103,6 +121,7 @@ function addRemoteVideo (videoToCreateData, fromPod, finalCallback) { authorId: author.id, duration: videoToCreateData.duration, createdAt: videoToCreateData.createdAt, + // FIXME: updatedAt does not seems to be considered by Sequelize updatedAt: videoToCreateData.updatedAt } @@ -142,7 +161,8 @@ function addRemoteVideo (videoToCreateData, fromPod, finalCallback) { ], function (err, t) { if (err) { - logger.error('Cannot insert the remote video.') + // This is just a debug because we will retry the insert + logger.debug('Cannot insert the remote video.', { error: err }) // Abort transaction? if (t) t.rollback() @@ -157,8 +177,25 @@ function addRemoteVideo (videoToCreateData, fromPod, finalCallback) { }) } +// Handle retries on fail +function updateRemoteVideoRetryWrapper (videoAttributesToUpdate, fromPod, finalCallback) { + utils.transactionRetryer( + function (callback) { + return updateRemoteVideo(videoAttributesToUpdate, fromPod, callback) + }, + function (err) { + if (err) { + logger.error('Cannot update the remote video with many retries.', { error: err }) + return finalCallback(err) + } + + return finalCallback() + } + ) +} + function updateRemoteVideo (videoAttributesToUpdate, fromPod, finalCallback) { - logger.debug('Updating remote video "%s".', videoAttributesToUpdate.name) + logger.debug('Updating remote video "%s".', videoAttributesToUpdate.remoteId) waterfall([ @@ -208,7 +245,8 @@ function updateRemoteVideo (videoAttributesToUpdate, fromPod, finalCallback) { ], function (err, t) { if (err) { - logger.error('Cannot update the remote video.') + // This is just a debug because we will retry the insert + logger.debug('Cannot update the remote video.', { error: err }) // Abort transaction? if (t) t.rollback() @@ -238,7 +276,7 @@ function reportAbuseRemoteVideo (reportData, fromPod, callback) { if (err || !video) { if (!err) err = new Error('video not found') - logger.error('Cannot load video from host and remote id.', { error: err }) + logger.error('Cannot load video from id.', { error: err, id: reportData.videoRemoteId }) return callback(err) } @@ -260,7 +298,7 @@ function fetchVideo (podHost, remoteId, callback) { if (err || !video) { if (!err) err = new Error('video not found') - logger.error('Cannot load video from host and remote id.', { error: err }) + logger.error('Cannot load video from host and remote id.', { error: err, podHost, remoteId }) return callback(err) } diff --git a/server/controllers/api/videos.js b/server/controllers/api/videos.js index 6829804ec..4d45c11c0 100644 --- a/server/controllers/api/videos.js +++ b/server/controllers/api/videos.js @@ -70,13 +70,13 @@ router.put('/:id', oAuth.authenticate, reqFiles, validatorsVideos.videosUpdate, - updateVideo + updateVideoRetryWrapper ) router.post('/', oAuth.authenticate, reqFiles, validatorsVideos.videosAdd, - addVideo + addVideoRetryWrapper ) router.get('/:id', validatorsVideos.videosGet, @@ -103,19 +103,37 @@ module.exports = router // --------------------------------------------------------------------------- -function addVideo (req, res, next) { - const videoFile = req.files.videofile[0] +// Wrapper to video add that retry the function if there is a database error +// We need this because we run the transaction in SERIALIZABLE isolation that can fail +function addVideoRetryWrapper (req, res, next) { + utils.transactionRetryer( + function (callback) { + return addVideo(req, res, req.files.videofile[0], callback) + }, + function (err) { + if (err) { + logger.error('Cannot insert the video with many retries.', { error: err }) + return next(err) + } + + // TODO : include Location of the new video -> 201 + return res.type('json').status(204).end() + } + ) +} + +function addVideo (req, res, videoFile, callback) { const videoInfos = req.body waterfall([ - function startTransaction (callback) { - db.sequelize.transaction().asCallback(function (err, t) { - return callback(err, t) + function startTransaction (callbackWaterfall) { + db.sequelize.transaction({ isolationLevel: 'SERIALIZABLE' }).asCallback(function (err, t) { + return callbackWaterfall(err, t) }) }, - function findOrCreateAuthor (t, callback) { + function findOrCreateAuthor (t, callbackWaterfall) { const user = res.locals.oauth.token.User const name = user.username @@ -124,19 +142,19 @@ function addVideo (req, res, next) { const userId = user.id db.Author.findOrCreateAuthor(name, podId, userId, t, function (err, authorInstance) { - return callback(err, t, authorInstance) + return callbackWaterfall(err, t, authorInstance) }) }, - function findOrCreateTags (t, author, callback) { + function findOrCreateTags (t, author, callbackWaterfall) { const tags = videoInfos.tags db.Tag.findOrCreateTags(tags, t, function (err, tagInstances) { - return callback(err, t, author, tagInstances) + return callbackWaterfall(err, t, author, tagInstances) }) }, - function createVideoObject (t, author, tagInstances, callback) { + function createVideoObject (t, author, tagInstances, callbackWaterfall) { const videoData = { name: videoInfos.name, remoteId: null, @@ -148,74 +166,97 @@ function addVideo (req, res, next) { const video = db.Video.build(videoData) - return callback(null, t, author, tagInstances, video) + return callbackWaterfall(null, t, author, tagInstances, video) }, // Set the videoname the same as the id - function renameVideoFile (t, author, tagInstances, video, callback) { + function renameVideoFile (t, author, tagInstances, video, callbackWaterfall) { const videoDir = constants.CONFIG.STORAGE.VIDEOS_DIR const source = path.join(videoDir, videoFile.filename) const destination = path.join(videoDir, video.getVideoFilename()) fs.rename(source, destination, function (err) { - return callback(err, t, author, tagInstances, video) + if (err) return callbackWaterfall(err) + + // This is important in case if there is another attempt + videoFile.filename = video.getVideoFilename() + return callbackWaterfall(null, t, author, tagInstances, video) }) }, - function insertVideoIntoDB (t, author, tagInstances, video, callback) { + function insertVideoIntoDB (t, author, tagInstances, video, callbackWaterfall) { const options = { transaction: t } // Add tags association video.save(options).asCallback(function (err, videoCreated) { - if (err) return callback(err) + if (err) return callbackWaterfall(err) // Do not forget to add Author informations to the created video videoCreated.Author = author - return callback(err, t, tagInstances, videoCreated) + return callbackWaterfall(err, t, tagInstances, videoCreated) }) }, - function associateTagsToVideo (t, tagInstances, video, callback) { + function associateTagsToVideo (t, tagInstances, video, callbackWaterfall) { const options = { transaction: t } video.setTags(tagInstances, options).asCallback(function (err) { video.Tags = tagInstances - return callback(err, t, video) + return callbackWaterfall(err, t, video) }) }, - function sendToFriends (t, video, callback) { + function sendToFriends (t, video, callbackWaterfall) { video.toAddRemoteJSON(function (err, remoteVideo) { - if (err) return callback(err) + if (err) return callbackWaterfall(err) // Now we'll add the video's meta data to our friends - friends.addVideoToFriends(remoteVideo) - - return callback(null, t) + friends.addVideoToFriends(remoteVideo, t, function (err) { + return callbackWaterfall(err, t) + }) }) } ], function andFinally (err, t) { if (err) { - logger.error('Cannot insert the video.') + // This is just a debug because we will retry the insert + logger.debug('Cannot insert the video.', { error: err }) // Abort transaction? if (t) t.rollback() - return next(err) + return callback(err) } // Commit transaction t.commit() - // TODO : include Location of the new video -> 201 - return res.type('json').status(204).end() + logger.info('Video with name %s created.', videoInfos.name) + + return callback(null) }) } -function updateVideo (req, res, next) { +function updateVideoRetryWrapper (req, res, next) { + utils.transactionRetryer( + function (callback) { + return updateVideo(req, res, callback) + }, + function (err) { + if (err) { + logger.error('Cannot update the video with many retries.', { error: err }) + return next(err) + } + + // TODO : include Location of the new video -> 201 + return res.type('json').status(204).end() + } + ) +} + +function updateVideo (req, res, finalCallback) { const videoInstance = res.locals.video const videoInfosToUpdate = req.body @@ -267,26 +308,25 @@ function updateVideo (req, res, next) { const json = videoInstance.toUpdateRemoteJSON() // Now we'll update the video's meta data to our friends - friends.updateVideoToFriends(json) - - return callback(null, t) + friends.updateVideoToFriends(json, t, function (err) { + return callback(err, t) + }) } ], function andFinally (err, t) { if (err) { - logger.error('Cannot insert the video.') + logger.debug('Cannot update the video.', { error: err }) // Abort transaction? if (t) t.rollback() - return next(err) + return finalCallback(err) } // Commit transaction t.commit() - // TODO : include Location of the new video -> 201 - return res.type('json').status(204).end() + return finalCallback(null) }) } diff --git a/server/helpers/utils.js b/server/helpers/utils.js index 9f4b14582..a902850cd 100644 --- a/server/helpers/utils.js +++ b/server/helpers/utils.js @@ -1,6 +1,7 @@ 'use strict' const crypto = require('crypto') +const retry = require('async/retry') const logger = require('./logger') @@ -9,7 +10,8 @@ const utils = { cleanForExit, generateRandomString, isTestInstance, - getFormatedObjects + getFormatedObjects, + transactionRetryer } function badRequest (req, res, next) { @@ -46,6 +48,18 @@ function getFormatedObjects (objects, objectsTotal) { } } +function transactionRetryer (func, callback) { + retry({ + times: 5, + + errorFilter: function (err) { + const willRetry = (err.name === 'SequelizeDatabaseError') + logger.debug('Maybe retrying the transaction function.', { willRetry }) + return willRetry + } + }, func, callback) +} + // --------------------------------------------------------------------------- module.exports = utils diff --git a/server/lib/friends.js b/server/lib/friends.js index 4afb91b8b..3d3d0fdee 100644 --- a/server/lib/friends.js +++ b/server/lib/friends.js @@ -24,16 +24,33 @@ const friends = { sendOwnedVideosToPod } -function addVideoToFriends (videoData) { - createRequest('add', constants.REQUEST_ENDPOINTS.VIDEOS, videoData) +function addVideoToFriends (videoData, transaction, callback) { + const options = { + type: 'add', + endpoint: constants.REQUEST_ENDPOINTS.VIDEOS, + data: videoData, + transaction + } + createRequest(options, callback) } -function updateVideoToFriends (videoData) { - createRequest('update', constants.REQUEST_ENDPOINTS.VIDEOS, videoData) +function updateVideoToFriends (videoData, transaction, callback) { + const options = { + type: 'update', + endpoint: constants.REQUEST_ENDPOINTS.VIDEOS, + data: videoData, + transaction + } + createRequest(options, callback) } function removeVideoToFriends (videoParams) { - createRequest('remove', constants.REQUEST_ENDPOINTS.VIDEOS, videoParams) + const options = { + type: 'remove', + endpoint: constants.REQUEST_ENDPOINTS.VIDEOS, + data: videoParams + } + createRequest(options) } function reportAbuseVideoToFriend (reportData, video) { @@ -258,25 +275,35 @@ function makeRequestsToWinningPods (cert, podsList, callback) { } // Wrapper that populate "toIds" argument with all our friends if it is not specified -function createRequest (type, endpoint, data, toIds) { - if (toIds) return _createRequest(type, endpoint, data, toIds) +// { type, endpoint, data, toIds, transaction } +function createRequest (options, callback) { + if (!callback) callback = function () {} + if (options.toIds) return _createRequest(options, callback) // If the "toIds" pods is not specified, we send the request to all our friends - db.Pod.listAllIds(function (err, podIds) { + db.Pod.listAllIds(options.transaction, function (err, podIds) { if (err) { logger.error('Cannot get pod ids', { error: err }) return } - return _createRequest(type, endpoint, data, podIds) + const newOptions = Object.assign(options, { toIds: podIds }) + return _createRequest(newOptions, callback) }) } -function _createRequest (type, endpoint, data, toIds) { +// { type, endpoint, data, toIds, transaction } +function _createRequest (options, callback) { + const type = options.type + const endpoint = options.endpoint + const data = options.data + const toIds = options.toIds + const transaction = options.transaction + const pods = [] // If there are no destination pods abort - if (toIds.length === 0) return + if (toIds.length === 0) return callback(null) toIds.forEach(function (toPod) { pods.push(db.Pod.build({ id: toPod })) @@ -290,17 +317,14 @@ function _createRequest (type, endpoint, data, toIds) { } } - // We run in transaction to keep coherency between Request and RequestToPod tables - db.sequelize.transaction(function (t) { - const dbRequestOptions = { - transaction: t - } + const dbRequestOptions = { + transaction + } - return db.Request.create(createQuery, dbRequestOptions).then(function (request) { - return request.setPods(pods, dbRequestOptions) - }) - }).asCallback(function (err) { - if (err) logger.error('Error in createRequest transaction.', { error: err }) + return db.Request.create(createQuery, dbRequestOptions).asCallback(function (err, request) { + if (err) return callback(err) + + return request.setPods(pods, dbRequestOptions).asCallback(callback) }) } diff --git a/server/models/pod.js b/server/models/pod.js index 83ecd732e..8e7dd1fd8 100644 --- a/server/models/pod.js +++ b/server/models/pod.js @@ -115,11 +115,18 @@ function list (callback) { return this.findAll().asCallback(callback) } -function listAllIds (callback) { +function listAllIds (transaction, callback) { + if (!callback) { + callback = transaction + transaction = null + } + const query = { attributes: [ 'id' ] } + if (transaction) query.transaction = transaction + return this.findAll(query).asCallback(function (err, pods) { if (err) return callback(err) diff --git a/server/models/request.js b/server/models/request.js index bae227c05..1d6038044 100644 --- a/server/models/request.js +++ b/server/models/request.js @@ -291,8 +291,8 @@ function listWithLimitAndRandom (limit, callback) { order: [ [ 'id', 'ASC' ] ], - offset: start, - limit: limit, + // offset: start, + // limit: limit, include: [ this.sequelize.models.Pod ] } -- cgit v1.2.3 From bd14d16a29e2f90805d04b48378188517741a071 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Tue, 10 Jan 2017 22:24:42 +0100 Subject: Server: improve requests scheduler --- server/initializers/constants.js | 9 ++-- server/lib/friends.js | 16 +++++- server/models/pod.js | 37 +++++++++++++ server/models/request.js | 110 ++++++++++++++++++++++++--------------- 4 files changed, 125 insertions(+), 47 deletions(-) (limited to 'server') diff --git a/server/initializers/constants.js b/server/initializers/constants.js index a6adb75bf..97e3c5296 100644 --- a/server/initializers/constants.js +++ b/server/initializers/constants.js @@ -108,8 +108,10 @@ let REQUESTS_INTERVAL = 600000 // Number of requests in parallel we can make const REQUESTS_IN_PARALLEL = 10 -// How many requests we put in request -const REQUESTS_LIMIT = 10 +// To how many pods we send requests +const REQUESTS_LIMIT_PODS = 10 +// How many requests we send to a pod per interval +const REQUESTS_LIMIT_PER_POD = 5 // Number of requests to retry for replay requests module const RETRY_REQUESTS = 5 @@ -184,7 +186,8 @@ module.exports = { REQUEST_ENDPOINTS, REQUESTS_IN_PARALLEL, REQUESTS_INTERVAL, - REQUESTS_LIMIT, + REQUESTS_LIMIT_PODS, + REQUESTS_LIMIT_PER_POD, RETRY_REQUESTS, SEARCHABLE_COLUMNS, SIGNATURE_ALGORITHM, diff --git a/server/lib/friends.js b/server/lib/friends.js index 3d3d0fdee..f0575ff2f 100644 --- a/server/lib/friends.js +++ b/server/lib/friends.js @@ -54,7 +54,13 @@ function removeVideoToFriends (videoParams) { } function reportAbuseVideoToFriend (reportData, video) { - createRequest('report-abuse', constants.REQUEST_ENDPOINTS.VIDEOS, reportData, [ video.Author.podId ]) + const options = { + type: 'report-abuse', + endpoint: constants.REQUEST_ENDPOINTS.VIDEOS, + data: reportData, + toIds: [ video.Author.podId ] + } + createRequest(options) } function hasFriends (callback) { @@ -161,7 +167,13 @@ function sendOwnedVideosToPod (podId) { return } - createRequest('add', constants.REQUEST_ENDPOINTS.VIDEOS, remoteVideo, [ podId ]) + const options = { + type: 'add', + endpoint: constants.REQUEST_ENDPOINTS.VIDEOS, + data: remoteVideo, + toIds: [ podId ] + } + createRequest(options) }) }) }) diff --git a/server/models/pod.js b/server/models/pod.js index 8e7dd1fd8..b3c6db8e8 100644 --- a/server/models/pod.js +++ b/server/models/pod.js @@ -50,6 +50,7 @@ module.exports = function (sequelize, DataTypes) { incrementScores, list, listAllIds, + listRandomPodIdsWithRequest, listBadPods, load, loadByHost, @@ -134,6 +135,42 @@ function listAllIds (transaction, callback) { }) } +function listRandomPodIdsWithRequest (limit, callback) { + const self = this + + self.count().asCallback(function (err, count) { + if (err) return callback(err) + + // Optimization... + if (count === 0) return callback(null, []) + + let start = Math.floor(Math.random() * count) - limit + if (start < 0) start = 0 + + const query = { + attributes: [ 'id' ], + order: [ + [ 'id', 'ASC' ] + ], + offset: start, + limit: limit, + where: { + id: { + $in: [ + this.sequelize.literal('SELECT "podId" FROM "RequestToPods"') + ] + } + } + } + + return this.findAll(query).asCallback(function (err, pods) { + if (err) return callback(err) + + return callback(null, map(pods, 'id')) + }) + }) +} + function listBadPods (callback) { const query = { where: { diff --git a/server/models/request.js b/server/models/request.js index 1d6038044..26953e5f5 100644 --- a/server/models/request.js +++ b/server/models/request.js @@ -138,9 +138,9 @@ function makeRequests () { const self = this const RequestToPod = this.sequelize.models.RequestToPod - // We limit the size of the requests (REQUESTS_LIMIT) + // We limit the size of the requests // We don't want to stuck with the same failing requests so we get a random list - listWithLimitAndRandom.call(self, constants.REQUESTS_LIMIT, function (err, requests) { + listWithLimitAndRandom.call(self, constants.REQUESTS_LIMIT_PODS, constants.REQUESTS_LIMIT_PER_POD, function (err, requests) { if (err) { logger.error('Cannot get the list of requests.', { err: err }) return // Abort @@ -156,13 +156,15 @@ function makeRequests () { // We want to group requests by destinations pod and endpoint const requestsToMakeGrouped = {} + Object.keys(requests).forEach(function (toPodId) { + requests[toPodId].forEach(function (data) { + const request = data.request + const pod = data.pod + const hashKey = toPodId + request.endpoint - requests.forEach(function (request) { - request.Pods.forEach(function (toPod) { - const hashKey = toPod.id + request.endpoint if (!requestsToMakeGrouped[hashKey]) { requestsToMakeGrouped[hashKey] = { - toPodId: toPod.id, + toPod: pod, endpoint: request.endpoint, ids: [], // request ids, to delete them from the DB in the future datas: [] // requests data, @@ -179,36 +181,29 @@ function makeRequests () { eachLimit(Object.keys(requestsToMakeGrouped), constants.REQUESTS_IN_PARALLEL, function (hashKey, callbackEach) { const requestToMake = requestsToMakeGrouped[hashKey] + const toPod = requestToMake.toPod - // FIXME: SQL request inside a loop :/ - self.sequelize.models.Pod.load(requestToMake.toPodId, function (err, toPod) { - if (err) { - logger.error('Error finding pod by id.', { err: err }) - return callbackEach() - } - - // Maybe the pod is not our friend anymore so simply remove it - if (!toPod) { - const requestIdsToDelete = requestToMake.ids + // Maybe the pod is not our friend anymore so simply remove it + if (!toPod) { + const requestIdsToDelete = requestToMake.ids - logger.info('Removing %d requests of unexisting pod %s.', requestIdsToDelete.length, requestToMake.toPodId) - RequestToPod.removePodOf.call(self, requestIdsToDelete, requestToMake.toPodId) - return callbackEach() - } + logger.info('Removing %d requests of unexisting pod %s.', requestIdsToDelete.length, requestToMake.toPod.id) + RequestToPod.removePodOf.call(self, requestIdsToDelete, requestToMake.toPod.id) + return callbackEach() + } - makeRequest(toPod, requestToMake.endpoint, requestToMake.datas, function (success) { - if (success === true) { - logger.debug('Removing requests for pod %s.', requestToMake.toPodId, { requestsIds: requestToMake.ids }) + makeRequest(toPod, requestToMake.endpoint, requestToMake.datas, function (success) { + if (success === true) { + logger.debug('Removing requests for pod %s.', requestToMake.toPod.id, { requestsIds: requestToMake.ids }) - goodPods.push(requestToMake.toPodId) + goodPods.push(requestToMake.toPod.id) - // Remove the pod id of these request ids - RequestToPod.removePodOf(requestToMake.ids, requestToMake.toPodId, callbackEach) - } else { - badPods.push(requestToMake.toPodId) - callbackEach() - } - }) + // Remove the pod id of these request ids + RequestToPod.removePodOf(requestToMake.ids, requestToMake.toPod.id, callbackEach) + } else { + badPods.push(requestToMake.toPod.id) + callbackEach() + } }) }, function () { // All the requests were made, we update the pods score @@ -275,29 +270,60 @@ function updatePodsScore (goodPods, badPods) { } } -function listWithLimitAndRandom (limit, callback) { +function listWithLimitAndRandom (limitPods, limitRequestsPerPod, callback) { const self = this + const Pod = this.sequelize.models.Pod - self.count().asCallback(function (err, count) { + Pod.listRandomPodIdsWithRequest(limitPods, function (err, podIds) { if (err) return callback(err) - // Optimization... - if (count === 0) return callback(null, []) - - let start = Math.floor(Math.random() * count) - limit - if (start < 0) start = 0 + // We don't have friends that have requests + if (podIds.length === 0) return callback(null, []) + // The the first x requests of these pods + // It is very important to sort by id ASC to keep the requests order! const query = { order: [ [ 'id', 'ASC' ] ], - // offset: start, - // limit: limit, - include: [ this.sequelize.models.Pod ] + include: [ + { + model: self.sequelize.models.Pod, + where: { + id: { + $in: podIds + } + } + } + ] } - self.findAll(query).asCallback(callback) + self.findAll(query).asCallback(function (err, requests) { + if (err) return callback(err) + + const requestsGrouped = groupAndTruncateRequests(requests, limitRequestsPerPod) + return callback(err, requestsGrouped) + }) + }) +} + +function groupAndTruncateRequests (requests, limitRequestsPerPod) { + const requestsGrouped = {} + + requests.forEach(function (request) { + request.Pods.forEach(function (pod) { + if (!requestsGrouped[pod.id]) requestsGrouped[pod.id] = [] + + if (requestsGrouped[pod.id].length < limitRequestsPerPod) { + requestsGrouped[pod.id].push({ + request, + pod + }) + } + }) }) + + return requestsGrouped } function removeAll (callback) { -- cgit v1.2.3 From bf4ff8fe0be63422c05d42e12f25b454bb95d1a5 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Tue, 10 Jan 2017 22:33:00 +0100 Subject: Server: retry video abuse requests too --- server/controllers/api/videos.js | 68 +++++++++++++++++++++++++++++++++------- 1 file changed, 56 insertions(+), 12 deletions(-) (limited to 'server') diff --git a/server/controllers/api/videos.js b/server/controllers/api/videos.js index 4d45c11c0..6573b1210 100644 --- a/server/controllers/api/videos.js +++ b/server/controllers/api/videos.js @@ -56,7 +56,7 @@ router.get('/abuse', router.post('/:id/abuse', oAuth.authenticate, validatorsVideos.videoAbuseReport, - reportVideoAbuse + reportVideoAbuseRetryWrapper ) router.get('/', @@ -375,7 +375,23 @@ function listVideoAbuses (req, res, next) { }) } -function reportVideoAbuse (req, res, next) { +function reportVideoAbuseRetryWrapper (req, res, next) { + utils.transactionRetryer( + function (callback) { + return reportVideoAbuse(req, res, callback) + }, + function (err) { + if (err) { + logger.error('Cannot report abuse to the video with many retries.', { error: err }) + return next(err) + } + + return res.type('json').status(204).end() + } + ) +} + +function reportVideoAbuse (req, res, finalCallback) { const videoInstance = res.locals.video const reporterUsername = res.locals.oauth.token.User.username @@ -386,21 +402,49 @@ function reportVideoAbuse (req, res, next) { reporterPodId: null // This is our pod that reported this abuse } - db.VideoAbuse.create(abuse).asCallback(function (err) { - if (err) return next(err) + waterfall([ + + function startTransaction (callback) { + db.sequelize.transaction().asCallback(function (err, t) { + return callback(err, t) + }) + }, + + function createAbuse (t, callback) { + db.VideoAbuse.create(abuse).asCallback(function (err, abuse) { + return callback(err, t, abuse) + }) + }, + + function sendToFriendsIfNeeded (t, abuse, callback) { + // We send the information to the destination pod + if (videoInstance.isOwned() === false) { + const reportData = { + reporterUsername, + reportReason: abuse.reason, + videoRemoteId: videoInstance.remoteId + } - // We send the information to the destination pod - if (videoInstance.isOwned() === false) { - const reportData = { - reporterUsername, - reportReason: abuse.reason, - videoRemoteId: videoInstance.remoteId + friends.reportAbuseVideoToFriend(reportData, videoInstance) } - friends.reportAbuseVideoToFriend(reportData, videoInstance) + return callback(null, t) } - return res.type('json').status(204).end() + ], function andFinally (err, t) { + if (err) { + logger.debug('Cannot update the video.', { error: err }) + + // Abort transaction? + if (t) t.rollback() + + return finalCallback(err) + } + + // Commit transaction + t.commit() + + return finalCallback(null) }) } -- cgit v1.2.3 From dea32aacde362a5fbd62a88cd32487768b788468 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Wed, 11 Jan 2017 16:22:50 +0100 Subject: Server: always check commit result --- server/controllers/api/remote/videos.js | 16 +++++++++++----- server/controllers/api/videos.js | 23 +++++++++++++++-------- server/initializers/migrator.js | 2 +- 3 files changed, 27 insertions(+), 14 deletions(-) (limited to 'server') diff --git a/server/controllers/api/remote/videos.js b/server/controllers/api/remote/videos.js index 6d768eae8..17bdce019 100644 --- a/server/controllers/api/remote/videos.js +++ b/server/controllers/api/remote/videos.js @@ -171,9 +171,12 @@ function addRemoteVideo (videoToCreateData, fromPod, finalCallback) { } // Commit transaction - t.commit() + t.commit().asCallback(function (err) { + if (err) return finalCallback(err) - return finalCallback() + logger.info('Remote video %s inserted.', videoToCreateData.videoToCreateData.name) + return finalCallback(null) + }) }) } @@ -254,10 +257,13 @@ function updateRemoteVideo (videoAttributesToUpdate, fromPod, finalCallback) { return finalCallback(err) } - // Commit transaction - t.commit() + // Commit transaction + t.commit().asCallback(function (err) { + if (err) return finalCallback(err) - return finalCallback() + logger.info('Remote video %s updated', videoAttributesToUpdate.name) + return finalCallback(null) + }) }) } diff --git a/server/controllers/api/videos.js b/server/controllers/api/videos.js index 6573b1210..df068f961 100644 --- a/server/controllers/api/videos.js +++ b/server/controllers/api/videos.js @@ -231,11 +231,12 @@ function addVideo (req, res, videoFile, callback) { } // Commit transaction - t.commit() + t.commit().asCallback(function (err) { + if (err) return callback(err) - logger.info('Video with name %s created.', videoInfos.name) - - return callback(null) + logger.info('Video with name %s created.', videoInfos.name) + return callback(null) + }) }) } @@ -324,9 +325,12 @@ function updateVideo (req, res, finalCallback) { } // Commit transaction - t.commit() + t.commit().asCallback(function (err) { + if (err) return finalCallback(err) - return finalCallback(null) + logger.info('Video with name %s updated.', videoInfosToUpdate.name) + return finalCallback(null) + }) }) } @@ -442,9 +446,12 @@ function reportVideoAbuse (req, res, finalCallback) { } // Commit transaction - t.commit() + t.commit().asCallback(function (err) { + if (err) return finalCallback(err) - return finalCallback(null) + logger.info('Abuse report for video %s created.', videoInstance.name) + return finalCallback(null) + }) }) } diff --git a/server/initializers/migrator.js b/server/initializers/migrator.js index eaecb4936..e5288b615 100644 --- a/server/initializers/migrator.js +++ b/server/initializers/migrator.js @@ -91,7 +91,7 @@ function executeMigration (actualVersion, entity, callback) { return callback(err) } - t.commit() + t.commit().asCallback(callback) }) }) }) -- cgit v1.2.3 From d8cc063e9775688a1631eda9203411a2dba0333c Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Wed, 11 Jan 2017 18:06:51 +0100 Subject: Server: do not break remote videos processing on error --- server/controllers/api/remote/videos.js | 33 ++++++++++++++++++++++++--------- server/models/request.js | 4 ++-- server/tests/real-world/real-world.js | 2 +- 3 files changed, 27 insertions(+), 12 deletions(-) (limited to 'server') diff --git a/server/controllers/api/remote/videos.js b/server/controllers/api/remote/videos.js index 17bdce019..b9494f602 100644 --- a/server/controllers/api/remote/videos.js +++ b/server/controllers/api/remote/videos.js @@ -73,10 +73,10 @@ function addRemoteVideoRetryWrapper (videoToCreateData, fromPod, finalCallback) function (err) { if (err) { logger.error('Cannot insert the remote video with many retries.', { error: err }) - return finalCallback(err) } - return finalCallback() + // Do not return the error, continue the process + return finalCallback(null) } ) } @@ -174,7 +174,7 @@ function addRemoteVideo (videoToCreateData, fromPod, finalCallback) { t.commit().asCallback(function (err) { if (err) return finalCallback(err) - logger.info('Remote video %s inserted.', videoToCreateData.videoToCreateData.name) + logger.info('Remote video %s inserted.', videoToCreateData.name) return finalCallback(null) }) }) @@ -189,10 +189,10 @@ function updateRemoteVideoRetryWrapper (videoAttributesToUpdate, fromPod, finalC function (err) { if (err) { logger.error('Cannot update the remote video with many retries.', { error: err }) - return finalCallback(err) } - return finalCallback() + // Do not return the error, continue the process + return finalCallback(null) } ) } @@ -270,10 +270,18 @@ function updateRemoteVideo (videoAttributesToUpdate, fromPod, finalCallback) { function removeRemoteVideo (videoToRemoveData, fromPod, callback) { // We need the instance because we have to remove some other stuffs (thumbnail etc) fetchVideo(fromPod.host, videoToRemoveData.remoteId, function (err, video) { - if (err) return callback(err) + // Do not return the error, continue the process + if (err) return callback(null) logger.debug('Removing remote video %s.', video.remoteId) - video.destroy().asCallback(callback) + video.destroy().asCallback(function (err) { + // Do not return the error, continue the process + if (err) { + logger.error('Cannot remove remote video with id %s.', videoToRemoveData.remoteId, { error: err }) + } + + return callback(null) + }) }) } @@ -283,7 +291,8 @@ function reportAbuseRemoteVideo (reportData, fromPod, callback) { if (!err) err = new Error('video not found') logger.error('Cannot load video from id.', { error: err, id: reportData.videoRemoteId }) - return callback(err) + // Do not return the error, continue the process + return callback(null) } logger.debug('Reporting remote abuse for video %s.', video.id) @@ -295,7 +304,13 @@ function reportAbuseRemoteVideo (reportData, fromPod, callback) { videoId: video.id } - db.VideoAbuse.create(videoAbuseData).asCallback(callback) + db.VideoAbuse.create(videoAbuseData).asCallback(function (err) { + if (err) { + logger.error('Cannot create remote abuse video.', { error: err }) + } + + return callback(null) + }) }) } diff --git a/server/models/request.js b/server/models/request.js index 26953e5f5..e048c288b 100644 --- a/server/models/request.js +++ b/server/models/request.js @@ -152,8 +152,6 @@ function makeRequests () { return } - logger.info('Making requests to friends.') - // We want to group requests by destinations pod and endpoint const requestsToMakeGrouped = {} Object.keys(requests).forEach(function (toPodId) { @@ -176,6 +174,8 @@ function makeRequests () { }) }) + logger.info('Making requests to friends.', { requests: requestsToMakeGrouped }) + const goodPods = [] const badPods = [] diff --git a/server/tests/real-world/real-world.js b/server/tests/real-world/real-world.js index 751d3923f..9a63860ad 100644 --- a/server/tests/real-world/real-world.js +++ b/server/tests/real-world/real-world.js @@ -36,7 +36,7 @@ const numberOfPods = 6 // Wait requests between pods const baseRequestInterval = integrityInterval < constants.REQUESTS_INTERVAL ? integrityInterval : constants.REQUESTS_INTERVAL const requestsMaxPerInterval = baseRequestInterval / actionInterval -const intervalsToMakeAllRequests = Math.ceil(requestsMaxPerInterval / constants.REQUESTS_LIMIT) +const intervalsToMakeAllRequests = Math.ceil(requestsMaxPerInterval / (constants.REQUESTS_LIMIT_PER_POD * numberOfPods)) const waitForBeforeIntegrityCheck = (intervalsToMakeAllRequests * constants.REQUESTS_INTERVAL) + 1000 console.log('Create weight: %d, remove weight: %d.', createWeight, removeWeight) -- cgit v1.2.3 From 45abb8b97b8313f8f58a4a73b527882ad7b4af9c Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Wed, 11 Jan 2017 18:41:09 +0100 Subject: Server: rights check for update a video --- server/middlewares/validators/videos.js | 8 ++++++++ server/tests/api/check-params/videos.js | 4 ++++ 2 files changed, 12 insertions(+) (limited to 'server') diff --git a/server/middlewares/validators/videos.js b/server/middlewares/validators/videos.js index ff18a99c2..3d7c04b60 100644 --- a/server/middlewares/validators/videos.js +++ b/server/middlewares/validators/videos.js @@ -53,6 +53,14 @@ function videosUpdate (req, res, next) { logger.debug('Checking videosUpdate parameters', { parameters: req.body }) checkErrors(req, res, function () { + if (res.locals.video.isOwned() === false) { + return res.status(403).send('Cannot update video of another pod') + } + + if (res.locals.video.Author.userId !== res.locals.oauth.token.User.id) { + return res.status(403).send('Cannot update video of another user') + } + checkVideoExists(req.params.id, res, next) }) } diff --git a/server/tests/api/check-params/videos.js b/server/tests/api/check-params/videos.js index d18305291..fac903715 100644 --- a/server/tests/api/check-params/videos.js +++ b/server/tests/api/check-params/videos.js @@ -378,6 +378,10 @@ describe('Test videos API validator', function () { } requestsUtils.makePutBodyRequest(server.url, path + videoId, server.accessToken, data, done) }) + + it('Should fail with a video of another user') + + it('Should fail with a video of another pod') }) describe('When getting a video', function () { -- cgit v1.2.3 From f2cdb86675c3783ee903640b5b6f794fa09cdff2 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Wed, 11 Jan 2017 18:41:28 +0100 Subject: Server: add update case to real world script --- server/tests/real-world/real-world.js | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) (limited to 'server') diff --git a/server/tests/real-world/real-world.js b/server/tests/real-world/real-world.js index 9a63860ad..896ba6cce 100644 --- a/server/tests/real-world/real-world.js +++ b/server/tests/real-world/real-world.js @@ -17,6 +17,7 @@ const videosUtils = require('../utils/videos') program .option('-c, --create [weight]', 'Weight for creating videos') .option('-r, --remove [weight]', 'Weight for removing videos') + .option('-u, --update [weight]', 'Weight for updating videos') .option('-p, --pods [n]', 'Number of pods to run (3 or 6)', /^3|6$/, 3) .option('-a, --action [interval]', 'Interval in ms for an action') .option('-i, --integrity [interval]', 'Interval in ms for an integrity check') @@ -26,6 +27,7 @@ program const createWeight = program.create !== undefined ? parseInt(program.create) : 5 const removeWeight = program.remove !== undefined ? parseInt(program.remove) : 4 +const updateWeight = program.update !== undefined ? parseInt(program.update) : 4 const flushAtExit = program.flush || false const actionInterval = program.action !== undefined ? parseInt(program.action) : 500 let integrityInterval = program.integrity !== undefined ? parseInt(program.integrity) : 60000 @@ -39,7 +41,7 @@ const requestsMaxPerInterval = baseRequestInterval / actionInterval const intervalsToMakeAllRequests = Math.ceil(requestsMaxPerInterval / (constants.REQUESTS_LIMIT_PER_POD * numberOfPods)) const waitForBeforeIntegrityCheck = (intervalsToMakeAllRequests * constants.REQUESTS_INTERVAL) + 1000 -console.log('Create weight: %d, remove weight: %d.', createWeight, removeWeight) +console.log('Create weight: %d, update weight: %d, remove weight: %d.', createWeight, updateWeight, removeWeight) if (flushAtExit) { console.log('Program will flush data on exit.') } else { @@ -71,10 +73,12 @@ runServers(numberOfPods, function (err, servers) { setInterval(function () { if (checking === true) return - const rand = getRandomInt(0, createWeight + removeWeight) + const rand = getRandomInt(0, createWeight + updateWeight + removeWeight) if (rand < createWeight) { upload(servers, getRandomNumServer(servers)) + } else if (rand < createWeight + updateWeight) { + update(servers, getRandomNumServer(servers)) } else { remove(servers, getRandomNumServer(servers)) } @@ -180,6 +184,26 @@ function upload (servers, numServer, callback) { videosUtils.uploadVideo(servers[numServer].url, servers[numServer].accessToken, name, description, tags, file, callback) } +function update (servers, numServer, callback) { + if (!callback) callback = function () {} + + videosUtils.getVideosList(servers[numServer].url, function (err, res) { + if (err) throw err + + const videos = res.body.data.filter(function (video) { return video.isLocal }) + if (videos.length === 0) return callback() + + const toUpdate = videos[getRandomInt(0, videos.length)].id + const name = Date.now() + ' name' + const description = Date.now() + ' description' + const tags = [ Date.now().toString().substring(0, 5) + 't1', Date.now().toString().substring(0, 5) + 't2' ] + + console.log('Updating video of server ' + numServer) + + videosUtils.updateVideo(servers[numServer].url, servers[numServer].accessToken, toUpdate, name, description, tags, callback) + }) +} + function remove (servers, numServer, callback) { if (!callback) callback = function () {} -- cgit v1.2.3 From edc5e86006bf5e4a2819c380bb65734fe9caa87e Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Wed, 11 Jan 2017 18:41:40 +0100 Subject: Server: transaction serializable for videos --- server/controllers/api/remote/videos.js | 2 +- server/controllers/api/videos.js | 2 +- server/models/request.js | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) (limited to 'server') diff --git a/server/controllers/api/remote/videos.js b/server/controllers/api/remote/videos.js index b9494f602..c45a86dbb 100644 --- a/server/controllers/api/remote/videos.js +++ b/server/controllers/api/remote/videos.js @@ -203,7 +203,7 @@ function updateRemoteVideo (videoAttributesToUpdate, fromPod, finalCallback) { waterfall([ function startTransaction (callback) { - db.sequelize.transaction().asCallback(function (err, t) { + db.sequelize.transaction({ isolationLevel: 'SERIALIZABLE' }).asCallback(function (err, t) { return callback(err, t) }) }, diff --git a/server/controllers/api/videos.js b/server/controllers/api/videos.js index df068f961..55d671f5b 100644 --- a/server/controllers/api/videos.js +++ b/server/controllers/api/videos.js @@ -264,7 +264,7 @@ function updateVideo (req, res, finalCallback) { waterfall([ function startTransaction (callback) { - db.sequelize.transaction().asCallback(function (err, t) { + db.sequelize.transaction({ isolationLevel: 'SERIALIZABLE' }).asCallback(function (err, t) { return callback(err, t) }) }, diff --git a/server/models/request.js b/server/models/request.js index e048c288b..cd52ea767 100644 --- a/server/models/request.js +++ b/server/models/request.js @@ -174,7 +174,7 @@ function makeRequests () { }) }) - logger.info('Making requests to friends.', { requests: requestsToMakeGrouped }) + logger.info('Making requests to friends.') const goodPods = [] const badPods = [] -- cgit v1.2.3 From 63d00f5ded0aad25eeb50111da65b6daa46bcb24 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Wed, 11 Jan 2017 19:15:23 +0100 Subject: Server: fix update right checks --- server/middlewares/validators/videos.js | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) (limited to 'server') diff --git a/server/middlewares/validators/videos.js b/server/middlewares/validators/videos.js index 3d7c04b60..4fe6dcd8b 100644 --- a/server/middlewares/validators/videos.js +++ b/server/middlewares/validators/videos.js @@ -53,15 +53,18 @@ function videosUpdate (req, res, next) { logger.debug('Checking videosUpdate parameters', { parameters: req.body }) checkErrors(req, res, function () { - if (res.locals.video.isOwned() === false) { - return res.status(403).send('Cannot update video of another pod') - } + checkVideoExists(req.params.id, res, function () { + // We need to make additional checks + if (res.locals.video.isOwned() === false) { + return res.status(403).send('Cannot update video of another pod') + } - if (res.locals.video.Author.userId !== res.locals.oauth.token.User.id) { - return res.status(403).send('Cannot update video of another user') - } + if (res.locals.video.Author.userId !== res.locals.oauth.token.User.id) { + return res.status(403).send('Cannot update video of another user') + } - checkVideoExists(req.params.id, res, next) + next() + }) }) } -- cgit v1.2.3 From 7f4e7c36373217b8e92cf227c71999a0ce9a15d9 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Thu, 12 Jan 2017 09:47:21 +0100 Subject: Server: fix update remote video infohash --- server/controllers/api/videos.js | 14 ++++++++++++-- server/models/video.js | 3 ++- server/tests/api/multiple-pods.js | 29 +++++++++++++++++++++++------ server/tests/api/single-pod.js | 19 +++++++++++++++++-- server/tests/real-world/real-world.js | 2 +- 5 files changed, 55 insertions(+), 12 deletions(-) (limited to 'server') diff --git a/server/controllers/api/videos.js b/server/controllers/api/videos.js index 55d671f5b..2c4af520e 100644 --- a/server/controllers/api/videos.js +++ b/server/controllers/api/videos.js @@ -259,6 +259,7 @@ function updateVideoRetryWrapper (req, res, next) { function updateVideo (req, res, finalCallback) { const videoInstance = res.locals.video + const videoFieldsSave = videoInstance.toJSON() const videoInfosToUpdate = req.body waterfall([ @@ -280,12 +281,13 @@ function updateVideo (req, res, finalCallback) { }, function updateVideoIntoDB (t, tagInstances, callback) { - const options = { transaction: t } + const options = { + transaction: t + } if (videoInfosToUpdate.name) videoInstance.set('name', videoInfosToUpdate.name) if (videoInfosToUpdate.description) videoInstance.set('description', videoInfosToUpdate.description) - // Add tags association videoInstance.save(options).asCallback(function (err) { return callback(err, t, tagInstances) }) @@ -321,6 +323,14 @@ function updateVideo (req, res, finalCallback) { // Abort transaction? if (t) t.rollback() + // Force fields we want to update + // If the transaction is retried, sequelize will think the object has not changed + // So it will skip the SQL request, even if the last one was ROLLBACKed! + Object.keys(videoFieldsSave).forEach(function (key) { + const value = videoFieldsSave[key] + videoInstance.set(key, value) + }) + return finalCallback(err) } diff --git a/server/models/video.js b/server/models/video.js index b3060705d..ceed976b0 100644 --- a/server/models/video.js +++ b/server/models/video.js @@ -141,7 +141,8 @@ module.exports = function (sequelize, DataTypes) { } function beforeValidate (video, options, next) { - if (video.isOwned()) { + // Put a fake infoHash if it does not exists yet + if (video.isOwned() && !video.infoHash) { // 40 hexa length video.infoHash = '0123456789abcdef0123456789abcdef01234567' } diff --git a/server/tests/api/multiple-pods.js b/server/tests/api/multiple-pods.js index 4442a7ff7..169a9f2e0 100644 --- a/server/tests/api/multiple-pods.js +++ b/server/tests/api/multiple-pods.js @@ -4,7 +4,8 @@ const chai = require('chai') const each = require('async/each') const expect = chai.expect const series = require('async/series') -const webtorrent = new (require('webtorrent'))() +const WebTorrent = require('webtorrent') +const webtorrent = new WebTorrent() const loginUtils = require('../utils/login') const miscsUtils = require('../utils/miscs') @@ -311,7 +312,7 @@ describe('Test multiple pods', function () { expect(torrent.files.length).to.equal(1) expect(torrent.files[0].path).to.exist.and.to.not.equal('') - done() + webtorrent.remove(video.magnetUri, done) }) }) }) @@ -330,7 +331,7 @@ describe('Test multiple pods', function () { expect(torrent.files.length).to.equal(1) expect(torrent.files[0].path).to.exist.and.to.not.equal('') - done() + webtorrent.remove(video.magnetUri, done) }) }) }) @@ -349,7 +350,7 @@ describe('Test multiple pods', function () { expect(torrent.files.length).to.equal(1) expect(torrent.files[0].path).to.exist.and.to.not.equal('') - done() + webtorrent.remove(video.magnetUri, done) }) }) }) @@ -368,7 +369,7 @@ describe('Test multiple pods', function () { expect(torrent.files.length).to.equal(1) expect(torrent.files[0].path).to.exist.and.to.not.equal('') - done() + webtorrent.remove(video.magnetUri, done) }) }) }) @@ -390,7 +391,12 @@ describe('Test multiple pods', function () { }) it('Should have the video 3 updated on each pod', function (done) { + this.timeout(200000) + each(servers, function (server, callback) { + // Avoid "duplicate torrent" errors + const webtorrent = new WebTorrent() + videosUtils.getVideosList(server.url, function (err, res) { if (err) throw err @@ -404,7 +410,18 @@ describe('Test multiple pods', function () { expect(videoUpdated.tags).to.deep.equal([ 'tagup1', 'tagup2' ]) expect(miscsUtils.dateIsValid(videoUpdated.updatedAt, 20000)).to.be.true - callback() + videosUtils.testVideoImage(server.url, 'video_short3.webm', videoUpdated.thumbnailPath, function (err, test) { + if (err) throw err + expect(test).to.equal(true) + + webtorrent.add(videoUpdated.magnetUri, function (torrent) { + expect(torrent.files).to.exist + expect(torrent.files.length).to.equal(1) + expect(torrent.files[0].path).to.exist.and.to.not.equal('') + + webtorrent.remove(videoUpdated.magnetUri, callback) + }) + }) }) }, done) }) diff --git a/server/tests/api/single-pod.js b/server/tests/api/single-pod.js index 29512dfc6..04b93fac7 100644 --- a/server/tests/api/single-pod.js +++ b/server/tests/api/single-pod.js @@ -96,7 +96,7 @@ describe('Test a single pod', function () { expect(torrent.files.length).to.equal(1) expect(torrent.files[0].path).to.exist.and.to.not.equal('') - done() + webtorrent.remove(video.magnetUri, done) }) }) }) @@ -515,6 +515,8 @@ describe('Test a single pod', function () { }) it('Should have the video updated', function (done) { + this.timeout(60000) + videosUtils.getVideo(server.url, videoId, function (err, res) { if (err) throw err @@ -529,7 +531,20 @@ describe('Test a single pod', function () { expect(miscsUtils.dateIsValid(video.createdAt)).to.be.true expect(miscsUtils.dateIsValid(video.updatedAt)).to.be.true - done() + videosUtils.testVideoImage(server.url, 'video_short3.webm', video.thumbnailPath, function (err, test) { + if (err) throw err + expect(test).to.equal(true) + + videoId = video.id + + webtorrent.add(video.magnetUri, function (torrent) { + expect(torrent.files).to.exist + expect(torrent.files.length).to.equal(1) + expect(torrent.files[0].path).to.exist.and.to.not.equal('') + + done() + }) + }) }) }) diff --git a/server/tests/real-world/real-world.js b/server/tests/real-world/real-world.js index 896ba6cce..941e43a2e 100644 --- a/server/tests/real-world/real-world.js +++ b/server/tests/real-world/real-world.js @@ -38,7 +38,7 @@ const numberOfPods = 6 // Wait requests between pods const baseRequestInterval = integrityInterval < constants.REQUESTS_INTERVAL ? integrityInterval : constants.REQUESTS_INTERVAL const requestsMaxPerInterval = baseRequestInterval / actionInterval -const intervalsToMakeAllRequests = Math.ceil(requestsMaxPerInterval / (constants.REQUESTS_LIMIT_PER_POD * numberOfPods)) +const intervalsToMakeAllRequests = Math.ceil(requestsMaxPerInterval / constants.REQUESTS_LIMIT_PER_POD) const waitForBeforeIntegrityCheck = (intervalsToMakeAllRequests * constants.REQUESTS_INTERVAL) + 1000 console.log('Create weight: %d, update weight: %d, remove weight: %d.', createWeight, updateWeight, removeWeight) -- cgit v1.2.3 From 790e65fcf7a0a9f065ecc68c5982efb80cd2e1ca Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Thu, 12 Jan 2017 10:06:03 +0100 Subject: Try to fix travis build --- server/tests/api/multiple-pods.js | 10 +++++----- server/tests/api/single-pod.js | 2 +- server/tests/real-world/real-world.js | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) (limited to 'server') diff --git a/server/tests/api/multiple-pods.js b/server/tests/api/multiple-pods.js index 169a9f2e0..df12ba0e9 100644 --- a/server/tests/api/multiple-pods.js +++ b/server/tests/api/multiple-pods.js @@ -312,7 +312,7 @@ describe('Test multiple pods', function () { expect(torrent.files.length).to.equal(1) expect(torrent.files[0].path).to.exist.and.to.not.equal('') - webtorrent.remove(video.magnetUri, done) + done() }) }) }) @@ -331,7 +331,7 @@ describe('Test multiple pods', function () { expect(torrent.files.length).to.equal(1) expect(torrent.files[0].path).to.exist.and.to.not.equal('') - webtorrent.remove(video.magnetUri, done) + done() }) }) }) @@ -350,7 +350,7 @@ describe('Test multiple pods', function () { expect(torrent.files.length).to.equal(1) expect(torrent.files[0].path).to.exist.and.to.not.equal('') - webtorrent.remove(video.magnetUri, done) + done() }) }) }) @@ -369,7 +369,7 @@ describe('Test multiple pods', function () { expect(torrent.files.length).to.equal(1) expect(torrent.files[0].path).to.exist.and.to.not.equal('') - webtorrent.remove(video.magnetUri, done) + done() }) }) }) @@ -419,7 +419,7 @@ describe('Test multiple pods', function () { expect(torrent.files.length).to.equal(1) expect(torrent.files[0].path).to.exist.and.to.not.equal('') - webtorrent.remove(videoUpdated.magnetUri, callback) + callback() }) }) }) diff --git a/server/tests/api/single-pod.js b/server/tests/api/single-pod.js index 04b93fac7..2ac83bbf4 100644 --- a/server/tests/api/single-pod.js +++ b/server/tests/api/single-pod.js @@ -96,7 +96,7 @@ describe('Test a single pod', function () { expect(torrent.files.length).to.equal(1) expect(torrent.files[0].path).to.exist.and.to.not.equal('') - webtorrent.remove(video.magnetUri, done) + done() }) }) }) diff --git a/server/tests/real-world/real-world.js b/server/tests/real-world/real-world.js index 941e43a2e..12ab06d6d 100644 --- a/server/tests/real-world/real-world.js +++ b/server/tests/real-world/real-world.js @@ -30,16 +30,16 @@ const removeWeight = program.remove !== undefined ? parseInt(program.remove) : 4 const updateWeight = program.update !== undefined ? parseInt(program.update) : 4 const flushAtExit = program.flush || false const actionInterval = program.action !== undefined ? parseInt(program.action) : 500 -let integrityInterval = program.integrity !== undefined ? parseInt(program.integrity) : 60000 +const integrityInterval = program.integrity !== undefined ? parseInt(program.integrity) : 60000 const displayDiffOnFail = program.integrity || false const numberOfPods = 6 // Wait requests between pods -const baseRequestInterval = integrityInterval < constants.REQUESTS_INTERVAL ? integrityInterval : constants.REQUESTS_INTERVAL +const baseRequestInterval = integrityInterval < constants.REQUESTS_INTERVAL ? constants.REQUESTS_INTERVAL : integrityInterval const requestsMaxPerInterval = baseRequestInterval / actionInterval const intervalsToMakeAllRequests = Math.ceil(requestsMaxPerInterval / constants.REQUESTS_LIMIT_PER_POD) -const waitForBeforeIntegrityCheck = (intervalsToMakeAllRequests * constants.REQUESTS_INTERVAL) + 1000 +const waitForBeforeIntegrityCheck = (intervalsToMakeAllRequests * constants.REQUESTS_INTERVAL) - integrityInterval + 1000 console.log('Create weight: %d, update weight: %d, remove weight: %d.', createWeight, updateWeight, removeWeight) if (flushAtExit) { -- cgit v1.2.3 From 91cc839af88730ba55f84997c56b85ea100070a7 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Thu, 12 Jan 2017 13:08:47 +0100 Subject: Server: fix single pod tests --- server/models/video.js | 2 +- server/tests/api/single-pod.js | 52 +++++++++++++++++++--------------------- server/tests/utils/videos.js | 54 +++++++++++++++++++++++++++--------------- 3 files changed, 61 insertions(+), 47 deletions(-) (limited to 'server') diff --git a/server/models/video.js b/server/models/video.js index ceed976b0..17eff6428 100644 --- a/server/models/video.js +++ b/server/models/video.js @@ -335,7 +335,7 @@ function toFormatedJSON () { author: this.Author.name, duration: this.duration, tags: map(this.Tags, 'name'), - thumbnailPath: constants.STATIC_PATHS.THUMBNAILS + '/' + this.getThumbnailName(), + thumbnailPath: pathUtils.join(constants.STATIC_PATHS.THUMBNAILS, this.getThumbnailName()), createdAt: this.createdAt, updatedAt: this.updatedAt } diff --git a/server/tests/api/single-pod.js b/server/tests/api/single-pod.js index 2ac83bbf4..2db60448f 100644 --- a/server/tests/api/single-pod.js +++ b/server/tests/api/single-pod.js @@ -338,69 +338,69 @@ describe('Test a single pod', function () { }) it('Should list only the two first videos', function (done) { - videosUtils.getVideosListPagination(server.url, 0, 2, function (err, res) { + videosUtils.getVideosListPagination(server.url, 0, 2, 'name', function (err, res) { if (err) throw err const videos = res.body.data expect(res.body.total).to.equal(6) expect(videos.length).to.equal(2) - expect(videos[0].name === videosListBase[0].name) - expect(videos[1].name === videosListBase[1].name) + expect(videos[0].name).to.equal(videosListBase[0].name) + expect(videos[1].name).to.equal(videosListBase[1].name) done() }) }) it('Should list only the next three videos', function (done) { - videosUtils.getVideosListPagination(server.url, 2, 3, function (err, res) { + videosUtils.getVideosListPagination(server.url, 2, 3, 'name', function (err, res) { if (err) throw err const videos = res.body.data expect(res.body.total).to.equal(6) expect(videos.length).to.equal(3) - expect(videos[0].name === videosListBase[2].name) - expect(videos[1].name === videosListBase[3].name) - expect(videos[2].name === videosListBase[4].name) + expect(videos[0].name).to.equal(videosListBase[2].name) + expect(videos[1].name).to.equal(videosListBase[3].name) + expect(videos[2].name).to.equal(videosListBase[4].name) done() }) }) it('Should list the last video', function (done) { - videosUtils.getVideosListPagination(server.url, 5, 6, function (err, res) { + videosUtils.getVideosListPagination(server.url, 5, 6, 'name', function (err, res) { if (err) throw err const videos = res.body.data expect(res.body.total).to.equal(6) expect(videos.length).to.equal(1) - expect(videos[0].name === videosListBase[5].name) + expect(videos[0].name).to.equal(videosListBase[5].name) done() }) }) it('Should search the first video', function (done) { - videosUtils.searchVideoWithPagination(server.url, 'webm', 'name', 0, 1, function (err, res) { + videosUtils.searchVideoWithPagination(server.url, 'webm', 'name', 0, 1, 'name', function (err, res) { if (err) throw err const videos = res.body.data expect(res.body.total).to.equal(4) expect(videos.length).to.equal(1) - expect(videos[0].name === 'video_short.webm name') + expect(videos[0].name).to.equal('video_short1.webm name') done() }) }) it('Should search the last two videos', function (done) { - videosUtils.searchVideoWithPagination(server.url, 'webm', 'name', 2, 2, function (err, res) { + videosUtils.searchVideoWithPagination(server.url, 'webm', 'name', 2, 2, 'name', function (err, res) { if (err) throw err const videos = res.body.data expect(res.body.total).to.equal(4) expect(videos.length).to.equal(2) - expect(videos[0].name === 'video_short2.webm name') - expect(videos[1].name === 'video_short3.webm name') + expect(videos[0].name).to.equal('video_short3.webm name') + expect(videos[1].name).to.equal('video_short.webm name') done() }) @@ -476,12 +476,12 @@ describe('Test a single pod', function () { const videos = res.body.data expect(res.body.total).to.equal(6) expect(videos.length).to.equal(6) - expect(videos[5].name === 'video_short.mp4 name') - expect(videos[4].name === 'video_short.ogv name') - expect(videos[3].name === 'video_short.webm name') - expect(videos[2].name === 'video_short1.webm name') - expect(videos[1].name === 'video_short2.webm name') - expect(videos[0].name === 'video_short3.webm name') + expect(videos[0].name).to.equal('video_short.webm name') + expect(videos[1].name).to.equal('video_short.ogv name') + expect(videos[2].name).to.equal('video_short.mp4 name') + expect(videos[3].name).to.equal('video_short3.webm name') + expect(videos[4].name).to.equal('video_short2.webm name') + expect(videos[5].name).to.equal('video_short1.webm name') done() }) @@ -495,12 +495,12 @@ describe('Test a single pod', function () { expect(res.body.total).to.equal(4) expect(videos.length).to.equal(4) - expect(videos[0].name === 'video_short.webm name') - expect(videos[1].name === 'video_short1.webm name') - expect(videos[2].name === 'video_short2.webm name') - expect(videos[3].name === 'video_short3.webm name') + expect(videos[0].name).to.equal('video_short1.webm name') + expect(videos[1].name).to.equal('video_short2.webm name') + expect(videos[2].name).to.equal('video_short3.webm name') + expect(videos[3].name).to.equal('video_short.webm name') - videoId = videos[3].id + videoId = videos[2].id done() }) @@ -535,8 +535,6 @@ describe('Test a single pod', function () { if (err) throw err expect(test).to.equal(true) - videoId = video.id - webtorrent.add(video.magnetUri, function (torrent) { expect(torrent.files).to.exist expect(torrent.files.length).to.equal(1) diff --git a/server/tests/utils/videos.js b/server/tests/utils/videos.js index beafd3cf5..f94368437 100644 --- a/server/tests/utils/videos.js +++ b/server/tests/utils/videos.js @@ -58,17 +58,25 @@ function getVideosList (url, end) { .end(end) } -function getVideosListPagination (url, start, count, end) { +function getVideosListPagination (url, start, count, sort, end) { + if (!end) { + end = sort + sort = null + } + const path = '/api/v1/videos' - request(url) - .get(path) - .query({ start: start }) - .query({ count: count }) - .set('Accept', 'application/json') - .expect(200) - .expect('Content-Type', /json/) - .end(end) + const req = request(url) + .get(path) + .query({ start: start }) + .query({ count: count }) + + if (sort) req.query({ sort }) + + req.set('Accept', 'application/json') + .expect(200) + .expect('Content-Type', /json/) + .end(end) } function getVideosListSort (url, sort, end) { @@ -116,18 +124,26 @@ function searchVideo (url, search, field, end) { .end(end) } -function searchVideoWithPagination (url, search, field, start, count, end) { +function searchVideoWithPagination (url, search, field, start, count, sort, end) { + if (!end) { + end = sort + sort = null + } + const path = '/api/v1/videos' - request(url) - .get(path + '/search/' + search) - .query({ start: start }) - .query({ count: count }) - .query({ field: field }) - .set('Accept', 'application/json') - .expect(200) - .expect('Content-Type', /json/) - .end(end) + const req = request(url) + .get(path + '/search/' + search) + .query({ start: start }) + .query({ count: count }) + .query({ field: field }) + + if (sort) req.query({ sort }) + + req.set('Accept', 'application/json') + .expect(200) + .expect('Content-Type', /json/) + .end(end) } function searchVideoWithSort (url, search, sort, end) { -- cgit v1.2.3