From aaf61f3810e6d57c5130af959bd2860df32775e7 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Fri, 24 Jun 2016 17:42:51 +0200 Subject: Video model refractoring -> use mongoose api --- server.js | 5 +- server/controllers/api/v1/pods.js | 17 +- server/controllers/api/v1/remote.js | 45 ++--- server/controllers/api/v1/videos.js | 124 +++--------- server/helpers/customValidators.js | 11 +- server/initializers/constants.js | 3 +- server/initializers/database.js | 3 + server/lib/friends.js | 21 +- server/lib/requestsScheduler.js | 18 +- server/lib/videos.js | 199 ------------------ server/middlewares/reqValidators/remote.js | 2 +- server/middlewares/reqValidators/videos.js | 20 +- server/models/video.js | 314 +++++++++++++++++++++++++++++ server/models/videos.js | 162 --------------- server/tests/api/multiplePods.js | 2 +- 15 files changed, 412 insertions(+), 534 deletions(-) delete mode 100644 server/lib/videos.js create mode 100644 server/models/video.js delete mode 100644 server/models/videos.js diff --git a/server.js b/server.js index 4c8e8cfd3..63aeb7145 100644 --- a/server.js +++ b/server.js @@ -31,7 +31,6 @@ const logger = require('./server/helpers/logger') const poolRequests = require('./server/lib/requestsScheduler') const routes = require('./server/controllers') const utils = require('./server/helpers/utils') -const videos = require('./server/lib/videos') const webtorrent = require('./server/lib/webtorrent') // Get configurations @@ -138,11 +137,11 @@ installer.installApplication(function (err) { // Activate the pool requests poolRequests.activate() - videos.seedAllExisting(function () { + // videos.seedAllExisting(function () { logger.info('Seeded all the videos') logger.info('Server listening on port %d', port) app.emit('ready') - }) + // }) }) }) }) diff --git a/server/controllers/api/v1/pods.js b/server/controllers/api/v1/pods.js index 881b2090d..9dd9197b3 100644 --- a/server/controllers/api/v1/pods.js +++ b/server/controllers/api/v1/pods.js @@ -2,6 +2,7 @@ const async = require('async') const express = require('express') +const mongoose = require('mongoose') const logger = require('../../../helpers/logger') const friends = require('../../../lib/friends') @@ -10,10 +11,9 @@ const Pods = require('../../../models/pods') const oAuth2 = middlewares.oauth2 const reqValidator = middlewares.reqValidators.pods const signatureValidator = middlewares.reqValidators.remote.signature -const videos = require('../../../lib/videos') -const Videos = require('../../../models/videos') const router = express.Router() +const Video = mongoose.model('Video') router.get('/', listPodsUrl) router.post('/', reqValidator.podsAdd, addPods) @@ -86,7 +86,7 @@ function removePods (req, res, next) { }, function (callback) { - Videos.listFromUrl(url, function (err, videosList) { + Video.listByUrls([ url ], function (err, videosList) { if (err) { logger.error('Cannot list videos from url.', { error: err }) return callback(err) @@ -97,14 +97,9 @@ function removePods (req, res, next) { }, function removeTheRemoteVideos (videosList, callback) { - videos.removeRemoteVideos(videosList, function (err) { - if (err) { - logger.error('Cannot remove remote videos.', { error: err }) - return callback(err) - } - - return callback(null) - }) + async.each(videosList, function (video, callbackEach) { + video.remove(callbackEach) + }, callback) } ], function (err) { if (err) return next(err) diff --git a/server/controllers/api/v1/remote.js b/server/controllers/api/v1/remote.js index ced8470d7..2d71c605d 100644 --- a/server/controllers/api/v1/remote.js +++ b/server/controllers/api/v1/remote.js @@ -2,15 +2,15 @@ const async = require('async') const express = require('express') +const mongoose = require('mongoose') const middlewares = require('../../../middlewares') const secureMiddleware = middlewares.secure const reqValidator = middlewares.reqValidators.remote const logger = require('../../../helpers/logger') -const Videos = require('../../../models/videos') -const videos = require('../../../lib/videos') const router = express.Router() +const Video = mongoose.model('Video') router.post('/videos', reqValidator.signature, @@ -33,48 +33,39 @@ function remoteVideos (req, res, next) { // We need to process in the same order to keep consistency // TODO: optimization async.eachSeries(requests, function (request, callbackEach) { - const video = request.data + const videoData = request.data if (request.type === 'add') { - addRemoteVideo(video, callbackEach) + addRemoteVideo(videoData, callbackEach) } else if (request.type === 'remove') { - removeRemoteVideo(video, fromUrl, callbackEach) + removeRemoteVideo(videoData, fromUrl, callbackEach) } + }, function (err) { + if (err) logger.error('Error managing remote videos.', { error: err }) }) // We don't need to keep the other pod waiting return res.type('json').status(204).end() } -function addRemoteVideo (videoToCreate, callback) { - videos.createRemoteVideos([ videoToCreate ], function (err, remoteVideos) { - if (err) { - logger.error('Cannot create remote videos.', { error: err }) - // Don't break the process - } +function addRemoteVideo (videoToCreateData, callback) { + // Mongoose pre hook will automatically create the thumbnail on disk + videoToCreateData.thumbnail = videoToCreateData.thumbnailBase64 - return callback() - }) + const video = new Video(videoToCreateData) + video.save(callback) } -function removeRemoteVideo (videoToRemove, fromUrl, callback) { - const magnetUris = [ videoToRemove.magnetUri ] - +function removeRemoteVideo (videoToRemoveData, fromUrl, callback) { // We need the list because we have to remove some other stuffs (thumbnail etc) - Videos.listFromUrlAndMagnets(fromUrl, magnetUris, function (err, videosList) { + Video.listByUrlAndMagnet(fromUrl, videoToRemoveData.magnetUri, function (err, videosList) { if (err) { logger.error('Cannot list videos from url and magnets.', { error: err }) - // Don't break the process - return callback() + return callback(err) } - videos.removeRemoteVideos(videosList, function (err) { - if (err) { - logger.error('Cannot remove remote videos.', { error: err }) - // Don't break the process - } - - return callback() - }) + async.each(videosList, function (video, callbackEach) { + video.remove(callbackEach) + }, callback) }) } diff --git a/server/controllers/api/v1/videos.js b/server/controllers/api/v1/videos.js index 2edb31122..83734b35e 100644 --- a/server/controllers/api/v1/videos.js +++ b/server/controllers/api/v1/videos.js @@ -3,9 +3,9 @@ const async = require('async') const config = require('config') const express = require('express') +const mongoose = require('mongoose') const multer = require('multer') -const constants = require('../../../initializers/constants') const logger = require('../../../helpers/logger') const friends = require('../../../lib/friends') const middlewares = require('../../../middlewares') @@ -18,12 +18,10 @@ const reqValidatorVideos = reqValidator.videos const search = middlewares.search const sort = middlewares.sort const utils = require('../../../helpers/utils') -const Videos = require('../../../models/videos') // model -const videos = require('../../../lib/videos') -const webtorrent = require('../../../lib/webtorrent') const router = express.Router() const uploads = config.get('storage.uploads') +const Video = mongoose.model('Video') // multer configuration const storage = multer.diskStorage({ @@ -88,55 +86,27 @@ function addVideo (req, res, next) { const videoInfos = req.body async.waterfall([ - function seedTheVideo (callback) { - videos.seed(videoFile.path, callback) - }, - - function createThumbnail (torrent, callback) { - videos.createVideoThumbnail(videoFile.path, function (err, thumbnailName) { - if (err) { - // TODO: unseed the video - logger.error('Cannot make a thumbnail of the video file.') - return callback(err) - } - - callback(null, torrent, thumbnailName) - }) - }, - function insertIntoDB (torrent, thumbnailName, callback) { + function insertIntoDB (callback) { const videoData = { name: videoInfos.name, namePath: videoFile.filename, description: videoInfos.description, - magnetUri: torrent.magnetURI, author: res.locals.oauth.token.user.username, duration: videoFile.duration, - thumbnail: thumbnailName, tags: videoInfos.tags } - Videos.add(videoData, function (err, insertedVideo) { - if (err) { - // TODO unseed the video - // TODO remove thumbnail - logger.error('Cannot insert this video in the database.') - return callback(err) - } - - return callback(null, insertedVideo) + const video = new Video(videoData) + video.save(function (err, video) { + // Assert there are only one argument sent to the next function (video) + return callback(err, video) }) }, - function sendToFriends (insertedVideo, callback) { - videos.convertVideoToRemote(insertedVideo, function (err, remoteVideo) { - if (err) { - // TODO unseed the video - // TODO remove thumbnail - // TODO delete from DB - logger.error('Cannot convert video to remote.') - return callback(err) - } + function sendToFriends (video, callback) { + video.toRemoteJSON(function (err, remoteVideo) { + if (err) return callback(err) // Now we'll add the video's meta data to our friends friends.addVideoToFriends(remoteVideo) @@ -147,6 +117,9 @@ function addVideo (req, res, next) { ], function andFinally (err) { if (err) { + // TODO unseed the video + // TODO remove thumbnail + // TODO delete from DB logger.error('Cannot insert the video.') return next(err) } @@ -157,23 +130,22 @@ function addVideo (req, res, next) { } function getVideo (req, res, next) { - Videos.get(req.params.id, function (err, videoObj) { + Video.load(req.params.id, function (err, video) { if (err) return next(err) - const state = videos.getVideoState(videoObj) - if (state.exist === false) { + if (!video) { return res.type('json').status(204).end() } - res.json(getFormatedVideo(videoObj)) + res.json(video.toFormatedJSON()) }) } function listVideos (req, res, next) { - Videos.list(req.query.start, req.query.count, req.query.sort, function (err, videosList, totalVideos) { + Video.list(req.query.start, req.query.count, req.query.sort, function (err, videosList, videosTotal) { if (err) return next(err) - res.json(getFormatedVideos(videosList, totalVideos)) + res.json(getFormatedVideos(videosList, videosTotal)) }) } @@ -182,31 +154,17 @@ function removeVideo (req, res, next) { async.waterfall([ function getVideo (callback) { - Videos.get(videoId, callback) - }, - - function removeVideoTorrent (video, callback) { - removeTorrent(video.magnetUri, function () { - return callback(null, video) - }) + Video.load(videoId, callback) }, function removeFromDB (video, callback) { - Videos.removeOwned(req.params.id, function (err) { + video.remove(function (err) { if (err) return callback(err) return callback(null, video) }) }, - function removeVideoData (video, callback) { - videos.removeVideosDataFromDisk([ video ], function (err) { - if (err) logger.error('Cannot remove video data from disk.', { video: video }) - - return callback(null, video) - }) - }, - function sendInformationToFriends (video, callback) { const params = { name: video.name, @@ -228,53 +186,25 @@ function removeVideo (req, res, next) { } function searchVideos (req, res, next) { - Videos.search(req.params.value, req.query.field, req.query.start, req.query.count, req.query.sort, - function (err, videosList, totalVideos) { + Video.search(req.params.value, req.query.field, req.query.start, req.query.count, req.query.sort, + function (err, videosList, videosTotal) { if (err) return next(err) - res.json(getFormatedVideos(videosList, totalVideos)) + res.json(getFormatedVideos(videosList, videosTotal)) }) } // --------------------------------------------------------------------------- -function getFormatedVideo (videoObj) { - const formatedVideo = { - id: videoObj._id, - name: videoObj.name, - description: videoObj.description, - podUrl: videoObj.podUrl.replace(/^https?:\/\//, ''), - isLocal: videos.getVideoState(videoObj).owned, - magnetUri: videoObj.magnetUri, - author: videoObj.author, - duration: videoObj.duration, - tags: videoObj.tags, - thumbnailPath: constants.THUMBNAILS_STATIC_PATH + '/' + videoObj.thumbnail, - createdDate: videoObj.createdDate - } - - return formatedVideo -} - -function getFormatedVideos (videosObj, totalVideos) { +function getFormatedVideos (videos, videosTotal) { const formatedVideos = [] - videosObj.forEach(function (videoObj) { - formatedVideos.push(getFormatedVideo(videoObj)) + videos.forEach(function (video) { + formatedVideos.push(video.toFormatedJSON()) }) return { - total: totalVideos, + total: videosTotal, data: formatedVideos } } - -// Maybe the torrent is not seeded, but we catch the error to don't stop the removing process -function removeTorrent (magnetUri, callback) { - try { - webtorrent.remove(magnetUri, callback) - } catch (err) { - logger.warn('Cannot remove the torrent from WebTorrent', { err: err }) - return callback(null) - } -} diff --git a/server/helpers/customValidators.js b/server/helpers/customValidators.js index a6cf680e5..4d6139a3d 100644 --- a/server/helpers/customValidators.js +++ b/server/helpers/customValidators.js @@ -17,7 +17,8 @@ const customValidators = { isVideoNameValid: isVideoNameValid, isVideoPodUrlValid: isVideoPodUrlValid, isVideoTagsValid: isVideoTagsValid, - isVideoThumbnailValid: isVideoThumbnailValid + isVideoThumbnailValid: isVideoThumbnailValid, + isVideoThumbnail64Valid: isVideoThumbnail64Valid } function exists (value) { @@ -37,7 +38,7 @@ function isEachRemoteVideosValid (requests) { isVideoNameValid(video.name) && isVideoPodUrlValid(video.podUrl) && isVideoTagsValid(video.tags) && - isVideoThumbnailValid(video.thumbnailBase64) + isVideoThumbnail64Valid(video.thumbnailBase64) ) || ( isRequestTypeRemoveValid(request.type) && @@ -97,8 +98,12 @@ function isVideoTagsValid (tags) { } function isVideoThumbnailValid (value) { + return validator.isLength(value, VIDEOS_CONSTRAINTS_FIELDS.THUMBNAIL) +} + +function isVideoThumbnail64Valid (value) { return validator.isBase64(value) && - validator.isByteLength(value, VIDEOS_CONSTRAINTS_FIELDS.THUMBNAIL) + validator.isByteLength(value, VIDEOS_CONSTRAINTS_FIELDS.THUMBNAIL64) } // --------------------------------------------------------------------------- diff --git a/server/initializers/constants.js b/server/initializers/constants.js index caeb340cf..1f9876c4b 100644 --- a/server/initializers/constants.js +++ b/server/initializers/constants.js @@ -48,7 +48,8 @@ const VIDEOS_CONSTRAINTS_FIELDS = { AUTHOR: { min: 3, max: 20 }, // Length TAGS: { min: 1, max: 3 }, // Number of total tags TAG: { min: 2, max: 10 }, // Length - THUMBNAIL: { min: 0, max: 20000 } // Bytes + THUMBNAIL: { min: 2, max: 30 }, + THUMBNAIL64: { min: 0, max: 20000 } // Bytes } // Special constants for a test instance diff --git a/server/initializers/database.js b/server/initializers/database.js index 830cc7dd8..5932a978b 100644 --- a/server/initializers/database.js +++ b/server/initializers/database.js @@ -5,6 +5,9 @@ const mongoose = require('mongoose') const logger = require('../helpers/logger') +// Bootstrap models +require('../models/video') + const dbname = 'peertube' + config.get('database.suffix') const host = config.get('database.host') const port = config.get('database.port') diff --git a/server/lib/friends.js b/server/lib/friends.js index d81a603ad..91cd69f86 100644 --- a/server/lib/friends.js +++ b/server/lib/friends.js @@ -3,6 +3,7 @@ const async = require('async') const config = require('config') const fs = require('fs') +const mongoose = require('mongoose') const request = require('request') const constants = require('../initializers/constants') @@ -11,12 +12,11 @@ const peertubeCrypto = require('../helpers/peertubeCrypto') const Pods = require('../models/pods') const requestsScheduler = require('../lib/requestsScheduler') const requests = require('../helpers/requests') -const videos = require('../lib/videos') -const Videos = require('../models/videos') const http = config.get('webserver.https') ? 'https' : 'http' const host = config.get('webserver.host') const port = config.get('webserver.port') +const Video = mongoose.model('Video') const pods = { addVideoToFriends: addVideoToFriends, @@ -117,18 +117,13 @@ function quitFriends (callback) { function listRemoteVideos (callbackAsync) { logger.info('Broke friends, so sad :(') - Videos.listFromRemotes(callbackAsync) + Video.listRemotes(callbackAsync) }, function removeTheRemoteVideos (videosList, callbackAsync) { - videos.removeRemoteVideos(videosList, function (err) { - if (err) { - logger.error('Cannot remove remote videos.', { error: err }) - return callbackAsync(err) - } - - return callbackAsync(null) - }) + async.each(videosList, function (video, callbackEach) { + video.remove(callbackEach) + }, callbackAsync) } ], function (err) { // Don't forget to re activate the scheduler, even if there was an error @@ -146,14 +141,14 @@ function removeVideoToFriends (video) { } function sendOwnedVideosToPod (podId) { - Videos.listOwned(function (err, videosList) { + Video.listOwned(function (err, videosList) { if (err) { logger.error('Cannot get the list of videos we own.') return } videosList.forEach(function (video) { - videos.convertVideoToRemote(video, function (err, remoteVideo) { + video.toRemoteJSON(function (err, remoteVideo) { if (err) { logger.error('Cannot convert video to remote.', { error: err }) // Don't break the process diff --git a/server/lib/requestsScheduler.js b/server/lib/requestsScheduler.js index ac75e5b93..b192d8299 100644 --- a/server/lib/requestsScheduler.js +++ b/server/lib/requestsScheduler.js @@ -2,14 +2,15 @@ const async = require('async') const map = require('lodash/map') +const mongoose = require('mongoose') const constants = require('../initializers/constants') const logger = require('../helpers/logger') const Pods = require('../models/pods') const Requests = require('../models/requests') const requests = require('../helpers/requests') -const videos = require('../lib/videos') -const Videos = require('../models/videos') + +const Video = mongoose.model('Video') let timer = null @@ -210,7 +211,7 @@ function removeBadPods () { const urls = map(pods, 'url') const ids = map(pods, '_id') - Videos.listFromUrls(urls, function (err, videosList) { + Video.listByUrls(urls, function (err, videosList) { if (err) { logger.error('Cannot list videos urls.', { error: err, urls: urls }) return callback(null, ids, []) @@ -224,9 +225,14 @@ function removeBadPods () { // We don't have to remove pods, skip if (typeof podIds === 'function') return podIds(null) - // Remove the remote videos - videos.removeRemoteVideos(videosList, function (err) { - if (err) logger.error('Cannot remove remote videos.', { error: err }) + async.each(videosList, function (video, callbackEach) { + video.remove(callbackEach) + }, function (err) { + if (err) { + // Don't stop the process + logger.error('Error while removing videos of bad pods.', { error: err }) + return + } return callback(null, podIds) }) diff --git a/server/lib/videos.js b/server/lib/videos.js deleted file mode 100644 index a74c77dc4..000000000 --- a/server/lib/videos.js +++ /dev/null @@ -1,199 +0,0 @@ -'use strict' - -const async = require('async') -const config = require('config') -const ffmpeg = require('fluent-ffmpeg') -const fs = require('fs') -const map = require('lodash/map') -const pathUtils = require('path') - -const constants = require('../initializers/constants') -const logger = require('../helpers/logger') -const utils = require('../helpers/utils') -const Videos = require('../models/videos') -const webtorrent = require('../lib/webtorrent') - -const uploadDir = pathUtils.join(__dirname, '..', '..', config.get('storage.uploads')) -const thumbnailsDir = pathUtils.join(__dirname, '..', '..', config.get('storage.thumbnails')) - -const videos = { - convertVideoToRemote: convertVideoToRemote, - createRemoteVideos: createRemoteVideos, - getVideoDuration: getVideoDuration, - getVideoState: getVideoState, - createVideoThumbnail: createVideoThumbnail, - removeVideosDataFromDisk: removeVideosDataFromDisk, - removeRemoteVideos: removeRemoteVideos, - seed: seed, - seedAllExisting: seedAllExisting -} - -function convertVideoToRemote (video, callback) { - fs.readFile(thumbnailsDir + video.thumbnail, function (err, thumbnailData) { - if (err) { - logger.error('Cannot read the thumbnail of the video') - return callback(err) - } - - const remoteVideo = { - name: video.name, - description: video.description, - magnetUri: video.magnetUri, - author: video.author, - duration: video.duration, - thumbnailBase64: new Buffer(thumbnailData).toString('base64'), - tags: video.tags, - createdDate: video.createdDate, - podUrl: video.podUrl - } - - return callback(null, remoteVideo) - }) -} - -function createRemoteVideos (videos, callback) { - // Create the remote videos from the new pod - createRemoteVideoObjects(videos, function (err, remoteVideos) { - if (err) return callback(err) - - Videos.addRemotes(remoteVideos, callback) - }) -} - -function getVideoDuration (videoPath, callback) { - ffmpeg.ffprobe(videoPath, function (err, metadata) { - if (err) return callback(err) - - return callback(null, Math.floor(metadata.format.duration)) - }) -} - -function getVideoState (video) { - const exist = (video !== null) - let owned = false - if (exist === true) { - owned = (video.namePath !== null) - } - - return { exist: exist, owned: owned } -} - -function createVideoThumbnail (videoPath, callback) { - const filename = pathUtils.basename(videoPath) + '.jpg' - ffmpeg(videoPath) - .on('error', callback) - .on('end', function () { - callback(null, filename) - }) - .thumbnail({ - count: 1, - folder: thumbnailsDir, - size: constants.THUMBNAILS_SIZE, - filename: filename - }) -} - -// Remove video datas from disk (video file, thumbnail...) -function removeVideosDataFromDisk (videos, callback) { - async.each(videos, function (video, callbackEach) { - fs.unlink(thumbnailsDir + video.thumbnail, function (err) { - if (err) logger.error('Cannot remove the video thumbnail') - - if (getVideoState(video).owned === true) { - fs.unlink(uploadDir + video.namePath, function (err) { - if (err) { - logger.error('Cannot remove this video file.') - return callbackEach(err) - } - - callbackEach(null) - }) - } else { - callbackEach(null) - } - }) - }, callback) -} - -function removeRemoteVideos (videos, callback) { - Videos.removeByIds(map(videos, '_id'), function (err) { - if (err) return callback(err) - - removeVideosDataFromDisk(videos, callback) - }) -} - -function seed (path, callback) { - logger.info('Seeding %s...', path) - - webtorrent.seed(path, function (torrent) { - logger.info('%s seeded (%s).', path, torrent.magnetURI) - - return callback(null, torrent) - }) -} - -function seedAllExisting (callback) { - Videos.listOwned(function (err, videosList) { - if (err) { - logger.error('Cannot get list of the videos to seed.') - return callback(err) - } - - async.each(videosList, function (video, callbackEach) { - seed(uploadDir + video.namePath, function (err) { - if (err) { - logger.error('Cannot seed this video.') - return callback(err) - } - - callbackEach(null) - }) - }, callback) - }) -} - -// --------------------------------------------------------------------------- - -module.exports = videos - -// --------------------------------------------------------------------------- - -function createRemoteVideoObjects (videos, callback) { - const remoteVideos = [] - - async.each(videos, function (video, callbackEach) { - // Creating the thumbnail for this remote video - utils.generateRandomString(16, function (err, randomString) { - if (err) return callbackEach(err) - - const thumbnailName = randomString + '.jpg' - createThumbnailFromBase64(thumbnailName, video.thumbnailBase64, function (err) { - if (err) return callbackEach(err) - - const params = { - name: video.name, - description: video.description, - magnetUri: video.magnetUri, - podUrl: video.podUrl, - duration: video.duration, - thumbnail: thumbnailName, - tags: video.tags, - author: video.author - } - remoteVideos.push(params) - - callbackEach(null) - }) - }) - }, - function (err) { - if (err) return callback(err) - - callback(null, remoteVideos) - }) -} - -function createThumbnailFromBase64 (thumbnailName, data, callback) { - fs.writeFile(thumbnailsDir + thumbnailName, data, { encoding: 'base64' }, callback) -} diff --git a/server/middlewares/reqValidators/remote.js b/server/middlewares/reqValidators/remote.js index a23673d89..dd8ee5f6e 100644 --- a/server/middlewares/reqValidators/remote.js +++ b/server/middlewares/reqValidators/remote.js @@ -22,7 +22,7 @@ function remoteVideos (req, res, next) { req.checkBody('data').isArray() req.checkBody('data').isEachRemoteVideosValid() - logger.debug('Checking remoteVideosAdd parameters', { parameters: req.body }) + logger.debug('Checking remoteVideos parameters', { parameters: req.body }) checkErrors(req, res, next) } diff --git a/server/middlewares/reqValidators/videos.js b/server/middlewares/reqValidators/videos.js index f31fd93a2..452fbc859 100644 --- a/server/middlewares/reqValidators/videos.js +++ b/server/middlewares/reqValidators/videos.js @@ -1,11 +1,13 @@ 'use strict' +const mongoose = require('mongoose') + const checkErrors = require('./utils').checkErrors const constants = require('../../initializers/constants') const customValidators = require('../../helpers/customValidators') const logger = require('../../helpers/logger') -const videos = require('../../lib/videos') -const Videos = require('../../models/videos') + +const Video = mongoose.model('Video') const reqValidatorsVideos = { videosAdd: videosAdd, @@ -26,7 +28,7 @@ function videosAdd (req, res, next) { checkErrors(req, res, function () { const videoFile = req.files.videofile[0] - videos.getVideoDuration(videoFile.path, function (err, duration) { + Video.getDurationFromFile(videoFile.path, function (err, duration) { if (err) { return res.status(400).send('Cannot retrieve metadata of the file.') } @@ -47,14 +49,13 @@ function videosGet (req, res, next) { logger.debug('Checking videosGet parameters', { parameters: req.params }) checkErrors(req, res, function () { - Videos.get(req.params.id, function (err, video) { + Video.load(req.params.id, function (err, video) { if (err) { logger.error('Error in videosGet request validator.', { error: err }) return res.sendStatus(500) } - const state = videos.getVideoState(video) - if (state.exist === false) return res.status(404).send('Video not found') + if (!video) return res.status(404).send('Video not found') next() }) @@ -67,15 +68,14 @@ function videosRemove (req, res, next) { logger.debug('Checking videosRemove parameters', { parameters: req.params }) checkErrors(req, res, function () { - Videos.get(req.params.id, function (err, video) { + Video.load(req.params.id, function (err, video) { if (err) { logger.error('Error in videosRemove request validator.', { error: err }) return res.sendStatus(500) } - const state = videos.getVideoState(video) - if (state.exist === false) return res.status(404).send('Video not found') - else if (state.owned === false) return res.status(403).send('Cannot remove video of another pod') + if (!video) return res.status(404).send('Video not found') + else if (video.isOwned() === false) return res.status(403).send('Cannot remove video of another pod') next() }) diff --git a/server/models/video.js b/server/models/video.js new file mode 100644 index 000000000..8b14e9b35 --- /dev/null +++ b/server/models/video.js @@ -0,0 +1,314 @@ +'use strict' + +const async = require('async') +const config = require('config') +const ffmpeg = require('fluent-ffmpeg') +const fs = require('fs') +const pathUtils = require('path') +const mongoose = require('mongoose') + +const constants = require('../initializers/constants') +const customValidators = require('../helpers/customValidators') +const logger = require('../helpers/logger') +const utils = require('../helpers/utils') +const webtorrent = require('../lib/webtorrent') + +const http = config.get('webserver.https') === true ? 'https' : 'http' +const host = config.get('webserver.host') +const port = config.get('webserver.port') +const uploadsDir = pathUtils.join(__dirname, '..', '..', config.get('storage.uploads')) +const thumbnailsDir = pathUtils.join(__dirname, '..', '..', config.get('storage.thumbnails')) + +// --------------------------------------------------------------------------- + +// TODO: add indexes on searchable columns +const VideoSchema = mongoose.Schema({ + name: String, + namePath: String, + description: String, + magnetUri: String, + podUrl: String, + author: String, + duration: Number, + thumbnail: String, + tags: [ String ], + createdDate: { + type: Date, + default: Date.now + } +}) + +VideoSchema.path('name').validate(customValidators.isVideoNameValid) +VideoSchema.path('description').validate(customValidators.isVideoDescriptionValid) +VideoSchema.path('magnetUri').validate(customValidators.isVideoMagnetUriValid) +VideoSchema.path('podUrl').validate(customValidators.isVideoPodUrlValid) +VideoSchema.path('author').validate(customValidators.isVideoAuthorValid) +VideoSchema.path('duration').validate(customValidators.isVideoDurationValid) +// The tumbnail can be the path or the data in base 64 +// The pre save hook will convert the base 64 data in a file on disk and replace the thumbnail key by the filename +VideoSchema.path('thumbnail').validate(function (value) { + return customValidators.isVideoThumbnailValid(value) || customValidators.isVideoThumbnail64Valid(value) +}) +VideoSchema.path('tags').validate(customValidators.isVideoTagsValid) + +VideoSchema.methods = { + isOwned: isOwned, + toFormatedJSON: toFormatedJSON, + toRemoteJSON: toRemoteJSON +} + +VideoSchema.statics = { + getDurationFromFile: getDurationFromFile, + list: list, + listByUrlAndMagnet: listByUrlAndMagnet, + listByUrls: listByUrls, + listOwned: listOwned, + listRemotes: listRemotes, + load: load, + search: search, + seedAllExisting: seedAllExisting +} + +VideoSchema.pre('remove', function (next) { + const video = this + const tasks = [] + + tasks.push( + function (callback) { + removeThumbnail(video, callback) + } + ) + + if (video.isOwned()) { + tasks.push( + function (callback) { + removeFile(video, callback) + }, + function (callback) { + removeTorrent(video, callback) + } + ) + } + + async.parallel(tasks, next) +}) + +VideoSchema.pre('save', function (next) { + const video = this + const tasks = [] + + if (video.isOwned()) { + const videoPath = pathUtils.join(uploadsDir, video.namePath) + this.podUrl = http + '://' + host + ':' + port + + tasks.push( + function (callback) { + seed(videoPath, callback) + }, + function (callback) { + createThumbnail(videoPath, callback) + } + ) + + async.parallel(tasks, function (err, results) { + if (err) return next(err) + + video.magnetUri = results[0].magnetURI + video.thumbnail = results[1] + + return next() + }) + } else { + generateThumbnailFromBase64(video.thumbnail, function (err, thumbnailName) { + if (err) return next(err) + + video.thumbnail = thumbnailName + + return next() + }) + } +}) + +mongoose.model('Video', VideoSchema) + +// ------------------------------ METHODS ------------------------------ + +function isOwned () { + return this.namePath !== null +} + +function toFormatedJSON () { + const json = { + id: this._id, + name: this.name, + description: this.description, + podUrl: this.podUrl.replace(/^https?:\/\//, ''), + isLocal: this.isOwned(), + magnetUri: this.magnetUri, + author: this.author, + duration: this.duration, + tags: this.tags, + thumbnailPath: constants.THUMBNAILS_STATIC_PATH + '/' + this.thumbnail, + createdDate: this.createdDate + } + + return json +} + +function toRemoteJSON (callback) { + const self = this + + // Convert thumbnail to base64 + fs.readFile(pathUtils.join(thumbnailsDir, this.thumbnail), function (err, thumbnailData) { + if (err) { + logger.error('Cannot read the thumbnail of the video') + return callback(err) + } + + const remoteVideo = { + name: self.name, + description: self.description, + magnetUri: self.magnetUri, + namePath: null, + author: self.author, + duration: self.duration, + thumbnailBase64: new Buffer(thumbnailData).toString('base64'), + tags: self.tags, + createdDate: self.createdDate, + podUrl: self.podUrl + } + + return callback(null, remoteVideo) + }) +} + +// ------------------------------ STATICS ------------------------------ + +function getDurationFromFile (videoPath, callback) { + ffmpeg.ffprobe(videoPath, function (err, metadata) { + if (err) return callback(err) + + return callback(null, Math.floor(metadata.format.duration)) + }) +} + +function list (start, count, sort, callback) { + const query = {} + return findWithCount.call(this, query, start, count, sort, callback) +} + +function listByUrlAndMagnet (fromUrl, magnetUri, callback) { + this.find({ podUrl: fromUrl, magnetUri: magnetUri }, callback) +} + +function listByUrls (fromUrls, callback) { + this.find({ podUrl: { $in: fromUrls } }, callback) +} + +function listOwned (callback) { + // If namePath is not null this is *our* video + this.find({ namePath: { $ne: null } }, callback) +} + +function listRemotes (callback) { + this.find({ namePath: null }, callback) +} + +function load (id, callback) { + this.findById(id, callback) +} + +function search (value, field, start, count, sort, callback) { + const query = {} + // Make an exact search with the magnet + if (field === 'magnetUri' || field === 'tags') { + query[field] = value + } else { + query[field] = new RegExp(value) + } + + findWithCount.call(this, query, start, count, sort, callback) +} + +// TODO +function seedAllExisting () { + +} + +// --------------------------------------------------------------------------- + +function findWithCount (query, start, count, sort, callback) { + const self = this + + async.parallel([ + function (asyncCallback) { + self.find(query).skip(start).limit(start + count).sort(sort).exec(asyncCallback) + }, + function (asyncCallback) { + self.count(query, asyncCallback) + } + ], function (err, results) { + if (err) return callback(err) + + const videos = results[0] + const totalVideos = results[1] + return callback(null, videos, totalVideos) + }) +} + +function removeThumbnail (video, callback) { + fs.unlink(thumbnailsDir + video.thumbnail, callback) +} + +function removeFile (video, callback) { + fs.unlink(uploadsDir + video.namePath, callback) +} + +// Maybe the torrent is not seeded, but we catch the error to don't stop the removing process +function removeTorrent (video, callback) { + try { + webtorrent.remove(video.magnetUri, callback) + } catch (err) { + logger.warn('Cannot remove the torrent from WebTorrent', { err: err }) + return callback(null) + } +} + +function createThumbnail (videoPath, callback) { + const filename = pathUtils.basename(videoPath) + '.jpg' + ffmpeg(videoPath) + .on('error', callback) + .on('end', function () { + callback(null, filename) + }) + .thumbnail({ + count: 1, + folder: thumbnailsDir, + size: constants.THUMBNAILS_SIZE, + filename: filename + }) +} + +function seed (path, callback) { + logger.info('Seeding %s...', path) + + webtorrent.seed(path, function (torrent) { + logger.info('%s seeded (%s).', path, torrent.magnetURI) + + return callback(null, torrent) + }) +} + +function generateThumbnailFromBase64 (data, callback) { + // Creating the thumbnail for this remote video + utils.generateRandomString(16, function (err, randomString) { + if (err) return callback(err) + + const thumbnailName = randomString + '.jpg' + fs.writeFile(thumbnailsDir + thumbnailName, data, { encoding: 'base64' }, function (err) { + if (err) return callback(err) + + return callback(null, thumbnailName) + }) + }) +} diff --git a/server/models/videos.js b/server/models/videos.js deleted file mode 100644 index c177b414c..000000000 --- a/server/models/videos.js +++ /dev/null @@ -1,162 +0,0 @@ -'use strict' - -const async = require('async') -const config = require('config') -const mongoose = require('mongoose') - -const logger = require('../helpers/logger') - -const http = config.get('webserver.https') === true ? 'https' : 'http' -const host = config.get('webserver.host') -const port = config.get('webserver.port') - -// --------------------------------------------------------------------------- - -// TODO: add indexes on searchable columns -const videosSchema = mongoose.Schema({ - name: String, - namePath: String, - description: String, - magnetUri: String, - podUrl: String, - author: String, - duration: Number, - thumbnail: String, - tags: [ String ], - createdDate: { - type: Date, - default: Date.now - } -}) -const VideosDB = mongoose.model('videos', videosSchema) - -// --------------------------------------------------------------------------- - -const Videos = { - add: add, - addRemotes: addRemotes, - get: get, - list: list, - listFromUrl: listFromUrl, - listFromUrls: listFromUrls, - listFromUrlAndMagnets: listFromUrlAndMagnets, - listFromRemotes: listFromRemotes, - listOwned: listOwned, - removeOwned: removeOwned, - removeByIds: removeByIds, - search: search -} - -function add (video, callback) { - logger.info('Adding %s video to database.', video.name) - - const params = video - params.podUrl = http + '://' + host + ':' + port - - VideosDB.create(params, function (err, insertedVideo) { - if (err) { - logger.error('Cannot insert this video into database.') - return callback(err) - } - - callback(null, insertedVideo) - }) -} - -function addRemotes (videos, callback) { - videos.forEach(function (video) { - // Ensure they are remote videos - video.namePath = null - }) - - VideosDB.create(videos, callback) -} - -function get (id, callback) { - VideosDB.findById(id, function (err, video) { - if (err) { - logger.error('Cannot get this video.') - return callback(err) - } - - return callback(null, video) - }) -} - -function list (start, count, sort, callback) { - const query = {} - return findWithCount(query, start, count, sort, callback) -} - -function listFromUrl (fromUrl, callback) { - VideosDB.find({ podUrl: fromUrl }, callback) -} - -function listFromUrls (fromUrls, callback) { - VideosDB.find({ podUrl: { $in: fromUrls } }, callback) -} - -function listFromUrlAndMagnets (fromUrl, magnets, callback) { - VideosDB.find({ podUrl: fromUrl, magnetUri: { $in: magnets } }, callback) -} - -function listFromRemotes (callback) { - VideosDB.find({ namePath: null }, callback) -} - -function listOwned (callback) { - // If namePath is not null this is *our* video - VideosDB.find({ namePath: { $ne: null } }, function (err, videosList) { - if (err) { - logger.error('Cannot get the list of owned videos.') - return callback(err) - } - - return callback(null, videosList) - }) -} - -// Return the video in the callback -function removeOwned (id, callback) { - VideosDB.findByIdAndRemove(id, callback) -} - -// Use the magnet Uri because the _id field is not the same on different servers -function removeByIds (ids, callback) { - VideosDB.remove({ _id: { $in: ids } }, callback) -} - -function search (value, field, start, count, sort, callback) { - const query = {} - // Make an exact search with the magnet - if (field === 'magnetUri' || field === 'tags') { - query[field] = value - } else { - query[field] = new RegExp(value) - } - - findWithCount(query, start, count, sort, callback) -} - -// --------------------------------------------------------------------------- - -module.exports = Videos - -// --------------------------------------------------------------------------- - -function findWithCount (query, start, count, sort, callback) { - async.parallel([ - function (asyncCallback) { - VideosDB.find(query).skip(start).limit(start + count).sort(sort).exec(asyncCallback) - }, - function (asyncCallback) { - VideosDB.count(query, asyncCallback) - } - ], function (err, results) { - if (err) return callback(err) - - const videos = results[0] - const totalVideos = results[1] - return callback(null, videos, totalVideos) - }) -} diff --git a/server/tests/api/multiplePods.js b/server/tests/api/multiplePods.js index 2a1bc64e6..52dfda137 100644 --- a/server/tests/api/multiplePods.js +++ b/server/tests/api/multiplePods.js @@ -414,7 +414,7 @@ describe('Test multiple pods', function () { // Keep the logs if the test failed if (this.ok) { - utils.flushTests(done) + // utils.flushTests(done) } else { done() } -- cgit v1.2.3