X-Git-Url: https://git.immae.eu/?a=blobdiff_plain;f=server%2Fmodels%2Fvideo.js;h=daa273845f5610de2b90f85df4a405a2cda2cb59;hb=9e167724f7e933f41d9ea2e1c31772bf4c560a28;hp=acb8353c2891b411e2b2e3fb64a32ec7bd1a033b;hpb=e4c556196d7b31111f17596840d2e1d60caa7dcb;p=github%2FChocobozzz%2FPeerTube.git diff --git a/server/models/video.js b/server/models/video.js index acb8353c2..daa273845 100644 --- a/server/models/video.js +++ b/server/models/video.js @@ -1,77 +1,214 @@ 'use strict' -const config = require('config') -const eachLimit = require('async/eachLimit') +const Buffer = require('safe-buffer').Buffer +const createTorrent = require('create-torrent') const ffmpeg = require('fluent-ffmpeg') const fs = require('fs') +const magnetUtil = require('magnet-uri') +const map = require('lodash/map') const parallel = require('async/parallel') +const parseTorrent = require('parse-torrent') const pathUtils = require('path') -const mongoose = require('mongoose') +const values = require('lodash/values') const constants = require('../initializers/constants') -const customVideosValidators = require('../helpers/custom-validators').videos const logger = require('../helpers/logger') -const utils = require('../helpers/utils') -const webtorrent = require('../lib/webtorrent') - -const http = config.get('webserver.https') === true ? 'https' : 'http' -const host = config.get('webserver.host') -const port = config.get('webserver.port') -const uploadsDir = pathUtils.join(__dirname, '..', '..', config.get('storage.uploads')) -const thumbnailsDir = pathUtils.join(__dirname, '..', '..', config.get('storage.thumbnails')) +const friends = require('../lib/friends') +const modelUtils = require('./utils') +const customVideosValidators = require('../helpers/custom-validators').videos // --------------------------------------------------------------------------- -// TODO: add indexes on searchable columns -const VideoSchema = mongoose.Schema({ - name: String, - filename: String, - description: String, - magnetUri: String, - podUrl: String, - author: String, - duration: Number, - thumbnail: String, - tags: [ String ], - createdDate: { - type: Date, - default: Date.now +module.exports = function (sequelize, DataTypes) { + const Video = sequelize.define('Video', + { + id: { + type: DataTypes.UUID, + defaultValue: DataTypes.UUIDV4, + primaryKey: true, + validate: { + isUUID: 4 + } + }, + name: { + type: DataTypes.STRING, + allowNull: false, + validate: { + nameValid: function (value) { + const res = customVideosValidators.isVideoNameValid(value) + if (res === false) throw new Error('Video name is not valid.') + } + } + }, + extname: { + type: DataTypes.ENUM(values(constants.CONSTRAINTS_FIELDS.VIDEOS.EXTNAME)), + allowNull: false + }, + remoteId: { + type: DataTypes.UUID, + allowNull: true, + validate: { + isUUID: 4 + } + }, + description: { + type: DataTypes.STRING, + allowNull: false, + validate: { + descriptionValid: function (value) { + const res = customVideosValidators.isVideoDescriptionValid(value) + if (res === false) throw new Error('Video description is not valid.') + } + } + }, + infoHash: { + type: DataTypes.STRING, + allowNull: false, + validate: { + infoHashValid: function (value) { + const res = customVideosValidators.isVideoInfoHashValid(value) + if (res === false) throw new Error('Video info hash is not valid.') + } + } + }, + duration: { + type: DataTypes.INTEGER, + allowNull: false, + validate: { + durationValid: function (value) { + const res = customVideosValidators.isVideoDurationValid(value) + if (res === false) throw new Error('Video duration is not valid.') + } + } + }, + views: { + type: DataTypes.INTEGER, + allowNull: false, + defaultValue: 0, + validate: { + min: 0, + isInt: true + } + } + }, + { + indexes: [ + { + fields: [ 'authorId' ] + }, + { + fields: [ 'remoteId' ] + }, + { + fields: [ 'name' ] + }, + { + fields: [ 'createdAt' ] + }, + { + fields: [ 'duration' ] + }, + { + fields: [ 'infoHash' ] + }, + { + fields: [ 'views' ] + } + ], + classMethods: { + associate, + + generateThumbnailFromData, + getDurationFromFile, + list, + listForApi, + listOwnedAndPopulateAuthorAndTags, + listOwnedByAuthor, + load, + loadByHostAndRemoteId, + loadAndPopulateAuthor, + loadAndPopulateAuthorAndPodAndTags, + searchAndPopulateAuthorAndPodAndTags + }, + instanceMethods: { + generateMagnetUri, + getVideoFilename, + getThumbnailName, + getPreviewName, + getTorrentName, + isOwned, + toFormatedJSON, + toAddRemoteJSON, + toUpdateRemoteJSON + }, + hooks: { + beforeValidate, + beforeCreate, + afterDestroy + } + } + ) + + return Video +} + +function beforeValidate (video, options, next) { + // Put a fake infoHash if it does not exists yet + if (video.isOwned() && !video.infoHash) { + // 40 hexa length + video.infoHash = '0123456789abcdef0123456789abcdef01234567' } -}) - -VideoSchema.path('name').validate(customVideosValidators.isVideoNameValid) -VideoSchema.path('description').validate(customVideosValidators.isVideoDescriptionValid) -VideoSchema.path('magnetUri').validate(customVideosValidators.isVideoMagnetUriValid) -VideoSchema.path('podUrl').validate(customVideosValidators.isVideoPodUrlValid) -VideoSchema.path('author').validate(customVideosValidators.isVideoAuthorValid) -VideoSchema.path('duration').validate(customVideosValidators.isVideoDurationValid) -// The tumbnail can be the path or the data in base 64 -// The pre save hook will convert the base 64 data in a file on disk and replace the thumbnail key by the filename -VideoSchema.path('thumbnail').validate(function (value) { - return customVideosValidators.isVideoThumbnailValid(value) || customVideosValidators.isVideoThumbnail64Valid(value) -}) -VideoSchema.path('tags').validate(customVideosValidators.isVideoTagsValid) - -VideoSchema.methods = { - isOwned: isOwned, - toFormatedJSON: toFormatedJSON, - toRemoteJSON: toRemoteJSON -} - -VideoSchema.statics = { - getDurationFromFile: getDurationFromFile, - list: list, - listByUrlAndMagnet: listByUrlAndMagnet, - listByUrls: listByUrls, - listOwned: listOwned, - listRemotes: listRemotes, - load: load, - search: search, - seedAllExisting: seedAllExisting -} - -VideoSchema.pre('remove', function (next) { - const video = this + + return next(null) +} + +function beforeCreate (video, options, next) { + const tasks = [] + + if (video.isOwned()) { + const videoPath = pathUtils.join(constants.CONFIG.STORAGE.VIDEOS_DIR, video.getVideoFilename()) + + tasks.push( + function createVideoTorrent (callback) { + const options = { + announceList: [ + [ constants.CONFIG.WEBSERVER.WS + '://' + constants.CONFIG.WEBSERVER.HOSTNAME + ':' + constants.CONFIG.WEBSERVER.PORT + '/tracker/socket' ] + ], + urlList: [ + constants.CONFIG.WEBSERVER.URL + constants.STATIC_PATHS.WEBSEED + video.getVideoFilename() + ] + } + + createTorrent(videoPath, options, function (err, torrent) { + if (err) return callback(err) + + const filePath = pathUtils.join(constants.CONFIG.STORAGE.TORRENTS_DIR, video.getTorrentName()) + fs.writeFile(filePath, torrent, function (err) { + if (err) return callback(err) + + const parsedTorrent = parseTorrent(torrent) + video.set('infoHash', parsedTorrent.infoHash) + video.validate().asCallback(callback) + }) + }) + }, + + function createVideoThumbnail (callback) { + createThumbnail(video, videoPath, callback) + }, + + function createVIdeoPreview (callback) { + createPreview(video, videoPath, callback) + } + ) + + return parallel(tasks, next) + } + + return next() +} + +function afterDestroy (video, options, next) { const tasks = [] tasks.push( @@ -82,85 +219,151 @@ VideoSchema.pre('remove', function (next) { if (video.isOwned()) { tasks.push( - function (callback) { + function removeVideoFile (callback) { removeFile(video, callback) }, - function (callback) { + + function removeVideoTorrent (callback) { removeTorrent(video, callback) + }, + + function removeVideoPreview (callback) { + removePreview(video, callback) + }, + + function removeVideoToFriends (callback) { + const params = { + remoteId: video.id + } + + friends.removeVideoToFriends(params) + + return callback() } ) } parallel(tasks, next) -}) +} -VideoSchema.pre('save', function (next) { - const video = this - const tasks = [] +// ------------------------------ METHODS ------------------------------ - if (video.isOwned()) { - const videoPath = pathUtils.join(uploadsDir, video.filename) - this.podUrl = http + '://' + host + ':' + port +function associate (models) { + this.belongsTo(models.Author, { + foreignKey: { + name: 'authorId', + allowNull: false + }, + onDelete: 'cascade' + }) - tasks.push( - function (callback) { - seed(videoPath, callback) - }, - function (callback) { - createThumbnail(videoPath, callback) - } - ) + this.belongsToMany(models.Tag, { + foreignKey: 'videoId', + through: models.VideoTag, + onDelete: 'cascade' + }) - parallel(tasks, function (err, results) { - if (err) return next(err) + this.hasMany(models.VideoAbuse, { + foreignKey: { + name: 'videoId', + allowNull: false + }, + onDelete: 'cascade' + }) +} - video.magnetUri = results[0].magnetURI - video.thumbnail = results[1] +function generateMagnetUri () { + let baseUrlHttp, baseUrlWs - return next() - }) + if (this.isOwned()) { + baseUrlHttp = constants.CONFIG.WEBSERVER.URL + baseUrlWs = constants.CONFIG.WEBSERVER.WS + '://' + constants.CONFIG.WEBSERVER.HOSTNAME + ':' + constants.CONFIG.WEBSERVER.PORT } else { - generateThumbnailFromBase64(video.thumbnail, function (err, thumbnailName) { - if (err) return next(err) + baseUrlHttp = constants.REMOTE_SCHEME.HTTP + '://' + this.Author.Pod.host + baseUrlWs = constants.REMOTE_SCHEME.WS + '://' + this.Author.Pod.host + } - video.thumbnail = thumbnailName + const xs = baseUrlHttp + constants.STATIC_PATHS.TORRENTS + this.getTorrentName() + const announce = baseUrlWs + '/tracker/socket' + const urlList = [ baseUrlHttp + constants.STATIC_PATHS.WEBSEED + this.getVideoFilename() ] - return next() - }) + const magnetHash = { + xs, + announce, + urlList, + infoHash: this.infoHash, + name: this.name } -}) -mongoose.model('Video', VideoSchema) + return magnetUtil.encode(magnetHash) +} -// ------------------------------ METHODS ------------------------------ +function getVideoFilename () { + if (this.isOwned()) return this.id + this.extname + + return this.remoteId + this.extname +} + +function getThumbnailName () { + // We always have a copy of the thumbnail + return this.id + '.jpg' +} + +function getPreviewName () { + const extension = '.jpg' + + if (this.isOwned()) return this.id + extension + + return this.remoteId + extension +} + +function getTorrentName () { + const extension = '.torrent' + + if (this.isOwned()) return this.id + extension + + return this.remoteId + extension +} function isOwned () { - return this.filename !== null + return this.remoteId === null } function toFormatedJSON () { + let podHost + + if (this.Author.Pod) { + podHost = this.Author.Pod.host + } else { + // It means it's our video + podHost = constants.CONFIG.WEBSERVER.HOST + } + const json = { - id: this._id, + id: this.id, name: this.name, description: this.description, - podUrl: this.podUrl.replace(/^https?:\/\//, ''), + podHost, isLocal: this.isOwned(), - magnetUri: this.magnetUri, - author: this.author, + magnetUri: this.generateMagnetUri(), + author: this.Author.name, duration: this.duration, - tags: this.tags, - thumbnailPath: constants.THUMBNAILS_STATIC_PATH + '/' + this.thumbnail, - createdDate: this.createdDate + views: this.views, + tags: map(this.Tags, 'name'), + thumbnailPath: pathUtils.join(constants.STATIC_PATHS.THUMBNAILS, this.getThumbnailName()), + createdAt: this.createdAt, + updatedAt: this.updatedAt } return json } -function toRemoteJSON (callback) { +function toAddRemoteJSON (callback) { const self = this - // Convert thumbnail to base64 - fs.readFile(pathUtils.join(thumbnailsDir, this.thumbnail), function (err, thumbnailData) { + // Get thumbnail data to send to the other pod + const thumbnailPath = pathUtils.join(constants.CONFIG.STORAGE.THUMBNAILS_DIR, this.getThumbnailName()) + fs.readFile(thumbnailPath, function (err, thumbnailData) { if (err) { logger.error('Cannot read the thumbnail of the video') return callback(err) @@ -169,22 +372,52 @@ function toRemoteJSON (callback) { const remoteVideo = { name: self.name, description: self.description, - magnetUri: self.magnetUri, - filename: null, - author: self.author, + infoHash: self.infoHash, + remoteId: self.id, + author: self.Author.name, duration: self.duration, - thumbnailBase64: new Buffer(thumbnailData).toString('base64'), - tags: self.tags, - createdDate: self.createdDate, - podUrl: self.podUrl + thumbnailData: thumbnailData.toString('binary'), + tags: map(self.Tags, 'name'), + createdAt: self.createdAt, + updatedAt: self.updatedAt, + extname: self.extname } return callback(null, remoteVideo) }) } +function toUpdateRemoteJSON (callback) { + const json = { + name: this.name, + description: this.description, + infoHash: this.infoHash, + remoteId: this.id, + author: this.Author.name, + duration: this.duration, + tags: map(this.Tags, 'name'), + createdAt: this.createdAt, + updatedAt: this.updatedAt, + extname: this.extname + } + + return json +} + // ------------------------------ STATICS ------------------------------ +function generateThumbnailFromData (video, thumbnailData, callback) { + // Creating the thumbnail for a remote video + + const thumbnailName = video.getThumbnailName() + const thumbnailPath = pathUtils.join(constants.CONFIG.STORAGE.THUMBNAILS_DIR, thumbnailName) + fs.writeFile(thumbnailPath, Buffer.from(thumbnailData, 'binary'), function (err) { + if (err) return callback(err) + + return callback(null, thumbnailName) + }) +} + function getDurationFromFile (videoPath, callback) { ffmpeg.ffprobe(videoPath, function (err, metadata) { if (err) return callback(err) @@ -193,129 +426,235 @@ function getDurationFromFile (videoPath, callback) { }) } -function list (start, count, sort, callback) { - const query = {} - return findWithCount.call(this, query, start, count, sort, callback) +function list (callback) { + return this.findAll().asCallback(callback) } -function listByUrlAndMagnet (fromUrl, magnetUri, callback) { - this.find({ podUrl: fromUrl, magnetUri: magnetUri }, callback) +function listForApi (start, count, sort, callback) { + const query = { + offset: start, + limit: count, + distinct: true, // For the count, a video can have many tags + order: [ modelUtils.getSort(sort), [ this.sequelize.models.Tag, 'name', 'ASC' ] ], + include: [ + { + model: this.sequelize.models.Author, + include: [ { model: this.sequelize.models.Pod, required: false } ] + }, + + this.sequelize.models.Tag + ] + } + + return this.findAndCountAll(query).asCallback(function (err, result) { + if (err) return callback(err) + + return callback(null, result.rows, result.count) + }) } -function listByUrls (fromUrls, callback) { - this.find({ podUrl: { $in: fromUrls } }, callback) +function loadByHostAndRemoteId (fromHost, remoteId, callback) { + const query = { + where: { + remoteId: remoteId + }, + include: [ + { + model: this.sequelize.models.Author, + include: [ + { + model: this.sequelize.models.Pod, + required: true, + where: { + host: fromHost + } + } + ] + } + ] + } + + return this.findOne(query).asCallback(callback) } -function listOwned (callback) { - // If filename is not null this is *our* video - this.find({ filename: { $ne: null } }, callback) +function listOwnedAndPopulateAuthorAndTags (callback) { + // If remoteId is null this is *our* video + const query = { + where: { + remoteId: null + }, + include: [ this.sequelize.models.Author, this.sequelize.models.Tag ] + } + + return this.findAll(query).asCallback(callback) } -function listRemotes (callback) { - this.find({ filename: null }, callback) +function listOwnedByAuthor (author, callback) { + const query = { + where: { + remoteId: null + }, + include: [ + { + model: this.sequelize.models.Author, + where: { + name: author + } + } + ] + } + + return this.findAll(query).asCallback(callback) } function load (id, callback) { - this.findById(id, callback) + return this.findById(id).asCallback(callback) } -function search (value, field, start, count, sort, callback) { - const query = {} - // Make an exact search with the magnet - if (field === 'magnetUri' || field === 'tags') { - query[field] = value - } else { - query[field] = new RegExp(value) +function loadAndPopulateAuthor (id, callback) { + const options = { + include: [ this.sequelize.models.Author ] } - findWithCount.call(this, query, start, count, sort, callback) + return this.findById(id, options).asCallback(callback) } -function seedAllExisting (callback) { - listOwned.call(this, function (err, videos) { - if (err) return callback(err) +function loadAndPopulateAuthorAndPodAndTags (id, callback) { + const options = { + include: [ + { + model: this.sequelize.models.Author, + include: [ { model: this.sequelize.models.Pod, required: false } ] + }, + this.sequelize.models.Tag + ] + } - eachLimit(videos, constants.SEEDS_IN_PARALLEL, function (video, callbackEach) { - const videoPath = pathUtils.join(uploadsDir, video.filename) - seed(videoPath, callbackEach) - }, callback) - }) + return this.findById(id, options).asCallback(callback) } -// --------------------------------------------------------------------------- +function searchAndPopulateAuthorAndPodAndTags (value, field, start, count, sort, callback) { + const podInclude = { + model: this.sequelize.models.Pod, + required: false + } -function findWithCount (query, start, count, sort, callback) { - const self = this + const authorInclude = { + model: this.sequelize.models.Author, + include: [ + podInclude + ] + } - parallel([ - function (asyncCallback) { - self.find(query).skip(start).limit(count).sort(sort).exec(asyncCallback) - }, - function (asyncCallback) { - self.count(query, asyncCallback) + const tagInclude = { + model: this.sequelize.models.Tag + } + + const query = { + where: {}, + offset: start, + limit: count, + distinct: true, // For the count, a video can have many tags + order: [ modelUtils.getSort(sort), [ this.sequelize.models.Tag, 'name', 'ASC' ] ] + } + + // Make an exact search with the magnet + if (field === 'magnetUri') { + const infoHash = magnetUtil.decode(value).infoHash + query.where.infoHash = infoHash + } else if (field === 'tags') { + const escapedValue = this.sequelize.escape('%' + value + '%') + query.where = { + id: { + $in: this.sequelize.literal( + '(SELECT "VideoTags"."videoId" FROM "Tags" INNER JOIN "VideoTags" ON "Tags"."id" = "VideoTags"."tagId" WHERE name LIKE ' + escapedValue + ')' + ) + } + } + } else if (field === 'host') { + // FIXME: Include our pod? (not stored in the database) + podInclude.where = { + host: { + $like: '%' + value + '%' + } + } + podInclude.required = true + } else if (field === 'author') { + authorInclude.where = { + name: { + $like: '%' + value + '%' + } + } + + // authorInclude.or = true + } else { + query.where[field] = { + $like: '%' + value + '%' } - ], function (err, results) { + } + + query.include = [ + authorInclude, tagInclude + ] + + if (tagInclude.where) { + // query.include.push([ this.sequelize.models.Tag ]) + } + + return this.findAndCountAll(query).asCallback(function (err, result) { if (err) return callback(err) - const videos = results[0] - const totalVideos = results[1] - return callback(null, videos, totalVideos) + return callback(null, result.rows, result.count) }) } +// --------------------------------------------------------------------------- + function removeThumbnail (video, callback) { - fs.unlink(thumbnailsDir + video.thumbnail, callback) + const thumbnailPath = pathUtils.join(constants.CONFIG.STORAGE.THUMBNAILS_DIR, video.getThumbnailName()) + fs.unlink(thumbnailPath, callback) } function removeFile (video, callback) { - fs.unlink(uploadsDir + video.filename, callback) + const filePath = pathUtils.join(constants.CONFIG.STORAGE.VIDEOS_DIR, video.getVideoFilename()) + fs.unlink(filePath, callback) } -// Maybe the torrent is not seeded, but we catch the error to don't stop the removing process function removeTorrent (video, callback) { - try { - webtorrent.remove(video.magnetUri, callback) - } catch (err) { - logger.warn('Cannot remove the torrent from WebTorrent', { err: err }) - return callback(null) - } + const torrenPath = pathUtils.join(constants.CONFIG.STORAGE.TORRENTS_DIR, video.getTorrentName()) + fs.unlink(torrenPath, callback) } -function createThumbnail (videoPath, callback) { - const filename = pathUtils.basename(videoPath) + '.jpg' - ffmpeg(videoPath) - .on('error', callback) - .on('end', function () { - callback(null, filename) - }) - .thumbnail({ - count: 1, - folder: thumbnailsDir, - size: constants.THUMBNAILS_SIZE, - filename: filename - }) +function removePreview (video, callback) { + // Same name than video thumnail + fs.unlink(constants.CONFIG.STORAGE.PREVIEWS_DIR + video.getPreviewName(), callback) } -function seed (path, callback) { - logger.info('Seeding %s...', path) - - webtorrent.seed(path, function (torrent) { - logger.info('%s seeded (%s).', path, torrent.magnetURI) +function createPreview (video, videoPath, callback) { + generateImage(video, videoPath, constants.CONFIG.STORAGE.PREVIEWS_DIR, video.getPreviewName(), callback) +} - return callback(null, torrent) - }) +function createThumbnail (video, videoPath, callback) { + generateImage(video, videoPath, constants.CONFIG.STORAGE.THUMBNAILS_DIR, video.getThumbnailName(), constants.THUMBNAILS_SIZE, callback) } -function generateThumbnailFromBase64 (data, callback) { - // Creating the thumbnail for this remote video - utils.generateRandomString(16, function (err, randomString) { - if (err) return callback(err) +function generateImage (video, videoPath, folder, imageName, size, callback) { + const options = { + filename: imageName, + count: 1, + folder + } - const thumbnailName = randomString + '.jpg' - fs.writeFile(thumbnailsDir + thumbnailName, data, { encoding: 'base64' }, function (err) { - if (err) return callback(err) + if (!callback) { + callback = size + } else { + options.size = size + } - return callback(null, thumbnailName) + ffmpeg(videoPath) + .on('error', callback) + .on('end', function () { + callback(null, imageName) }) - }) + .thumbnail(options) }