From 052937db8a8d282eccdbdf38d487ed8d85d3c0a7 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Tue, 26 Jul 2016 22:30:46 +0200 Subject: First draft using only webseed for server --- server/initializers/constants.js | 12 ++- server/lib/webtorrent-process.js | 92 ---------------------- server/lib/webtorrent.js | 161 --------------------------------------- server/models/video.js | 56 ++++++-------- 4 files changed, 30 insertions(+), 291 deletions(-) delete mode 100644 server/lib/webtorrent-process.js delete mode 100644 server/lib/webtorrent.js (limited to 'server') diff --git a/server/initializers/constants.js b/server/initializers/constants.js index 467816f2c..e0ea188af 100644 --- a/server/initializers/constants.js +++ b/server/initializers/constants.js @@ -49,12 +49,16 @@ const SORTABLE_COLUMNS = { VIDEOS: [ 'name', '-name', 'duration', '-duration', 'createdDate', '-createdDate' ] } +// Express static paths (router) +const STATIC_PATHS = { + THUMBNAILS: '/static/thumbnails', + TORRENTS: '/static/torrents/', + WEBSEED: '/static/webseed/' +} + // Videos thumbnail size const THUMBNAILS_SIZE = '200x110' -// Path for access to thumbnails with express router -const THUMBNAILS_STATIC_PATH = '/static/thumbnails' - const VIDEOS_CONSTRAINTS_FIELDS = { NAME: { min: 3, max: 50 }, // Length DESCRIPTION: { min: 3, max: 250 }, // Length @@ -89,8 +93,8 @@ module.exports = { SEARCHABLE_COLUMNS: SEARCHABLE_COLUMNS, SEEDS_IN_PARALLEL: SEEDS_IN_PARALLEL, SORTABLE_COLUMNS: SORTABLE_COLUMNS, + STATIC_PATHS: STATIC_PATHS, THUMBNAILS_SIZE: THUMBNAILS_SIZE, - THUMBNAILS_STATIC_PATH: THUMBNAILS_STATIC_PATH, VIDEOS_CONSTRAINTS_FIELDS: VIDEOS_CONSTRAINTS_FIELDS } diff --git a/server/lib/webtorrent-process.js b/server/lib/webtorrent-process.js deleted file mode 100644 index be7ac5bb4..000000000 --- a/server/lib/webtorrent-process.js +++ /dev/null @@ -1,92 +0,0 @@ -'use strict' - -const WebTorrent = require('webtorrent') -const ipc = require('node-ipc') - -function webtorrent (args) { - if (args.length !== 3) { - throw new Error('Wrong arguments number: ' + args.length + '/3') - } - - const host = args[1] - const port = args[2] - const nodeKey = 'webtorrentnode' + port - const processKey = 'webtorrentprocess' + port - - ipc.config.silent = true - ipc.config.id = processKey - - if (host === 'client' && port === '1') global.WEBTORRENT_ANNOUNCE = [] - else global.WEBTORRENT_ANNOUNCE = 'ws://' + host + ':' + port + '/tracker/socket' - const wt = new WebTorrent({ dht: false }) - - function seed (data) { - const args = data.args - const path = args.path - const _id = data._id - - wt.seed(path, { announceList: '' }, function (torrent) { - const toSend = { - magnetUri: torrent.magnetURI - } - - ipc.of[nodeKey].emit(nodeKey + '.seedDone.' + _id, toSend) - }) - } - - function add (data) { - const args = data.args - const magnetUri = args.magnetUri - const _id = data._id - - wt.add(magnetUri, function (torrent) { - const toSend = { - files: [] - } - - torrent.files.forEach(function (file) { - toSend.files.push({ path: file.path }) - }) - - ipc.of[nodeKey].emit(nodeKey + '.addDone.' + _id, toSend) - }) - } - - function remove (data) { - const args = data.args - const magnetUri = args.magnetUri - const _id = data._id - - try { - wt.remove(magnetUri, callback) - } catch (err) { - console.log('Cannot remove the torrent from WebTorrent.') - return callback(null) - } - - function callback () { - const toSend = {} - ipc.of[nodeKey].emit(nodeKey + '.removeDone.' + _id, toSend) - } - } - - console.log('Configuration: ' + host + ':' + port) - console.log('Connecting to IPC...') - - ipc.connectTo(nodeKey, function () { - ipc.of[nodeKey].on(processKey + '.seed', seed) - ipc.of[nodeKey].on(processKey + '.add', add) - ipc.of[nodeKey].on(processKey + '.remove', remove) - - ipc.of[nodeKey].emit(processKey + '.ready') - console.log('Ready.') - }) - - process.on('uncaughtException', function (e) { - ipc.of[nodeKey].emit(processKey + '.exception', { exception: e.toString() }) - }) -} - -// --------------------------------------------------------------------------- - -module.exports = webtorrent diff --git a/server/lib/webtorrent.js b/server/lib/webtorrent.js deleted file mode 100644 index bcd30139e..000000000 --- a/server/lib/webtorrent.js +++ /dev/null @@ -1,161 +0,0 @@ -'use strict' - -const config = require('config') -const ipc = require('node-ipc') -const pathUtils = require('path') -const spawn = require('electron-spawn') - -const logger = require('../helpers/logger') - -const electronDebug = config.get('electron.debug') -let host = config.get('webserver.host') -let port = config.get('webserver.port') -let nodeKey = 'webtorrentnode' + port -let processKey = 'webtorrentprocess' + port -ipc.config.silent = true -ipc.config.id = nodeKey - -const webtorrent = { - add: add, - app: null, // Pid of the app - create: create, - remove: remove, - seed: seed, - silent: false // Useful for beautiful tests -} - -function create (options, callback) { - if (typeof options === 'function') { - callback = options - options = {} - } - - // Override options - if (options.host) host = options.host - if (options.port) { - port = options.port - nodeKey = 'webtorrentnode' + port - processKey = 'webtorrentprocess' + port - ipc.config.id = nodeKey - } - - ipc.serve(function () { - if (!webtorrent.silent) logger.info('IPC server ready.') - - // Run a timeout of 30s after which we exit the process - const timeoutWebtorrentProcess = setTimeout(function () { - throw new Error('Timeout : cannot run the webtorrent process. Please ensure you have electron-prebuilt npm package installed with xvfb-run.') - }, 30000) - - ipc.server.on(processKey + '.ready', function () { - if (!webtorrent.silent) logger.info('Webtorrent process ready.') - clearTimeout(timeoutWebtorrentProcess) - callback() - }) - - ipc.server.on(processKey + '.exception', function (data) { - throw new Error('Received exception error from webtorrent process : ' + data.exception) - }) - - const webtorrentProcess = spawn(pathUtils.join(__dirname, 'webtorrent-process.js'), host, port, { detached: true }) - - if (electronDebug === true) { - webtorrentProcess.stderr.on('data', function (data) { - logger.debug('Webtorrent process stderr: ', data.toString()) - }) - - webtorrentProcess.stdout.on('data', function (data) { - logger.debug('Webtorrent process:', data.toString()) - }) - } - - webtorrent.app = webtorrentProcess - }) - - ipc.server.start() -} - -function seed (path, callback) { - const extension = pathUtils.extname(path) - const basename = pathUtils.basename(path, extension) - const data = { - _id: basename, - args: { - path: path - } - } - - if (!webtorrent.silent) logger.debug('Node wants to seed %s.', data._id) - - // Finish signal - const eventKey = nodeKey + '.seedDone.' + data._id - ipc.server.on(eventKey, function listener (received) { - if (!webtorrent.silent) logger.debug('Process seeded torrent %s.', received.magnetUri) - - // This is a fake object, we just use the magnetUri in this project - const torrent = { - magnetURI: received.magnetUri - } - - ipc.server.off(eventKey, '*') - callback(torrent) - }) - - ipc.server.broadcast(processKey + '.seed', data) -} - -function add (magnetUri, callback) { - const data = { - _id: magnetUri, - args: { - magnetUri: magnetUri - } - } - - if (!webtorrent.silent) logger.debug('Node wants to add ' + data._id) - - // Finish signal - const eventKey = nodeKey + '.addDone.' + data._id - ipc.server.on(eventKey, function (received) { - if (!webtorrent.silent) logger.debug('Process added torrent.') - - // This is a fake object, we just use the magnetUri in this project - const torrent = { - files: received.files - } - - ipc.server.off(eventKey, '*') - callback(torrent) - }) - - ipc.server.broadcast(processKey + '.add', data) -} - -function remove (magnetUri, callback) { - const data = { - _id: magnetUri, - args: { - magnetUri: magnetUri - } - } - - if (!webtorrent.silent) logger.debug('Node wants to stop seeding %s.', data._id) - - // Finish signal - const eventKey = nodeKey + '.removeDone.' + data._id - ipc.server.on(eventKey, function (received) { - if (!webtorrent.silent) logger.debug('Process removed torrent %s.', data._id) - - let err = null - if (received.err) err = received.err - - ipc.server.off(eventKey, '*') - callback(err) - }) - - ipc.server.broadcast(processKey + '.remove', data) -} - -// --------------------------------------------------------------------------- - -module.exports = webtorrent diff --git a/server/models/video.js b/server/models/video.js index 396aa505d..14e0df6f2 100644 --- a/server/models/video.js +++ b/server/models/video.js @@ -1,24 +1,27 @@ 'use strict' const config = require('config') -const eachLimit = require('async/eachLimit') +const createTorrent = require('create-torrent') const ffmpeg = require('fluent-ffmpeg') const fs = require('fs') const parallel = require('async/parallel') +const parseTorrent = require('parse-torrent') const pathUtils = require('path') +const magnet = require('magnet-uri') const mongoose = require('mongoose') const constants = require('../initializers/constants') const customValidators = require('../helpers/custom-validators') const logger = require('../helpers/logger') const utils = require('../helpers/utils') -const webtorrent = require('../lib/webtorrent') const http = config.get('webserver.https') === true ? 'https' : 'http' const host = config.get('webserver.host') const port = config.get('webserver.port') const uploadsDir = pathUtils.join(__dirname, '..', '..', config.get('storage.uploads')) const thumbnailsDir = pathUtils.join(__dirname, '..', '..', config.get('storage.thumbnails')) +const torrentsDir = pathUtils.join(__dirname, '..', '..', config.get('storage.torrents')) +const webseedBaseUrl = http + '://' + host + ':' + port + constants.STATIC_PATHS.WEBSEED // --------------------------------------------------------------------------- @@ -66,8 +69,7 @@ VideoSchema.statics = { listOwned: listOwned, listRemotes: listRemotes, load: load, - search: search, - seedAllExisting: seedAllExisting + search: search } VideoSchema.pre('remove', function (next) { @@ -103,8 +105,21 @@ VideoSchema.pre('save', function (next) { this.podUrl = http + '://' + host + ':' + port tasks.push( + // TODO: refractoring function (callback) { - seed(videoPath, callback) + createTorrent(videoPath, { announceList: [ [ 'ws://' + host + ':' + port + '/tracker/socket' ] ], urlList: [ webseedBaseUrl + video.filename ] }, function (err, torrent) { + if (err) return callback(err) + + fs.writeFile(torrentsDir + video.filename + '.torrent', torrent, function (err) { + if (err) return callback(err) + + const parsedTorrent = parseTorrent(torrent) + parsedTorrent.xs = video.podUrl + constants.STATIC_PATHS.TORRENTS + video.filename + '.torrent' + video.magnetUri = magnet.encode(parsedTorrent) + + callback(null) + }) + }) }, function (callback) { createThumbnail(videoPath, callback) @@ -114,7 +129,6 @@ VideoSchema.pre('save', function (next) { parallel(tasks, function (err, results) { if (err) return next(err) - video.magnetUri = results[0].magnetURI video.thumbnail = results[1] return next() @@ -149,7 +163,7 @@ function toFormatedJSON () { author: this.author, duration: this.duration, tags: this.tags, - thumbnailPath: constants.THUMBNAILS_STATIC_PATH + '/' + this.thumbnail, + thumbnailPath: constants.STATIC_PATHS.THUMBNAILS + '/' + this.thumbnail, createdDate: this.createdDate } @@ -231,17 +245,6 @@ function search (value, field, start, count, sort, callback) { findWithCount.call(this, query, start, count, sort, callback) } -function seedAllExisting (callback) { - listOwned.call(this, function (err, videos) { - if (err) return callback(err) - - eachLimit(videos, constants.SEEDS_IN_PARALLEL, function (video, callbackEach) { - const videoPath = pathUtils.join(uploadsDir, video.filename) - seed(videoPath, callbackEach) - }, callback) - }) -} - // --------------------------------------------------------------------------- function findWithCount (query, start, count, sort, callback) { @@ -273,12 +276,7 @@ function removeFile (video, callback) { // Maybe the torrent is not seeded, but we catch the error to don't stop the removing process function removeTorrent (video, callback) { - try { - webtorrent.remove(video.magnetUri, callback) - } catch (err) { - logger.warn('Cannot remove the torrent from WebTorrent', { err: err }) - return callback(null) - } + fs.unlink(torrentsDir + video.filename + '.torrent') } function createThumbnail (videoPath, callback) { @@ -296,16 +294,6 @@ function createThumbnail (videoPath, callback) { }) } -function seed (path, callback) { - logger.info('Seeding %s...', path) - - webtorrent.seed(path, function (torrent) { - logger.info('%s seeded (%s).', path, torrent.magnetURI) - - return callback(null, torrent) - }) -} - function generateThumbnailFromBase64 (data, callback) { // Creating the thumbnail for this remote video utils.generateRandomString(16, function (err, randomString) { -- cgit v1.2.3