First draft using only webseed for server

15 files changed, 58 insertions, 326 deletions

diff --git a/.gitignore b/.gitignore
index debe118e1..ec5b4b2f0 100644
--- a/.gitignore
+++ b/.gitignore
@@ -14,3 +14,4 @@ uploads
 thumbnails
 config/production.yaml
 ffmpeg
+torrents
diff --git a/client/package.json b/client/package.json
index b01e768b3..a5c5d092b 100644
--- a/client/package.json
+++ b/client/package.json
@@ -63,7 +63,7 @@
     "webpack-md5-hash": "0.0.5",
     "webpack-merge": "^0.13.0",
     "webpack-notifier": "^1.3.0",
-    "webtorrent": "^0.93.2",
+    "webtorrent": "^0.95.2",
     "zone.js": "0.6.12"
   },
   "devDependencies": {
diff --git a/config/default.yaml b/config/default.yaml
index b5fd7b06d..9a8a57879 100644
--- a/config/default.yaml
+++ b/config/default.yaml
@@ -17,6 +17,7 @@ storage:
   uploads: 'uploads/'
   logs: 'logs/'
   thumbnails: 'thumbnails/'
+  torrents: 'torrents/'
 
 network:
   friends: []
diff --git a/config/test-1.yaml b/config/test-1.yaml
index 3afcb1b04..0998eaea1 100644
--- a/config/test-1.yaml
+++ b/config/test-1.yaml
@@ -14,6 +14,7 @@ storage:
   uploads: 'test1/uploads/'
   logs: 'test1/logs/'
   thumbnails: 'test1/thumbnails/'
+  torrents: 'test1/torrents/'
 
 network:
   friends:
diff --git a/config/test-2.yaml b/config/test-2.yaml
index 51cc186eb..ec2cff811 100644
--- a/config/test-2.yaml
+++ b/config/test-2.yaml
@@ -14,6 +14,7 @@ storage:
   uploads: 'test2/uploads/'
   logs: 'test2/logs/'
   thumbnails: 'test2/thumbnails/'
+  torrents: 'test2/torrents/'
 
 network:
   friends:
diff --git a/config/test-3.yaml b/config/test-3.yaml
index 7ef01ba4d..24f5533e0 100644
--- a/config/test-3.yaml
+++ b/config/test-3.yaml
@@ -14,6 +14,7 @@ storage:
   uploads: 'test3/uploads/'
   logs: 'test3/logs/'
   thumbnails: 'test3/thumbnails/'
+  torrents: 'test3/torrents/'
 
 network:
   friends:
diff --git a/config/test-4.yaml b/config/test-4.yaml
index a4d3bb164..1f884dbf2 100644
--- a/config/test-4.yaml
+++ b/config/test-4.yaml
@@ -14,6 +14,7 @@ storage:
   uploads: 'test4/uploads/'
   logs: 'test4/logs/'
   thumbnails: 'test4/thumbnails/'
+  torrents: 'test4/torrents/'
 
 network:
   friends:
diff --git a/config/test-5.yaml b/config/test-5.yaml
index 0435c17fe..08ed9f068 100644
--- a/config/test-5.yaml
+++ b/config/test-5.yaml
@@ -14,6 +14,7 @@ storage:
   uploads: 'test5/uploads/'
   logs: 'test5/logs/'
   thumbnails: 'test5/thumbnails/'
+  torrents: 'test5/torrents/'
 
 network:
   friends:
diff --git a/config/test-6.yaml b/config/test-6.yaml
index b7dbd4bdd..a57784cca 100644
--- a/config/test-6.yaml
+++ b/config/test-6.yaml
@@ -14,6 +14,7 @@ storage:
   uploads: 'test6/uploads/'
   logs: 'test6/logs/'
   thumbnails: 'test6/thumbnails/'
+  torrents: 'test6/torrents/'
 
 network:
   friends:
diff --git a/package.json b/package.json
index 9a0d4ecaa..63d014376 100644
--- a/package.json
+++ b/package.json
@@ -39,28 +39,28 @@
     "body-parser": "^1.12.4",
     "concurrently": "^2.0.0",
     "config": "^1.14.0",
+    "create-torrent": "^3.24.5",
     "debug": "^2.2.0",
     "dezalgo": "^1.0.3",
-    "electron-spawn": "https://github.com/Chocobozzz/electron-spawn",
     "express": "^4.12.4",
     "express-oauth-server": "https://github.com/Chocobozzz/express-oauth-server",
     "express-validator": "^2.11.0",
     "fluent-ffmpeg": "^2.1.0",
     "js-yaml": "^3.5.4",
     "lodash": "^4.11.1",
+    "magnet-uri": "^5.1.4",
     "mkdirp": "^0.5.1",
     "mongoose": "^4.0.5",
     "morgan": "^1.5.3",
     "multer": "^1.1.0",
-    "node-ipc": "^8.0.0",
     "openssl-wrapper": "^0.3.4",
+    "parse-torrent": "^5.8.0",
     "password-generator": "^2.0.2",
     "request": "^2.57.0",
     "request-replay": "^1.0.2",
     "scripty": "^1.5.0",
     "segfault-handler": "^1.0.0",
     "ursa": "^0.9.1",
-    "webtorrent": "^0.93.2",
     "winston": "^2.1.1",
     "ws": "^1.1.1"
   },
diff --git a/server.js b/server.js
index b2eeeff70..0033ed1db 100644
--- a/server.js
+++ b/server.js
@@ -34,10 +34,7 @@ const customValidators = require('./server/helpers/custom-validators')
 const installer = require('./server/initializers/installer')
 const mongoose = require('mongoose')
 const routes = require('./server/controllers')
-const utils = require('./server/helpers/utils')
-const webtorrent = require('./server/lib/webtorrent')
 const Request = mongoose.model('Request')
-const Video = mongoose.model('Video')
 
 // Get configurations
 const port = config.get('listen.port')
@@ -72,9 +69,16 @@ app.use('/client/*', function (req, res, next) {
   res.sendStatus(404)
 })
 
+const torrentsPhysicalPath = path.join(__dirname, config.get('storage.torrents'))
+app.use(constants.STATIC_PATHS.TORRENTS, express.static(torrentsPhysicalPath, { maxAge: 0 }))
+
+// Uploads path for webseeding
+const uploadsPhysicalPath = path.join(__dirname, config.get('storage.uploads'))
+app.use(constants.STATIC_PATHS.WEBSEED, express.static(uploadsPhysicalPath, { maxAge: 0 }))
+
 // Thumbnails path for express
 const thumbnailsPhysicalPath = path.join(__dirname, config.get('storage.thumbnails'))
-app.use(constants.THUMBNAILS_STATIC_PATH, express.static(thumbnailsPhysicalPath, { maxAge: 0 }))
+app.use(constants.STATIC_PATHS.THUMBNAILS, express.static(thumbnailsPhysicalPath, { maxAge: 0 }))
 
 // Client application
 app.use('/*', function (req, res, next) {
@@ -121,33 +125,14 @@ app.use(function (err, req, res, next) {
 installer.installApplication(function (err) {
   if (err) throw err
 
-  // Create/activate the webtorrent module
-  webtorrent.create(function () {
-    function cleanForExit () {
-      utils.cleanForExit(webtorrent.app)
-    }
-
-    function exitGracefullyOnSignal () {
-      process.exit(-1)
-    }
-
-    process.on('exit', cleanForExit)
-    process.on('SIGINT', exitGracefullyOnSignal)
-    process.on('SIGTERM', exitGracefullyOnSignal)
-
-    // ----------- Make the server listening -----------
-    server.listen(port, function () {
-      // Activate the pool requests
-      Request.activate()
-
-      Video.seedAllExisting(function (err) {
-        if (err) throw err
-
-        logger.info('Seeded all the videos')
-        logger.info('Server listening on port %d', port)
-        app.emit('ready')
-      })
-    })
+  // ----------- Make the server listening -----------
+  server.listen(port, function () {
+    // Activate the pool requests
+    Request.activate()
+
+    logger.info('Seeded all the videos')
+    logger.info('Server listening on port %d', port)
+    app.emit('ready')
   })
 })
 
diff --git a/server/initializers/constants.js b/server/initializers/constants.js
index 467816f2c..e0ea188af 100644
--- a/server/initializers/constants.js
+++ b/server/initializers/constants.js
@@ -49,12 +49,16 @@ const SORTABLE_COLUMNS = {
   VIDEOS: [ 'name', '-name', 'duration', '-duration', 'createdDate', '-createdDate' ]
 }
 
+// Express static paths (router)
+const STATIC_PATHS = {
+  THUMBNAILS: '/static/thumbnails',
+  TORRENTS: '/static/torrents/',
+  WEBSEED: '/static/webseed/'
+}
+
 // Videos thumbnail size
 const THUMBNAILS_SIZE = '200x110'
 
-// Path for access to thumbnails with express router
-const THUMBNAILS_STATIC_PATH = '/static/thumbnails'
-
 const VIDEOS_CONSTRAINTS_FIELDS = {
   NAME: { min: 3, max: 50 }, // Length
   DESCRIPTION: { min: 3, max: 250 }, // Length
@@ -89,8 +93,8 @@ module.exports = {
   SEARCHABLE_COLUMNS: SEARCHABLE_COLUMNS,
   SEEDS_IN_PARALLEL: SEEDS_IN_PARALLEL,
   SORTABLE_COLUMNS: SORTABLE_COLUMNS,
+  STATIC_PATHS: STATIC_PATHS,
   THUMBNAILS_SIZE: THUMBNAILS_SIZE,
-  THUMBNAILS_STATIC_PATH: THUMBNAILS_STATIC_PATH,
   VIDEOS_CONSTRAINTS_FIELDS: VIDEOS_CONSTRAINTS_FIELDS
 }
 
diff --git a/server/lib/webtorrent-process.js b/server/lib/webtorrent-process.js
deleted file mode 100644
index be7ac5bb4..000000000
--- a/server/lib/webtorrent-process.js
+++ /dev/null
@@ -1,92 +0,0 @@
-'use strict'
-
-const WebTorrent = require('webtorrent')
-const ipc = require('node-ipc')
-
-function webtorrent (args) {
-  if (args.length !== 3) {
-    throw new Error('Wrong arguments number: ' + args.length + '/3')
-  }
-
-  const host = args[1]
-  const port = args[2]
-  const nodeKey = 'webtorrentnode' + port
-  const processKey = 'webtorrentprocess' + port
-
-  ipc.config.silent = true
-  ipc.config.id = processKey
-
-  if (host === 'client' && port === '1') global.WEBTORRENT_ANNOUNCE = []
-  else global.WEBTORRENT_ANNOUNCE = 'ws://' + host + ':' + port + '/tracker/socket'
-  const wt = new WebTorrent({ dht: false })
-
-  function seed (data) {
-    const args = data.args
-    const path = args.path
-    const _id = data._id
-
-    wt.seed(path, { announceList: '' }, function (torrent) {
-      const toSend = {
-        magnetUri: torrent.magnetURI
-      }
-
-      ipc.of[nodeKey].emit(nodeKey + '.seedDone.' + _id, toSend)
-    })
-  }
-
-  function add (data) {
-    const args = data.args
-    const magnetUri = args.magnetUri
-    const _id = data._id
-
-    wt.add(magnetUri, function (torrent) {
-      const toSend = {
-        files: []
-      }
-
-      torrent.files.forEach(function (file) {
-        toSend.files.push({ path: file.path })
-      })
-
-      ipc.of[nodeKey].emit(nodeKey + '.addDone.' + _id, toSend)
-    })
-  }
-
-  function remove (data) {
-    const args = data.args
-    const magnetUri = args.magnetUri
-    const _id = data._id
-
-    try {
-      wt.remove(magnetUri, callback)
-    } catch (err) {
-      console.log('Cannot remove the torrent from WebTorrent.')
-      return callback(null)
-    }
-
-    function callback () {
-      const toSend = {}
-      ipc.of[nodeKey].emit(nodeKey + '.removeDone.' + _id, toSend)
-    }
-  }
-
-  console.log('Configuration: ' + host + ':' + port)
-  console.log('Connecting to IPC...')
-
-  ipc.connectTo(nodeKey, function () {
-    ipc.of[nodeKey].on(processKey + '.seed', seed)
-    ipc.of[nodeKey].on(processKey + '.add', add)
-    ipc.of[nodeKey].on(processKey + '.remove', remove)
-
-    ipc.of[nodeKey].emit(processKey + '.ready')
-    console.log('Ready.')
-  })
-
-  process.on('uncaughtException', function (e) {
-    ipc.of[nodeKey].emit(processKey + '.exception', { exception: e.toString() })
-  })
-}
-
-// ---------------------------------------------------------------------------
-
-module.exports = webtorrent
diff --git a/server/lib/webtorrent.js b/server/lib/webtorrent.js
deleted file mode 100644
index bcd30139e..000000000
--- a/server/lib/webtorrent.js
+++ /dev/null
@@ -1,161 +0,0 @@
-'use strict'
-
-const config = require('config')
-const ipc = require('node-ipc')
-const pathUtils = require('path')
-const spawn = require('electron-spawn')
-
-const logger = require('../helpers/logger')
-
-const electronDebug = config.get('electron.debug')
-let host = config.get('webserver.host')
-let port = config.get('webserver.port')
-let nodeKey = 'webtorrentnode' + port
-let processKey = 'webtorrentprocess' + port
-ipc.config.silent = true
-ipc.config.id = nodeKey
-
-const webtorrent = {
-  add: add,
-  app: null, // Pid of the app
-  create: create,
-  remove: remove,
-  seed: seed,
-  silent: false // Useful for beautiful tests
-}
-
-function create (options, callback) {
-  if (typeof options === 'function') {
-    callback = options
-    options = {}
-  }
-
-  // Override options
-  if (options.host) host = options.host
-  if (options.port) {
-    port = options.port
-    nodeKey = 'webtorrentnode' + port
-    processKey = 'webtorrentprocess' + port
-    ipc.config.id = nodeKey
-  }
-
-  ipc.serve(function () {
-    if (!webtorrent.silent) logger.info('IPC server ready.')
-
-    // Run a timeout of 30s after which we exit the process
-    const timeoutWebtorrentProcess = setTimeout(function () {
-      throw new Error('Timeout : cannot run the webtorrent process. Please ensure you have electron-prebuilt npm package installed with xvfb-run.')
-    }, 30000)
-
-    ipc.server.on(processKey + '.ready', function () {
-      if (!webtorrent.silent) logger.info('Webtorrent process ready.')
-      clearTimeout(timeoutWebtorrentProcess)
-      callback()
-    })
-
-    ipc.server.on(processKey + '.exception', function (data) {
-      throw new Error('Received exception error from webtorrent process : ' + data.exception)
-    })
-
-    const webtorrentProcess = spawn(pathUtils.join(__dirname, 'webtorrent-process.js'), host, port, { detached: true })
-
-    if (electronDebug === true) {
-      webtorrentProcess.stderr.on('data', function (data) {
-        logger.debug('Webtorrent process stderr: ', data.toString())
-      })
-
-      webtorrentProcess.stdout.on('data', function (data) {
-        logger.debug('Webtorrent process:', data.toString())
-      })
-    }
-
-    webtorrent.app = webtorrentProcess
-  })
-
-  ipc.server.start()
-}
-
-function seed (path, callback) {
-  const extension = pathUtils.extname(path)
-  const basename = pathUtils.basename(path, extension)
-  const data = {
-    _id: basename,
-    args: {
-      path: path
-    }
-  }
-
-  if (!webtorrent.silent) logger.debug('Node wants to seed %s.', data._id)
-
-  // Finish signal
-  const eventKey = nodeKey + '.seedDone.' + data._id
-  ipc.server.on(eventKey, function listener (received) {
-    if (!webtorrent.silent) logger.debug('Process seeded torrent %s.', received.magnetUri)
-
-    // This is a fake object, we just use the magnetUri in this project
-    const torrent = {
-      magnetURI: received.magnetUri
-    }
-
-    ipc.server.off(eventKey, '*')
-    callback(torrent)
-  })
-
-  ipc.server.broadcast(processKey + '.seed', data)
-}
-
-function add (magnetUri, callback) {
-  const data = {
-    _id: magnetUri,
-    args: {
-      magnetUri: magnetUri
-    }
-  }
-
-  if (!webtorrent.silent) logger.debug('Node wants to add ' + data._id)
-
-  // Finish signal
-  const eventKey = nodeKey + '.addDone.' + data._id
-  ipc.server.on(eventKey, function (received) {
-    if (!webtorrent.silent) logger.debug('Process added torrent.')
-
-    // This is a fake object, we just use the magnetUri in this project
-    const torrent = {
-      files: received.files
-    }
-
-    ipc.server.off(eventKey, '*')
-    callback(torrent)
-  })
-
-  ipc.server.broadcast(processKey + '.add', data)
-}
-
-function remove (magnetUri, callback) {
-  const data = {
-    _id: magnetUri,
-    args: {
-      magnetUri: magnetUri
-    }
-  }
-
-  if (!webtorrent.silent) logger.debug('Node wants to stop seeding %s.', data._id)
-
-  // Finish signal
-  const eventKey = nodeKey + '.removeDone.' + data._id
-  ipc.server.on(eventKey, function (received) {
-    if (!webtorrent.silent) logger.debug('Process removed torrent %s.', data._id)
-
-    let err = null
-    if (received.err) err = received.err
-
-    ipc.server.off(eventKey, '*')
-    callback(err)
-  })
-
-  ipc.server.broadcast(processKey + '.remove', data)
-}
-
-// ---------------------------------------------------------------------------
-
-module.exports = webtorrent
diff --git a/server/models/video.js b/server/models/video.js
index 396aa505d..14e0df6f2 100644
--- a/server/models/video.js
+++ b/server/models/video.js
@@ -1,24 +1,27 @@
 'use strict'
 
 const config = require('config')
-const eachLimit = require('async/eachLimit')
+const createTorrent = require('create-torrent')
 const ffmpeg = require('fluent-ffmpeg')
 const fs = require('fs')
 const parallel = require('async/parallel')
+const parseTorrent = require('parse-torrent')
 const pathUtils = require('path')
+const magnet = require('magnet-uri')
 const mongoose = require('mongoose')
 
 const constants = require('../initializers/constants')
 const customValidators = require('../helpers/custom-validators')
 const logger = require('../helpers/logger')
 const utils = require('../helpers/utils')
-const webtorrent = require('../lib/webtorrent')
 
 const http = config.get('webserver.https') === true ? 'https' : 'http'
 const host = config.get('webserver.host')
 const port = config.get('webserver.port')
 const uploadsDir = pathUtils.join(__dirname, '..', '..', config.get('storage.uploads'))
 const thumbnailsDir = pathUtils.join(__dirname, '..', '..', config.get('storage.thumbnails'))
+const torrentsDir = pathUtils.join(__dirname, '..', '..', config.get('storage.torrents'))
+const webseedBaseUrl = http + '://' + host + ':' + port + constants.STATIC_PATHS.WEBSEED
 
 // ---------------------------------------------------------------------------
 
@@ -66,8 +69,7 @@ VideoSchema.statics = {
   listOwned: listOwned,
   listRemotes: listRemotes,
   load: load,
-  search: search,
-  seedAllExisting: seedAllExisting
+  search: search
 }
 
 VideoSchema.pre('remove', function (next) {
@@ -103,8 +105,21 @@ VideoSchema.pre('save', function (next) {
     this.podUrl = http + '://' + host + ':' + port
 
     tasks.push(
+      // TODO: refractoring
       function (callback) {
-        seed(videoPath, callback)
+        createTorrent(videoPath, { announceList: [ [ 'ws://' + host + ':' + port + '/tracker/socket' ] ], urlList: [ webseedBaseUrl + video.filename ] }, function (err, torrent) {
+          if (err) return callback(err)
+
+          fs.writeFile(torrentsDir + video.filename + '.torrent', torrent, function (err) {
+            if (err) return callback(err)
+
+            const parsedTorrent = parseTorrent(torrent)
+            parsedTorrent.xs = video.podUrl + constants.STATIC_PATHS.TORRENTS + video.filename + '.torrent'
+            video.magnetUri = magnet.encode(parsedTorrent)
+
+            callback(null)
+          })
+        })
       },
       function (callback) {
         createThumbnail(videoPath, callback)
@@ -114,7 +129,6 @@ VideoSchema.pre('save', function (next) {
     parallel(tasks, function (err, results) {
       if (err) return next(err)
 
-      video.magnetUri = results[0].magnetURI
       video.thumbnail = results[1]
 
       return next()
@@ -149,7 +163,7 @@ function toFormatedJSON () {
     author: this.author,
     duration: this.duration,
     tags: this.tags,
-    thumbnailPath: constants.THUMBNAILS_STATIC_PATH + '/' + this.thumbnail,
+    thumbnailPath: constants.STATIC_PATHS.THUMBNAILS + '/' + this.thumbnail,
     createdDate: this.createdDate
   }
 
@@ -231,17 +245,6 @@ function search (value, field, start, count, sort, callback) {
   findWithCount.call(this, query, start, count, sort, callback)
 }
 
-function seedAllExisting (callback) {
-  listOwned.call(this, function (err, videos) {
-    if (err) return callback(err)
-
-    eachLimit(videos, constants.SEEDS_IN_PARALLEL, function (video, callbackEach) {
-      const videoPath = pathUtils.join(uploadsDir, video.filename)
-      seed(videoPath, callbackEach)
-    }, callback)
-  })
-}
-
 // ---------------------------------------------------------------------------
 
 function findWithCount (query, start, count, sort, callback) {
@@ -273,12 +276,7 @@ function removeFile (video, callback) {
 
 // Maybe the torrent is not seeded, but we catch the error to don't stop the removing process
 function removeTorrent (video, callback) {
-  try {
-    webtorrent.remove(video.magnetUri, callback)
-  } catch (err) {
-    logger.warn('Cannot remove the torrent from WebTorrent', { err: err })
-    return callback(null)
-  }
+  fs.unlink(torrentsDir + video.filename + '.torrent')
 }
 
 function createThumbnail (videoPath, callback) {
@@ -296,16 +294,6 @@ function createThumbnail (videoPath, callback) {
     })
 }
 
-function seed (path, callback) {
-  logger.info('Seeding %s...', path)
-
-  webtorrent.seed(path, function (torrent) {
-    logger.info('%s seeded (%s).', path, torrent.magnetURI)
-
-    return callback(null, torrent)
-  })
-}
-
 function generateThumbnailFromBase64 (data, callback) {
   // Creating the thumbnail for this remote video
   utils.generateRandomString(16, function (err, randomString) {