diff options
Diffstat (limited to 'server')
-rw-r--r-- | server/initializers/constants.js | 12 | ||||
-rw-r--r-- | server/lib/webtorrent-process.js | 92 | ||||
-rw-r--r-- | server/lib/webtorrent.js | 160 | ||||
-rw-r--r-- | server/models/video.js | 63 |
4 files changed, 37 insertions, 290 deletions
diff --git a/server/initializers/constants.js b/server/initializers/constants.js index b1d033377..be2e3e943 100644 --- a/server/initializers/constants.js +++ b/server/initializers/constants.js | |||
@@ -122,12 +122,16 @@ const RETRY_REQUESTS = 5 | |||
122 | // Password encryption | 122 | // Password encryption |
123 | const BCRYPT_SALT_SIZE = 10 | 123 | const BCRYPT_SALT_SIZE = 10 |
124 | 124 | ||
125 | // Express static paths (router) | ||
126 | const STATIC_PATHS = { | ||
127 | THUMBNAILS: '/static/thumbnails', | ||
128 | TORRENTS: '/static/torrents/', | ||
129 | WEBSEED: '/static/webseed/' | ||
130 | } | ||
131 | |||
125 | // Videos thumbnail size | 132 | // Videos thumbnail size |
126 | const THUMBNAILS_SIZE = '200x110' | 133 | const THUMBNAILS_SIZE = '200x110' |
127 | 134 | ||
128 | // Path for access to thumbnails with express router | ||
129 | const THUMBNAILS_STATIC_PATH = '/static/thumbnails' | ||
130 | |||
131 | const USER_ROLES = { | 135 | const USER_ROLES = { |
132 | ADMIN: 'admin', | 136 | ADMIN: 'admin', |
133 | USER: 'user' | 137 | USER: 'user' |
@@ -166,8 +170,8 @@ module.exports = { | |||
166 | SEARCHABLE_COLUMNS, | 170 | SEARCHABLE_COLUMNS, |
167 | SEEDS_IN_PARALLEL, | 171 | SEEDS_IN_PARALLEL, |
168 | SORTABLE_COLUMNS, | 172 | SORTABLE_COLUMNS, |
173 | STATIC_PATHS, | ||
169 | THUMBNAILS_SIZE, | 174 | THUMBNAILS_SIZE, |
170 | THUMBNAILS_STATIC_PATH, | ||
171 | USER_ROLES | 175 | USER_ROLES |
172 | } | 176 | } |
173 | 177 | ||
diff --git a/server/lib/webtorrent-process.js b/server/lib/webtorrent-process.js deleted file mode 100644 index be7ac5bb4..000000000 --- a/server/lib/webtorrent-process.js +++ /dev/null | |||
@@ -1,92 +0,0 @@ | |||
1 | 'use strict' | ||
2 | |||
3 | const WebTorrent = require('webtorrent') | ||
4 | const ipc = require('node-ipc') | ||
5 | |||
6 | function webtorrent (args) { | ||
7 | if (args.length !== 3) { | ||
8 | throw new Error('Wrong arguments number: ' + args.length + '/3') | ||
9 | } | ||
10 | |||
11 | const host = args[1] | ||
12 | const port = args[2] | ||
13 | const nodeKey = 'webtorrentnode' + port | ||
14 | const processKey = 'webtorrentprocess' + port | ||
15 | |||
16 | ipc.config.silent = true | ||
17 | ipc.config.id = processKey | ||
18 | |||
19 | if (host === 'client' && port === '1') global.WEBTORRENT_ANNOUNCE = [] | ||
20 | else global.WEBTORRENT_ANNOUNCE = 'ws://' + host + ':' + port + '/tracker/socket' | ||
21 | const wt = new WebTorrent({ dht: false }) | ||
22 | |||
23 | function seed (data) { | ||
24 | const args = data.args | ||
25 | const path = args.path | ||
26 | const _id = data._id | ||
27 | |||
28 | wt.seed(path, { announceList: '' }, function (torrent) { | ||
29 | const toSend = { | ||
30 | magnetUri: torrent.magnetURI | ||
31 | } | ||
32 | |||
33 | ipc.of[nodeKey].emit(nodeKey + '.seedDone.' + _id, toSend) | ||
34 | }) | ||
35 | } | ||
36 | |||
37 | function add (data) { | ||
38 | const args = data.args | ||
39 | const magnetUri = args.magnetUri | ||
40 | const _id = data._id | ||
41 | |||
42 | wt.add(magnetUri, function (torrent) { | ||
43 | const toSend = { | ||
44 | files: [] | ||
45 | } | ||
46 | |||
47 | torrent.files.forEach(function (file) { | ||
48 | toSend.files.push({ path: file.path }) | ||
49 | }) | ||
50 | |||
51 | ipc.of[nodeKey].emit(nodeKey + '.addDone.' + _id, toSend) | ||
52 | }) | ||
53 | } | ||
54 | |||
55 | function remove (data) { | ||
56 | const args = data.args | ||
57 | const magnetUri = args.magnetUri | ||
58 | const _id = data._id | ||
59 | |||
60 | try { | ||
61 | wt.remove(magnetUri, callback) | ||
62 | } catch (err) { | ||
63 | console.log('Cannot remove the torrent from WebTorrent.') | ||
64 | return callback(null) | ||
65 | } | ||
66 | |||
67 | function callback () { | ||
68 | const toSend = {} | ||
69 | ipc.of[nodeKey].emit(nodeKey + '.removeDone.' + _id, toSend) | ||
70 | } | ||
71 | } | ||
72 | |||
73 | console.log('Configuration: ' + host + ':' + port) | ||
74 | console.log('Connecting to IPC...') | ||
75 | |||
76 | ipc.connectTo(nodeKey, function () { | ||
77 | ipc.of[nodeKey].on(processKey + '.seed', seed) | ||
78 | ipc.of[nodeKey].on(processKey + '.add', add) | ||
79 | ipc.of[nodeKey].on(processKey + '.remove', remove) | ||
80 | |||
81 | ipc.of[nodeKey].emit(processKey + '.ready') | ||
82 | console.log('Ready.') | ||
83 | }) | ||
84 | |||
85 | process.on('uncaughtException', function (e) { | ||
86 | ipc.of[nodeKey].emit(processKey + '.exception', { exception: e.toString() }) | ||
87 | }) | ||
88 | } | ||
89 | |||
90 | // --------------------------------------------------------------------------- | ||
91 | |||
92 | module.exports = webtorrent | ||
diff --git a/server/lib/webtorrent.js b/server/lib/webtorrent.js deleted file mode 100644 index dde046828..000000000 --- a/server/lib/webtorrent.js +++ /dev/null | |||
@@ -1,160 +0,0 @@ | |||
1 | 'use strict' | ||
2 | |||
3 | const ipc = require('node-ipc') | ||
4 | const pathUtils = require('path') | ||
5 | const spawn = require('electron-spawn') | ||
6 | |||
7 | const constants = require('../initializers/constants') | ||
8 | const logger = require('../helpers/logger') | ||
9 | |||
10 | let host = constants.CONFIG.WEBSERVER.HOST | ||
11 | let port = constants.CONFIG.WEBSERVER.PORT | ||
12 | let nodeKey = 'webtorrentnode' + port | ||
13 | let processKey = 'webtorrentprocess' + port | ||
14 | ipc.config.silent = true | ||
15 | ipc.config.id = nodeKey | ||
16 | |||
17 | const webtorrent = { | ||
18 | add: add, | ||
19 | app: null, // Pid of the app | ||
20 | create: create, | ||
21 | remove: remove, | ||
22 | seed: seed, | ||
23 | silent: false // Useful for beautiful tests | ||
24 | } | ||
25 | |||
26 | function create (options, callback) { | ||
27 | if (typeof options === 'function') { | ||
28 | callback = options | ||
29 | options = {} | ||
30 | } | ||
31 | |||
32 | // Override options | ||
33 | if (options.host) host = options.host | ||
34 | if (options.port) { | ||
35 | port = options.port | ||
36 | nodeKey = 'webtorrentnode' + port | ||
37 | processKey = 'webtorrentprocess' + port | ||
38 | ipc.config.id = nodeKey | ||
39 | } | ||
40 | |||
41 | ipc.serve(function () { | ||
42 | if (!webtorrent.silent) logger.info('IPC server ready.') | ||
43 | |||
44 | // Run a timeout of 30s after which we exit the process | ||
45 | const timeoutWebtorrentProcess = setTimeout(function () { | ||
46 | throw new Error('Timeout : cannot run the webtorrent process. Please ensure you have electron npm package installed with xvfb-run.') | ||
47 | }, 30000) | ||
48 | |||
49 | ipc.server.on(processKey + '.ready', function () { | ||
50 | if (!webtorrent.silent) logger.info('Webtorrent process ready.') | ||
51 | clearTimeout(timeoutWebtorrentProcess) | ||
52 | callback() | ||
53 | }) | ||
54 | |||
55 | ipc.server.on(processKey + '.exception', function (data) { | ||
56 | throw new Error('Received exception error from webtorrent process : ' + data.exception) | ||
57 | }) | ||
58 | |||
59 | const webtorrentProcess = spawn(pathUtils.join(__dirname, 'webtorrent-process.js'), host, port, { detached: true }) | ||
60 | |||
61 | if (constants.CONFIG.ELECTRON.DEBUG === true) { | ||
62 | webtorrentProcess.stderr.on('data', function (data) { | ||
63 | logger.debug('Webtorrent process stderr: ', data.toString()) | ||
64 | }) | ||
65 | |||
66 | webtorrentProcess.stdout.on('data', function (data) { | ||
67 | logger.debug('Webtorrent process:', data.toString()) | ||
68 | }) | ||
69 | } | ||
70 | |||
71 | webtorrent.app = webtorrentProcess | ||
72 | }) | ||
73 | |||
74 | ipc.server.start() | ||
75 | } | ||
76 | |||
77 | function seed (path, callback) { | ||
78 | const extension = pathUtils.extname(path) | ||
79 | const basename = pathUtils.basename(path, extension) | ||
80 | const data = { | ||
81 | _id: basename, | ||
82 | args: { | ||
83 | path: path | ||
84 | } | ||
85 | } | ||
86 | |||
87 | if (!webtorrent.silent) logger.debug('Node wants to seed %s.', data._id) | ||
88 | |||
89 | // Finish signal | ||
90 | const eventKey = nodeKey + '.seedDone.' + data._id | ||
91 | ipc.server.on(eventKey, function listener (received) { | ||
92 | if (!webtorrent.silent) logger.debug('Process seeded torrent %s.', received.magnetUri) | ||
93 | |||
94 | // This is a fake object, we just use the magnetUri in this project | ||
95 | const torrent = { | ||
96 | magnetURI: received.magnetUri | ||
97 | } | ||
98 | |||
99 | ipc.server.off(eventKey, '*') | ||
100 | callback(torrent) | ||
101 | }) | ||
102 | |||
103 | ipc.server.broadcast(processKey + '.seed', data) | ||
104 | } | ||
105 | |||
106 | function add (magnetUri, callback) { | ||
107 | const data = { | ||
108 | _id: magnetUri, | ||
109 | args: { | ||
110 | magnetUri: magnetUri | ||
111 | } | ||
112 | } | ||
113 | |||
114 | if (!webtorrent.silent) logger.debug('Node wants to add ' + data._id) | ||
115 | |||
116 | // Finish signal | ||
117 | const eventKey = nodeKey + '.addDone.' + data._id | ||
118 | ipc.server.on(eventKey, function (received) { | ||
119 | if (!webtorrent.silent) logger.debug('Process added torrent.') | ||
120 | |||
121 | // This is a fake object, we just use the magnetUri in this project | ||
122 | const torrent = { | ||
123 | files: received.files | ||
124 | } | ||
125 | |||
126 | ipc.server.off(eventKey, '*') | ||
127 | callback(torrent) | ||
128 | }) | ||
129 | |||
130 | ipc.server.broadcast(processKey + '.add', data) | ||
131 | } | ||
132 | |||
133 | function remove (magnetUri, callback) { | ||
134 | const data = { | ||
135 | _id: magnetUri, | ||
136 | args: { | ||
137 | magnetUri: magnetUri | ||
138 | } | ||
139 | } | ||
140 | |||
141 | if (!webtorrent.silent) logger.debug('Node wants to stop seeding %s.', data._id) | ||
142 | |||
143 | // Finish signal | ||
144 | const eventKey = nodeKey + '.removeDone.' + data._id | ||
145 | ipc.server.on(eventKey, function (received) { | ||
146 | if (!webtorrent.silent) logger.debug('Process removed torrent %s.', data._id) | ||
147 | |||
148 | let err = null | ||
149 | if (received.err) err = received.err | ||
150 | |||
151 | ipc.server.off(eventKey, '*') | ||
152 | callback(err) | ||
153 | }) | ||
154 | |||
155 | ipc.server.broadcast(processKey + '.remove', data) | ||
156 | } | ||
157 | |||
158 | // --------------------------------------------------------------------------- | ||
159 | |||
160 | module.exports = webtorrent | ||
diff --git a/server/models/video.js b/server/models/video.js index b9999c8f6..7d073cffa 100644 --- a/server/models/video.js +++ b/server/models/video.js | |||
@@ -1,10 +1,13 @@ | |||
1 | 'use strict' | 1 | 'use strict' |
2 | 2 | ||
3 | const eachLimit = require('async/eachLimit') | 3 | const config = require('config') |
4 | const createTorrent = require('create-torrent') | ||
4 | const ffmpeg = require('fluent-ffmpeg') | 5 | const ffmpeg = require('fluent-ffmpeg') |
5 | const fs = require('fs') | 6 | const fs = require('fs') |
6 | const parallel = require('async/parallel') | 7 | const parallel = require('async/parallel') |
8 | const parseTorrent = require('parse-torrent') | ||
7 | const pathUtils = require('path') | 9 | const pathUtils = require('path') |
10 | const magnet = require('magnet-uri') | ||
8 | const mongoose = require('mongoose') | 11 | const mongoose = require('mongoose') |
9 | 12 | ||
10 | const constants = require('../initializers/constants') | 13 | const constants = require('../initializers/constants') |
@@ -12,7 +15,14 @@ const customVideosValidators = require('../helpers/custom-validators').videos | |||
12 | const logger = require('../helpers/logger') | 15 | const logger = require('../helpers/logger') |
13 | const modelUtils = require('./utils') | 16 | const modelUtils = require('./utils') |
14 | const utils = require('../helpers/utils') | 17 | const utils = require('../helpers/utils') |
15 | const webtorrent = require('../lib/webtorrent') | 18 | |
19 | const http = config.get('webserver.https') === true ? 'https' : 'http' | ||
20 | const host = config.get('webserver.host') | ||
21 | const port = config.get('webserver.port') | ||
22 | const uploadsDir = pathUtils.join(__dirname, '..', '..', config.get('storage.uploads')) | ||
23 | const thumbnailsDir = pathUtils.join(__dirname, '..', '..', config.get('storage.thumbnails')) | ||
24 | const torrentsDir = pathUtils.join(__dirname, '..', '..', config.get('storage.torrents')) | ||
25 | const webseedBaseUrl = http + '://' + host + ':' + port + constants.STATIC_PATHS.WEBSEED | ||
16 | 26 | ||
17 | // --------------------------------------------------------------------------- | 27 | // --------------------------------------------------------------------------- |
18 | 28 | ||
@@ -61,8 +71,7 @@ VideoSchema.statics = { | |||
61 | listOwnedByAuthor, | 71 | listOwnedByAuthor, |
62 | listRemotes, | 72 | listRemotes, |
63 | load, | 73 | load, |
64 | search, | 74 | search |
65 | seedAllExisting | ||
66 | } | 75 | } |
67 | 76 | ||
68 | VideoSchema.pre('remove', function (next) { | 77 | VideoSchema.pre('remove', function (next) { |
@@ -98,8 +107,21 @@ VideoSchema.pre('save', function (next) { | |||
98 | this.podUrl = constants.CONFIG.WEBSERVER.URL | 107 | this.podUrl = constants.CONFIG.WEBSERVER.URL |
99 | 108 | ||
100 | tasks.push( | 109 | tasks.push( |
110 | // TODO: refractoring | ||
101 | function (callback) { | 111 | function (callback) { |
102 | seed(videoPath, callback) | 112 | createTorrent(videoPath, { announceList: [ [ 'ws://' + host + ':' + port + '/tracker/socket' ] ], urlList: [ webseedBaseUrl + video.filename ] }, function (err, torrent) { |
113 | if (err) return callback(err) | ||
114 | |||
115 | fs.writeFile(torrentsDir + video.filename + '.torrent', torrent, function (err) { | ||
116 | if (err) return callback(err) | ||
117 | |||
118 | const parsedTorrent = parseTorrent(torrent) | ||
119 | parsedTorrent.xs = video.podUrl + constants.STATIC_PATHS.TORRENTS + video.filename + '.torrent' | ||
120 | video.magnetUri = magnet.encode(parsedTorrent) | ||
121 | |||
122 | callback(null) | ||
123 | }) | ||
124 | }) | ||
103 | }, | 125 | }, |
104 | function (callback) { | 126 | function (callback) { |
105 | createThumbnail(videoPath, callback) | 127 | createThumbnail(videoPath, callback) |
@@ -109,7 +131,6 @@ VideoSchema.pre('save', function (next) { | |||
109 | parallel(tasks, function (err, results) { | 131 | parallel(tasks, function (err, results) { |
110 | if (err) return next(err) | 132 | if (err) return next(err) |
111 | 133 | ||
112 | video.magnetUri = results[0].magnetURI | ||
113 | video.thumbnail = results[1] | 134 | video.thumbnail = results[1] |
114 | 135 | ||
115 | return next() | 136 | return next() |
@@ -144,7 +165,7 @@ function toFormatedJSON () { | |||
144 | author: this.author, | 165 | author: this.author, |
145 | duration: this.duration, | 166 | duration: this.duration, |
146 | tags: this.tags, | 167 | tags: this.tags, |
147 | thumbnailPath: constants.THUMBNAILS_STATIC_PATH + '/' + this.thumbnail, | 168 | thumbnailPath: constants.STATIC_PATHS.THUMBNAILS + '/' + this.thumbnail, |
148 | createdDate: this.createdDate | 169 | createdDate: this.createdDate |
149 | } | 170 | } |
150 | 171 | ||
@@ -230,17 +251,6 @@ function search (value, field, start, count, sort, callback) { | |||
230 | modelUtils.listForApiWithCount.call(this, query, start, count, sort, callback) | 251 | modelUtils.listForApiWithCount.call(this, query, start, count, sort, callback) |
231 | } | 252 | } |
232 | 253 | ||
233 | function seedAllExisting (callback) { | ||
234 | listOwned.call(this, function (err, videos) { | ||
235 | if (err) return callback(err) | ||
236 | |||
237 | eachLimit(videos, constants.SEEDS_IN_PARALLEL, function (video, callbackEach) { | ||
238 | const videoPath = pathUtils.join(constants.CONFIG.STORAGE.UPLOAD_DIR, video.filename) | ||
239 | seed(videoPath, callbackEach) | ||
240 | }, callback) | ||
241 | }) | ||
242 | } | ||
243 | |||
244 | // --------------------------------------------------------------------------- | 254 | // --------------------------------------------------------------------------- |
245 | 255 | ||
246 | function removeThumbnail (video, callback) { | 256 | function removeThumbnail (video, callback) { |
@@ -253,12 +263,7 @@ function removeFile (video, callback) { | |||
253 | 263 | ||
254 | // Maybe the torrent is not seeded, but we catch the error to don't stop the removing process | 264 | // Maybe the torrent is not seeded, but we catch the error to don't stop the removing process |
255 | function removeTorrent (video, callback) { | 265 | function removeTorrent (video, callback) { |
256 | try { | 266 | fs.unlink(torrentsDir + video.filename + '.torrent') |
257 | webtorrent.remove(video.magnetUri, callback) | ||
258 | } catch (err) { | ||
259 | logger.warn('Cannot remove the torrent from WebTorrent', { err: err }) | ||
260 | return callback(null) | ||
261 | } | ||
262 | } | 267 | } |
263 | 268 | ||
264 | function createThumbnail (videoPath, callback) { | 269 | function createThumbnail (videoPath, callback) { |
@@ -276,16 +281,6 @@ function createThumbnail (videoPath, callback) { | |||
276 | }) | 281 | }) |
277 | } | 282 | } |
278 | 283 | ||
279 | function seed (path, callback) { | ||
280 | logger.info('Seeding %s...', path) | ||
281 | |||
282 | webtorrent.seed(path, function (torrent) { | ||
283 | logger.info('%s seeded (%s).', path, torrent.magnetURI) | ||
284 | |||
285 | return callback(null, torrent) | ||
286 | }) | ||
287 | } | ||
288 | |||
289 | function generateThumbnailFromBase64 (data, callback) { | 284 | function generateThumbnailFromBase64 (data, callback) { |
290 | // Creating the thumbnail for this remote video | 285 | // Creating the thumbnail for this remote video |
291 | utils.generateRandomString(16, function (err, randomString) { | 286 | utils.generateRandomString(16, function (err, randomString) { |