diff options
author | Chocobozzz <florian.bigard@gmail.com> | 2016-05-10 21:19:24 +0200 |
---|---|---|
committer | Chocobozzz <florian.bigard@gmail.com> | 2016-05-10 21:19:24 +0200 |
commit | cbe2f7c34822b1bd3b1f8c691f79f0c29cf21f07 (patch) | |
tree | ce678124210db8b03b2a523e3b92a14cc403eeee /server | |
parent | f1dae018681936b556b2496b7f2d872c004cfda3 (diff) | |
download | PeerTube-cbe2f7c34822b1bd3b1f8c691f79f0c29cf21f07.tar.gz PeerTube-cbe2f7c34822b1bd3b1f8c691f79f0c29cf21f07.tar.zst PeerTube-cbe2f7c34822b1bd3b1f8c691f79f0c29cf21f07.zip |
Refractoring and add thumbnails support (without tests)
Diffstat (limited to 'server')
-rw-r--r-- | server/controllers/api/v1/pods.js | 27 | ||||
-rw-r--r-- | server/controllers/api/v1/remoteVideos.js | 32 | ||||
-rw-r--r-- | server/controllers/api/v1/videos.js | 82 | ||||
-rw-r--r-- | server/helpers/utils.js | 13 | ||||
-rw-r--r-- | server/initializers/constants.js | 10 | ||||
-rw-r--r-- | server/lib/friends.js | 39 | ||||
-rw-r--r-- | server/lib/requestsScheduler.js | 12 | ||||
-rw-r--r-- | server/lib/videos.js | 104 | ||||
-rw-r--r-- | server/models/videos.js | 126 | ||||
-rw-r--r-- | server/tests/api/multiplePods.js | 13 |
10 files changed, 298 insertions, 160 deletions
diff --git a/server/controllers/api/v1/pods.js b/server/controllers/api/v1/pods.js index dbe1a7372..d08b7860d 100644 --- a/server/controllers/api/v1/pods.js +++ b/server/controllers/api/v1/pods.js | |||
@@ -1,17 +1,16 @@ | |||
1 | 'use strict' | 1 | 'use strict' |
2 | 2 | ||
3 | const express = require('express') | 3 | const express = require('express') |
4 | const fs = require('fs') | ||
5 | 4 | ||
6 | const logger = require('../../../helpers/logger') | 5 | const logger = require('../../../helpers/logger') |
7 | const friends = require('../../../lib/friends') | 6 | const friends = require('../../../lib/friends') |
8 | const middleware = require('../../../middlewares') | 7 | const middleware = require('../../../middlewares') |
9 | const cacheMiddleware = middleware.cache | 8 | const cacheMiddleware = middleware.cache |
10 | const peertubeCrypto = require('../../../helpers/peertubeCrypto') | ||
11 | const Pods = require('../../../models/pods') | 9 | const Pods = require('../../../models/pods') |
12 | const reqValidator = middleware.reqValidators.pods | 10 | const reqValidator = middleware.reqValidators.pods |
13 | const secureMiddleware = middleware.secure | 11 | const secureMiddleware = middleware.secure |
14 | const secureRequest = middleware.reqValidators.remote.secureRequest | 12 | const secureRequest = middleware.reqValidators.remote.secureRequest |
13 | const videos = require('../../../lib/videos') | ||
15 | const Videos = require('../../../models/videos') | 14 | const Videos = require('../../../models/videos') |
16 | 15 | ||
17 | const router = express.Router() | 16 | const router = express.Router() |
@@ -34,9 +33,12 @@ function addPods (req, res, next) { | |||
34 | Pods.add(informations, function (err) { | 33 | Pods.add(informations, function (err) { |
35 | if (err) return next(err) | 34 | if (err) return next(err) |
36 | 35 | ||
37 | Videos.addRemotes(informations.videos) | 36 | // Create the remote videos from the new pod |
37 | videos.createRemoteVideos(informations.videos, function (err) { | ||
38 | if (err) logger.error('Cannot create remote videos.', { error: err }) | ||
39 | }) | ||
38 | 40 | ||
39 | fs.readFile(peertubeCrypto.getCertDir() + 'peertube.pub', 'utf8', function (err, cert) { | 41 | friends.getMyCertificate(function (err, cert) { |
40 | if (err) { | 42 | if (err) { |
41 | logger.error('Cannot read cert file.') | 43 | logger.error('Cannot read cert file.') |
42 | return next(err) | 44 | return next(err) |
@@ -75,11 +77,20 @@ function removePods (req, res, next) { | |||
75 | Pods.remove(url, function (err) { | 77 | Pods.remove(url, function (err) { |
76 | if (err) return next(err) | 78 | if (err) return next(err) |
77 | 79 | ||
78 | Videos.removeAllRemotesOf(url, function (err) { | 80 | Videos.listFromUrl(url, function (err, videos_list) { |
79 | if (err) logger.error('Cannot remove all remote videos of %s.', url) | 81 | if (err) { |
80 | else logger.info('%s pod removed.', url) | 82 | logger.error('Cannot list videos from url.', { error: err }) |
83 | next(err) | ||
84 | } | ||
81 | 85 | ||
82 | res.type('json').status(204).end() | 86 | videos.removeRemoteVideos(videos_list, function (err) { |
87 | if (err) { | ||
88 | logger.error('Cannot remove remote videos.', { error: err }) | ||
89 | next(err) | ||
90 | } | ||
91 | |||
92 | res.type('json').status(204).end() | ||
93 | }) | ||
83 | }) | 94 | }) |
84 | }) | 95 | }) |
85 | } | 96 | } |
diff --git a/server/controllers/api/v1/remoteVideos.js b/server/controllers/api/v1/remoteVideos.js index 5e9e71fdd..8ff212b7f 100644 --- a/server/controllers/api/v1/remoteVideos.js +++ b/server/controllers/api/v1/remoteVideos.js | |||
@@ -7,7 +7,9 @@ const middleware = require('../../../middlewares') | |||
7 | const secureMiddleware = middleware.secure | 7 | const secureMiddleware = middleware.secure |
8 | const cacheMiddleware = middleware.cache | 8 | const cacheMiddleware = middleware.cache |
9 | const reqValidator = middleware.reqValidators.remote | 9 | const reqValidator = middleware.reqValidators.remote |
10 | const videos = require('../../../models/videos') | 10 | const logger = require('../../../helpers/logger') |
11 | const Videos = require('../../../models/videos') | ||
12 | const videos = require('../../../lib/videos') | ||
11 | 13 | ||
12 | const router = express.Router() | 14 | const router = express.Router() |
13 | 15 | ||
@@ -34,20 +36,34 @@ module.exports = router | |||
34 | // --------------------------------------------------------------------------- | 36 | // --------------------------------------------------------------------------- |
35 | 37 | ||
36 | function addRemoteVideos (req, res, next) { | 38 | function addRemoteVideos (req, res, next) { |
37 | videos.addRemotes(req.body.data, function (err, videos) { | 39 | const videos_to_create = req.body.data |
38 | if (err) return next(err) | 40 | videos.createRemoteVideos(videos_to_create, function (err, remote_videos) { |
41 | if (err) { | ||
42 | logger.error('Cannot create remote videos.', { error: err }) | ||
43 | return next(err) | ||
44 | } | ||
39 | 45 | ||
40 | res.json(videos) | 46 | res.type('json').status(201).end() |
41 | }) | 47 | }) |
42 | } | 48 | } |
43 | 49 | ||
44 | function removeRemoteVideo (req, res, next) { | 50 | function removeRemoteVideo (req, res, next) { |
45 | const url = req.body.signature.url | 51 | const fromUrl = req.body.signature.url |
46 | const magnetUris = map(req.body.data, 'magnetUri') | 52 | const magnetUris = map(req.body.data, 'magnetUri') |
47 | 53 | ||
48 | videos.removeRemotesOfByMagnetUris(url, magnetUris, function (err) { | 54 | Videos.listFromUrlAndMagnets(fromUrl, magnetUris, function (err, videos_list) { |
49 | if (err) return next(err) | 55 | if (err) { |
56 | logger.error('Cannot list videos from url and magnets.', { error: err }) | ||
57 | return next(err) | ||
58 | } | ||
50 | 59 | ||
51 | res.type('json').status(204).end() | 60 | videos.removeRemoteVideos(videos_list, function (err) { |
61 | if (err) { | ||
62 | logger.error('Cannot remove remote videos.', { error: err }) | ||
63 | return next(err) | ||
64 | } | ||
65 | |||
66 | res.type('json').status(204).end() | ||
67 | }) | ||
52 | }) | 68 | }) |
53 | } | 69 | } |
diff --git a/server/controllers/api/v1/videos.js b/server/controllers/api/v1/videos.js index 7fdc50e52..98f17ac6c 100644 --- a/server/controllers/api/v1/videos.js +++ b/server/controllers/api/v1/videos.js | |||
@@ -1,16 +1,19 @@ | |||
1 | 'use strict' | 1 | 'use strict' |
2 | 2 | ||
3 | const config = require('config') | 3 | const config = require('config') |
4 | const crypto = require('crypto') | ||
5 | const express = require('express') | 4 | const express = require('express') |
5 | const fs = require('fs') | ||
6 | const path = require('path') | ||
6 | const multer = require('multer') | 7 | const multer = require('multer') |
7 | 8 | ||
9 | const constants = require('../../../initializers/constants') | ||
8 | const logger = require('../../../helpers/logger') | 10 | const logger = require('../../../helpers/logger') |
9 | const friends = require('../../../lib/friends') | 11 | const friends = require('../../../lib/friends') |
10 | const middleware = require('../../../middlewares') | 12 | const middleware = require('../../../middlewares') |
11 | const oAuth2 = require('../../../middlewares/oauth2') | 13 | const oAuth2 = require('../../../middlewares/oauth2') |
12 | const cacheMiddleware = middleware.cache | 14 | const cacheMiddleware = middleware.cache |
13 | const reqValidator = middleware.reqValidators.videos | 15 | const reqValidator = middleware.reqValidators.videos |
16 | const utils = require('../../../helpers/utils') | ||
14 | const Videos = require('../../../models/videos') // model | 17 | const Videos = require('../../../models/videos') // model |
15 | const videos = require('../../../lib/videos') | 18 | const videos = require('../../../lib/videos') |
16 | const webtorrent = require('../../../lib/webtorrent') | 19 | const webtorrent = require('../../../lib/webtorrent') |
@@ -29,14 +32,15 @@ const storage = multer.diskStorage({ | |||
29 | if (file.mimetype === 'video/webm') extension = 'webm' | 32 | if (file.mimetype === 'video/webm') extension = 'webm' |
30 | else if (file.mimetype === 'video/mp4') extension = 'mp4' | 33 | else if (file.mimetype === 'video/mp4') extension = 'mp4' |
31 | else if (file.mimetype === 'video/ogg') extension = 'ogv' | 34 | else if (file.mimetype === 'video/ogg') extension = 'ogv' |
32 | crypto.pseudoRandomBytes(16, function (err, raw) { | 35 | utils.generateRandomString(16, function (err, random_string) { |
33 | const fieldname = err ? undefined : raw.toString('hex') | 36 | const fieldname = err ? undefined : random_string |
34 | cb(null, fieldname + '.' + extension) | 37 | cb(null, fieldname + '.' + extension) |
35 | }) | 38 | }) |
36 | } | 39 | } |
37 | }) | 40 | }) |
38 | 41 | ||
39 | const reqFiles = multer({ storage: storage }).fields([{ name: 'videofile', maxCount: 1 }]) | 42 | const reqFiles = multer({ storage: storage }).fields([{ name: 'videofile', maxCount: 1 }]) |
43 | const thumbnailsDir = path.join(__dirname, '..', '..', '..', '..', config.get('storage.thumbnails')) | ||
40 | 44 | ||
41 | router.get('/', cacheMiddleware.cache(false), listVideos) | 45 | router.get('/', cacheMiddleware.cache(false), listVideos) |
42 | router.post('/', oAuth2.authenticate, reqFiles, reqValidator.videosAdd, cacheMiddleware.cache(false), addVideo) | 46 | router.post('/', oAuth2.authenticate, reqFiles, reqValidator.videosAdd, cacheMiddleware.cache(false), addVideo) |
@@ -63,31 +67,50 @@ function addVideo (req, res, next) { | |||
63 | videos.getVideoDuration(video_file.path, function (err, duration) { | 67 | videos.getVideoDuration(video_file.path, function (err, duration) { |
64 | if (err) { | 68 | if (err) { |
65 | // TODO: unseed the video | 69 | // TODO: unseed the video |
66 | logger.error('Cannot retrieve metadata of the file') | 70 | logger.error('Cannot retrieve metadata of the file.') |
67 | return next(err) | 71 | return next(err) |
68 | } | 72 | } |
69 | 73 | ||
70 | const video_data = { | 74 | videos.getVideoThumbnail(video_file.path, function (err, thumbnail_name) { |
71 | name: video_infos.name, | ||
72 | namePath: video_file.filename, | ||
73 | description: video_infos.description, | ||
74 | magnetUri: torrent.magnetURI, | ||
75 | author: res.locals.oauth.token.user.username, | ||
76 | duration: duration | ||
77 | } | ||
78 | |||
79 | Videos.add(video_data, function (err) { | ||
80 | if (err) { | 75 | if (err) { |
81 | // TODO unseed the video | 76 | // TODO: unseed the video |
82 | logger.error('Cannot insert this video in the database.') | 77 | logger.error('Cannot make a thumbnail of the video file.') |
83 | return next(err) | 78 | return next(err) |
84 | } | 79 | } |
85 | 80 | ||
86 | // Now we'll add the video's meta data to our friends | 81 | const video_data = { |
87 | friends.addVideoToFriends(video_data) | 82 | name: video_infos.name, |
83 | namePath: video_file.filename, | ||
84 | description: video_infos.description, | ||
85 | magnetUri: torrent.magnetURI, | ||
86 | author: res.locals.oauth.token.user.username, | ||
87 | duration: duration, | ||
88 | thumbnail: thumbnail_name | ||
89 | } | ||
88 | 90 | ||
89 | // TODO : include Location of the new video -> 201 | 91 | Videos.add(video_data, function (err) { |
90 | res.type('json').status(204).end() | 92 | if (err) { |
93 | // TODO unseed the video | ||
94 | logger.error('Cannot insert this video in the database.') | ||
95 | return next(err) | ||
96 | } | ||
97 | |||
98 | fs.readFile(thumbnailsDir + thumbnail_name, function (err, data) { | ||
99 | if (err) { | ||
100 | // TODO: remove video? | ||
101 | logger.error('Cannot read the thumbnail of the video') | ||
102 | return next(err) | ||
103 | } | ||
104 | |||
105 | // Set the image in base64 | ||
106 | video_data.thumbnail_base64 = new Buffer(data).toString('base64') | ||
107 | // Now we'll add the video's meta data to our friends | ||
108 | friends.addVideoToFriends(video_data) | ||
109 | |||
110 | // TODO : include Location of the new video -> 201 | ||
111 | res.type('json').status(204).end() | ||
112 | }) | ||
113 | }) | ||
91 | }) | 114 | }) |
92 | }) | 115 | }) |
93 | }) | 116 | }) |
@@ -123,13 +146,17 @@ function removeVideo (req, res, next) { | |||
123 | Videos.removeOwned(req.params.id, function (err) { | 146 | Videos.removeOwned(req.params.id, function (err) { |
124 | if (err) return next(err) | 147 | if (err) return next(err) |
125 | 148 | ||
126 | const params = { | 149 | videos.removeVideosDataFromDisk([ video ], function (err) { |
127 | name: video.name, | 150 | if (err) logger.error('Cannot remove video data from disk.', { video: video }) |
128 | magnetUri: video.magnetUri | 151 | |
129 | } | 152 | const params = { |
153 | name: video.name, | ||
154 | magnetUri: video.magnetUri | ||
155 | } | ||
130 | 156 | ||
131 | friends.removeVideoToFriends(params) | 157 | friends.removeVideoToFriends(params) |
132 | res.type('json').status(204).end() | 158 | res.type('json').status(204).end() |
159 | }) | ||
133 | }) | 160 | }) |
134 | }) | 161 | }) |
135 | }) | 162 | }) |
@@ -154,7 +181,8 @@ function getFormatedVideo (video_obj) { | |||
154 | isLocal: videos.getVideoState(video_obj).owned, | 181 | isLocal: videos.getVideoState(video_obj).owned, |
155 | magnetUri: video_obj.magnetUri, | 182 | magnetUri: video_obj.magnetUri, |
156 | author: video_obj.author, | 183 | author: video_obj.author, |
157 | duration: video_obj.duration | 184 | duration: video_obj.duration, |
185 | thumbnail_path: constants.THUMBNAILS_STATIC_PATH + '/' + video_obj.thumbnail | ||
158 | } | 186 | } |
159 | 187 | ||
160 | return formated_video | 188 | return formated_video |
diff --git a/server/helpers/utils.js b/server/helpers/utils.js index 1f7839673..9d4d51c46 100644 --- a/server/helpers/utils.js +++ b/server/helpers/utils.js | |||
@@ -1,9 +1,20 @@ | |||
1 | 'use strict' | 1 | 'use strict' |
2 | 2 | ||
3 | const crypto = require('crypto') | ||
4 | |||
3 | const logger = require('./logger') | 5 | const logger = require('./logger') |
4 | 6 | ||
5 | const utils = { | 7 | const utils = { |
6 | cleanForExit: cleanForExit | 8 | cleanForExit: cleanForExit, |
9 | generateRandomString: generateRandomString | ||
10 | } | ||
11 | |||
12 | function generateRandomString (size, callback) { | ||
13 | crypto.pseudoRandomBytes(size, function (err, raw) { | ||
14 | if (err) return callback(err) | ||
15 | |||
16 | callback(null, raw.toString('hex')) | ||
17 | }) | ||
7 | } | 18 | } |
8 | 19 | ||
9 | function cleanForExit (webtorrent_process) { | 20 | function cleanForExit (webtorrent_process) { |
diff --git a/server/initializers/constants.js b/server/initializers/constants.js index dc08805b9..bb141456a 100644 --- a/server/initializers/constants.js +++ b/server/initializers/constants.js | |||
@@ -18,6 +18,12 @@ const PODS_SCORE = { | |||
18 | // Number of retries we make for the make retry requests (to friends...) | 18 | // Number of retries we make for the make retry requests (to friends...) |
19 | let REQUEST_RETRIES = 10 | 19 | let REQUEST_RETRIES = 10 |
20 | 20 | ||
21 | // Videos thumbnail size | ||
22 | const THUMBNAILS_SIZE = '200x110' | ||
23 | |||
24 | // Path for access to thumbnails with express router | ||
25 | const THUMBNAILS_STATIC_PATH = '/static/thumbnails' | ||
26 | |||
21 | // Special constants for a test instance | 27 | // Special constants for a test instance |
22 | if (isTestInstance() === true) { | 28 | if (isTestInstance() === true) { |
23 | FRIEND_BASE_SCORE = 20 | 29 | FRIEND_BASE_SCORE = 20 |
@@ -32,7 +38,9 @@ module.exports = { | |||
32 | FRIEND_BASE_SCORE: FRIEND_BASE_SCORE, | 38 | FRIEND_BASE_SCORE: FRIEND_BASE_SCORE, |
33 | INTERVAL: INTERVAL, | 39 | INTERVAL: INTERVAL, |
34 | PODS_SCORE: PODS_SCORE, | 40 | PODS_SCORE: PODS_SCORE, |
35 | REQUEST_RETRIES: REQUEST_RETRIES | 41 | REQUEST_RETRIES: REQUEST_RETRIES, |
42 | THUMBNAILS_SIZE: THUMBNAILS_SIZE, | ||
43 | THUMBNAILS_STATIC_PATH: THUMBNAILS_STATIC_PATH | ||
36 | } | 44 | } |
37 | 45 | ||
38 | // --------------------------------------------------------------------------- | 46 | // --------------------------------------------------------------------------- |
diff --git a/server/lib/friends.js b/server/lib/friends.js index 9a2c5c06e..3b8a52060 100644 --- a/server/lib/friends.js +++ b/server/lib/friends.js | |||
@@ -11,6 +11,7 @@ const peertubeCrypto = require('../helpers/peertubeCrypto') | |||
11 | const Pods = require('../models/pods') | 11 | const Pods = require('../models/pods') |
12 | const requestsScheduler = require('../lib/requestsScheduler') | 12 | const requestsScheduler = require('../lib/requestsScheduler') |
13 | const requests = require('../helpers/requests') | 13 | const requests = require('../helpers/requests') |
14 | const videos = require('../lib/videos') | ||
14 | const Videos = require('../models/videos') | 15 | const Videos = require('../models/videos') |
15 | 16 | ||
16 | const http = config.get('webserver.https') ? 'https' : 'http' | 17 | const http = config.get('webserver.https') ? 'https' : 'http' |
@@ -20,6 +21,7 @@ const port = config.get('webserver.port') | |||
20 | const pods = { | 21 | const pods = { |
21 | addVideoToFriends: addVideoToFriends, | 22 | addVideoToFriends: addVideoToFriends, |
22 | hasFriends: hasFriends, | 23 | hasFriends: hasFriends, |
24 | getMyCertificate: getMyCertificate, | ||
23 | makeFriends: makeFriends, | 25 | makeFriends: makeFriends, |
24 | quitFriends: quitFriends, | 26 | quitFriends: quitFriends, |
25 | removeVideoToFriends: removeVideoToFriends | 27 | removeVideoToFriends: removeVideoToFriends |
@@ -42,11 +44,15 @@ function hasFriends (callback) { | |||
42 | }) | 44 | }) |
43 | } | 45 | } |
44 | 46 | ||
47 | function getMyCertificate (callback) { | ||
48 | fs.readFile(peertubeCrypto.getCertDir() + 'peertube.pub', 'utf8', callback) | ||
49 | } | ||
50 | |||
45 | function makeFriends (callback) { | 51 | function makeFriends (callback) { |
46 | const pods_score = {} | 52 | const pods_score = {} |
47 | 53 | ||
48 | logger.info('Make friends!') | 54 | logger.info('Make friends!') |
49 | fs.readFile(peertubeCrypto.getCertDir() + 'peertube.pub', 'utf8', function (err, cert) { | 55 | getMyCertificate(function (err, cert) { |
50 | if (err) { | 56 | if (err) { |
51 | logger.error('Cannot read public cert.') | 57 | logger.error('Cannot read public cert.') |
52 | return callback(err) | 58 | return callback(err) |
@@ -54,8 +60,8 @@ function makeFriends (callback) { | |||
54 | 60 | ||
55 | const urls = config.get('network.friends') | 61 | const urls = config.get('network.friends') |
56 | 62 | ||
57 | async.each(urls, function (url, callback) { | 63 | async.each(urls, function (url, callback_each) { |
58 | computeForeignPodsList(url, pods_score, callback) | 64 | computeForeignPodsList(url, pods_score, callback_each) |
59 | }, function (err) { | 65 | }, function (err) { |
60 | if (err) return callback(err) | 66 | if (err) return callback(err) |
61 | 67 | ||
@@ -96,11 +102,18 @@ function quitFriends (callback) { | |||
96 | 102 | ||
97 | logger.info('Broke friends, so sad :(') | 103 | logger.info('Broke friends, so sad :(') |
98 | 104 | ||
99 | Videos.removeAllRemotes(function (err) { | 105 | Videos.listFromRemotes(function (err, videos_list) { |
100 | if (err) return callback(err) | 106 | if (err) return callback(err) |
101 | 107 | ||
102 | logger.info('Removed all remote videos.') | 108 | videos.removeRemoteVideos(videos_list, function (err) { |
103 | callback(null) | 109 | if (err) { |
110 | logger.error('Cannot remove remote videos.', { error: err }) | ||
111 | return callback(err) | ||
112 | } | ||
113 | |||
114 | logger.info('Removed all remote videos.') | ||
115 | callback(null) | ||
116 | }) | ||
104 | }) | 117 | }) |
105 | }) | 118 | }) |
106 | }) | 119 | }) |
@@ -127,16 +140,14 @@ function computeForeignPodsList (url, pods_score, callback) { | |||
127 | if (err) return callback(err) | 140 | if (err) return callback(err) |
128 | if (foreign_pods_list.length === 0) return callback() | 141 | if (foreign_pods_list.length === 0) return callback() |
129 | 142 | ||
130 | async.each(foreign_pods_list, function (foreign_pod, callback_each) { | 143 | foreign_pods_list.forEach(function (foreign_pod) { |
131 | const foreign_url = foreign_pod.url | 144 | const foreign_url = foreign_pod.url |
132 | 145 | ||
133 | if (pods_score[foreign_url]) pods_score[foreign_url]++ | 146 | if (pods_score[foreign_url]) pods_score[foreign_url]++ |
134 | else pods_score[foreign_url] = 1 | 147 | else pods_score[foreign_url] = 1 |
135 | |||
136 | callback_each() | ||
137 | }, function () { | ||
138 | callback() | ||
139 | }) | 148 | }) |
149 | |||
150 | callback() | ||
140 | }) | 151 | }) |
141 | } | 152 | } |
142 | 153 | ||
@@ -194,13 +205,15 @@ function makeRequestsToWinningPods (cert, pods_list, callback) { | |||
194 | logger.error('Error with adding %s pod.', pod.url, { error: err }) | 205 | logger.error('Error with adding %s pod.', pod.url, { error: err }) |
195 | return callback_each_request() | 206 | return callback_each_request() |
196 | } | 207 | } |
197 | 208 | console.log('hihi') | |
198 | Videos.addRemotes(body.videos, function (err) { | 209 | videos.createRemoteVideos(body.videos, function (err) { |
199 | if (err) { | 210 | if (err) { |
200 | logger.error('Error with adding videos of pod.', pod.url, { error: err }) | 211 | logger.error('Error with adding videos of pod.', pod.url, { error: err }) |
201 | return callback_each_request() | 212 | return callback_each_request() |
202 | } | 213 | } |
203 | 214 | ||
215 | console.log('kik') | ||
216 | |||
204 | logger.debug('Adding remote videos from %s.', pod.url, { videos: body.videos }) | 217 | logger.debug('Adding remote videos from %s.', pod.url, { videos: body.videos }) |
205 | return callback_each_request() | 218 | return callback_each_request() |
206 | }) | 219 | }) |
diff --git a/server/lib/requestsScheduler.js b/server/lib/requestsScheduler.js index 2c5474e51..4953f6a91 100644 --- a/server/lib/requestsScheduler.js +++ b/server/lib/requestsScheduler.js | |||
@@ -8,6 +8,7 @@ const logger = require('../helpers/logger') | |||
8 | const Pods = require('../models/pods') | 8 | const Pods = require('../models/pods') |
9 | const Requests = require('../models/requests') | 9 | const Requests = require('../models/requests') |
10 | const requests = require('../helpers/requests') | 10 | const requests = require('../helpers/requests') |
11 | const videos = require('../lib/videos') | ||
11 | const Videos = require('../models/videos') | 12 | const Videos = require('../models/videos') |
12 | 13 | ||
13 | let timer = null | 14 | let timer = null |
@@ -99,7 +100,7 @@ function makeRequest (type, requests_to_make, callback) { | |||
99 | requests.makeMultipleRetryRequest(params, pods, callbackEachPodFinished, callbackAllPodsFinished) | 100 | requests.makeMultipleRetryRequest(params, pods, callbackEachPodFinished, callbackAllPodsFinished) |
100 | 101 | ||
101 | function callbackEachPodFinished (err, response, body, url, pod, callback_each_pod_finished) { | 102 | function callbackEachPodFinished (err, response, body, url, pod, callback_each_pod_finished) { |
102 | if (err || (response.statusCode !== 200 && response.statusCode !== 204)) { | 103 | if (err || (response.statusCode !== 200 && response.statusCode !== 201 && response.statusCode !== 204)) { |
103 | bad_pods.push(pod._id) | 104 | bad_pods.push(pod._id) |
104 | logger.error('Error sending secure request to %s pod.', url, { error: err || new Error('Status code not 20x') }) | 105 | logger.error('Error sending secure request to %s pod.', url, { error: err || new Error('Status code not 20x') }) |
105 | } else { | 106 | } else { |
@@ -187,12 +188,13 @@ function removeBadPods () { | |||
187 | const urls = map(pods, 'url') | 188 | const urls = map(pods, 'url') |
188 | const ids = map(pods, '_id') | 189 | const ids = map(pods, '_id') |
189 | 190 | ||
190 | Videos.removeAllRemotesOf(urls, function (err, r) { | 191 | Videos.listFromUrls(urls, function (err, videos_list) { |
191 | if (err) { | 192 | if (err) { |
192 | logger.error('Cannot remove videos from a pod that we removing.', { error: err }) | 193 | logger.error('Cannot list videos urls.', { error: err, urls: urls }) |
193 | } else { | 194 | } else { |
194 | const videos_removed = r.result.n | 195 | videos.removeRemoteVideos(videos_list, function (err) { |
195 | logger.info('Removed %d videos.', videos_removed) | 196 | if (err) logger.error('Cannot remove remote videos.', { error: err }) |
197 | }) | ||
196 | } | 198 | } |
197 | 199 | ||
198 | Pods.removeAllByIds(ids, function (err, r) { | 200 | Pods.removeAllByIds(ids, function (err, r) { |
diff --git a/server/lib/videos.js b/server/lib/videos.js index 43d6c6b99..b3497743a 100644 --- a/server/lib/videos.js +++ b/server/lib/videos.js | |||
@@ -3,21 +3,39 @@ | |||
3 | const async = require('async') | 3 | const async = require('async') |
4 | const config = require('config') | 4 | const config = require('config') |
5 | const ffmpeg = require('fluent-ffmpeg') | 5 | const ffmpeg = require('fluent-ffmpeg') |
6 | const fs = require('fs') | ||
7 | const map = require('lodash/map') | ||
6 | const pathUtils = require('path') | 8 | const pathUtils = require('path') |
7 | const webtorrent = require('../lib/webtorrent') | ||
8 | 9 | ||
10 | const constants = require('../initializers/constants') | ||
9 | const logger = require('../helpers/logger') | 11 | const logger = require('../helpers/logger') |
12 | const utils = require('../helpers/utils') | ||
10 | const Videos = require('../models/videos') | 13 | const Videos = require('../models/videos') |
14 | const webtorrent = require('../lib/webtorrent') | ||
11 | 15 | ||
12 | const uploadDir = pathUtils.join(__dirname, '..', '..', config.get('storage.uploads')) | 16 | const uploadDir = pathUtils.join(__dirname, '..', '..', config.get('storage.uploads')) |
17 | const thumbnailsDir = pathUtils.join(__dirname, '..', '..', config.get('storage.thumbnails')) | ||
13 | 18 | ||
14 | const videos = { | 19 | const videos = { |
20 | createRemoteVideos: createRemoteVideos, | ||
15 | getVideoDuration: getVideoDuration, | 21 | getVideoDuration: getVideoDuration, |
16 | getVideoState: getVideoState, | 22 | getVideoState: getVideoState, |
23 | getVideoThumbnail: getVideoThumbnail, | ||
24 | removeVideosDataFromDisk: removeVideosDataFromDisk, | ||
25 | removeRemoteVideos: removeRemoteVideos, | ||
17 | seed: seed, | 26 | seed: seed, |
18 | seedAllExisting: seedAllExisting | 27 | seedAllExisting: seedAllExisting |
19 | } | 28 | } |
20 | 29 | ||
30 | function createRemoteVideos (videos, callback) { | ||
31 | // Create the remote videos from the new pod | ||
32 | createRemoteVideoObjects(videos, function (err, remote_videos) { | ||
33 | if (err) return callback(err) | ||
34 | |||
35 | Videos.addRemotes(remote_videos, callback) | ||
36 | }) | ||
37 | } | ||
38 | |||
21 | function getVideoDuration (video_path, callback) { | 39 | function getVideoDuration (video_path, callback) { |
22 | ffmpeg.ffprobe(video_path, function (err, metadata) { | 40 | ffmpeg.ffprobe(video_path, function (err, metadata) { |
23 | if (err) return callback(err) | 41 | if (err) return callback(err) |
@@ -36,6 +54,51 @@ function getVideoState (video) { | |||
36 | return { exist: exist, owned: owned } | 54 | return { exist: exist, owned: owned } |
37 | } | 55 | } |
38 | 56 | ||
57 | function getVideoThumbnail (video_path, callback) { | ||
58 | const filename = pathUtils.basename(video_path) + '.jpg' | ||
59 | ffmpeg(video_path) | ||
60 | .on('error', callback) | ||
61 | .on('end', function () { | ||
62 | callback(null, filename) | ||
63 | }) | ||
64 | .thumbnail({ | ||
65 | count: 1, | ||
66 | folder: thumbnailsDir, | ||
67 | size: constants.THUMBNAILS_SIZE, | ||
68 | filename: filename | ||
69 | }) | ||
70 | } | ||
71 | |||
72 | // Remove video datas from disk (video file, thumbnail...) | ||
73 | function removeVideosDataFromDisk (videos, callback) { | ||
74 | async.each(videos, function (video, callback_each) { | ||
75 | fs.unlink(thumbnailsDir + video.thumbnail, function (err) { | ||
76 | if (err) logger.error('Cannot remove the video thumbnail') | ||
77 | |||
78 | if (getVideoState(video).owned === true) { | ||
79 | fs.unlink(uploadDir + video.namePath, function (err) { | ||
80 | if (err) { | ||
81 | logger.error('Cannot remove this video file.') | ||
82 | return callback_each(err) | ||
83 | } | ||
84 | |||
85 | callback_each(null) | ||
86 | }) | ||
87 | } else { | ||
88 | callback_each(null) | ||
89 | } | ||
90 | }) | ||
91 | }, callback) | ||
92 | } | ||
93 | |||
94 | function removeRemoteVideos (videos, callback) { | ||
95 | Videos.removeByIds(map(videos, '_id'), function (err) { | ||
96 | if (err) return callback(err) | ||
97 | |||
98 | removeVideosDataFromDisk(videos, callback) | ||
99 | }) | ||
100 | } | ||
101 | |||
39 | function seed (path, callback) { | 102 | function seed (path, callback) { |
40 | logger.info('Seeding %s...', path) | 103 | logger.info('Seeding %s...', path) |
41 | 104 | ||
@@ -69,3 +132,42 @@ function seedAllExisting (callback) { | |||
69 | // --------------------------------------------------------------------------- | 132 | // --------------------------------------------------------------------------- |
70 | 133 | ||
71 | module.exports = videos | 134 | module.exports = videos |
135 | |||
136 | // --------------------------------------------------------------------------- | ||
137 | |||
138 | function createRemoteVideoObjects (videos, callback) { | ||
139 | const remote_videos = [] | ||
140 | |||
141 | async.each(videos, function (video, callback_each) { | ||
142 | // Creating the thumbnail for this remote video | ||
143 | utils.generateRandomString(16, function (err, random_string) { | ||
144 | if (err) return callback_each(err) | ||
145 | |||
146 | const thumbnail_name = random_string + '.jpg' | ||
147 | createThumbnailFromBase64(thumbnail_name, video.thumbnail_base64, function (err) { | ||
148 | if (err) return callback_each(err) | ||
149 | |||
150 | const params = { | ||
151 | name: video.name, | ||
152 | description: video.description, | ||
153 | magnetUri: video.magnetUri, | ||
154 | podUrl: video.podUrl, | ||
155 | duration: video.duration, | ||
156 | thumbnail: thumbnail_name | ||
157 | } | ||
158 | remote_videos.push(params) | ||
159 | |||
160 | callback_each(null) | ||
161 | }) | ||
162 | }) | ||
163 | }, | ||
164 | function (err) { | ||
165 | if (err) return callback(err) | ||
166 | |||
167 | callback(null, remote_videos) | ||
168 | }) | ||
169 | } | ||
170 | |||
171 | function createThumbnailFromBase64 (thumbnail_name, data, callback) { | ||
172 | fs.writeFile(thumbnailsDir + thumbnail_name, data, { encoding: 'base64' }, callback) | ||
173 | } | ||
diff --git a/server/models/videos.js b/server/models/videos.js index e02158be7..eedb6eb58 100644 --- a/server/models/videos.js +++ b/server/models/videos.js | |||
@@ -1,18 +1,13 @@ | |||
1 | 'use strict' | 1 | 'use strict' |
2 | 2 | ||
3 | const async = require('async') | ||
4 | const config = require('config') | 3 | const config = require('config') |
5 | const dz = require('dezalgo') | ||
6 | const fs = require('fs') | ||
7 | const mongoose = require('mongoose') | 4 | const mongoose = require('mongoose') |
8 | const path = require('path') | ||
9 | 5 | ||
10 | const logger = require('../helpers/logger') | 6 | const logger = require('../helpers/logger') |
11 | 7 | ||
12 | const http = config.get('webserver.https') === true ? 'https' : 'http' | 8 | const http = config.get('webserver.https') === true ? 'https' : 'http' |
13 | const host = config.get('webserver.host') | 9 | const host = config.get('webserver.host') |
14 | const port = config.get('webserver.port') | 10 | const port = config.get('webserver.port') |
15 | const uploadDir = path.join(__dirname, '..', '..', config.get('storage.uploads')) | ||
16 | 11 | ||
17 | // --------------------------------------------------------------------------- | 12 | // --------------------------------------------------------------------------- |
18 | 13 | ||
@@ -23,7 +18,8 @@ const videosSchema = mongoose.Schema({ | |||
23 | magnetUri: String, | 18 | magnetUri: String, |
24 | podUrl: String, | 19 | podUrl: String, |
25 | author: String, | 20 | author: String, |
26 | duration: Number | 21 | duration: Number, |
22 | thumbnail: String | ||
27 | }) | 23 | }) |
28 | const VideosDB = mongoose.model('videos', videosSchema) | 24 | const VideosDB = mongoose.model('videos', videosSchema) |
29 | 25 | ||
@@ -34,11 +30,13 @@ const Videos = { | |||
34 | addRemotes: addRemotes, | 30 | addRemotes: addRemotes, |
35 | get: get, | 31 | get: get, |
36 | list: list, | 32 | list: list, |
33 | listFromUrl: listFromUrl, | ||
34 | listFromUrls: listFromUrls, | ||
35 | listFromUrlAndMagnets: listFromUrlAndMagnets, | ||
36 | listFromRemotes: listFromRemotes, | ||
37 | listOwned: listOwned, | 37 | listOwned: listOwned, |
38 | removeOwned: removeOwned, | 38 | removeOwned: removeOwned, |
39 | removeAllRemotes: removeAllRemotes, | 39 | removeByIds: removeByIds, |
40 | removeAllRemotesOf: removeAllRemotesOf, | ||
41 | removeRemotesOfByMagnetUris: removeRemotesOfByMagnetUris, | ||
42 | search: search | 40 | search: search |
43 | } | 41 | } |
44 | 42 | ||
@@ -58,38 +56,13 @@ function add (video, callback) { | |||
58 | }) | 56 | }) |
59 | } | 57 | } |
60 | 58 | ||
61 | // TODO: avoid doublons | ||
62 | function addRemotes (videos, callback) { | 59 | function addRemotes (videos, callback) { |
63 | if (!callback) callback = function () {} | 60 | videos.forEach(function (video) { |
64 | 61 | // Ensure they are remote videos | |
65 | const to_add = [] | 62 | video.namePath = null |
66 | |||
67 | async.each(videos, function (video, callback_each) { | ||
68 | callback_each = dz(callback_each) | ||
69 | logger.debug('Add remote video from pod: %s', video.podUrl) | ||
70 | |||
71 | const params = { | ||
72 | name: video.name, | ||
73 | namePath: null, | ||
74 | description: video.description, | ||
75 | magnetUri: video.magnetUri, | ||
76 | podUrl: video.podUrl, | ||
77 | duration: video.duration | ||
78 | } | ||
79 | |||
80 | to_add.push(params) | ||
81 | |||
82 | callback_each() | ||
83 | }, function () { | ||
84 | VideosDB.create(to_add, function (err, videos) { | ||
85 | if (err) { | ||
86 | logger.error('Cannot insert this remote video.') | ||
87 | return callback(err) | ||
88 | } | ||
89 | |||
90 | return callback(null, videos) | ||
91 | }) | ||
92 | }) | 63 | }) |
64 | |||
65 | VideosDB.create(videos, callback) | ||
93 | } | 66 | } |
94 | 67 | ||
95 | function get (id, callback) { | 68 | function get (id, callback) { |
@@ -114,6 +87,22 @@ function list (callback) { | |||
114 | }) | 87 | }) |
115 | } | 88 | } |
116 | 89 | ||
90 | function listFromUrl (fromUrl, callback) { | ||
91 | VideosDB.find({ podUrl: fromUrl }, callback) | ||
92 | } | ||
93 | |||
94 | function listFromUrls (fromUrls, callback) { | ||
95 | VideosDB.find({ podUrl: { $in: fromUrls } }, callback) | ||
96 | } | ||
97 | |||
98 | function listFromUrlAndMagnets (fromUrl, magnets, callback) { | ||
99 | VideosDB.find({ podUrl: fromUrl, magnetUri: { $in: magnets } }, callback) | ||
100 | } | ||
101 | |||
102 | function listFromRemotes (callback) { | ||
103 | VideosDB.find({ namePath: null }, callback) | ||
104 | } | ||
105 | |||
117 | function listOwned (callback) { | 106 | function listOwned (callback) { |
118 | // If namePath is not null this is *our* video | 107 | // If namePath is not null this is *our* video |
119 | VideosDB.find({ namePath: { $ne: null } }, function (err, videos_list) { | 108 | VideosDB.find({ namePath: { $ne: null } }, function (err, videos_list) { |
@@ -126,65 +115,14 @@ function listOwned (callback) { | |||
126 | }) | 115 | }) |
127 | } | 116 | } |
128 | 117 | ||
118 | // Return the video in the callback | ||
129 | function removeOwned (id, callback) { | 119 | function removeOwned (id, callback) { |
130 | VideosDB.findByIdAndRemove(id, function (err, video) { | 120 | VideosDB.findByIdAndRemove(id, callback) |
131 | if (err) { | ||
132 | logger.error('Cannot remove the torrent.') | ||
133 | return callback(err) | ||
134 | } | ||
135 | |||
136 | fs.unlink(uploadDir + video.namePath, function (err) { | ||
137 | if (err) { | ||
138 | logger.error('Cannot remove this video file.') | ||
139 | return callback(err) | ||
140 | } | ||
141 | |||
142 | callback(null) | ||
143 | }) | ||
144 | }) | ||
145 | } | ||
146 | |||
147 | function removeAllRemotes (callback) { | ||
148 | VideosDB.remove({ namePath: null }, callback) | ||
149 | } | ||
150 | |||
151 | function removeAllRemotesOf (fromUrl, callback) { | ||
152 | VideosDB.remove({ podUrl: fromUrl }, callback) | ||
153 | } | 121 | } |
154 | 122 | ||
155 | // Use the magnet Uri because the _id field is not the same on different servers | 123 | // Use the magnet Uri because the _id field is not the same on different servers |
156 | function removeRemotesOfByMagnetUris (fromUrl, magnetUris, callback) { | 124 | function removeByIds (ids, callback) { |
157 | if (callback === undefined) callback = function () {} | 125 | VideosDB.remove({ _id: { $in: ids } }, callback) |
158 | |||
159 | VideosDB.find({ magnetUri: { $in: magnetUris } }, function (err, videos) { | ||
160 | if (err || !videos) { | ||
161 | logger.error('Cannot find the torrent URI of these remote videos.') | ||
162 | return callback(err) | ||
163 | } | ||
164 | |||
165 | const to_remove = [] | ||
166 | async.each(videos, function (video, callback_async) { | ||
167 | callback_async = dz(callback_async) | ||
168 | |||
169 | if (video.podUrl !== fromUrl) { | ||
170 | logger.error('The pod %s has not the rights on the video of %s.', fromUrl, video.podUrl) | ||
171 | } else { | ||
172 | to_remove.push(video._id) | ||
173 | } | ||
174 | |||
175 | callback_async() | ||
176 | }, function () { | ||
177 | VideosDB.remove({ _id: { $in: to_remove } }, function (err) { | ||
178 | if (err) { | ||
179 | logger.error('Cannot remove the remote videos.') | ||
180 | return callback(err) | ||
181 | } | ||
182 | |||
183 | logger.info('Removed remote videos from %s.', fromUrl) | ||
184 | callback(null) | ||
185 | }) | ||
186 | }) | ||
187 | }) | ||
188 | } | 126 | } |
189 | 127 | ||
190 | function search (name, callback) { | 128 | function search (name, callback) { |
diff --git a/server/tests/api/multiplePods.js b/server/tests/api/multiplePods.js index adc99f714..4a4532a0f 100644 --- a/server/tests/api/multiplePods.js +++ b/server/tests/api/multiplePods.js | |||
@@ -193,14 +193,23 @@ describe('Test multiple pods', function () { | |||
193 | expect(videos).to.be.an('array') | 193 | expect(videos).to.be.an('array') |
194 | expect(videos.length).to.equal(4) | 194 | expect(videos.length).to.equal(4) |
195 | 195 | ||
196 | const video1 = videos[2] | 196 | // We not sure about the order of the two last uploads |
197 | let video1 = null | ||
198 | let video2 = null | ||
199 | if (videos[2].name === 'my super name for pod 3') { | ||
200 | video1 = videos[2] | ||
201 | video2 = videos[3] | ||
202 | } else { | ||
203 | video1 = videos[3] | ||
204 | video2 = videos[2] | ||
205 | } | ||
206 | |||
197 | expect(video1.name).to.equal('my super name for pod 3') | 207 | expect(video1.name).to.equal('my super name for pod 3') |
198 | expect(video1.description).to.equal('my super description for pod 3') | 208 | expect(video1.description).to.equal('my super description for pod 3') |
199 | expect(video1.podUrl).to.equal('http://localhost:9003') | 209 | expect(video1.podUrl).to.equal('http://localhost:9003') |
200 | expect(video1.magnetUri).to.exist | 210 | expect(video1.magnetUri).to.exist |
201 | expect(video1.duration).to.equal(5) | 211 | expect(video1.duration).to.equal(5) |
202 | 212 | ||
203 | const video2 = videos[3] | ||
204 | expect(video2.name).to.equal('my super name for pod 3-2') | 213 | expect(video2.name).to.equal('my super name for pod 3-2') |
205 | expect(video2.description).to.equal('my super description for pod 3-2') | 214 | expect(video2.description).to.equal('my super description for pod 3-2') |
206 | expect(video2.podUrl).to.equal('http://localhost:9003') | 215 | expect(video2.podUrl).to.equal('http://localhost:9003') |