aboutsummaryrefslogtreecommitdiffhomepage
diff options
context:
space:
mode:
-rw-r--r--.gitignore1
-rw-r--r--config/default.yaml1
-rw-r--r--config/test-1.yaml1
-rw-r--r--config/test-2.yaml1
-rw-r--r--config/test-3.yaml1
-rw-r--r--config/test-4.yaml1
-rw-r--r--config/test-5.yaml1
-rw-r--r--config/test-6.yaml1
-rw-r--r--server.js4
-rw-r--r--server/controllers/api/v1/pods.js27
-rw-r--r--server/controllers/api/v1/remoteVideos.js32
-rw-r--r--server/controllers/api/v1/videos.js82
-rw-r--r--server/helpers/utils.js13
-rw-r--r--server/initializers/constants.js10
-rw-r--r--server/lib/friends.js39
-rw-r--r--server/lib/requestsScheduler.js12
-rw-r--r--server/lib/videos.js104
-rw-r--r--server/models/videos.js126
-rw-r--r--server/tests/api/multiplePods.js13
19 files changed, 310 insertions, 160 deletions
diff --git a/.gitignore b/.gitignore
index 6c0165c41..9d9b1978b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -11,3 +11,4 @@ test6/
11public/stylesheets/global.css 11public/stylesheets/global.css
12public/stylesheets/vendor 12public/stylesheets/vendor
13uploads 13uploads
14thumbnails
diff --git a/config/default.yaml b/config/default.yaml
index be161640c..87b72aa3b 100644
--- a/config/default.yaml
+++ b/config/default.yaml
@@ -16,6 +16,7 @@ storage:
16 certs: 'certs/' 16 certs: 'certs/'
17 uploads: 'uploads/' 17 uploads: 'uploads/'
18 logs: 'logs/' 18 logs: 'logs/'
19 thumbnails: 'thumbnails/'
19 20
20network: 21network:
21 friends: [] 22 friends: []
diff --git a/config/test-1.yaml b/config/test-1.yaml
index 031e4893f..3afcb1b04 100644
--- a/config/test-1.yaml
+++ b/config/test-1.yaml
@@ -13,6 +13,7 @@ storage:
13 certs: 'test1/certs/' 13 certs: 'test1/certs/'
14 uploads: 'test1/uploads/' 14 uploads: 'test1/uploads/'
15 logs: 'test1/logs/' 15 logs: 'test1/logs/'
16 thumbnails: 'test1/thumbnails/'
16 17
17network: 18network:
18 friends: 19 friends:
diff --git a/config/test-2.yaml b/config/test-2.yaml
index 8a2a80158..51cc186eb 100644
--- a/config/test-2.yaml
+++ b/config/test-2.yaml
@@ -13,6 +13,7 @@ storage:
13 certs: 'test2/certs/' 13 certs: 'test2/certs/'
14 uploads: 'test2/uploads/' 14 uploads: 'test2/uploads/'
15 logs: 'test2/logs/' 15 logs: 'test2/logs/'
16 thumbnails: 'test2/thumbnails/'
16 17
17network: 18network:
18 friends: 19 friends:
diff --git a/config/test-3.yaml b/config/test-3.yaml
index 3e906d125..7ef01ba4d 100644
--- a/config/test-3.yaml
+++ b/config/test-3.yaml
@@ -13,6 +13,7 @@ storage:
13 certs: 'test3/certs/' 13 certs: 'test3/certs/'
14 uploads: 'test3/uploads/' 14 uploads: 'test3/uploads/'
15 logs: 'test3/logs/' 15 logs: 'test3/logs/'
16 thumbnails: 'test3/thumbnails/'
16 17
17network: 18network:
18 friends: 19 friends:
diff --git a/config/test-4.yaml b/config/test-4.yaml
index 6db8a5d03..a4d3bb164 100644
--- a/config/test-4.yaml
+++ b/config/test-4.yaml
@@ -13,6 +13,7 @@ storage:
13 certs: 'test4/certs/' 13 certs: 'test4/certs/'
14 uploads: 'test4/uploads/' 14 uploads: 'test4/uploads/'
15 logs: 'test4/logs/' 15 logs: 'test4/logs/'
16 thumbnails: 'test4/thumbnails/'
16 17
17network: 18network:
18 friends: 19 friends:
diff --git a/config/test-5.yaml b/config/test-5.yaml
index 7b3f18d35..0435c17fe 100644
--- a/config/test-5.yaml
+++ b/config/test-5.yaml
@@ -13,6 +13,7 @@ storage:
13 certs: 'test5/certs/' 13 certs: 'test5/certs/'
14 uploads: 'test5/uploads/' 14 uploads: 'test5/uploads/'
15 logs: 'test5/logs/' 15 logs: 'test5/logs/'
16 thumbnails: 'test5/thumbnails/'
16 17
17network: 18network:
18 friends: 19 friends:
diff --git a/config/test-6.yaml b/config/test-6.yaml
index 0c7675cc2..b7dbd4bdd 100644
--- a/config/test-6.yaml
+++ b/config/test-6.yaml
@@ -13,6 +13,7 @@ storage:
13 certs: 'test6/certs/' 13 certs: 'test6/certs/'
14 uploads: 'test6/uploads/' 14 uploads: 'test6/uploads/'
15 logs: 'test6/logs/' 15 logs: 'test6/logs/'
16 thumbnails: 'test6/thumbnails/'
16 17
17network: 18network:
18 friends: 19 friends:
diff --git a/server.js b/server.js
index 204cc146f..ef26ea773 100644
--- a/server.js
+++ b/server.js
@@ -75,6 +75,10 @@ app.use('/app/*', function (req, res, next) {
75 res.sendStatus(404) 75 res.sendStatus(404)
76}) 76})
77 77
78// Thumbnails path for express
79const thumbnails_physical_path = path.join(__dirname, config.get('storage.thumbnails'))
80app.use(constants.THUMBNAILS_STATIC_PATH, express.static(thumbnails_physical_path, { maxAge: 0 }))
81
78// Client application 82// Client application
79app.use('/*', function (req, res, next) { 83app.use('/*', function (req, res, next) {
80 res.sendFile(path.join(__dirname, 'client/index.html')) 84 res.sendFile(path.join(__dirname, 'client/index.html'))
diff --git a/server/controllers/api/v1/pods.js b/server/controllers/api/v1/pods.js
index dbe1a7372..d08b7860d 100644
--- a/server/controllers/api/v1/pods.js
+++ b/server/controllers/api/v1/pods.js
@@ -1,17 +1,16 @@
1'use strict' 1'use strict'
2 2
3const express = require('express') 3const express = require('express')
4const fs = require('fs')
5 4
6const logger = require('../../../helpers/logger') 5const logger = require('../../../helpers/logger')
7const friends = require('../../../lib/friends') 6const friends = require('../../../lib/friends')
8const middleware = require('../../../middlewares') 7const middleware = require('../../../middlewares')
9const cacheMiddleware = middleware.cache 8const cacheMiddleware = middleware.cache
10const peertubeCrypto = require('../../../helpers/peertubeCrypto')
11const Pods = require('../../../models/pods') 9const Pods = require('../../../models/pods')
12const reqValidator = middleware.reqValidators.pods 10const reqValidator = middleware.reqValidators.pods
13const secureMiddleware = middleware.secure 11const secureMiddleware = middleware.secure
14const secureRequest = middleware.reqValidators.remote.secureRequest 12const secureRequest = middleware.reqValidators.remote.secureRequest
13const videos = require('../../../lib/videos')
15const Videos = require('../../../models/videos') 14const Videos = require('../../../models/videos')
16 15
17const router = express.Router() 16const router = express.Router()
@@ -34,9 +33,12 @@ function addPods (req, res, next) {
34 Pods.add(informations, function (err) { 33 Pods.add(informations, function (err) {
35 if (err) return next(err) 34 if (err) return next(err)
36 35
37 Videos.addRemotes(informations.videos) 36 // Create the remote videos from the new pod
37 videos.createRemoteVideos(informations.videos, function (err) {
38 if (err) logger.error('Cannot create remote videos.', { error: err })
39 })
38 40
39 fs.readFile(peertubeCrypto.getCertDir() + 'peertube.pub', 'utf8', function (err, cert) { 41 friends.getMyCertificate(function (err, cert) {
40 if (err) { 42 if (err) {
41 logger.error('Cannot read cert file.') 43 logger.error('Cannot read cert file.')
42 return next(err) 44 return next(err)
@@ -75,11 +77,20 @@ function removePods (req, res, next) {
75 Pods.remove(url, function (err) { 77 Pods.remove(url, function (err) {
76 if (err) return next(err) 78 if (err) return next(err)
77 79
78 Videos.removeAllRemotesOf(url, function (err) { 80 Videos.listFromUrl(url, function (err, videos_list) {
79 if (err) logger.error('Cannot remove all remote videos of %s.', url) 81 if (err) {
80 else logger.info('%s pod removed.', url) 82 logger.error('Cannot list videos from url.', { error: err })
83 next(err)
84 }
81 85
82 res.type('json').status(204).end() 86 videos.removeRemoteVideos(videos_list, function (err) {
87 if (err) {
88 logger.error('Cannot remove remote videos.', { error: err })
89 next(err)
90 }
91
92 res.type('json').status(204).end()
93 })
83 }) 94 })
84 }) 95 })
85} 96}
diff --git a/server/controllers/api/v1/remoteVideos.js b/server/controllers/api/v1/remoteVideos.js
index 5e9e71fdd..8ff212b7f 100644
--- a/server/controllers/api/v1/remoteVideos.js
+++ b/server/controllers/api/v1/remoteVideos.js
@@ -7,7 +7,9 @@ const middleware = require('../../../middlewares')
7const secureMiddleware = middleware.secure 7const secureMiddleware = middleware.secure
8const cacheMiddleware = middleware.cache 8const cacheMiddleware = middleware.cache
9const reqValidator = middleware.reqValidators.remote 9const reqValidator = middleware.reqValidators.remote
10const videos = require('../../../models/videos') 10const logger = require('../../../helpers/logger')
11const Videos = require('../../../models/videos')
12const videos = require('../../../lib/videos')
11 13
12const router = express.Router() 14const router = express.Router()
13 15
@@ -34,20 +36,34 @@ module.exports = router
34// --------------------------------------------------------------------------- 36// ---------------------------------------------------------------------------
35 37
36function addRemoteVideos (req, res, next) { 38function addRemoteVideos (req, res, next) {
37 videos.addRemotes(req.body.data, function (err, videos) { 39 const videos_to_create = req.body.data
38 if (err) return next(err) 40 videos.createRemoteVideos(videos_to_create, function (err, remote_videos) {
41 if (err) {
42 logger.error('Cannot create remote videos.', { error: err })
43 return next(err)
44 }
39 45
40 res.json(videos) 46 res.type('json').status(201).end()
41 }) 47 })
42} 48}
43 49
44function removeRemoteVideo (req, res, next) { 50function removeRemoteVideo (req, res, next) {
45 const url = req.body.signature.url 51 const fromUrl = req.body.signature.url
46 const magnetUris = map(req.body.data, 'magnetUri') 52 const magnetUris = map(req.body.data, 'magnetUri')
47 53
48 videos.removeRemotesOfByMagnetUris(url, magnetUris, function (err) { 54 Videos.listFromUrlAndMagnets(fromUrl, magnetUris, function (err, videos_list) {
49 if (err) return next(err) 55 if (err) {
56 logger.error('Cannot list videos from url and magnets.', { error: err })
57 return next(err)
58 }
50 59
51 res.type('json').status(204).end() 60 videos.removeRemoteVideos(videos_list, function (err) {
61 if (err) {
62 logger.error('Cannot remove remote videos.', { error: err })
63 return next(err)
64 }
65
66 res.type('json').status(204).end()
67 })
52 }) 68 })
53} 69}
diff --git a/server/controllers/api/v1/videos.js b/server/controllers/api/v1/videos.js
index 7fdc50e52..98f17ac6c 100644
--- a/server/controllers/api/v1/videos.js
+++ b/server/controllers/api/v1/videos.js
@@ -1,16 +1,19 @@
1'use strict' 1'use strict'
2 2
3const config = require('config') 3const config = require('config')
4const crypto = require('crypto')
5const express = require('express') 4const express = require('express')
5const fs = require('fs')
6const path = require('path')
6const multer = require('multer') 7const multer = require('multer')
7 8
9const constants = require('../../../initializers/constants')
8const logger = require('../../../helpers/logger') 10const logger = require('../../../helpers/logger')
9const friends = require('../../../lib/friends') 11const friends = require('../../../lib/friends')
10const middleware = require('../../../middlewares') 12const middleware = require('../../../middlewares')
11const oAuth2 = require('../../../middlewares/oauth2') 13const oAuth2 = require('../../../middlewares/oauth2')
12const cacheMiddleware = middleware.cache 14const cacheMiddleware = middleware.cache
13const reqValidator = middleware.reqValidators.videos 15const reqValidator = middleware.reqValidators.videos
16const utils = require('../../../helpers/utils')
14const Videos = require('../../../models/videos') // model 17const Videos = require('../../../models/videos') // model
15const videos = require('../../../lib/videos') 18const videos = require('../../../lib/videos')
16const webtorrent = require('../../../lib/webtorrent') 19const webtorrent = require('../../../lib/webtorrent')
@@ -29,14 +32,15 @@ const storage = multer.diskStorage({
29 if (file.mimetype === 'video/webm') extension = 'webm' 32 if (file.mimetype === 'video/webm') extension = 'webm'
30 else if (file.mimetype === 'video/mp4') extension = 'mp4' 33 else if (file.mimetype === 'video/mp4') extension = 'mp4'
31 else if (file.mimetype === 'video/ogg') extension = 'ogv' 34 else if (file.mimetype === 'video/ogg') extension = 'ogv'
32 crypto.pseudoRandomBytes(16, function (err, raw) { 35 utils.generateRandomString(16, function (err, random_string) {
33 const fieldname = err ? undefined : raw.toString('hex') 36 const fieldname = err ? undefined : random_string
34 cb(null, fieldname + '.' + extension) 37 cb(null, fieldname + '.' + extension)
35 }) 38 })
36 } 39 }
37}) 40})
38 41
39const reqFiles = multer({ storage: storage }).fields([{ name: 'videofile', maxCount: 1 }]) 42const reqFiles = multer({ storage: storage }).fields([{ name: 'videofile', maxCount: 1 }])
43const thumbnailsDir = path.join(__dirname, '..', '..', '..', '..', config.get('storage.thumbnails'))
40 44
41router.get('/', cacheMiddleware.cache(false), listVideos) 45router.get('/', cacheMiddleware.cache(false), listVideos)
42router.post('/', oAuth2.authenticate, reqFiles, reqValidator.videosAdd, cacheMiddleware.cache(false), addVideo) 46router.post('/', oAuth2.authenticate, reqFiles, reqValidator.videosAdd, cacheMiddleware.cache(false), addVideo)
@@ -63,31 +67,50 @@ function addVideo (req, res, next) {
63 videos.getVideoDuration(video_file.path, function (err, duration) { 67 videos.getVideoDuration(video_file.path, function (err, duration) {
64 if (err) { 68 if (err) {
65 // TODO: unseed the video 69 // TODO: unseed the video
66 logger.error('Cannot retrieve metadata of the file') 70 logger.error('Cannot retrieve metadata of the file.')
67 return next(err) 71 return next(err)
68 } 72 }
69 73
70 const video_data = { 74 videos.getVideoThumbnail(video_file.path, function (err, thumbnail_name) {
71 name: video_infos.name,
72 namePath: video_file.filename,
73 description: video_infos.description,
74 magnetUri: torrent.magnetURI,
75 author: res.locals.oauth.token.user.username,
76 duration: duration
77 }
78
79 Videos.add(video_data, function (err) {
80 if (err) { 75 if (err) {
81 // TODO unseed the video 76 // TODO: unseed the video
82 logger.error('Cannot insert this video in the database.') 77 logger.error('Cannot make a thumbnail of the video file.')
83 return next(err) 78 return next(err)
84 } 79 }
85 80
86 // Now we'll add the video's meta data to our friends 81 const video_data = {
87 friends.addVideoToFriends(video_data) 82 name: video_infos.name,
83 namePath: video_file.filename,
84 description: video_infos.description,
85 magnetUri: torrent.magnetURI,
86 author: res.locals.oauth.token.user.username,
87 duration: duration,
88 thumbnail: thumbnail_name
89 }
88 90
89 // TODO : include Location of the new video -> 201 91 Videos.add(video_data, function (err) {
90 res.type('json').status(204).end() 92 if (err) {
93 // TODO unseed the video
94 logger.error('Cannot insert this video in the database.')
95 return next(err)
96 }
97
98 fs.readFile(thumbnailsDir + thumbnail_name, function (err, data) {
99 if (err) {
100 // TODO: remove video?
101 logger.error('Cannot read the thumbnail of the video')
102 return next(err)
103 }
104
105 // Set the image in base64
106 video_data.thumbnail_base64 = new Buffer(data).toString('base64')
107 // Now we'll add the video's meta data to our friends
108 friends.addVideoToFriends(video_data)
109
110 // TODO : include Location of the new video -> 201
111 res.type('json').status(204).end()
112 })
113 })
91 }) 114 })
92 }) 115 })
93 }) 116 })
@@ -123,13 +146,17 @@ function removeVideo (req, res, next) {
123 Videos.removeOwned(req.params.id, function (err) { 146 Videos.removeOwned(req.params.id, function (err) {
124 if (err) return next(err) 147 if (err) return next(err)
125 148
126 const params = { 149 videos.removeVideosDataFromDisk([ video ], function (err) {
127 name: video.name, 150 if (err) logger.error('Cannot remove video data from disk.', { video: video })
128 magnetUri: video.magnetUri 151
129 } 152 const params = {
153 name: video.name,
154 magnetUri: video.magnetUri
155 }
130 156
131 friends.removeVideoToFriends(params) 157 friends.removeVideoToFriends(params)
132 res.type('json').status(204).end() 158 res.type('json').status(204).end()
159 })
133 }) 160 })
134 }) 161 })
135 }) 162 })
@@ -154,7 +181,8 @@ function getFormatedVideo (video_obj) {
154 isLocal: videos.getVideoState(video_obj).owned, 181 isLocal: videos.getVideoState(video_obj).owned,
155 magnetUri: video_obj.magnetUri, 182 magnetUri: video_obj.magnetUri,
156 author: video_obj.author, 183 author: video_obj.author,
157 duration: video_obj.duration 184 duration: video_obj.duration,
185 thumbnail_path: constants.THUMBNAILS_STATIC_PATH + '/' + video_obj.thumbnail
158 } 186 }
159 187
160 return formated_video 188 return formated_video
diff --git a/server/helpers/utils.js b/server/helpers/utils.js
index 1f7839673..9d4d51c46 100644
--- a/server/helpers/utils.js
+++ b/server/helpers/utils.js
@@ -1,9 +1,20 @@
1'use strict' 1'use strict'
2 2
3const crypto = require('crypto')
4
3const logger = require('./logger') 5const logger = require('./logger')
4 6
5const utils = { 7const utils = {
6 cleanForExit: cleanForExit 8 cleanForExit: cleanForExit,
9 generateRandomString: generateRandomString
10}
11
12function generateRandomString (size, callback) {
13 crypto.pseudoRandomBytes(size, function (err, raw) {
14 if (err) return callback(err)
15
16 callback(null, raw.toString('hex'))
17 })
7} 18}
8 19
9function cleanForExit (webtorrent_process) { 20function cleanForExit (webtorrent_process) {
diff --git a/server/initializers/constants.js b/server/initializers/constants.js
index dc08805b9..bb141456a 100644
--- a/server/initializers/constants.js
+++ b/server/initializers/constants.js
@@ -18,6 +18,12 @@ const PODS_SCORE = {
18// Number of retries we make for the make retry requests (to friends...) 18// Number of retries we make for the make retry requests (to friends...)
19let REQUEST_RETRIES = 10 19let REQUEST_RETRIES = 10
20 20
21// Videos thumbnail size
22const THUMBNAILS_SIZE = '200x110'
23
24// Path for access to thumbnails with express router
25const THUMBNAILS_STATIC_PATH = '/static/thumbnails'
26
21// Special constants for a test instance 27// Special constants for a test instance
22if (isTestInstance() === true) { 28if (isTestInstance() === true) {
23 FRIEND_BASE_SCORE = 20 29 FRIEND_BASE_SCORE = 20
@@ -32,7 +38,9 @@ module.exports = {
32 FRIEND_BASE_SCORE: FRIEND_BASE_SCORE, 38 FRIEND_BASE_SCORE: FRIEND_BASE_SCORE,
33 INTERVAL: INTERVAL, 39 INTERVAL: INTERVAL,
34 PODS_SCORE: PODS_SCORE, 40 PODS_SCORE: PODS_SCORE,
35 REQUEST_RETRIES: REQUEST_RETRIES 41 REQUEST_RETRIES: REQUEST_RETRIES,
42 THUMBNAILS_SIZE: THUMBNAILS_SIZE,
43 THUMBNAILS_STATIC_PATH: THUMBNAILS_STATIC_PATH
36} 44}
37 45
38// --------------------------------------------------------------------------- 46// ---------------------------------------------------------------------------
diff --git a/server/lib/friends.js b/server/lib/friends.js
index 9a2c5c06e..3b8a52060 100644
--- a/server/lib/friends.js
+++ b/server/lib/friends.js
@@ -11,6 +11,7 @@ const peertubeCrypto = require('../helpers/peertubeCrypto')
11const Pods = require('../models/pods') 11const Pods = require('../models/pods')
12const requestsScheduler = require('../lib/requestsScheduler') 12const requestsScheduler = require('../lib/requestsScheduler')
13const requests = require('../helpers/requests') 13const requests = require('../helpers/requests')
14const videos = require('../lib/videos')
14const Videos = require('../models/videos') 15const Videos = require('../models/videos')
15 16
16const http = config.get('webserver.https') ? 'https' : 'http' 17const http = config.get('webserver.https') ? 'https' : 'http'
@@ -20,6 +21,7 @@ const port = config.get('webserver.port')
20const pods = { 21const pods = {
21 addVideoToFriends: addVideoToFriends, 22 addVideoToFriends: addVideoToFriends,
22 hasFriends: hasFriends, 23 hasFriends: hasFriends,
24 getMyCertificate: getMyCertificate,
23 makeFriends: makeFriends, 25 makeFriends: makeFriends,
24 quitFriends: quitFriends, 26 quitFriends: quitFriends,
25 removeVideoToFriends: removeVideoToFriends 27 removeVideoToFriends: removeVideoToFriends
@@ -42,11 +44,15 @@ function hasFriends (callback) {
42 }) 44 })
43} 45}
44 46
47function getMyCertificate (callback) {
48 fs.readFile(peertubeCrypto.getCertDir() + 'peertube.pub', 'utf8', callback)
49}
50
45function makeFriends (callback) { 51function makeFriends (callback) {
46 const pods_score = {} 52 const pods_score = {}
47 53
48 logger.info('Make friends!') 54 logger.info('Make friends!')
49 fs.readFile(peertubeCrypto.getCertDir() + 'peertube.pub', 'utf8', function (err, cert) { 55 getMyCertificate(function (err, cert) {
50 if (err) { 56 if (err) {
51 logger.error('Cannot read public cert.') 57 logger.error('Cannot read public cert.')
52 return callback(err) 58 return callback(err)
@@ -54,8 +60,8 @@ function makeFriends (callback) {
54 60
55 const urls = config.get('network.friends') 61 const urls = config.get('network.friends')
56 62
57 async.each(urls, function (url, callback) { 63 async.each(urls, function (url, callback_each) {
58 computeForeignPodsList(url, pods_score, callback) 64 computeForeignPodsList(url, pods_score, callback_each)
59 }, function (err) { 65 }, function (err) {
60 if (err) return callback(err) 66 if (err) return callback(err)
61 67
@@ -96,11 +102,18 @@ function quitFriends (callback) {
96 102
97 logger.info('Broke friends, so sad :(') 103 logger.info('Broke friends, so sad :(')
98 104
99 Videos.removeAllRemotes(function (err) { 105 Videos.listFromRemotes(function (err, videos_list) {
100 if (err) return callback(err) 106 if (err) return callback(err)
101 107
102 logger.info('Removed all remote videos.') 108 videos.removeRemoteVideos(videos_list, function (err) {
103 callback(null) 109 if (err) {
110 logger.error('Cannot remove remote videos.', { error: err })
111 return callback(err)
112 }
113
114 logger.info('Removed all remote videos.')
115 callback(null)
116 })
104 }) 117 })
105 }) 118 })
106 }) 119 })
@@ -127,16 +140,14 @@ function computeForeignPodsList (url, pods_score, callback) {
127 if (err) return callback(err) 140 if (err) return callback(err)
128 if (foreign_pods_list.length === 0) return callback() 141 if (foreign_pods_list.length === 0) return callback()
129 142
130 async.each(foreign_pods_list, function (foreign_pod, callback_each) { 143 foreign_pods_list.forEach(function (foreign_pod) {
131 const foreign_url = foreign_pod.url 144 const foreign_url = foreign_pod.url
132 145
133 if (pods_score[foreign_url]) pods_score[foreign_url]++ 146 if (pods_score[foreign_url]) pods_score[foreign_url]++
134 else pods_score[foreign_url] = 1 147 else pods_score[foreign_url] = 1
135
136 callback_each()
137 }, function () {
138 callback()
139 }) 148 })
149
150 callback()
140 }) 151 })
141} 152}
142 153
@@ -194,13 +205,15 @@ function makeRequestsToWinningPods (cert, pods_list, callback) {
194 logger.error('Error with adding %s pod.', pod.url, { error: err }) 205 logger.error('Error with adding %s pod.', pod.url, { error: err })
195 return callback_each_request() 206 return callback_each_request()
196 } 207 }
197 208 console.log('hihi')
198 Videos.addRemotes(body.videos, function (err) { 209 videos.createRemoteVideos(body.videos, function (err) {
199 if (err) { 210 if (err) {
200 logger.error('Error with adding videos of pod.', pod.url, { error: err }) 211 logger.error('Error with adding videos of pod.', pod.url, { error: err })
201 return callback_each_request() 212 return callback_each_request()
202 } 213 }
203 214
215 console.log('kik')
216
204 logger.debug('Adding remote videos from %s.', pod.url, { videos: body.videos }) 217 logger.debug('Adding remote videos from %s.', pod.url, { videos: body.videos })
205 return callback_each_request() 218 return callback_each_request()
206 }) 219 })
diff --git a/server/lib/requestsScheduler.js b/server/lib/requestsScheduler.js
index 2c5474e51..4953f6a91 100644
--- a/server/lib/requestsScheduler.js
+++ b/server/lib/requestsScheduler.js
@@ -8,6 +8,7 @@ const logger = require('../helpers/logger')
8const Pods = require('../models/pods') 8const Pods = require('../models/pods')
9const Requests = require('../models/requests') 9const Requests = require('../models/requests')
10const requests = require('../helpers/requests') 10const requests = require('../helpers/requests')
11const videos = require('../lib/videos')
11const Videos = require('../models/videos') 12const Videos = require('../models/videos')
12 13
13let timer = null 14let timer = null
@@ -99,7 +100,7 @@ function makeRequest (type, requests_to_make, callback) {
99 requests.makeMultipleRetryRequest(params, pods, callbackEachPodFinished, callbackAllPodsFinished) 100 requests.makeMultipleRetryRequest(params, pods, callbackEachPodFinished, callbackAllPodsFinished)
100 101
101 function callbackEachPodFinished (err, response, body, url, pod, callback_each_pod_finished) { 102 function callbackEachPodFinished (err, response, body, url, pod, callback_each_pod_finished) {
102 if (err || (response.statusCode !== 200 && response.statusCode !== 204)) { 103 if (err || (response.statusCode !== 200 && response.statusCode !== 201 && response.statusCode !== 204)) {
103 bad_pods.push(pod._id) 104 bad_pods.push(pod._id)
104 logger.error('Error sending secure request to %s pod.', url, { error: err || new Error('Status code not 20x') }) 105 logger.error('Error sending secure request to %s pod.', url, { error: err || new Error('Status code not 20x') })
105 } else { 106 } else {
@@ -187,12 +188,13 @@ function removeBadPods () {
187 const urls = map(pods, 'url') 188 const urls = map(pods, 'url')
188 const ids = map(pods, '_id') 189 const ids = map(pods, '_id')
189 190
190 Videos.removeAllRemotesOf(urls, function (err, r) { 191 Videos.listFromUrls(urls, function (err, videos_list) {
191 if (err) { 192 if (err) {
192 logger.error('Cannot remove videos from a pod that we removing.', { error: err }) 193 logger.error('Cannot list videos urls.', { error: err, urls: urls })
193 } else { 194 } else {
194 const videos_removed = r.result.n 195 videos.removeRemoteVideos(videos_list, function (err) {
195 logger.info('Removed %d videos.', videos_removed) 196 if (err) logger.error('Cannot remove remote videos.', { error: err })
197 })
196 } 198 }
197 199
198 Pods.removeAllByIds(ids, function (err, r) { 200 Pods.removeAllByIds(ids, function (err, r) {
diff --git a/server/lib/videos.js b/server/lib/videos.js
index 43d6c6b99..b3497743a 100644
--- a/server/lib/videos.js
+++ b/server/lib/videos.js
@@ -3,21 +3,39 @@
3const async = require('async') 3const async = require('async')
4const config = require('config') 4const config = require('config')
5const ffmpeg = require('fluent-ffmpeg') 5const ffmpeg = require('fluent-ffmpeg')
6const fs = require('fs')
7const map = require('lodash/map')
6const pathUtils = require('path') 8const pathUtils = require('path')
7const webtorrent = require('../lib/webtorrent')
8 9
10const constants = require('../initializers/constants')
9const logger = require('../helpers/logger') 11const logger = require('../helpers/logger')
12const utils = require('../helpers/utils')
10const Videos = require('../models/videos') 13const Videos = require('../models/videos')
14const webtorrent = require('../lib/webtorrent')
11 15
12const uploadDir = pathUtils.join(__dirname, '..', '..', config.get('storage.uploads')) 16const uploadDir = pathUtils.join(__dirname, '..', '..', config.get('storage.uploads'))
17const thumbnailsDir = pathUtils.join(__dirname, '..', '..', config.get('storage.thumbnails'))
13 18
14const videos = { 19const videos = {
20 createRemoteVideos: createRemoteVideos,
15 getVideoDuration: getVideoDuration, 21 getVideoDuration: getVideoDuration,
16 getVideoState: getVideoState, 22 getVideoState: getVideoState,
23 getVideoThumbnail: getVideoThumbnail,
24 removeVideosDataFromDisk: removeVideosDataFromDisk,
25 removeRemoteVideos: removeRemoteVideos,
17 seed: seed, 26 seed: seed,
18 seedAllExisting: seedAllExisting 27 seedAllExisting: seedAllExisting
19} 28}
20 29
30function createRemoteVideos (videos, callback) {
31 // Create the remote videos from the new pod
32 createRemoteVideoObjects(videos, function (err, remote_videos) {
33 if (err) return callback(err)
34
35 Videos.addRemotes(remote_videos, callback)
36 })
37}
38
21function getVideoDuration (video_path, callback) { 39function getVideoDuration (video_path, callback) {
22 ffmpeg.ffprobe(video_path, function (err, metadata) { 40 ffmpeg.ffprobe(video_path, function (err, metadata) {
23 if (err) return callback(err) 41 if (err) return callback(err)
@@ -36,6 +54,51 @@ function getVideoState (video) {
36 return { exist: exist, owned: owned } 54 return { exist: exist, owned: owned }
37} 55}
38 56
57function getVideoThumbnail (video_path, callback) {
58 const filename = pathUtils.basename(video_path) + '.jpg'
59 ffmpeg(video_path)
60 .on('error', callback)
61 .on('end', function () {
62 callback(null, filename)
63 })
64 .thumbnail({
65 count: 1,
66 folder: thumbnailsDir,
67 size: constants.THUMBNAILS_SIZE,
68 filename: filename
69 })
70}
71
72// Remove video datas from disk (video file, thumbnail...)
73function removeVideosDataFromDisk (videos, callback) {
74 async.each(videos, function (video, callback_each) {
75 fs.unlink(thumbnailsDir + video.thumbnail, function (err) {
76 if (err) logger.error('Cannot remove the video thumbnail')
77
78 if (getVideoState(video).owned === true) {
79 fs.unlink(uploadDir + video.namePath, function (err) {
80 if (err) {
81 logger.error('Cannot remove this video file.')
82 return callback_each(err)
83 }
84
85 callback_each(null)
86 })
87 } else {
88 callback_each(null)
89 }
90 })
91 }, callback)
92}
93
94function removeRemoteVideos (videos, callback) {
95 Videos.removeByIds(map(videos, '_id'), function (err) {
96 if (err) return callback(err)
97
98 removeVideosDataFromDisk(videos, callback)
99 })
100}
101
39function seed (path, callback) { 102function seed (path, callback) {
40 logger.info('Seeding %s...', path) 103 logger.info('Seeding %s...', path)
41 104
@@ -69,3 +132,42 @@ function seedAllExisting (callback) {
69// --------------------------------------------------------------------------- 132// ---------------------------------------------------------------------------
70 133
71module.exports = videos 134module.exports = videos
135
136// ---------------------------------------------------------------------------
137
138function createRemoteVideoObjects (videos, callback) {
139 const remote_videos = []
140
141 async.each(videos, function (video, callback_each) {
142 // Creating the thumbnail for this remote video
143 utils.generateRandomString(16, function (err, random_string) {
144 if (err) return callback_each(err)
145
146 const thumbnail_name = random_string + '.jpg'
147 createThumbnailFromBase64(thumbnail_name, video.thumbnail_base64, function (err) {
148 if (err) return callback_each(err)
149
150 const params = {
151 name: video.name,
152 description: video.description,
153 magnetUri: video.magnetUri,
154 podUrl: video.podUrl,
155 duration: video.duration,
156 thumbnail: thumbnail_name
157 }
158 remote_videos.push(params)
159
160 callback_each(null)
161 })
162 })
163 },
164 function (err) {
165 if (err) return callback(err)
166
167 callback(null, remote_videos)
168 })
169}
170
171function createThumbnailFromBase64 (thumbnail_name, data, callback) {
172 fs.writeFile(thumbnailsDir + thumbnail_name, data, { encoding: 'base64' }, callback)
173}
diff --git a/server/models/videos.js b/server/models/videos.js
index e02158be7..eedb6eb58 100644
--- a/server/models/videos.js
+++ b/server/models/videos.js
@@ -1,18 +1,13 @@
1'use strict' 1'use strict'
2 2
3const async = require('async')
4const config = require('config') 3const config = require('config')
5const dz = require('dezalgo')
6const fs = require('fs')
7const mongoose = require('mongoose') 4const mongoose = require('mongoose')
8const path = require('path')
9 5
10const logger = require('../helpers/logger') 6const logger = require('../helpers/logger')
11 7
12const http = config.get('webserver.https') === true ? 'https' : 'http' 8const http = config.get('webserver.https') === true ? 'https' : 'http'
13const host = config.get('webserver.host') 9const host = config.get('webserver.host')
14const port = config.get('webserver.port') 10const port = config.get('webserver.port')
15const uploadDir = path.join(__dirname, '..', '..', config.get('storage.uploads'))
16 11
17// --------------------------------------------------------------------------- 12// ---------------------------------------------------------------------------
18 13
@@ -23,7 +18,8 @@ const videosSchema = mongoose.Schema({
23 magnetUri: String, 18 magnetUri: String,
24 podUrl: String, 19 podUrl: String,
25 author: String, 20 author: String,
26 duration: Number 21 duration: Number,
22 thumbnail: String
27}) 23})
28const VideosDB = mongoose.model('videos', videosSchema) 24const VideosDB = mongoose.model('videos', videosSchema)
29 25
@@ -34,11 +30,13 @@ const Videos = {
34 addRemotes: addRemotes, 30 addRemotes: addRemotes,
35 get: get, 31 get: get,
36 list: list, 32 list: list,
33 listFromUrl: listFromUrl,
34 listFromUrls: listFromUrls,
35 listFromUrlAndMagnets: listFromUrlAndMagnets,
36 listFromRemotes: listFromRemotes,
37 listOwned: listOwned, 37 listOwned: listOwned,
38 removeOwned: removeOwned, 38 removeOwned: removeOwned,
39 removeAllRemotes: removeAllRemotes, 39 removeByIds: removeByIds,
40 removeAllRemotesOf: removeAllRemotesOf,
41 removeRemotesOfByMagnetUris: removeRemotesOfByMagnetUris,
42 search: search 40 search: search
43} 41}
44 42
@@ -58,38 +56,13 @@ function add (video, callback) {
58 }) 56 })
59} 57}
60 58
61// TODO: avoid doublons
62function addRemotes (videos, callback) { 59function addRemotes (videos, callback) {
63 if (!callback) callback = function () {} 60 videos.forEach(function (video) {
64 61 // Ensure they are remote videos
65 const to_add = [] 62 video.namePath = null
66
67 async.each(videos, function (video, callback_each) {
68 callback_each = dz(callback_each)
69 logger.debug('Add remote video from pod: %s', video.podUrl)
70
71 const params = {
72 name: video.name,
73 namePath: null,
74 description: video.description,
75 magnetUri: video.magnetUri,
76 podUrl: video.podUrl,
77 duration: video.duration
78 }
79
80 to_add.push(params)
81
82 callback_each()
83 }, function () {
84 VideosDB.create(to_add, function (err, videos) {
85 if (err) {
86 logger.error('Cannot insert this remote video.')
87 return callback(err)
88 }
89
90 return callback(null, videos)
91 })
92 }) 63 })
64
65 VideosDB.create(videos, callback)
93} 66}
94 67
95function get (id, callback) { 68function get (id, callback) {
@@ -114,6 +87,22 @@ function list (callback) {
114 }) 87 })
115} 88}
116 89
90function listFromUrl (fromUrl, callback) {
91 VideosDB.find({ podUrl: fromUrl }, callback)
92}
93
94function listFromUrls (fromUrls, callback) {
95 VideosDB.find({ podUrl: { $in: fromUrls } }, callback)
96}
97
98function listFromUrlAndMagnets (fromUrl, magnets, callback) {
99 VideosDB.find({ podUrl: fromUrl, magnetUri: { $in: magnets } }, callback)
100}
101
102function listFromRemotes (callback) {
103 VideosDB.find({ namePath: null }, callback)
104}
105
117function listOwned (callback) { 106function listOwned (callback) {
118 // If namePath is not null this is *our* video 107 // If namePath is not null this is *our* video
119 VideosDB.find({ namePath: { $ne: null } }, function (err, videos_list) { 108 VideosDB.find({ namePath: { $ne: null } }, function (err, videos_list) {
@@ -126,65 +115,14 @@ function listOwned (callback) {
126 }) 115 })
127} 116}
128 117
118// Return the video in the callback
129function removeOwned (id, callback) { 119function removeOwned (id, callback) {
130 VideosDB.findByIdAndRemove(id, function (err, video) { 120 VideosDB.findByIdAndRemove(id, callback)
131 if (err) {
132 logger.error('Cannot remove the torrent.')
133 return callback(err)
134 }
135
136 fs.unlink(uploadDir + video.namePath, function (err) {
137 if (err) {
138 logger.error('Cannot remove this video file.')
139 return callback(err)
140 }
141
142 callback(null)
143 })
144 })
145}
146
147function removeAllRemotes (callback) {
148 VideosDB.remove({ namePath: null }, callback)
149}
150
151function removeAllRemotesOf (fromUrl, callback) {
152 VideosDB.remove({ podUrl: fromUrl }, callback)
153} 121}
154 122
155// Use the magnet Uri because the _id field is not the same on different servers 123// Use the magnet Uri because the _id field is not the same on different servers
156function removeRemotesOfByMagnetUris (fromUrl, magnetUris, callback) { 124function removeByIds (ids, callback) {
157 if (callback === undefined) callback = function () {} 125 VideosDB.remove({ _id: { $in: ids } }, callback)
158
159 VideosDB.find({ magnetUri: { $in: magnetUris } }, function (err, videos) {
160 if (err || !videos) {
161 logger.error('Cannot find the torrent URI of these remote videos.')
162 return callback(err)
163 }
164
165 const to_remove = []
166 async.each(videos, function (video, callback_async) {
167 callback_async = dz(callback_async)
168
169 if (video.podUrl !== fromUrl) {
170 logger.error('The pod %s has not the rights on the video of %s.', fromUrl, video.podUrl)
171 } else {
172 to_remove.push(video._id)
173 }
174
175 callback_async()
176 }, function () {
177 VideosDB.remove({ _id: { $in: to_remove } }, function (err) {
178 if (err) {
179 logger.error('Cannot remove the remote videos.')
180 return callback(err)
181 }
182
183 logger.info('Removed remote videos from %s.', fromUrl)
184 callback(null)
185 })
186 })
187 })
188} 126}
189 127
190function search (name, callback) { 128function search (name, callback) {
diff --git a/server/tests/api/multiplePods.js b/server/tests/api/multiplePods.js
index adc99f714..4a4532a0f 100644
--- a/server/tests/api/multiplePods.js
+++ b/server/tests/api/multiplePods.js
@@ -193,14 +193,23 @@ describe('Test multiple pods', function () {
193 expect(videos).to.be.an('array') 193 expect(videos).to.be.an('array')
194 expect(videos.length).to.equal(4) 194 expect(videos.length).to.equal(4)
195 195
196 const video1 = videos[2] 196 // We not sure about the order of the two last uploads
197 let video1 = null
198 let video2 = null
199 if (videos[2].name === 'my super name for pod 3') {
200 video1 = videos[2]
201 video2 = videos[3]
202 } else {
203 video1 = videos[3]
204 video2 = videos[2]
205 }
206
197 expect(video1.name).to.equal('my super name for pod 3') 207 expect(video1.name).to.equal('my super name for pod 3')
198 expect(video1.description).to.equal('my super description for pod 3') 208 expect(video1.description).to.equal('my super description for pod 3')
199 expect(video1.podUrl).to.equal('http://localhost:9003') 209 expect(video1.podUrl).to.equal('http://localhost:9003')
200 expect(video1.magnetUri).to.exist 210 expect(video1.magnetUri).to.exist
201 expect(video1.duration).to.equal(5) 211 expect(video1.duration).to.equal(5)
202 212
203 const video2 = videos[3]
204 expect(video2.name).to.equal('my super name for pod 3-2') 213 expect(video2.name).to.equal('my super name for pod 3-2')
205 expect(video2.description).to.equal('my super description for pod 3-2') 214 expect(video2.description).to.equal('my super description for pod 3-2')
206 expect(video2.podUrl).to.equal('http://localhost:9003') 215 expect(video2.podUrl).to.equal('http://localhost:9003')