diff options
Diffstat (limited to 'server/initializers')
-rw-r--r-- | server/initializers/checker.js | 12 | ||||
-rw-r--r-- | server/initializers/constants.js | 83 | ||||
-rw-r--r-- | server/initializers/database.js | 90 | ||||
-rw-r--r-- | server/initializers/installer.js | 36 | ||||
-rw-r--r-- | server/initializers/migrations/0005-create-application.js | 17 | ||||
-rw-r--r-- | server/initializers/migrations/0005-example.js | 14 | ||||
-rw-r--r-- | server/initializers/migrations/0010-users-password.js | 22 | ||||
-rw-r--r-- | server/initializers/migrations/0015-admin-role.js | 16 | ||||
-rw-r--r-- | server/initializers/migrations/0020-requests-endpoint.js | 15 | ||||
-rw-r--r-- | server/initializers/migrations/0025-video-filenames.js | 57 | ||||
-rw-r--r-- | server/initializers/migrations/0030-video-magnet.js | 32 | ||||
-rw-r--r-- | server/initializers/migrations/0035-url-to-host.js | 30 | ||||
-rw-r--r-- | server/initializers/migrations/0040-video-remote-id.js | 59 | ||||
-rw-r--r-- | server/initializers/migrator.js | 92 |
14 files changed, 204 insertions, 371 deletions
diff --git a/server/initializers/checker.js b/server/initializers/checker.js index aea013fa9..6471bb4f1 100644 --- a/server/initializers/checker.js +++ b/server/initializers/checker.js | |||
@@ -1,10 +1,8 @@ | |||
1 | 'use strict' | 1 | 'use strict' |
2 | 2 | ||
3 | const config = require('config') | 3 | const config = require('config') |
4 | const mongoose = require('mongoose') | ||
5 | 4 | ||
6 | const Client = mongoose.model('OAuthClient') | 5 | const db = require('./database') |
7 | const User = mongoose.model('User') | ||
8 | 6 | ||
9 | const checker = { | 7 | const checker = { |
10 | checkConfig, | 8 | checkConfig, |
@@ -29,7 +27,7 @@ function checkConfig () { | |||
29 | function checkMissedConfig () { | 27 | function checkMissedConfig () { |
30 | const required = [ 'listen.port', | 28 | const required = [ 'listen.port', |
31 | 'webserver.https', 'webserver.hostname', 'webserver.port', | 29 | 'webserver.https', 'webserver.hostname', 'webserver.port', |
32 | 'database.hostname', 'database.port', 'database.suffix', | 30 | 'database.hostname', 'database.port', 'database.suffix', 'database.username', 'database.password', |
33 | 'storage.certs', 'storage.videos', 'storage.logs', 'storage.thumbnails', 'storage.previews' | 31 | 'storage.certs', 'storage.videos', 'storage.logs', 'storage.thumbnails', 'storage.previews' |
34 | ] | 32 | ] |
35 | const miss = [] | 33 | const miss = [] |
@@ -44,15 +42,15 @@ function checkMissedConfig () { | |||
44 | } | 42 | } |
45 | 43 | ||
46 | function clientsExist (callback) { | 44 | function clientsExist (callback) { |
47 | Client.list(function (err, clients) { | 45 | db.OAuthClient.countTotal(function (err, totalClients) { |
48 | if (err) return callback(err) | 46 | if (err) return callback(err) |
49 | 47 | ||
50 | return callback(null, clients.length !== 0) | 48 | return callback(null, totalClients !== 0) |
51 | }) | 49 | }) |
52 | } | 50 | } |
53 | 51 | ||
54 | function usersExist (callback) { | 52 | function usersExist (callback) { |
55 | User.countTotal(function (err, totalUsers) { | 53 | db.User.countTotal(function (err, totalUsers) { |
56 | if (err) return callback(err) | 54 | if (err) return callback(err) |
57 | 55 | ||
58 | return callback(null, totalUsers !== 0) | 56 | return callback(null, totalUsers !== 0) |
diff --git a/server/initializers/constants.js b/server/initializers/constants.js index 3ddf87454..97e3c5296 100644 --- a/server/initializers/constants.js +++ b/server/initializers/constants.js | |||
@@ -1,7 +1,6 @@ | |||
1 | 'use strict' | 1 | 'use strict' |
2 | 2 | ||
3 | const config = require('config') | 3 | const config = require('config') |
4 | const maxBy = require('lodash/maxBy') | ||
5 | const path = require('path') | 4 | const path = require('path') |
6 | 5 | ||
7 | // --------------------------------------------------------------------------- | 6 | // --------------------------------------------------------------------------- |
@@ -14,13 +13,14 @@ const PAGINATION_COUNT_DEFAULT = 15 | |||
14 | 13 | ||
15 | // Sortable columns per schema | 14 | // Sortable columns per schema |
16 | const SEARCHABLE_COLUMNS = { | 15 | const SEARCHABLE_COLUMNS = { |
17 | VIDEOS: [ 'name', 'magnetUri', 'podHost', 'author', 'tags' ] | 16 | VIDEOS: [ 'name', 'magnetUri', 'host', 'author', 'tags' ] |
18 | } | 17 | } |
19 | 18 | ||
20 | // Sortable columns per schema | 19 | // Sortable columns per schema |
21 | const SORTABLE_COLUMNS = { | 20 | const SORTABLE_COLUMNS = { |
22 | USERS: [ 'username', '-username', 'createdDate', '-createdDate' ], | 21 | USERS: [ 'username', '-username', 'createdAt', '-createdAt' ], |
23 | VIDEOS: [ 'name', '-name', 'duration', '-duration', 'createdDate', '-createdDate' ] | 22 | VIDEO_ABUSES: [ 'createdAt', '-createdAt' ], |
23 | VIDEOS: [ 'name', '-name', 'duration', '-duration', 'createdAt', '-createdAt' ] | ||
24 | } | 24 | } |
25 | 25 | ||
26 | const OAUTH_LIFETIME = { | 26 | const OAUTH_LIFETIME = { |
@@ -37,7 +37,9 @@ const CONFIG = { | |||
37 | DATABASE: { | 37 | DATABASE: { |
38 | DBNAME: 'peertube' + config.get('database.suffix'), | 38 | DBNAME: 'peertube' + config.get('database.suffix'), |
39 | HOSTNAME: config.get('database.hostname'), | 39 | HOSTNAME: config.get('database.hostname'), |
40 | PORT: config.get('database.port') | 40 | PORT: config.get('database.port'), |
41 | USERNAME: config.get('database.username'), | ||
42 | PASSWORD: config.get('database.password') | ||
41 | }, | 43 | }, |
42 | STORAGE: { | 44 | STORAGE: { |
43 | CERT_DIR: path.join(__dirname, '..', '..', config.get('storage.certs')), | 45 | CERT_DIR: path.join(__dirname, '..', '..', config.get('storage.certs')), |
@@ -64,17 +66,19 @@ const CONSTRAINTS_FIELDS = { | |||
64 | USERNAME: { min: 3, max: 20 }, // Length | 66 | USERNAME: { min: 3, max: 20 }, // Length |
65 | PASSWORD: { min: 6, max: 255 } // Length | 67 | PASSWORD: { min: 6, max: 255 } // Length |
66 | }, | 68 | }, |
69 | VIDEO_ABUSES: { | ||
70 | REASON: { min: 2, max: 300 } // Length | ||
71 | }, | ||
67 | VIDEOS: { | 72 | VIDEOS: { |
68 | NAME: { min: 3, max: 50 }, // Length | 73 | NAME: { min: 3, max: 50 }, // Length |
69 | DESCRIPTION: { min: 3, max: 250 }, // Length | 74 | DESCRIPTION: { min: 3, max: 250 }, // Length |
70 | MAGNET: { | 75 | EXTNAME: [ '.mp4', '.ogv', '.webm' ], |
71 | INFO_HASH: { min: 10, max: 50 } // Length | 76 | INFO_HASH: { min: 40, max: 40 }, // Length, infohash is 20 bytes length but we represent it in hexa so 20 * 2 |
72 | }, | ||
73 | DURATION: { min: 1, max: 7200 }, // Number | 77 | DURATION: { min: 1, max: 7200 }, // Number |
74 | TAGS: { min: 1, max: 3 }, // Number of total tags | 78 | TAGS: { min: 1, max: 3 }, // Number of total tags |
75 | TAG: { min: 2, max: 10 }, // Length | 79 | TAG: { min: 2, max: 10 }, // Length |
76 | THUMBNAIL: { min: 2, max: 30 }, | 80 | THUMBNAIL: { min: 2, max: 30 }, |
77 | THUMBNAIL64: { min: 0, max: 20000 } // Bytes | 81 | THUMBNAIL_DATA: { min: 0, max: 20000 } // Bytes |
78 | } | 82 | } |
79 | } | 83 | } |
80 | 84 | ||
@@ -88,41 +92,7 @@ const FRIEND_SCORE = { | |||
88 | 92 | ||
89 | // --------------------------------------------------------------------------- | 93 | // --------------------------------------------------------------------------- |
90 | 94 | ||
91 | const MONGO_MIGRATION_SCRIPTS = [ | 95 | const LAST_MIGRATION_VERSION = 0 |
92 | { | ||
93 | script: '0005-create-application', | ||
94 | version: 5 | ||
95 | }, | ||
96 | { | ||
97 | script: '0010-users-password', | ||
98 | version: 10 | ||
99 | }, | ||
100 | { | ||
101 | script: '0015-admin-role', | ||
102 | version: 15 | ||
103 | }, | ||
104 | { | ||
105 | script: '0020-requests-endpoint', | ||
106 | version: 20 | ||
107 | }, | ||
108 | { | ||
109 | script: '0025-video-filenames', | ||
110 | version: 25 | ||
111 | }, | ||
112 | { | ||
113 | script: '0030-video-magnet', | ||
114 | version: 30 | ||
115 | }, | ||
116 | { | ||
117 | script: '0035-url-to-host', | ||
118 | version: 35 | ||
119 | }, | ||
120 | { | ||
121 | script: '0040-video-remote-id', | ||
122 | version: 40 | ||
123 | } | ||
124 | ] | ||
125 | const LAST_MONGO_SCHEMA_VERSION = (maxBy(MONGO_MIGRATION_SCRIPTS, 'version'))['version'] | ||
126 | 96 | ||
127 | // --------------------------------------------------------------------------- | 97 | // --------------------------------------------------------------------------- |
128 | 98 | ||
@@ -138,8 +108,10 @@ let REQUESTS_INTERVAL = 600000 | |||
138 | // Number of requests in parallel we can make | 108 | // Number of requests in parallel we can make |
139 | const REQUESTS_IN_PARALLEL = 10 | 109 | const REQUESTS_IN_PARALLEL = 10 |
140 | 110 | ||
141 | // How many requests we put in request | 111 | // To how many pods we send requests |
142 | const REQUESTS_LIMIT = 10 | 112 | const REQUESTS_LIMIT_PODS = 10 |
113 | // How many requests we send to a pod per interval | ||
114 | const REQUESTS_LIMIT_PER_POD = 5 | ||
143 | 115 | ||
144 | // Number of requests to retry for replay requests module | 116 | // Number of requests to retry for replay requests module |
145 | const RETRY_REQUESTS = 5 | 117 | const RETRY_REQUESTS = 5 |
@@ -148,16 +120,21 @@ const REQUEST_ENDPOINTS = { | |||
148 | VIDEOS: 'videos' | 120 | VIDEOS: 'videos' |
149 | } | 121 | } |
150 | 122 | ||
151 | // --------------------------------------------------------------------------- | ||
152 | |||
153 | const REMOTE_SCHEME = { | 123 | const REMOTE_SCHEME = { |
154 | HTTP: 'https', | 124 | HTTP: 'https', |
155 | WS: 'wss' | 125 | WS: 'wss' |
156 | } | 126 | } |
157 | 127 | ||
128 | // --------------------------------------------------------------------------- | ||
129 | |||
130 | const SIGNATURE_ALGORITHM = 'RSA-SHA256' | ||
131 | const SIGNATURE_ENCODING = 'hex' | ||
132 | |||
158 | // Password encryption | 133 | // Password encryption |
159 | const BCRYPT_SALT_SIZE = 10 | 134 | const BCRYPT_SALT_SIZE = 10 |
160 | 135 | ||
136 | // --------------------------------------------------------------------------- | ||
137 | |||
161 | // Express static paths (router) | 138 | // Express static paths (router) |
162 | const STATIC_PATHS = { | 139 | const STATIC_PATHS = { |
163 | PREVIEWS: '/static/previews/', | 140 | PREVIEWS: '/static/previews/', |
@@ -173,6 +150,8 @@ let STATIC_MAX_AGE = '30d' | |||
173 | const THUMBNAILS_SIZE = '200x110' | 150 | const THUMBNAILS_SIZE = '200x110' |
174 | const PREVIEWS_SIZE = '640x480' | 151 | const PREVIEWS_SIZE = '640x480' |
175 | 152 | ||
153 | // --------------------------------------------------------------------------- | ||
154 | |||
176 | const USER_ROLES = { | 155 | const USER_ROLES = { |
177 | ADMIN: 'admin', | 156 | ADMIN: 'admin', |
178 | USER: 'user' | 157 | USER: 'user' |
@@ -198,8 +177,7 @@ module.exports = { | |||
198 | CONFIG, | 177 | CONFIG, |
199 | CONSTRAINTS_FIELDS, | 178 | CONSTRAINTS_FIELDS, |
200 | FRIEND_SCORE, | 179 | FRIEND_SCORE, |
201 | LAST_MONGO_SCHEMA_VERSION, | 180 | LAST_MIGRATION_VERSION, |
202 | MONGO_MIGRATION_SCRIPTS, | ||
203 | OAUTH_LIFETIME, | 181 | OAUTH_LIFETIME, |
204 | PAGINATION_COUNT_DEFAULT, | 182 | PAGINATION_COUNT_DEFAULT, |
205 | PODS_SCORE, | 183 | PODS_SCORE, |
@@ -208,9 +186,12 @@ module.exports = { | |||
208 | REQUEST_ENDPOINTS, | 186 | REQUEST_ENDPOINTS, |
209 | REQUESTS_IN_PARALLEL, | 187 | REQUESTS_IN_PARALLEL, |
210 | REQUESTS_INTERVAL, | 188 | REQUESTS_INTERVAL, |
211 | REQUESTS_LIMIT, | 189 | REQUESTS_LIMIT_PODS, |
190 | REQUESTS_LIMIT_PER_POD, | ||
212 | RETRY_REQUESTS, | 191 | RETRY_REQUESTS, |
213 | SEARCHABLE_COLUMNS, | 192 | SEARCHABLE_COLUMNS, |
193 | SIGNATURE_ALGORITHM, | ||
194 | SIGNATURE_ENCODING, | ||
214 | SORTABLE_COLUMNS, | 195 | SORTABLE_COLUMNS, |
215 | STATIC_MAX_AGE, | 196 | STATIC_MAX_AGE, |
216 | STATIC_PATHS, | 197 | STATIC_PATHS, |
diff --git a/server/initializers/database.js b/server/initializers/database.js index 0564e4e77..f8f68adeb 100644 --- a/server/initializers/database.js +++ b/server/initializers/database.js | |||
@@ -1,37 +1,77 @@ | |||
1 | 'use strict' | 1 | 'use strict' |
2 | 2 | ||
3 | const mongoose = require('mongoose') | 3 | const fs = require('fs') |
4 | const path = require('path') | ||
5 | const Sequelize = require('sequelize') | ||
4 | 6 | ||
5 | const constants = require('../initializers/constants') | 7 | const constants = require('../initializers/constants') |
6 | const logger = require('../helpers/logger') | 8 | const logger = require('../helpers/logger') |
9 | const utils = require('../helpers/utils') | ||
7 | 10 | ||
8 | // Bootstrap models | 11 | const database = {} |
9 | require('../models/application') | ||
10 | require('../models/oauth-token') | ||
11 | require('../models/user') | ||
12 | require('../models/oauth-client') | ||
13 | require('../models/video') | ||
14 | // Request model needs Video model | ||
15 | require('../models/pods') | ||
16 | // Request model needs Pod model | ||
17 | require('../models/request') | ||
18 | |||
19 | const database = { | ||
20 | connect: connect | ||
21 | } | ||
22 | 12 | ||
23 | function connect () { | 13 | const dbname = constants.CONFIG.DATABASE.DBNAME |
24 | mongoose.Promise = global.Promise | 14 | const username = constants.CONFIG.DATABASE.USERNAME |
25 | mongoose.connect('mongodb://' + constants.CONFIG.DATABASE.HOSTNAME + ':' + constants.CONFIG.DATABASE.PORT + '/' + constants.CONFIG.DATABASE.DBNAME) | 15 | const password = constants.CONFIG.DATABASE.PASSWORD |
26 | mongoose.connection.on('error', function () { | ||
27 | throw new Error('Mongodb connection error.') | ||
28 | }) | ||
29 | 16 | ||
30 | mongoose.connection.on('open', function () { | 17 | const sequelize = new Sequelize(dbname, username, password, { |
31 | logger.info('Connected to mongodb.') | 18 | dialect: 'postgres', |
32 | }) | 19 | host: constants.CONFIG.DATABASE.HOSTNAME, |
33 | } | 20 | port: constants.CONFIG.DATABASE.PORT, |
21 | benchmark: utils.isTestInstance(), | ||
22 | |||
23 | logging: function (message, benchmark) { | ||
24 | let newMessage = message | ||
25 | if (benchmark !== undefined) { | ||
26 | newMessage += ' | ' + benchmark + 'ms' | ||
27 | } | ||
28 | |||
29 | logger.debug(newMessage) | ||
30 | } | ||
31 | }) | ||
32 | |||
33 | database.sequelize = sequelize | ||
34 | database.Sequelize = Sequelize | ||
35 | database.init = init | ||
34 | 36 | ||
35 | // --------------------------------------------------------------------------- | 37 | // --------------------------------------------------------------------------- |
36 | 38 | ||
37 | module.exports = database | 39 | module.exports = database |
40 | |||
41 | // --------------------------------------------------------------------------- | ||
42 | |||
43 | function init (silent, callback) { | ||
44 | if (!callback) { | ||
45 | callback = silent | ||
46 | silent = false | ||
47 | } | ||
48 | |||
49 | if (!callback) callback = function () {} | ||
50 | |||
51 | const modelDirectory = path.join(__dirname, '..', 'models') | ||
52 | fs.readdir(modelDirectory, function (err, files) { | ||
53 | if (err) throw err | ||
54 | |||
55 | files.filter(function (file) { | ||
56 | // For all models but not utils.js | ||
57 | if (file === 'utils.js') return false | ||
58 | |||
59 | return true | ||
60 | }) | ||
61 | .forEach(function (file) { | ||
62 | const model = sequelize.import(path.join(modelDirectory, file)) | ||
63 | |||
64 | database[model.name] = model | ||
65 | }) | ||
66 | |||
67 | Object.keys(database).forEach(function (modelName) { | ||
68 | if ('associate' in database[modelName]) { | ||
69 | database[modelName].associate(database) | ||
70 | } | ||
71 | }) | ||
72 | |||
73 | if (!silent) logger.info('Database is ready.') | ||
74 | |||
75 | return callback(null) | ||
76 | }) | ||
77 | } | ||
diff --git a/server/initializers/installer.js b/server/initializers/installer.js index 1df300ba8..fb63b81ac 100644 --- a/server/initializers/installer.js +++ b/server/initializers/installer.js | |||
@@ -3,26 +3,27 @@ | |||
3 | const config = require('config') | 3 | const config = require('config') |
4 | const each = require('async/each') | 4 | const each = require('async/each') |
5 | const mkdirp = require('mkdirp') | 5 | const mkdirp = require('mkdirp') |
6 | const mongoose = require('mongoose') | ||
7 | const passwordGenerator = require('password-generator') | 6 | const passwordGenerator = require('password-generator') |
8 | const path = require('path') | 7 | const path = require('path') |
9 | const series = require('async/series') | 8 | const series = require('async/series') |
10 | 9 | ||
11 | const checker = require('./checker') | 10 | const checker = require('./checker') |
12 | const constants = require('./constants') | 11 | const constants = require('./constants') |
12 | const db = require('./database') | ||
13 | const logger = require('../helpers/logger') | 13 | const logger = require('../helpers/logger') |
14 | const peertubeCrypto = require('../helpers/peertube-crypto') | 14 | const peertubeCrypto = require('../helpers/peertube-crypto') |
15 | 15 | ||
16 | const Application = mongoose.model('Application') | ||
17 | const Client = mongoose.model('OAuthClient') | ||
18 | const User = mongoose.model('User') | ||
19 | |||
20 | const installer = { | 16 | const installer = { |
21 | installApplication | 17 | installApplication |
22 | } | 18 | } |
23 | 19 | ||
24 | function installApplication (callback) { | 20 | function installApplication (callback) { |
25 | series([ | 21 | series([ |
22 | function createDatabase (callbackAsync) { | ||
23 | db.sequelize.sync().asCallback(callbackAsync) | ||
24 | // db.sequelize.sync({ force: true }).asCallback(callbackAsync) | ||
25 | }, | ||
26 | |||
26 | function createDirectories (callbackAsync) { | 27 | function createDirectories (callbackAsync) { |
27 | createDirectoriesIfNotExist(callbackAsync) | 28 | createDirectoriesIfNotExist(callbackAsync) |
28 | }, | 29 | }, |
@@ -65,16 +66,18 @@ function createOAuthClientIfNotExist (callback) { | |||
65 | 66 | ||
66 | logger.info('Creating a default OAuth Client.') | 67 | logger.info('Creating a default OAuth Client.') |
67 | 68 | ||
68 | const secret = passwordGenerator(32, false) | 69 | const id = passwordGenerator(32, false, /[a-z0-9]/) |
69 | const client = new Client({ | 70 | const secret = passwordGenerator(32, false, /[a-zA-Z0-9]/) |
71 | const client = db.OAuthClient.build({ | ||
72 | clientId: id, | ||
70 | clientSecret: secret, | 73 | clientSecret: secret, |
71 | grants: [ 'password', 'refresh_token' ] | 74 | grants: [ 'password', 'refresh_token' ] |
72 | }) | 75 | }) |
73 | 76 | ||
74 | client.save(function (err, createdClient) { | 77 | client.save().asCallback(function (err, createdClient) { |
75 | if (err) return callback(err) | 78 | if (err) return callback(err) |
76 | 79 | ||
77 | logger.info('Client id: ' + createdClient._id) | 80 | logger.info('Client id: ' + createdClient.clientId) |
78 | logger.info('Client secret: ' + createdClient.clientSecret) | 81 | logger.info('Client secret: ' + createdClient.clientSecret) |
79 | 82 | ||
80 | return callback(null) | 83 | return callback(null) |
@@ -93,6 +96,7 @@ function createOAuthAdminIfNotExist (callback) { | |||
93 | 96 | ||
94 | const username = 'root' | 97 | const username = 'root' |
95 | const role = constants.USER_ROLES.ADMIN | 98 | const role = constants.USER_ROLES.ADMIN |
99 | const createOptions = {} | ||
96 | let password = '' | 100 | let password = '' |
97 | 101 | ||
98 | // Do not generate a random password for tests | 102 | // Do not generate a random password for tests |
@@ -102,25 +106,27 @@ function createOAuthAdminIfNotExist (callback) { | |||
102 | if (process.env.NODE_APP_INSTANCE) { | 106 | if (process.env.NODE_APP_INSTANCE) { |
103 | password += process.env.NODE_APP_INSTANCE | 107 | password += process.env.NODE_APP_INSTANCE |
104 | } | 108 | } |
109 | |||
110 | // Our password is weak so do not validate it | ||
111 | createOptions.validate = false | ||
105 | } else { | 112 | } else { |
106 | password = passwordGenerator(8, true) | 113 | password = passwordGenerator(8, true) |
107 | } | 114 | } |
108 | 115 | ||
109 | const user = new User({ | 116 | const userData = { |
110 | username, | 117 | username, |
111 | password, | 118 | password, |
112 | role | 119 | role |
113 | }) | 120 | } |
114 | 121 | ||
115 | user.save(function (err, createdUser) { | 122 | db.User.create(userData, createOptions).asCallback(function (err, createdUser) { |
116 | if (err) return callback(err) | 123 | if (err) return callback(err) |
117 | 124 | ||
118 | logger.info('Username: ' + username) | 125 | logger.info('Username: ' + username) |
119 | logger.info('User password: ' + password) | 126 | logger.info('User password: ' + password) |
120 | 127 | ||
121 | logger.info('Creating Application collection.') | 128 | logger.info('Creating Application table.') |
122 | const application = new Application({ mongoSchemaVersion: constants.LAST_MONGO_SCHEMA_VERSION }) | 129 | db.Application.create({ migrationVersion: constants.LAST_MIGRATION_VERSION }).asCallback(callback) |
123 | application.save(callback) | ||
124 | }) | 130 | }) |
125 | }) | 131 | }) |
126 | } | 132 | } |
diff --git a/server/initializers/migrations/0005-create-application.js b/server/initializers/migrations/0005-create-application.js deleted file mode 100644 index e99dec019..000000000 --- a/server/initializers/migrations/0005-create-application.js +++ /dev/null | |||
@@ -1,17 +0,0 @@ | |||
1 | /* | ||
2 | Create the application collection in MongoDB. | ||
3 | Used to store the actual MongoDB scheme version. | ||
4 | */ | ||
5 | |||
6 | const mongoose = require('mongoose') | ||
7 | |||
8 | const Application = mongoose.model('Application') | ||
9 | |||
10 | exports.up = function (callback) { | ||
11 | const application = new Application() | ||
12 | application.save(callback) | ||
13 | } | ||
14 | |||
15 | exports.down = function (callback) { | ||
16 | throw new Error('Not implemented.') | ||
17 | } | ||
diff --git a/server/initializers/migrations/0005-example.js b/server/initializers/migrations/0005-example.js new file mode 100644 index 000000000..cedc42919 --- /dev/null +++ b/server/initializers/migrations/0005-example.js | |||
@@ -0,0 +1,14 @@ | |||
1 | // /* | ||
2 | // This is just an example. | ||
3 | // */ | ||
4 | |||
5 | // const db = require('../database') | ||
6 | |||
7 | // // options contains the transaction | ||
8 | // exports.up = function (options, callback) { | ||
9 | // db.Application.create({ migrationVersion: 42 }, { transaction: options.transaction }).asCallback(callback) | ||
10 | // } | ||
11 | |||
12 | // exports.down = function (options, callback) { | ||
13 | // throw new Error('Not implemented.') | ||
14 | // } | ||
diff --git a/server/initializers/migrations/0010-users-password.js b/server/initializers/migrations/0010-users-password.js deleted file mode 100644 index a0616a269..000000000 --- a/server/initializers/migrations/0010-users-password.js +++ /dev/null | |||
@@ -1,22 +0,0 @@ | |||
1 | /* | ||
2 | Convert plain user password to encrypted user password. | ||
3 | */ | ||
4 | |||
5 | const eachSeries = require('async/eachSeries') | ||
6 | const mongoose = require('mongoose') | ||
7 | |||
8 | const User = mongoose.model('User') | ||
9 | |||
10 | exports.up = function (callback) { | ||
11 | User.list(function (err, users) { | ||
12 | if (err) return callback(err) | ||
13 | |||
14 | eachSeries(users, function (user, callbackEach) { | ||
15 | user.save(callbackEach) | ||
16 | }, callback) | ||
17 | }) | ||
18 | } | ||
19 | |||
20 | exports.down = function (callback) { | ||
21 | throw new Error('Not implemented.') | ||
22 | } | ||
diff --git a/server/initializers/migrations/0015-admin-role.js b/server/initializers/migrations/0015-admin-role.js deleted file mode 100644 index af06dca9e..000000000 --- a/server/initializers/migrations/0015-admin-role.js +++ /dev/null | |||
@@ -1,16 +0,0 @@ | |||
1 | /* | ||
2 | Set the admin role to the root user. | ||
3 | */ | ||
4 | |||
5 | const constants = require('../constants') | ||
6 | const mongoose = require('mongoose') | ||
7 | |||
8 | const User = mongoose.model('User') | ||
9 | |||
10 | exports.up = function (callback) { | ||
11 | User.update({ username: 'root' }, { role: constants.USER_ROLES.ADMIN }, callback) | ||
12 | } | ||
13 | |||
14 | exports.down = function (callback) { | ||
15 | throw new Error('Not implemented.') | ||
16 | } | ||
diff --git a/server/initializers/migrations/0020-requests-endpoint.js b/server/initializers/migrations/0020-requests-endpoint.js deleted file mode 100644 index 55feec571..000000000 --- a/server/initializers/migrations/0020-requests-endpoint.js +++ /dev/null | |||
@@ -1,15 +0,0 @@ | |||
1 | /* | ||
2 | Set the endpoint videos for requests. | ||
3 | */ | ||
4 | |||
5 | const mongoose = require('mongoose') | ||
6 | |||
7 | const Request = mongoose.model('Request') | ||
8 | |||
9 | exports.up = function (callback) { | ||
10 | Request.update({ }, { endpoint: 'videos' }, callback) | ||
11 | } | ||
12 | |||
13 | exports.down = function (callback) { | ||
14 | throw new Error('Not implemented.') | ||
15 | } | ||
diff --git a/server/initializers/migrations/0025-video-filenames.js b/server/initializers/migrations/0025-video-filenames.js deleted file mode 100644 index df21494d7..000000000 --- a/server/initializers/migrations/0025-video-filenames.js +++ /dev/null | |||
@@ -1,57 +0,0 @@ | |||
1 | /* | ||
2 | Rename thumbnails and video filenames to _id.extension | ||
3 | */ | ||
4 | |||
5 | const each = require('async/each') | ||
6 | const fs = require('fs') | ||
7 | const path = require('path') | ||
8 | const mongoose = require('mongoose') | ||
9 | |||
10 | const constants = require('../constants') | ||
11 | const logger = require('../../helpers/logger') | ||
12 | |||
13 | const Video = mongoose.model('Video') | ||
14 | |||
15 | exports.up = function (callback) { | ||
16 | // Use of lean because the new Video scheme does not have filename field | ||
17 | Video.find({ filename: { $ne: null } }).lean().exec(function (err, videos) { | ||
18 | if (err) throw err | ||
19 | |||
20 | each(videos, function (video, callbackEach) { | ||
21 | const torrentName = video.filename + '.torrent' | ||
22 | const thumbnailName = video.thumbnail | ||
23 | const thumbnailExtension = path.extname(thumbnailName) | ||
24 | const videoName = video.filename | ||
25 | const videoExtension = path.extname(videoName) | ||
26 | |||
27 | const newTorrentName = video._id + '.torrent' | ||
28 | const newThumbnailName = video._id + thumbnailExtension | ||
29 | const newVideoName = video._id + videoExtension | ||
30 | |||
31 | const torrentsDir = constants.CONFIG.STORAGE.TORRENTS_DIR | ||
32 | const thumbnailsDir = constants.CONFIG.STORAGE.THUMBNAILS_DIR | ||
33 | const videosDir = constants.CONFIG.STORAGE.VIDEOS_DIR | ||
34 | |||
35 | logger.info('Renaming %s to %s.', torrentsDir + torrentName, torrentsDir + newTorrentName) | ||
36 | fs.renameSync(torrentsDir + torrentName, torrentsDir + newTorrentName) | ||
37 | |||
38 | logger.info('Renaming %s to %s.', thumbnailsDir + thumbnailName, thumbnailsDir + newThumbnailName) | ||
39 | fs.renameSync(thumbnailsDir + thumbnailName, thumbnailsDir + newThumbnailName) | ||
40 | |||
41 | logger.info('Renaming %s to %s.', videosDir + videoName, videosDir + newVideoName) | ||
42 | fs.renameSync(videosDir + videoName, videosDir + newVideoName) | ||
43 | |||
44 | Video.load(video._id, function (err, videoObj) { | ||
45 | if (err) return callbackEach(err) | ||
46 | |||
47 | videoObj.extname = videoExtension | ||
48 | videoObj.remoteId = null | ||
49 | videoObj.save(callbackEach) | ||
50 | }) | ||
51 | }, callback) | ||
52 | }) | ||
53 | } | ||
54 | |||
55 | exports.down = function (callback) { | ||
56 | throw new Error('Not implemented.') | ||
57 | } | ||
diff --git a/server/initializers/migrations/0030-video-magnet.js b/server/initializers/migrations/0030-video-magnet.js deleted file mode 100644 index b9119d61c..000000000 --- a/server/initializers/migrations/0030-video-magnet.js +++ /dev/null | |||
@@ -1,32 +0,0 @@ | |||
1 | /* | ||
2 | Change video magnet structures | ||
3 | */ | ||
4 | |||
5 | const each = require('async/each') | ||
6 | const magnet = require('magnet-uri') | ||
7 | const mongoose = require('mongoose') | ||
8 | |||
9 | const Video = mongoose.model('Video') | ||
10 | |||
11 | exports.up = function (callback) { | ||
12 | // Use of lean because the new Video scheme does not have magnetUri field | ||
13 | Video.find({ }).lean().exec(function (err, videos) { | ||
14 | if (err) throw err | ||
15 | |||
16 | each(videos, function (video, callbackEach) { | ||
17 | const parsed = magnet.decode(video.magnetUri) | ||
18 | const infoHash = parsed.infoHash | ||
19 | |||
20 | Video.load(video._id, function (err, videoObj) { | ||
21 | if (err) return callbackEach(err) | ||
22 | |||
23 | videoObj.magnet.infoHash = infoHash | ||
24 | videoObj.save(callbackEach) | ||
25 | }) | ||
26 | }, callback) | ||
27 | }) | ||
28 | } | ||
29 | |||
30 | exports.down = function (callback) { | ||
31 | throw new Error('Not implemented.') | ||
32 | } | ||
diff --git a/server/initializers/migrations/0035-url-to-host.js b/server/initializers/migrations/0035-url-to-host.js deleted file mode 100644 index 6243304d5..000000000 --- a/server/initializers/migrations/0035-url-to-host.js +++ /dev/null | |||
@@ -1,30 +0,0 @@ | |||
1 | /* | ||
2 | Change video magnet structures | ||
3 | */ | ||
4 | |||
5 | const each = require('async/each') | ||
6 | const mongoose = require('mongoose') | ||
7 | |||
8 | const Video = mongoose.model('Video') | ||
9 | |||
10 | exports.up = function (callback) { | ||
11 | // Use of lean because the new Video scheme does not have podUrl field | ||
12 | Video.find({ }).lean().exec(function (err, videos) { | ||
13 | if (err) throw err | ||
14 | |||
15 | each(videos, function (video, callbackEach) { | ||
16 | Video.load(video._id, function (err, videoObj) { | ||
17 | if (err) return callbackEach(err) | ||
18 | |||
19 | const host = video.podUrl.split('://')[1] | ||
20 | |||
21 | videoObj.podHost = host | ||
22 | videoObj.save(callbackEach) | ||
23 | }) | ||
24 | }, callback) | ||
25 | }) | ||
26 | } | ||
27 | |||
28 | exports.down = function (callback) { | ||
29 | throw new Error('Not implemented.') | ||
30 | } | ||
diff --git a/server/initializers/migrations/0040-video-remote-id.js b/server/initializers/migrations/0040-video-remote-id.js deleted file mode 100644 index 46a14a689..000000000 --- a/server/initializers/migrations/0040-video-remote-id.js +++ /dev/null | |||
@@ -1,59 +0,0 @@ | |||
1 | /* | ||
2 | Use remote id as identifier | ||
3 | */ | ||
4 | |||
5 | const map = require('lodash/map') | ||
6 | const mongoose = require('mongoose') | ||
7 | const readline = require('readline') | ||
8 | |||
9 | const rl = readline.createInterface({ | ||
10 | input: process.stdin, | ||
11 | output: process.stdout | ||
12 | }) | ||
13 | |||
14 | const logger = require('../../helpers/logger') | ||
15 | const friends = require('../../lib/friends') | ||
16 | |||
17 | const Pod = mongoose.model('Pod') | ||
18 | const Video = mongoose.model('Video') | ||
19 | |||
20 | exports.up = function (callback) { | ||
21 | Pod.find({}).lean().exec(function (err, pods) { | ||
22 | if (err) return callback(err) | ||
23 | |||
24 | // We need to quit friends first | ||
25 | if (pods.length === 0) { | ||
26 | return setVideosRemoteId(callback) | ||
27 | } | ||
28 | |||
29 | const timeout = setTimeout(function () { | ||
30 | throw new Error('You need to enter a value!') | ||
31 | }, 10000) | ||
32 | |||
33 | rl.question('I am sorry but I need to quit friends for upgrading. Do you want to continue? (yes/*)', function (answer) { | ||
34 | if (answer !== 'yes') throw new Error('I cannot continue.') | ||
35 | |||
36 | clearTimeout(timeout) | ||
37 | rl.close() | ||
38 | |||
39 | const urls = map(pods, 'url') | ||
40 | logger.info('Saying goodbye to: ' + urls.join(', ')) | ||
41 | |||
42 | setVideosRemoteId(function () { | ||
43 | friends.quitFriends(callback) | ||
44 | }) | ||
45 | }) | ||
46 | }) | ||
47 | } | ||
48 | |||
49 | exports.down = function (callback) { | ||
50 | throw new Error('Not implemented.') | ||
51 | } | ||
52 | |||
53 | function setVideosRemoteId (callback) { | ||
54 | Video.update({ filename: { $ne: null } }, { remoteId: null }, function (err) { | ||
55 | if (err) throw err | ||
56 | |||
57 | Video.update({ filename: null }, { remoteId: mongoose.Types.ObjectId() }, callback) | ||
58 | }) | ||
59 | } | ||
diff --git a/server/initializers/migrator.js b/server/initializers/migrator.js index 6b31d994f..e5288b615 100644 --- a/server/initializers/migrator.js +++ b/server/initializers/migrator.js | |||
@@ -1,48 +1,36 @@ | |||
1 | 'use strict' | 1 | 'use strict' |
2 | 2 | ||
3 | const eachSeries = require('async/eachSeries') | 3 | const eachSeries = require('async/eachSeries') |
4 | const mongoose = require('mongoose') | 4 | const fs = require('fs') |
5 | const path = require('path') | 5 | const path = require('path') |
6 | 6 | ||
7 | const constants = require('./constants') | 7 | const constants = require('./constants') |
8 | const db = require('./database') | ||
8 | const logger = require('../helpers/logger') | 9 | const logger = require('../helpers/logger') |
9 | 10 | ||
10 | const Application = mongoose.model('Application') | ||
11 | |||
12 | const migrator = { | 11 | const migrator = { |
13 | migrate: migrate | 12 | migrate: migrate |
14 | } | 13 | } |
15 | 14 | ||
16 | function migrate (callback) { | 15 | function migrate (callback) { |
17 | Application.loadMongoSchemaVersion(function (err, actualVersion) { | 16 | db.Application.loadMigrationVersion(function (err, actualVersion) { |
18 | if (err) return callback(err) | 17 | if (err) return callback(err) |
19 | 18 | ||
20 | // If there are a new mongo schemas | 19 | // If there are a new migration scripts |
21 | if (!actualVersion || actualVersion < constants.LAST_MONGO_SCHEMA_VERSION) { | 20 | if (actualVersion < constants.LAST_MIGRATION_VERSION) { |
22 | logger.info('Begin migrations.') | 21 | logger.info('Begin migrations.') |
23 | 22 | ||
24 | eachSeries(constants.MONGO_MIGRATION_SCRIPTS, function (entity, callbackEach) { | 23 | getMigrationScripts(function (err, migrationScripts) { |
25 | const versionScript = entity.version | 24 | if (err) return callback(err) |
26 | |||
27 | // Do not execute old migration scripts | ||
28 | if (versionScript <= actualVersion) return callbackEach(null) | ||
29 | |||
30 | // Load the migration module and run it | ||
31 | const migrationScriptName = entity.script | ||
32 | logger.info('Executing %s migration script.', migrationScriptName) | ||
33 | 25 | ||
34 | const migrationScript = require(path.join(__dirname, 'migrations', migrationScriptName)) | 26 | eachSeries(migrationScripts, function (entity, callbackEach) { |
35 | migrationScript.up(function (err) { | 27 | executeMigration(actualVersion, entity, callbackEach) |
36 | if (err) return callbackEach(err) | 28 | }, function (err) { |
29 | if (err) return callback(err) | ||
37 | 30 | ||
38 | // Update the new mongo version schema | 31 | logger.info('Migrations finished. New migration version schema: %s', constants.LAST_MIGRATION_VERSION) |
39 | Application.updateMongoSchemaVersion(versionScript, callbackEach) | 32 | return callback(null) |
40 | }) | 33 | }) |
41 | }, function (err) { | ||
42 | if (err) return callback(err) | ||
43 | |||
44 | logger.info('Migrations finished. New mongo version schema: %s', constants.LAST_MONGO_SCHEMA_VERSION) | ||
45 | return callback(null) | ||
46 | }) | 34 | }) |
47 | } else { | 35 | } else { |
48 | return callback(null) | 36 | return callback(null) |
@@ -54,3 +42,57 @@ function migrate (callback) { | |||
54 | 42 | ||
55 | module.exports = migrator | 43 | module.exports = migrator |
56 | 44 | ||
45 | // --------------------------------------------------------------------------- | ||
46 | |||
47 | function getMigrationScripts (callback) { | ||
48 | fs.readdir(path.join(__dirname, 'migrations'), function (err, files) { | ||
49 | if (err) return callback(err) | ||
50 | |||
51 | const filesToMigrate = [] | ||
52 | |||
53 | files.forEach(function (file) { | ||
54 | // Filename is something like 'version-blabla.js' | ||
55 | const version = file.split('-')[0] | ||
56 | filesToMigrate.push({ | ||
57 | version, | ||
58 | script: file | ||
59 | }) | ||
60 | }) | ||
61 | |||
62 | return callback(err, filesToMigrate) | ||
63 | }) | ||
64 | } | ||
65 | |||
66 | function executeMigration (actualVersion, entity, callback) { | ||
67 | const versionScript = entity.version | ||
68 | |||
69 | // Do not execute old migration scripts | ||
70 | if (versionScript <= actualVersion) return callback(null) | ||
71 | |||
72 | // Load the migration module and run it | ||
73 | const migrationScriptName = entity.script | ||
74 | logger.info('Executing %s migration script.', migrationScriptName) | ||
75 | |||
76 | const migrationScript = require(path.join(__dirname, 'migrations', migrationScriptName)) | ||
77 | |||
78 | db.sequelize.transaction().asCallback(function (err, t) { | ||
79 | if (err) return callback(err) | ||
80 | |||
81 | migrationScript.up({ transaction: t }, function (err) { | ||
82 | if (err) { | ||
83 | t.rollback() | ||
84 | return callback(err) | ||
85 | } | ||
86 | |||
87 | // Update the new migration version | ||
88 | db.Application.updateMigrationVersion(versionScript, t, function (err) { | ||
89 | if (err) { | ||
90 | t.rollback() | ||
91 | return callback(err) | ||
92 | } | ||
93 | |||
94 | t.commit().asCallback(callback) | ||
95 | }) | ||
96 | }) | ||
97 | }) | ||
98 | } | ||