From b769007f733769d3afe2d29a3eb23e2e7693f301 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Sun, 25 Dec 2016 09:44:57 +0100 Subject: Update migrations code --- server/initializers/checker.js | 2 +- server/initializers/constants.js | 43 ++--------- server/initializers/database.js | 65 ++++++++++------ server/initializers/installer.js | 5 +- .../migrations/0005-create-application.js | 17 ----- server/initializers/migrations/0005-example.js | 14 ++++ .../initializers/migrations/0010-users-password.js | 22 ------ server/initializers/migrations/0015-admin-role.js | 16 ---- .../migrations/0020-requests-endpoint.js | 15 ---- .../migrations/0025-video-filenames.js | 57 -------------- .../initializers/migrations/0030-video-magnet.js | 32 -------- server/initializers/migrations/0035-url-to-host.js | 30 -------- .../migrations/0040-video-remote-id.js | 59 --------------- server/initializers/migrator.js | 88 ++++++++++++++++------ 14 files changed, 130 insertions(+), 335 deletions(-) delete mode 100644 server/initializers/migrations/0005-create-application.js create mode 100644 server/initializers/migrations/0005-example.js delete mode 100644 server/initializers/migrations/0010-users-password.js delete mode 100644 server/initializers/migrations/0015-admin-role.js delete mode 100644 server/initializers/migrations/0020-requests-endpoint.js delete mode 100644 server/initializers/migrations/0025-video-filenames.js delete mode 100644 server/initializers/migrations/0030-video-magnet.js delete mode 100644 server/initializers/migrations/0035-url-to-host.js delete mode 100644 server/initializers/migrations/0040-video-remote-id.js (limited to 'server/initializers') diff --git a/server/initializers/checker.js b/server/initializers/checker.js index 7b402de82..2753604dc 100644 --- a/server/initializers/checker.js +++ b/server/initializers/checker.js @@ -27,7 +27,7 @@ function checkConfig () { function checkMissedConfig () { const required = [ 'listen.port', 'webserver.https', 'webserver.hostname', 'webserver.port', - 'database.hostname', 'database.port', 'database.suffix', + 'database.hostname', 'database.port', 'database.suffix', 'database.username', 'database.password', 'storage.certs', 'storage.videos', 'storage.logs', 'storage.thumbnails', 'storage.previews' ] const miss = [] diff --git a/server/initializers/constants.js b/server/initializers/constants.js index 1ad0c82a0..6f39b65da 100644 --- a/server/initializers/constants.js +++ b/server/initializers/constants.js @@ -37,7 +37,9 @@ const CONFIG = { DATABASE: { DBNAME: 'peertube' + config.get('database.suffix'), HOSTNAME: config.get('database.hostname'), - PORT: config.get('database.port') + PORT: config.get('database.port'), + USERNAME: config.get('database.username'), + PASSWORD: config.get('database.password') }, STORAGE: { CERT_DIR: path.join(__dirname, '..', '..', config.get('storage.certs')), @@ -87,41 +89,7 @@ const FRIEND_SCORE = { // --------------------------------------------------------------------------- -const MIGRATION_SCRIPTS = [ - { - script: '0005-create-application', - version: 5 - }, - { - script: '0010-users-password', - version: 10 - }, - { - script: '0015-admin-role', - version: 15 - }, - { - script: '0020-requests-endpoint', - version: 20 - }, - { - script: '0025-video-filenames', - version: 25 - }, - { - script: '0030-video-magnet', - version: 30 - }, - { - script: '0035-url-to-host', - version: 35 - }, - { - script: '0040-video-remote-id', - version: 40 - } -] -const LAST_SQL_SCHEMA_VERSION = (maxBy(MIGRATION_SCRIPTS, 'version'))['version'] +const LAST_MIGRATION_VERSION = 0 // --------------------------------------------------------------------------- @@ -197,8 +165,7 @@ module.exports = { CONFIG, CONSTRAINTS_FIELDS, FRIEND_SCORE, - LAST_SQL_SCHEMA_VERSION, - MIGRATION_SCRIPTS, + LAST_MIGRATION_VERSION, OAUTH_LIFETIME, PAGINATION_COUNT_DEFAULT, PODS_SCORE, diff --git a/server/initializers/database.js b/server/initializers/database.js index 9642231b9..f8f68adeb 100644 --- a/server/initializers/database.js +++ b/server/initializers/database.js @@ -10,7 +10,11 @@ const utils = require('../helpers/utils') const database = {} -const sequelize = new Sequelize(constants.CONFIG.DATABASE.DBNAME, 'peertube', 'peertube', { +const dbname = constants.CONFIG.DATABASE.DBNAME +const username = constants.CONFIG.DATABASE.USERNAME +const password = constants.CONFIG.DATABASE.PASSWORD + +const sequelize = new Sequelize(dbname, username, password, { dialect: 'postgres', host: constants.CONFIG.DATABASE.HOSTNAME, port: constants.CONFIG.DATABASE.PORT, @@ -26,33 +30,48 @@ const sequelize = new Sequelize(constants.CONFIG.DATABASE.DBNAME, 'peertube', 'p } }) -const modelDirectory = path.join(__dirname, '..', 'models') -fs.readdir(modelDirectory, function (err, files) { - if (err) throw err +database.sequelize = sequelize +database.Sequelize = Sequelize +database.init = init - files.filter(function (file) { - if (file === 'utils.js') return false +// --------------------------------------------------------------------------- - return true - }) - .forEach(function (file) { - const model = sequelize.import(path.join(modelDirectory, file)) +module.exports = database - database[model.name] = model - }) +// --------------------------------------------------------------------------- - Object.keys(database).forEach(function (modelName) { - if ('associate' in database[modelName]) { - database[modelName].associate(database) - } - }) +function init (silent, callback) { + if (!callback) { + callback = silent + silent = false + } - logger.info('Database is ready.') -}) + if (!callback) callback = function () {} -database.sequelize = sequelize -database.Sequelize = Sequelize + const modelDirectory = path.join(__dirname, '..', 'models') + fs.readdir(modelDirectory, function (err, files) { + if (err) throw err -// --------------------------------------------------------------------------- + files.filter(function (file) { + // For all models but not utils.js + if (file === 'utils.js') return false -module.exports = database + return true + }) + .forEach(function (file) { + const model = sequelize.import(path.join(modelDirectory, file)) + + database[model.name] = model + }) + + Object.keys(database).forEach(function (modelName) { + if ('associate' in database[modelName]) { + database[modelName].associate(database) + } + }) + + if (!silent) logger.info('Database is ready.') + + return callback(null) + }) +} diff --git a/server/initializers/installer.js b/server/initializers/installer.js index 4823bc8c8..d5382364e 100644 --- a/server/initializers/installer.js +++ b/server/initializers/installer.js @@ -121,9 +121,8 @@ function createOAuthAdminIfNotExist (callback) { logger.info('Username: ' + username) logger.info('User password: ' + password) - logger.info('Creating Application collection.') - const application = db.Application.build({ sqlSchemaVersion: constants.LAST_SQL_SCHEMA_VERSION }) - application.save().asCallback(callback) + logger.info('Creating Application table.') + db.Application.create({ migrationVersion: constants.LAST_MIGRATION_VERSION }).asCallback(callback) }) }) } diff --git a/server/initializers/migrations/0005-create-application.js b/server/initializers/migrations/0005-create-application.js deleted file mode 100644 index e99dec019..000000000 --- a/server/initializers/migrations/0005-create-application.js +++ /dev/null @@ -1,17 +0,0 @@ -/* - Create the application collection in MongoDB. - Used to store the actual MongoDB scheme version. -*/ - -const mongoose = require('mongoose') - -const Application = mongoose.model('Application') - -exports.up = function (callback) { - const application = new Application() - application.save(callback) -} - -exports.down = function (callback) { - throw new Error('Not implemented.') -} diff --git a/server/initializers/migrations/0005-example.js b/server/initializers/migrations/0005-example.js new file mode 100644 index 000000000..481c2c4dd --- /dev/null +++ b/server/initializers/migrations/0005-example.js @@ -0,0 +1,14 @@ +/* + This is just an example. +*/ + +const db = require('../database') + +// options contains the transaction +exports.up = function (options, callback) { + // db.Application.create({ migrationVersion: 42 }, { transaction: options.transaction }).asCallback(callback) +} + +exports.down = function (options, callback) { + throw new Error('Not implemented.') +} diff --git a/server/initializers/migrations/0010-users-password.js b/server/initializers/migrations/0010-users-password.js deleted file mode 100644 index a0616a269..000000000 --- a/server/initializers/migrations/0010-users-password.js +++ /dev/null @@ -1,22 +0,0 @@ -/* - Convert plain user password to encrypted user password. -*/ - -const eachSeries = require('async/eachSeries') -const mongoose = require('mongoose') - -const User = mongoose.model('User') - -exports.up = function (callback) { - User.list(function (err, users) { - if (err) return callback(err) - - eachSeries(users, function (user, callbackEach) { - user.save(callbackEach) - }, callback) - }) -} - -exports.down = function (callback) { - throw new Error('Not implemented.') -} diff --git a/server/initializers/migrations/0015-admin-role.js b/server/initializers/migrations/0015-admin-role.js deleted file mode 100644 index af06dca9e..000000000 --- a/server/initializers/migrations/0015-admin-role.js +++ /dev/null @@ -1,16 +0,0 @@ -/* - Set the admin role to the root user. -*/ - -const constants = require('../constants') -const mongoose = require('mongoose') - -const User = mongoose.model('User') - -exports.up = function (callback) { - User.update({ username: 'root' }, { role: constants.USER_ROLES.ADMIN }, callback) -} - -exports.down = function (callback) { - throw new Error('Not implemented.') -} diff --git a/server/initializers/migrations/0020-requests-endpoint.js b/server/initializers/migrations/0020-requests-endpoint.js deleted file mode 100644 index 55feec571..000000000 --- a/server/initializers/migrations/0020-requests-endpoint.js +++ /dev/null @@ -1,15 +0,0 @@ -/* - Set the endpoint videos for requests. -*/ - -const mongoose = require('mongoose') - -const Request = mongoose.model('Request') - -exports.up = function (callback) { - Request.update({ }, { endpoint: 'videos' }, callback) -} - -exports.down = function (callback) { - throw new Error('Not implemented.') -} diff --git a/server/initializers/migrations/0025-video-filenames.js b/server/initializers/migrations/0025-video-filenames.js deleted file mode 100644 index df21494d7..000000000 --- a/server/initializers/migrations/0025-video-filenames.js +++ /dev/null @@ -1,57 +0,0 @@ -/* - Rename thumbnails and video filenames to _id.extension -*/ - -const each = require('async/each') -const fs = require('fs') -const path = require('path') -const mongoose = require('mongoose') - -const constants = require('../constants') -const logger = require('../../helpers/logger') - -const Video = mongoose.model('Video') - -exports.up = function (callback) { - // Use of lean because the new Video scheme does not have filename field - Video.find({ filename: { $ne: null } }).lean().exec(function (err, videos) { - if (err) throw err - - each(videos, function (video, callbackEach) { - const torrentName = video.filename + '.torrent' - const thumbnailName = video.thumbnail - const thumbnailExtension = path.extname(thumbnailName) - const videoName = video.filename - const videoExtension = path.extname(videoName) - - const newTorrentName = video._id + '.torrent' - const newThumbnailName = video._id + thumbnailExtension - const newVideoName = video._id + videoExtension - - const torrentsDir = constants.CONFIG.STORAGE.TORRENTS_DIR - const thumbnailsDir = constants.CONFIG.STORAGE.THUMBNAILS_DIR - const videosDir = constants.CONFIG.STORAGE.VIDEOS_DIR - - logger.info('Renaming %s to %s.', torrentsDir + torrentName, torrentsDir + newTorrentName) - fs.renameSync(torrentsDir + torrentName, torrentsDir + newTorrentName) - - logger.info('Renaming %s to %s.', thumbnailsDir + thumbnailName, thumbnailsDir + newThumbnailName) - fs.renameSync(thumbnailsDir + thumbnailName, thumbnailsDir + newThumbnailName) - - logger.info('Renaming %s to %s.', videosDir + videoName, videosDir + newVideoName) - fs.renameSync(videosDir + videoName, videosDir + newVideoName) - - Video.load(video._id, function (err, videoObj) { - if (err) return callbackEach(err) - - videoObj.extname = videoExtension - videoObj.remoteId = null - videoObj.save(callbackEach) - }) - }, callback) - }) -} - -exports.down = function (callback) { - throw new Error('Not implemented.') -} diff --git a/server/initializers/migrations/0030-video-magnet.js b/server/initializers/migrations/0030-video-magnet.js deleted file mode 100644 index b9119d61c..000000000 --- a/server/initializers/migrations/0030-video-magnet.js +++ /dev/null @@ -1,32 +0,0 @@ -/* - Change video magnet structures -*/ - -const each = require('async/each') -const magnet = require('magnet-uri') -const mongoose = require('mongoose') - -const Video = mongoose.model('Video') - -exports.up = function (callback) { - // Use of lean because the new Video scheme does not have magnetUri field - Video.find({ }).lean().exec(function (err, videos) { - if (err) throw err - - each(videos, function (video, callbackEach) { - const parsed = magnet.decode(video.magnetUri) - const infoHash = parsed.infoHash - - Video.load(video._id, function (err, videoObj) { - if (err) return callbackEach(err) - - videoObj.magnet.infoHash = infoHash - videoObj.save(callbackEach) - }) - }, callback) - }) -} - -exports.down = function (callback) { - throw new Error('Not implemented.') -} diff --git a/server/initializers/migrations/0035-url-to-host.js b/server/initializers/migrations/0035-url-to-host.js deleted file mode 100644 index 6243304d5..000000000 --- a/server/initializers/migrations/0035-url-to-host.js +++ /dev/null @@ -1,30 +0,0 @@ -/* - Change video magnet structures -*/ - -const each = require('async/each') -const mongoose = require('mongoose') - -const Video = mongoose.model('Video') - -exports.up = function (callback) { - // Use of lean because the new Video scheme does not have podUrl field - Video.find({ }).lean().exec(function (err, videos) { - if (err) throw err - - each(videos, function (video, callbackEach) { - Video.load(video._id, function (err, videoObj) { - if (err) return callbackEach(err) - - const host = video.podUrl.split('://')[1] - - videoObj.podHost = host - videoObj.save(callbackEach) - }) - }, callback) - }) -} - -exports.down = function (callback) { - throw new Error('Not implemented.') -} diff --git a/server/initializers/migrations/0040-video-remote-id.js b/server/initializers/migrations/0040-video-remote-id.js deleted file mode 100644 index 46a14a689..000000000 --- a/server/initializers/migrations/0040-video-remote-id.js +++ /dev/null @@ -1,59 +0,0 @@ -/* - Use remote id as identifier -*/ - -const map = require('lodash/map') -const mongoose = require('mongoose') -const readline = require('readline') - -const rl = readline.createInterface({ - input: process.stdin, - output: process.stdout -}) - -const logger = require('../../helpers/logger') -const friends = require('../../lib/friends') - -const Pod = mongoose.model('Pod') -const Video = mongoose.model('Video') - -exports.up = function (callback) { - Pod.find({}).lean().exec(function (err, pods) { - if (err) return callback(err) - - // We need to quit friends first - if (pods.length === 0) { - return setVideosRemoteId(callback) - } - - const timeout = setTimeout(function () { - throw new Error('You need to enter a value!') - }, 10000) - - rl.question('I am sorry but I need to quit friends for upgrading. Do you want to continue? (yes/*)', function (answer) { - if (answer !== 'yes') throw new Error('I cannot continue.') - - clearTimeout(timeout) - rl.close() - - const urls = map(pods, 'url') - logger.info('Saying goodbye to: ' + urls.join(', ')) - - setVideosRemoteId(function () { - friends.quitFriends(callback) - }) - }) - }) -} - -exports.down = function (callback) { - throw new Error('Not implemented.') -} - -function setVideosRemoteId (callback) { - Video.update({ filename: { $ne: null } }, { remoteId: null }, function (err) { - if (err) throw err - - Video.update({ filename: null }, { remoteId: mongoose.Types.ObjectId() }, callback) - }) -} diff --git a/server/initializers/migrator.js b/server/initializers/migrator.js index 9e5350e60..eaecb4936 100644 --- a/server/initializers/migrator.js +++ b/server/initializers/migrator.js @@ -1,6 +1,7 @@ 'use strict' const eachSeries = require('async/eachSeries') +const fs = require('fs') const path = require('path') const constants = require('./constants') @@ -12,35 +13,24 @@ const migrator = { } function migrate (callback) { - db.Application.loadSqlSchemaVersion(function (err, actualVersion) { + db.Application.loadMigrationVersion(function (err, actualVersion) { if (err) return callback(err) - // If there are a new mongo schemas - if (!actualVersion || actualVersion < constants.LAST_SQL_SCHEMA_VERSION) { + // If there are a new migration scripts + if (actualVersion < constants.LAST_MIGRATION_VERSION) { logger.info('Begin migrations.') - eachSeries(constants.MONGO_MIGRATION_SCRIPTS, function (entity, callbackEach) { - const versionScript = entity.version - - // Do not execute old migration scripts - if (versionScript <= actualVersion) return callbackEach(null) - - // Load the migration module and run it - const migrationScriptName = entity.script - logger.info('Executing %s migration script.', migrationScriptName) + getMigrationScripts(function (err, migrationScripts) { + if (err) return callback(err) - const migrationScript = require(path.join(__dirname, 'migrations', migrationScriptName)) - migrationScript.up(function (err) { - if (err) return callbackEach(err) + eachSeries(migrationScripts, function (entity, callbackEach) { + executeMigration(actualVersion, entity, callbackEach) + }, function (err) { + if (err) return callback(err) - // Update the new mongo version schema - db.Application.updateSqlSchemaVersion(versionScript, callbackEach) + logger.info('Migrations finished. New migration version schema: %s', constants.LAST_MIGRATION_VERSION) + return callback(null) }) - }, function (err) { - if (err) return callback(err) - - logger.info('Migrations finished. New SQL version schema: %s', constants.LAST_SQL_SCHEMA_VERSION) - return callback(null) }) } else { return callback(null) @@ -52,3 +42,57 @@ function migrate (callback) { module.exports = migrator +// --------------------------------------------------------------------------- + +function getMigrationScripts (callback) { + fs.readdir(path.join(__dirname, 'migrations'), function (err, files) { + if (err) return callback(err) + + const filesToMigrate = [] + + files.forEach(function (file) { + // Filename is something like 'version-blabla.js' + const version = file.split('-')[0] + filesToMigrate.push({ + version, + script: file + }) + }) + + return callback(err, filesToMigrate) + }) +} + +function executeMigration (actualVersion, entity, callback) { + const versionScript = entity.version + + // Do not execute old migration scripts + if (versionScript <= actualVersion) return callback(null) + + // Load the migration module and run it + const migrationScriptName = entity.script + logger.info('Executing %s migration script.', migrationScriptName) + + const migrationScript = require(path.join(__dirname, 'migrations', migrationScriptName)) + + db.sequelize.transaction().asCallback(function (err, t) { + if (err) return callback(err) + + migrationScript.up({ transaction: t }, function (err) { + if (err) { + t.rollback() + return callback(err) + } + + // Update the new migration version + db.Application.updateMigrationVersion(versionScript, t, function (err) { + if (err) { + t.rollback() + return callback(err) + } + + t.commit() + }) + }) + }) +} -- cgit v1.2.3