From b769007f733769d3afe2d29a3eb23e2e7693f301 Mon Sep 17 00:00:00 2001 From: Chocobozzz Date: Sun, 25 Dec 2016 09:44:57 +0100 Subject: [PATCH] Update migrations code --- config/default.yaml | 2 + server.js | 1 + server/initializers/checker.js | 2 +- server/initializers/constants.js | 43 ++------- server/initializers/database.js | 65 +++++++++----- server/initializers/installer.js | 5 +- .../migrations/0005-create-application.js | 17 ---- .../initializers/migrations/0005-example.js | 14 +++ .../migrations/0010-users-password.js | 22 ----- .../migrations/0015-admin-role.js | 16 ---- .../migrations/0020-requests-endpoint.js | 15 ---- .../migrations/0025-video-filenames.js | 57 ------------ .../migrations/0030-video-magnet.js | 32 ------- .../migrations/0035-url-to-host.js | 30 ------- .../migrations/0040-video-remote-id.js | 59 ------------- server/initializers/migrator.js | 88 ++++++++++++++----- server/models/application.js | 26 ++++-- server/models/video.js | 7 +- server/tests/utils/servers.js | 2 +- 19 files changed, 158 insertions(+), 345 deletions(-) delete mode 100644 server/initializers/migrations/0005-create-application.js create mode 100644 server/initializers/migrations/0005-example.js delete mode 100644 server/initializers/migrations/0010-users-password.js delete mode 100644 server/initializers/migrations/0015-admin-role.js delete mode 100644 server/initializers/migrations/0020-requests-endpoint.js delete mode 100644 server/initializers/migrations/0025-video-filenames.js delete mode 100644 server/initializers/migrations/0030-video-magnet.js delete mode 100644 server/initializers/migrations/0035-url-to-host.js delete mode 100644 server/initializers/migrations/0040-video-remote-id.js diff --git a/config/default.yaml b/config/default.yaml index 631400f7d..2dd5e05f9 100644 --- a/config/default.yaml +++ b/config/default.yaml @@ -10,6 +10,8 @@ database: hostname: 'localhost' port: 5432 suffix: '_dev' + username: peertube + password: peertube # From the project root directory storage: diff --git a/server.js b/server.js index e54ffe69f..f4ca53907 100644 --- a/server.js +++ b/server.js @@ -20,6 +20,7 @@ const constants = require('./server/initializers/constants') const logger = require('./server/helpers/logger') // Initialize database and models const db = require('./server/initializers/database') +db.init() // ----------- Checker ----------- const checker = require('./server/initializers/checker') diff --git a/server/initializers/checker.js b/server/initializers/checker.js index 7b402de82..2753604dc 100644 --- a/server/initializers/checker.js +++ b/server/initializers/checker.js @@ -27,7 +27,7 @@ function checkConfig () { function checkMissedConfig () { const required = [ 'listen.port', 'webserver.https', 'webserver.hostname', 'webserver.port', - 'database.hostname', 'database.port', 'database.suffix', + 'database.hostname', 'database.port', 'database.suffix', 'database.username', 'database.password', 'storage.certs', 'storage.videos', 'storage.logs', 'storage.thumbnails', 'storage.previews' ] const miss = [] diff --git a/server/initializers/constants.js b/server/initializers/constants.js index 1ad0c82a0..6f39b65da 100644 --- a/server/initializers/constants.js +++ b/server/initializers/constants.js @@ -37,7 +37,9 @@ const CONFIG = { DATABASE: { DBNAME: 'peertube' + config.get('database.suffix'), HOSTNAME: config.get('database.hostname'), - PORT: config.get('database.port') + PORT: config.get('database.port'), + USERNAME: config.get('database.username'), + PASSWORD: config.get('database.password') }, STORAGE: { CERT_DIR: path.join(__dirname, '..', '..', config.get('storage.certs')), @@ -87,41 +89,7 @@ const FRIEND_SCORE = { // --------------------------------------------------------------------------- -const MIGRATION_SCRIPTS = [ - { - script: '0005-create-application', - version: 5 - }, - { - script: '0010-users-password', - version: 10 - }, - { - script: '0015-admin-role', - version: 15 - }, - { - script: '0020-requests-endpoint', - version: 20 - }, - { - script: '0025-video-filenames', - version: 25 - }, - { - script: '0030-video-magnet', - version: 30 - }, - { - script: '0035-url-to-host', - version: 35 - }, - { - script: '0040-video-remote-id', - version: 40 - } -] -const LAST_SQL_SCHEMA_VERSION = (maxBy(MIGRATION_SCRIPTS, 'version'))['version'] +const LAST_MIGRATION_VERSION = 0 // --------------------------------------------------------------------------- @@ -197,8 +165,7 @@ module.exports = { CONFIG, CONSTRAINTS_FIELDS, FRIEND_SCORE, - LAST_SQL_SCHEMA_VERSION, - MIGRATION_SCRIPTS, + LAST_MIGRATION_VERSION, OAUTH_LIFETIME, PAGINATION_COUNT_DEFAULT, PODS_SCORE, diff --git a/server/initializers/database.js b/server/initializers/database.js index 9642231b9..f8f68adeb 100644 --- a/server/initializers/database.js +++ b/server/initializers/database.js @@ -10,7 +10,11 @@ const utils = require('../helpers/utils') const database = {} -const sequelize = new Sequelize(constants.CONFIG.DATABASE.DBNAME, 'peertube', 'peertube', { +const dbname = constants.CONFIG.DATABASE.DBNAME +const username = constants.CONFIG.DATABASE.USERNAME +const password = constants.CONFIG.DATABASE.PASSWORD + +const sequelize = new Sequelize(dbname, username, password, { dialect: 'postgres', host: constants.CONFIG.DATABASE.HOSTNAME, port: constants.CONFIG.DATABASE.PORT, @@ -26,33 +30,48 @@ const sequelize = new Sequelize(constants.CONFIG.DATABASE.DBNAME, 'peertube', 'p } }) -const modelDirectory = path.join(__dirname, '..', 'models') -fs.readdir(modelDirectory, function (err, files) { - if (err) throw err +database.sequelize = sequelize +database.Sequelize = Sequelize +database.init = init - files.filter(function (file) { - if (file === 'utils.js') return false +// --------------------------------------------------------------------------- - return true - }) - .forEach(function (file) { - const model = sequelize.import(path.join(modelDirectory, file)) +module.exports = database - database[model.name] = model - }) +// --------------------------------------------------------------------------- - Object.keys(database).forEach(function (modelName) { - if ('associate' in database[modelName]) { - database[modelName].associate(database) - } - }) +function init (silent, callback) { + if (!callback) { + callback = silent + silent = false + } - logger.info('Database is ready.') -}) + if (!callback) callback = function () {} -database.sequelize = sequelize -database.Sequelize = Sequelize + const modelDirectory = path.join(__dirname, '..', 'models') + fs.readdir(modelDirectory, function (err, files) { + if (err) throw err -// --------------------------------------------------------------------------- + files.filter(function (file) { + // For all models but not utils.js + if (file === 'utils.js') return false -module.exports = database + return true + }) + .forEach(function (file) { + const model = sequelize.import(path.join(modelDirectory, file)) + + database[model.name] = model + }) + + Object.keys(database).forEach(function (modelName) { + if ('associate' in database[modelName]) { + database[modelName].associate(database) + } + }) + + if (!silent) logger.info('Database is ready.') + + return callback(null) + }) +} diff --git a/server/initializers/installer.js b/server/initializers/installer.js index 4823bc8c8..d5382364e 100644 --- a/server/initializers/installer.js +++ b/server/initializers/installer.js @@ -121,9 +121,8 @@ function createOAuthAdminIfNotExist (callback) { logger.info('Username: ' + username) logger.info('User password: ' + password) - logger.info('Creating Application collection.') - const application = db.Application.build({ sqlSchemaVersion: constants.LAST_SQL_SCHEMA_VERSION }) - application.save().asCallback(callback) + logger.info('Creating Application table.') + db.Application.create({ migrationVersion: constants.LAST_MIGRATION_VERSION }).asCallback(callback) }) }) } diff --git a/server/initializers/migrations/0005-create-application.js b/server/initializers/migrations/0005-create-application.js deleted file mode 100644 index e99dec019..000000000 --- a/server/initializers/migrations/0005-create-application.js +++ /dev/null @@ -1,17 +0,0 @@ -/* - Create the application collection in MongoDB. - Used to store the actual MongoDB scheme version. -*/ - -const mongoose = require('mongoose') - -const Application = mongoose.model('Application') - -exports.up = function (callback) { - const application = new Application() - application.save(callback) -} - -exports.down = function (callback) { - throw new Error('Not implemented.') -} diff --git a/server/initializers/migrations/0005-example.js b/server/initializers/migrations/0005-example.js new file mode 100644 index 000000000..481c2c4dd --- /dev/null +++ b/server/initializers/migrations/0005-example.js @@ -0,0 +1,14 @@ +/* + This is just an example. +*/ + +const db = require('../database') + +// options contains the transaction +exports.up = function (options, callback) { + // db.Application.create({ migrationVersion: 42 }, { transaction: options.transaction }).asCallback(callback) +} + +exports.down = function (options, callback) { + throw new Error('Not implemented.') +} diff --git a/server/initializers/migrations/0010-users-password.js b/server/initializers/migrations/0010-users-password.js deleted file mode 100644 index a0616a269..000000000 --- a/server/initializers/migrations/0010-users-password.js +++ /dev/null @@ -1,22 +0,0 @@ -/* - Convert plain user password to encrypted user password. -*/ - -const eachSeries = require('async/eachSeries') -const mongoose = require('mongoose') - -const User = mongoose.model('User') - -exports.up = function (callback) { - User.list(function (err, users) { - if (err) return callback(err) - - eachSeries(users, function (user, callbackEach) { - user.save(callbackEach) - }, callback) - }) -} - -exports.down = function (callback) { - throw new Error('Not implemented.') -} diff --git a/server/initializers/migrations/0015-admin-role.js b/server/initializers/migrations/0015-admin-role.js deleted file mode 100644 index af06dca9e..000000000 --- a/server/initializers/migrations/0015-admin-role.js +++ /dev/null @@ -1,16 +0,0 @@ -/* - Set the admin role to the root user. -*/ - -const constants = require('../constants') -const mongoose = require('mongoose') - -const User = mongoose.model('User') - -exports.up = function (callback) { - User.update({ username: 'root' }, { role: constants.USER_ROLES.ADMIN }, callback) -} - -exports.down = function (callback) { - throw new Error('Not implemented.') -} diff --git a/server/initializers/migrations/0020-requests-endpoint.js b/server/initializers/migrations/0020-requests-endpoint.js deleted file mode 100644 index 55feec571..000000000 --- a/server/initializers/migrations/0020-requests-endpoint.js +++ /dev/null @@ -1,15 +0,0 @@ -/* - Set the endpoint videos for requests. -*/ - -const mongoose = require('mongoose') - -const Request = mongoose.model('Request') - -exports.up = function (callback) { - Request.update({ }, { endpoint: 'videos' }, callback) -} - -exports.down = function (callback) { - throw new Error('Not implemented.') -} diff --git a/server/initializers/migrations/0025-video-filenames.js b/server/initializers/migrations/0025-video-filenames.js deleted file mode 100644 index df21494d7..000000000 --- a/server/initializers/migrations/0025-video-filenames.js +++ /dev/null @@ -1,57 +0,0 @@ -/* - Rename thumbnails and video filenames to _id.extension -*/ - -const each = require('async/each') -const fs = require('fs') -const path = require('path') -const mongoose = require('mongoose') - -const constants = require('../constants') -const logger = require('../../helpers/logger') - -const Video = mongoose.model('Video') - -exports.up = function (callback) { - // Use of lean because the new Video scheme does not have filename field - Video.find({ filename: { $ne: null } }).lean().exec(function (err, videos) { - if (err) throw err - - each(videos, function (video, callbackEach) { - const torrentName = video.filename + '.torrent' - const thumbnailName = video.thumbnail - const thumbnailExtension = path.extname(thumbnailName) - const videoName = video.filename - const videoExtension = path.extname(videoName) - - const newTorrentName = video._id + '.torrent' - const newThumbnailName = video._id + thumbnailExtension - const newVideoName = video._id + videoExtension - - const torrentsDir = constants.CONFIG.STORAGE.TORRENTS_DIR - const thumbnailsDir = constants.CONFIG.STORAGE.THUMBNAILS_DIR - const videosDir = constants.CONFIG.STORAGE.VIDEOS_DIR - - logger.info('Renaming %s to %s.', torrentsDir + torrentName, torrentsDir + newTorrentName) - fs.renameSync(torrentsDir + torrentName, torrentsDir + newTorrentName) - - logger.info('Renaming %s to %s.', thumbnailsDir + thumbnailName, thumbnailsDir + newThumbnailName) - fs.renameSync(thumbnailsDir + thumbnailName, thumbnailsDir + newThumbnailName) - - logger.info('Renaming %s to %s.', videosDir + videoName, videosDir + newVideoName) - fs.renameSync(videosDir + videoName, videosDir + newVideoName) - - Video.load(video._id, function (err, videoObj) { - if (err) return callbackEach(err) - - videoObj.extname = videoExtension - videoObj.remoteId = null - videoObj.save(callbackEach) - }) - }, callback) - }) -} - -exports.down = function (callback) { - throw new Error('Not implemented.') -} diff --git a/server/initializers/migrations/0030-video-magnet.js b/server/initializers/migrations/0030-video-magnet.js deleted file mode 100644 index b9119d61c..000000000 --- a/server/initializers/migrations/0030-video-magnet.js +++ /dev/null @@ -1,32 +0,0 @@ -/* - Change video magnet structures -*/ - -const each = require('async/each') -const magnet = require('magnet-uri') -const mongoose = require('mongoose') - -const Video = mongoose.model('Video') - -exports.up = function (callback) { - // Use of lean because the new Video scheme does not have magnetUri field - Video.find({ }).lean().exec(function (err, videos) { - if (err) throw err - - each(videos, function (video, callbackEach) { - const parsed = magnet.decode(video.magnetUri) - const infoHash = parsed.infoHash - - Video.load(video._id, function (err, videoObj) { - if (err) return callbackEach(err) - - videoObj.magnet.infoHash = infoHash - videoObj.save(callbackEach) - }) - }, callback) - }) -} - -exports.down = function (callback) { - throw new Error('Not implemented.') -} diff --git a/server/initializers/migrations/0035-url-to-host.js b/server/initializers/migrations/0035-url-to-host.js deleted file mode 100644 index 6243304d5..000000000 --- a/server/initializers/migrations/0035-url-to-host.js +++ /dev/null @@ -1,30 +0,0 @@ -/* - Change video magnet structures -*/ - -const each = require('async/each') -const mongoose = require('mongoose') - -const Video = mongoose.model('Video') - -exports.up = function (callback) { - // Use of lean because the new Video scheme does not have podUrl field - Video.find({ }).lean().exec(function (err, videos) { - if (err) throw err - - each(videos, function (video, callbackEach) { - Video.load(video._id, function (err, videoObj) { - if (err) return callbackEach(err) - - const host = video.podUrl.split('://')[1] - - videoObj.podHost = host - videoObj.save(callbackEach) - }) - }, callback) - }) -} - -exports.down = function (callback) { - throw new Error('Not implemented.') -} diff --git a/server/initializers/migrations/0040-video-remote-id.js b/server/initializers/migrations/0040-video-remote-id.js deleted file mode 100644 index 46a14a689..000000000 --- a/server/initializers/migrations/0040-video-remote-id.js +++ /dev/null @@ -1,59 +0,0 @@ -/* - Use remote id as identifier -*/ - -const map = require('lodash/map') -const mongoose = require('mongoose') -const readline = require('readline') - -const rl = readline.createInterface({ - input: process.stdin, - output: process.stdout -}) - -const logger = require('../../helpers/logger') -const friends = require('../../lib/friends') - -const Pod = mongoose.model('Pod') -const Video = mongoose.model('Video') - -exports.up = function (callback) { - Pod.find({}).lean().exec(function (err, pods) { - if (err) return callback(err) - - // We need to quit friends first - if (pods.length === 0) { - return setVideosRemoteId(callback) - } - - const timeout = setTimeout(function () { - throw new Error('You need to enter a value!') - }, 10000) - - rl.question('I am sorry but I need to quit friends for upgrading. Do you want to continue? (yes/*)', function (answer) { - if (answer !== 'yes') throw new Error('I cannot continue.') - - clearTimeout(timeout) - rl.close() - - const urls = map(pods, 'url') - logger.info('Saying goodbye to: ' + urls.join(', ')) - - setVideosRemoteId(function () { - friends.quitFriends(callback) - }) - }) - }) -} - -exports.down = function (callback) { - throw new Error('Not implemented.') -} - -function setVideosRemoteId (callback) { - Video.update({ filename: { $ne: null } }, { remoteId: null }, function (err) { - if (err) throw err - - Video.update({ filename: null }, { remoteId: mongoose.Types.ObjectId() }, callback) - }) -} diff --git a/server/initializers/migrator.js b/server/initializers/migrator.js index 9e5350e60..eaecb4936 100644 --- a/server/initializers/migrator.js +++ b/server/initializers/migrator.js @@ -1,6 +1,7 @@ 'use strict' const eachSeries = require('async/eachSeries') +const fs = require('fs') const path = require('path') const constants = require('./constants') @@ -12,35 +13,24 @@ const migrator = { } function migrate (callback) { - db.Application.loadSqlSchemaVersion(function (err, actualVersion) { + db.Application.loadMigrationVersion(function (err, actualVersion) { if (err) return callback(err) - // If there are a new mongo schemas - if (!actualVersion || actualVersion < constants.LAST_SQL_SCHEMA_VERSION) { + // If there are a new migration scripts + if (actualVersion < constants.LAST_MIGRATION_VERSION) { logger.info('Begin migrations.') - eachSeries(constants.MONGO_MIGRATION_SCRIPTS, function (entity, callbackEach) { - const versionScript = entity.version - - // Do not execute old migration scripts - if (versionScript <= actualVersion) return callbackEach(null) - - // Load the migration module and run it - const migrationScriptName = entity.script - logger.info('Executing %s migration script.', migrationScriptName) + getMigrationScripts(function (err, migrationScripts) { + if (err) return callback(err) - const migrationScript = require(path.join(__dirname, 'migrations', migrationScriptName)) - migrationScript.up(function (err) { - if (err) return callbackEach(err) + eachSeries(migrationScripts, function (entity, callbackEach) { + executeMigration(actualVersion, entity, callbackEach) + }, function (err) { + if (err) return callback(err) - // Update the new mongo version schema - db.Application.updateSqlSchemaVersion(versionScript, callbackEach) + logger.info('Migrations finished. New migration version schema: %s', constants.LAST_MIGRATION_VERSION) + return callback(null) }) - }, function (err) { - if (err) return callback(err) - - logger.info('Migrations finished. New SQL version schema: %s', constants.LAST_SQL_SCHEMA_VERSION) - return callback(null) }) } else { return callback(null) @@ -52,3 +42,57 @@ function migrate (callback) { module.exports = migrator +// --------------------------------------------------------------------------- + +function getMigrationScripts (callback) { + fs.readdir(path.join(__dirname, 'migrations'), function (err, files) { + if (err) return callback(err) + + const filesToMigrate = [] + + files.forEach(function (file) { + // Filename is something like 'version-blabla.js' + const version = file.split('-')[0] + filesToMigrate.push({ + version, + script: file + }) + }) + + return callback(err, filesToMigrate) + }) +} + +function executeMigration (actualVersion, entity, callback) { + const versionScript = entity.version + + // Do not execute old migration scripts + if (versionScript <= actualVersion) return callback(null) + + // Load the migration module and run it + const migrationScriptName = entity.script + logger.info('Executing %s migration script.', migrationScriptName) + + const migrationScript = require(path.join(__dirname, 'migrations', migrationScriptName)) + + db.sequelize.transaction().asCallback(function (err, t) { + if (err) return callback(err) + + migrationScript.up({ transaction: t }, function (err) { + if (err) { + t.rollback() + return callback(err) + } + + // Update the new migration version + db.Application.updateMigrationVersion(versionScript, t, function (err) { + if (err) { + t.rollback() + return callback(err) + } + + t.commit() + }) + }) + }) +} diff --git a/server/models/application.js b/server/models/application.js index ec1d7b122..4114ed76d 100644 --- a/server/models/application.js +++ b/server/models/application.js @@ -1,15 +1,15 @@ module.exports = function (sequelize, DataTypes) { const Application = sequelize.define('Application', { - sqlSchemaVersion: { + migrationVersion: { type: DataTypes.INTEGER, defaultValue: 0 } }, { classMethods: { - loadSqlSchemaVersion, - updateSqlSchemaVersion + loadMigrationVersion, + updateMigrationVersion } } ) @@ -19,18 +19,28 @@ module.exports = function (sequelize, DataTypes) { // --------------------------------------------------------------------------- -function loadSqlSchemaVersion (callback) { +function loadMigrationVersion (callback) { const query = { - attributes: [ 'sqlSchemaVersion' ] + attributes: [ 'migrationVersion' ] } return this.findOne(query).asCallback(function (err, data) { - const version = data ? data.sqlSchemaVersion : 0 + const version = data ? data.migrationVersion : 0 return callback(err, version) }) } -function updateSqlSchemaVersion (newVersion, callback) { - return this.update({ sqlSchemaVersion: newVersion }).asCallback(callback) +function updateMigrationVersion (newVersion, transaction, callback) { + const options = { + where: {} + } + + if (!callback) { + transaction = callback + } else { + options.transaction = transaction + } + + return this.update({ migrationVersion: newVersion }, options).asCallback(callback) } diff --git a/server/models/video.js b/server/models/video.js index 0023a24e1..af05a861f 100644 --- a/server/models/video.js +++ b/server/models/video.js @@ -16,7 +16,7 @@ const modelUtils = require('./utils') // --------------------------------------------------------------------------- module.exports = function (sequelize, DataTypes) { -// TODO: add indexes on searchable columns + // TODO: add indexes on searchable columns const Video = sequelize.define('Video', { id: { @@ -50,6 +50,7 @@ module.exports = function (sequelize, DataTypes) { generateThumbnailFromBase64, getDurationFromFile, + list, listForApi, listByHostAndRemoteId, listOwnedAndPopulateAuthorAndTags, @@ -310,6 +311,10 @@ function getDurationFromFile (videoPath, callback) { }) } +function list (callback) { + return this.find().asCallback() +} + function listForApi (start, count, sort, callback) { const query = { offset: start, diff --git a/server/tests/utils/servers.js b/server/tests/utils/servers.js index 4e55f8f5c..e7c756499 100644 --- a/server/tests/utils/servers.js +++ b/server/tests/utils/servers.js @@ -103,7 +103,7 @@ function runServer (number, callback) { if (serverRunString[key] === false) dontContinue = true } - // If no, there is maybe one thing not already initialized (mongodb...) + // If no, there is maybe one thing not already initialized (client/user credentials generation...) if (dontContinue === true) return server.app.stdout.removeListener('data', onStdout) -- 2.41.0