scripts/mongo-to-postgre.js

   1 #!/usr/bin/env node
   2
   3 'use strict'
   4
   5 // TODO: document this script
   6
   7 const program = require('commander')
   8 const eachSeries = require('async/eachSeries')
   9 const series = require('async/series')
  10 const waterfall = require('async/waterfall')
  11 const fs = require('fs')
  12 const path = require('path')
  13 const MongoClient = require('mongodb').MongoClient
  14
  15 const constants = require('../server/initializers/constants')
  16
  17 program
  18   .option('-mh, --mongo-host [host]', 'MongoDB host', 'localhost')
  19   .option('-mp, --mongo-port [weight]', 'MongoDB port', '27017')
  20   .option('-md, --mongo-database [dbname]', 'MongoDB database')
  21   .parse(process.argv)
  22
  23 if (!program.mongoDatabase) {
  24   console.error('The mongodb database is mandatory.')
  25   process.exit(-1)
  26 }
  27
  28 const mongoUrl = 'mongodb://' + program.mongoHost + ':' + program.mongoPort + '/' + program.mongoDatabase
  29 const dbSequelize = require('../server/initializers/database')
  30
  31 console.log('Connecting to ' + mongoUrl)
  32 MongoClient.connect(mongoUrl, function (err, dbMongo) {
  33   if (err) throw err
  34
  35   console.log('Connected to ' + mongoUrl)
  36
  37   const videoMongo = dbMongo.collection('videos')
  38   const userMongo = dbMongo.collection('users')
  39   const podMongo = dbMongo.collection('pods')
  40
  41   podMongo.count(function (err, podsLength) {
  42     if (err) throw err
  43
  44     if (podsLength > 0) {
  45       console.error('You need to quit friends first.')
  46       process.exit(-1)
  47     }
  48
  49     console.log('Connecting to ' + dbSequelize.sequelize.config.database)
  50     dbSequelize.init(true, function (err) {
  51       if (err) throw err
  52
  53       console.log('Connected to SQL database %s.', dbSequelize.sequelize.config.database)
  54
  55       series([
  56         function (next) {
  57           dbSequelize.sequelize.sync({ force: true }).asCallback(next)
  58         },
  59
  60         function (next) {
  61           migrateVideos(videoMongo, dbSequelize, next)
  62         },
  63
  64         function (next) {
  65           migrateUsers(userMongo, dbSequelize, next)
  66         }
  67       ], function (err) {
  68         if (err) console.error(err)
  69
  70         process.exit(0)
  71       })
  72     })
  73   })
  74 })
  75
  76 // ---------------------------------------------------------------------------
  77
  78 function migrateUsers (userMongo, dbSequelize, callback) {
  79   userMongo.find().toArray(function (err, mongoUsers) {
  80     if (err) return callback(err)
  81
  82     eachSeries(mongoUsers, function (mongoUser, callbackEach) {
  83       console.log('Migrating user %s', mongoUser.username)
  84
  85       const userData = {
  86         username: mongoUser.username,
  87         password: mongoUser.password,
  88         role: mongoUser.role
  89       }
  90       const options = {
  91         hooks: false
  92       }
  93
  94       dbSequelize.User.create(userData, options).asCallback(callbackEach)
  95     }, callback)
  96   })
  97 }
  98
  99 function migrateVideos (videoMongo, dbSequelize, finalCallback) {
 100   videoMongo.find().toArray(function (err, mongoVideos) {
 101     if (err) return finalCallback(err)
 102
 103     eachSeries(mongoVideos, function (mongoVideo, callbackEach) {
 104       console.log('Migrating video %s.', mongoVideo.name)
 105
 106       waterfall([
 107
 108         function startTransaction (callback) {
 109           dbSequelize.sequelize.transaction().asCallback(function (err, t) {
 110             return callback(err, t)
 111           })
 112         },
 113
 114         function findOrCreatePod (t, callback) {
 115           if (mongoVideo.remoteId === null) return callback(null, t, null)
 116
 117           const query = {
 118             where: {
 119               host: mongoVideo.podHost
 120             },
 121             defaults: {
 122               host: mongoVideo.podHost
 123             },
 124             transaction: t
 125           }
 126
 127           dbSequelize.Pod.findOrCreate(query).asCallback(function (err, result) {
 128             // [ instance, wasCreated ]
 129             const res = result ? result[0] : null
 130             return callback(err, t, res)
 131           })
 132         },
 133
 134         function findOrCreateAuthor (t, pod, callback) {
 135           const podId = pod ? pod.id : null
 136           const username = mongoVideo.author
 137
 138           const query = {
 139             where: {
 140               podId,
 141               name: username
 142             },
 143             defaults: {
 144               podId,
 145               name: username
 146             },
 147             transaction: t
 148           }
 149
 150           dbSequelize.Author.findOrCreate(query).asCallback(function (err, result) {
 151             // [ instance, wasCreated ]
 152             const res = result ? result[0] : null
 153             return callback(err, t, res)
 154           })
 155         },
 156
 157         function findOrCreateTags (t, author, callback) {
 158           const tags = mongoVideo.tags
 159           const tagInstances = []
 160
 161           eachSeries(tags, function (tag, callbackEach) {
 162             const query = {
 163               where: {
 164                 name: tag
 165               },
 166               defaults: {
 167                 name: tag
 168               },
 169               transaction: t
 170             }
 171
 172             dbSequelize.Tag.findOrCreate(query).asCallback(function (err, res) {
 173               if (err) return callbackEach(err)
 174
 175               // res = [ tag, isCreated ]
 176               const tag = res[0]
 177               tagInstances.push(tag)
 178               return callbackEach()
 179             })
 180           }, function (err) {
 181             return callback(err, t, author, tagInstances)
 182           })
 183         },
 184
 185         function createVideoObject (t, author, tagInstances, callback) {
 186           const videoData = {
 187             name: mongoVideo.name,
 188             remoteId: mongoVideo.remoteId,
 189             extname: mongoVideo.extname,
 190             infoHash: mongoVideo.magnet.infoHash,
 191             description: mongoVideo.description,
 192             authorId: author.id,
 193             duration: mongoVideo.duration,
 194             createdAt: mongoVideo.createdDate
 195           }
 196
 197           const video = dbSequelize.Video.build(videoData)
 198
 199           return callback(null, t, tagInstances, video)
 200         },
 201
 202         function moveVideoFile (t, tagInstances, video, callback) {
 203           const basePath = constants.CONFIG.STORAGE.VIDEOS_DIR
 204           const src = path.join(basePath, mongoVideo._id.toString()) + video.extname
 205           const dst = path.join(basePath, video.id) + video.extname
 206           fs.rename(src, dst, function (err) {
 207             if (err) return callback(err)
 208
 209             return callback(null, t, tagInstances, video)
 210           })
 211         },
 212
 213         function insertVideoIntoDB (t, tagInstances, video, callback) {
 214           const options = {
 215             transaction: t
 216           }
 217
 218           video.save(options).asCallback(function (err, videoCreated) {
 219             return callback(err, t, tagInstances, videoCreated)
 220           })
 221         },
 222
 223         function associateTagsToVideo (t, tagInstances, video, callback) {
 224           const options = { transaction: t }
 225
 226           video.setTags(tagInstances, options).asCallback(function (err) {
 227             return callback(err, t)
 228           })
 229         }
 230
 231       ], function (err, t) {
 232         if (err) {
 233           // Abort transaction?
 234           if (t) t.rollback()
 235
 236           return callbackEach(err)
 237         }
 238
 239         // Commit transaction
 240         return t.commit().asCallback(callbackEach)
 241       })
 242     }, finalCallback)
 243   })
 244 }