Merge branch 'postgresql'

7 files changed, 278 insertions, 35 deletions

diff --git a/scripts/clean/server/test.sh b/scripts/clean/server/test.sh
index 927671dd4..35d3ad50f 100755
--- a/scripts/clean/server/test.sh
+++ b/scripts/clean/server/test.sh
@@ -1,6 +1,7 @@
 #!/usr/bin/env sh
 
 for i in $(seq 1 6); do
-  printf "use peertube-test%s;\ndb.dropDatabase();" "$i" | mongo
+  dropdb "peertube_test$i"
   rm -rf "./test$i"
+  createdb "peertube_test$i"
 done
diff --git a/scripts/danger/clean/cleaner.js b/scripts/danger/clean/cleaner.js
index 10e91e700..1a1e3dee7 100644
--- a/scripts/danger/clean/cleaner.js
+++ b/scripts/danger/clean/cleaner.js
@@ -1,24 +1,25 @@
+const eachSeries = require('async/eachSeries')
 const rimraf = require('rimraf')
-const mongoose = require('mongoose')
-mongoose.Promise = global.Promise
 
 const constants = require('../../../server/initializers/constants')
+const db = require('../../../server/initializers/database')
 
-const mongodbUrl = 'mongodb://' + constants.CONFIG.DATABASE.HOSTNAME + ':' + constants.CONFIG.DATABASE.PORT + '/' + constants.CONFIG.DATABASE.DBNAME
-mongoose.connect(mongodbUrl, function () {
-  console.info('Deleting MongoDB %s database.', constants.CONFIG.DATABASE.DBNAME)
-  mongoose.connection.dropDatabase(function () {
-    mongoose.connection.close()
-  })
-})
+db.init(true, function () {
+  db.sequelize.drop().asCallback(function (err) {
+    if (err) throw err
 
-const STORAGE = constants.CONFIG.STORAGE
-Object.keys(STORAGE).forEach(function (storage) {
-  const storageDir = STORAGE[storage]
+    console.info('Tables of %s deleted.', db.sequelize.config.database)
 
-  rimraf(storageDir, function (err) {
-    if (err) throw err
+    const STORAGE = constants.CONFIG.STORAGE
+    eachSeries(Object.keys(STORAGE), function (storage, callbackEach) {
+      const storageDir = STORAGE[storage]
 
-    console.info('Deleting %s.', storageDir)
+      rimraf(storageDir, function (err) {
+        console.info('%s deleted.', storageDir)
+        return callbackEach(err)
+      })
+    }, function () {
+      process.exit(0)
+    })
   })
 })
diff --git a/scripts/danger/clean/dev.sh b/scripts/danger/clean/dev.sh
index 5f09565cf..f4a5d1e8a 100755
--- a/scripts/danger/clean/dev.sh
+++ b/scripts/danger/clean/dev.sh
@@ -1,6 +1,7 @@
 #!/bin/bash
 
-read -p "This will remove all directories and Mongo database. Are you sure? " -n 1 -r
+read -p "This will remove all directories and SQL tables. Are you sure? (y/*) " -n 1 -r
+echo
 
 if [[ "$REPLY" =~ ^[Yy]$ ]]; then
   NODE_ENV=test node "./scripts/danger/clean/cleaner"
diff --git a/scripts/danger/clean/prod.sh b/scripts/danger/clean/prod.sh
index e6c92671d..7ee13ba83 100755
--- a/scripts/danger/clean/prod.sh
+++ b/scripts/danger/clean/prod.sh
@@ -1,6 +1,7 @@
 #!/bin/bash
 
-read -p "This will remove all directories and Mongo database. Are you sure? " -n 1 -r
+read -p "This will remove all directories and SQL tables. Are you sure? (y/*) " -n 1 -r
+echo
 
 if [[ "$REPLY" =~ ^[Yy]$ ]]; then
   NODE_ENV=production node "./scripts/danger/clean/cleaner"
diff --git a/scripts/mongo-to-postgre.js b/scripts/mongo-to-postgre.js
new file mode 100755
index 000000000..4a581b46a
--- /dev/null
+++ b/scripts/mongo-to-postgre.js
@@ -0,0 +1,244 @@
+#!/usr/bin/env node
+
+'use strict'
+
+// TODO: document this script
+
+const program = require('commander')
+const eachSeries = require('async/eachSeries')
+const series = require('async/series')
+const waterfall = require('async/waterfall')
+const fs = require('fs')
+const path = require('path')
+const MongoClient = require('mongodb').MongoClient
+
+const constants = require('../server/initializers/constants')
+
+program
+  .option('-mh, --mongo-host [host]', 'MongoDB host', 'localhost')
+  .option('-mp, --mongo-port [weight]', 'MongoDB port', '27017')
+  .option('-md, --mongo-database [dbname]', 'MongoDB database')
+  .parse(process.argv)
+
+if (!program.mongoDatabase) {
+  console.error('The mongodb database is mandatory.')
+  process.exit(-1)
+}
+
+const mongoUrl = 'mongodb://' + program.mongoHost + ':' + program.mongoPort + '/' + program.mongoDatabase
+const dbSequelize = require('../server/initializers/database')
+
+console.log('Connecting to ' + mongoUrl)
+MongoClient.connect(mongoUrl, function (err, dbMongo) {
+  if (err) throw err
+
+  console.log('Connected to ' + mongoUrl)
+
+  const videoMongo = dbMongo.collection('videos')
+  const userMongo = dbMongo.collection('users')
+  const podMongo = dbMongo.collection('pods')
+
+  podMongo.count(function (err, podsLength) {
+    if (err) throw err
+
+    if (podsLength > 0) {
+      console.error('You need to quit friends first.')
+      process.exit(-1)
+    }
+
+    console.log('Connecting to ' + dbSequelize.sequelize.config.database)
+    dbSequelize.init(true, function (err) {
+      if (err) throw err
+
+      console.log('Connected to SQL database %s.', dbSequelize.sequelize.config.database)
+
+      series([
+        function (next) {
+          dbSequelize.sequelize.sync({ force: true }).asCallback(next)
+        },
+
+        function (next) {
+          migrateVideos(videoMongo, dbSequelize, next)
+        },
+
+        function (next) {
+          migrateUsers(userMongo, dbSequelize, next)
+        }
+      ], function (err) {
+        if (err) console.error(err)
+
+        process.exit(0)
+      })
+    })
+  })
+})
+
+// ---------------------------------------------------------------------------
+
+function migrateUsers (userMongo, dbSequelize, callback) {
+  userMongo.find().toArray(function (err, mongoUsers) {
+    if (err) return callback(err)
+
+    eachSeries(mongoUsers, function (mongoUser, callbackEach) {
+      console.log('Migrating user %s', mongoUser.username)
+
+      const userData = {
+        username: mongoUser.username,
+        password: mongoUser.password,
+        role: mongoUser.role
+      }
+      const options = {
+        hooks: false
+      }
+
+      dbSequelize.User.create(userData, options).asCallback(callbackEach)
+    }, callback)
+  })
+}
+
+function migrateVideos (videoMongo, dbSequelize, finalCallback) {
+  videoMongo.find().toArray(function (err, mongoVideos) {
+    if (err) return finalCallback(err)
+
+    eachSeries(mongoVideos, function (mongoVideo, callbackEach) {
+      console.log('Migrating video %s.', mongoVideo.name)
+
+      waterfall([
+
+        function startTransaction (callback) {
+          dbSequelize.sequelize.transaction().asCallback(function (err, t) {
+            return callback(err, t)
+          })
+        },
+
+        function findOrCreatePod (t, callback) {
+          if (mongoVideo.remoteId === null) return callback(null, t, null)
+
+          const query = {
+            where: {
+              host: mongoVideo.podHost
+            },
+            defaults: {
+              host: mongoVideo.podHost
+            },
+            transaction: t
+          }
+
+          dbSequelize.Pod.findOrCreate(query).asCallback(function (err, result) {
+            // [ instance, wasCreated ]
+            return callback(err, t, result[0])
+          })
+        },
+
+        function findOrCreateAuthor (t, pod, callback) {
+          const podId = pod ? pod.id : null
+          const username = mongoVideo.author
+
+          const query = {
+            where: {
+              podId,
+              name: username
+            },
+            defaults: {
+              podId,
+              name: username
+            },
+            transaction: t
+          }
+
+          dbSequelize.Author.findOrCreate(query).asCallback(function (err, result) {
+            // [ instance, wasCreated ]
+            return callback(err, t, result[0])
+          })
+        },
+
+        function findOrCreateTags (t, author, callback) {
+          const tags = mongoVideo.tags
+          const tagInstances = []
+
+          eachSeries(tags, function (tag, callbackEach) {
+            const query = {
+              where: {
+                name: tag
+              },
+              defaults: {
+                name: tag
+              },
+              transaction: t
+            }
+
+            dbSequelize.Tag.findOrCreate(query).asCallback(function (err, res) {
+              if (err) return callbackEach(err)
+
+              // res = [ tag, isCreated ]
+              const tag = res[0]
+              tagInstances.push(tag)
+              return callbackEach()
+            })
+          }, function (err) {
+            return callback(err, t, author, tagInstances)
+          })
+        },
+
+        function createVideoObject (t, author, tagInstances, callback) {
+          const videoData = {
+            name: mongoVideo.name,
+            remoteId: mongoVideo.remoteId,
+            extname: mongoVideo.extname,
+            infoHash: mongoVideo.magnet.infoHash,
+            description: mongoVideo.description,
+            authorId: author.id,
+            duration: mongoVideo.duration,
+            createdAt: mongoVideo.createdDate
+          }
+
+          const video = dbSequelize.Video.build(videoData)
+
+          return callback(null, t, tagInstances, video)
+        },
+
+        function moveVideoFile (t, tagInstances, video, callback) {
+          const basePath = constants.CONFIG.STORAGE.VIDEOS_DIR
+          const src = path.join(basePath, mongoVideo._id.toString()) + video.extname
+          const dst = path.join(basePath, video.id) + video.extname
+          fs.rename(src, dst, function (err) {
+            if (err) return callback(err)
+
+            return callback(null, t, tagInstances, video)
+          })
+        },
+
+        function insertVideoIntoDB (t, tagInstances, video, callback) {
+          const options = {
+            transaction: t
+          }
+
+          video.save(options).asCallback(function (err, videoCreated) {
+            return callback(err, t, tagInstances, videoCreated)
+          })
+        },
+
+        function associateTagsToVideo (t, tagInstances, video, callback) {
+          const options = { transaction: t }
+
+          video.setTags(tagInstances, options).asCallback(function (err) {
+            return callback(err, t)
+          })
+        }
+
+      ], function (err, t) {
+        if (err) {
+          // Abort transaction?
+          if (t) t.rollback()
+
+          return callbackEach(err)
+        }
+
+        // Commit transaction
+        t.commit()
+
+        return callbackEach()
+      })
+    }, finalCallback)
+  })
+}
diff --git a/scripts/play.sh b/scripts/play.sh
index 33dc1a545..ab812025c 100755
--- a/scripts/play.sh
+++ b/scripts/play.sh
@@ -5,7 +5,9 @@ if [ ! -f server.js ]; then
   exit -1
 fi
 
-for i in 1 2 3; do
+max=${1:-3}
+
+for i in $(seq 1 $max); do
   NODE_ENV=test NODE_APP_INSTANCE=$i node server.js &
   sleep 1
 done
diff --git a/scripts/update-host.js b/scripts/update-host.js
index 8a17f2402..91051e68b 100755
--- a/scripts/update-host.js
+++ b/scripts/update-host.js
@@ -5,31 +5,24 @@
 // TODO: document this script
 
 const fs = require('fs')
-const mongoose = require('mongoose')
 const parseTorrent = require('parse-torrent')
 
 const constants = require('../server/initializers/constants')
-const database = require('../server/initializers/database')
-
-database.connect()
+const db = require('../server/initializers/database')
 
 const friends = require('../server/lib/friends')
-const Video = mongoose.model('Video')
-
-friends.hasFriends(function (err, hasFriends) {
-  if (err) throw err
 
-  if (hasFriends === true) {
-    console.log('Cannot update host because you have friends!')
-    process.exit(-1)
-  }
-
-  console.log('Updating videos host in database.')
-  Video.update({ }, { podHost: constants.CONFIG.WEBSERVER.HOST }, { multi: true }, function (err) {
+db.init(true, function () {
+  friends.hasFriends(function (err, hasFriends) {
     if (err) throw err
 
+    if (hasFriends === true) {
+      console.log('Cannot update host because you have friends!')
+      process.exit(-1)
+    }
+
     console.log('Updating torrent files.')
-    Video.find().lean().exec(function (err, videos) {
+    db.Video.list(function (err, videos) {
       if (err) throw err
 
       videos.forEach(function (video) {