aboutsummaryrefslogtreecommitdiffhomepage
diff options
context:
space:
mode:
authorChocobozzz <florian.bigard@gmail.com>2016-12-27 18:33:38 +0100
committerChocobozzz <florian.bigard@gmail.com>2016-12-27 18:34:47 +0100
commit00d6a41e46e4c4948b0d5b4cf21433150a57c067 (patch)
tree8366fe4229c96b85b6f2d5e7c88c2565d43e0a50
parent178edb20259f90b1c59f40728aaf8073f097f1f5 (diff)
downloadPeerTube-00d6a41e46e4c4948b0d5b4cf21433150a57c067.tar.gz
PeerTube-00d6a41e46e4c4948b0d5b4cf21433150a57c067.tar.zst
PeerTube-00d6a41e46e4c4948b0d5b4cf21433150a57c067.zip
Add script to migrate from mongodb to postgresql
Usage: NODE_ENV=production ./scripts/mongo-to-postgre.js --mongo-database peertube-prod
-rw-r--r--scripts/danger/clean/cleaner.js10
-rwxr-xr-xscripts/mongo-to-postgre.js244
-rw-r--r--server/helpers/peertube-crypto.js1
3 files changed, 250 insertions, 5 deletions
diff --git a/scripts/danger/clean/cleaner.js b/scripts/danger/clean/cleaner.js
index d1e218145..1a1e3dee7 100644
--- a/scripts/danger/clean/cleaner.js
+++ b/scripts/danger/clean/cleaner.js
@@ -1,3 +1,4 @@
1const eachSeries = require('async/eachSeries')
1const rimraf = require('rimraf') 2const rimraf = require('rimraf')
2 3
3const constants = require('../../../server/initializers/constants') 4const constants = require('../../../server/initializers/constants')
@@ -10,14 +11,15 @@ db.init(true, function () {
10 console.info('Tables of %s deleted.', db.sequelize.config.database) 11 console.info('Tables of %s deleted.', db.sequelize.config.database)
11 12
12 const STORAGE = constants.CONFIG.STORAGE 13 const STORAGE = constants.CONFIG.STORAGE
13 Object.keys(STORAGE).forEach(function (storage) { 14 eachSeries(Object.keys(STORAGE), function (storage, callbackEach) {
14 const storageDir = STORAGE[storage] 15 const storageDir = STORAGE[storage]
15 16
16 rimraf(storageDir, function (err) { 17 rimraf(storageDir, function (err) {
17 if (err) throw err 18 console.info('%s deleted.', storageDir)
18 19 return callbackEach(err)
19 console.info('Deleting %s.', storageDir)
20 }) 20 })
21 }, function () {
22 process.exit(0)
21 }) 23 })
22 }) 24 })
23}) 25})
diff --git a/scripts/mongo-to-postgre.js b/scripts/mongo-to-postgre.js
new file mode 100755
index 000000000..4a581b46a
--- /dev/null
+++ b/scripts/mongo-to-postgre.js
@@ -0,0 +1,244 @@
1#!/usr/bin/env node
2
3'use strict'
4
5// TODO: document this script
6
7const program = require('commander')
8const eachSeries = require('async/eachSeries')
9const series = require('async/series')
10const waterfall = require('async/waterfall')
11const fs = require('fs')
12const path = require('path')
13const MongoClient = require('mongodb').MongoClient
14
15const constants = require('../server/initializers/constants')
16
17program
18 .option('-mh, --mongo-host [host]', 'MongoDB host', 'localhost')
19 .option('-mp, --mongo-port [weight]', 'MongoDB port', '27017')
20 .option('-md, --mongo-database [dbname]', 'MongoDB database')
21 .parse(process.argv)
22
23if (!program.mongoDatabase) {
24 console.error('The mongodb database is mandatory.')
25 process.exit(-1)
26}
27
28const mongoUrl = 'mongodb://' + program.mongoHost + ':' + program.mongoPort + '/' + program.mongoDatabase
29const dbSequelize = require('../server/initializers/database')
30
31console.log('Connecting to ' + mongoUrl)
32MongoClient.connect(mongoUrl, function (err, dbMongo) {
33 if (err) throw err
34
35 console.log('Connected to ' + mongoUrl)
36
37 const videoMongo = dbMongo.collection('videos')
38 const userMongo = dbMongo.collection('users')
39 const podMongo = dbMongo.collection('pods')
40
41 podMongo.count(function (err, podsLength) {
42 if (err) throw err
43
44 if (podsLength > 0) {
45 console.error('You need to quit friends first.')
46 process.exit(-1)
47 }
48
49 console.log('Connecting to ' + dbSequelize.sequelize.config.database)
50 dbSequelize.init(true, function (err) {
51 if (err) throw err
52
53 console.log('Connected to SQL database %s.', dbSequelize.sequelize.config.database)
54
55 series([
56 function (next) {
57 dbSequelize.sequelize.sync({ force: true }).asCallback(next)
58 },
59
60 function (next) {
61 migrateVideos(videoMongo, dbSequelize, next)
62 },
63
64 function (next) {
65 migrateUsers(userMongo, dbSequelize, next)
66 }
67 ], function (err) {
68 if (err) console.error(err)
69
70 process.exit(0)
71 })
72 })
73 })
74})
75
76// ---------------------------------------------------------------------------
77
78function migrateUsers (userMongo, dbSequelize, callback) {
79 userMongo.find().toArray(function (err, mongoUsers) {
80 if (err) return callback(err)
81
82 eachSeries(mongoUsers, function (mongoUser, callbackEach) {
83 console.log('Migrating user %s', mongoUser.username)
84
85 const userData = {
86 username: mongoUser.username,
87 password: mongoUser.password,
88 role: mongoUser.role
89 }
90 const options = {
91 hooks: false
92 }
93
94 dbSequelize.User.create(userData, options).asCallback(callbackEach)
95 }, callback)
96 })
97}
98
99function migrateVideos (videoMongo, dbSequelize, finalCallback) {
100 videoMongo.find().toArray(function (err, mongoVideos) {
101 if (err) return finalCallback(err)
102
103 eachSeries(mongoVideos, function (mongoVideo, callbackEach) {
104 console.log('Migrating video %s.', mongoVideo.name)
105
106 waterfall([
107
108 function startTransaction (callback) {
109 dbSequelize.sequelize.transaction().asCallback(function (err, t) {
110 return callback(err, t)
111 })
112 },
113
114 function findOrCreatePod (t, callback) {
115 if (mongoVideo.remoteId === null) return callback(null, t, null)
116
117 const query = {
118 where: {
119 host: mongoVideo.podHost
120 },
121 defaults: {
122 host: mongoVideo.podHost
123 },
124 transaction: t
125 }
126
127 dbSequelize.Pod.findOrCreate(query).asCallback(function (err, result) {
128 // [ instance, wasCreated ]
129 return callback(err, t, result[0])
130 })
131 },
132
133 function findOrCreateAuthor (t, pod, callback) {
134 const podId = pod ? pod.id : null
135 const username = mongoVideo.author
136
137 const query = {
138 where: {
139 podId,
140 name: username
141 },
142 defaults: {
143 podId,
144 name: username
145 },
146 transaction: t
147 }
148
149 dbSequelize.Author.findOrCreate(query).asCallback(function (err, result) {
150 // [ instance, wasCreated ]
151 return callback(err, t, result[0])
152 })
153 },
154
155 function findOrCreateTags (t, author, callback) {
156 const tags = mongoVideo.tags
157 const tagInstances = []
158
159 eachSeries(tags, function (tag, callbackEach) {
160 const query = {
161 where: {
162 name: tag
163 },
164 defaults: {
165 name: tag
166 },
167 transaction: t
168 }
169
170 dbSequelize.Tag.findOrCreate(query).asCallback(function (err, res) {
171 if (err) return callbackEach(err)
172
173 // res = [ tag, isCreated ]
174 const tag = res[0]
175 tagInstances.push(tag)
176 return callbackEach()
177 })
178 }, function (err) {
179 return callback(err, t, author, tagInstances)
180 })
181 },
182
183 function createVideoObject (t, author, tagInstances, callback) {
184 const videoData = {
185 name: mongoVideo.name,
186 remoteId: mongoVideo.remoteId,
187 extname: mongoVideo.extname,
188 infoHash: mongoVideo.magnet.infoHash,
189 description: mongoVideo.description,
190 authorId: author.id,
191 duration: mongoVideo.duration,
192 createdAt: mongoVideo.createdDate
193 }
194
195 const video = dbSequelize.Video.build(videoData)
196
197 return callback(null, t, tagInstances, video)
198 },
199
200 function moveVideoFile (t, tagInstances, video, callback) {
201 const basePath = constants.CONFIG.STORAGE.VIDEOS_DIR
202 const src = path.join(basePath, mongoVideo._id.toString()) + video.extname
203 const dst = path.join(basePath, video.id) + video.extname
204 fs.rename(src, dst, function (err) {
205 if (err) return callback(err)
206
207 return callback(null, t, tagInstances, video)
208 })
209 },
210
211 function insertVideoIntoDB (t, tagInstances, video, callback) {
212 const options = {
213 transaction: t
214 }
215
216 video.save(options).asCallback(function (err, videoCreated) {
217 return callback(err, t, tagInstances, videoCreated)
218 })
219 },
220
221 function associateTagsToVideo (t, tagInstances, video, callback) {
222 const options = { transaction: t }
223
224 video.setTags(tagInstances, options).asCallback(function (err) {
225 return callback(err, t)
226 })
227 }
228
229 ], function (err, t) {
230 if (err) {
231 // Abort transaction?
232 if (t) t.rollback()
233
234 return callbackEach(err)
235 }
236
237 // Commit transaction
238 t.commit()
239
240 return callbackEach()
241 })
242 }, finalCallback)
243 })
244}
diff --git a/server/helpers/peertube-crypto.js b/server/helpers/peertube-crypto.js
index 302ddca58..610cb16cd 100644
--- a/server/helpers/peertube-crypto.js
+++ b/server/helpers/peertube-crypto.js
@@ -1,7 +1,6 @@
1'use strict' 1'use strict'
2 2
3const bcrypt = require('bcrypt') 3const bcrypt = require('bcrypt')
4const crypto = require('crypto')
5const fs = require('fs') 4const fs = require('fs')
6const openssl = require('openssl-wrapper') 5const openssl = require('openssl-wrapper')
7const ursa = require('ursa') 6const ursa = require('ursa')