NMIG reconstruction, ram consumption reduction, bug fixes
This commit is contained in:
parent
279d7d740c
commit
02cdc3817f
|
@ -36,6 +36,12 @@
|
||||||
"Maximal amount of simultaneous connections to your PostgreSQL database during migration"
|
"Maximal amount of simultaneous connections to your PostgreSQL database during migration"
|
||||||
],
|
],
|
||||||
"max_pool_size_target" : 10,
|
"max_pool_size_target" : 10,
|
||||||
|
|
||||||
|
"pipe_width_description" : [
|
||||||
|
"Maximal amount of data-chunks, processed in a single 'DataPipe' iteration.",
|
||||||
|
"Note: 'pipe_width' should not be greater than 'max_pool_size_target'."
|
||||||
|
],
|
||||||
|
"pipe_width" : 10,
|
||||||
|
|
||||||
"encoding_description" : [
|
"encoding_description" : [
|
||||||
"JavaScript encoding type.",
|
"JavaScript encoding type.",
|
||||||
|
@ -47,7 +53,7 @@
|
||||||
"schema - a name of the schema, that will contain all migrated tables.",
|
"schema - a name of the schema, that will contain all migrated tables.",
|
||||||
"If not supplied, then a new schema will be created automatically."
|
"If not supplied, then a new schema will be created automatically."
|
||||||
],
|
],
|
||||||
"schema" : "test_schema",
|
"schema" : "public",
|
||||||
|
|
||||||
"data_chunk_size_description" : [
|
"data_chunk_size_description" : [
|
||||||
"During migration each table's data will be split into chunks of data_chunk_size (in KB).",
|
"During migration each table's data will be split into chunks of data_chunk_size (in KB).",
|
||||||
|
|
|
@ -29,7 +29,7 @@
|
||||||
module.exports = function(arrTableColumns) {
|
module.exports = function(arrTableColumns) {
|
||||||
let strRetVal = '';
|
let strRetVal = '';
|
||||||
|
|
||||||
for (let i = 0; i < arrTableColumns.length; i++) {
|
for (let i = 0; i < arrTableColumns.length; ++i) {
|
||||||
if (
|
if (
|
||||||
arrTableColumns[i].Type.indexOf('geometry') !== -1
|
arrTableColumns[i].Type.indexOf('geometry') !== -1
|
||||||
|| arrTableColumns[i].Type.indexOf('point') !== -1
|
|| arrTableColumns[i].Type.indexOf('point') !== -1
|
||||||
|
@ -47,9 +47,8 @@ module.exports = function(arrTableColumns) {
|
||||||
) {
|
) {
|
||||||
strRetVal += 'BIN(`' + arrTableColumns[i].Field + '`),';
|
strRetVal += 'BIN(`' + arrTableColumns[i].Field + '`),';
|
||||||
} else if (
|
} else if (
|
||||||
arrTableColumns[i].Type.indexOf('date') !== -1
|
arrTableColumns[i].Type.indexOf('timestamp') !== -1
|
||||||
|| arrTableColumns[i].Type.indexOf('datetime') !== -1
|
|| arrTableColumns[i].Type.indexOf('date') !== -1
|
||||||
|| arrTableColumns[i].Type.indexOf('timestamp') !== -1
|
|
||||||
) {
|
) {
|
||||||
strRetVal += 'IF(`' + arrTableColumns[i].Field
|
strRetVal += 'IF(`' + arrTableColumns[i].Field
|
||||||
+ '` IN(\'0000-00-00\', \'0000-00-00 00:00:00\'), \'-INFINITY\', `'
|
+ '` IN(\'0000-00-00\', \'0000-00-00 00:00:00\'), \'-INFINITY\', `'
|
||||||
|
|
279
migration/fmtp/CsvStringifyModified.js
Normal file
279
migration/fmtp/CsvStringifyModified.js
Normal file
|
@ -0,0 +1,279 @@
|
||||||
|
/*
|
||||||
|
* This is a copy of "csv-stringify" module, that was slightly modified.
|
||||||
|
*/
|
||||||
|
var Stringifier, stream, util;
|
||||||
|
|
||||||
|
stream = require('stream');
|
||||||
|
|
||||||
|
util = require('util');
|
||||||
|
|
||||||
|
module.exports = function() {
|
||||||
|
var callback, chunks, data, options, stringifier;
|
||||||
|
if (arguments.length === 3) {
|
||||||
|
data = arguments[0];
|
||||||
|
options = arguments[1];
|
||||||
|
callback = arguments[2];
|
||||||
|
} else if (arguments.length === 2) {
|
||||||
|
if (Array.isArray(arguments[0])) {
|
||||||
|
data = arguments[0];
|
||||||
|
} else {
|
||||||
|
options = arguments[0];
|
||||||
|
}
|
||||||
|
if (typeof arguments[1] === 'function') {
|
||||||
|
callback = arguments[1];
|
||||||
|
} else {
|
||||||
|
options = arguments[1];
|
||||||
|
}
|
||||||
|
} else if (arguments.length === 1) {
|
||||||
|
if (typeof arguments[0] === 'function') {
|
||||||
|
callback = arguments[0];
|
||||||
|
} else if (Array.isArray(arguments[0])) {
|
||||||
|
data = arguments[0];
|
||||||
|
} else {
|
||||||
|
options = arguments[0];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (options == null) {
|
||||||
|
options = {};
|
||||||
|
}
|
||||||
|
stringifier = new Stringifier(options);
|
||||||
|
if (data) {
|
||||||
|
process.nextTick(function() {
|
||||||
|
var d, j, len;
|
||||||
|
for (j = 0, len = data.length; j < len; j++) {
|
||||||
|
d = data[j];
|
||||||
|
stringifier.write(d);
|
||||||
|
}
|
||||||
|
return stringifier.end();
|
||||||
|
});
|
||||||
|
}
|
||||||
|
if (callback) {
|
||||||
|
chunks = [];
|
||||||
|
stringifier.on('readable', function() {
|
||||||
|
var chunk, results;
|
||||||
|
results = [];
|
||||||
|
while (chunk = stringifier.read()) {
|
||||||
|
results.push(chunks.push(chunk));
|
||||||
|
}
|
||||||
|
return results;
|
||||||
|
});
|
||||||
|
stringifier.on('error', function(err) {
|
||||||
|
return callback(err);
|
||||||
|
});
|
||||||
|
stringifier.on('end', function() {
|
||||||
|
return callback(null, chunks.join(''));
|
||||||
|
});
|
||||||
|
}
|
||||||
|
return stringifier;
|
||||||
|
};
|
||||||
|
|
||||||
|
Stringifier = function(options) {
|
||||||
|
var base, base1, base2, base3, base4, base5, base6, base7, base8;
|
||||||
|
if (options == null) {
|
||||||
|
options = {};
|
||||||
|
}
|
||||||
|
stream.Transform.call(this, options);
|
||||||
|
this.options = options;
|
||||||
|
if ((base = this.options).delimiter == null) {
|
||||||
|
base.delimiter = ',';
|
||||||
|
}
|
||||||
|
if ((base1 = this.options).quote == null) {
|
||||||
|
base1.quote = '"';
|
||||||
|
}
|
||||||
|
if ((base2 = this.options).quoted == null) {
|
||||||
|
base2.quoted = false;
|
||||||
|
}
|
||||||
|
if ((base3 = this.options).quotedString == null) {
|
||||||
|
base3.quotedString = false;
|
||||||
|
}
|
||||||
|
if ((base4 = this.options).eof == null) {
|
||||||
|
base4.eof = true;
|
||||||
|
}
|
||||||
|
if ((base5 = this.options).escape == null) {
|
||||||
|
base5.escape = '"';
|
||||||
|
}
|
||||||
|
if ((base6 = this.options).columns == null) {
|
||||||
|
base6.columns = null;
|
||||||
|
}
|
||||||
|
if ((base7 = this.options).header == null) {
|
||||||
|
base7.header = false;
|
||||||
|
}
|
||||||
|
if ((base8 = this.options).rowDelimiter == null) {
|
||||||
|
base8.rowDelimiter = '\n';
|
||||||
|
}
|
||||||
|
if (this.countWriten == null) {
|
||||||
|
this.countWriten = 0;
|
||||||
|
}
|
||||||
|
switch (this.options.rowDelimiter) {
|
||||||
|
case 'auto':
|
||||||
|
this.options.rowDelimiter = null;
|
||||||
|
break;
|
||||||
|
case 'unix':
|
||||||
|
this.options.rowDelimiter = "\n";
|
||||||
|
break;
|
||||||
|
case 'mac':
|
||||||
|
this.options.rowDelimiter = "\r";
|
||||||
|
break;
|
||||||
|
case 'windows':
|
||||||
|
this.options.rowDelimiter = "\r\n";
|
||||||
|
break;
|
||||||
|
case 'unicode':
|
||||||
|
this.options.rowDelimiter = "\u2028";
|
||||||
|
}
|
||||||
|
return this;
|
||||||
|
};
|
||||||
|
|
||||||
|
util.inherits(Stringifier, stream.Transform);
|
||||||
|
|
||||||
|
module.exports.Stringifier = Stringifier;
|
||||||
|
|
||||||
|
Stringifier.prototype.headers = function() {
|
||||||
|
var k, label, labels;
|
||||||
|
if (!this.options.header) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (!this.options.columns) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
labels = this.options.columns;
|
||||||
|
if (typeof labels === 'object') {
|
||||||
|
labels = (function() {
|
||||||
|
var results;
|
||||||
|
results = [];
|
||||||
|
for (k in labels) {
|
||||||
|
label = labels[k];
|
||||||
|
results.push(label);
|
||||||
|
}
|
||||||
|
return results;
|
||||||
|
})();
|
||||||
|
}
|
||||||
|
if (this.options.eof) {
|
||||||
|
labels = this.stringify(labels) + this.options.rowDelimiter;
|
||||||
|
} else {
|
||||||
|
labels = this.stringify(labels);
|
||||||
|
}
|
||||||
|
return stream.Transform.prototype.write.call(this, labels);
|
||||||
|
};
|
||||||
|
|
||||||
|
Stringifier.prototype.end = function(chunk, encoding, callback) {
|
||||||
|
if (this.countWriten === 0) {
|
||||||
|
this.headers();
|
||||||
|
}
|
||||||
|
return stream.Transform.prototype.end.apply(this, arguments);
|
||||||
|
};
|
||||||
|
|
||||||
|
Stringifier.prototype.write = function(chunk, encoding, callback) {
|
||||||
|
var base, e, preserve;
|
||||||
|
if (chunk == null) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
preserve = typeof chunk !== 'object';
|
||||||
|
if (!preserve) {
|
||||||
|
if (this.countWriten === 0 && !Array.isArray(chunk)) {
|
||||||
|
if ((base = this.options).columns == null) {
|
||||||
|
base.columns = Object.keys(chunk);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
this.emit('record', chunk, this.countWriten);
|
||||||
|
} catch (_error) {
|
||||||
|
e = _error;
|
||||||
|
return this.emit('error', e);
|
||||||
|
}
|
||||||
|
if (this.options.eof) {
|
||||||
|
chunk = this.stringify(chunk) + this.options.rowDelimiter;
|
||||||
|
} else {
|
||||||
|
chunk = this.stringify(chunk);
|
||||||
|
if (this.options.header || this.countWriten) {
|
||||||
|
chunk = this.options.rowDelimiter + chunk;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (typeof chunk === 'number') {
|
||||||
|
chunk = "" + chunk;
|
||||||
|
}
|
||||||
|
if (this.countWriten === 0) {
|
||||||
|
this.headers();
|
||||||
|
}
|
||||||
|
if (!preserve) {
|
||||||
|
this.countWriten++;
|
||||||
|
}
|
||||||
|
return stream.Transform.prototype.write.call(this, chunk, encoding, callback);
|
||||||
|
};
|
||||||
|
|
||||||
|
Stringifier.prototype._transform = function(chunk, encoding, callback) {
|
||||||
|
this.push(chunk);
|
||||||
|
return callback();
|
||||||
|
};
|
||||||
|
|
||||||
|
Stringifier.prototype.stringify = function(line) {
|
||||||
|
var _line, column, columns, containsLinebreak, containsQuote, containsdelimiter, delimiter, escape, field, i, j, l, newLine, quote, ref, ref1, regexp;
|
||||||
|
if (typeof line !== 'object') {
|
||||||
|
return line;
|
||||||
|
}
|
||||||
|
columns = this.options.columns;
|
||||||
|
if (typeof columns === 'object' && columns !== null && !Array.isArray(columns)) {
|
||||||
|
columns = Object.keys(columns);
|
||||||
|
}
|
||||||
|
delimiter = this.options.delimiter;
|
||||||
|
quote = this.options.quote;
|
||||||
|
escape = this.options.escape;
|
||||||
|
if (!Array.isArray(line)) {
|
||||||
|
_line = [];
|
||||||
|
if (columns) {
|
||||||
|
for (i = j = 0, ref = columns.length; 0 <= ref ? j < ref : j > ref; i = 0 <= ref ? ++j : --j) {
|
||||||
|
column = columns[i];
|
||||||
|
_line[i] = typeof line[column] === 'undefined' || line[column] === null ? '' : line[column];
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
for (column in line) {
|
||||||
|
_line.push(line[column]);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
line = _line;
|
||||||
|
_line = null;
|
||||||
|
} else if (columns) {
|
||||||
|
line.splice(columns.length);
|
||||||
|
}
|
||||||
|
if (Array.isArray(line)) {
|
||||||
|
newLine = '';
|
||||||
|
for (i = l = 0, ref1 = line.length; 0 <= ref1 ? l < ref1 : l > ref1; i = 0 <= ref1 ? ++l : --l) {
|
||||||
|
field = line[i];
|
||||||
|
if (typeof field === 'string') {
|
||||||
|
// No code here in original version.
|
||||||
|
} else if (typeof field === 'number') {
|
||||||
|
field = '' + field;
|
||||||
|
} else if (typeof field === 'boolean') {
|
||||||
|
field = field ? '1' : '';
|
||||||
|
} else if (field instanceof Date) {
|
||||||
|
//field = '' + field.getTime(); // In original version.
|
||||||
|
field = field.getFullYear() + '-' + (field.getMonth() + 1) + '-' + field.getDate()
|
||||||
|
+ ' ' + field.getHours() + ':' + field.getMinutes() + ':' + field.getSeconds();
|
||||||
|
|
||||||
|
} else if (typeof field === 'object' && field !== null) {
|
||||||
|
field = JSON.stringify(field);
|
||||||
|
}
|
||||||
|
if (field) {
|
||||||
|
containsdelimiter = field.indexOf(delimiter) >= 0;
|
||||||
|
containsQuote = field.indexOf(quote) >= 0;
|
||||||
|
containsLinebreak = field.indexOf('\r') >= 0 || field.indexOf('\n') >= 0;
|
||||||
|
if (containsQuote) {
|
||||||
|
regexp = new RegExp(quote, 'g');
|
||||||
|
field = field.replace(regexp, escape + quote);
|
||||||
|
}
|
||||||
|
if (containsQuote || containsdelimiter || containsLinebreak || this.options.quoted || (this.options.quotedString && typeof line[i] === 'string')) {
|
||||||
|
field = quote + field + quote;
|
||||||
|
}
|
||||||
|
newLine += field;
|
||||||
|
} else if (this.options.quotedEmpty || ((this.options.quotedEmpty == null) && line[i] === '' && this.options.quotedString)) {
|
||||||
|
newLine += quote + quote;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (i !== line.length - 1) {
|
||||||
|
newLine += delimiter;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
line = newLine;
|
||||||
|
}
|
||||||
|
return line;
|
||||||
|
};
|
File diff suppressed because it is too large
Load diff
33
migration/fmtp/Table.js
Normal file
33
migration/fmtp/Table.js
Normal file
|
@ -0,0 +1,33 @@
|
||||||
|
/*
|
||||||
|
* This file is a part of "NMIG" - the database migration tool.
|
||||||
|
*
|
||||||
|
* Copyright 2016 Anatoly Khaytovich <anatolyuss@gmail.com>
|
||||||
|
*
|
||||||
|
* This program is free software: you can redistribute it and/or modify
|
||||||
|
* it under the terms of the GNU General Public License as published by
|
||||||
|
* the Free Software Foundation, either version 3 of the License.
|
||||||
|
*
|
||||||
|
* This program is distributed in the hope that it will be useful,
|
||||||
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
* GNU General Public License for more details.
|
||||||
|
*
|
||||||
|
* You should have received a copy of the GNU General Public License
|
||||||
|
* along with this program (please see the "LICENSE.md" file).
|
||||||
|
* If not, see <http://www.gnu.org/licenses/gpl.txt>.
|
||||||
|
*
|
||||||
|
* @author Anatoly Khaytovich <anatolyuss@gmail.com>
|
||||||
|
*/
|
||||||
|
'use strict';
|
||||||
|
|
||||||
|
/**
|
||||||
|
* This function represents table related metadata.
|
||||||
|
* Constructor.
|
||||||
|
*
|
||||||
|
* @param {String} tableLogPath
|
||||||
|
*/
|
||||||
|
module.exports = function Table(tableLogPath) {
|
||||||
|
this.tableLogPath = tableLogPath;
|
||||||
|
this.arrTableColumns = [];
|
||||||
|
this.totalRowsInserted = 0;
|
||||||
|
};
|
|
@ -20,13 +20,6 @@
|
||||||
*/
|
*/
|
||||||
'use strict';
|
'use strict';
|
||||||
|
|
||||||
/**
|
|
||||||
* Constructor.
|
|
||||||
*/
|
|
||||||
function ViewGenerator() {
|
|
||||||
// No code should be put here.
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Attempts to convert MySQL view to PostgreSQL view.
|
* Attempts to convert MySQL view to PostgreSQL view.
|
||||||
*
|
*
|
||||||
|
@ -35,13 +28,13 @@ function ViewGenerator() {
|
||||||
* @param {String} mysqlViewCode
|
* @param {String} mysqlViewCode
|
||||||
* @returns {String}
|
* @returns {String}
|
||||||
*/
|
*/
|
||||||
ViewGenerator.prototype.generateView = function(schema, viewName, mysqlViewCode) {
|
module.exports = function(schema, viewName, mysqlViewCode) {
|
||||||
mysqlViewCode = mysqlViewCode.split('`').join('"');
|
mysqlViewCode = mysqlViewCode.split('`').join('"');
|
||||||
let queryStart = mysqlViewCode.indexOf('AS');
|
let queryStart = mysqlViewCode.indexOf('AS');
|
||||||
mysqlViewCode = mysqlViewCode.slice(queryStart);
|
mysqlViewCode = mysqlViewCode.slice(queryStart);
|
||||||
let arrMysqlViewCode = mysqlViewCode.split(' ');
|
let arrMysqlViewCode = mysqlViewCode.split(' ');
|
||||||
|
|
||||||
for (let i = 0; i < arrMysqlViewCode.length; i++) {
|
for (let i = 0; i < arrMysqlViewCode.length; ++i) {
|
||||||
if (
|
if (
|
||||||
arrMysqlViewCode[i].toLowerCase() === 'from'
|
arrMysqlViewCode[i].toLowerCase() === 'from'
|
||||||
|| arrMysqlViewCode[i].toLowerCase() === 'join'
|
|| arrMysqlViewCode[i].toLowerCase() === 'join'
|
||||||
|
@ -53,5 +46,3 @@ ViewGenerator.prototype.generateView = function(schema, viewName, mysqlViewCode)
|
||||||
|
|
||||||
return 'CREATE OR REPLACE VIEW "' + schema + '"."' + viewName + '" ' + arrMysqlViewCode.join(' ') + ';';
|
return 'CREATE OR REPLACE VIEW "' + schema + '"."' + viewName + '" ' + arrMysqlViewCode.join(' ') + ';';
|
||||||
};
|
};
|
||||||
|
|
||||||
module.exports.ViewGenerator = ViewGenerator;
|
|
||||||
|
|
39
nmig.js
Normal file
39
nmig.js
Normal file
|
@ -0,0 +1,39 @@
|
||||||
|
/*
|
||||||
|
* This file is a part of "NMIG" - the database migration tool.
|
||||||
|
*
|
||||||
|
* Copyright 2016 Anatoly Khaytovich <anatolyuss@gmail.com>
|
||||||
|
*
|
||||||
|
* This program is free software: you can redistribute it and/or modify
|
||||||
|
* it under the terms of the GNU General Public License as published by
|
||||||
|
* the Free Software Foundation, either version 3 of the License.
|
||||||
|
*
|
||||||
|
* This program is distributed in the hope that it will be useful,
|
||||||
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
* GNU General Public License for more details.
|
||||||
|
*
|
||||||
|
* You should have received a copy of the GNU General Public License
|
||||||
|
* along with this program (please see the "LICENSE.md" file).
|
||||||
|
* If not, see <http://www.gnu.org/licenses/gpl.txt>.
|
||||||
|
*
|
||||||
|
* @author Anatoly Khaytovich <anatolyuss@gmail.com>
|
||||||
|
*/
|
||||||
|
'use strict';
|
||||||
|
|
||||||
|
const fs = require('fs');
|
||||||
|
const fmtp = require('./migration/fmtp/FromMySQL2PostgreSQL');
|
||||||
|
|
||||||
|
fs.readFile(__dirname + '/config.json', (error, data) => {
|
||||||
|
if (error) {
|
||||||
|
console.log('\n\t--Cannot run migration\nCannot read configuration info from ' + __dirname + '/config.json');
|
||||||
|
} else {
|
||||||
|
try {
|
||||||
|
let config = JSON.parse(data.toString());
|
||||||
|
config.tempDirPath = __dirname + '/temporary_directory';
|
||||||
|
config.logsDirPath = __dirname + '/logs_directory';
|
||||||
|
fmtp(config);
|
||||||
|
} catch (err) {
|
||||||
|
console.log('\n\t--Cannot parse JSON from' + __dirname + '/config.json');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
Loading…
Reference in a new issue