NMIG reconstruction, ram consumption reduction, bug fixes
This commit is contained in:
parent
279d7d740c
commit
02cdc3817f
|
@ -37,6 +37,12 @@
|
|||
],
|
||||
"max_pool_size_target" : 10,
|
||||
|
||||
"pipe_width_description" : [
|
||||
"Maximal amount of data-chunks, processed in a single 'DataPipe' iteration.",
|
||||
"Note: 'pipe_width' should not be greater than 'max_pool_size_target'."
|
||||
],
|
||||
"pipe_width" : 10,
|
||||
|
||||
"encoding_description" : [
|
||||
"JavaScript encoding type.",
|
||||
"If not supplied, then utf-8 will be used as a default."
|
||||
|
@ -47,7 +53,7 @@
|
|||
"schema - a name of the schema, that will contain all migrated tables.",
|
||||
"If not supplied, then a new schema will be created automatically."
|
||||
],
|
||||
"schema" : "test_schema",
|
||||
"schema" : "public",
|
||||
|
||||
"data_chunk_size_description" : [
|
||||
"During migration each table's data will be split into chunks of data_chunk_size (in KB).",
|
||||
|
|
|
@ -29,7 +29,7 @@
|
|||
module.exports = function(arrTableColumns) {
|
||||
let strRetVal = '';
|
||||
|
||||
for (let i = 0; i < arrTableColumns.length; i++) {
|
||||
for (let i = 0; i < arrTableColumns.length; ++i) {
|
||||
if (
|
||||
arrTableColumns[i].Type.indexOf('geometry') !== -1
|
||||
|| arrTableColumns[i].Type.indexOf('point') !== -1
|
||||
|
@ -47,9 +47,8 @@ module.exports = function(arrTableColumns) {
|
|||
) {
|
||||
strRetVal += 'BIN(`' + arrTableColumns[i].Field + '`),';
|
||||
} else if (
|
||||
arrTableColumns[i].Type.indexOf('date') !== -1
|
||||
|| arrTableColumns[i].Type.indexOf('datetime') !== -1
|
||||
|| arrTableColumns[i].Type.indexOf('timestamp') !== -1
|
||||
arrTableColumns[i].Type.indexOf('timestamp') !== -1
|
||||
|| arrTableColumns[i].Type.indexOf('date') !== -1
|
||||
) {
|
||||
strRetVal += 'IF(`' + arrTableColumns[i].Field
|
||||
+ '` IN(\'0000-00-00\', \'0000-00-00 00:00:00\'), \'-INFINITY\', `'
|
||||
|
|
279
migration/fmtp/CsvStringifyModified.js
Normal file
279
migration/fmtp/CsvStringifyModified.js
Normal file
|
@ -0,0 +1,279 @@
|
|||
/*
|
||||
* This is a copy of "csv-stringify" module, that was slightly modified.
|
||||
*/
|
||||
var Stringifier, stream, util;
|
||||
|
||||
stream = require('stream');
|
||||
|
||||
util = require('util');
|
||||
|
||||
module.exports = function() {
|
||||
var callback, chunks, data, options, stringifier;
|
||||
if (arguments.length === 3) {
|
||||
data = arguments[0];
|
||||
options = arguments[1];
|
||||
callback = arguments[2];
|
||||
} else if (arguments.length === 2) {
|
||||
if (Array.isArray(arguments[0])) {
|
||||
data = arguments[0];
|
||||
} else {
|
||||
options = arguments[0];
|
||||
}
|
||||
if (typeof arguments[1] === 'function') {
|
||||
callback = arguments[1];
|
||||
} else {
|
||||
options = arguments[1];
|
||||
}
|
||||
} else if (arguments.length === 1) {
|
||||
if (typeof arguments[0] === 'function') {
|
||||
callback = arguments[0];
|
||||
} else if (Array.isArray(arguments[0])) {
|
||||
data = arguments[0];
|
||||
} else {
|
||||
options = arguments[0];
|
||||
}
|
||||
}
|
||||
if (options == null) {
|
||||
options = {};
|
||||
}
|
||||
stringifier = new Stringifier(options);
|
||||
if (data) {
|
||||
process.nextTick(function() {
|
||||
var d, j, len;
|
||||
for (j = 0, len = data.length; j < len; j++) {
|
||||
d = data[j];
|
||||
stringifier.write(d);
|
||||
}
|
||||
return stringifier.end();
|
||||
});
|
||||
}
|
||||
if (callback) {
|
||||
chunks = [];
|
||||
stringifier.on('readable', function() {
|
||||
var chunk, results;
|
||||
results = [];
|
||||
while (chunk = stringifier.read()) {
|
||||
results.push(chunks.push(chunk));
|
||||
}
|
||||
return results;
|
||||
});
|
||||
stringifier.on('error', function(err) {
|
||||
return callback(err);
|
||||
});
|
||||
stringifier.on('end', function() {
|
||||
return callback(null, chunks.join(''));
|
||||
});
|
||||
}
|
||||
return stringifier;
|
||||
};
|
||||
|
||||
Stringifier = function(options) {
|
||||
var base, base1, base2, base3, base4, base5, base6, base7, base8;
|
||||
if (options == null) {
|
||||
options = {};
|
||||
}
|
||||
stream.Transform.call(this, options);
|
||||
this.options = options;
|
||||
if ((base = this.options).delimiter == null) {
|
||||
base.delimiter = ',';
|
||||
}
|
||||
if ((base1 = this.options).quote == null) {
|
||||
base1.quote = '"';
|
||||
}
|
||||
if ((base2 = this.options).quoted == null) {
|
||||
base2.quoted = false;
|
||||
}
|
||||
if ((base3 = this.options).quotedString == null) {
|
||||
base3.quotedString = false;
|
||||
}
|
||||
if ((base4 = this.options).eof == null) {
|
||||
base4.eof = true;
|
||||
}
|
||||
if ((base5 = this.options).escape == null) {
|
||||
base5.escape = '"';
|
||||
}
|
||||
if ((base6 = this.options).columns == null) {
|
||||
base6.columns = null;
|
||||
}
|
||||
if ((base7 = this.options).header == null) {
|
||||
base7.header = false;
|
||||
}
|
||||
if ((base8 = this.options).rowDelimiter == null) {
|
||||
base8.rowDelimiter = '\n';
|
||||
}
|
||||
if (this.countWriten == null) {
|
||||
this.countWriten = 0;
|
||||
}
|
||||
switch (this.options.rowDelimiter) {
|
||||
case 'auto':
|
||||
this.options.rowDelimiter = null;
|
||||
break;
|
||||
case 'unix':
|
||||
this.options.rowDelimiter = "\n";
|
||||
break;
|
||||
case 'mac':
|
||||
this.options.rowDelimiter = "\r";
|
||||
break;
|
||||
case 'windows':
|
||||
this.options.rowDelimiter = "\r\n";
|
||||
break;
|
||||
case 'unicode':
|
||||
this.options.rowDelimiter = "\u2028";
|
||||
}
|
||||
return this;
|
||||
};
|
||||
|
||||
util.inherits(Stringifier, stream.Transform);
|
||||
|
||||
module.exports.Stringifier = Stringifier;
|
||||
|
||||
Stringifier.prototype.headers = function() {
|
||||
var k, label, labels;
|
||||
if (!this.options.header) {
|
||||
return;
|
||||
}
|
||||
if (!this.options.columns) {
|
||||
return;
|
||||
}
|
||||
labels = this.options.columns;
|
||||
if (typeof labels === 'object') {
|
||||
labels = (function() {
|
||||
var results;
|
||||
results = [];
|
||||
for (k in labels) {
|
||||
label = labels[k];
|
||||
results.push(label);
|
||||
}
|
||||
return results;
|
||||
})();
|
||||
}
|
||||
if (this.options.eof) {
|
||||
labels = this.stringify(labels) + this.options.rowDelimiter;
|
||||
} else {
|
||||
labels = this.stringify(labels);
|
||||
}
|
||||
return stream.Transform.prototype.write.call(this, labels);
|
||||
};
|
||||
|
||||
Stringifier.prototype.end = function(chunk, encoding, callback) {
|
||||
if (this.countWriten === 0) {
|
||||
this.headers();
|
||||
}
|
||||
return stream.Transform.prototype.end.apply(this, arguments);
|
||||
};
|
||||
|
||||
Stringifier.prototype.write = function(chunk, encoding, callback) {
|
||||
var base, e, preserve;
|
||||
if (chunk == null) {
|
||||
return;
|
||||
}
|
||||
preserve = typeof chunk !== 'object';
|
||||
if (!preserve) {
|
||||
if (this.countWriten === 0 && !Array.isArray(chunk)) {
|
||||
if ((base = this.options).columns == null) {
|
||||
base.columns = Object.keys(chunk);
|
||||
}
|
||||
}
|
||||
try {
|
||||
this.emit('record', chunk, this.countWriten);
|
||||
} catch (_error) {
|
||||
e = _error;
|
||||
return this.emit('error', e);
|
||||
}
|
||||
if (this.options.eof) {
|
||||
chunk = this.stringify(chunk) + this.options.rowDelimiter;
|
||||
} else {
|
||||
chunk = this.stringify(chunk);
|
||||
if (this.options.header || this.countWriten) {
|
||||
chunk = this.options.rowDelimiter + chunk;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (typeof chunk === 'number') {
|
||||
chunk = "" + chunk;
|
||||
}
|
||||
if (this.countWriten === 0) {
|
||||
this.headers();
|
||||
}
|
||||
if (!preserve) {
|
||||
this.countWriten++;
|
||||
}
|
||||
return stream.Transform.prototype.write.call(this, chunk, encoding, callback);
|
||||
};
|
||||
|
||||
Stringifier.prototype._transform = function(chunk, encoding, callback) {
|
||||
this.push(chunk);
|
||||
return callback();
|
||||
};
|
||||
|
||||
Stringifier.prototype.stringify = function(line) {
|
||||
var _line, column, columns, containsLinebreak, containsQuote, containsdelimiter, delimiter, escape, field, i, j, l, newLine, quote, ref, ref1, regexp;
|
||||
if (typeof line !== 'object') {
|
||||
return line;
|
||||
}
|
||||
columns = this.options.columns;
|
||||
if (typeof columns === 'object' && columns !== null && !Array.isArray(columns)) {
|
||||
columns = Object.keys(columns);
|
||||
}
|
||||
delimiter = this.options.delimiter;
|
||||
quote = this.options.quote;
|
||||
escape = this.options.escape;
|
||||
if (!Array.isArray(line)) {
|
||||
_line = [];
|
||||
if (columns) {
|
||||
for (i = j = 0, ref = columns.length; 0 <= ref ? j < ref : j > ref; i = 0 <= ref ? ++j : --j) {
|
||||
column = columns[i];
|
||||
_line[i] = typeof line[column] === 'undefined' || line[column] === null ? '' : line[column];
|
||||
}
|
||||
} else {
|
||||
for (column in line) {
|
||||
_line.push(line[column]);
|
||||
}
|
||||
}
|
||||
line = _line;
|
||||
_line = null;
|
||||
} else if (columns) {
|
||||
line.splice(columns.length);
|
||||
}
|
||||
if (Array.isArray(line)) {
|
||||
newLine = '';
|
||||
for (i = l = 0, ref1 = line.length; 0 <= ref1 ? l < ref1 : l > ref1; i = 0 <= ref1 ? ++l : --l) {
|
||||
field = line[i];
|
||||
if (typeof field === 'string') {
|
||||
// No code here in original version.
|
||||
} else if (typeof field === 'number') {
|
||||
field = '' + field;
|
||||
} else if (typeof field === 'boolean') {
|
||||
field = field ? '1' : '';
|
||||
} else if (field instanceof Date) {
|
||||
//field = '' + field.getTime(); // In original version.
|
||||
field = field.getFullYear() + '-' + (field.getMonth() + 1) + '-' + field.getDate()
|
||||
+ ' ' + field.getHours() + ':' + field.getMinutes() + ':' + field.getSeconds();
|
||||
|
||||
} else if (typeof field === 'object' && field !== null) {
|
||||
field = JSON.stringify(field);
|
||||
}
|
||||
if (field) {
|
||||
containsdelimiter = field.indexOf(delimiter) >= 0;
|
||||
containsQuote = field.indexOf(quote) >= 0;
|
||||
containsLinebreak = field.indexOf('\r') >= 0 || field.indexOf('\n') >= 0;
|
||||
if (containsQuote) {
|
||||
regexp = new RegExp(quote, 'g');
|
||||
field = field.replace(regexp, escape + quote);
|
||||
}
|
||||
if (containsQuote || containsdelimiter || containsLinebreak || this.options.quoted || (this.options.quotedString && typeof line[i] === 'string')) {
|
||||
field = quote + field + quote;
|
||||
}
|
||||
newLine += field;
|
||||
} else if (this.options.quotedEmpty || ((this.options.quotedEmpty == null) && line[i] === '' && this.options.quotedString)) {
|
||||
newLine += quote + quote;
|
||||
}
|
||||
|
||||
if (i !== line.length - 1) {
|
||||
newLine += delimiter;
|
||||
}
|
||||
}
|
||||
line = newLine;
|
||||
}
|
||||
return line;
|
||||
};
|
File diff suppressed because it is too large
Load diff
33
migration/fmtp/Table.js
Normal file
33
migration/fmtp/Table.js
Normal file
|
@ -0,0 +1,33 @@
|
|||
/*
|
||||
* This file is a part of "NMIG" - the database migration tool.
|
||||
*
|
||||
* Copyright 2016 Anatoly Khaytovich <anatolyuss@gmail.com>
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with this program (please see the "LICENSE.md" file).
|
||||
* If not, see <http://www.gnu.org/licenses/gpl.txt>.
|
||||
*
|
||||
* @author Anatoly Khaytovich <anatolyuss@gmail.com>
|
||||
*/
|
||||
'use strict';
|
||||
|
||||
/**
|
||||
* This function represents table related metadata.
|
||||
* Constructor.
|
||||
*
|
||||
* @param {String} tableLogPath
|
||||
*/
|
||||
module.exports = function Table(tableLogPath) {
|
||||
this.tableLogPath = tableLogPath;
|
||||
this.arrTableColumns = [];
|
||||
this.totalRowsInserted = 0;
|
||||
};
|
|
@ -20,13 +20,6 @@
|
|||
*/
|
||||
'use strict';
|
||||
|
||||
/**
|
||||
* Constructor.
|
||||
*/
|
||||
function ViewGenerator() {
|
||||
// No code should be put here.
|
||||
}
|
||||
|
||||
/**
|
||||
* Attempts to convert MySQL view to PostgreSQL view.
|
||||
*
|
||||
|
@ -35,13 +28,13 @@ function ViewGenerator() {
|
|||
* @param {String} mysqlViewCode
|
||||
* @returns {String}
|
||||
*/
|
||||
ViewGenerator.prototype.generateView = function(schema, viewName, mysqlViewCode) {
|
||||
module.exports = function(schema, viewName, mysqlViewCode) {
|
||||
mysqlViewCode = mysqlViewCode.split('`').join('"');
|
||||
let queryStart = mysqlViewCode.indexOf('AS');
|
||||
mysqlViewCode = mysqlViewCode.slice(queryStart);
|
||||
let arrMysqlViewCode = mysqlViewCode.split(' ');
|
||||
|
||||
for (let i = 0; i < arrMysqlViewCode.length; i++) {
|
||||
for (let i = 0; i < arrMysqlViewCode.length; ++i) {
|
||||
if (
|
||||
arrMysqlViewCode[i].toLowerCase() === 'from'
|
||||
|| arrMysqlViewCode[i].toLowerCase() === 'join'
|
||||
|
@ -53,5 +46,3 @@ ViewGenerator.prototype.generateView = function(schema, viewName, mysqlViewCode)
|
|||
|
||||
return 'CREATE OR REPLACE VIEW "' + schema + '"."' + viewName + '" ' + arrMysqlViewCode.join(' ') + ';';
|
||||
};
|
||||
|
||||
module.exports.ViewGenerator = ViewGenerator;
|
||||
|
|
39
nmig.js
Normal file
39
nmig.js
Normal file
|
@ -0,0 +1,39 @@
|
|||
/*
|
||||
* This file is a part of "NMIG" - the database migration tool.
|
||||
*
|
||||
* Copyright 2016 Anatoly Khaytovich <anatolyuss@gmail.com>
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with this program (please see the "LICENSE.md" file).
|
||||
* If not, see <http://www.gnu.org/licenses/gpl.txt>.
|
||||
*
|
||||
* @author Anatoly Khaytovich <anatolyuss@gmail.com>
|
||||
*/
|
||||
'use strict';
|
||||
|
||||
const fs = require('fs');
|
||||
const fmtp = require('./migration/fmtp/FromMySQL2PostgreSQL');
|
||||
|
||||
fs.readFile(__dirname + '/config.json', (error, data) => {
|
||||
if (error) {
|
||||
console.log('\n\t--Cannot run migration\nCannot read configuration info from ' + __dirname + '/config.json');
|
||||
} else {
|
||||
try {
|
||||
let config = JSON.parse(data.toString());
|
||||
config.tempDirPath = __dirname + '/temporary_directory';
|
||||
config.logsDirPath = __dirname + '/logs_directory';
|
||||
fmtp(config);
|
||||
} catch (err) {
|
||||
console.log('\n\t--Cannot parse JSON from' + __dirname + '/config.json');
|
||||
}
|
||||
}
|
||||
});
|
Loading…
Reference in a new issue