NMIG reconstruction, ram consumption reduction, bug fixes

config.json

@@ -36,6 +36,12 @@
         "Maximal amount of simultaneous connections to your PostgreSQL database during migration"
     ],
     "max_pool_size_target" : 10,
+
+	"pipe_width_description" : [
+        "Maximal amount of data-chunks, processed in a single 'DataPipe' iteration.",
+        "Note: 'pipe_width' should not be greater than 'max_pool_size_target'."
+    ],
+    "pipe_width" : 10,
     
     "encoding_description" : [
         "JavaScript encoding type.",
@@ -47,7 +53,7 @@
         "schema - a name of the schema, that will contain all migrated tables.",
         "If not supplied, then a new schema will be created automatically."
     ],
-    "schema" : "test_schema",
+    "schema" : "public",
     
     "data_chunk_size_description" : [
         "During migration each table's data will be split into chunks of data_chunk_size (in KB).",

migration/fmtp/ColumnsDataArranger.js

@@ -29,7 +29,7 @@
 module.exports = function(arrTableColumns) {
     let strRetVal = '';
 
-    for (let i = 0; i < arrTableColumns.length; i++) {
+    for (let i = 0; i < arrTableColumns.length; ++i) {
         if (
             arrTableColumns[i].Type.indexOf('geometry') !== -1
             || arrTableColumns[i].Type.indexOf('point') !== -1
@@ -47,9 +47,8 @@ module.exports = function(arrTableColumns) {
         ) {
             strRetVal += 'BIN(`' + arrTableColumns[i].Field + '`),';
         } else if (
-            arrTableColumns[i].Type.indexOf('date') !== -1
-            || arrTableColumns[i].Type.indexOf('datetime') !== -1
-            || arrTableColumns[i].Type.indexOf('timestamp') !== -1
+            arrTableColumns[i].Type.indexOf('timestamp') !== -1
+            || arrTableColumns[i].Type.indexOf('date') !== -1
         ) {
             strRetVal += 'IF(`' + arrTableColumns[i].Field
                       +  '` IN(\'0000-00-00\', \'0000-00-00 00:00:00\'), \'-INFINITY\', `'

migration/fmtp/CsvStringifyModified.js

@@ -0,0 +1,279 @@
+/*
+ * This is a copy of "csv-stringify" module, that was slightly modified.
+ */
+ var Stringifier, stream, util;
+
+ stream = require('stream');
+
+ util = require('util');
+
+ module.exports = function() {
+   var callback, chunks, data, options, stringifier;
+   if (arguments.length === 3) {
+     data = arguments[0];
+     options = arguments[1];
+     callback = arguments[2];
+   } else if (arguments.length === 2) {
+     if (Array.isArray(arguments[0])) {
+       data = arguments[0];
+     } else {
+       options = arguments[0];
+     }
+     if (typeof arguments[1] === 'function') {
+       callback = arguments[1];
+     } else {
+       options = arguments[1];
+     }
+   } else if (arguments.length === 1) {
+     if (typeof arguments[0] === 'function') {
+       callback = arguments[0];
+     } else if (Array.isArray(arguments[0])) {
+       data = arguments[0];
+     } else {
+       options = arguments[0];
+     }
+   }
+   if (options == null) {
+     options = {};
+   }
+   stringifier = new Stringifier(options);
+   if (data) {
+     process.nextTick(function() {
+       var d, j, len;
+       for (j = 0, len = data.length; j < len; j++) {
+         d = data[j];
+         stringifier.write(d);
+       }
+       return stringifier.end();
+     });
+   }
+   if (callback) {
+     chunks = [];
+     stringifier.on('readable', function() {
+       var chunk, results;
+       results = [];
+       while (chunk = stringifier.read()) {
+         results.push(chunks.push(chunk));
+       }
+       return results;
+     });
+     stringifier.on('error', function(err) {
+       return callback(err);
+     });
+     stringifier.on('end', function() {
+       return callback(null, chunks.join(''));
+     });
+   }
+   return stringifier;
+ };
+
+ Stringifier = function(options) {
+   var base, base1, base2, base3, base4, base5, base6, base7, base8;
+   if (options == null) {
+     options = {};
+   }
+   stream.Transform.call(this, options);
+   this.options = options;
+   if ((base = this.options).delimiter == null) {
+     base.delimiter = ',';
+   }
+   if ((base1 = this.options).quote == null) {
+     base1.quote = '"';
+   }
+   if ((base2 = this.options).quoted == null) {
+     base2.quoted = false;
+   }
+   if ((base3 = this.options).quotedString == null) {
+     base3.quotedString = false;
+   }
+   if ((base4 = this.options).eof == null) {
+     base4.eof = true;
+   }
+   if ((base5 = this.options).escape == null) {
+     base5.escape = '"';
+   }
+   if ((base6 = this.options).columns == null) {
+     base6.columns = null;
+   }
+   if ((base7 = this.options).header == null) {
+     base7.header = false;
+   }
+   if ((base8 = this.options).rowDelimiter == null) {
+     base8.rowDelimiter = '\n';
+   }
+   if (this.countWriten == null) {
+     this.countWriten = 0;
+   }
+   switch (this.options.rowDelimiter) {
+     case 'auto':
+       this.options.rowDelimiter = null;
+       break;
+     case 'unix':
+       this.options.rowDelimiter = "\n";
+       break;
+     case 'mac':
+       this.options.rowDelimiter = "\r";
+       break;
+     case 'windows':
+       this.options.rowDelimiter = "\r\n";
+       break;
+     case 'unicode':
+       this.options.rowDelimiter = "\u2028";
+   }
+   return this;
+ };
+
+ util.inherits(Stringifier, stream.Transform);
+
+ module.exports.Stringifier = Stringifier;
+
+ Stringifier.prototype.headers = function() {
+   var k, label, labels;
+   if (!this.options.header) {
+     return;
+   }
+   if (!this.options.columns) {
+     return;
+   }
+   labels = this.options.columns;
+   if (typeof labels === 'object') {
+     labels = (function() {
+       var results;
+       results = [];
+       for (k in labels) {
+         label = labels[k];
+         results.push(label);
+       }
+       return results;
+     })();
+   }
+   if (this.options.eof) {
+     labels = this.stringify(labels) + this.options.rowDelimiter;
+   } else {
+     labels = this.stringify(labels);
+   }
+   return stream.Transform.prototype.write.call(this, labels);
+ };
+
+ Stringifier.prototype.end = function(chunk, encoding, callback) {
+   if (this.countWriten === 0) {
+     this.headers();
+   }
+   return stream.Transform.prototype.end.apply(this, arguments);
+ };
+
+ Stringifier.prototype.write = function(chunk, encoding, callback) {
+   var base, e, preserve;
+   if (chunk == null) {
+     return;
+   }
+   preserve = typeof chunk !== 'object';
+   if (!preserve) {
+     if (this.countWriten === 0 && !Array.isArray(chunk)) {
+       if ((base = this.options).columns == null) {
+         base.columns = Object.keys(chunk);
+       }
+     }
+     try {
+       this.emit('record', chunk, this.countWriten);
+     } catch (_error) {
+       e = _error;
+       return this.emit('error', e);
+     }
+     if (this.options.eof) {
+       chunk = this.stringify(chunk) + this.options.rowDelimiter;
+     } else {
+       chunk = this.stringify(chunk);
+       if (this.options.header || this.countWriten) {
+         chunk = this.options.rowDelimiter + chunk;
+       }
+     }
+   }
+   if (typeof chunk === 'number') {
+     chunk = "" + chunk;
+   }
+   if (this.countWriten === 0) {
+     this.headers();
+   }
+   if (!preserve) {
+     this.countWriten++;
+   }
+   return stream.Transform.prototype.write.call(this, chunk, encoding, callback);
+ };
+
+ Stringifier.prototype._transform = function(chunk, encoding, callback) {
+   this.push(chunk);
+   return callback();
+ };
+
+ Stringifier.prototype.stringify = function(line) {
+   var _line, column, columns, containsLinebreak, containsQuote, containsdelimiter, delimiter, escape, field, i, j, l, newLine, quote, ref, ref1, regexp;
+   if (typeof line !== 'object') {
+     return line;
+   }
+   columns = this.options.columns;
+   if (typeof columns === 'object' && columns !== null && !Array.isArray(columns)) {
+     columns = Object.keys(columns);
+   }
+   delimiter = this.options.delimiter;
+   quote = this.options.quote;
+   escape = this.options.escape;
+   if (!Array.isArray(line)) {
+     _line = [];
+     if (columns) {
+       for (i = j = 0, ref = columns.length; 0 <= ref ? j < ref : j > ref; i = 0 <= ref ? ++j : --j) {
+         column = columns[i];
+         _line[i] = typeof line[column] === 'undefined' || line[column] === null ? '' : line[column];
+       }
+     } else {
+       for (column in line) {
+         _line.push(line[column]);
+       }
+     }
+     line = _line;
+     _line = null;
+   } else if (columns) {
+     line.splice(columns.length);
+   }
+   if (Array.isArray(line)) {
+     newLine = '';
+     for (i = l = 0, ref1 = line.length; 0 <= ref1 ? l < ref1 : l > ref1; i = 0 <= ref1 ? ++l : --l) {
+       field = line[i];
+       if (typeof field === 'string') {
+          // No code here in original version.
+       } else if (typeof field === 'number') {
+         field = '' + field;
+       } else if (typeof field === 'boolean') {
+         field = field ? '1' : '';
+       } else if (field instanceof Date) {
+         //field = '' + field.getTime(); // In original version.
+         field = field.getFullYear() + '-' + (field.getMonth() + 1) + '-' + field.getDate()
+               + ' ' + field.getHours() + ':' + field.getMinutes() + ':' + field.getSeconds();
+
+       } else if (typeof field === 'object' && field !== null) {
+         field = JSON.stringify(field);
+       }
+       if (field) {
+         containsdelimiter = field.indexOf(delimiter) >= 0;
+         containsQuote = field.indexOf(quote) >= 0;
+         containsLinebreak = field.indexOf('\r') >= 0 || field.indexOf('\n') >= 0;
+         if (containsQuote) {
+           regexp = new RegExp(quote, 'g');
+           field = field.replace(regexp, escape + quote);
+         }
+         if (containsQuote || containsdelimiter || containsLinebreak || this.options.quoted || (this.options.quotedString && typeof line[i] === 'string')) {
+           field = quote + field + quote;
+         }
+         newLine += field;
+       } else if (this.options.quotedEmpty || ((this.options.quotedEmpty == null) && line[i] === '' && this.options.quotedString)) {
+         newLine += quote + quote;
+       }
+       
+       if (i !== line.length - 1) {
+         newLine += delimiter;
+       }
+     }
+     line = newLine;
+   }
+   return line;
+ };

migration/fmtp/Table.js

@@ -0,0 +1,33 @@
+/*
+ * This file is a part of "NMIG" - the database migration tool.
+ *
+ * Copyright 2016 Anatoly Khaytovich <anatolyuss@gmail.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program (please see the "LICENSE.md" file).
+ * If not, see <http://www.gnu.org/licenses/gpl.txt>.
+ *
+ * @author Anatoly Khaytovich <anatolyuss@gmail.com>
+ */
+'use strict';
+
+/**
+ * This function represents table related metadata.
+ * Constructor.
+ *
+ * @param {String} tableLogPath
+ */
+module.exports = function Table(tableLogPath) {
+    this.tableLogPath      = tableLogPath;
+    this.arrTableColumns   = [];
+    this.totalRowsInserted = 0;
+};

migration/fmtp/ViewGenerator.js

@@ -20,13 +20,6 @@
  */
 'use strict';
 
-/**
- * Constructor.
- */
-function ViewGenerator() {
-    // No code should be put here.
-}
-
 /**
  * Attempts to convert MySQL view to PostgreSQL view.
  *
@@ -35,13 +28,13 @@ function ViewGenerator() {
  * @param   {String} mysqlViewCode
  * @returns {String}
  */
-ViewGenerator.prototype.generateView = function(schema, viewName, mysqlViewCode) {
+module.exports = function(schema, viewName, mysqlViewCode) {
     mysqlViewCode        = mysqlViewCode.split('`').join('"');
     let queryStart       = mysqlViewCode.indexOf('AS');
     mysqlViewCode        = mysqlViewCode.slice(queryStart);
     let arrMysqlViewCode = mysqlViewCode.split(' ');
-
-    for (let i = 0; i < arrMysqlViewCode.length; i++) {
+    
+    for (let i = 0; i < arrMysqlViewCode.length; ++i) {
         if (
             arrMysqlViewCode[i].toLowerCase() === 'from'
             || arrMysqlViewCode[i].toLowerCase() === 'join'
@@ -53,5 +46,3 @@ ViewGenerator.prototype.generateView = function(schema, viewName, mysqlViewCode)
 
     return 'CREATE OR REPLACE VIEW "' + schema + '"."' + viewName + '" ' + arrMysqlViewCode.join(' ') + ';';
 };
-
-module.exports.ViewGenerator = ViewGenerator;

nmig.js

@@ -0,0 +1,39 @@
+/*
+ * This file is a part of "NMIG" - the database migration tool.
+ *
+ * Copyright 2016 Anatoly Khaytovich <anatolyuss@gmail.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program (please see the "LICENSE.md" file).
+ * If not, see <http://www.gnu.org/licenses/gpl.txt>.
+ *
+ * @author Anatoly Khaytovich <anatolyuss@gmail.com>
+ */
+'use strict';
+
+const fs   = require('fs');
+const fmtp = require('./migration/fmtp/FromMySQL2PostgreSQL');
+
+fs.readFile(__dirname + '/config.json', (error, data) => {
+    if (error) {
+        console.log('\n\t--Cannot run migration\nCannot read configuration info from ' + __dirname + '/config.json');
+    } else {
+        try {
+            let config         = JSON.parse(data.toString());
+            config.tempDirPath = __dirname + '/temporary_directory';
+            config.logsDirPath = __dirname + '/logs_directory';
+            fmtp(config);
+        } catch (err) {
+            console.log('\n\t--Cannot parse JSON from' + __dirname + '/config.json');
+        }
+    }
+});