oreolek/nmig
1
0
Fork 0
Code Releases Activity

prevent the "invalid byte sequence for encoding UTF8: 0x00" issue

Browse source
This commit is contained in:
AnatolyUss 2017-03-25 15:19:48 +03:00
parent 6e32b736c0
commit c97a026b88
2 changed files with 21 additions and 5 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

24
migration/fmtp/CsvStringifyModified.js
View file

@ -8,11 +8,23 @@
util = require('util'); util = require('util');
module.exports = function() { module.exports = function() {
var callback, chunks, data, options, stringifier; var callback, chunks, data, options, stringifier, encoding;
if (arguments.length === 3) { if (arguments.length === 3) {
data = arguments[0]; // Original version.
/*data = arguments[0];
options = arguments[1]; options = arguments[1];
callback = arguments[2]; callback = arguments[2];*/
if (Array.isArray(arguments[0])) {
data = arguments[0];
} else {
options = arguments[0];
}
if (typeof arguments[1] === 'function') {
callback = arguments[1];
} else {
options = arguments[1];
}
encoding = arguments[2];
} else if (arguments.length === 2) { } else if (arguments.length === 2) {
if (Array.isArray(arguments[0])) { if (Array.isArray(arguments[0])) {
data = arguments[0]; data = arguments[0];
@ -36,7 +48,10 @@
if (options == null) { if (options == null) {
options = {}; options = {};
} }
stringifier = new Stringifier(options); stringifier = new Stringifier(options);
stringifier._encoding = encoding;
if (data) { if (data) {
process.nextTick(function() { process.nextTick(function() {
var d, j, len; var d, j, len;
@ -254,6 +269,7 @@
field = JSON.stringify(field); field = JSON.stringify(field);
} }
if (field) { if (field) {
field = this._encoding === 'utf8' ? field.replace(/\u0000/g, ' ') : field;
containsdelimiter = field.indexOf(delimiter) >= 0; containsdelimiter = field.indexOf(delimiter) >= 0;
containsQuote = field.indexOf(quote) >= 0; containsQuote = field.indexOf(quote) >= 0;
containsLinebreak = field.indexOf('\r') >= 0 || field.indexOf('\n') >= 0; containsLinebreak = field.indexOf('\r') >= 0 || field.indexOf('\n') >= 0;
@ -268,7 +284,7 @@
} else if (this.options.quotedEmpty || ((this.options.quotedEmpty == null) && line[i] === '' && this.options.quotedString)) { } else if (this.options.quotedEmpty || ((this.options.quotedEmpty == null) && line[i] === '' && this.options.quotedString)) {
newLine += quote + quote; newLine += quote + quote;
} }
if (i !== line.length - 1) { if (i !== line.length - 1) {
newLine += delimiter; newLine += delimiter;
} }

2
migration/fmtp/DataLoader.js
View file

@ -285,7 +285,7 @@ const populateTableWorker = (self, tableName, strSelectFieldList, offset, rowsIn
} }
}); });
} }
}); }, self._encoding);
} }
}); });
} }