我编写了一个简单的节点程序来解析从公司ERP返回的excel格式化HTML表，提取数据并将其另存为JSON。
这使用FS打开文件，使用Cheerio来提取数据。
该程序适用于小文件（<10MB）但大型文件需要很长时间（> 30MB）
我遇到问题的数据文件是38MB，有大约30,000行数据。
问题1：这不应该更快吗？ 问题2：我只能输出一个console.log语句。我可以在任何地方放置一个语句并且它可以工作，如果我添加多个，只有第一个输出任何东西。
var fs = require('fs'); // for file system streaming
function oracleParse(file, callback) {
var headers = []; // array to store the data table column headers
var myError; // module error holder
var XMLdata = []; // array to store the parsed XML data to be returned
var cheerio = require('cheerio');
// open relevant file
var reader = fs.readFile(file, function (err, data) {
if (err) {
myError = err; // catch errors returned from file open
} else {
$ = cheerio.load(data); // load data returned from fs into cheerio for parsing
// the data retruned from Oracle consists of a variable number of tables however the last one is
// always the one that contains the data. We can select this with cheerio and reset the cherrio $ object
var dataTable = $('table').last();
$ = cheerio.load(dataTable);
// table column headers in the table of data returned from Oracle include headers under 'tr td b' elements
// We extract these headers and load these into the 'headers' array for future use as keys in the JSON
// data array to be constucted
$('tr td b').each(function (i, elem) {
headers.push($(this).text());
});
// remove the headers from the cheerio data object so that they don't interfere with the data
$('tr td b').remove();
// for the actual data, each row of data (this corresponds to a customer, account, transation record etc) is
// extracted using cheerio and stored in a key/value object. These objects are then stored in an array
var dataElements = [];
var dataObj = {};
var headersLength = headers.length;
var headerNum;
// the actual data is returned from Oracle in 'tr td nobr' elements. Using cheerio, we can extract all of
// these elements although they are not separated into individual rows. It is possible to return individual
// rows using cheeris (e.g. 'tr') but this is very slow as cheerio needs to requery each subsequent row.
// In our case, we simply select all data elements using the 'tr td nobr' selector and then iterate through
// them, aligning them with the relevant key and grouping them into relevant rows by taking the modulus of
// the element number returned and the number of headers there are.
$('tr td nobr').each(function (i, elem) {
headerNum = i % headersLength; // pick which column is associated with each element
dataObj[headers[headerNum]] = $(this).text(); // build the row object
// if we find the header number is equal to the header length less one, we have reached the end of
// elements for the row and push the row object onto the array in which we store the final result
if (headerNum === headersLength - 1) {
XMLdata.push(dataObj);
dataObj = {};
}
});
console.log(headersLength);
// once all the data in the file has been parsed, run the call back function passed in
callback(JSON.stringify(XMLdata));
}
});
return myError;
}
// parse promo dates data
var file = './data/Oracle/signups_01.html';
var output = './data/Oracle/signups_01.JSON';
//var file = './data/Oracle/detailed_data.html';
//var output = './data/Oracle/detailed_data.JSON';
var test = oracleParse(file, function(data) {
fs.writeFile(output, data, function(err) {
if (err) throw err;
console.log('File write complete: ' + output);
});
});
console.log(test);