我有以下代码,使用fast-csv解析2GB csv文件并插入MongoDB。
当Pipe为大约17K +记录执行此代码时,Nodej消耗所有内存并停止并显示错误:
"CALL_AND_RETRY_LAST Allocation failed - process out of memory"
我试图为变量设置null以避免可能的泄漏。
我使用的是Mongoose最新版本和Node.js 0.12 ..
PS:如果我评论p.save()方法并运行此代码,它的执行没有任何内存泄漏。我检查了5百万条记录和它的好......
所以我怀疑mongoose保存功能可能存在内存泄漏。
var csv = require( "fast-csv" );
var fs = require( "fs" );
var DBMong = require( "./db-mongoose.js" );
function processCSVnImportDB( CSVFilePath , callback )
{
var rowcounter = 0;
var dbObj = DB.getDBObject( ); //Mongoose connection object
var stream = fs.createReadStream( CSVFilePath );
var csvStream;
csvStream = csv( )
.on( "data",
function ( data )
{
csvStream.pause( );
try
{
rowcounter++;
var p = new DBMong.ProductModel; //Mongoose Schema Object
var su = new models.supplierinfo();
su.cd = "FK";
p._id = "FK-" + data.productId;
p.supp = [];
p.supp.push( su );
p.save(
function ( err,res )
{
if ( err )
{
if ( err.code == "11000" )
{
console.log( CSVFilePath + "- duplicate ID - " );
} else
console.log( CSVFilePath + "- save error - " + err );
} else
{
console.log( CSVFilePath + "- save success " );
}
su = null;
p = null;
item = null;
data = null;
csvStream.resume( );
}
);
}
catch ( exc )
{
errcounter++;
console.log( exc.stack );
console.log( "ERROR: Filename:" + CSVFilePath + errcounter.toString( ) );
csvStream.resume( );
}
} )
.on( 'error', function ( error )
{
console.log( "ERROR: " + error.stack );
})
.on( "end", function ()
{
console.log( "Total ERROR: " + errcounter.toString( ) );
console.log( "done" );
});
stream.pipe( csvStream );
}