Mongoose Node.js中的内存泄漏

时间:2015-04-24 14:04:16

标签: node.js mongodb memory-leaks mongoose

我有以下代码,使用fast-csv解析2GB csv文件并插入MongoDB。

当Pipe为大约17K +记录执行此代码时,Nodej消耗所有内存并停止并显示错误:

"CALL_AND_RETRY_LAST Allocation failed - process out of memory"

我试图为变量设置null以避免可能的泄漏。

我使用的是Mongoose最新版本和Node.js 0.12 ..

PS:如果我评论p.save()方法并运行此代码,它的执行没有任何内存泄漏。我检查了5百万条记录和它的好......

所以我怀疑mongoose保存功能可能存在内存泄漏。

var csv = require( "fast-csv" );
var fs = require( "fs" );
var DBMong = require( "./db-mongoose.js" );

function processCSVnImportDB( CSVFilePath , callback )
{

    var rowcounter = 0;
    var dbObj = DB.getDBObject( ); //Mongoose connection object

    var stream = fs.createReadStream( CSVFilePath );
    var csvStream;
    csvStream = csv( )
        .on( "data",
         function ( data )
    {
        csvStream.pause( );
        try
        {
            rowcounter++;

            var p = new DBMong.ProductModel; //Mongoose Schema Object
            var su = new models.supplierinfo();
            su.cd = "FK";  
            p._id = "FK-" + data.productId;
            p.supp = [];
            p.supp.push( su );

            p.save(
                function ( err,res )
                {
                    if ( err )
                    {
                        if ( err.code == "11000" )
                        {
                            console.log( CSVFilePath + "- duplicate ID - "   );
                        } else
                            console.log( CSVFilePath + "- save error  - " + err );
                    } else
                    {
                                console.log( CSVFilePath + "- save success " );
                    }

                    su = null;
                    p = null;
                    item = null;
                    data = null;

                    csvStream.resume( );
                }
            ); 
        }
        catch ( exc )
        {
            errcounter++;
            console.log( exc.stack );
            console.log( "ERROR: Filename:" + CSVFilePath + errcounter.toString( ) );
            csvStream.resume( );
        }

    } )
       .on( 'error', function ( error )
    {
        console.log( "ERROR: " + error.stack );
    })
    .on( "end", function ()
    {
        console.log( "Total ERROR: " + errcounter.toString( ) );
        console.log( "done" );
    });
    stream.pipe( csvStream );
}

0 个答案:

没有答案