Question

我正在尝试使用nodejs rest服务从mongodb集合中获取巨大的json数据。但是在获取数据时出现此错误：

<---后几个GC --->

[8744：00000000003A9550] 164892毫秒：标记扫描1407.6（1468.2）-> 1407.5（1437.2）MB，2510.7 / 0.0毫秒，要求在旧空间中使用最后一个GC
  [8744：00000000003A9550] 167480毫秒：标记扫描1407.5（1437.2）-> 1407.5（1436.7）MB，2587.7 / 0.0毫秒，要求在旧空间中使用最后一个GC

<--- JS堆栈跟踪--->

==== JS堆栈跟踪======================================== =

安全上下文：000000E816AA5879
  1：_writeGeneric [net.js：755] [bytecode = 000001F3DEF03FC1 offset = 351]（this = 000003C99D221C61，writev = 000001EC53F02371，data = 0000010D3E5FBCA9，encoding = 000001EC53F02431，cb = 000003C99D221F59）           2：_writev [net.js：778] [字节码= 0000000C48D59649 offset = 23]（this = 000003C99D221C ...

FATAL ERROR: CALL_AND_RETRY_LAST Allocation failed - JavaScript heap out of memory
 1: node_module_register
 2: v8::internal::FatalProcessOutOfMemory
 3: v8::internal::FatalProcessOutOfMemory
 4: v8::internal::Factory::NewRawOneByteString
 5: v8::internal::Smi::SmiPrint
 6: v8::internal::StackGuard::HandleInterrupts
 7: v8::String::WriteUtf8
 8: v8::internal::wasm::AsmType::Extern
 9: v8::internal::wasm::AsmType::Extern
10: v8::internal::wasm::AsmType::Extern
11: v8::internal::wasm::SignatureMap::Find
12: v8::internal::Builtins::CallableFor
13: v8::internal::Builtins::CallableFor
14: v8::internal::Builtins::CallableFor
15: 000000BC8E7843C1

D:\new>node --max_old_space_size=2000 service.js
23-10-2018 07-08-46.1046 - info: log information extraData=somve_value

节点服务：

var getFiletredTimestamp = function(alarm , fromDate , toDate){
    var resultData = [];
    for(var i=0;i<alarm.length;i++){
        var obj = alarm[i];
        var x;
        for( x in obj['alarms'] ){
             if(new Date(obj['alarms'][x]['timestamp']).getTime()>=new Date(fromDate).getTime() && new Date(obj['alarms'][x]['timestamp']).getTime()<=new Date(toDate).getTime()){
                 resultData.push(obj);
                 break;
             }
       }
    }
    //console.log(resultData);
    console.log(resultData.length);
    return resultData;
};

function getAlarmHistory(req,res){
   var queryObject = url.parse(req.url,true).query;
   db.collection("HISTORY").find({},{"no":1,"type":1,"ID":1,"id":1,"ptype":1,"alarms":1}).toArray(function (err , result){
       console.log(result.length);
       var index = 0 , resultset = [];
       var docs = getFiletredTimestamp(result , queryObject.fromdate, queryObject.todate);
        var count = docs.length;
        console.log("count " , count);
        if(count==0){
                console.log("resulttttttttt",resultset);

                res.writeHead(200, {
                    'Content-Type': 'application/json'
                });
                res.write('');
                res.end();  
            }else{
       docs.forEach(function(doc){
           console.log()
               if (doc!=null){
                var valdata=doc.alarms;
                var fields = [];
                var queryString ="SELECT datafield1,LOC_NAME FROM details c inner join ldetails l on c.loc_id=l.loc_id where no='"+doc.no+"' limit 1;";
               var dtfield1 = null;
               var dtfield2 = null;
                connection.query(queryString, function(err,result){
                           index++;
                           console.log("result query" , result);
                        if(err){
                            console.log("err",err);
                        }else{
                            if(result.length>0)
                                {
                                     dtfield1 = result[0].datafield1;
                                     dtfield2 = result[0].LOC_NAME;
                                     if(dtfield1!=null){
                                         for (var x in valdata) {
                                            resultset.push({"Name":dtfield1,"LName":dtfield2,"no":doc.no,"type":doc.type,"ID":doc.ID,"aType":x,"aValue":valdata[x].value,"occurTimestamp":valdata[x].occurTimestamp,"Timestamp":valdata[x].timestamp});
                                         }} 

                                         }
                                          if(index == count){
                                                console.log("resulttttttttt",resultset);

                                                res.writeHead(200, {
                                                    'Content-Type': 'application/json'
                                                });
                                                res.write(JSON.stringify(resultset));
                                                res.end();
                                            }
                                         }
                    }); 
   }
       });}

   })

}

如何处理错误。请帮忙。

Answer 1

使用分页方法，一次仅处理小块数据，或流式传输结果： http://mongodb.github.io/node-mongodb-native/2.0/tutorials/streams/

来自NodeJS中MongoDB的Json数据

1 个答案: