Node JS异步库系列

时间:2014-12-23 07:13:50

标签: javascript node.js node-async

如果我的记录少于4000,以下代码可以正常工作。

async.eachSeries(results, function (item, innerCallback) {

                        if (item[lang]) {
                            document = item[lang];
                            document._id = item._id;
                            documents.push(document);
                            innerCallback();
                        }else{
                            innerCallback();
                        }


                    }, function (err) {

                        if (err) {
                            console.log('Error in processing')
                            return callback(err);
                        } else {

                            return callback(null, documents);
                        }

                    });

如果我有超过4000条记录,则会出现以下错误。

"domain": {
    "domain": null,
    "_events": {},
    "_maxListeners": 10,
    "members": [{
        "_readableState": {
            "highWaterMark": 16384,
            "buffer": [],
            "length": 0,
            "pipes": null,
            "pipesCount": 0,
            "flowing": false,
            "ended": true,
            "endEmitted": true,
            "reading": false,
            "calledRead": true,
            "sync": false,
            "needReadable": true,
            "emittedReadable": false,
            "readableListening": false,
            "objectMode": false,
            "defaultEncoding": "utf8",
            "ranOut": false,
            "awaitDrain": 0,
            "readingMore": false,
            "oldMode": true,
            "decoder": null,
            "encoding": null
        },
        "readable": false,
        "domain": "[Circular]",
        "_events": {},
        "_maxListeners": 1000,
        "socket": {
            "_connecting": false,
            "_handle": {
                "fd": null,
                "writeQueueSize": 0,
                "owner": "[Circular]",
                "reading": true
            },
            "_readableState": {
                "highWaterMark": 16384,
                "buffer": [],
                "length": 0,
                "pipes": null,
                "pipesCount": 0,
                "flowing": false,
                "ended": false,
                "endEmitted": false,
                "reading": true,
                "calledRead": true,
                "sync": false,
                "needReadable": true,
                "emittedReadable": false,
                "readableListening": false,
                "objectMode": false,
                "defaultEncoding": "utf8",
                "ranOut": false,
                "awaitDrain": 0,
                "readingMore": false,
                "oldMode": false,
                "decoder": null,
                "encoding": null
            },
            "readable": true,
            "domain": null,
            "_events": {
                "drain": [null, null],
                "close": [null, null]
            },
            "_maxListeners": 10,
            "_writableState": {
                "highWaterMark": 16384,
                "objectMode": false,
                "needDrain": false,
                "ending": false,
                "ended": false,
                "finished": false,
                "decodeStrings": false,
                "defaultEncoding": "utf8",
                "length": 0,
                "writing": false,
                "sync": true,
                "bufferProcessing": false,
                "writecb": null,
                "writelen": 0,
                "buffer": [],
                "errorEmitted": false
            },
            "writable": true,
            "allowHalfOpen": true,
            "destroyed": false,
            "bytesRead": 423,
            "_bytesDispatched": 0,
            "_pendingData": null,
            "_pendingEncoding": "",
            "server": {
                "domain": null,
                "_events": {
                    "connection": [null, null],
                    "clientError": [null, null],
                    "listening": [null]
                },
                "_maxListeners": 10,
                "_connections": 2,
                "connections": 2,
                "_handle": {
                    "fd": null,
                    "writeQueueSize": 0,
                    "owner": "[Circular]"
                },
                "_usingSlaves": false,
                "_slaves": [],
                "allowHalfOpen": true,
                "httpAllowHalfOpen": false,
                "timeout": 120000,
                "_connectionKey": "4:0.0.0.0:8500"
            },
            "_idleTimeout": 120000,
            "_idleNext": {
                "_idleNext": {
                    "_connecting": false,
                    "_handle": {
                        "fd": null,
                        "writeQueueSize": 0,
                        "owner": "[Circular]",
                        "reading": true
                    },
                    "_readableState": {
                        "highWaterMark": 16384,
                        "buffer": [],
                        "length": 0,
                        "pipes": null,
                        "pipesCount": 0,
                        "flowing": false,
                        "ended": false,
                        "endEmitted": false,
                        "reading": true,
                        "calledRead": true,
                        "sync": false,
                        "needReadable": true,
                        "emittedReadable": false,
                        "readableListening": false,
                        "objectMode": false,
                        "defaultEncoding": "utf8",
                        "ranOut": false,
                        "awaitDrain": 0,
                        "readingMore": false,
                        "oldMode": false,
                        "decoder": null,
                        "encoding": null
                    },
                    "readable": true,
                    "domain": null,
                    "_events": {
                        "drain": [null, null]
                    },
                    "_maxListeners": 10,
                    "_writableState": {
                        "highWaterMark": 16384,
                        "objectMode": false,
                        "needDrain": false,
                        "ending": false,
                        "ended": false,
                        "finished": false,
                        "decodeStrings": false,
                        "defaultEncoding": "utf8",
                        "length": 0,
                        "writing": false,
                        "sync": true,
                        "bufferProcessing": false,
                        "writecb": null,
                        "writelen": 0,
                        "buffer": [],
                        "errorEmitted": false
                    },
                    "writable": true,
                    "allowHalfOpen": true,
                    "destroyed": false,
                    "bytesRead": 0,
                    "_bytesDispatched": 0,
                    "_pendingData": null,
                    "_pendingEncoding": "",
                    "server": "[Circular]",
                    "_idleTimeout": 120000,
                    "_idleNext": "[Circular]",
                    "_idlePrev": "[Circular]",
                    "_idleStart": 1419317275504,
                    "parser": {
                        "_headers": [],
                        "_url": "",
                        "socket": "[Circular]",
                        "incoming": null,
                        "maxHeaderPairs": 2000
                    },
                    "_paused": false
                },
                "_idlePrev": "[Circular]"
            },
            "_idlePrev": "[Circular]",
            "_idleStart": 1419317275251,
            "parser": {
                "_headers": [],
                "_url": "",
                "socket": "[Circular]",
                "incoming": "[Circular]",
                "maxHeaderPairs": 2000
            },
            "_paused": false,
            "_httpMessage": {
                "domain": "[Circular]",
                "_events": {},
                "_maxListeners": 10,
                "output": [],
                "outputEncodings": [],
                "writable": true,
                "_last": false,
                "chunkedEncoding": false,
                "shouldKeepAlive": true,
                "useChunkedEncodingByDefault": true,
                "sendDate": true,
                "_headerSent": false,
                "_header": "",
                "_hasBody": true,
                "_trailer": "",
                "finished": false,
                "_hangupClose": false,
                "socket": "[Circular]",
                "connection": "[Circular]",
                "log": {
                    "domain": null,
                    "_events": {},
                    "_maxListeners": 10,
                    "_level": 30,
                    "streams": [{
                        "level": 30,
                        "path": "E:\\vidyartha-com\\trunk\\node\\businessapp\\businessapp.log",
                        "type": "rotating-file",
                        "period": "1d",
                        "count": 7,
                        "raw": false,
                        "stream": {
                            "path": "E:\\vidyartha-com\\trunk\\node\\businessapp\\businessapp.log",
                            "stream": {
                                "_writableState": {
                                    "highWaterMark": 16384,
                                    "objectMode": false,
                                    "needDrain": false,
                                    "ending": false,
                                    "ended": false,
                                    "finished": false,
                                    "decodeStrings": true,
                                    "defaultEncoding": "utf8",
                                    "length": 0,
                                    "writing": false,
                                    "sync": false,
                                    "bufferProcessing": false,
                                    "writecb": null,
                                    "writelen": 0,
                                    "buffer": [],
                                    "errorEmitted": false
                                },
                                "writable": true,
                                "domain": null,
                                "_events": {},
                                "_maxListeners": 10,
                                "path": "E:\\vidyartha-com\\trunk\\node\\businessapp\\businessapp.log",
                                "fd": 3,
                                "flags": "a",
                                "mode": 438,
                                "bytesWritten": 11626
                            },
                            "count": 7,
                            "periodNum": 1,
                            "periodScope": "d",
                            "rotQueue": [],
                            "rotating": false,
                            "rotAt": 1419379200000,
                            "timeout": {
                                "_idleTimeout": 61943637,
                                "_idlePrev": {
                                    "_idleTimeout": 61943637,
                                    "_idlePrev": {
                                        "_idleNext": "[Circular]",
                                        "_idlePrev": "[Circular]",
                                        "msecs": 61943637
                                    },
                                    "_idleNext": "[Circular]",
                                    "_idleStart": 1419317256363,
                                    "_repeat": false
                                },
                                "_idleNext": "[Circular]",
                                "_idleStart": 1419317256363,
                                "_repeat": false
                            }
                        },
                        "closeOnExit": true
                    }, {
                        "level": 50,
                        "path": "E:\\vidyartha-com\\trunk\\node\\businessapp\\businessapp_error.log",
                        "type": "rotating-file",
                        "period": "1d",
                        "count": 7,
                        "raw": false,
                        "stream": {
                            "path": "E:\\vidyartha-com\\trunk\\node\\businessapp\\businessapp_error.log",
                            "stream": {
                                "_writableState": {
                                    "highWaterMark": 16384,
                                    "objectMode": false,
                                    "needDrain": false,
                                    "ending": false,
                                    "ended": false,
                                    "finished": false,
                                    "decodeStrings": true,
                                    "defaultEncoding": "utf8",
                                    "length": 0,
                                    "writing": false,
                                    "sync": true,
                                    "bufferProcessing": false,
                                    "writecb": null,
                                    "writelen": 0,
                                    "buffer": [],
                                    "errorEmitted": false
                                },
                                "writable": true,
                                "domain": null,
                                "_events": {},
                                "_maxListeners": 10,
                                "path": "E:\\vidyartha-com\\trunk\\node\\businessapp\\businessapp_error.log",
                                "fd": 4,
                                "flags": "a",
                                "mode": 438,
                                "bytesWritten": 0
                            },
                            "count": 7,
                            "periodNum": 1,
                            "periodScope": "d",
                            "rotQueue": [],
                            "rotating": false,
                            "rotAt": 1419379200000,
                            "timeout": "[Circular]"
                        },
                        "closeOnExit": true
                    }],
                    "serializers": {},
                    "src": false,
                    "fields": {
                        "name": "businessapp",
                        "hostname": "rakesh",
                        "pid": 7176
                    },
                    "haveNonRawStreams": true
                },
                "_time": 1419317275253,
                "acceptable": ["application/json", "text/plain", "application/octet-stream", "application/javascript"],
                "formatters": {},
                "req": "[Circular]",
                "serverName": "businessapp",
                "methods": ["GET"],
                "_headers": {
                    "content-encoding": "gzip"
                },
                "_headerNames": {
                    "content-encoding": "Content-Encoding"
                },
                "locale": "en",
                "_charSet": "utf-8",
                "contentType": "json"
            },
            "_peername": {
                "address": "127.0.0.1",
                "family": "IPv4",
                "port": 62452
            }
        },
        "connection": "[Circular]",
        "httpVersion": "1.1",
        "complete": true,
        "headers": {
            "host": "localhost:8500",
            "connection": "keep-alive",
            "user-agent": "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36",
            "content-type": "text/plain; charset=utf-8",
            "accept": "*/*",
            "accept-encoding": "gzip, deflate, sdch",
            "accept-language": "en-GB,en-US;q=0.8,en;q=0.6",
            "cookie": "ATOKEN=fe8c94a1-de6d-440c-82d7-871ef71e391b"
        },
        "trailers": {},
        "_pendings": [],
        "_pendingIndex": 0,
        "url": "/business/services/updateAptitudeResults?lang=en",
        "method": "GET",
        "statusCode": null,
        "client": "[Circular]",
        "_consuming": true,
        "_dumped": false,
        "httpVersionMajor": 1,
        "httpVersionMinor": 1,
        "upgrade": false,
        "log": "[Circular]",
        "_time": 1419317275253,
        "timers": [{
            "name": "parseAccept",
            "time": [0, 2185089]
        }, {
            "name": "parseAuthorization",
            "time": [0, 269017]
        }, {
            "name": "parseDate",
            "time": [0, 204734]
        }, {
            "name": "parseQueryString",
            "time": [0, 2522171]
        }, {
            "name": "readBody",
            "time": [0, 2562685]
        }, {
            "name": "parseJson",
            "time": [0, 220940]
        }, {
            "name": "gzip",
            "time": [0, 4035797]
        }, {
            "name": "i18nInit",
            "time": [0, 1456906]
        }, {
            "name": "rateLimit",
            "time": [0, 557481]
        }, {
            "name": "handler-9",
            "time": [0, 308992]
        }],
        "_version": "*",
        "_url": {
            "protocol": null,
            "slashes": null,
            "auth": null,
            "host": null,
            "port": null,
            "hostname": null,
            "hash": null,
            "search": "?lang=en",
            "query": "lang=en",
            "pathname": "/business/services/updateAptitudeResults",
            "path": "/business/services/updateAptitudeResults?lang=en",
            "href": "/business/services/updateAptitudeResults?lang=en"
        },
        "_path": "/business/services/updateAptitudeResults",
        "params": {
            "lang": "en",
            "rbody": ""
        },
        "context": "[Circular]",
        "route": {
            "path": "/business/services/updateAptitudeResults",
            "method": "GET",
            "versions": [],
            "name": "getbusinessservicesupdateaptituderesults"
        },
        "_negotiator": {
            "request": {
                "headers": {
                    "accept": "*/*",
                    "accept-encoding": "gzip, deflate, sdch"
                }
            }
        },
        "authorization": {},
        "username": "anonymous",
        "_query": {
            "lang": "en"
        },
        "query": "[Circular]",
        "_clen": false,
        "_contentType": "text/plain",
        "body": "",
        "languages": ["en", "en", "en"],
        "regions": ["gb", "us"],
        "language": "en",
        "region": "gb",
        "locale": "en",
        "unauthorized": true
    }, "[Circular]"]
},
"domainThrown": true

2 个答案:

答案 0 :(得分:0)

eachLimit可以解决问题:

  

eachLimit(arr,limit,iterator,callback)

     

与每个相同,只有不超过限制迭代器   随时同时运行。

答案 1 :(得分:0)

尝试使用:

mapSeries(arr, iterator, callback);它可以解决您的问题。

另见:https://github.com/caolan/async#mapseriesarr-iterator-callback