nodejs mongodb timesSeries有时挂起而没有错误

时间:2017-07-03 21:19:39

标签: node.js mongodb google-bigquery

我使用timesSeries循环运行一个包含多个查询的脚本。

有时它会毫无理由地挂起或出错。首先它使用计数,

确定页面的循环数。然后它循环以减少结果。

然后它使用异步循环来构建要发送给google bigquery的文档。

有时它会停止,然后我必须手动杀死脚本。我应该先关闭计数吗?

mongo_client.connect(mongo_url, function(err, db) {

    var query = { "date_visited" : {$gt: new Date("2016-01-01T00:00:00.000Z"),$lt: new Date("2016-02-01T00:00:00.000Z")},_id:{$gte: ObjectId("56aabfe8c6edfd0d608b480e")} };

    db.collection(collection_name).count(query,function(err, count){
        var pages = Math.ceil(count/limit);
    //var pages = 1;
        console.log("count: "+count);
        console.log("pages: "+pages);

        var page = 0

        async.timesSeries(pages, function(page, next)
        {
            var skip = page * limit;
            var current_page = page+1;
            console.log("skip: "+skip);
            console.log("page: "+page);
            console.log("current_page: "+current_page+" of "+pages);

            var documents = [];
            var x=1;
            db.collection(collection_name).find(query).limit(limit).skip(skip).toArray(function(err, db_results) {  
                async.each(db_results, function (db_resultsx, cb) {
                    var account_id = db_resultsx.account_id;

                    delete db_resultsx.upsell_detect;

                    db_resultsx._id = db_resultsx._id.toString();
                    db_resultsx.xdate_visited = moment(db_resultsx.date_visited).format("YYYY-MM-DD hh:mm:ss");
                    documents.push(db_resultsx);
                    //console.log(documents);

                    if(documents.length == db_results.length) { 

                        bigqueryClient
                          .dataset(dataset)
                          .table(collection_name)
                          .insert(documents)
                          .then((insertErrors) => {
                            console.log("Inserted: "+page);
                            //console.error("Inserted: "+page);
                            //console.error(insertErrors);
                            next();
                            if (insertErrors && insertErrors.length > 0) {
                              console.log("start errors: "+page+"("+insertErrors.length+")");
                              insertErrors.forEach((err) => console.log(JSON.stringify(err, null, 4)));
                              console.log("done errors: "+page);
                              console.log("done on: "+moment().format("YYYY-MM-DD hh:mm:ss"));

                            }
                          })
                          .catch((err) => {
                            console.log('ERROR:');
                            console.log(JSON.stringify(err, null, 4));

                            console.log('done catch:'+page);
                            process.exit();
                        });
                    }
                });


        });
    },
    function(err, coll)
    {
        console.log("done all");
        console.log(query);

        setTimeout(function () {
            db.close();
            console.error("DONE");
            process.exit();
        }, 60000);
    });


});

});

0 个答案:

没有答案