我使用timesSeries循环运行一个包含多个查询的脚本。
有时它会毫无理由地挂起或出错。首先它使用计数,
确定页面的循环数。然后它循环以减少结果。
然后它使用异步循环来构建要发送给google bigquery的文档。
有时它会停止,然后我必须手动杀死脚本。我应该先关闭计数吗?
mongo_client.connect(mongo_url, function(err, db) {
var query = { "date_visited" : {$gt: new Date("2016-01-01T00:00:00.000Z"),$lt: new Date("2016-02-01T00:00:00.000Z")},_id:{$gte: ObjectId("56aabfe8c6edfd0d608b480e")} };
db.collection(collection_name).count(query,function(err, count){
var pages = Math.ceil(count/limit);
//var pages = 1;
console.log("count: "+count);
console.log("pages: "+pages);
var page = 0
async.timesSeries(pages, function(page, next)
{
var skip = page * limit;
var current_page = page+1;
console.log("skip: "+skip);
console.log("page: "+page);
console.log("current_page: "+current_page+" of "+pages);
var documents = [];
var x=1;
db.collection(collection_name).find(query).limit(limit).skip(skip).toArray(function(err, db_results) {
async.each(db_results, function (db_resultsx, cb) {
var account_id = db_resultsx.account_id;
delete db_resultsx.upsell_detect;
db_resultsx._id = db_resultsx._id.toString();
db_resultsx.xdate_visited = moment(db_resultsx.date_visited).format("YYYY-MM-DD hh:mm:ss");
documents.push(db_resultsx);
//console.log(documents);
if(documents.length == db_results.length) {
bigqueryClient
.dataset(dataset)
.table(collection_name)
.insert(documents)
.then((insertErrors) => {
console.log("Inserted: "+page);
//console.error("Inserted: "+page);
//console.error(insertErrors);
next();
if (insertErrors && insertErrors.length > 0) {
console.log("start errors: "+page+"("+insertErrors.length+")");
insertErrors.forEach((err) => console.log(JSON.stringify(err, null, 4)));
console.log("done errors: "+page);
console.log("done on: "+moment().format("YYYY-MM-DD hh:mm:ss"));
}
})
.catch((err) => {
console.log('ERROR:');
console.log(JSON.stringify(err, null, 4));
console.log('done catch:'+page);
process.exit();
});
}
});
});
},
function(err, coll)
{
console.log("done all");
console.log(query);
setTimeout(function () {
db.close();
console.error("DONE");
process.exit();
}, 60000);
});
});
});