我们正在尝试读取目录下的大量xml文件,该目录下有一个数据列表(约50K对象),并在DB中检查每条记录并插入(如果不存在)。在完成一个文件数据之后,它应该转到第二个文件,这并不是全部。下面是我们编写的代码。
仅供参考:我们最近开始在node,猫鼬上工作,并开始为cron作业编写脚本。
## we recently starting working on node, mongoose and started writing a script the cron job.
var fs = require('fs');
function log(arg) {
console.log(arg);
}
## step-1: read the files in dir
fs.readdir('./scripts/test/', function(err, files) {
files.filter(fn => fn.endsWith('.xml')).forEach(file => {
new Promise((resolve, reject) => {
fs.readFile('/scripts/test/'+file, 'utf-8', (err, data) => {
if (!err) {
log('parsing XML to JSON.....');
resolve(data);
} else {
log('reading file error: '+err)
reject(err);
}
});
}).then((data) => {
var productData = JSON.parse(data);
## calling another function
format(productData);
}).catch(err => log('Catch Error: '+err));
});
});
async function format(items) {
if (productData !== undefined) {
await productBatchesloop(items);
log('completed');
} else {
log('@@@@@@@ action:productData undefined @@@@@@@@');
}
}
async function productBatchesloop(items) {
var formatted_products, items_batch, promises;
formatted_products = promises = []; var next = 1;
var formatfn = function formatItems(product) {
let pro = ProductModel.findOneAndUpdate(
{id: product.id},
product,
{upsert: true, new: true, runValidators: true}).exec();
pro.then((pro) => {
log('inserted: '+pro.id);
});
log(pro);//not waiting for resolve, showing Promise <pending>
}
var res2 = Promise.all(items.map(formatfn));
res2.then((data) => {
log('*********** COMPLETED **************');
// log(data);
});
}
答案 0 :(得分:1)
在猫鼬查询中使用异步等待。
var formatfn = async formatItems(product) => {
let pro = await ProductModel.findOneAndUpdate(
{id: product.id},
product,
{upsert: true, new: true, runValidators: true}).exec();
pro.then((pro) => {
log('inserted: '+pro.id);
});
log(pro);//not waiting for resolve, showing Promise <pending>
}