Question

我知道，老派的for循环以传统方式工作-等待await完成结果。但是在我的用例中，我需要从local / s3中读取文件并逐行处理它，并且对于每一行，我都需要调用一个外部API。通常，我在循环内使用await，因为它们都在lambda内运行，并且我不想使用所有内存来并行运行它。在这里，我正在使用stream.on()方法读取文件，为了在其中使用await，我需要在read方法中添加async，如下所示：

stream.on('data',async () =>{
         while(data=stream.read()!==null){
           console.log('line');
           const requests = getRequests(); // sync code,no pblms
           for(let i=0;i<requests.length;i++){
             const result = await apiCall(request[i);
             console.log('result from api')
             const finalResult = await anotherapiCall(result.data);
             }
        }
});

这是可行的，但是不能保证处理行的顺序。我需要全部同步。有帮助吗？

完整代码

async function processSOIFileLocal (options, params) {
console.log('Process SOI file');

const readStream = byline.createStream(fs.createReadStream(key));
readStream.setEncoding('utf8');
const pattern = /^UHL\s|^UTL\s/;
const regExp = new RegExp(pattern);
readStream.on('readable', () => {
    let line;
    while (null !== (line = readStream.read())) {
        if (!regExp.test(line.toString())) {
            totalRecordsCount++;
            dataObject = soiParser(line);
            const { id } = dataObject;
            const XMLRequests = createLoSTRequestXML(
                options,
                { mapping: event.mapping, row: dataObject }
            );
            console.log('Read line');
            console.log(id);
            try {
                for (let i = 0;i < XMLRequests.length;i++) {
                    totalRequestsCount++;
                    console.log('Sending request');
                    const response = await sendLoSTRequest(
                        options,
                        { data: XMLRequests[i],
                            url: LOST_URL }
                    );
                    console.log("got response");
                    const responseObj = await xml2js.
                        parseStringPromise(response.data);
                    if (Object.keys(responseObj).indexOf('errors') !== -1) {
               
                        fs.writeFileSync(`${ERR_DIR}/${generateKey()}-${id}.xml`, response.data);
                        failedRequestsCount++;
                    } else {
                        successRequestsCount++;
                        console.log('Response from the Lost Server');
                        console.log(response[i].data);
                    }
                }
            } catch (err) {
                console.log(err);
            }
        }
    }
})
    .on('end', () => {
        console.log('file processed');
        console.log(`
        ************************************************
        Total Records Processed:${totalRecordsCount}
        Total Requests Sent: ${totalRequestsCount}
        Success Requests: ${successRequestsCount}
        Failed Requests: ${failedRequestsCount}
        ************************************************
        `);
    });
}

async function sendLoSTRequest (options, params) {
const { axios } = options;
const { url, data } = params;
if (url) {
    return  axios.post(url, data);
// eslint-disable-next-line no-else-return
} else {
    console.log('URL is not found');
    return null;
}

}

代码需要这样流动：以同步方式读取一行处理线并将线转换为两个成员的数组为每个成员调用API做些事情生产线完成后，寻找另一条生产线，一切按顺序完成

更新：有一种解决方法。但是它会在不等待流完成读取的情况下触发stream.end（）

async function processSOIFileLocal (options, params) {
console.log('Process SOI file');
const { ERR_DIR, fs, xml2js, LOST_URL, byline, event } = options;
const { key } = params;
const responseObject = {};
let totalRecordsCount = 0;
let totalRequestsCount = 0;
let failedRequestsCount = 0;
let successRequestsCount = 0;
let dataObject = {};
const queue = (() => {
    let q = Promise.resolve();
    return fn => (q = q.then(fn));
})();
const readStream = byline.createStream(fs.createReadStream(key));
readStream.setEncoding('utf8');
const pattern = /^UHL\s|^UTL\s/;
const regExp = new RegExp(pattern);
readStream.on('readable', () => {
    let line;
    while (null !== (line = readStream.read())) {
        if (!regExp.test(line.toString())) {
            totalRecordsCount++;
            dataObject = soiParser(line);
            const { id } = dataObject;
            const XMLRequests = createLoSTRequestXML(
                options,
                { mapping: event.mapping, row: dataObject }
            );
                // eslint-disable-next-line no-loop-func
            queue(async () => {
                try {
                    for (let i = 0;i < XMLRequests.length;i++) {
                        console.log('Sending request');
                        console.log(id);
                        totalRequestsCount++;
                        const response = await sendLoSTRequest(
                            options,
                            { data: XMLRequests[i],
                                url: LOST_URL }
                        );
                        console.log('got response');
                        const responseObj = await xml2js.
                            parseStringPromise(response.data);
                        if (Object.keys(responseObj).indexOf('errors') !== -1) {
                            // console.log('Response have the error:');
                            // await handleError(options, { err: responseObj, id });
                            failedRequestsCount++;
                            fs.writeFileSync(`${ERR_DIR}/${generateKey()}-${id}.xml`, response.data);
                        } else {
                            console.log('Response from the Lost Server');
                            console.log(response[i].data);
                            successRequestsCount++;
                        }
                    }
                } catch (err) {
                    console.log(err);
                }
            });
        }
    }
})
    .on('end', () => {
        console.log('file processed');
        console.log(`
            ************************************************
            Total Records Processed:${totalRecordsCount}
            Total Requests Sent: ${totalRequestsCount}
            Success Requests: ${successRequestsCount}
            Failed Requests: ${failedRequestsCount}
            ************************************************
            `);
        Object.assign(responseObject, {
            failedRequestsCount,
            successRequestsCount,
            totalRecordsCount,
            totalRequestsCount
        });
    });

}

谢谢

Answer 1

您可以将问题顶部的示例代码重写为

const queue = (() => {
    let q = Promise.resolve();
    return (fn) => (q = q.then(fn));
})();
stream.on('data', async() => {
    while (data = stream.read() !== null) {
        console.log('line');
        const requests = getRequests(); // sync code,no pblms
        queue(async () => {
            for (let i = 0; i < requests.length; i++) {
                const result = await apiCall(request[i]);
                console.log('result from api');
                const finalResult = await anotherapiCall(result.data);
            }
        });
    }
});

希望这对完整的代码很有用

Answer 2

如果任何人都希望有一种解决方案来同步处理文件，即逐行读取并执行一些Async调用，建议使用内置流转换。在那里，我们可以创建一个转换函数，并在完成时返回一个回调。这将帮助任何面对此问题的人。 Through2是一个小的npm库，也可以用于该库。

异步/等待for循环NodeJS不阻止循环执行

完整代码

2 个答案: