Question

我正在用node.js编写一个后端API，需要用户能够上传带有数据的文件，然后调用存储过程以将数据插入MySQL的功能。我正在考虑使用fast-csv作为解析器，但是我在为如何在csv流中设置对存储过程的调用而苦苦挣扎。这个想法是这样的：

var fs = require("fs");
var csv = require("fast-csv");
var stream1 = fs.createReadStream("files/testCsvFile.csv");

csv
  .fromStream(stream2, { headers: true })
  .on("data", function(data) {

   //CALL TO SP with params from "data"//

    numlines++;
  })
  .on("end", function() {
    console.log("done");
  });

在应用程序的其他部分，我设置了以下路由：

auth.post("/verified", async (req, res) => {
    var user = req.session.passwordless;
    if (user) {
      const rawCredentials = await admin.raw(getUserRoleCredentials(user));
      const { user_end, role } = await normalizeCredentials(rawCredentials);
      const user_data = { user_end, role };
      res.send(user_data);
    } else {
      res.sendStatus(401);
    }
  });

..也就是-路由以异步/等待方式编写，并且查询（都称为存储过程）被定义为Promises。。我想在上载/解析csv / call SP的每个行函数中遵循此模式

Answer 1

如评论中所述，我使scramjet可以轻松处理这种用例...如果我理解错了，请更正我，但我知道您想为每个调用两个等待行测试中的CSV行。

如果是这样，您的代码应如下所示（已更新为与您的评论/答案匹配）：

var fs = require("fs");
var csv = require("fast-csv");
var stream1 = fs.createReadStream("files/testCsvFile.csv");
var {DataStream} = require("scramjet");

DataStream
    // the following line will convert any stream to scramjet.DataStream
    .from(csv.fromStream(stream2, { headers: true }))
    // the next lines controls how many simultaneous operations are made
    // I assumed 16, but if you're fine with 40 or you want 1 - go for it.
    .setOptions({maxParallel: 16})
    // the next line will call your async function and wait until it's completed
    // and control the back-pressure of the stream
    .do(async (data) => {
        const query = await queryBuilder({
          schema,
          routine,
          parameters,
          request
        }); //here we prepare query for calling the SP with parameters from data

        winston.info(query + JSON.stringify(data));
        const rawResponse = await session.raw(query); //here the query gets executed
        return data; // push each row - for testing only)
    })
    // next line will run the stream until end and return a promise
    .toArray()
    .then(fileRows => {
        console.log(fileRows);
        fs.unlinkSync(form.FileName); // remove temp file
        //process "fileRows" and respond
        res.end(JSON.stringify(fileRows)); // - for testing
    })
    .catch(e => {
        res.writeHead(500); // some error handling
        res.end(e.message);
    })
    ;
    // you may want to put an await statement before this, or call then to check
    // for errors, which I assume is your use case.
;

要回答您的评论问题-如果要在on("data")事件中使用异步函数-您需要创建一个Promise数组并等待Promise。流end上的所有该数组-但这需要同步完成-因此事件处理程序中的async函数不会这么做。

在scramjet中，这种情况发生在幕后，因此您可以使用该功能。

Answer 2

这正在为我完成工作--请您描述一下如何通过您的框架实现--我认为应该以某种方式完成，我只需要正确配置它即可即可

 //use fast-csv to stream data from a file
    csv
      .fromPath(form.FileName, { headers: true })
      .on("data", async data => {
        const query = await queryBuilder({
          schema,
          routine,
          parameters,
          request
        }); //here we prepare query for calling the SP with parameters from data

        winston.info(query + JSON.stringify(data));
        const rawResponse = await session.raw(query); //here the query gets executed
        fileRows.push(data); // push each row - for testing only
      })

      .on("end", function() {
        console.log(fileRows);
        fs.unlinkSync(form.FileName); // remove temp file
        //process "fileRows" and respond

        res.end(JSON.stringify(fileRows)) // - for testing
      });

在fast-csv中异步调用存储过程

2 个答案: