Question

这是我的节点js代码 我想使用nodejs和angularjs上传一个csv文件，然后将其保存到cassandra。

    app.use(function(req, res, next) { //allow cross origin requests
        res.setHeader("Access-Control-Allow-Methods", "POST, PUT, OPTIONS, DELETE, GET");
        res.header("Access-Control-Allow-Origin", "http://localhost");
        res.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept");
        next();
    });

    app.use(express.static('../client'));
    app.use(bodyParser.json());  

    var storage = multer.diskStorage({ //multers disk storage settings
        destination: function (req, file, cb) {
            cb(null, './uploads/');
        },
        filename: function (req, file, cb) {
            var datetimestamp = Date.now();
            cb(null, file.fieldname + '-' + datetimestamp + '.' + file.originalname.split('.')[file.originalname.split('.').length -1]);
        }
    });

    var upload = multer({ //multer settings
                    storage: storage
                }).single('file');

    /** API path that will upload the files */
    app.post('/upload', function(req, res) {
        upload(req,res,function(err){
            if(err){
                 res.json({error_code:1,err_desc:err});
                 return;
            }
             res.json({error_code:0,err_desc:null});
        });
    });

我想通过gui将我的文件直接上传到cassandra。任何领导或帮助都是值得的。

Answer 1

您应该找到一种方法来从多部分表单数据读取而不存储到网络服务器硬盘驱动器。

您可以使用busboy（中间件connect-busboy）从响应中读取文件数据作为流。一旦您可以阅读即将推出的数据，就可以将其插入Cassandra。

您必须定义：

最大并发级别：您希望并行插入C *的csv行数。
如何处理故障：单个插入失败时该怎么办。

之后，您可以读取行并将这些行并行插入Cassandra，如下所示：

// Use some control flow methods (dependency)
const async = require('async');
const csv = stream; //using busboy to obtain the file stream
// Maximum amount of rows to insert in parallel
const limit = 256;
csv.on('readable', function () {
  //its emitted once there is a chunk of data
  var r;
  var rows = [];
  async.whilst(function condition() {
    while ((r = csv.read()) != null && rows.length < limit) {
      rows.push(r);
    }
    return rows.length > 0;
  }, function eachGroup(next) {
    // We have a group of 256 rows or less to save
    // we can do it in a batch
    // or we can do it in parallel with async.each()
    async.each(rows, function (r, eachCallback) {
      //adapt the csv row to parameters
      //sample
      var params = r.split(',);
      client.execute(query, params, { prepare: true}, eachCallback);
    }, next);
  }, function groupFinished(err) {
    if (err) {
      //something happened when saving
      //do something with err
      return;
    }
    // The chunk of csv rows emitted by csv stream where saved
  });
}).on('end', function () {
  // no more data
});

我想使用nodejs和angularjs上传一个csv文件，然后将其保存到cassandra

1 个答案: