Question

使用Meteor并在服务器上尝试通过循环Meteor集合并插入行来生成大型csv文件。在某些时候，服务器将出现内存不足错误 - 我的猜测是在循环结束之前我的内存耗尽，具体取决于集合大小。我该如何解决这个问题（以某种方式清除记忆）？这是代码：

    var job = Jobs.findOne();
    var fs = Npm.require('fs');
    var file = '/tmp/csv-' + job._id + '.csv';
    var headers = ["Email", "Processed?", "Integration", "Passed?", "Reason", "Date"];
    var stream = fs.createWriteStream(file);
    var first_line = headers.join() + '\n';
    var wstream = fs.createWriteStream(file);
    var emails = rawEmails.find();
    wstream.write(first_line);
    emails.forEach(function(rawemail) {
        var line_item = [];
        line_item.push(rawemail.email);
        if (rawemail.processed === true || rawemail.processed === false)
            line_item.push(rawemail.processed);
        if (rawemail.integration)
            line_item.push(rawemail.integration);
        if (rawemail.passed === true || rawemail.passed === false)
            line_item.push(rawemail.passed);
        if (rawemail.reason)
            line_item.push(rawemail.reason);
        if (rawemail.updated_at)
            line_item.push(rawemail.updated_at);
        var to_write = line_item.join() + '\n';
        wstream.write(to_write);
    });
    wstream.end();

Answer 1

var emails = rawEmails.find();

不好。您需要限制，分页并将大量记录写入文件

var skip = 0
var emails = rawEmails.find({}, {limit: 100, skip: skip})

while (emails) {
   // write to buffer 
   skip = (++skip) * 100 
   emails = rawEmails.find({}, {limit: 100, skip: skip})
}

请注意，如果记录数太大，节点进程也会为writeStream消耗大量内存，因此会再次使内存不足。考虑写入多个文件并将其压缩以发送回客户端（如果客户想要下载它）

内存不足试图创建大型csv文件

1 个答案: