并行读取多个文件(Nodejs)

时间:2018-02-02 04:14:53

标签: node.js file filesystems line-by-line

我有2个文件(或更多)1.txt和2.txt。
1.txt

1-a
1-b
1-c

2.txt

2-a
2-b
2-c

现在我希望1.txt和2.txt逐行并行读取,并在读取后将该行写入result.txt。的Result.txt。

1.a + 2.a
1.b + 2.b
1.c + 2.c

我该怎么做?
编辑:这是一个简单的例子。我想处理大文件,所以我想尽可能地保存我的内存,我不想使用数组。

3 个答案:

答案 0 :(得分:1)

我认为使用异步I / O和尽可能少的内存(仅缓冲readStream在data事件上提供的内容)并输出每一行时,这将是一个有趣的挑战。

所以,这是一种方法。这不是在非常大的文件上进行广泛测试,但似乎在我尝试的测试用例中有效。如果两个文件具有不同的行数,则在读取较短的文件时将停止输出行。这是代码:

const LineReader = require('line-by-line');
const fs = require('fs');

class DualLineReader extends LineReader {
    constructor(filename, outputStream) {
        super(filename);
        this.myBuffer = [];
        this.output = outputStream;
    }
    setOtherReader(lr) {
        this.otherLineReader = lr;
    }

    resumeIfEmpty() {
        if (!this.myBuffer.length) {
            this.resume();
        }
    }

    // if data in both buffers, write the next line
    // call the callback when OK to call writeLine again
    writeLine(cb) {
        // if both buffers contain at least one line, output the first line in each
        if (this.myBuffer.length && this.otherLineReader.myBuffer.length) {
            let ready = this.output.write(this.myBuffer.shift() + " + " + this.otherLineReader.myBuffer.shift() + '\n');
            if (!ready) {
                // need to wait for drain event before writing any more
                this.output.once('drain', () => {
                    cb(true);
                });
            } else {
                process.nextTick(() => {
                    cb(true);
                });
            }
        } else {
            // nothing else to write at the moment
            // call the callback on next tick
            process.nextTick(() => {
                cb(false);
            });
        }
    }

    closeOutput(cb) {
        if (!this.output.closed) {
            this.output.end(cb);
        }
    }

    // loop asynchronously until no more data in buffer to write
    // call callback when done
    writeAllLines(cb = function() {}) {
        this.writeLine(more => {
            if (more) {
                this.writeAllLines();
            } else {
                // if either buffer is empty, start it flowing again
                this.resumeIfEmpty();
                this.otherLineReader.resumeIfEmpty();
                cb();
            }
        });
    }

    run(cb) {
        this.on('line', line => {
            this.myBuffer.push(line);
            this.pause();
            this.writeAllLines();
        });

        this.on('end', () => {
            this.writeAllLines(() => {
                this.close();
                this.otherLineReader.close();
                this.closeOutput(cb);
            });
        });

        this.on('error', (err) => {
            console.log(err);
            this.close();
            this.otherLineReader.close();
            this.closeOutput(() => {
                cb(err);
            });
        });

    }
}

let output = fs.createWriteStream("results.txt");
output.on('close', () => {
    this.closed = true;
});

let lr1 = new DualLineReader("file1.txt", output);
let lr2 = new DualLineReader("file2.txt", output);

lr1.setOtherReader(lr2);
lr2.setOtherReader(lr1);

function done() {
    console.log("all done");
}
lr1.run(done);
lr2.run(done);

答案 1 :(得分:0)

我认为您想要的简单方法是全部读取文件并加上它们。您可以使用ReadFileSync函数来读取文件。我举个例子。希望可以帮到你。

var fs = require("fs");

var file_1 = fs.readFileSync("files/1.txt");
var file_2 = fs.readFileSync("files/2.txt");

array_file_1 = file_1.toString().split("\r\n");
array_file_2 = file_2.toString().split("\r\n");

string_file = '';
array_file_1.forEach((element, index) => {
    string_file += element + array_file_2[index] + '\r\n';
});

fs.writeFileSync("files/result.txt", string_file);

console.log(string_file);

答案 2 :(得分:0)

阅读10篇以上的文章花费了2个小时以上,以找到合适的解决方案。下面工作正常。

安装节点模块:

npm install --save n-readlines

代码:

const lineByLine = require("n-readlines");

var filePath1 = "./file1.txt", // <------ NOTE: Make sure that this file exists
    filePath2 = "./file2.txt"; // <------ NOTE: Make sure that this file exists
var file1 = new lineByLine(filePath1),
    file2 = new lineByLine(filePath2);
var line1, line2;

while (true) {
    line1 = file1.next(); // Read next line from 1st file
    if (!line1) break; // EOF

    line2 = file2.next(); // Read next line from 2nd file
    if (!line2) break; // EOF

    line1 = line1.toString("ascii").trim(); // Buffer to text, then trim spaces
    line2 = line2.toString("ascii").trim(); // Buffer to text, then trim spaces

    console.log(line1, line2);
}