Question

我有一个来自mongo的流式查询，我正在将它传递给一个通过“2”间谍“可写流。它完全包括“结束”回调和一小部分5个文档。然而，随着344个文档的更大集合，只有前15个文档通过，然后它永远挂起，并且“结束”事件永远不会触发。这是一个MCVE：

var spy = require("through2-spy").obj;
var MongoClient = require("mongodb").MongoClient;

function getStream() {
  var stream = spy(function() {
    console.log("@bug counting", stream.total++);
  });
  stream.total = 0;
  return stream;
}

function onEnd() {
  console.log("ended");
}

MongoClient.connect(process.argv[2], function(error, db) {
  if (error) {
    console.error(error);
    return;
  }
  var stream = db.collection(process.argv[3]).find().stream();
  stream
    // behavior is the same with the follow line commented out or not
    .on("end", db.close.bind(db))
    .on("error", console.error)
    .on("end", onEnd)
    .pipe(getStream());
});

Answer 1

问题是through2-spy默认使用16的highWaterMark。为了处理流量控制，流维护一个内部缓冲区，当从中消耗数据时清除它们。因为没有可读的流消耗getStream返回的转换流中的数据，所以内部缓冲区被填充并到达highWaterMark。增加highWaterMark应该修复它：

var stream = spy({highWaterMark: 350}, function() {
  console.log("@bug counting", stream.total++);
});

另一种非标准替代方法是重置变换流的可读状态：

var stream = spy(function() {
    console.log("@bug counting", stream.total++);
    this._readableState.length = 0;
});

Answer 2

解决此问题的另一种方法是确保下游的某些内容能够完全读取上游源。我最后在我的流的末尾添加了一个额外的.pipe(terminus.devnull({objectMode: true});，这也是诀窍。

var MongoClient = require("mongodb").MongoClient;
var spy = require("through2-spy").obj;
var terminus = require("terminus");

function getStream() {
  var stream = spy(function() {
    console.log("@bug counting", stream.total++);
  });
  stream.total = 0;
  return stream;
}

function onEnd() {
  console.log("ended");
}

MongoClient.connect(process.argv[2], function(error, db) {
  if (error) {
    console.error(error);
    return;
  }
  var stream = db.collection(process.argv[3]).find().stream();
  stream
    // behavior is the same with the follow line commented out or not
    .on("end", db.close.bind(db))
    .on("error", console.error)
    .on("end", onEnd)
    .pipe(getStream())
    .pipe(terminus.devnull({objectMode: true}));
});

没有调用带有through2.spy结尾的mongodb流查询

2 个答案: