Question

我已经实现了一个从触发器开始的滑动窗口。编写了测试代码，使用Akka Streams TestKit测试我的滑动窗口阶段。

我的测试没有完成，并且onUpstreamFinish方法中的while循环没有出现控制。我看到了这个例外：

case class SlidingOnTrigger[T](duration: Duration, trigger: T => Boolean, collector: T => Boolean, timeEpochExtractor: T => Long) extends GraphStage[FlowShape[T, collection.Seq[T]]] {


  val in = Inlet[T]("TriggeredSliding.in")
  val out = Outlet[collection.Seq[T]]("TriggeredSliding.out")
  override val shape: FlowShape[T, collection.Seq[T]] = FlowShape(in, out)

  override protected val initialAttributes: Attributes = Attributes.name("sliding")


  override def createLogic(inheritedAttributes: Attributes): GraphStageLogic = new GraphStageLogic(shape) with InHandler with OutHandler {
    private var inPursuit = Vector.empty[mutable.MutableList[T]]

    private val bufSeq = mutable.Queue.empty[collection.Seq[T]]

    var watermark = -1L

    private def untilDoneIndex(current: T) = {
      inPursuit.indexWhere { buf =>
        if (buf.nonEmpty) {
          val hts = timeEpochExtractor(buf.head)
          val cts = timeEpochExtractor(current)

          cts >= hts + duration.toMillis
        } else false
      }

    }


    override def onPush(): Unit = {
      val data = grab(in)

      val timeStamp = timeEpochExtractor(data)

      if (timeStamp > watermark) {
        watermark = timeStamp

        if (trigger(data)) {
          inPursuit :+= mutable.MutableList.empty[T]
        }

        val indexUntilDone = untilDoneIndex(data)

        inPursuit.indices.foreach { i =>
          if (i <= indexUntilDone) {
            bufSeq.enqueue(inPursuit(i))
          } else {
            if (collector(data)) {
              inPursuit(i) += data
            }
          }
        }

        inPursuit = inPursuit.drop(indexUntilDone + 1)
        pull(in)

      } else {
        pull(in)
      }

      checkAndPush()
    }

     private def checkAndPush() = {
      if (bufSeq.nonEmpty && isAvailable(out)) {
        push(out, bufSeq.dequeue())
      }else if(isClosed(in) && inPursuit.nonEmpty && isAvailable(out)){
        push(out, inPursuit.head)
        inPursuit = inPursuit.drop(1)
      }

      if(isClosed(in) && bufSeq.isEmpty && inPursuit.isEmpty){
        completeStage()
      }
    }

    override def onPull(): Unit = {
      if (!isClosed(in) && !hasBeenPulled(in) && bufSeq.isEmpty) {
        pull(in)
      } else {
        checkAndPush()
      }
    }


    override def onUpstreamFinish(): Unit = {

    }

    this.setHandlers(in, out, this)
  }

}

谷歌搜索出口无法使用但无法获得满意的结果。

Akka，Akka Streams，Akka Streams TestKit版本：2.5.9
Scala版本：2.12.4

滑动窗口：

object SlidingWindowOnTriggerTest extends App {

  import akka.actor.ActorSystem
  import akka.stream.ActorMaterializer
  import akka.stream.scaladsl.{Keep, Sink, Source}
  import scala.concurrent.duration._

  implicit val as = ActorSystem("WindowTest")
  implicit val m = ActorMaterializer()

  val expectedResultStream = Stream.from(0).map(_.toLong)

  val testIt = Iterator.from(0).take(20).map(_.toLong)

  val (_, ts) = Source
    .fromIterator(() => testIt)
    .via(SlidingOnTrigger[Long](10 millis, x => x % 3 == 0, _ => true, identity))
    .toMat(TestSink.probe[Seq[Long]])(Keep.both)
    .run()

  ts
    .request(10)
    .expectNext(expectedResultStream.take(10))
    .expectNext(expectedResultStream.take(13).drop(3))
    .expectNext(expectedResultStream.take(16).drop(6))
    .expectNext(expectedResultStream.take(19).drop(9))
    .request(10)
    .expectNext(expectedResultStream.take(20).drop(12))
    .expectNext(expectedResultStream.take(20).drop(15))
    .expectNext(expectedResultStream.take(20).drop(18))
    .expectComplete()

  as.terminate()

}

测试代码：

MyApp

另外，提供任何改善实施的建议

Answer 1

有一种情况，下游没有发出任何需求信号，但上游完成了。鉴于你有自己的缓冲区，如果发生这种情况，你需要处理其中的元素。

在onUpstreamFinish中使用永不完成的while循环阻塞的解决方案并不好。即使在最好的情况下，你也永远不会有多个可能的突出推送，在最糟糕的情况下你会永远阻止一个线程。

如果调用onUpstreamFinish时缓冲区中有东西，则需要做的是不完成阶段，而是确保可以拉出其余元素。这可以通过两种方式完成，或者通过拉入检查（可以使用isClosed(in)完成），或者通过切换到仅从缓冲区向下游提供的不同OutHandler直到空，然后完成。

Akka Streams：在我的Sliding窗口实现中上游关闭时，Outlet不可用

1 个答案: