Question

我正在尝试在Scala中为Kafka编写集成测试（对这两者都有点新意）;通过集成测试，我的意思是我的主代码中有ClosedShape RunnableGraph，我想通过Kafka主题提供数据，然后检查通过Kafka主题发出的内容（而不是单位） - 在RunnableGraph）范围内测试单个流程。

这是一个简化的例子：

import akka.NotUsed
import akka.actor.ActorSystem
import akka.kafka.{ProducerSettings, ConsumerSettings}
import akka.kafka.scaladsl.{Producer, Consumer}
import akka.kafka.scaladsl.Consumer.Control
import akka.stream.ClosedShape
import akka.stream.scaladsl._
import org.apache.kafka.clients.consumer.{ConsumerRecord, ConsumerConfig}
import org.apache.kafka.clients.producer.ProducerRecord
import org.apache.kafka.common.serialization.{StringSerializer, StringDeserializer}
import GraphDSL.Implicits._

object SimpleKafkaStream {

  def apply(sourceTopic: String, targetTopic: String, kafkaBootstrapServer: String) (implicit actorSystem: ActorSystem) = {

    RunnableGraph.fromGraph (GraphDSL.create() { implicit builder: GraphDSL.Builder[NotUsed] =>
      source(sourceTopic, kafkaBootstrapServer) ~> transformMessage(targetTopic) ~> target(kafkaBootstrapServer)
      ClosedShape
    })
  }

  private def transformMessage (targetTopic: String) = Flow[ConsumerRecord[String, String]]
    .map (_.value())
    .map ("hello " + _)
    .map (message => { new ProducerRecord[String, String] (targetTopic, message) })

  private def source (topic: String, bootstrapServer: String) (implicit actorSystem: ActorSystem) : Source[ConsumerRecord[String, String], Control] = {
    val consumerSettings = ConsumerSettings(actorSystem, new StringDeserializer, new StringDeserializer, Set(topic))
      .withBootstrapServers(bootstrapServer)
      .withGroupId(s"consumer_1_.$topic")
      .withClientId(s"consumer_1_.$topic")
      .withProperty(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest")
      .withProperty(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG, "1000")
      .withProperty(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "true")
    Consumer.plainSource(consumerSettings)
  }

  private def target (bootstrapServer: String) (implicit actorSystem: ActorSystem) = {
    Producer.plainSink(ProducerSettings(actorSystem, new StringSerializer, new StringSerializer)
      .withBootstrapServers(bootstrapServer))
  }
}

然后用以下方法测试：

import java.util.UUID

import akka.actor.ActorSystem
import akka.kafka.{ConsumerSettings, ProducerSettings}
import akka.kafka.scaladsl.{Consumer, Producer}
import akka.stream.ActorMaterializer
import akka.stream.testkit.javadsl.TestSink
import akka.stream.testkit.scaladsl.TestSource
import org.apache.kafka.clients.consumer.ConsumerConfig
import org.apache.kafka.clients.producer.ProducerRecord
import org.apache.kafka.common.serialization.{StringDeserializer, StringSerializer}
import org.scalatest.{Matchers, WordSpec}

class SimpleKafkaStreamTest extends WordSpec with Matchers {

  "A person should be greeted" in new TestScope {
    startStream()
    send("World")
    requestNext() shouldBe "hello World"
  }

  trait TestScope extends E2EConfiguration with Kafka

  trait E2EConfiguration {
    implicit val actorSystem = ActorSystem("e2e-system")
    implicit val actorMaterializer = ActorMaterializer()
    val kafkaBootstrapServer = "192.168.99.100:9092"
    val sourceTopic = "person"
    val targetTopic = "greeting"
  }

  trait Kafka {
    this: E2EConfiguration =>

    private val consumerSettings = ConsumerSettings(actorSystem, new StringDeserializer, new StringDeserializer, Set(targetTopic))
      .withBootstrapServers(kafkaBootstrapServer)
      .withGroupId(UUID.randomUUID().toString)
      .withClientId(UUID.randomUUID().toString)
      .withProperty(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "latest")

    val kafkaInputSource =
      TestSource.probe[String].map( name => {
        new ProducerRecord[String, String] (sourceTopic, name)
    }).to(Producer.plainSink(ProducerSettings(actorSystem, new StringSerializer, new StringSerializer)
      .withBootstrapServers(bootstrapServers = kafkaBootstrapServer))).run()

    val kafkaOutput = Consumer.plainSource(consumerSettings).runWith(TestSink.probe(actorSystem))
    def requestNext() = kafkaOutput.requestNext.value

    def send(name: String) = kafkaInputSource.sendNext(name)

    def startStream() = {
      SimpleKafkaStream(sourceTopic = sourceTopic, targetTopic = targetTopic, kafkaBootstrapServer = kafkaBootstrapServer).run()
    }
  }
}

所以，这应该将“世界”写入主题“人”，并在“问候”主题中找回“你好世界”......偶尔会发生这种情况。但是，大多数情况下，我得到：

Expected OnNext(_), yet no element signaled during 3 seconds
java.lang.AssertionError: Expected OnNext(_), yet no element signaled during 3 seconds
    at akka.stream.testkit.TestSubscriber$ManualProbe.expectNext(StreamTestKit.scala:268)
    at akka.stream.testkit.TestSubscriber$ManualProbe.expectNext(StreamTestKit.scala:259)
    at akka.stream.testkit.TestSubscriber$Probe.requestNext(StreamTestKit.scala:631)
    at kafka.SimpleKafkaStreamTest$Kafka$class.requestNext(SimpleKafkaStreamTest.scala:56)
    at kafka.SimpleKafkaStreamTest$$anonfun$1$$anon$1.requestNext(SimpleKafkaStreamTest.scala:18)
    at kafka.SimpleKafkaStreamTest$$anonfun$1$$anon$1.<init>(SimpleKafkaStreamTest.scala:22)
    at kafka.SimpleKafkaStreamTest$$anonfun$1.apply$mcV$sp(SimpleKafkaStreamTest.scala:18)
    at kafka.SimpleKafkaStreamTest$$anonfun$1.apply(SimpleKafkaStreamTest.scala:18)
    at kafka.SimpleKafkaStreamTest$$anonfun$1.apply(SimpleKafkaStreamTest.scala:18)

Kafka根本没有收集到这些数据。我做错了什么？

Answer 1

是的，在没有百万网络的任何贡献的情况下，我自己想出来了。为了遇到遇到同样问题的其他人的利益，这里需要修复上述代码：

ConsumerConfig.AUTO_OFFSET_RESET_CONFIG需要"latest"，而不是"earliest"才能从队列中获取最新条目。
其次，上面的代码没有给ActorSystem一个正确关闭的机会，并通知Kafka两个消费者（测试代码中的一个，被测试代码中的一个）现在已经死了。如果没有这个，队列将保持锁定状态，直到会话超时期限（默认为30＆＃34;）过去，并且任何后续运行的测试都将无法读取Kafka队列。通过让测试类也扩展为BeforeAndAfterAll来修复，并在afterAll方法中包含Await.result (actorSystem.terminate(), 20.seconds)（10＆＃34;不够长）。
第三，我发现偏移提交有时会一再失败并立即重新安排，并且这可能会持续多达24＆＃34;秒（虽然我确定更长的时间）。这使得kafkaOutput.requestNext()（kafkaOutput实际上是TestSubscriber.Probe[String]）不适合目的;有必要使用kafkaOutput.requestNext(2.seconds)}（以便让代码有机会在try阻止AssertionError "Expected OnNext(_), yet no element signaled "形式的public class AppController extends Application { private static AppController mInstance; public Context context(){ return mInstance.getApplicationContext(); } @Override public void onTerminate() { context().deleteDatabase(DbHelper.DATABASE_NAME); super.onTerminate(); } } [等等] {{ 1}}并重试足够的时间，使其显着超过上述24＆＃34;期间。

Scala中Akka中的集成测试只是间歇性地传递

1 个答案: