我正在尝试使用Spring Cloud Stream框架构建一个简单的Kafka Streams应用程序。我可以连接到流以推送原始数据进行处理。但是，当我尝试处理流以按键进行事件计数时，运行应用程序时出现Serde class not found: org.apache.kafka.common.serialization.Serde$StringSerde异常。我检入了包含库的项目，并且可以找到Serde类，但它并不缺少。我不确定为什么在运行时没有加载！
下面是我的源文件。
com.pgp.learn.kafka.analytics.AnalyticsApplication
package com.pgp.learn.kafka.analytics;
import com.pgp.learn.kafka.analytics.model.PageViewEvent;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.kafka.common.serialization.Serde;
import org.apache.kafka.common.serialization.Serdes;
import org.apache.kafka.streams.KeyValue;
import org.apache.kafka.streams.kstream.*;
import org.springframework.boot.ApplicationArguments;
import org.springframework.boot.ApplicationRunner;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.cloud.stream.annotation.EnableBinding;
import org.springframework.cloud.stream.annotation.Input;
import org.springframework.cloud.stream.annotation.StreamListener;
import org.springframework.kafka.support.KafkaHeaders;
import org.springframework.messaging.Message;
import org.springframework.messaging.MessageChannel;
import org.springframework.messaging.handler.annotation.SendTo;
import org.springframework.messaging.support.MessageBuilder;
import org.springframework.stereotype.Component;
import java.util.Arrays;
import java.util.List;
import java.util.Random;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
@SpringBootApplication
@EnableBinding(AnalyticsBinding.class)
public class AnalyticsApplication {
public static void main(String[] args) {
SpringApplication.run(AnalyticsApplication.class, args);
}
@Component
public static class PageViewEventSource implements ApplicationRunner {
private final MessageChannel pageViewOut;
private final Log log = LogFactory.getLog(getClass());
public PageViewEventSource(AnalyticsBinding binding) {
this.pageViewOut = binding.pageViewsOut();
}
@Override
public void run(ApplicationArguments args) throws Exception {
List<String> names = Arrays.asList("Peter", "Tom", "Ady", "Nency", "George", "Kevin", "Chelsey", "Thomas");
List<String> pages = Arrays.asList("About", "Contact", "Blogs", "Gallery", "Music", "Site-map", "News");
Runnable runner = () -> {
String rName = names.get(new Random().nextInt(names.size()));
String rPage = pages.get(new Random().nextInt(pages.size()));
PageViewEvent event = new PageViewEvent(rName, rPage, Math.random() > .5 ? 10L : 1000L);
Message<PageViewEvent> message = MessageBuilder
.withPayload(event)
.setHeader(KafkaHeaders.MESSAGE_KEY, event.getUserId().getBytes())
.build();
try {
this.pageViewOut.send(message);
log.info("Sent: " + message.toString());
} catch (Exception e) {
log.error(e);
}
};
Executors.newScheduledThreadPool(1).scheduleAtFixedRate(runner, 1, 1, TimeUnit.SECONDS);
}
}
@Component
public static class PageViewEventProcessor {
private final Log log = LogFactory.getLog(getClass());
@StreamListener
@SendTo(AnalyticsBinding.PAGE_COUNT_OUT)
public static KStream<String, Long> process(
@Input(AnalyticsBinding.PAGE_VIEW_IN_CHANNEL) KStream<String, PageViewEvent> events) {
return events
.filter((s, pageViewEvent) -> pageViewEvent.getDuration() > 10)
.map((s, pageViewEvent) -> new KeyValue<>(pageViewEvent.getPage(), 0L))
.groupByKey()
//Also tried using below line, but no luck
//.groupByKey(Serialized.with(Serdes.String(), Serdes.Long()))
.count(Materialized.as(AnalyticsBinding.PAGE_COUNT_MV))
.toStream();
}
}
}
com.pgp.learn.kafka.analytics.AnalyticsBinding
package com.pgp.learn.kafka.analytics;
import com.pgp.learn.kafka.analytics.model.PageViewEvent;
import org.apache.kafka.streams.kstream.KStream;
import org.springframework.cloud.stream.annotation.Input;
import org.springframework.cloud.stream.annotation.Output;
import org.springframework.messaging.MessageChannel;
public interface AnalyticsBinding {
String PAGE_VIEW_OUT_CHANNEL = "pvout";
String PAGE_VIEW_IN_CHANNEL = "pvin";
String PAGE_COUNT_MV = "pcmvc";
String PAGE_COUNT_OUT = "pcout";
String PAGE_COUNT_IN = "pcin";
@Input (PAGE_VIEW_IN_CHANNEL)
KStream<String, PageViewEvent> pageViewsIn();
@Output (PAGE_VIEW_OUT_CHANNEL)
MessageChannel pageViewsOut();
@Output (PAGE_COUNT_OUT)
KStream<String, Long> pageCountOut();
}
application.properties
# defaults
spring.cloud.stream.kafka.streams.binder.configuration.commit.interval.mms=1000
spring.cloud.stream.kafka.streams.binder.configuration.default.key.serde=org.apache.kafka.common.serialization.Serde$StringSerde
spring.cloud.stream.kafka.streams.binder.configuration.default.value.serde=org.apache.kafka.common.serialization.Serde$StringSerde
spring.cloud.stream.kafka.binder.brokers=54.173.206.255
#
# page views out
spring.cloud.stream.bindings.pvout.destination=pvs
spring.cloud.stream.bindings.pvout.producer.header-mode=raw
#
# page views in
spring.cloud.stream.bindings.pvin.destination=pvs
spring.cloud.stream.bindings.pvin.consumer.header-mode=raw
#
# page count out
spring.cloud.stream.bindings.pcout.destination=pcs
spring.cloud.stream.bindings.pcout.producer.use-native-encoding=true
spring.cloud.stream.kafka.streams.bindings.pcout.producer.keySerde=org.apache.kafka.common.serialization.Serdes$StringSerde
spring.cloud.stream.kafka.streams.bindings.pcout.producer.valueSerde=org.apache.kafka.common.serialization.Serdes$LongSerde
pom.xml
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-parent</artifactId>
<version>2.1.1.RELEASE</version>
<relativePath/> <!-- lookup parent from repository -->
</parent>
<groupId>com.pgp.learn.kafka</groupId>
<artifactId>analytics</artifactId>
<version>0.0.1-SNAPSHOT</version>
<name>analytics</name>
<description>Demo project for Spring Boot</description>
<properties>
<java.version>1.8</java.version>
<spring-cloud.version>Greenwich.RC2</spring-cloud.version>
</properties>
<dependencies>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.cloud</groupId>
<artifactId>spring-cloud-stream</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.cloud</groupId>
<artifactId>spring-cloud-stream-binder-kafka</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.cloud</groupId>
<artifactId>spring-cloud-stream-binder-kafka-streams</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.kafka</groupId>
<artifactId>spring-kafka</artifactId>
</dependency>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
<optional>true</optional>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.springframework.cloud</groupId>
<artifactId>spring-cloud-stream-test-support</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.springframework.kafka</groupId>
<artifactId>spring-kafka-test</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
<dependencyManagement>
<dependencies>
<dependency>
<groupId>org.springframework.cloud</groupId>
<artifactId>spring-cloud-dependencies</artifactId>
<version>${spring-cloud.version}</version>
<type>pom</type>
<scope>import</scope>
</dependency>
</dependencies>
</dependencyManagement>
<build>
<plugins>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
</plugin>
</plugins>
</build>
<repositories>
<repository>
<id>spring-milestones</id>
<name>Spring Milestones</name>
<url>https://repo.spring.io/milestone</url>
</repository>
</repositories>
</project>
运行时发生异常
java.lang.IllegalStateException: java.lang.IllegalStateException: Serde class not found:
at org.springframework.cloud.stream.binder.kafka.streams.KafkaStreamsStreamListenerSetupMethodOrchestrator.adaptAndRetrieveInboundArguments(KafkaStreamsStreamListenerSetupMethodOrchestrator.java:308) ~[spring-cloud-stream-binder-kafka-streams-2.1.0.RC4.jar:2.1.0.RC4]
at org.springframework.cloud.stream.binder.kafka.streams.KafkaStreamsStreamListenerSetupMethodOrchestrator.orchestrateStreamListenerSetupMethod(KafkaStreamsStreamListenerSetupMethodOrchestrator.java:164) ~[spring-cloud-stream-binder-kafka-streams-2.1.0.RC4.jar:2.1.0.RC4]
at org.springframework.cloud.stream.binding.StreamListenerAnnotationBeanPostProcessor.doPostProcess(StreamListenerAnnotationBeanPostProcessor.java:195) ~[spring-cloud-stream-2.1.0.RC4.jar:2.1.0.RC4]
at org.springframework.cloud.stream.binding.StreamListenerAnnotationBeanPostProcessor.lambda$postProcessAfterInitialization$0(StreamListenerAnnotationBeanPostProcessor.java:167) ~[spring-cloud-stream-2.1.0.RC4.jar:2.1.0.RC4]
at java.lang.Iterable.forEach(Iterable.java:75) ~[na:1.8.0_161]
at org.springframework.cloud.stream.binding.StreamListenerAnnotationBeanPostProcessor.injectAndPostProcessDependencies(StreamListenerAnnotationBeanPostProcessor.java:285) ~[spring-cloud-stream-2.1.0.RC4.jar:2.1.0.RC4]
at org.springframework.cloud.stream.binding.StreamListenerAnnotationBeanPostProcessor.afterSingletonsInstantiated(StreamListenerAnnotationBeanPostProcessor.java:105) ~[spring-cloud-stream-2.1.0.RC4.jar:2.1.0.RC4]
at org.springframework.beans.factory.support.DefaultListableBeanFactory.preInstantiateSingletons(DefaultListableBeanFactory.java:863) ~[spring-beans-5.1.3.RELEASE.jar:5.1.3.RELEASE]
at org.springframework.context.support.AbstractApplicationContext.finishBeanFactoryInitialization(AbstractApplicationContext.java:863) ~[spring-context-5.1.3.RELEASE.jar:5.1.3.RELEASE]
at org.springframework.context.support.AbstractApplicationContext.refresh(AbstractApplicationContext.java:546) ~[spring-context-5.1.3.RELEASE.jar:5.1.3.RELEASE]
at org.springframework.boot.web.servlet.context.ServletWebServerApplicationContext.refresh(ServletWebServerApplicationContext.java:142) ~[spring-boot-2.1.1.RELEASE.jar:2.1.1.RELEASE]
at org.springframework.boot.SpringApplication.refresh(SpringApplication.java:775) [spring-boot-2.1.1.RELEASE.jar:2.1.1.RELEASE]
at org.springframework.boot.SpringApplication.refreshContext(SpringApplication.java:397) [spring-boot-2.1.1.RELEASE.jar:2.1.1.RELEASE]
at org.springframework.boot.SpringApplication.run(SpringApplication.java:316) [spring-boot-2.1.1.RELEASE.jar:2.1.1.RELEASE]
at org.springframework.boot.SpringApplication.run(SpringApplication.java:1260) [spring-boot-2.1.1.RELEASE.jar:2.1.1.RELEASE]
at org.springframework.boot.SpringApplication.run(SpringApplication.java:1248) [spring-boot-2.1.1.RELEASE.jar:2.1.1.RELEASE]
at com.pgp.learn.kafka.analytics.AnalyticsApplication.main(AnalyticsApplication.java:35) [classes/:na]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[na:1.8.0_161]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[na:1.8.0_161]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[na:1.8.0_161]
at java.lang.reflect.Method.invoke(Method.java:498) ~[na:1.8.0_161]
at org.springframework.boot.maven.AbstractRunMojo$LaunchRunner.run(AbstractRunMojo.java:558) [spring-boot-maven-plugin-2.1.1.RELEASE.jar:2.1.1.RELEASE]
at java.lang.Thread.run(Thread.java:748) [na:1.8.0_161]
Caused by: java.lang.IllegalStateException: Serde class not found:
at org.springframework.cloud.stream.binder.kafka.streams.KeyValueSerdeResolver.getKeySerde(KeyValueSerdeResolver.java:177) ~[spring-cloud-stream-binder-kafka-streams-2.1.0.RC4.jar:2.1.0.RC4]
at org.springframework.cloud.stream.binder.kafka.streams.KeyValueSerdeResolver.getInboundKeySerde(KeyValueSerdeResolver.java:74) ~[spring-cloud-stream-binder-kafka-streams-2.1.0.RC4.jar:2.1.0.RC4]
at org.springframework.cloud.stream.binder.kafka.streams.KafkaStreamsStreamListenerSetupMethodOrchestrator.adaptAndRetrieveInboundArguments(KafkaStreamsStreamListenerSetupMethodOrchestrator.java:246) ~[spring-cloud-stream-binder-kafka-streams-2.1.0.RC4.jar:2.1.0.RC4]
... 22 common frames omitted
Caused by: java.lang.ClassNotFoundException: org.apache.kafka.common.serialization.Serde$StringSerde
at java.net.URLClassLoader.findClass(URLClassLoader.java:381) ~[na:1.8.0_161]
at java.lang.ClassLoader.loadClass(ClassLoader.java:424) ~[na:1.8.0_161]
at java.lang.ClassLoader.loadClass(ClassLoader.java:357) ~[na:1.8.0_161]
at java.lang.Class.forName0(Native Method) ~[na:1.8.0_161]
at java.lang.Class.forName(Class.java:348) ~[na:1.8.0_161]
at org.apache.kafka.common.utils.Utils.loadClass(Utils.java:333) ~[kafka-clients-2.0.1.jar:na]
at org.apache.kafka.common.utils.Utils.newInstance(Utils.java:322) ~[kafka-clients-2.0.1.jar:na]
at org.springframework.cloud.stream.binder.kafka.streams.KeyValueSerdeResolver.getKeySerde(KeyValueSerdeResolver.java:171) ~[spring-cloud-stream-binder-kafka-streams-2.1.0.RC4.jar:2.1.0.RC4]
... 24 common frames omitted
在这里找到我做错的任何帮助吗？我是Kafka和Stream处理的新手。因此，如果我犯了任何愚蠢的错误，请原谅。
答案 0 :(得分：4)
您的配置属性中有一个错字：
spring.cloud.stream.kafka.streams.binder.configuration.default.key.serde=org.apache.kafka.common.serialization.Serde$StringSerde
在 Serde 中缺少 的 。应该是...serialization.Serdes$StringSerde