如何评估kafka流应用程序中的消耗时间

时间:2018-01-05 08:05:27

标签: apache-kafka apache-kafka-streams

我有1.0.0 kafka流应用程序,有两个类,如下所示' class FilterByPolicyStreamsApp'和'类FilterByPolicyTransformerSupplier'。在我的应用程序中,我阅读事件,执行一些条件检查并转发到另一个主题中的相同kafka。我能够通过'eventsForwardTimeInMs'来获得制作时间。 FilterByPolicyTransformerSupplier类中的变量。但是我无法获得消耗时间(有和没有(反)序列化)。我怎么会得到这个时间?请帮帮我。

FilterByPolicyStreamsApp .java:

public class FilterByPolicyStreamsApp implements CommandLineRunner {
    String policyKafkaTopicName="policy";
    String policyFilterDataKafkaTopicName = "policy.filter.data";
    String bootstrapServers="11.1.1.1:9092";
    String sampleEventsKafkaTopicName = 'sample-.*";
    String applicationId="filter-by-policy-app";
    String policyFilteredEventsKafkaTopicName = "policy.filter.events";

    public static void main(String[] args) {
        SpringApplication.run(FilterByPolicyStreamsApp.class, args);
    }

    @Override
    public void run(String... arg0) {
        String policyGlobalTableName = policyKafkaTopicName + ".table";
        String policyFilterDataGlobalTable = policyFilterDataKafkaTopicName + ".table";

        Properties config = new Properties();
        config.put(StreamsConfig.APPLICATION_ID_CONFIG, applicationId);
        config.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers);
        config.put(StreamsConfig.DEFAULT_TIMESTAMP_EXTRACTOR_CLASS_CONFIG, WallclockTimestampExtractor.class);

        KStreamBuilder builder = new KStreamBuilder();
        builder.globalTable(Serdes.String(), new JsonSerde<>(List.class), policyKafkaTopicName,
                policyGlobalTableName);
        builder.globalTable(Serdes.String(), new JsonSerde<>(PolicyFilterData.class), policyFilterDataKafkaTopicName,
                policyFilterDataGlobalTable);

        KStream<String, SampleEvent> events = builder.stream(Serdes.String(),
                new JsonSerde<>(SampleEvent.class), Pattern.compile(sampleEventsKafkaTopicName));
        events = events.transform(new FilterByPolicyTransformerSupplier(policyGlobalTableName,
                policyFilterDataGlobalTable));

        events.to(Serdes.String(), new JsonSerde<>(SampleEvent.class), policyFilteredEventsKafkaTopicName);

        KafkaStreams streams = new KafkaStreams(builder, config);

        streams.start();
        streams.setUncaughtExceptionHandler(new Thread.UncaughtExceptionHandler() {
            @Override
            public void uncaughtException(Thread t, Throwable e) {
                logger.error(e.getMessage(), e);
            }
        });
    }
}

FilterByPolicyTransformerSupplier.java:

public class FilterByPolicyTransformerSupplier
            implements TransformerSupplier<String, SampleEvent, KeyValue<String, SampleEvent>> {
        private String policyGlobalTableName;
        private String policyFilterDataGlobalTable;

        public FilterByPolicyTransformerSupplier(String policyGlobalTableName,
            String policyFilterDataGlobalTable) {
            this.policyGlobalTableName = policyGlobalTableName;
            this.policyFilterDataGlobalTable = policyFilterDataGlobalTable;
        }
        @Override
        public Transformer<String, SampleEvent, KeyValue<String, SampleEvent>> get() {
            return new Transformer<String, SampleEvent, KeyValue<String, SampleEvent>>() {
                private KeyValueStore<String, List<String>> policyStore;
                private KeyValueStore<String, PolicyFilterData> policyMetadataStore;
                private ProcessorContext context;

                @Override
                public void close() {
                }

                @Override
                public void init(ProcessorContext context) {
                    this.context = context;
                    // Call punctuate every 1 second
                    this.context.schedule(1000);
                    policyStore = (KeyValueStore<String, List<String>>) this.context
                            .getStateStore(policyGlobalTableName);
                    policyMetadataStore = (KeyValueStore<String, PolicyFilterData>) this.context
                            .getStateStore(policyFilterDataGlobalTable);
                }

                @Override
                public KeyValue<String, SampleEvent> punctuate(long arg0) {
                    return null;
                }

                @Override
                public KeyValue<String, SampleEvent> transform(String key, SampleEvent event) {
                    long eventsForwardTimeInMs = 0;
                    long forwardedEventCouunt = 0;
                    List<String> policyIds = policyStore.get(event.getCustomerCode().toLowerCase());
                    if (policyIds != null) {
                        for (String policyId : policyIds) {

                            /*
                                PolicyFilterData policyFilterMetadata = policyMetadataStore.get(policyId);

                                Do some condition checks on the event. If it satisfies then will forward them.
                                if(policyFilterMetadata == null){
                                    continue;
                                }
                            */

                            // Using context forward as event can map to multiple policies
                            long startForwardTime = System.currentTimeMillis();
                            context.forward(policyId, event);
                            forwardedEventCouunt++;
                            eventsForwardTimeInMs += System.currentTimeMillis() - startForwardTime;
                        }
                    }
                    return null;
                }
            };
    }
}

0 个答案:

没有答案