Spring Cloud Stream多主题事务管理

时间：2019-08-06 15:38:14

标签： java apache-kafka spring-kafka spring-cloud-stream

我正在尝试使用Java创建PoC应用程序，以弄清楚使用Kafka进行消息发布时如何在Spring Cloud Stream中进行事务管理。我尝试模拟的用例是一个接收消息的处理器。然后，它进行一些处理，并生成目的地为两个单独主题的两个新消息。我希望能够将两个消息发布为一个事务。因此，如果发布第二条消息失败，我想滚动（而不是提交）第一条消息。 Spring Cloud Stream是否支持这种用例？

我已经设置了@Transactional批注，并且可以看到在将消息传递给使用者之前开始了全局事务。但是，当我尝试通过MessageChannel.send()方法发布消息时，可以看到在KafkaProducerMessageHandler类的handleRequestMessage()方法中新的本地事务已启动和完成。这意味着消息的发送不参与全局事务。因此，如果在发布第一条消息后引发异常，则该消息将不会回滚。全局事务被回滚，但是自从第一条消息已经提交以来，它实际上并没有做任何事情。

spring:
  cloud:
    stream:
      kafka:
        binder:
          brokers: localhost:9092
          transaction:
            transaction-id-prefix: txn.
            producer: # these apply to all producers that participate in the transaction
              partition-key-extractor-name: partitionKeyExtractorStrategy
              partition-selector-name: partitionSelectorStrategy
              partition-count: 3
              configuration:
               acks: all
               enable:
                 idempotence: true
               retries: 10
        bindings:
          input-customer-data-change-topic:
            consumer:
              configuration:
                isolation:
                  level: read_committed
              enable-dlq: true
      bindings:
        input-customer-data-change-topic:
          content-type: application/json
          destination: com.fis.customer
          group: com.fis.ec
          consumer:
            partitioned: true
            max-attempts: 1
        output-name-change-topic:
          content-type: application/json
          destination: com.fis.customer.name          
        output-email-change-topic:
          content-type: application/json
          destination: com.fis.customer.email

@SpringBootApplication
@EnableBinding(CustomerDataChangeStreams.class)
public class KafkaCloudStreamCustomerDemoApplication
{
   public static void main(final String[] args)
   {
      SpringApplication.run(KafkaCloudStreamCustomerDemoApplication.class, args);
   }
}

public interface CustomerDataChangeStreams
{
   @Input("input-customer-data-change-topic")
   SubscribableChannel inputCustomerDataChange();

   @Output("output-email-change-topic")
   MessageChannel outputEmailDataChange();

   @Output("output-name-change-topic")
   MessageChannel outputNameDataChange();
}

@Component
public class CustomerDataChangeListener
{
   @Autowired
   private CustomerDataChangeProcessor mService;

   @StreamListener("input-customer-data-change-topic")
   public Message<String> handleCustomerDataChangeMessages(
      @Payload final ImmutableCustomerDetails customerDetails)
   {
      return mService.processMessage(customerDetails);
   }
}

@Component
public class CustomerDataChangeProcessor
{
   private final CustomerDataChangeStreams mStreams;

   @Value("${spring.cloud.stream.bindings.output-email-change-topic.destination}")
   private String mEmailChangeTopic;

   @Value("${spring.cloud.stream.bindings.output-name-change-topic.destination}")
   private String mNameChangeTopic;

   public CustomerDataChangeProcessor(final CustomerDataChangeStreams streams)
   {
      mStreams = streams;
   }

   public void processMessage(final CustomerDetails customerDetails)
   {
      try
      {
         sendNameMessage(customerDetails);
         sendEmailMessage(customerDetails);
      }
      catch (final JSONException ex)
      {
         LOGGER.error("Failed to send messages.", ex);
      }
   }

   public void sendNameMessage(final CustomerDetails customerDetails)
      throws JSONException
   {
      final JSONObject nameChangeDetails = new JSONObject();
      nameChangeDetails.put(KafkaConst.BANK_ID_KEY, customerDetails.bankId());
      nameChangeDetails.put(KafkaConst.CUSTOMER_ID_KEY, customerDetails.customerId());
      nameChangeDetails.put(KafkaConst.FIRST_NAME_KEY, customerDetails.firstName());
      nameChangeDetails.put(KafkaConst.LAST_NAME_KEY, customerDetails.lastName());
      final String action = customerDetails.action();
      nameChangeDetails.put(KafkaConst.ACTION_KEY, action);

      final MessageChannel nameChangeMessageChannel = mStreams.outputNameDataChange();
      emailChangeMessageChannel.send(MessageBuilder.withPayload(nameChangeDetails.toString())
         .setHeader(MessageHeaders.CONTENT_TYPE, MimeTypeUtils.APPLICATION_JSON)
         .setHeader(KafkaHeaders.TOPIC, mNameChangeTopic).build());

      if ("fail_name_illegal".equalsIgnoreCase(action))
      {
         throw new IllegalArgumentException("Customer name failure!");
      }
   }

   public void sendEmailMessage(final CustomerDetails customerDetails) throws JSONException
   {
      final JSONObject emailChangeDetails = new JSONObject();
      emailChangeDetails.put(KafkaConst.BANK_ID_KEY, customerDetails.bankId());
      emailChangeDetails.put(KafkaConst.CUSTOMER_ID_KEY, customerDetails.customerId());
      emailChangeDetails.put(KafkaConst.EMAIL_ADDRESS_KEY, customerDetails.email());
      final String action = customerDetails.action();
      emailChangeDetails.put(KafkaConst.ACTION_KEY, action);

      final MessageChannel emailChangeMessageChannel = mStreams.outputEmailDataChange();
      emailChangeMessageChannel.send(MessageBuilder.withPayload(emailChangeDetails.toString())
         .setHeader(MessageHeaders.CONTENT_TYPE, MimeTypeUtils.APPLICATION_JSON)
         .setHeader(KafkaHeaders.TOPIC, mEmailChangeTopic).build());

      if ("fail_email_illegal".equalsIgnoreCase(action))
      {
         throw new IllegalArgumentException("E-mail address failure!");
      }
   }
}

编辑

我们越来越近了。本地事务不再创建。但是，即使发生异常，全局事务仍然会被提交。据我所知，异常不会传播到TransactionTemplate.execute()方法。因此，事务被提交。似乎MessageProducerSupport方法中的sendMessage()类“吞噬”了catch子句中的异常。如果定义了错误通道，则会向其发布消息，因此不会重新引发异常。我尝试关闭错误通道（spring.cloud.stream.kafka.binder.transaction.producer.error-channel-enabled = false），但并没有关闭错误通道。因此，仅出于测试目的，我只需在调试器中将错误通道设置为null即可强制抛出异常。这似乎做到了。但是，即使我已将原始消费者的max-attempts设置为1，原始消息仍会继续传递给初始消费者。

1 个答案:

答案 0 :(得分：0)

请参见the documentation。

  

spring.cloud.stream.kafka.binder.transaction.transactionIdPrefix

     

在活页夹中启用事务。请参阅Kafka文档中的transaction.id和spring-kafka文档中的Transactions。启用事务后，单个生产者属性将被忽略，所有生产者都将使用spring.cloud.stream.kafka.binder.transaction.producer。*属性。

     

默认为空（无交易）

     

spring.cloud.stream.kafka.binder.transaction.producer.*

     

交易绑定中生产者的全球生产者属性。请参阅spring.cloud.stream.kafka.binder.transaction.transactionIdPrefix和Kafka生产者属性以及所有绑定程序支持的常规生产者属性。

     

默认：查看各个生产者属性。

您必须配置共享的全局生产者。

请勿添加@Transactional-容器将在提交交易之前启动交易并将偏移量发送到交易。

如果侦听器引发异常，则事务将回滚，DefaultAfterRollbackPostProcessor将重新搜索主题/分区，以便重新发送记录。

编辑

活页夹的事务管理器配置中存在一个错误，该错误导致输出绑定启动新的本地事务。

要解决此问题，请使用以下容器定制程序bean重新配置TM ... 

@Bean
public ListenerContainerCustomizer<AbstractMessageListenerContainer<byte[], byte[]>> customizer() {
    return (container, dest, group) -> {
        KafkaTransactionManager<?, ?> tm = (KafkaTransactionManager<?, ?>) container.getContainerProperties()
                .getTransactionManager();
        tm.setTransactionSynchronization(AbstractPlatformTransactionManager.SYNCHRONIZATION_ON_ACTUAL_TRANSACTION);
    };
}

EDIT2

您不能使用活页夹的DLQ支持，因为从容器的角度来看，交付是成功的。我们需要将异常传播到容器以强制回滚。因此，您需要将无效字母移到AfterRollbackProcessor上。这是我完整的测试课：

@SpringBootApplication
@EnableBinding(Processor.class)
public class So57379575Application {

    public static void main(String[] args) {
        SpringApplication.run(So57379575Application.class, args);
    }

    @Autowired
    private MessageChannel output;

    @StreamListener(Processor.INPUT)
    public void listen(String in) {
        System.out.println("in:" + in);
        this.output.send(new GenericMessage<>(in.toUpperCase()));
        if (in.equals("two")) {
            throw new RuntimeException("fail");
        }
    }

    @KafkaListener(id = "so57379575", topics = "so57379575out")
    public void listen2(String in) {
        System.out.println("out:" + in);
    }

    @KafkaListener(id = "so57379575DLT", topics = "so57379575dlt")
    public void listen3(String in) {
        System.out.println("dlt:" + in);
    }

    @Bean
    public ApplicationRunner runner(KafkaTemplate<byte[], byte[]> template) {
        return args -> {
            template.send("so57379575in", "one".getBytes());
            template.send("so57379575in", "two".getBytes());
        };
    }

    @Bean
    public ListenerContainerCustomizer<AbstractMessageListenerContainer<byte[], byte[]>> customizer(
            KafkaTemplate<Object, Object> template) {

        return (container, dest, group) -> {
            // enable transaction synchronization
            KafkaTransactionManager<?, ?> tm = (KafkaTransactionManager<?, ?>) container.getContainerProperties()
                    .getTransactionManager();
            tm.setTransactionSynchronization(AbstractPlatformTransactionManager.SYNCHRONIZATION_ON_ACTUAL_TRANSACTION);
            // container dead-lettering
            DefaultAfterRollbackProcessor<? super byte[], ? super byte[]> afterRollbackProcessor =
                    new DefaultAfterRollbackProcessor<>(new DeadLetterPublishingRecoverer(template,
                            (ex, tp) -> new TopicPartition("so57379575dlt", -1)), 0);
            container.setAfterRollbackProcessor(afterRollbackProcessor);
        };
    }

}

和

spring:
  kafka:
    bootstrap-servers:
    - 10.0.0.8:9092
    - 10.0.0.8:9093
    - 10.0.0.8:9094
    consumer:
      auto-offset-reset: earliest
      enable-auto-commit: false
      properties:
        isolation.level: read_committed
  cloud:
    stream:
      bindings:
        input:
          destination: so57379575in
          group: so57379575in
          consumer:
            max-attempts: 1
        output:
          destination: so57379575out
      kafka:
        binder:
          transaction:
            transaction-id-prefix: so57379575tx.
            producer:
              configuration:
                acks: all
                retries: 10

#logging:
#  level:
#    org.springframework.kafka: trace
#    org.springframework.transaction: trace

和

in:two
2019-08-07 12:43:33.457 ERROR 36532 --- [container-0-C-1] o.s.integration.handler.LoggingHandler   : org.springframework.messaging.MessagingException: Exception thrown while 
...
Caused by: java.lang.RuntimeException: fail
...
in:one
dlt:two
out:ONE
相关问题
最新问题