Question

我的流程如下：

通知进入
通知产生0个或更多文档
文档以批处理大小N进行批处理
批次已发送以进行处理
在处理一些文档时可能会失败

任务：将失败的文档发送到步骤2。理想情况下，重试/延迟的次数很多，但是现在只要重新排队就足够了。我不想重新启动整个流，仅重试失败的元素。

我是Akka流的新手，我已经阅读了流文档，但是仍然不清楚什么是实现这样的正确方法。我应该使用某种策略还是广播？欢迎任何提示。

现在我有这样的东西：

Flow.of(Notification.class)
        .mapConcat(Notification::getDocuments)
        .grouped(50)
        .map(DocumentProcessor::process)
        .map(result -> {
            List<Document> succeeded = result.succeeded;
            List<Document> failed = result.failed;
            // what to do with failed?
        });

谢谢

Answer 1

分享我的解决方案。为简单起见，我将整数用作文档，当整数达到8时便完成文档处理；

import static java.util.concurrent.TimeUnit.MILLISECONDS;
import static java.util.concurrent.TimeUnit.SECONDS;
import static java.util.stream.Collectors.toList;
import static org.junit.Assert.fail;

import java.util.List;
import java.util.concurrent.CompletionStage;
import java.util.concurrent.Executors;
import java.util.stream.IntStream;

import akka.NotUsed;
import akka.actor.ActorSystem;
import akka.stream.ActorMaterializer;
import akka.stream.ClosedShape;
import akka.stream.DelayOverflowStrategy;
import akka.stream.Graph;
import akka.stream.KillSwitches;
import akka.stream.OverflowStrategy;
import akka.stream.SharedKillSwitch;
import akka.stream.UniformFanInShape;
import akka.stream.UniformFanOutShape;
import akka.stream.javadsl.Broadcast;
import akka.stream.javadsl.Flow;
import akka.stream.javadsl.GraphDSL;
import akka.stream.javadsl.Merge;
import akka.stream.javadsl.RunnableGraph;
import akka.stream.javadsl.Sink;
import akka.stream.javadsl.Source;
import scala.concurrent.duration.Duration;
import scala.concurrent.duration.FiniteDuration;

import org.assertj.core.api.SoftAssertions;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;

public class BulkRetryFlowProviderTest {
    private ActorSystem actorSystem;
    private SharedKillSwitch killSwitch;

    @Before
    public void setUp() {
        actorSystem = ActorSystem.create();
        killSwitch = KillSwitches.shared("my-kill-switch");
    }

    @After
    public void tearDown() {
        actorSystem.terminate();
    }


    @Test
    public void test() throws Exception {
        Source<Integer, NotUsed> source = Source.from(IntStream.range(0, 5)
            .boxed()
            .collect(toList()));

        // each input creates 5 items to process
        Flow<Integer, Integer, NotUsed> createDocuments = Flow.of(Integer.class)
            .mapConcat(i -> IntStream.range(0, 5).boxed().collect(toList()));

        // buffer items and do bulk processing
        Flow<Integer, Integer, NotUsed> bulkProcess = Flow.of(Integer.class)
            .groupedWithin(50, Duration.create(10, MILLISECONDS))
            .mapConcat(integers -> integers.stream()
                .map(i -> i + 1)
                .collect(toList()));

        // items are ready
        Flow<Integer, Integer, NotUsed> filterProcessed = Flow.of(Integer.class)
            .filter(i -> i > 7)
            .map(i -> {
                System.out.println("Done: " + i);
                return i;
            });

        // items should be processed again
        Flow<Integer, Integer, NotUsed> filterRecoverable = Flow.of(Integer.class)
            .filter(i -> i <= 7);

        Flow<Integer, Integer, NotUsed> bufferRetry = Flow.of(Integer.class)
            .buffer(3, OverflowStrategy.backpressure())
            .delay(FiniteDuration.apply(10, MILLISECONDS), DelayOverflowStrategy.backpressure());

        Graph<ClosedShape, CompletionStage<List<Integer>>> graph = GraphDSL.create(Sink.seq(), (builder, out) -> {
            UniformFanOutShape<Integer, Integer> broadcast =
                builder.add(Broadcast.create(2));
            UniformFanInShape<Integer, Integer> merge = builder.add(Merge.create(2));
            builder
                .from(builder.add(source).out())
                .via(builder.add(createDocuments))
                .viaFanIn(merge)
                .via(builder.add(bulkProcess))
                .viaFanOut(broadcast)
                .via(builder.add(filterProcessed))
                .via(builder.add(killSwitch.flow()))
                .to(out);

            builder
                .from(broadcast)
                .via(builder.add(filterRecoverable))
                .via(builder.add(bufferRetry))
                .toFanIn(merge);

            return ClosedShape.getInstance();
        });

        CompletionStage<List<Integer>> completionStage = RunnableGraph
            .fromGraph(graph)
            .run(ActorMaterializer.create(actorSystem)).exceptionally(e -> {
                fail("Stream filed: " + e);
                return null;
            });

        // give it some time to complete
        Executors.newCachedThreadPool().submit(() -> {
            try {
                Thread.sleep(10000);
            } catch (InterruptedException e) {
                e.printStackTrace();
            }
            killSwitch.shutdown();
        });

        List<Integer> result = completionStage.toCompletableFuture().get(10000, SECONDS);

        SoftAssertions.assertSoftly(softly -> {
            softly
                .assertThat(result)
                .hasSize(25);

            softly
                .assertThat(result)
                .allMatch(i -> i == 8);
        });
    }
}

Akka流：重试元素处理

1 个答案: