Hazelcast Jet Query

时间：2018-04-10 06:51:23

标签： hazelcast-jet

我有关于Hazelcast Jet的以下查询

用例如下

有一个应用程序（应用程序＆＃39; A＆＃39;，部署在群集中）使用Hazelcast IMDG并将数百万条记录/交易放入hazelcast IMap中。

已为此IMap配置了事件日志。

还有另一个应用程序（应用程序B，部署在集群中）实例化JetInstance并在每个节点上单独运行作业以处理记录。

目前，此作业从事件日志中读取数据并添加到IList中（参考 - hazelcast-jet-0.5.1 \ code-samples \ streaming \ map-journal-source \ src \ main \ java \ RemoteMapJournalSource.java）< / p>

当作业在多个节点上运行时，Event Journal中的记录由多个节点处理。这会在IList中产生多个条目。

是否可以确保，记录仅由“应用程序B”的一个节点处理。并且没有被其他节点处理以避免重复？

如果没有，这是否意味着该作业将由“应用程序B”的单个节点运行。集群？

以下是示例代码（应用B） 

        Pipeline p = Pipeline.create();
        p.drawFrom(Sources.<Integer, Integer, Integer>remoteMapJournal(MAP_NAME, clientConfig,
                e -> e.getType() == EntryEventType.ADDED, EventJournalMapEvent::getNewValue, true))
         .peek()
         .drainTo(Sinks.list(SINK_NAME));

        JobConfig jc= new JobConfig();
        jc.setProcessingGuarantee(ProcessingGuarantee.EXACTLY_ONCE);

        localJet.newJob(p,jc);

以下是完整的代码。

应用程序源代码。 

public class RemoteMapJournalSourceSrv1 {

private static final String MAP_NAME = "map";
private static final String SINK_NAME = "list";

public static void main(String[] args) throws Exception {
    System.setProperty("remoteHz.logging.type", "log4j");

    Config hzConfig = getConfig();
    HazelcastInstance remoteHz = startRemoteHzCluster(hzConfig);

    try {
        IMap<Integer, Integer> map = remoteHz.getMap(MAP_NAME);
        System.out.println("*************** Initial Map  address " +  map.size() );

        while(true) {
            System.out.println("***************map size "+map.size());  
            TimeUnit.SECONDS.sleep(20);
        }

    } finally {
        Hazelcast.shutdownAll();
    }
}

private static HazelcastInstance startRemoteHzCluster(Config config) {
    HazelcastInstance remoteHz = Hazelcast.newHazelcastInstance(config);
    return remoteHz;
}

private static Config getConfig() {
    Config config = new Config();
    // Add an event journal config for map which has custom capacity of 1000 (default 10_000)
    // and time to live seconds as 10 seconds (default 0 which means infinite)
    config.addEventJournalConfig(new EventJournalConfig().setEnabled(true)
                                                         .setMapName(MAP_NAME)
                                                         .setCapacity(10000)
                                                         .setTimeToLiveSeconds(100));
    return config;
}

这是应用程序B - 节点1示例代码 

public class RemoteMapJournalSourceCL1 {

private static final String MAP_NAME = "map";
private static final String SINK_NAME = "list";

public static void main(String[] args) throws Exception {
    System.setProperty("remoteHz.logging.type", "log4j");

      JetInstance localJet = startLocalJetCluster();

    try {
        ClientConfig clientConfig = new ClientConfig();
        GroupConfig groupConfig = new GroupConfig();

        clientConfig.getNetworkConfig().addAddress("localhost:5701");
        clientConfig.setGroupConfig(groupConfig);

        IList list1 = localJet.getList(SINK_NAME);

        int size1 = list1.size();

        System.out.println("***************List Initial size "+size1);

        Pipeline p = Pipeline.create();
        p.drawFrom(Sources.<Integer, Integer, Integer>remoteMapJournal(MAP_NAME, clientConfig,
                e -> e.getType() == EntryEventType.ADDED, EventJournalMapEvent::getNewValue, false))
         .peek()
         .drainTo(Sinks.list(SINK_NAME));

        JobConfig jc= new JobConfig();
        jc.setProcessingGuarantee(ProcessingGuarantee.EXACTLY_ONCE);
        localJet.newJob(p,jc);

        while(true){
            TimeUnit.SECONDS.sleep(10); 
            System.out.println("***************Read " + list1.size() + " entries from remote map journal.");
        }           

    } finally {
        Hazelcast.shutdownAll();
        Jet.shutdownAll();
    }

}

private static String getAddress(HazelcastInstance remoteHz) {
    Address address = remoteHz.getCluster().getLocalMember().getAddress();
    System.out.println("***************Remote address " + address.getHost() + ":" + address.getPort() );
    return address.getHost() + ":" + address.getPort();
}

private static JetInstance startLocalJetCluster() {
      JetInstance localJet = Jet.newJetInstance();
    return localJet;
}

这是应用程序B - 节点2示例代码 

public class RemoteMapJournalSourceCL2 {

private static final String MAP_NAME = "map";
private static final String SINK_NAME = "list";

public static void main(String[] args) throws Exception {
    System.setProperty("remoteHz.logging.type", "log4j");

      JetInstance localJet = startLocalJetCluster();

    try {
        ClientConfig clientConfig = new ClientConfig();
        GroupConfig groupConfig = new GroupConfig();

        clientConfig.getNetworkConfig().addAddress("localhost:5701");
        clientConfig.setGroupConfig(groupConfig);

        IList list1 = localJet.getList(SINK_NAME);
        int size1 = list1.size();
        System.out.println("***************List Initial size "+size1);


        Pipeline p = Pipeline.create();
        p.drawFrom(Sources.<Integer, Integer, Integer>remoteMapJournal(MAP_NAME, clientConfig,
                e -> e.getType() == EntryEventType.ADDED, EventJournalMapEvent::getNewValue, true))
         .peek()
         .drainTo(Sinks.list(SINK_NAME));

        JobConfig jc= new JobConfig();
        jc.setProcessingGuarantee(ProcessingGuarantee.EXACTLY_ONCE);

        localJet.newJob(p,jc);

        while(true){
            TimeUnit.SECONDS.sleep(10); 
            System.out.println("***************Read " + list1.size() + " entries from remote map journal.");
        }           
    } finally {
        Hazelcast.shutdownAll();
        Jet.shutdownAll();
    }

}
private static JetInstance startLocalJetCluster() {
      JetInstance localJet = Jet.newJetInstance();
    return localJet;
}

Hazelcast客户端 - 在Hazelcast地图中显示条目（应用A） 

public class HZClient {

  public static void main(String[] args) {

    ClientConfig clientConfig = new ClientConfig();
    GroupConfig groupConfig = new GroupConfig();

    clientConfig.getNetworkConfig().addAddress("localhost:5701");
    clientConfig.setGroupConfig(groupConfig);

    HazelcastInstance client = HazelcastClient.newHazelcastClient(clientConfig);
    IMap<Integer, Integer> map = client.getMap("map");
    Scanner in = new Scanner(System.in);
    int startIndex= 0;
    int endIndex= 0;

    while(true) {
        if(args !=null && args.length > 0 && args[0].equals("BATCH")) {

            System.out.println("Please input the batch size");
            int b = in.nextInt();
            startIndex= endIndex + 1;
            endIndex+= b;
            System.out.println("Batch starts from  "+ startIndex +"ends at"+endIndex);
            putBatch(map,startIndex,endIndex);

        }
        else {
            System.out.println("Please input the map entry");
            int a = in.nextInt();
            System.out.println("You entered integer "+a);
            put(map,a,a);
        }
    }
}

public static void putBatch(IMap map,int startIndex, int endIndex) {
    int index= startIndex;
    System.out.println("Start Index" + startIndex +"End Index"+endIndex );
    while(index<=endIndex){
        System.out.println("Map Values"+ index);
        put(map,index,index);
        index+=1;
    }

}

public static void put(IMap map,int key,int value) {
    map.set(key, value);
}

以下是执行此操作的步骤。

  1. 运行应用程序A - Java程序RemoteMapJournalSourceSrv1

  2. 运行应用程序B节点1 - Java程序RemoteMapJournalSourceCL1

  3. 运行应用程序B节点2 - Java程序RemoteMapJournalSourceCL2

  4. 运行应用程序A的Hazelcast客户端 - Java程序HZClient

    5. 此客户端程序根据控制台输入将条目放入地图。请提供整数输入。

    观察

    执行时，.peek（）记录应用程序B的两个节点的值，并且在应用程序A映射中插入1个条目时列表计数变为2。

1 个答案:

答案 0 :(得分：0)

您似乎正在从两个Jet客户端提交两个独立的作业。每个作业都会收到所有IMap事件日志项并将它们推送到同一个IList，因此预期的结果是IList包含每个项的两个实例。

请记住，您只能从Jet客户端提交作业，但它实际上在Jet集群内部同时在其所有成员上运行。如果您只想在接收器中存储一个数据副本，请不要两次提交相同的作业。

相关问题
最新问题