kafka到具有融合源代码的hdfs

时间:2018-07-27 13:44:33

标签: java apache-kafka apache-kafka-connect

对于我的项目要求,我需要从融合的Java代码中构建一个类,以将数据从kafka主题写入hdfs文件系统。

它实际上可以在带有独立连接的CLI中工作,但是我需要对成功构建的源代码执行相同的操作。

我对SinkTask和hdfsConnector类有问题。 put方法中显示异常。

以下是我的课程代码:

package io.confluent.connect.hdfs;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;

import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.TimeUnit;

import org.apache.kafka.connect.errors.ConnectException;
import org.apache.kafka.connect.sink.SinkConnector;
import org.apache.kafka.connect.sink.SinkRecord;
import org.apache.kafka.connect.sink.SinkTaskContext;

import io.confluent.connect.avro.AvroData;
import io.confluent.connect.hdfs.avro.AvroFormat;
import io.confluent.connect.hdfs.partitioner.DefaultPartitioner;
import io.confluent.connect.storage.common.StorageCommonConfig;
import io.confluent.connect.storage.partitioner.PartitionerConfig;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.kafka.common.TopicPartition;
import org.apache.kafka.common.config.ConfigDef;

public class main{
    private static Map<String, String> props = new HashMap<>(); 
    protected static final TopicPartition TOPIC_PARTITION = new TopicPartition(TOPIC, PARTITION);
      protected static String url = "hdfs://localhost:9000";
    protected static SinkTaskContext context;

    public static void main(String[] args) {
        HdfsSinkConnector hk = new HdfsSinkConnector();
        HdfsSinkTask h = new HdfsSinkTask();
        props.put(StorageCommonConfig.STORE_URL_CONFIG, url);
        props.put(HdfsSinkConnectorConfig.HDFS_URL_CONFIG, url);
        props.put(HdfsSinkConnectorConfig.FLUSH_SIZE_CONFIG, "3");
        props.put(HdfsSinkConnectorConfig.FORMAT_CLASS_CONFIG, AvroFormat.class.getName());    
        try {
            hk.start(props);
            Collection<SinkRecord> sinkRecords = new ArrayList<>();
            SinkRecord record = new SinkRecord("test", 0, null, null, null, null, 0);
            sinkRecords.add(record);
            h.initialize(context);
            h.put(sinkRecords);
            hk.stop();
        } catch (Exception e) {
            throw new ConnectException("Couldn't start HdfsSinkConnector due to configuration error", e);
        }
    }
}

0 个答案:

没有答案