Question

Spring for new for new as non web project。请指导我如何在spring boot中编写Spark流，我已经在java-spark项目中工作，并希望在spring boot非web应用程序中进行转换。任何帮助或建议请。

这是我的Spark配置

@Bean
public SparkConf sparkConf() {
    SparkConf sparkConf = new SparkConf();
    sparkConf.set("spark.app.name", "SparkReceiver"); //The name of application. This will appear in the UI and in log data.
    //conf.set("spark.ui.port", "7077");    //Port for application's dashboard, which shows memory and workload data.
    sparkConf.set("dynamicAllocation.enabled","false");  //Which scales the number of executors registered with this application up and down based on the workload
    //conf.set("spark.cassandra.connection.host", "localhost"); //Cassandra Host Adddress/IP
    sparkConf.set("spark.serializer","org.apache.spark.serializer.KryoSerializer");  //For serializing objects that will be sent over the network or need to be cached in serialized form.
    sparkConf.set("spark.driver.allowMultipleContexts", "true");
    sparkConf.setMaster("local[4]");

    return sparkConf;
}

 @Bean
    public JavaSparkContext javaSparkContext() {
        return new JavaSparkContext(sparkConf());
    }

@Bean
public SparkSession sparkSession() {
    return SparkSession
            .builder()
            .sparkContext(javaSparkContext().sc())
            .appName("Java Spark SQL basic example")
            .getOrCreate();

}

@Bean
public  JavaStreamingContext javaStreamingContext(){
    return new JavaStreamingContext(sparkConf(), new Duration(2000));
}

这是我的测试类

 @Autowired
   private JavaSparkContext sc;

  @Autowired
   private SparkSession session;


public void testMessage() throws InterruptedException{

    JavaStreamingContext jsc = new JavaStreamingContext(sc, new Duration(2000));

    Map<String, String> kafkaParams = new HashMap<String, String>();

    kafkaParams.put("zookeeper.connect", "localhost:2181"); //Make all kafka data for this cluster appear under a particular path. 
    kafkaParams.put("group.id", "testgroup");   //String that uniquely identifies the group of consumer processes to which this consumer belongs
    kafkaParams.put("metadata.broker.list", "localhost:9092"); //Producer can find a one or more Brokers to determine the Leader for each topic.
    kafkaParams.put("serializer.class", "kafka.serializer.StringEncoder"); //Serializer to use when preparing the message for transmission to the Broker.
    kafkaParams.put("request.required.acks", "1");  //Producer to require an acknowledgement from the Broker that the message was received.

    Set<String> topics = Collections.singleton("16jnfbtopic");

    //Create an input DStream for Receiving data from socket
    JavaPairInputDStream<String, String> directKafkaStream = KafkaUtils.createDirectStream(jsc,
            String.class, 
            String.class, 
            StringDecoder.class, 
            StringDecoder.class, 
            kafkaParams, topics);

    //Create JavaDStream<String>
    JavaDStream<String> msgDataStream = directKafkaStream.map(new Function<Tuple2<String, String>, String>() {
        @Override
        public String call(Tuple2<String, String> tuple2) {
          return tuple2._2();
        }
      });
    //Create JavaRDD<Row>
    msgDataStream.foreachRDD(new VoidFunction<JavaRDD<String>>() {
          @Override
          public void call(JavaRDD<String> rdd) { 
              JavaRDD<Row> rowRDD = rdd.map(new Function<String, Row>() {
                  @Override
                  public Row call(String msg) {
                    Row row = RowFactory.create(msg);
                    return row;
                  }
                });
    //Create Schema       
    StructType schema = DataTypes.createStructType(new StructField[] {DataTypes.createStructField("Message", DataTypes.StringType, true)});


    Dataset<Row> msgDataFrame = session.createDataFrame(rowRDD, schema);
    msgDataFrame.show();
          }
    });

    jsc.start();            
    jsc.awaitTermination();

在运行此应用程序时我收到错误请指导我。

这是我的错误日志 Eclipse Error Log

使用弹簧靴启动火花

0 个答案: