集成SQL和Spark Streaming时不可序列化的异常

时间:2014-12-24 05:12:46

标签: apache-spark spark-streaming apache-spark-sql

Not Serializable exception when integrating Spark SQL and Spark Streaming

之外

我的源代码

public static void main(String args[]) {
    SparkConf sparkConf = new SparkConf().setAppName("NumberCount");
    JavaSparkContext jc = new JavaSparkContext(sparkConf);
    JavaStreamingContext jssc = new JavaStreamingContext(jc, new Duration(2000));
    jssc.addStreamingListener(new WorkCountMonitor());
    int numThreads = Integer.parseInt(args[3]);
    Map<String,Integer> topicMap = new HashMap<String,Integer>();
    String[] topics = args[2].split(",");
    for (String topic : topics) {
        topicMap.put(topic, numThreads);
    }
    JavaPairReceiverInputDStream<String,String> data = KafkaUtils.createStream(jssc, args[0], args[1], topicMap);
    data.print();

    JavaDStream<Person> streamData = data.map(new Function<Tuple2<String, String>, Person>() {
            public Person call(Tuple2<String,String> v1) throws Exception {
                String[] stringArray = v1._2.split(",");
                Person Person = new Person();
                Person.setName(stringArray[0]);
                Person.setAge(stringArray[1]);
                return Person;
            }

        });


    final JavaSQLContext sqlContext = new JavaSQLContext(jc);
    streamData.foreachRDD(new Function<JavaRDD<Person>,Void>() {
        public Void call(JavaRDD<Person> rdd) {

            JavaSchemaRDD subscriberSchema = sqlContext.applySchema(rdd, Person.class);

            subscriberSchema.registerAsTable("people");
            System.out.println("all data");
            JavaSchemaRDD names = sqlContext.sql("SELECT name FROM people");
            System.out.println("afterwards");

            List<String> males = new ArrayList<String>();

            males = names.map(new Function<Row,String>() {
                public String call(Row row) {
                    return row.getString(0);
                }
            }).collect();
            System.out.println("before for");
            for (String name : males) {
                System.out.println(name);
            }
            return null;
        }
    });
    jssc.start();
    jssc.awaitTermination();
}

JavaSQLContext也在ForeachRDD循环之外声明,但我仍然得到NonSerializableException

  

14/12/23 23:49:38错误JobScheduler:运行作业流作业时出错1419378578000 ms.1   org.apache.spark.SparkException:任务不可序列化           在org.apache.spark.util.ClosureCleaner $ .ensureSerializable(ClosureCleaner.scala:166)           在org.apache.spark.util.ClosureCleaner $ .clean(ClosureCleaner.scala:158)           在org.apache.spark.SparkContext.clean(SparkContext.scala:1435)           在org.apache.spark.rdd.RDD.map(RDD.scala:271)           在org.apache.spark.api.java.JavaRDDLike $ class.map(JavaRDDLike.scala:78)           在org.apache.spark.sql.api.java.JavaSchemaRDD.map(JavaSchemaRDD.scala:42)           在com.basic.spark.NumberCount $ 2.call(NumberCount.java:79)           在com.basic.spark.NumberCount $ 2.call(NumberCount.java:67)           在org.apache.spark.streaming.api.java.JavaDStreamLike $$ anonfun $ foreachRDD $ 1.apply(JavaDStreamLike.scala:274)           在org.apache.spark.streaming.api.java.JavaDStreamLike $$ anonfun $ foreachRDD $ 1.apply(JavaDStreamLike.scala:274)           在org.apache.spark.streaming.dstream.DStream $$ anonfun $ foreachRDD $ 1.apply(DStream.scala:529)           在org.apache.spark.streaming.dstream.DStream $$ anonfun $ foreachRDD $ 1.apply(DStream.scala:529)           在org.apache.spark.streaming.dstream.ForEachDStream $$ anonfun $ 1.apply $ mcV $ sp(ForEachDStream.scala:42)           在org.apache.spark.streaming.dstream.ForEachDStream $$ anonfun $ 1.apply(ForEachDStream.scala:40)           在org.apache.spark.streaming.dstream.ForEachDStream $$ anonfun $ 1.apply(ForEachDStream.scala:40)           在scala.util.Try $ .apply(Try.scala:161)           在org.apache.spark.streaming.scheduler.Job.run(Job.scala:32)           在org.apache.spark.streaming.scheduler.JobScheduler $ JobHandler.run(JobScheduler.scala:171)           在java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)           at java.util.concurrent.ThreadPoolExecutor $ Worker.run(ThreadPoolExecutor.java:615)           在java.lang.Thread.run(Thread.java:724)   引起:java.io.NotSerializableException:org.apache.spark.sql.api.java.JavaSQLContext           at java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1181)           at java.io.ObjectOutputStream.defaultWriteFields(ObjectOutputStream.java:1541)           at java.io.ObjectOutputStream.writeSerialData(ObjectOutputStream.java:1506)           at java.io.ObjectOutputStream.writeOrdinaryObject(ObjectOutputStream.java:1429)           at java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1175)           at java.io.ObjectOutputStream.defaultWriteFields(ObjectOutputStream.java:1541)           at java.io.ObjectOutputStream.writeSerialData(ObjectOutputStream.java:1506)           at java.io.ObjectOutputStream.writeOrdinaryObject(ObjectOutputStream.java:1429)           at java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1175)           at java.io.ObjectOutputStream.defaultWriteFields(ObjectOutputStream.java:1541)           at java.io.ObjectOutputStream.writeSerialData(ObjectOutputStream.java:1506)           at java.io.ObjectOutputStream.writeOrdinaryObject(ObjectOutputStream.java:1429)           at java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1175)           at java.io.ObjectOutputStream.writeObject(ObjectOutputStream.java:347)           在org.apache.spark.serializer.JavaSerializationStream.writeObject(JavaSerializer.scala:42)           在org.apache.spark.serializer.JavaSerializerInstance.serialize(JavaSerializer.scala:73)           在org.apache.spark.util.ClosureCleaner $ .ensureSerializable(ClosureCleaner.scala:164)           ......还有20个

如果您有任何建议我很感激。

2 个答案:

答案 0 :(得分:0)

您是否在Person pojo类中实现了Serializable接口。您也可以尝试将topicMap声明为最终

答案 1 :(得分:0)

这是工作代码

package com.basic.spark;

import java.io.Serializable;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;

import kafka.javaapi.producer.Producer;
import kafka.producer.KeyedMessage;
import kafka.producer.ProducerConfig;

import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.function.Function;
import org.apache.spark.sql.api.java.JavaSQLContext;
import org.apache.spark.sql.api.java.JavaSchemaRDD;
import org.apache.spark.sql.api.java.Row;
import org.apache.spark.streaming.Duration;
import org.apache.spark.streaming.api.java.JavaDStream;
import org.apache.spark.streaming.api.java.JavaPairReceiverInputDStream;
import org.apache.spark.streaming.api.java.JavaStreamingContext;
import org.apache.spark.streaming.kafka.KafkaUtils;

import scala.Tuple2;

public class NumberCount implements Serializable {

    transient SparkConf sparkConf = new SparkConf().setAppName("NumberCount");
    transient JavaSparkContext jc = new JavaSparkContext(sparkConf);
    transient JavaStreamingContext jssc_1 = new JavaStreamingContext(jc, new Duration(1000));
    transient JavaSQLContext sqlContext = new JavaSQLContext(jc);
    transient Producer producer = configureKafka();

    public static void main(String args[]) {
        (new NumberCount()).job_1(args);
    }

    public void job_1(String...args) {
        jssc_1.addStreamingListener(new WorkCountMonitor());
        int numThreads = Integer.parseInt(args[3]);
        Map<String,Integer> topicMap = new HashMap<String,Integer>();
        String[] topics = args[2].split(",");
        for (String topic : topics) {
            topicMap.put(topic, numThreads);
        }

        JavaPairReceiverInputDStream<String,String> data = KafkaUtils.createStream(jssc_1, args[0], args[1], topicMap);
        data.window(new Duration(10000), new Duration(2000));

        JavaDStream<String> streamData = data.map(new Function<Tuple2<String, String>, String>() {
            public String call(Tuple2<String,String> v1) {
                return v1._2;
            }
        });

        streamData.foreachRDD(new Function<JavaRDD<String>,Void>() {
            public Void call(JavaRDD<String> rdd) {

                if (rdd.count() < 1)
                    return null;

                try {
                    JavaSchemaRDD eventSchema = sqlContext.jsonRDD(rdd);
                    eventSchema.registerTempTable("event");
                    System.out.println("all data");
                    JavaSchemaRDD names = sqlContext.sql("SELECT deviceId, count(*) FROM event group by deviceId");
                    System.out.println("afterwards");

//                    List<Long> males = new ArrayList<Long>();
//
//                    males = names.map(new Function<Row,Long>() {
//                        public Long call(Row row) {
//                            return row.getLong(0);
//                        }
//                    }).collect();
//                    System.out.println("before for");
//                    ArrayList<KeyedMessage<String, String>> data = new ArrayList<KeyedMessage<String, String>>();
//                    for (Long name : males) {
//                        System.out.println("**************"+name);
//                        writeToKafka_1(data, String.valueOf(name));
//                    }
//                    producer.send(data);

                    List<String> deviceDetails = new ArrayList<String>();

                    deviceDetails = names.map(new Function<Row,String>() {
                        public String call(Row row) {
                            return row.getString(0) +":" + row.getLong(1);
                        }
                    }).collect();

                    System.out.println("before for");
                    ArrayList<KeyedMessage<String, String>> data = new ArrayList<KeyedMessage<String, String>>();
                    for (String name : deviceDetails) {
                        System.out.println("**************"+name);
                        writeToKafka_1(data, name);
                    }
                    producer.send(data);

                } catch (Exception e) {
                    System.out.println("#ERROR_1#   #" + rdd);
                    e.printStackTrace();
                }

                return null;
            }
        });
        jssc_1.start();
        jssc_1.awaitTermination();
    }

    public Producer<String, String> configureKafka() {
        Properties props = new Properties();
        props.put("metadata.broker.list", "xx.xx.xx.xx:9092");
        props.put("serializer.class", "kafka.serializer.StringEncoder");
        props.put("compression.codec", "2");
        props.put("request.required.acks", "0");
        props.put("producer.type", "sync");

        ProducerConfig config = new ProducerConfig(props);

        Producer<String, String> producer = new Producer<String, String>(config);

        return producer;
    }

    public void writeToKafka_1(ArrayList<KeyedMessage<String,String>> list, String msg) {
        list.add(new KeyedMessage<String,String>("my-replicated-topic-1", "", msg));
    }
}