public class SparkDemo {
@SuppressWarnings({ "resource" })
public static void main(String[] args) {
SparkConf conf = new SparkConf().setAppName("Spark APP").setMaster("spark://xxx.xxx.xxx.xx:7077");
JavaSparkContext sc = new JavaSparkContext(conf);
JavaRDD<String> lines = sc.textFile("/Users/mchaurasia/file.txt");
JavaRDD<String> words = lines.flatMap((String s) -> {
return Arrays.asList(s.split(" "));
});
JavaPairRDD<String, Integer> pairs = words.mapToPair((String s) -> {
return new Tuple2<String, Integer>(s, 1);
});
JavaPairRDD<String, Integer> counts = pairs.reduceByKey((a, b) -> a + b);
for (Tuple2<String, Integer> result : counts.collect()) {
System.out.println(result._1 + " , " + result._2);
}
sc.close();
System.out.println("================= DONE ===================");
}
当我使用setMaster（＆＃34; spark：//xxx.xxx.xxx）将master设置为local（setMaster（＆＃34; local＆＃34;））时，上面的代码工作正常.XX：7077＆＃34）， 我收到了以下错误 -
6/06/21 09:28:59 WARN TaskSetManager: Lost task 1.0 in stage 0.0 (TID 1, xxx.xxx.xxx.xx): java.lang.ClassCastException: cannot assign instance of java.lang.invoke.SerializedLambda to field org.apache.spark.api.java.JavaRDDLike$$anonfun$fn$1$1.f$3 of type org.apache.spark.api.java.function.FlatMapFunction in instance of org.apache.spark.api.java.JavaRDDLike$$anonfun$fn$1$1
at java.io.ObjectStreamClass$FieldReflector.setObjFieldValues(ObjectStreamClass.java:2089)
at java.io.ObjectStreamClass.setObjFieldValues(ObjectStreamClass.java:1261)
at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1999)
at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1918)
at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1801)
at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1351)
at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1993)
at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1918)
at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1801)
at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1351)
at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1993)
at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1918)
at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1801)
at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1351)
at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1993)
at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1918)
at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1801)
at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1351)
at java.io.ObjectInputStream.readObject(ObjectInputStream.java:371)
at scala.collection.immutable.$colon$colon.readObject(List.scala:362)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:483)
at java.io.ObjectStreamClass.invokeReadObject(ObjectStreamClass.java:1017)
at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1896)
at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1801)
at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1351)
at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1993)
at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1918)
at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1801)
at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1351)
at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1993)
at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1918)
线程中的异常＆＃34; main＆＃34; org.apache.spark.SparkException：作业因阶段失败而中止：阶段0.0中的任务0失败4次，最近失败：阶段0.0中失去的任务0.3（TID 5，xxx.xxx.xxx.xx）：java.lang .ClassCastException：无法将java.lang.invoke.SerializedLambda的实例分配给字段org.apache.apark.api.java.JavaRDDLike $$ anonfun $ fn $ 1 $ 1.f $ 3类型为org.apache.spark.api.java.function在org.apache.spark.api.java.JavaRDDLike $$ anonfun $ fn $ 1 $ 1的实例中.FlatMapFunction at java.io.ObjectStreamClass $ FieldReflector.setObjFieldValues（ObjectStreamClass.java:2089） at java.io.ObjectStreamClass.setObjFieldValues（ObjectStreamClass.java:1261） at java.io.ObjectInputStream.defaultReadFields（ObjectInputStream.java:1999） at java.io.ObjectInputStream.readSerialData（ObjectInputStream.java:1918） at java.io.ObjectInputStream.readOrdinaryObject（ObjectInputStream.java:1801） 在java.io.ObjectInputStream.readObject0（ObjectInputStream.java:1351）
请帮忙。