Question

运行读取文本文件并收集结果的简单作业时，我收到EOFException。这在我的开发机器上运行正常，但在独立模式（单机，主机+工作器）中执行时失败。我的设置是Apache Spark 0.9.1 Hadoop 2预建。

我使用sbt-assembly插件部署我的代码并生成可执行的jar文件。

相关堆栈跟踪：

14/05/27 08:22:03 WARN scheduler.TaskSetManager: Loss was due to java.io.EOFException
java.io.EOFException
    at java.io.ObjectInputStream$BlockDataInputStream.readFully(ObjectInputStream.java:2742)
    at java.io.ObjectInputStream.readFully(ObjectInputStream.java:1014)
    at org.apache.hadoop.io.WritableUtils.readCompressedByteArray(WritableUtils.java:39)
    at org.apache.hadoop.io.WritableUtils.readCompressedString(WritableUtils.java:87)
    at org.apache.hadoop.io.WritableUtils.readCompressedStringArray(WritableUtils.java:185)
    at org.apache.hadoop.conf.Configuration.readFields(Configuration.java:2378)
    at org.apache.hadoop.io.ObjectWritable.readObject(ObjectWritable.java:285)
    at org.apache.hadoop.io.ObjectWritable.readFields(ObjectWritable.java:77)
    at org.apache.spark.SerializableWritable.readObject(SerializableWritable.scala:39)
    at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
    at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.lang.reflect.Method.invoke(Method.java:622)
    at java.io.ObjectStreamClass.invokeReadObject(ObjectStreamClass.java:1001)
    at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1892)
    at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1797)
    at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1349)
    at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369)
    at org.apache.spark.serializer.JavaDeserializationStream.readObject(JavaSerializer.scala:40)
    at org.apache.spark.broadcast.HttpBroadcast$.read(HttpBroadcast.scala:165)
    at org.apache.spark.broadcast.HttpBroadcast.readObject(HttpBroadcast.scala:56)
    at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
    at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.lang.reflect.Method.invoke(Method.java:622)
    at java.io.ObjectStreamClass.invokeReadObject(ObjectStreamClass.java:1001)
    at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1892)
    at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1797)
    at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1349)
    at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1989)
    at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1914)
    at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1797)
    at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1349)
    at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1989)
    at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1914)
    at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1797)
    at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1349)
    at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369)
    at scala.collection.immutable.$colon$colon.readObject(List.scala:362)
    at sun.reflect.GeneratedMethodAccessor10.invoke(Unknown Source)
    at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.lang.reflect.Method.invoke(Method.java:622)
    at java.io.ObjectStreamClass.invokeReadObject(ObjectStreamClass.java:1001)
    at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1892)
    at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1797)
    at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1349)
    at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1989)
    at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1914)
    at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1797)
    at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1349)
    at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1989)
    at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1914)
    at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1797)
    at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1349)
    at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369)
    at scala.collection.immutable.$colon$colon.readObject(List.scala:362)
    at sun.reflect.GeneratedMethodAccessor10.invoke(Unknown Source)
    at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.lang.reflect.Method.invoke(Method.java:622)
    at java.io.ObjectStreamClass.invokeReadObject(ObjectStreamClass.java:1001)
    at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1892)
    at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1797)
    at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1349)
    at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1989)
    at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1914)
    at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1797)
    at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1349)
    at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1989)
    at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1914)
    at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1797)
    at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1349)
    at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369)
    at scala.collection.immutable.$colon$colon.readObject(List.scala:362)
    at sun.reflect.GeneratedMethodAccessor10.invoke(Unknown Source)
    at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.lang.reflect.Method.invoke(Method.java:622)
    at java.io.ObjectStreamClass.invokeReadObject(ObjectStreamClass.java:1001)
    at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1892)
    at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1797)
    at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1349)
    at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1989)
    at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1914)
    at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1797)
    at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1349)
    at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1989)
    at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1914)
    at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1797)
    at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1349)
    at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369)
    at scala.collection.immutable.$colon$colon.readObject(List.scala:362)
    at sun.reflect.GeneratedMethodAccessor9.invoke(Unknown Source)
    at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.lang.reflect.Method.invoke(Method.java:622)
    at java.io.ObjectStreamClass.invokeReadObject(ObjectStreamClass.java:1001)
    at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1892)
    at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1797)
    at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1349)
    at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1989)
    at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1914)
    at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1797)
    at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1349)
    at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1989)
    at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1914)
    at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1797)
    at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1349)
    at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369)
    at scala.collection.immutable.$colon$colon.readObject(List.scala:362)
    at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
    at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.lang.reflect.Method.invoke(Method.java:622)
    at java.io.ObjectStreamClass.invokeReadObject(ObjectStreamClass.java:1001)
    at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1892)
    at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1797)
    at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1349)
    at java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1989)
    at java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1914)
    at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1797)
    at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1349)
    at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369)
    at org.apache.spark.serializer.JavaDeserializationStream.readObject(JavaSerializer.scala:40)
    at org.apache.spark.scheduler.ResultTask$.deserializeInfo(ResultTask.scala:63)
    at org.apache.spark.scheduler.ResultTask.readExternal(ResultTask.scala:139)
    at java.io.ObjectInputStream.readExternalData(ObjectInputStream.java:1836)
    at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1795)
    at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1349)
    at java.io.ObjectInputStream.readObject(ObjectInputStream.java:369)
    at org.apache.spark.serializer.JavaDeserializationStream.readObject(JavaSerializer.scala:40)
    at org.apache.spark.serializer.JavaSerializerInstance.deserialize(JavaSerializer.scala:62)
    at org.apache.spark.executor.Executor$TaskRunner$$anonfun$run$1.apply$mcV$sp(Executor.scala:193)
    at org.apache.spark.deploy.SparkHadoopUtil$$anon$1.run(SparkHadoopUtil.scala:42)
    at org.apache.spark.deploy.SparkHadoopUtil$$anon$1.run(SparkHadoopUtil.scala:41)
    at java.security.AccessController.doPrivileged(Native Method)
    at javax.security.auth.Subject.doAs(Subject.java:416)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1491)
    at org.apache.spark.deploy.SparkHadoopUtil.runAsUser(SparkHadoopUtil.scala:41)
    at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:176)
    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1146)
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
    at java.lang.Thread.run(Thread.java:701)

[编辑]

请注意我更改了序列化程序，我现在正在使用Kryo（只是尝试过来看看是否存在问题）。

我的火花背景：

//Load Spark config file
lazy val conf = ConfigFactory.load

//Set Spark config object
val sparkConf = new SparkConf()
      .setMaster(conf.getString("spark.prod.master"))  //Something like spark://host:port
      .setAppName(conf.getString("app.name"))
      .set("spark.executor.memory", conf.getString("spark.prod.config.executorMemory"))
      .set("spark.cores.max", conf.getString("spark.prod.config.coresMax"))
      .set("spark.serializer", conf.getString("spark.prod.config.serializer"))
      .set("spark.kryo.registrator", conf.getString("spark.prod.config.kryoRegistrator"))
      .set("spark.kryoserializer.buffer.mb", conf.getString("spark.prod.config.kryoSerializerBufferSize"))
      .set("spark.logConf", conf.getString("spark.prod.config.logConf"))

任何提示？

Answer 1

经过几天的努力，我终于想出了一个解决方案。我不得不添加相应的hadoop-client依赖项以避免这种奇怪的异常。

之后，其他一些已经reported errors出现了。连接拒绝问题的解决方案是：

更改sbin/start-master.sh和/或sbin/start-slaves.sh并将$SPARK_MASTER_IP设置为hostname -f而不是hostname。似乎Akka只能使用完全限定的名称，而不能使用主机名或IP地址。
还将$SPARK_MASTER_IP中的conf/spark-env.sh设置为hostname -f，以便群集工作人员可以联系到主人。
确保conf/slaves也使用完全限定的域名，而不是主机名/ IP地址。

经过这些改变，一切正常。

希望它可以帮助别人！

Apache Spark EOF异常

1 个答案: