Apache Spark Pregel NullPointerException

时间:2016-03-20 09:23:25

标签: scala apache-spark

我找到了this pregel示例,并希望将其用于图形着色。

我已将顶点定义为:(VertexId,(Int,Set[Int]())

def getColor(vertexId: VertexId):Int = {
    pcGraph.vertices.filter{case(id,_) => id == vertexId}.collect.toList(0)._2._1
}

def sendMsg(triplet: EdgeTriplet[(Int, Set[Int]),Boolean]): Iterator[(VertexId, Int)] = {
    var srcColor = getColor(triplet.srcId)
    var dstColor = getColor(triplet.dstId)
    if (srcColor != dstColor)
        Iterator.empty
    else 
        Iterator((triplet.dstId, srcColor))
}

执行时出现NullPointerException:

val minGraph = pcGraph.pregel(initialMsg, Int.MaxValue, EdgeDirection.Out)(vprog, sendMsg, mergeMsg)

这是堆栈跟踪:

[Stage 6:>                                                          (0 + 0) / 2]16/03/20 01:52:17 ERROR Executor: Exception in task 0.0 in stage 7.0 (TID 14)
java.lang.NullPointerException
    at $line45.$read$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC.getColor(<console>:53)
    at $line48.$read$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC.sendMsg(<console>:55)
    at $line50.$read$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$anonfun$3.apply(<console>:66)
    at $line50.$read$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$anonfun$3.apply(<console>:66)
    at org.apache.spark.graphx.impl.GraphImpl.org$apache$spark$graphx$impl$GraphImpl$$sendMsg$1(GraphImpl.scala:197)
    at org.apache.spark.graphx.impl.GraphImpl$$anonfun$mapReduceTriplets$1.apply(GraphImpl.scala:213)
    at org.apache.spark.graphx.impl.GraphImpl$$anonfun$mapReduceTriplets$1.apply(GraphImpl.scala:213)
    at org.apache.spark.graphx.impl.EdgePartition.aggregateMessagesEdgeScan(EdgePartition.scala:409)
    at org.apache.spark.graphx.impl.GraphImpl$$anonfun$13$$anonfun$apply$3.apply(GraphImpl.scala:265)
    at org.apache.spark.graphx.impl.GraphImpl$$anonfun$13$$anonfun$apply$3.apply(GraphImpl.scala:235)
    at scala.collection.Iterator$$anon$13.hasNext(Iterator.scala:371)
    at org.apache.spark.shuffle.sort.BypassMergeSortShuffleWriter.write(BypassMergeSortShuffleWriter.java:126)
    at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:73)
    at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:41)
    at org.apache.spark.scheduler.Task.run(Task.scala:89)
    at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:213)
    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
    at java.lang.Thread.run(Thread.java:745)
16/03/20 01:52:18 ERROR Executor: Exception in task 1.0 in stage 7.0 (TID 15)

我不明白为什么会发生异常。

0 个答案:

没有答案