在spark-shell中,我运行我的代码:
$found_max_class=false;
foreach ($array as $class) {
if (get_class($class) == 'MaxClass') {
if($found_max_class)
{
throw new Exception('Too many MaxClass!');
}
$found_max_class =true;
}
}
这样的错误,我的火花版本是2.1.1。
import org.apache.spark.{SparkConf, SparkContext}
import org.apache.spark.graphx.{Edge, Graph, GraphLoader, VertexId}
import org.apache.spark.rdd.RDD
import org.apache.spark.streaming.{Seconds, StreamingContext}
val users: RDD[(VertexId, Map[String, String])] =
sc.parallelize(Array(
(1L, Map("a" -> "a")),
(2L, Map("b" -> "b")),
(3L, Map("c" -> "c")),
(4L, Map("d" -> "d"))
))
val edgs: RDD[Edge[String]] =
sc.parallelize(Array(
Edge(1L, 2L, "1_2"),
Edge(2L, 3L, "2_3"),
Edge(3L, 1L, "3_1"),
Edge(4L, 1L, "4_1"),
Edge(4L, 3L, "4_3")
))
val graph = Graph.apply(users, edgs)
graph.edges.saveAsObjectFile("/Users/test/edges")
graph.vertices.saveAsObjectFile("/Users/test/vertices")
val vertices = sc.objectFile[(VertexId, Map[String, String])]("/Users/test/edges")
val edges = sc.objectFile[Edge[String]]("/Users/test/vertices")
val un = Graph.apply(vertices, edges)
un.edges.foreach(println)
因此错误看起来像是无法将obejectfile转换为图形。 我的问题是如何将graphx存储为目标文件并重新加载它,谢谢!
答案 0 :(得分:0)
我犯了一个愚蠢的错误: val vertices = sc.objectFile(VertexId,Map [String,String]) val edges = sc.objectFileEdge [String]
从错误的文件加载边缘。
所以,通过这种方式使用是没有问题的