Spark数据集:使用groupBy时出现异常MissingRequirementError

时间:2016-09-20 01:17:34

标签: scala apache-spark apache-spark-dataset apache-spark-1.6

我开始使用Spark数据集,当我在Spark 1.6.1中执行groupby时,我遇到了这个异常

case class RecordIdDate(recordId: String, date: String)
val ds = sc.parallelize(List(RecordIdDate("hello","1"), RecordIdDate("bye","2"))).toDS
ds.groupBy(_.recordId).mapGroups{case (id, it) => it.toSeq }

这是例外

  

scala.reflect.internal.MissingRequirementError:object $ line19。$ read not found。       在scala.reflect.internal.MissingRequirementError $ .signal(MissingRequirementError.scala:16)       在scala.reflect.internal.MissingRequirementError $ .notFound(MissingRequirementError.scala:17)       在scala.reflect.internal.Mirrors $ RootsBase.ensureModuleSymbol(Mirrors.scala:126)       在scala.reflect.internal.Mirrors $ RootsBase.staticModule(Mirrors.scala:161)       在scala.reflect.internal.Mirrors $ RootsBase.staticModule(Mirrors.scala:21)       at $ iwC $$ iwC $$ iwC $$ iwC $$ iwC $$ iwC $$ iwC $$ iwC $$ typecreator2 $ 1.apply(:32)       at scala.reflect.api.TypeTags $ WeakTypeTagImpl.tpe $ lzycompute(TypeTags.scala:231)       at scala.reflect.api.TypeTags $ WeakTypeTagImpl.tpe(TypeTags.scala:231)       at org.apache.spark.sql.SQLImplicits $$ typecreator10 $ 1.apply(SQLImplicits.scala:96)       at scala.reflect.api.TypeTags $ WeakTypeTagImpl.tpe $ lzycompute(TypeTags.scala:231)       at scala.reflect.api.TypeTags $ WeakTypeTagImpl.tpe(TypeTags.scala:231)       at org.apache.spark.sql.catalyst.encoders.ExpressionEncoder $ .apply(ExpressionEncoder.scala:50)       at org.apache.spark.sql.SQLImplicits.newProductSeqEncoder(SQLImplicits.scala:96)       at $ iwC $$ iwC $$ iwC $$ iwC $$ iwC $$ iwC $$ iwC $$ iwC。(:32)       at $ iwC $$ iwC $$ iwC $$ iwC $$ iwC $$ iwC $$ iwC。(:37)       at $ iwC $$ iwC $$ iwC $$ iwC $$ iwC $$ iwC。(:39)       $ iwC $$ iwC $$ iwC $$ iwC $$ iwC。(:41)       $ iwC $$ iwC $$ iwC $$ iwC。(:43)       $ iwC $ iwC $$ iwC。(:45)       $ iwC $$ iwC。(:47)       在$ iwC。(:49)       在(:51)       at。(:55)       在 。()       7岁时)       在 。()       at $ print()       at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)       at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)       at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)       at java.lang.reflect.Method.invoke(Method.java:606)       在org.apache.spark.repl.SparkIMain $ ReadEvalPrint.call(SparkIMain.scala:1065)       在org.apache.spark.repl.SparkIMain $ Request.loadAndRun(SparkIMain.scala:1346)       在org.apache.spark.repl.SparkIMain.loadAndRunReq $ 1(SparkIMain.scala:840)       在org.apache.spark.repl.SparkIMain.interpret(SparkIMain.scala:871)       在org.apache.spark.repl.SparkIMain.interpret(SparkIMain.scala:819)       在org.apache.spark.repl.SparkILoop.reallyInterpret $ 1(SparkILoop.scala:857)       在org.apache.spark.repl.SparkILoop.interpretStartingWith(SparkILoop.scala:902)       在org.apache.spark.repl.SparkILoop.command(SparkILoop.scala:814)       在org.apache.spark.repl.SparkILoop.processLine $ 1(SparkILoop.scala:657)       在org.apache.spark.repl.SparkILoop.innerLoop $ 1(SparkILoop.scala:665)       在org.apache.spark.repl.SparkILoop.org $ apache $ spark $ repl $ SparkILoop $$ loop(SparkILoop.scala:670)       在org.apache.spark.repl.SparkILoop $$ anonfun $ org $ apache $ spark $ repl $ SparkILoop $$ process $ 1.apply $ mcZ $ sp(SparkILoop.scala:997)       在org.apache.spark.repl.SparkILoop $$ anonfun $ org $ apache $ spark $ repl $ SparkILoop $$ process $ 1.apply(SparkILoop.scala:945)       在org.apache.spark.repl.SparkILoop $$ anonfun $ org $ apache $ spark $ repl $ SparkILoop $$ process $ 1.apply(SparkILoop.scala:945)       在scala.tools.nsc.util.ScalaClassLoader $ .savingContextLoader(ScalaClassLoader.scala:135)       在org.apache.spark.repl.SparkILoop.org $ apache $ spark $ repl $ SparkILoop $$进程(SparkILoop.scala:945)       在org.apache.spark.repl.SparkILoop.process(SparkILoop.scala:1059)       在org.apache.spark.repl.Main $ .main(Main.scala:31)       在org.apache.spark.repl.Main.main(Main.scala)       at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)       at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)       at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)       at java.lang.reflect.Method.invoke(Method.java:606)       在org.apache.spark.deploy.SparkSubmit $ .org $ apache $ spark $ deploy $ SparkSubmit $$ runMain(SparkSubmit.scala:731)       在org.apache.spark.deploy.SparkSubmit $ .doRunMain $ 1(SparkSubmit.scala:181)       在org.apache.spark.deploy.SparkSubmit $ .submit(SparkSubmit.scala:206)       在org.apache.spark.deploy.SparkSubmit $ .main(SparkSubmit.scala:121)       在org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)

任何人都可以提供帮助吗?

0 个答案:

没有答案