我试图为我的sparkJob编写scala测试用例,但它没有工作,并且给出了InvocationTargetException。
我正在尝试创建新的sqlContext并从我的localbox读取一个json文件。 我正在使用eclipse。
sc = new SparkContext(conf)
sqlContext = new org.apache.spark.sql.hive.HiveContext(sc)
完整的例外是:
java.lang.reflect.InvocationTargetException at java.lang.reflect.Constructor.newInstance(Constrructor.java:423)at org.apache.spark.io.CompressionCodec $ .createCodec(CompressionCodec.scala:67) 在 org.apache.spark.io.CompressionCodec $ .createCodec(CompressionCodec.scala:60) 在 org.apache.spark.broadcast.TorrentBroadcast.org $阿帕奇$火花$ $播出$$ TorrentBroadcast setConf(TorrentBroadcast.scala:73) 在 。org.apache.spark.broadcast.TorrentBroadcast(TorrentBroadcast.scala:80) 在 org.apache.spark.broadcast.TorrentBroadcastFactory.newBroadcast(TorrentBroadcastFactory.scala:34) 在 org.apache.spark.broadcast.BroadcastManager.newBroadcast(BroadcastManager..scala:63) 在org.apache.spark.SparkContext.broadcast(SparkContext.scala:1318)at org.apache.spark.SparkContext $$ anonfun $ hadoopFile $ 1.适用(SparkContext.scala:1006) 在 org.apache.spark.SparkContext $$ anonfun $ hadoopFile $ 1.适用(SparkContext.scala:1003) 在 org.apache.spark.rdd.RDDOperationScope $ .withScope(RDDOperationScope.scala:147) 在 org.apache.spark.rdd.RDDOperationScope $ .withScope(RDDOperationScope.scala:108) 在org.apache.spark.SparkCContext.withScope(SparkContext.scala:700)at at org.apache.spark.SparkContext.hadoopFile(SparkContext.scala:1003)at at org.apache.spark.SparkContext $$ anonfun $ TEXTFILE $ 1.appply(SparkContext.scala:818) 在 org.apache.spark.SparkContext $$ anonfun $ TEXTFILE $ 1.适用(SparkContext.scala:816)) 在 org.apache.spark.rdd.RDDOperationScope $ .withScope(RDDOperationScope.scala:147) 在 org.apache.spark.rdd.RDDOperationScope $ .withScope(RDDOperationScope.scala:108) 在org.apache.spark.SparkContext.withScope(SparkContext.scala:700)at org.apache.spark.SparkContext.texxtFile(SparkContext.scala:816)at at com.databricks.spark.csv.util.TextFile $ .withCharset(TextFile.scalaa:30) 在 com.databricks.spark.csv.DefaultSource $$ anonfuun $ $ createRelation 1.适用(DefaultSource.scala:146)
在 com.databricks.spark.csv.DefaultSource $$ anonfun $ createRelation $ 1.适用(DDefaultSource.scala:146) 在 com.databricks.spark.csv.CsvRelation.firstLine $ lzycompute(CsvRelation.scalaa:265) 在 com.databricks.spark.csv.CsvRelation.firstLine(CsvRelation.scala:263) at com.databricks.spark.csv.CsvRelation.tokenRdd(CsvRelation.scala:89) 在 com.databricks.spark.csv.CsvRRelation.buildScan(CsvRelation.scala:173) 在 org.apache.spark.sql.execution.datasources.DataSourceStrategy $$ anonfun $ 4.适用(DataSourceStrategy.scala:60) 在 org.apache.spark.sql.execution.datasources.DataSourceStrategy $$ anonfun $ 4.适用(DataSourceStrategy.scala:60) 在 org.apache.spark.sql.execution.datasources.DataSourceStrategy $$$ anonfun $ $ pruneFilterProject 1.适用(DataSourceStrategy.scala:279)
在 org.apache.spark.sql.execution.datasources.DataSourceStrategy $$ anonfun $ pruneFilterProject $ 1.适用(DataSourceStrattegy.scala:278) 在 org.apache.spark.sql.execution.datasources.DataSourceStrategy $ .pruneFilterProjectRaw(DataSourceStrategy.scala:3100) 在 org.apache.spark.sql.execution.datasources.DataSourceStrategy $ .pruneFilterProject(DataSourceStrategy.scala:274) 在 org.apache.spark.sql.execution.datasources.DataSourceStrategy $。适用(DataSourceStrategy.scala:556) 在 org.apache.spark.sql.catalyst.planning.QuueryPlanner $$ anonfun $ 1.适用(QueryPlanner.scala:58) 在 org.apache.spark.sqll.catalyst.planning.QueryPlanner $$ anonfun $ 1.适用(QueryPlanner.scala:58) 在scala.collection.Iterator $$ anon $ 13.hasNext(Iterator.scala:371)at at org.apache.spark.sql.catalyst.planning.QueryPlanner.plan(QueryPlanner.scala:59) 在 org.apache.spark.sql.catalyst.planning.QueryPlanner.planLater(QueryPlanner.scala:54) 在 org.apache.spark.sql.execution.SparkStrategies $ AAggregation $。适用(SparkStrategies.scala:235) 在 org.apache.spark.sql.catalyst.pllanning.QueryPlanner $$ anonfun $ 1.适用(QueryPlanner.scala:58) 在 org.apache.spark.sql.catalyst.planning.QueryPlanner $$ anonfun $ 1.适用(QueryPlaanner.scala:58) 在scala.collection.Iterator $$ anon $ 13.hasNext(Iterator.scala:371)at at org.apache.spark.sql.catalyst.planning.QueryPlanner.plan(QueryPlanner.scala:59) 在 org.apache.spark.sql.SQLContext $ QueryExecuution.sparkPlan $ lzycompute(SQLContext.scala:920) 在 org.apache.spark.sql.SQLContext $ QueryExecution.sparkPlan(SQLContext.scala:918) 在 org.apache.spark.sql.SQLContext $ QueryExecution.executedPlan $ lzycompute(SQLContext.scala:924) 在 org.apache.spark.sql.SQLContext $ QueryExecution.executedPlan(SQLContext.scala:924) 在 org.apache.spark.sql.execution.SQLExecution $ .withNewExecutionId(SQLExecution.scala:53) 在 org.apache..spark.sql.DataFrame.withNewExecutionId(DataFrame.scala:1904) 在org.apache.spark.sql.DataFrame.collect(DataFrame.scala:1385))at org.apache.spark.sql.DataFrame.count(DataFrame.scala:14033)at com.amazon.smt.ec.datapopulator.common.DataFrameOperationsTest $$ anonfun $ 1.适用$ MCV $ SP(DataFrameOperationsTest.scala:55) 在 com.amazon.smt.ec.datapopulator.common.DataFrameOperatiionsTest $$ anonfun $ 1.适用(DataFrameOperationsTest.scala:47)
在 com..amazon.smt.ec.datapopulator.common.DataFrameOperationsTest $$ anonfun $ 1.适用(DataFrameOperationsTest.scala:47) 在 org.scalatest.Transformer $$ anonfun $ $适用1.适用(Transformer.scala:22) 在 org.scalatest.Transformer $$ anonfun $ $适用1.适用(Transformer.scala:22) 在org.scalatest.OutcomeOf $ class.outcomeOf(OutcomeOf.scala:85)at at org.scalatest.OutcomeOf $ .outcomeOf(OutcomeOf.scala:104)at at org.scalatest.Transformer.apply(Transformer.scala:22)at org.scalatest.Transformer.apply(Transformer.scala:20)at org.scalatest.FunSuiteLike $$ anon $ 1.apply(FunSuuiteLike.scala:158)at at org.scalatest.Suite $ class.withFixture(Suite.scala:1121)at org.scalatest.FunSuite.withFixture(FunSuite.scala:1559)at org.scalatest.FuunSuiteLike $ class.invokeWithFixture $ 1(FunSuiteLike.scala:155) 在 org.scalatest.FunSuiteLike $$ anonfun $ $的runTest 1.适用(FunSuiteLike.scala:167) 在 org.scalatest.FunSuiteLike $$ anonfun $ $的runTest 1.适用(FunSuiteLike.sscala:167) 在orrg.scalatest.SuperEngine.runTestImpl(Engine.scala:306)at at org.scalatest.FunSuiteLike $ class.runTest(FunSuiteLike.scala:167)at at org.scalatest.FunSuite.runTestt(FunSuite.scala:1559)at org.scalatest..FunSuiteLike $$ anonfun $ runTests $ 1.适用(FunSuiteLike.scala:200) 在 org.scalatest.FunSuiteLike $$ anonfun $ runTests $ 1.适用(FunSuiteLike.scala:200) 在 org.scalatest.SuperEngine $$ anonfun $ traaverseSubNodes $ 1 $ 1.适用(Engine.scala:413)
在 org.scalatest.SuperEngine $$$ anonfun $ traverseSubNodes $ 1 $ 1.适用(Engine.scala:401) 在scala.collection.immutable.List.foreach(LList.scala:318)at org.scalatest.SuperEngine.traverseSubNodes $ 1(Engine.scala:401)at at org.scalatest.SuperEngine.org $ scalatest $ SuperEngine $$ runTestsInBranch(Engine.scala:396) 在org.scalatest.SuperEngine.runTestsImpl(Engine.scala:483)at at org.scalatest.FunSuiteLike $ class.runTests(FunSuiteLike.scala:200)at at org.scalatest.FunSuite.runTests(FunSuite.scala:15559)at org.scalatest.Suite $ class.run(Suite.scala:1423)at org.scalatest.FunSSuite.org $ scalatest $ FunSuiteLike $$超$运行(FunSuite.scala:1559) 在 org.scalatest.FunSuiteLike $$ anonfun $运行$ 1.适用(FunSuiteLike.scala:204) 在 org.scalatest.FunSuiteLike $$ anonfun $运行$ 1.适用(FunSuiteLike.scala:204) 在org.scalatest.SuperEngine.runImpl(Engine.scala:545)at org.scalatest.FunSuiteLike $ class.run(FuunSuiteLike.scala:204)at at com.amazon.smt.ec.datapopulator.common.DataFrameOperationsTest.org $ scalatest $ BeforeAndAfterAll $$超$运行(DataFrameOperationsTest.scala:21) 在 org.scalatest.BBeforeAndAfterAll $ class.liftedTree1 $ 1(BeforeAndAfterAll.scala:257) 在 org.scalatest.BeforeAndAfterAll $ class.run(BeforeAndAfterAll.scala:256) 在 com.amazon.smt.ec.datapopulator.common.DattaFrameOperationsTest.run(DataFrameOperationsTest.scala:21) 在oorg.scalatest.junit.JUnitRunner.run(JUnitRunner.scala:99)引起 by:java.lang.IllegalArgumentException at org.apache.spaark.io.SnappyCompressionCodec(CompressionCodec.scala:151)。
有人能帮助我吗?我在这里错过了什么?