Spark代码失败

Question

我是Spark和Scala的新手，下面是我的代码。一切都成功编译，但是当我使用下面的spark-submit运行jar文件时，我收到如下错误。无法看到我的任何printf语句，也没有得到任何明确的错误，Application master失败的确切原因。请帮助我如何调试这种类型的分布式应用程序，并请指出我的代码是否有任何错误。谢谢！

火花提交： -

spark-submit  --master yarn-cluster --jars jdbc-oracle.jar --class Read test1-1.0-SNAPSHOT.jar

错误： -

16/07/28 17:08:29 INFO yarn.Client: Application report for application_1469585636371_0793 (state: RUNNING)
16/07/28 17:08:30 INFO yarn.Client: Application report for application_1469585636371_0793 (state: FINISHED)
16/07/28 17:08:30 INFO yarn.Client:
         client token: Token { kind: YARN_CLIENT_TOKEN, service:  }
         diagnostics: N/A
         ApplicationMaster host: 10.45.56.102
         ApplicationMaster RPC port: 0
         queue: root.srusr
         start time: 1469740093365
         final status: FAILED
         tracking URL: https://cdts1hd01d.rxcorp.com:8090/proxy/application_1469585636371_0793/history/application_1469585636371_0793/2
         user: srusr
16/07/28 17:08:30 INFO yarn.Client: Deleting staging directory .sparkStaging/application_1469585636371_0793
Exception in thread "main" org.apache.spark.SparkException: Application application_1469585636371_0793 finished with failed status
        at org.apache.spark.deploy.yarn.Client.run(Client.scala:1035)
        at org.apache.spark.deploy.yarn.Client$.main(Client.scala:1082)
        at org.apache.spark.deploy.yarn.Client.main(Client.scala)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:497)
        at org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:731)
        at org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:181)
        at org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:206)
        at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:121)
        at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
16/07/28 17:08:30 INFO util.ShutdownHookManager: Shutdown hook called
16/07/28 17:08:30 INFO util.ShutdownHookManager: Deleting directory /tmp/spark-3f2a4d69-30f4-4cf9-bf48-ef4f95b90ffb

第一个代码文件： -

 package com.DWaDay.conf
    import org.apache.spark._
    import org.apache.spark.sql._
    import org.apache.spark.sql.hive._
    import org.apache.spark.{SparkConf,SparkContext}


    object sparkContextBuilder {



      println("I am inside Builder")
      val dwid = new SparkConf().setAppName("RddToDataFrame") //.setMaster("local[1]")
      val sparkContext = new org.apache.spark.SparkContext(dwid)

      val sqlContext = new org.apache.spark.sql.SQLContext(sparkContext)

      val hiveContext = new org.apache.spark.sql.hive.HiveContext(sparkContext)

    }

第二个代码文件： -

import com.DWaDay.conf.sparkContextBuilder
import org.apache.spark.SparkContext
import org.apache.spark.SparkConf
import org.apache.spark.sql.DataFrame
import org.apache.spark.sql.SQLContext
import org.apache.spark.sql.jdbc.{JdbcDialects, JdbcType, JdbcDialect}
import org.apache.spark.sql.types._



trait SourceReader  {

  //var dataFrame: DataFrame
  def createURL(sqlContext : SQLContext,argument : Map[String,String]) : Unit
  def createTempTable(tableName: String) : String
  def getDataFrame(): DataFrame
}


class OracleSourceReader(sqlContext: SQLContext, ConnectionDetails: Map[String,String]) extends SourceReader {


  var dataFrame: DataFrame=  _
   //def OracleSourceReader(){
    println("Inside connection OracleReader")
    this.createURL(sqlContext,ConnectionDetails)
   //}

  override def createURL(sqlContext: SQLContext, argument: Map[String, String]): Unit = {


    val connectionURL = ConnectionDetails.getOrElse("connectionURL",null)
    val userName = ConnectionDetails.getOrElse("userName",null)
    val password = ConnectionDetails.getOrElse("password",null)
    //val sourceDF = ConnectionDetls.values("sourceDfName")

    println(connectionURL + " " + userName + " " + password)
    val sourceTable = ConnectionDetails.getOrElse("sourceTableName",null)
     var dataFrame= sqlContext.read.format("jdbc").options(
      Map("url" -> connectionURL, "user" -> userName, "password" -> password, "dbtable" -> sourceTable,
        "driver" -> "oracle.jdbc.OracleDriver")).
      load()


    println(connectionURL)
    println(userName)
    println(password)
    //println(sourceDF)
    //println(sourceTable)

  }

  override def getDataFrame(): DataFrame = {

    return dataFrame
  }

  override def createTempTable(tableName : String): String ={

    dataFrame.registerTempTable(tableName)
    return tableName
  }
}


object Read1 extends App {

  println("Starting  Read")
  val sqlcontext= sparkContextBuilder.sqlContext
  val connString=Map("connectionURL" -> "jdbc:oracle:thin:CR_PRO/CR_PRO@//tr03.rxcorp.com:1521/DMR" ,"userName" -> "CR_PROC","password" -> "CR_PROC","sourceTableName" -> "MRR_DLY_CURR_ECHG", "driver" -> "oracle.jdbc.OracleDriver")
  println("calling  Oracle readerRead")
  val v1 = new OracleSourceReader(sqlcontext,connString)
  val v2= v1.getDataFrame()
  v2.first()
}