Question

在curl命令的帮助下在sparkJobServer上运行作业时，我遇到了classnotfound异常错误，但我期待一些不同的答案来代替版本不匹配，因为我已经尝试过了，请整体帮助配置属性，如何为作业服务器设置日志。下面给出的版本是我当前正在为此特定工作运行的版本。

借助curl命令-curl -d

在sparkJobServer上运行作业时

"listOfDates = [2018-01-01], channelName = bmb, standardEventColumns = {eventId: event_id, sessionId : sessionid, customerId : customer_id,eventdatetime : eventdatetime,errorCode : messageresponsecode, eventType : operationcode}, standardCustomerColumns = {customerId : customer_id},tableFilters = [{tableName: customer,colFilters: [{column: {name:segment},expression : equals,value : Premium}]}], fsPaths = {eventsInputBasePath: JourneyAnalytics/src/test/resources/data/Events,customerInputBasePath : JourneyAnalytics/src/test/resources/data/Customers},report=[{rType : StepAnalysis,eventName : Login,fSteps : 1,bSteps : 2}]" "localhost:8090/jobs?appName=ij1&classPath=com.barclays.jobserver.finalJob&context=c2"

我的finalJob.scala代码-

    package com.sample.jobserver
    import com.sample.journeyanalytics.json.{MyJsonProtocol, RequestConfig}
    import com.sample.journeyanalytics.report.PrepareReport
    import com.typesafe.config.{Config, ConfigFactory}
    import MyJsonProtocol._
    import spray.json._
    import spark.jobserver.{SparkJob,SparkJobValid, SparkJobValidation}
    import org.apache.spark.sql.SparkSession
    import org.apache.spark._

    object finalJob extends SparkJob {
      val conf = new SparkConf().setMaster("local[4]").setAppName("PrepareReport")
      def main(args: Array[String]) {
        val sc = new SparkContext(conf)
        val config = ConfigFactory.parseString(" ")
        runJob(sc, config)
      }
      override def validate(sc: SparkContext, config: Config): SparkJobValidation = {
        SparkJobValid
        /*Try(config.getString("input.string"))
          .map(x => SparkJobValid)
          .getOrElse(SparkJobInvalid("No input.string config param"))*/
      }
      case class data(listOfDates:Object, channelName: String, standardEventColumns:Object,standardCustomerColumns:Object,
                      tableFilters:Object,fsPaths:Object,report:Object)

      //case class standardEventColumnsFormat(eventId: String, sessionId: String, customerId:String, eventdatetime:String,errorCode:String,eventType:String)
      //case class standardCustomerColumnsFormat(customerId:String)
      //case class tableFilters(tableName:String, colFilters:t1)
      //case class t1(column:tc1,expression:String,value:String)
      //case class tc1(name:String)
      //case class fsPaths(eventsInputBasePath: String, customerInputBasePath: String)
      //case class report(rType: String, eventName: String,fSteps:Int,bSteps:Int)

      override def runJob(sc: SparkContext, config: Config): Any = {
        val spark = SparkSession.builder().config(conf).getOrCreate()
        val cr1 = config.getAnyRef("listOfDates")
        val cr2 = config.getString("channelName")
        val cr3 = config.getAnyRef("standardEventColumns")
        val cr4 = config.getAnyRef("standardCustomerColumns")
        val cr5 = config.getAnyRef(" tableFilters")
        val cr6 = config.getAnyRef("fsPaths")
        val cr7 = config.getAnyRef("report")
        //val cr1: String = config.getString("input.string")
        val p = data(cr1,cr2,cr3,cr4,cr5,cr6,cr7)

        import com.google.gson.Gson
        val gson= new Gson
        val jsonString = gson.toJson(p).mkString
        val requestConfig = JsonParser(jsonString).convertTo[RequestConfig]
        PrepareReport.prepare(spark,requestConfig)

      }
    }

errorStack-

      "duration": "0.011 secs",
      "classPath": "com.barclays.jobserver.finalJob",
      "startTime": "2019-05-14T13:42:10.180+05:30",
      "context": "c2",
      "result": {
        "message": "org/apache/spark/sql/SparkSession$",
        "errorClass": "java.lang.NoClassDefFoundError",
        "stack": "java.lang.NoClassDefFoundError: org/apache/spark/sql/SparkSession$\r\n\tat com.barclays.jobserver.finalJob$.runJob(finalJob.scala:36)\r\n\tat com.barclays.jobserver.finalJob$.runJob(finalJob.scala:11)\r\n\tat spark.jobserver.SparkJobBase$class.runJob(SparkJob.scala:31)\r\n\tat com.barclays.jobserver.finalJob$.runJob(finalJob.scala:11)\r\n\tat com.barclays.jobserver.finalJob$.runJob(finalJob.scala:11)\r\n\tat spark.jobserver.JobManagerActor$$anonfun$getJobFuture$4.apply(JobManagerActor.scala:594)\r\n\tat scala.concurrent.impl.Future$PromiseCompletingRunnable.liftedTree1$1(Future.scala:24)\r\n\tat scala.concurrent.impl.Future$PromiseCompletingRunnable.run(Future.scala:24)\r\n\tat java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)\r\n\tat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)\r\n\tat java.lang.Thread.run(Thread.java:748)\r\nCaused by: java.lang.ClassNotFoundException: org.apache.spark.sql.SparkSession$\r\n\tat java.net.URLClassLoader.findClass(URLClassLoader.java:382)\r\n\tat java.lang.ClassLoader.loadClass(ClassLoader.java:424)\r\n\tat java.lang.ClassLoader.loadClass(ClassLoader.java:357)\r\n\t... 11 more\r\n"
      },
      "status": "ERROR",
      "jobId": "611dad20-726d-46aa-bf66-693ef2bdcb5a",
      "contextId": ""
    }

这整个工作非常好，在IntelliJ上运行或调试时，我也得到了输出，但是当涉及到sparkJobserver时，我遇到了这个问题。

scala版本-2.11.8 pom.xml-sparkJobserver版本-0.8.0 pom.xml-Spark版本-2.2.0

机器上的火花版本-2.2.3 在群集上运行的spark版本-2.3.2（最有可能是因为我使用的是0.9.1-sparkJobserver的快照版本）

有些人说当我保持相同的spark版本仍然是版本不匹配问题时，我仍然收到此错误，我遇到此错误已有5天了，请帮帮我。

与sparkJobserver Extras中的日志文件的配置和设置有关的任何帮助也将受到赞赏！！

现在我要把这个留给你们！！

java.lang.ClassNotFoundException：org.apache.spark.sql.SparkSession $

0 个答案: