Spark Submit命令：
/opt/cmsgraph/spark/default/bin/spark-submit -v \
--driver-java-options -Djava.io.tmpdir=/opt/cmsgraph/temp
--conf spark.cassandra.connection.timeout_ms=60000 \
--conf spark.cassandra.input.fetch.size_in_rows=1 \
--conf spark.cassandra.input.split.size_in_mb=8 \
--conf spark.cassandra.input.reads_per_sec=1 \
--conf spark.cassandra.output.consistency.level=LOCAL_ONE \
--jars spark-cassandra-connector_2.11-2.0.5.jar,cms-api-4.1.0-SNAPSHOT.jar \
--master spark://************:7077 \
--class "com.til.cms.sparkjobs.SolrBulkJob" \
/opt/cmsgraph/sparkjobs/spark/sparkjobs-1.0.jar
这适用于spark独立群集版本2.1.1，但是当我升级到spark 2.2.0时。它开始给出class not found异常，缺少的类是我的主jar（sparkjobs-1.0.jar）的一部分。 我正在使用1个火花大师和10个执行者奴隶。
堆栈追踪：
13:04:04,165 INFO CassandraConnector:35 - Connected to Cassandra cluster: CLUSTER_CMSENTITY
Exception in thread "main" java.lang.ClassNotFoundException: com.til.cms.sparkjobs.CmsEntitySolrParentTemp
at java.net.URLClassLoader.findClass(URLClassLoader.java:381)
at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
at sun.misc.Launcher$AppClassLoader.loadClass(Launcher.java:331)
at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
at java.lang.Class.forName0(Native Method)
at java.lang.Class.forName(Class.java:348)
at scala.reflect.runtime.JavaMirrors$JavaMirror.javaClass(JavaMirrors.scala:555)
at scala.reflect.runtime.JavaMirrors$JavaMirror$$anonfun$classToJava$1.apply(JavaMirrors.scala:1211)
at scala.reflect.runtime.JavaMirrors$JavaMirror$$anonfun$classToJava$1.apply(JavaMirrors.scala:1203)
at scala.reflect.runtime.TwoWayCaches$TwoWayCache$$anonfun$toJava$1.apply(TwoWayCaches.scala:49)
at scala.reflect.runtime.Gil$class.gilSynchronized(Gil.scala:19)
at scala.reflect.runtime.JavaUniverse.gilSynchronized(JavaUniverse.scala:16)
at scala.reflect.runtime.TwoWayCaches$TwoWayCache.toJava(TwoWayCaches.scala:44)
at scala.reflect.runtime.JavaMirrors$JavaMirror.classToJava(JavaMirrors.scala:1203)
at scala.reflect.runtime.JavaMirrors$JavaMirror.typeToJavaClass(JavaMirrors.scala:1297)
at scala.reflect.runtime.JavaMirrors$JavaMirror.runtimeClass(JavaMirrors.scala:192)
at scala.reflect.runtime.JavaMirrors$JavaMirror.runtimeClass(JavaMirrors.scala:54)
at com.datastax.spark.connector.util.ReflectionUtil$.classTag(ReflectionUtil.scala:148)
at com.datastax.spark.connector.rdd.reader.GettableDataToMappedTypeConverter.columnMapper(GettableDataToMappedTypeConverter.scala:69)
at com.datastax.spark.connector.rdd.reader.GettableDataToMappedTypeConverter.converter(GettableDataToMappedTypeConverter.scala:91)
at com.datastax.spark.connector.rdd.reader.GettableDataToMappedTypeConverter.com$datastax$spark$connector$rdd$reader$GettableDataToMappedTypeConverter$$converter(GettableDataToMappedTypeConverter.scala:117)
at com.datastax.spark.connector.rdd.reader.GettableDataToMappedTypeConverter.converter(GettableDataToMappedTypeConverter.scala:95)
at com.datastax.spark.connector.rdd.reader.GettableDataToMappedTypeConverter.com$datastax$spark$connector$rdd$reader$GettableDataToMappedTypeConverter$$converter(GettableDataToMappedTypeConverter.scala:117)
at com.datastax.spark.connector.rdd.reader.GettableDataToMappedTypeConverter$$anonfun$7.apply(GettableDataToMappedTypeConverter.scala:184)
at com.datastax.spark.connector.rdd.reader.GettableDataToMappedTypeConverter$$anonfun$7.apply(GettableDataToMappedTypeConverter.scala:181)
at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234)
at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234)
at scala.collection.Iterator$class.foreach(Iterator.scala:893)
at scala.collection.AbstractIterator.foreach(Iterator.scala:1336)
at scala.collection.MapLike$DefaultKeySet.foreach(MapLike.scala:174)
at scala.collection.TraversableLike$class.map(TraversableLike.scala:234)
at scala.collection.AbstractSet.scala$collection$SetLike$$super$map(Set.scala:47)
at scala.collection.SetLike$class.map(SetLike.scala:92)
at scala.collection.AbstractSet.map(Set.scala:47)
at com.datastax.spark.connector.rdd.reader.GettableDataToMappedTypeConverter.<init>(GettableDataToMappedTypeConverter.scala:181)
at com.datastax.spark.connector.rdd.reader.ClassBasedRowReader.<init>(ClassBasedRowReader.scala:21)
at com.datastax.spark.connector.rdd.reader.ClassBasedRowReaderFactory.rowReader(ClassBasedRowReader.scala:44)
at com.datastax.spark.connector.rdd.reader.ClassBasedRowReaderFactory.rowReader(ClassBasedRowReader.scala:39)
at com.datastax.spark.connector.rdd.CassandraTableRowReaderProvider$class.rowReader(CassandraTableRowReaderProvider.scala:48)
at com.datastax.spark.connector.rdd.CassandraTableScanRDD.rowReader$lzycompute(CassandraTableScanRDD.scala:62)
at com.datastax.spark.connector.rdd.CassandraTableScanRDD.rowReader(CassandraTableScanRDD.scala:62)
at com.datastax.spark.connector.rdd.CassandraTableRowReaderProvider$class.verify(CassandraTableRowReaderProvider.scala:138)
at com.datastax.spark.connector.rdd.CassandraTableScanRDD.verify(CassandraTableScanRDD.scala:62)
at com.datastax.spark.connector.rdd.CassandraTableScanRDD.getPartitions(CassandraTableScanRDD.scala:262)
at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:252)
at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:250)
at scala.Option.getOrElse(Option.scala:121)
at org.apache.spark.rdd.RDD.partitions(RDD.scala:250)
at org.apache.spark.rdd.MapPartitionsRDD.getPartitions(MapPartitionsRDD.scala:35)
at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:252)
at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:250)
at scala.Option.getOrElse(Option.scala:121)
at org.apache.spark.rdd.RDD.partitions(RDD.scala:250)
at org.apache.spark.rdd.MapPartitionsRDD.getPartitions(MapPartitionsRDD.scala:35)
at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:252)
at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:250)
at scala.Option.getOrElse(Option.scala:121)
at org.apache.spark.rdd.RDD.partitions(RDD.scala:250)
at org.apache.spark.rdd.MapPartitionsRDD.getPartitions(MapPartitionsRDD.scala:35)
at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:252)
at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:250)
at scala.Option.getOrElse(Option.scala:121)
at org.apache.spark.rdd.RDD.partitions(RDD.scala:250)
at org.apache.spark.SparkContext.runJob(SparkContext.scala:2075)
at com.datastax.spark.connector.RDDFunctions.saveToCassandra(RDDFunctions.scala:36)
at com.datastax.spark.connector.japi.RDDJavaFunctions.saveToCassandra(RDDJavaFunctions.java:61)
at com.datastax.spark.connector.japi.RDDAndDStreamCommonJavaFunctions$WriterBuilder.saveToCassandra(RDDAndDStreamCommonJavaFunctions.java:486)
at com.til.cms.sparkjobs.SolrBulkJob.main(SolrBulkJob.java:74)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:497)
at org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:755)
at org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:180)
at org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:205)
at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:119)
at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
13:04:12,195 INFO CassandraConnector:35 - Disconnected from Cassandra cluster: CLUSTER_CMSENTITY
13:04:13,176 INFO SparkContext:54 - Invoking stop() from shutdown hook
13:04:13,178 INFO SerialShutdownHooks:35 - Successfully executed shutdown hook: Clearing session cache for C* connector