Question

I am trying to get data from Cloudant using Java code and getting error,

I tried with below Spark and cloudant-spark version,

Spark 2.0.0,

Spark 2.0.1,

Spark 2.0.2

Getting same error for all version as error posted below.

If I add scala dependencies to resolve error this error than it is conflicting with Spark library.

Below is my java code,

package spark.cloudant.connecter;

import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.SQLContext;
import com.cloudant.spark.*;

public class cloudantconnecter {
    public static void main(String[] args) throws Exception {

        try {
            SparkConf sparkConf = new SparkConf().setAppName("spark cloudant connecter").setMaster("local[*]");
            sparkConf.set("spark.streaming.concurrentJobs", "30");

            JavaSparkContext sc = new JavaSparkContext(sparkConf);

            SQLContext sqlContext = new SQLContext(sc);
            System.out.print("initialization successfully");


            Dataset<org.apache.spark.sql.Row> st = sqlContext.read().format("com.cloudant.spark")
                    .option("cloudant.host", "HOSTNAME").option("cloudant.username", "USERNAME")
                    .option("cloudant.password", "PASSWORD").load("DATABASENAME");

            st.printSchema();


        } catch (

        Exception e) {
            e.printStackTrace();
        }
    }
}

Maven Dependencies

<dependencies>
        <dependency>
            <groupId>org.apache.spark</groupId>
            <artifactId>spark-core_2.10</artifactId>
            <version>2.0.0</version>
        </dependency>
        <dependency>
            <groupId>org.apache.spark</groupId>
            <artifactId>spark-mllib_2.10</artifactId>
            <version>2.0.0</version>
        </dependency>
        <dependency>
            <groupId>cloudant-labs</groupId>
            <artifactId>spark-cloudant</artifactId>
            <version>2.0.0-s_2.11</version>
        </dependency>
    </dependencies>

Getting error details,

Exception in thread "main" java.lang.NoSuchMethodError: scala/Predef$.ArrowAssoc(Ljava/lang/Object;)Ljava/lang/Object; (loaded from file:/C:/Users/Administrator/.m2/repository/org/scala-lang/scala-library/2.10.6/scala-library-2.10.6.jar by sun.misc.Launcher$AppClassLoader@9f916f97) called from class scalaj.http.HttpConstants$ (loaded from file:/C:/Users/Administrator/.m2/repository/org/scalaj/scalaj-http_2.11/2.3.0/scalaj-http_2.11-2.3.0.jar by sun.misc.Launcher$AppClassLoader@9f916f97).
    at scalaj.http.HttpConstants$.liftedTree1$1(Http.scala:637)
    at scalaj.http.HttpConstants$.<init>(Http.scala:636)
    at scalaj.http.HttpConstants$.<clinit>(Http.scala)
    at scalaj.http.BaseHttp$.$lessinit$greater$default$2(Http.scala:754)
    at scalaj.http.Http$.<init>(Http.scala:738)
    at scalaj.http.Http$.<clinit>(Http.scala)
    at com.cloudant.spark.common.JsonStoreDataAccess.getQueryResult(JsonStoreDataAccess.scala:152)
    at com.cloudant.spark.common.JsonStoreDataAccess.getTotalRows(JsonStoreDataAccess.scala:99)
    at com.cloudant.spark.common.JsonStoreRDD.totalRows$lzycompute(JsonStoreRDD.scala:56)
    at com.cloudant.spark.common.JsonStoreRDD.totalRows(JsonStoreRDD.scala:55)
    at com.cloudant.spark.common.JsonStoreRDD.totalPartition$lzycompute(JsonStoreRDD.scala:59)
    at com.cloudant.spark.common.JsonStoreRDD.totalPartition(JsonStoreRDD.scala:58)
    at com.cloudant.spark.common.JsonStoreRDD.getPartitions(JsonStoreRDD.scala:81)
    at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:248)
    at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:246)
    at scala.Option.getOrElse(Option.scala:120)
    at org.apache.spark.rdd.RDD.partitions(RDD.scala:246)
    at org.apache.spark.rdd.MapPartitionsRDD.getPartitions(MapPartitionsRDD.scala:35)
    at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:248)
    at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:246)
    at scala.Option.getOrElse(Option.scala:120)
    at org.apache.spark.rdd.RDD.partitions(RDD.scala:246)
    at org.apache.spark.SparkContext.runJob(SparkContext.scala:1934)
    at org.apache.spark.rdd.RDD$$anonfun$fold$1.apply(RDD.scala:1046)
    at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
    at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:112)
    at org.apache.spark.rdd.RDD.withScope(RDD.scala:358)
    at org.apache.spark.rdd.RDD.fold(RDD.scala:1040)
    at org.apache.spark.sql.execution.datasources.json.InferSchema$.infer(InferSchema.scala:68)
    at org.apache.spark.sql.DataFrameReader$$anonfun$3.apply(DataFrameReader.scala:317)
    at org.apache.spark.sql.DataFrameReader$$anonfun$3.apply(DataFrameReader.scala:317)
    at scala.Option.getOrElse(Option.scala:120)
    at org.apache.spark.sql.DataFrameReader.json(DataFrameReader.scala:316)
    at com.cloudant.spark.DefaultSource.create(DefaultSource.scala:127)
    at com.cloudant.spark.DefaultSource.createRelation(DefaultSource.scala:105)
    at com.cloudant.spark.DefaultSource.createRelation(DefaultSource.scala:100)
    at org.apache.spark.sql.execution.datasources.DataSource.resolveRelation(DataSource.scala:315)
    at org.apache.spark.sql.DataFrameReader.load(DataFrameReader.scala:149)
    at org.apache.spark.sql.DataFrameReader.load(DataFrameReader.scala:132)
    at spark.cloudant.connecter.cloudantconnecter.main(cloudantconnecter.java:24)

Answer 1

错误显示是因为提到的库使用scala 2.10并且使用2.11提到了包火花cloudant库

所以请将库spark-core_2.10更改为spark-core_2.11

现在依赖是，

<dependency>
            <groupId>org.apache.spark</groupId>
            <artifactId>spark-core_2.11</artifactId>
            <version>2.0.1</version>
        </dependency>
        <dependency>
            <groupId>org.apache.spark</groupId>
            <artifactId>spark-mllib_2.11</artifactId>
            <version>2.0.1</version>
        </dependency>   
        <dependency>
            <groupId>cloudant-labs</groupId>
            <artifactId>spark-cloudant</artifactId>
            <version>2.0.0-s_2.11</version>
        </dependency>

Not able to connect Spark-Cloudant

1 个答案: