只想使用spark-cassandra-connector
从密钥空间playerinfo
访问表player
。
import org.apache.commons.lang3.StringUtils;
import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.function.Function;
import com.datastax.spark.connector.cql.CassandraConnector;
import static com.datastax.spark.connector.japi.CassandraJavaUtil.*;
import com.datastax.spark.connector.japi.CassandraRow;
public class SparkTest implements Serializable {
private transient SparkConf conf;
private SparkTest(SparkConf conf) {
this.conf = conf;
}
private void run() {
JavaSparkContext sc = new JavaSparkContext(conf);
CassandraConnector connector = CassandraConnector.apply(sc.getConf());
JavaRDD<String> rdd = javaFunctions(sc).cassandraTable("player", "playerinfo")
.select("id").map(new Function<CassandraRow, String>() {
@Override
public String call(CassandraRow cassandraRow) throws Exception {
return cassandraRow.toString();
}
});
System.out.println("Data with only 'id' column fetched: \n" + StringUtils.join(rdd.toArray(), "\n"));
System.out.println("finished!");
}
public static void main(String[] args) {
SparkConf conf = new SparkConf();
conf.setAppName("Java API demo");
conf.setMaster("local[1]");
System.out.println("---------------------------------");
conf.set("spark.cassandra.connection.host", "10.6.1.244");
SparkTest app = new SparkTest(conf);
app.run();
}
}
这是错误堆栈:
Exception in thread "main" java.io.IOException: Failed to open thrift connection to Cassandra at 10.6.1.244:9160
at com.datastax.spark.connector.cql.CassandraConnector.createThriftClient(CassandraConnector.scala:132)
at com.datastax.spark.connector.cql.CassandraConnector.createThriftClient(CassandraConnector.scala:138)
at com.datastax.spark.connector.cql.CassandraConnector.withCassandraClientDo(CassandraConnector.scala:144)
at com.datastax.spark.connector.rdd.partitioner.CassandraRDDPartitioner.partitions(CassandraRDDPartitioner.scala:158)
at com.datastax.spark.connector.rdd.CassandraRDD.getPartitions(CassandraRDD.scala:294)
at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:204)
at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:202)
at scala.Option.getOrElse(Option.scala:120)
at org.apache.spark.rdd.RDD.partitions(RDD.scala:202)
at org.apache.spark.rdd.MappedRDD.getPartitions(MappedRDD.scala:28)
at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:204)
at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:202)
at scala.Option.getOrElse(Option.scala:120)
at org.apache.spark.rdd.RDD.partitions(RDD.scala:202)
at org.apache.spark.SparkContext.runJob(SparkContext.scala:1135)
at org.apache.spark.rdd.RDD.collect(RDD.scala:774)
at org.apache.spark.api.java.JavaRDDLike$class.collect(JavaRDDLike.scala:305)
at org.apache.spark.api.java.JavaRDD.collect(JavaRDD.scala:32)
at org.apache.spark.api.java.JavaRDDLike$class.toArray(JavaRDDLike.scala:325)
at org.apache.spark.api.java.JavaRDD.toArray(JavaRDD.scala:32)
at sparkDemo.SparkTest.run(SparkTest.java:36)
at sparkDemo.SparkTest.main(SparkTest.java:49)
Caused by: java.lang.NoSuchMethodError: org.apache.cassandra.thrift.TFramedTransportFactory.openTransport(Ljava/lang/String;I)Lorg/apache/thrift/transport/TTransport;
at com.datastax.spark.connector.cql.DefaultConnectionFactory$.createThriftClient(CassandraConnectionFactory.scala:47)
at com.datastax.spark.connector.cql.CassandraConnector.createThriftClient(CassandraConnector.scala:127)
... 21 more
为什么我会出现这样的错误:
Failed to open thrift connection to Cassandra at 10.6.1.244:9160
任何人都知道这个问题是什么?这个错误让我发疯了......
答案 0 :(得分:2)
Cassandra on&#34; 10.6.1.244&#34;似乎没有在Thrift客户端上侦听端口9160。您可以通过telnet 10.6.1.244 9160
进行确认。如果你看到&#34;连接被拒绝&#34;消息,那么你的Cassandra没有收听端口号。请检查您的Cassandra配置。