我想在我的hdfs中读取一个二进制文件。 Hadoop服务已正常启动并运行。但是,这是怎么回事此错误。我已经彻底检查过我的hdfs端口localhost:9000 /。
sample_hdfs_path = "hdfs://localhost:9000/IOCL_DATA /DATA_2_18.12.2015_20.32.34.bin"
from pyspark import SparkContext, SparkConf
from pyspark.sql.types import StructType
spark = SparkSession.builder.getOrCreate()
bin_RDD = sc.binaryRecords(sample_hdfs_path,bit_lenght)
bin_RDD.take(1)[enter image description here][1]
在运行此行之后,它始终显示。我认为这是连接问题,但我无法纠正。
Py4JJavaError Traceback (most recent call last)
<ipython-input-21-5fd499e19671> in <module>()
----> 1 bin_RDD.take(1)
/home/alethe/Downloads/spark-2.4.0-bin-hadoop2.7/python/pyspark/rdd.py in take(self, num)
1325 """
1326 items = []
-> 1327 totalParts = self.getNumPartitions()
1328 partsScanned = 0
1329
/home/alethe/Downloads/spark-2.4.0-bin-hadoop2.7/python/pyspark/rdd.py in getNumPartitions(self)
389 2
390 """
--> 391 return self._jrdd.partitions().size()
392
393 def filter(self, f):
/home/alethe/Downloads/spark-2.4.0-bin-hadoop2.7/python/lib/py4j-0.10.7-src.zip/py4j/java_gateway.py in __call__(self, *args)
1255 answer = self.gateway_client.send_command(command)
1256 return_value = get_return_value(
-> 1257 answer, self.gateway_client, self.target_id, self.name)
1258
1259 for temp_arg in temp_args:
home/alethe/Downloads/spark-2.4.0-bin-hadoop2.7/python/pyspark/sql/utils.py in deco(*a, **kw)
61 def deco(*a, **kw):
62 try:
---> 63 return f(*a, **kw)
64 except py4j.protocol.Py4JJavaError as e:
65 s = e.java_exception.toString()
/home/alethe/Downloads/spark-2.4.0-bin-hadoop2.7/python/lib/py4j-0.10.7-src.zip/py4j/protocol.py in get_return_value(answer, gateway_client, target_id, name)
326 raise Py4JJavaError(
327 "An error occurred while calling {0}{1}{2}.\n".
--> 328 format(target_id, ".", name), value)
329 else:
330 raise Py4JError(
Py4JJavaError: An error occurred while calling o204.partitions.
: java.net.ConnectException: Call From test/127.0.0.1 to localhost:9000
failed on connection exception: java.net.ConnectException: Connection refused;
For more details see: http://wiki.apache.org/hadoop/ConnectionRefused