Pyspark saveAsTextFile错误

时间:2018-04-03 07:40:29

标签: python apache-spark pyspark

这是我的代码:

alist = [('A',1),('B',2),('C',3)]
rdd = sc.parallelize(alist)
import json
rdd.map(lambda o: json.dumps(o)).saveAsTextFile("data/jsonedTSV")

这是错误

Py4JJavaError                             Traceback (most recent call last)
<ipython-input-56-4fc40f9b6100> in <module>()
      1 import json
----> 2 rdd.map(lambda o: json.dumps(o)).saveAsTextFile("data/jsonedTSV")

C:\spark\python\pyspark\rdd.pyc in saveAsTextFile(self, path, compressionCodecClass)
   1566             keyed._jrdd.map(self.ctx._jvm.BytesToString()).saveAsTextFile(path, compressionCodec)
   1567         else:
-> 1568             keyed._jrdd.map(self.ctx._jvm.BytesToString()).saveAsTextFile(path)
   1569 
   1570     # Pair functions

C:\spark\python\lib\py4j-0.10.6-src.zip\py4j\java_gateway.py in __call__(self, *args)
   1158         answer = self.gateway_client.send_command(command)
   1159         return_value = get_return_value(
-> 1160             answer, self.gateway_client, self.target_id, self.name)
   1161 
   1162         for temp_arg in temp_args:

C:\spark\python\pyspark\sql\utils.pyc in deco(*a, **kw)
     61     def deco(*a, **kw):
     62         try:
---> 63             return f(*a, **kw)
     64         except py4j.protocol.Py4JJavaError as e:
     65             s = e.java_exception.toString()

C:\spark\python\lib\py4j-0.10.6-src.zip\py4j\protocol.py in get_return_value(answer, gateway_client, target_id, name)
    318                 raise Py4JJavaError(
    319                     "An error occurred while calling {0}{1}{2}.\n".
--> 320                     format(target_id, ".", name), value)
    321             else:
    322                 raise Py4JError(

Py4JJavaError: An error occurred while calling o523.saveAsTextFile.

在互联网上同样的人说,这可能是因为hadoop版本,但我不知道该怎么办,如果这是原因。

0 个答案:

没有答案