我在EMR群集上运行pyspark，使用以下查询将pyspark sql查询转换为pandas时，始终收到错误消息。 我正在尝试在AWS Glue中查询数据。昨天有效
df = spark.sql("select * from flat_demography limit 10").toPandas()
Py4JJavaError Traceback (most recent call last)
<ipython-input-31-b9ae2ecc55b2> in <module>()
----> 3 df = spark.sql("select * from agg_clicks limit 300000").toPandas()
4 df = df.loc[df.ad_id != "/13/131/185"]
/usr/lib/spark/python/pyspark/sql/dataframe.py in toPandas(self)
1966 raise RuntimeError("%s\n%s" % (_exception_message(e), msg))
1967 else:
-> 1968 pdf = pd.DataFrame.from_records(self.collect(), columns=self.columns)
1969
1970 dtype = {}
/usr/lib/spark/python/pyspark/sql/dataframe.py in collect(self)
464 """
465 with SCCallSiteSync(self._sc) as css:
--> 466 sock_info = self._jdf.collectToPython()
467 return list(_load_from_socket(sock_info, BatchedSerializer(PickleSerializer())))
468
/usr/lib/spark/python/lib/py4j-0.10.7-src.zip/py4j/java_gateway.py in __call__(self, *args)
1255 answer = self.gateway_client.send_command(command)
1256 return_value = get_return_value(
-> 1257 answer, self.gateway_client, self.target_id, self.name)
1258
1259 for temp_arg in temp_args:
/usr/lib/spark/python/pyspark/sql/utils.py in deco(*a, **kw)
61 def deco(*a, **kw):
62 try:
---> 63 return f(*a, **kw)
64 except py4j.protocol.Py4JJavaError as e:
65 s = e.java_exception.toString()
/usr/lib/spark/python/lib/py4j-0.10.7-src.zip/py4j/protocol.py in get_return_value(answer, gateway_client, target_id, name)
326 raise Py4JJavaError(
327 "An error occurred while calling {0}{1}{2}.\n".
--> 328 format(target_id, ".", name), value)
329 else:
330 raise Py4JError(
Py4JJavaError: An error occurred while calling o216.collectToPython.
: org.apache.spark.SparkException: Job aborted due to stage failure: Task serialization failed: java.io.IOException: Failed to create local dir in /mnt/tmp/blockmgr-f6b2927d-b478-4699-9f9d-9475151e442d/25.
java.io.IOException: Failed to create local dir in /mnt/tmp/blockmgr-f6b2927d-b478-4699-9f9d-9475151e442d/25.