我认为是由Window函数引起的错误。
当我应用此脚本并仅保留几个示例行时，它就可以正常工作，但是当我将其应用于整个数据集时（只有几个GB） 在尝试持久保存到hdfs时，它在最后一步失败并显示了这个奇怪的错误...当我不使用Window Function时，脚本可以工作，因此问题必定是这样的（我大约有325个要素列正在运行for循环）。
任何想法都可能引起问题吗？我的目标是通过前向填充方法将时间序列数据插入数据框中的每个变量。
from pyspark.sql import SparkSession
from pyspark.sql import functions as F
from pyspark.sql import types as T
from pyspark.sql import Window
import sys
print(spark.version)
'2.3.0'
# sample data
df = spark.createDataFrame([('2019-05-10 7:30:05', '1', '10', '0.5', 'FALSE'),\
('2019-05-10 7:30:10', '2', 'UNKNOWN', '0.24', 'FALSE'),\
('2019-05-10 7:30:15', '3', '6', 'UNKNOWN', 'TRUE'),\
('2019-05-10 7:30:20', '4', '7', 'UNKNOWN', 'UNKNOWN'),\
('2019-05-10 7:30:25', '5', '10', '1.1', 'UNKNOWN'),\
('2019-05-10 7:30:30', '6', 'UNKNOWN', '1.1', 'NULL'),\
('2019-05-10 7:30:35', '7', 'UNKNOWN', 'UNKNOWN', 'TRUE'),\
('2019-05-10 7:30:49', '8', '50', 'UNKNOWN', 'UNKNOWN')], ["date", "id", "v1", "v2", "v3"])
df = df.withColumn("date", F.col("date").cast("timestamp"))
# imputer process / all cols that need filled are strings
def stringReplacer(x, y):
return F.when(x != y, x).otherwise(F.lit(None)) # replace with NULL
def forwardFillImputer(df, cols=[], partitioner="date", value="UNKNOWN"):
for i in cols:
window = Window\
.partitionBy(F.month(partitioner))\
.orderBy(partitioner)\
.rowsBetween(-sys.maxsize, 0)
df = df\
.withColumn(i, stringReplacer(F.col(i), value))
fill = F.last(df[i], ignorenulls=True).over(window)
df = df.withColumn(i, fill)
return df
df2 = forwardFillImputer(df, cols=[i for i in df.columns])
# errors here
df2\
.write\
.format("csv")\
.mode("overwrite")\
.option("header", "true")\
.save("test_window_func.csv")
Py4JJavaError: An error occurred while calling o13504.save.
: org.apache.spark.SparkException: Job aborted.
at org.apache.spark.sql.execution.datasources.FileFormatWriter$.write(FileFormatWriter.scala:224)
at org.apache.spark.sql.execution.datasources.InsertIntoHadoopFsRelationCommand.run(InsertIntoHadoopFsRelationCommand.scala:154)
at org.apache.spark.sql.execution.command.DataWritingCommandExec.sideEffectResult$lzycompute(commands.scala:104)
at org.apache.spark.sql.execution.command.DataWritingCommandExec.sideEffectResult(commands.scala:102)
at org.apache.spark.sql.execution.command.DataWritingCommandExec.doExecute(commands.scala:122)
at org.apache.spark.sql.execution.SparkPlan$$anonfun$execute$1.apply(SparkPlan.scala:131)
at org.apache.spark.sql.execution.SparkPlan$$anonfun$execute$1.apply(SparkPlan.scala:127)
at org.apache.spark.sql.execution.SparkPlan$$anonfun$executeQuery$1.apply(SparkPlan.scala:155)
at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:152)
at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:127)
at org.apache.spark.sql.execution.QueryExecution.toRdd$lzycompute(QueryExecution.scala:80)
at org.apache.spark.sql.execution.QueryExecution.toRdd(QueryExecution.scala:80)
at org.apache.spark.sql.DataFrameWriter$$anonfun$runCommand$1.apply(DataFrameWriter.scala:654)
at org.apache.spark.sql.DataFrameWriter$$anonfun$runCommand$1.apply(DataFrameWriter.scala:654)
at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:77)
at org.apache.spark.sql.DataFrameWriter.runCommand(DataFrameWriter.scala:654)
at org.apache.spark.sql.DataFrameWriter.saveToV1Source(DataFrameWriter.scala:273)
at org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:267)
at org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:225)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)
at py4j.Gateway.invoke(Gateway.java:282)
at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
at py4j.commands.CallCommand.execute(CallCommand.java:79)
at py4j.GatewayConnection.run(GatewayConnection.java:238)
at java.lang.Thread.run(Thread.java:748)
Caused by: java.lang.StackOverflowError
at org.apache.spark.sql.execution.SparkPlan.prepare(SparkPlan.scala:200)
at org.apache.spark.sql.execution.SparkPlan$$anonfun$prepare$1.apply(SparkPlan.scala:200)
at org.apache.spark.sql.execution.SparkPlan$$anonfun$prepare$1.apply(SparkPlan.scala:200)
at scala.collection.immutable.List.foreach(List.scala:381)
def forwardFillImputer(df, cols=[], partitioner="date", value="UNKNOWN"):
window = Window \
.partitionBy(F.month(partitioner)) \
.orderBy(partitioner) \
.rowsBetween(-sys.maxsize, 0)
imputed_cols = [F.last(stringReplacer(F.col(i), value), ignorenulls=True).over(window).alias(i)
for i in cols]
missing_cols = [i for i in df.columns if i not in cols]
return df.select(missing_cols+imputed_cols)
df2 = forwardFillImputer(df, cols=[i for i in df.columns[1:]])
df2.printSchema()
root
|-- date: timestamp (nullable = true)
|-- id: string (nullable = true)
|-- v1: string (nullable = true)
|-- v2: string (nullable = true)
|-- v3: string (nullable = true)
df2.show()
+-------------------+---+---+----+-----+
| date| id| v1| v2| v3|
+-------------------+---+---+----+-----+
|2019-05-10 07:30:05| 1| 10| 0.5|FALSE|
|2019-05-10 07:30:10| 2| 10|0.24|FALSE|
|2019-05-10 07:30:15| 3| 6|0.24| TRUE|
|2019-05-10 07:30:20| 4| 7|0.24| TRUE|
|2019-05-10 07:30:25| 5| 10| 1.1| TRUE|
|2019-05-10 07:30:30| 6| 10| 1.1| NULL|
|2019-05-10 07:30:35| 7| 10| 1.1| TRUE|
|2019-05-10 07:30:49| 8| 50| 1.1| TRUE|
+-------------------+---+---+----+-----+
答案 0 :(得分：2)
通过stacktrace，我相信错误来自执行计划的准备，正如它说的那样：
Caused by: java.lang.StackOverflowError
at org.apache.spark.sql.execution.SparkPlan.prepare(SparkPlan.scala:200)
我相信这样做的原因是因为您在循环中两次调用了.withColumn方法。 .withColumn在Spark执行计划中所做的基本上是对所有列的select语句，其中按方法指定更改了1列。如果您有325列，则对于单次迭代，它将对325列调用select两次-> 650列传递到计划程序中。这样做325次，您将看到它如何产生开销。
但是，很有趣的是，虽然您不会因为一小部分样本而收到此错误，但我希望不会。
无论如何，您可以尝试像这样替换您的forwardFillImputer：
def forwardFillImputer(df, cols=[], partitioner="date", value="UNKNOWN"):
window = Window \
.partitionBy(F.month(partitioner)) \
.orderBy(partitioner) \
.rowsBetween(-sys.maxsize, 0)
imputed_cols = [F.last(stringReplacer(F.col(i), value), ignorenulls=True).over(window).alias(i)
for i in cols]
missing_cols = [F.col(i) for i in df.columns if i not in cols]
return df.select(missing_cols + imputed_cols)
这样，您基本上只需将一个select语句解析到计划器中，该语句应该更易于处理。
仅作为警告，通常Spark在列数过多时效果不佳，因此您可能会在此过程中看到其他 strange 问题。