由以下原因导致：java.time.format.DateTimeParseException：无法解析文本'2020-05-12 10:23:45'，在索引10处找到了未解析的文本

时间：2020-07-02 05:35:52

标签： scala apache-spark databricks

我正在创建一个UDF，它将为我找到一周的第一天。

UDF的输入将是来自数据帧的字符串列，该日期将日期时间存储在yyyy-MM-dd hh:MM:ss中。

我同意可以在没有UDF的情况下建立相同的对象，但是我想探索这样做的所有选择。到目前为止，我对通过UDF的实现感到困惑。

重要注意事项-每周的开始日期是星期一。

代码-

import org.apache.spark.sql.functions._
import java.time.format.DateTimeFormatter
import java.time.LocalDate
import org.joda.time.DateTimeConstants

val df1 = Seq((1, "2020-05-12 10:23:45", 5000), (2, "2020-11-11 12:12:12", 2000)).toDF("id", "DateTime", "miliseconds")
val findFirstDayOfWeek = udf((x:String) => {
  
  val dateFormat = DateTimeFormatter.ofPattern("yyyy-MM-dd")
  val dayOfWeek = LocalDate.parse(x,dateFormat).getDayOfWeek
  
  if (dayOfWeek != DateTimeConstants.MONDAY )
    {
      val newDate = LocalDate.parse(x).plusDays(DateTimeConstants.MONDAY - dayOfWeek.getValue())
      val firstDateOfTheWeek = newDate.format(dateFormat)
      firstDateOfTheWeek
      
    }
  else
  {
    val newDate = x
    newDate.format(dateFormat)
    
  }
})

val udf_new_df1 = df1.withColumn("week",findFirstDayOfWeek(col("DateTime")))

但是当我运行display(udf_new_df1)时，出现此错误-（在Databricks上）

org.apache.spark.SparkException: Failed to execute user defined function($anonfun$1: (string) => string)
    at org.apache.spark.sql.catalyst.expressions.ScalaUDF.eval(ScalaUDF.scala:1066)
    at org.apache.spark.sql.catalyst.expressions.Alias.eval(namedExpressions.scala:152)
    at org.apache.spark.sql.catalyst.expressions.InterpretedMutableProjection.apply(InterpretedMutableProjection.scala:62)
    at org.apache.spark.sql.catalyst.optimizer.ConvertToLocalRelation$$anonfun$apply$23$$anonfun$applyOrElse$23.apply(Optimizer.scala:1471)
    at org.apache.spark.sql.catalyst.optimizer.ConvertToLocalRelation$$anonfun$apply$23$$anonfun$applyOrElse$23.apply(Optimizer.scala:1471)
    at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234)
    at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234)
    at scala.collection.immutable.List.foreach(List.scala:392)
    at scala.collection.TraversableLike$class.map(TraversableLike.scala:234)
    at scala.collection.immutable.List.map(List.scala:296)
    at org.apache.spark.sql.catalyst.optimizer.ConvertToLocalRelation$$anonfun$apply$23.applyOrElse(Optimizer.scala:1471)
    at org.apache.spark.sql.catalyst.optimizer.ConvertToLocalRelation$$anonfun$apply$23.applyOrElse(Optimizer.scala:1466)
    at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$2.apply(TreeNode.scala:280)
    at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$2.apply(TreeNode.scala:280)
    at org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(TreeNode.scala:77)
    at org.apache.spark.sql.catalyst.trees.TreeNode.transformDown(TreeNode.scala:279)
    at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.org$apache$spark$sql$catalyst$plans$logical$AnalysisHelper$$super$transformDown(LogicalPlan.scala:29)
    at org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper$class.transformDown(AnalysisHelper.scala:149)
    at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.transformDown(LogicalPlan.scala:29)
    at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.transformDown(LogicalPlan.scala:29)
    at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$3.apply(TreeNode.scala:285)
    at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$3.apply(TreeNode.scala:285)
    at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$8.apply(TreeNode.scala:354)
    at org.apache.spark.sql.catalyst.trees.TreeNode.mapProductIterator(TreeNode.scala:208)
    at org.apache.spark.sql.catalyst.trees.TreeNode.mapChildren(TreeNode.scala:352)
    at org.apache.spark.sql.catalyst.trees.TreeNode.transformDown(TreeNode.scala:285)
    at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.org$apache$spark$sql$catalyst$plans$logical$AnalysisHelper$$super$transformDown(LogicalPlan.scala:29)
    at org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper$class.transformDown(AnalysisHelper.scala:149)
    at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.transformDown(LogicalPlan.scala:29)
    at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.transformDown(LogicalPlan.scala:29)
    at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$3.apply(TreeNode.scala:285)
    at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$3.apply(TreeNode.scala:285)
    at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$8.apply(TreeNode.scala:354)
    at org.apache.spark.sql.catalyst.trees.TreeNode.mapProductIterator(TreeNode.scala:208)
    at org.apache.spark.sql.catalyst.trees.TreeNode.mapChildren(TreeNode.scala:352)
    at org.apache.spark.sql.catalyst.trees.TreeNode.transformDown(TreeNode.scala:285)
    at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.org$apache$spark$sql$catalyst$plans$logical$AnalysisHelper$$super$transformDown(LogicalPlan.scala:29)
    at org.apache.spark.sql.catalyst.plans.logical.AnalysisHelper$class.transformDown(AnalysisHelper.scala:149)
    at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.transformDown(LogicalPlan.scala:29)
    at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.transformDown(LogicalPlan.scala:29)
    at org.apache.spark.sql.catalyst.trees.TreeNode.transform(TreeNode.scala:269)
    at org.apache.spark.sql.catalyst.optimizer.ConvertToLocalRelation$.apply(Optimizer.scala:1466)
    at org.apache.spark.sql.catalyst.optimizer.ConvertToLocalRelation$.apply(Optimizer.scala:1465)
    at org.apache.spark.sql.catalyst.rules.RuleExecutor$$anonfun$execute$1$$anonfun$apply$1.apply(RuleExecutor.scala:112)
    at org.apache.spark.sql.catalyst.rules.RuleExecutor$$anonfun$execute$1$$anonfun$apply$1.apply(RuleExecutor.scala:109)
    at scala.collection.IndexedSeqOptimized$class.foldl(IndexedSeqOptimized.scala:57)
    at scala.collection.IndexedSeqOptimized$class.foldLeft(IndexedSeqOptimized.scala:66)
    at scala.collection.mutable.WrappedArray.foldLeft(WrappedArray.scala:35)
    at org.apache.spark.sql.catalyst.rules.RuleExecutor$$anonfun$execute$1.apply(RuleExecutor.scala:109)
    at org.apache.spark.sql.catalyst.rules.RuleExecutor$$anonfun$execute$1.apply(RuleExecutor.scala:101)
    at scala.collection.immutable.List.foreach(List.scala:392)
    at org.apache.spark.sql.catalyst.rules.RuleExecutor.execute(RuleExecutor.scala:101)
    at org.apache.spark.sql.catalyst.rules.RuleExecutor$$anonfun$executeAndTrack$1.apply(RuleExecutor.scala:80)
    at org.apache.spark.sql.catalyst.rules.RuleExecutor$$anonfun$executeAndTrack$1.apply(RuleExecutor.scala:80)
    at org.apache.spark.sql.catalyst.QueryPlanningTracker$.withTracker(QueryPlanningTracker.scala:88)
    at org.apache.spark.sql.catalyst.rules.RuleExecutor.executeAndTrack(RuleExecutor.scala:79)
    at org.apache.spark.sql.execution.QueryExecution$$anonfun$optimizedPlan$1.apply(QueryExecution.scala:94)
    at org.apache.spark.sql.execution.QueryExecution$$anonfun$optimizedPlan$1.apply(QueryExecution.scala:94)
    at org.apache.spark.sql.catalyst.QueryPlanningTracker.measurePhase(QueryPlanningTracker.scala:111)
    at org.apache.spark.sql.execution.QueryExecution.optimizedPlan$lzycompute(QueryExecution.scala:93)
    at org.apache.spark.sql.execution.QueryExecution.optimizedPlan(QueryExecution.scala:93)
    at org.apache.spark.sql.execution.QueryExecution$$anonfun$toString$2.apply(QueryExecution.scala:263)
    at org.apache.spark.sql.execution.QueryExecution$$anonfun$toString$2.apply(QueryExecution.scala:263)
    at org.apache.spark.sql.execution.QueryExecution.stringOrError(QueryExecution.scala:147)
    at org.apache.spark.sql.execution.QueryExecution.toString(QueryExecution.scala:263)
    at org.apache.spark.sql.execution.SQLExecution$$anonfun$withCustomExecutionEnv$1.apply(SQLExecution.scala:102)
    at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:240)
    at org.apache.spark.sql.execution.SQLExecution$.withCustomExecutionEnv(SQLExecution.scala:97)
    at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:170)
    at org.apache.spark.sql.Dataset.org$apache$spark$sql$Dataset$$withAction(Dataset.scala:3441)
    at org.apache.spark.sql.Dataset.collectResult(Dataset.scala:2832)
    at com.databricks.backend.daemon.driver.OutputAggregator$.withOutputAggregation0(OutputAggregator.scala:149)
    at com.databricks.backend.daemon.driver.OutputAggregator$.withOutputAggregation(OutputAggregator.scala:54)
    at com.databricks.backend.daemon.driver.ScalaDriverLocal$$anonfun$getResultBufferInternal$1$$anonfun$apply$1.apply(ScalaDriverLocal.scala:318)
    at com.databricks.backend.daemon.driver.ScalaDriverLocal$$anonfun$getResultBufferInternal$1$$anonfun$apply$1.apply(ScalaDriverLocal.scala:303)
    at scala.Option.map(Option.scala:146)
    at com.databricks.backend.daemon.driver.ScalaDriverLocal$$anonfun$getResultBufferInternal$1.apply(ScalaDriverLocal.scala:303)
    at com.databricks.backend.daemon.driver.ScalaDriverLocal$$anonfun$getResultBufferInternal$1.apply(ScalaDriverLocal.scala:267)
    at scala.Option.map(Option.scala:146)
    at com.databricks.backend.daemon.driver.ScalaDriverLocal.getResultBufferInternal(ScalaDriverLocal.scala:267)
    at com.databricks.backend.daemon.driver.DriverLocal.getResultBuffer(DriverLocal.scala:463)
    at com.databricks.backend.daemon.driver.ScalaDriverLocal.repl(ScalaDriverLocal.scala:244)
    at com.databricks.backend.daemon.driver.DriverLocal$$anonfun$execute$8.apply(DriverLocal.scala:373)
    at com.databricks.backend.daemon.driver.DriverLocal$$anonfun$execute$8.apply(DriverLocal.scala:350)
    at com.databricks.logging.UsageLogging$$anonfun$withAttributionContext$1.apply(UsageLogging.scala:238)
    at scala.util.DynamicVariable.withValue(DynamicVariable.scala:58)
    at com.databricks.logging.UsageLogging$class.withAttributionContext(UsageLogging.scala:233)
    at com.databricks.backend.daemon.driver.DriverLocal.withAttributionContext(DriverLocal.scala:48)
    at com.databricks.logging.UsageLogging$class.withAttributionTags(UsageLogging.scala:271)
    at com.databricks.backend.daemon.driver.DriverLocal.withAttributionTags(DriverLocal.scala:48)
    at com.databricks.backend.daemon.driver.DriverLocal.execute(DriverLocal.scala:350)
    at com.databricks.backend.daemon.driver.DriverWrapper$$anonfun$tryExecutingCommand$2.apply(DriverWrapper.scala:644)
    at com.databricks.backend.daemon.driver.DriverWrapper$$anonfun$tryExecutingCommand$2.apply(DriverWrapper.scala:644)
    at scala.util.Try$.apply(Try.scala:192)
    at com.databricks.backend.daemon.driver.DriverWrapper.tryExecutingCommand(DriverWrapper.scala:639)
    at com.databricks.backend.daemon.driver.DriverWrapper.getCommandOutputAndError(DriverWrapper.scala:485)
    at com.databricks.backend.daemon.driver.DriverWrapper.executeCommand(DriverWrapper.scala:597)
    at com.databricks.backend.daemon.driver.DriverWrapper.runInnerLoop(DriverWrapper.scala:390)
    at com.databricks.backend.daemon.driver.DriverWrapper.runInner(DriverWrapper.scala:337)
    at com.databricks.backend.daemon.driver.DriverWrapper.run(DriverWrapper.scala:219)
    at java.lang.Thread.run(Thread.java:748)
Caused by: java.time.format.DateTimeParseException: Text '2020-05-12 10:23:45' could not be parsed, unparsed text found at index 10
    at java.time.format.DateTimeFormatter.parseResolved0(DateTimeFormatter.java:1952)
    at java.time.format.DateTimeFormatter.parse(DateTimeFormatter.java:1851)
    at java.time.LocalDate.parse(LocalDate.java:400)
    at linedde9e8e2c7794f68a6e16898b7ed370036.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$anonfun$1.apply(command-14467074:14)
    at linedde9e8e2c7794f68a6e16898b7ed370036.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$anonfun$1.apply(command-14467074:11)
    at org.apache.spark.sql.catalyst.expressions.ScalaUDF$$anonfun$2.apply(ScalaUDF.scala:108)
    at org.apache.spark.sql.catalyst.expressions.ScalaUDF$$anonfun$2.apply(ScalaUDF.scala:107)
    at org.apache.spark.sql.catalyst.expressions.ScalaUDF.eval(ScalaUDF.scala:1063)
    ... 100 more

所以我的问题是，为什么在解析类型为String且格式为yyyy-MM-dd hh：MM：ss的dateTime时出现问题？

2 个答案:

答案 0 :(得分：1)

不确定为什么要使用UDF，但是可以在一周的第一天不使用UDF，如下所示-

一周从Monday

开始

使用spark内置函数date_trunc 

 val df1 = Seq((1, "2020-05-12 10:23:45", 5000), (2, "2020-11-11 12:12:12", 2000)).toDF("id", "DateTime", "miliseconds")
    df1.withColumn("week", date_trunc("week", $"DateTime"))
      .show(false)

    /**
      * +---+-------------------+-----------+-------------------+
      * |id |DateTime           |miliseconds|week               |
      * +---+-------------------+-----------+-------------------+
      * |1  |2020-05-12 10:23:45|5000       |2020-05-11 00:00:00|
      * |2  |2020-11-11 12:12:12|2000       |2020-11-09 00:00:00|
      * +---+-------------------+-----------+-------------------+
      */

使用UDF 

  // convert dateTime -> date truncated to the first day of week
    val findFirstDayOfWeek = udf((x:String) => {

      val dateFormat = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss")
      val time = LocalDateTime.parse(x,dateFormat)
      val dayOfWeek = time.getDayOfWeek

      if (dayOfWeek.getValue != DateTimeConstants.MONDAY ) {
        val newDateTime = time.plusDays(DateTimeConstants.MONDAY - dayOfWeek.getValue())
        java.sql.Date.valueOf(newDateTime.toLocalDate)
      } else {
        java.sql.Date.valueOf(time.toLocalDate)
      }
    })

    val udf_new_df1 = df1.withColumn("week",findFirstDayOfWeek(col("DateTime")))
    udf_new_df1.show(false)
    udf_new_df1.printSchema()

    /**
      * +---+-------------------+-----------+----------+
      * |id |DateTime           |miliseconds|week      |
      * +---+-------------------+-----------+----------+
      * |1  |2020-05-12 10:23:45|5000       |2020-05-11|
      * |2  |2020-11-11 12:12:12|2000       |2020-11-09|
      * +---+-------------------+-----------+----------+
      *
      * root
      * |-- id: integer (nullable = false)
      * |-- DateTime: string (nullable = true)
      * |-- miliseconds: integer (nullable = false)
      * |-- week: date (nullable = true)
      */

答案 1 :(得分：0)

使用LocalDateTime.parse(x.replace(' ', 'T'))或LocalDate.parse(x.split(' ')(0))代替LocalDate.parse(x)和LocalDate.parse(x,dateFormat)。

$ scala
Welcome to Scala 2.13.0 (OpenJDK 64-Bit Server VM, Java 1.8.0_252).
Type in expressions for evaluation. Or try :help.

scala> java.time.LocalDateTime.parse("2020-05-12 10:23:45".replace(' ', 'T'))
res0: java.time.LocalDateTime = 2020-05-12T10:23:45

scala> java.time.LocalDate.parse("2020-05-12 10:23:45".split(' ')(0))
res1: java.time.LocalDate = 2020-05-12
相关问题
最新问题