Question

我通过添加

在build.sbt中添加了作业服务器0.9.0依赖项

scalaVersion := "2.11.0"
resolvers += "Job Server Bintray" at "https://dl.bintray.com/spark-jobserver/maven"
libraryDependencies ++= Seq(
"spark.jobserver" %% "job-server-api" % "0.9.0",
"org.apache.spark" %% "spark-core" % "2.3.2",
"org.apache.spark" %% "spark-sql" % "2.3.2",
)

但是，当我尝试测试这样的代码（从https://github.com/spark-jobserver/spark-jobserver#new-sparkjob-api-with-spark-v21复制）时：

import com.typesafe.config.Config
import org.apache.spark.sql.SparkSession
import org.scalactic._
import spark.jobserver.api.{JobEnvironment, SingleProblem, ValidationProblem}
import spark.jobserver.SparkSessionJob
import scala.util.Try
object WordCountExampleSparkSession extends SparkSessionJob {
  type JobData = Seq[String]
  type JobOutput = collection.Map[String, Long]
  override def runJob(sparkSession: SparkSession, runtime: JobEnvironment, data: JobData): JobOutput =
sparkSession.sparkContext.parallelize(data).countByValue

  override def validate(sparkSession: SparkSession, runtime: JobEnvironment, config: Config): JobData Or Every[ValidationProblem] = {
Try(config.getString("input.string").split(" ").toSeq)
  .map(words => Good(words))
  .getOrElse(Bad(One(SingleProblem("No input.string param"))))
  }
}

IntelliJ显示错误：“无法解析符号'SparkSessionJob'” jar文件如下所示： enter image description here

Answer 1

我发现，sparkSessionJob位于job-server-extras中，所以我只需要添加

let count: Int = String(text).utf16.count
textDocumentProxy.adjustTextPosition(byCharacterOffset: count)

如何导入spark.jobserver.SparkSessionJob

1 个答案: