Question

我正在将Kafka结构化流中的json数据写入文件路径，当我从shell中执行该操作时，我就可以做到。当我将其编译到jar并执行spark2提交时，仅_spark_metadata创建并且未找到数据

我尝试从外壳执行此操作，因此能够在文件路径中看到json文件。我使用“ sbt clean package”编译该程序，然后尝试使用spark-submit来运行，它不会创建任何数据。

export SPARK_KAFKA_VERSION=0.10
spark2-submit --jars /opt/cloudera/parcels/CDH-5.13.3-1.cdh5.13.3.p0.2/jars/kafka-clients-0.9.0-kafka-2.0.2.jar,/opt/cloudera/parcels/CDH-5.13.3-1.cdh5.13.3.p0.2/jars/spark-sql-kafka-0-10_2.11-2.3.0.cloudera5-20190107.080402-22.jar,/opt/cloudera/parcels/CDH-5.13.3-1.cdh5.13.3.p0.2/jars/mongo-spark-connector_2.11-2.3.2.jar,/opt/cloudera/parcels/CDH-5.13.3-1.cdh5.13.3.p0.2/jars/bson-3.9.0.jar,/opt/cloudera/parcels/CDH-5.13.3-1.cdh5.13.3.p0.2/jars/mongo-java-driver-3.9.0.jar,/opt/cloudera/parcels/CDH-5.13.3-1.cdh5.13.3.p0.2/jars/mongodb-driver-core-3.9.0.jar,/opt/cloudera/parcels/CDH-5.13.3-1.cdh5.13.3.p0.2/jars/mongodb-driver-3.9.0.jar,/opt/cloudera/parcels/CDH-5.13.3-1.cdh5.13.3.p0.2/jars/mongodb-driver-async-3.9.0.jar,/opt/cloudera/parcels/CDH-5.13.3-1.cdh5.13.3.p0.2/jars/casbah-commons_2.11-2.8.2.jar,/opt/cloudera/parcels/CDH-5.13.3-1.cdh5.13.3.p0.2/jars/casbah-core_2.11-2.8.2.jar,/opt/cloudera/parcels/CDH-5.13.3-1.cdh5.13.3.p0.2/jars/casbah-query_2.11-2.8.2.jar --class OSB_kafkaToSpark --master yarn --deploy-mode client /home/streaming_osb_2.11-0.1.0-SNAPSHOT.jar

import com.mongodb.client.MongoCollection
import com.mongodb.spark.config.WriteConfig
import com.mongodb.spark.{MongoConnector}
import org.apache.spark.sql.SparkSession
import org.apache.spark.sql.functions.{col, _}
import scala.collection.JavaConverters._
import scala.collection.mutable
import org.apache.spark.sql._
import org.bson.Document
import org.apache.spark.sql.streaming.Trigger

object OSB_kafkaToSpark {

  def main(args: Array[String]): Unit = {

    val spark = SparkSession.
      builder().
      appName("OSB_kafkaToSpark").
      getOrCreate()

println("SparkSession -> "+spark)

    import spark.implicits._

    val df = spark
      .readStream
      .format("kafka")
      .option("kafka.bootstrap.servers", "bootstrap1.com:9092, bootstrap2.com:9092")
      .option("subscribe", "topictest")
      .option("failOnDataLoss", "false")
      .load()
      val dfs = df.selectExpr("CAST(value AS STRING)")

val data = dfs.withColumn("splitted", split($"value", "/"))
      .select($"splitted".getItem(4).alias("region"), $"splitted".getItem(5).alias("service"), col("value"))
      .withColumn("service_type", regexp_extract($"service", """.*(Inbound|Outbound|Outound).*""", 1))
      .withColumn("region_type", concat(
        when(col("region").isNotNull, col("region")).otherwise(lit("null")), lit(" "),
        when(col("service").isNotNull, col("service_type")).otherwise(lit("null"))))
      .withColumn("datetime", regexp_extract($"value", """\d{4}-[01]\d-[0-3]\d [0-2]\d:[0-5]\d:[0-5]\d""", 0))

    val extractedDF = data.filter(
      col("region").isNotNull &&
        col("service").isNotNull &&
        col("value").isNotNull &&
        col("service_type").isNotNull &&
        col("region_type").isNotNull &&
      col("datetime").isNotNull)
        .filter("region != ''")
        .filter("service != ''")
        .filter("value != ''")
        .filter("service_type != ''")
        .filter("region_type != ''")
        .filter("datetime != ''")

     val pathstring = "/user/spark_streaming".concat(args(0))

    val query = extractedDF.writeStream
      .format("json")
      .option("path", pathstring)
      .option("checkpointLocation", "/user/checkpoint")
      .outputMode("append")
      .trigger(Trigger.ProcessingTime("20 seconds"))
      .start()
query.stop()
  }
}

我想在运行spark2-submit时将流数据保存到一个文件夹中

Answer 1

我想出了答案，我将需要使用query.awaitTermination（）

使用结构化流时，无法将json格式的数据写入路径。进行spark2提交时仅创建_spark_metadata

1 个答案: