Question

    import org.apache.hadoop.conf.Configuration
    import org.apache.hadoop.fs.{FileSystem, FileUtil, Path}
    import org.apache.spark.sql.SparkSession

    object APP{

      def merge(srcPath: String, dstPath: String): Unit = {
        val hadoopConfig = new Configuration()
        val hdfs = FileSystem.get(hadoopConfig)
        FileUtil.copyMerge(hdfs, new Path(srcPath), hdfs, new Path(dstPath), false, hadoopConfig, null)
        // the "true" setting deletes the source files once they are merged into the new output
      }

      def main(args: Array[String]): Unit = {

        val url = "jdbc:sqlserver://dc-bir-cdb01;databaseName=dbapp;integratedSecurity=true";
        val driver = "com.microsoft.sqlserver.jdbc.SQLServerDriver"
        val BusinessDate = "2019-02-28"
        val destination = "src/main/resources/out/"
        val filename = s"Example@$BusinessDate.csv.gz"
        val outputFileName = destination + "/temp_" + filename
        val mergedFileName = destination + "/merged_" + filename
        val mergeFindGlob = outputFileName


        val spark = SparkSession.
          builder.master("local[*]")
          //.config("spark.debug.maxToStringFields", "100")
          .appName("Application Big Data")
          .getOrCreate()
        val query = s"""(SELECT a,b,c From table') tmp """.stripMargin

        val responseWithSelectedColumns = spark
          .read
          .format("jdbc")
          .option("url", url)
          .option("driver", driver)
          .option("dbtable", query)
          .load()

        print("TOTAL: "+responseWithSelectedColumns.count())

        responseWithSelectedColumns
          .coalesce(1) //So just a single part- file will be created
          .repartition(10)
          .write.mode("overwrite")
          .option("codec", "org.apache.hadoop.io.compress.GzipCodec")
          .format("com.databricks.spark.csv")
          .option("charset", "UTF8")
          .option("mapreduce.fileoutputcommitter.marksuccessfuljobs", "false") //Avoid creating of crc files
          .option("header", "true") //Write the header

          .save(outputFileName)
        merge(mergeFindGlob, mergedFileName)
        responseWithSelectedColumns.unpersist()

        spark.stop()
      }
    }

上面的代码生成一个带有多个标题的文件。

如何修改代码以使一个文件中仅包含一个标头？

Answer 1

基本上，您正在尝试生成仅具有所有文件头的csv文件。一种简单的解决方案是使用coalesce(1)并删除您引入的repartition(10)。这样做的问题是所有数据都进入一个分区。抛出OOM错误可能非常慢，甚至更糟。但是（如果可行）您将获得一个带有一个标头的大文件。

要继续利用spark的并行性a，您可以像这样单独编写标头（假设我们有一个数据帧df）

    val output = "hdfs:///...path.../output.csv"
    val merged_output = "hdfs:///...path.../merged_output.csv"

    import spark.implicits._
    // Let's build the header
    val header = responseWithSelectedColumns
        .schema.fieldNames.reduceLeft(_+","+_)

    // Let's write the data
    responseWithSelectedColumns.write.csv(output)

    // Let's write the header without spark
    val hadoopConfig = new Configuration()
    val hdfs = FileSystem.get(hadoopConfig)
    val f = hdfs.create(new Path(output + "/header"))
    f.write(header.getBytes)
    f.close()

    // Let's merge everything into one file
    FileUtil.copyMerge(hdfs, new Path(output), hdfs, new Path(merged_output),
                                    false,hadoopConfig, null)

还请注意，spark 2.x支持开箱即用地编写csv。这是我用来代替databricks库的东西，它使事情变得更加冗长。

合并标头在文件中只有一个标头

1 个答案: