如何使用Spark流Scala程序将文件从LFS复制到HDFS

时间:2015-10-28 06:46:32

标签: scala apache-spark spark-streaming

我把这个写到复制文件从LFS复制到HDFS,我仍有错误,任何人帮助我;

import org.apache.spark._
import org.apache.spark.streaming.{Seconds, StreamingContext}
import StreamingContext._
import org.apache.hadoop.conf._
import org.apache.hadoop.fs._
import org.apache.hadoop.conf.Configuration;
import org.apache.commons.io.IOUtils;

object HdfsWordCount {
  def main(args: Array[String]) {
    if (args.length < 1) {
      System.err.println("Usage: HdfsWordCount args(0) args(1)")
      System.exit(1)
    }
    val sparkConf = new SparkConf().setAppName("File_Load")
       val fs = FileSystem.get(sparkConf) error code
    val ssc = new StreamingContext(fs, Seconds(2))

    val outFileStream = ssc.create(new Path(args(1)))
    val inStream = FileSystem.open(args(0)) // error code   

    IOUtils.copy(inStream, outFileStream)
    inStream.close()
    outFileStream.close()
  }
}    

0 个答案:

没有答案