我在哪里可以找到apache spark terasort source?

时间:2014-10-13 19:12:49

标签: sorting hadoop apache-spark benchmarking

import org.apache.spark.rdd._    
import org.apache.spark._
import org.apache.spark.SparkContext._

object ScalaTeraSort {

  def main(args: Array[String])
  {

    if (args.length < 2)
    {
      System.err.println(s"Usage: $ScalaTeraSort <INPUT_HDFS> <OUTPUT_HDFS>"
    )
    System.exit(1)
  }

  val sparkConf = new SparkConf().setAppName("ScalaTeraSort")
  val sc = new SparkContext(sparkConf)

  val file = sc.textFile(args(0))
  val data = file.map(line => (line.substring(0, 10), line.substring(10)))
                 .sortByKey().map{case(k, v) => k + v}
  data.saveAsTextFile(args(1))

  sc.stop()
}

我尝试从stackoverflow帖子中运行此代码

0 个答案:

没有答案