使用Kafka的Spark Streaming - 消息不能打印到控制台

时间:2016-09-05 19:20:02

标签: apache-spark apache-kafka spark-streaming

我使用以下代码来使用和打印来自kafka的消息。但是,消息不打印。我可以确认问题不在于卡夫卡。你如何打印消息?

package com.paras.KafkaSpark
import java.util.HashMap
import java.util.Properties
import kafka.serializer.StringDecoder
import org.apache.kafka.clients.producer.{KafkaProducer, ProducerConfig, ProducerRecord}
import org.apache.spark.SparkConf
import org.apache.spark.streaming._
import org.apache.spark.streaming.kafka._



object StreamKafkaLearn {
  def main(args: Array[String]) {
    if (args.length < 4) {
      System.err.println("Usage: KafkaWordCount <zkQuorum> <group> <topics> <numThreads>")
     System.exit(1)
    }

    val Array(zkQuorum, group, topics, numThreads) = args

    // Create context with 2 second batch interval
    val sparkConf = new SparkConf().setAppName("DirectKafkaTest")
    val ssc = new StreamingContext(sparkConf, Seconds(2))

    // Create direct kafka stream with brokers and topics
    val topicMap = topics.split(",").map((_, numThreads.toInt)).toMap
    val lines = KafkaUtils.createStream(ssc, zkQuorum, group, topicMap).map(_._2)

    // Get the lines, split them into words, count the words and print
    //lines.print()

   lines.foreachRDD{ rdd =>
      val testString = rdd.collect()
      println(testString.mkString("\n")) // prints to the stdout of the driver
      }


      ssc.start()
      ssc.awaitTermination()
      }
}

0 个答案:

没有答案