使用com.ning.http.client.AsyncHandler时出现NoClassDefFoundError

时间:2015-10-13 18:55:07

标签: apache spark-streaming receiver ning

我正在为szelvenskiy模板生成一个接收器,用于获取http-URL作为输入。我可以使用sbt打包jar,但在运行时,应用程序会抛出java.lang.NoClassDefFoundError: com.ning.http.client.AsyncHandler异常。请在此处找到实现接收器的代码:

import org.apache.spark.streaming.receiver.Receiver
import org.apache.spark.storage.StorageLevel
import org.apache.spark.Logging
import com.ning.http.client.AsyncHttpClientConfig
import com.ning.http.client.AsyncHandler
import com.ning.http.client.AsyncHttpClient
import com.ning.http.client._
import scala.collection.mutable.ArrayBuffer
import java.io.OutputStream
import java.io.ByteArrayInputStream
import java.io.InputStreamReader
import java.io.BufferedReader
import java.io.InputStream
import java.io.PipedInputStream
import java.io.PipedOutputStream

class ParkingReceiver(url: String) extends Receiver[String](StorageLevel.MEMORY_AND_DISK_2) with Logging {

  @transient var client: AsyncHttpClient = _ 
  @transient var inputPipe: PipedInputStream = _
  @transient var outputPipe: PipedOutputStream = _  

  def onStart() {    
    val cf = new AsyncHttpClientConfig.Builder()
    cf.setRequestTimeout(Integer.MAX_VALUE)
    cf.setReadTimeout(Integer.MAX_VALUE)
    cf.setPooledConnectionIdleTimeout(Integer.MAX_VALUE)      
    client= new AsyncHttpClient(cf.build())

    inputPipe = new PipedInputStream(1024 * 1024)
    outputPipe = new PipedOutputStream(inputPipe)
    val producerThread = new Thread(new DataConsumer(inputPipe))
    producerThread.start()

    client.prepareGet(url).execute(new AsyncHandler[Unit]() {

      def onBodyPartReceived(bodyPart: HttpResponseBodyPart) = {
        bodyPart.writeTo(outputPipe)
        AsyncHandler.STATE.CONTINUE        
      }

      def onStatusReceived(status: HttpResponseStatus) = {
        AsyncHandler.STATE.CONTINUE
      }

      def onHeadersReceived(headers: HttpResponseHeaders) = {
        AsyncHandler.STATE.CONTINUE
      }

      def onCompleted = {
        println("completed")
      }


      def onThrowable(t: Throwable)={
        t.printStackTrace()
      }

    })    


  }

  def onStop() {
    if (Option(client).isDefined) client.close()
    if (Option(outputPipe).isDefined) {
     outputPipe.flush()
     outputPipe.close() 
    }
    if (Option(inputPipe).isDefined) {
     inputPipe.close() 
    }    
  }

  class DataConsumer(inputStream: InputStream) extends Runnable 
  {

      override
      def run()
      {        
        val bufferedReader = new BufferedReader( new InputStreamReader( inputStream ))
        var input=bufferedReader.readLine()
        while(input!=null){          
          store(input)
          input=bufferedReader.readLine()
        }            
      }     
  }
}

你们之前是否有过这方面的经历或者可能知道代码中缺少什么?

0 个答案:

没有答案