Question

我试图通过直接从结构化流传递值来调用REST API。我试图以下面的方式实现它。我不期望从这个REST API调用中得到任何响应。但是我需要确保每次对终点的调用都是成功的。我怎样才能确保这一点？目前，所有拨打电话都没有成功，无法跟踪。有什么建议。

import org.apache.spark.sql.ForeachWriter
val writer = new ForeachWriter[String] {
  override def open(partitionId: Long, version: Long) = true
  override def process(value: String) = {
     import sys.process._
     val command = """curl -d '{"Id":"""+ value + """}'  -H "Content-Type: application/json"  -X POST http://xx.xxx.xxx.xxx:xxxx/xxx/xxxxx/xxxx"""
     Seq("/bin/bash","-c",command).!!
  }
  override def close(errorOrNull: Throwable) = {}
}

val results = output.map(r => r.getString(0))

results.writeStream
  .queryName("rest-api-processor")
  .foreach(writer)
  .start
  .awaitTermination

Answer 1

如上所述，在没有.!!的Spark中执行Bash命令。或者，我们也可以打印声明。

    import org.apache.spark.sql.ForeachWriter
val writer = new ForeachWriter[Row] {
  override def open(partitionId: Long, version: Long) = true
  override def process(value: Row) = {
     import sys.process._
     val command = """curl -d '{"Id":"""+ value + """}'  -H "Content-Type: application/json"  -X POST http://xx.xxx.xxx.xxx:xxxx/xxx/xxxxx/xxxx"""
     Seq("/bin/bash","-c",command)
  }
  override def close(errorOrNull: Throwable) = {}
}

Azure Databricks使用ForeachWriter进行结构化流式处理

1 个答案: