Question

从Spark读取oracle表时，出现以下错误错误PoolWatchThread：尝试获取连接时出错。重试7000ms java.security.AccessControlException：访问被拒绝的org.apache.derby.security.SystemPermission（“ engine”，“ usederbyinternals”）

val theSameDF = spark.read.option("header", "true").csv("wasbs://testdata@datamotionpcfstorage.blob.core.windows.net/convertcsv1.csv")//.repartition(30)

val ds = theSameDF.as[(String, String, String)].repartition(30)

var value:List[String] = null
def getListOfSelect() = {
  val connectionforselect = Datasource.connectionPool.getConnection
  val sqlSelect = s"Select UserId from UserTable"
  val stmtForSelect = connectionforselect.createStatement()
  val result = stmtForSelect.executeQuery(sqlSelect)

  value = new Iterator[String] {
    def hasNext = result.next()

    def next() = result.getString(1)
  }.toList
  //connectionforselect.close()
  value
}


ds.foreachPartition { rdd =>
  //val notSerializable = new NotSerializable()
  val connection = Datasource.connectionPool.getConnection
  //forupdate
  val rddList = rdd.toList

   val list = if(value==null) getListOfSelect() else value
  // @transient val myNonSerializableObjectThatIDoNotUseInATransformation
  // val returnIds = rdd1.map(_._1).toList

  //val returnIds2 = rdd.map(_._1).toList

  val update = rddList.map(_._1).filter(list.contains(_))

  if (update.length > 0) {
    update.grouped(1000).foreach{up=>
    val deletequery = "delete from usertable where userid in ('"
    val deletelistquery = s"${deletequery}${up.mkString("','")}')"
    val stmt = connection.createStatement()
       stmt.execute(deletelistquery)
    }
 }
  val datetime = Calendar.getInstance().getTime()
  val v = rddList.foldLeft(List[String]()){(z,f)=>
    z:+ s"""('${f._1}','${f._2}','${f._3}','${datetime}')"""
  }
  v.grouped(1000).foreach{ x=>
    val values = x mkString(",")
    val sql=s"""INSERT INTO UserTable (USERID,NAME,AGE,Date) VALUES ${values}"""
    val stmt = connection.createStatement()
    stmt.execute(sql)
  }
}
object Datasource  
{
val dbUrl = s"jdbc:sqlserver://testanishserver.database.windows.net:1433;database=TestAnish"
val connectionPool = new BasicDataSource()
connectionPool.setUsername("admin123!")
connectionPool.setPassword("ihsmarkit123!") 
connectionPool.setDriverClassName
("com.microsoft.sqlserver.jdbc.SQLServerDriver")
connectionPool.setUrl(dbUrl)
connectionPool.setInitialSize(40)
connectionPool.addConnectionProperty("rewriteBatchedStatements", "true");
connectionPool.addConnectionProperty("allowMultiQueries", "true");
}

如果我做错了事，请告诉我！

Spark和JDBC连接问题

0 个答案: