我正在尝试使用groovy脚本将数据从SQL SERVER
提取到HDFS
。
流定义部署流,部署后不会获取脚本。
流定义:
stream create --name WhatIFScenarioRun --definition "trigger --initialDelay=10 --fixedDelay=30 | script --script=file:/opt/spring-xd-1.3.1.RELEASE/xd/modules/processor/scripts/WhatIF.groovy | log" --deploy
WhatIF.groovy
import groovy.sql.Sql
import org.springframework.data.hadoop.fs.FsShell
import org.apache.hadoop.conf.Configuration
def dbDriver = "com.microsoft.sqlserver.jdbc.SQLServerDriver"
def dbUrl = "jdbc:sqlserver://machine_name:1433/db_name"
def dbUser = "username"
def dbPassword = "password"
def stagingFilePath = "Stagging_Folder_Path"
def hdfsPath = "HDFS_Path"
def hdfsFs="hdfs_URl"
Sql sql = null
def today = new Date().format( 'yyyy-MM-dd' )
def stagedData = false
try {
println "connecting to source database........"
sql = Sql.newInstance(dbUrl, dbUser, dbPassword, dbDriver)
println "running sql to check data availability ...."
def row1 = sql.firstRow(" select flag from cpsdev.polling_table")
def flag = row1.flag
println "flag = " + flag
def flag = 1
if( flag == 1) {
println "new Data avaialble and moving data ...."
def fileName = new Date().format( 'yyyyMMddHHmmSS' ) + ".txt"
println fileName
def stagingFile = new File(stagingFilePath, fileName)
println "File Created ..."
def writer = stagingFile.newWriter()
println "Writer Opened ...."
def sqlStr = "EXEC Scenario.dbo.Get_ScenarioResults"
println "Runng sql and extracting data ....."
sql.eachRow(sqlStr) { row1 ->
def str = row1.InstanceID+"|"+row1.ScenarioID+"|"+row1.UasTagName+"|"+row2.TimeStamp+"|"+row2.ValInCusUom+"|"+row2.ValInUopUom+"|"+row2.Asset+"|"+row1.Attribute+"|"+row1.BaseValueCusUom+"|"+BaseValueUopUom
println str
writer.writeLine(str)
}
println "closing writer ..."
writer.close()
println "Done creating local file "+ fileName
println "opening hadoop connection ...."
def configuration = new Configuration()
configuration.set("fs.defaultFS", hdfsFs)
def fsShell = new FsShell(configuration)
def dt = new Date().format( 'yyyy-MM-dd' )
targetfolder = hdfsPath + dt
println "checking the hadoop folder and create if not exist ....."
if (!fsShell.test(targetfolder)) {
fsShell.mkdir(targetfolder)
}
println "copy file to hadoop ...."
def sourceFile = stagingFilePath+fileName
fsShell.moveFromLocal(sourceFile, targetfolder)
println "moved files to hdfs...."
} else {
println "No new data avaialble. Skipping execution...."
}
} catch (Exception e) {
e.printStackTrace()
}