我试图从Spring XD shell运行Groovy脚本。但脚本不会触发

时间:2017-07-19 13:09:02

标签: hadoop groovy hdfs spring-xd

我正在尝试使用groovy脚本将数据从SQL SERVER提取到HDFS。 流定义部署流,部署后不会获取脚本。

流定义:

stream create --name WhatIFScenarioRun --definition "trigger --initialDelay=10 --fixedDelay=30 | script --script=file:/opt/spring-xd-1.3.1.RELEASE/xd/modules/processor/scripts/WhatIF.groovy | log" --deploy
  

WhatIF.groovy

import groovy.sql.Sql
import org.springframework.data.hadoop.fs.FsShell
import org.apache.hadoop.conf.Configuration


def dbDriver   = "com.microsoft.sqlserver.jdbc.SQLServerDriver"
    def dbUrl      = "jdbc:sqlserver://machine_name:1433/db_name"
    def dbUser     = "username"
    def dbPassword = "password"
    def stagingFilePath = "Stagging_Folder_Path"
    def hdfsPath = "HDFS_Path"
    def hdfsFs="hdfs_URl"
    Sql sql = null
    def today = new Date().format( 'yyyy-MM-dd' )
    def stagedData = false
try {
    println "connecting to source database........"
    sql = Sql.newInstance(dbUrl, dbUser, dbPassword, dbDriver)
    println "running sql to check data availability ...."
    def row1 = sql.firstRow(" select flag from cpsdev.polling_table")
    def flag = row1.flag
    println "flag = " + flag
    def flag = 1
    if( flag == 1) {
        println "new Data avaialble and moving data ...."   
        def fileName = new Date().format( 'yyyyMMddHHmmSS' ) + ".txt"
        println fileName
        def stagingFile = new File(stagingFilePath, fileName)   
        println "File Created ..."
        def writer = stagingFile.newWriter()
        println "Writer Opened ...."
        def sqlStr = "EXEC Scenario.dbo.Get_ScenarioResults"
        println "Runng sql and extracting data ....."                
        sql.eachRow(sqlStr) { row1 ->
            def str = row1.InstanceID+"|"+row1.ScenarioID+"|"+row1.UasTagName+"|"+row2.TimeStamp+"|"+row2.ValInCusUom+"|"+row2.ValInUopUom+"|"+row2.Asset+"|"+row1.Attribute+"|"+row1.BaseValueCusUom+"|"+BaseValueUopUom
            println str
            writer.writeLine(str)
        }
        println "closing writer ..."
        writer.close()
        println "Done creating local file "+ fileName
        println "opening hadoop connection ...."
        def configuration = new Configuration()
        configuration.set("fs.defaultFS", hdfsFs)
        def fsShell = new FsShell(configuration)
        def dt = new Date().format( 'yyyy-MM-dd' )
        targetfolder = hdfsPath + dt
        println "checking the hadoop folder and create if not exist ....."
        if (!fsShell.test(targetfolder)) {      
            fsShell.mkdir(targetfolder)
        }
        println "copy file to hadoop  ...."
        def sourceFile = stagingFilePath+fileName
        fsShell.moveFromLocal(sourceFile, targetfolder)
        println "moved files to hdfs...."

    } else {
        println "No new data avaialble. Skipping execution...."
    }
} catch (Exception e) { 
    e.printStackTrace()
}

0 个答案:

没有答案