如何使用Powershell从SQL表批量插入Postgres

时间:2020-11-04 20:35:56

标签: sql-server postgresql powershell database-migration bulkinsert

我必须使用Powershell在Postgres中播种表,而没有使用文件的机会。信息的来源是SQL Server查询。

我找到了一种创建“自动”插入语句和脚本的方法,但是我对这种解决方案不满意,我正在寻求一种更好的方法,例如使用查询的结果集作为输入大量插入Postgres表中,但是我不知道是否可行以及如何使用Powershell进行操作,因为我是该工具的新手

这是我当前的PS1脚本中的代码:

Import-Module SQLServer

$SQLSERVER = "LAPTOP-201MM1BF"
$MSSQLDB = "origin"
$Username = "dbuser"
$Password = "password"

$Query="BEGIN
SET NOCOUNT ON;
SET TRANSACTION ISOLATION LEVEL READ COMMITTED;

Declare @target_time_zone varchar(50);
Set @target_time_zone = 'US Eastern Standard Time'; --'Hawaiian Standard Time', 'Alaskan Standard Time', 'Pacific Standard Time', 'US Mountain Standard Time', 'Central Standard Time'

BEGIN TRANSACTION
    SELECT
        ID as source_id,                                                        --Authrize Table's Id
        (SELECT TOP 1 CONCAT_WS( '\', (SELECT @@SERVERNAME), (SELECT DB_NAME()))) as source_facility_id, --Facility's ID
        FINITIALS as source_worker_id,                                          --Worker's id in MRS DB Initials or Employee Id
        null as okta_id,                                                        --Not defined field
        FTITLE as worker_role,                                                  --Worker's title in MRS DB with default value
        FNAME as full_name,                                                     --Worker's Full Name registered in MRS DB with default value
        1 as source_worker_status_id,                                           --All migrated workers will be considered active because in MRS DB doesn't exist this flag
        iif(isnull(FLOADING,0)=0,'false','true') as casepick,                   --Flag to know if the worker is trained to do loading work or not
        iif(isnull(FDROP,0)=0,'false','true') as drops,                         --The training flag is the same than case pick work
        iif(isnull(FFULL,0)=0,'false','true') as fulls,                         --Flag to know if the worker is trained to do full work or not
        iif(isnull(FLOADING,0)=0,'false','true') as loading,                    --Flag to know if the worker is trained to do loading work or not
        iif(isnull(FPUTAWAY_A,0)=0,'false','true') as putaway_a,                --Flag to know if the worker is trained to do putaway air work or not
        iif(isnull(FPUTAWAY_F,0)=0,'false','true') as putaway_f,                --Flag to know if the worker is trained to do putaway floor work or not
        iif(isnull(FRECEIVE,0)=0,'false','true') as receiving,                  --Flag to know if the worker is trained to do receiving work or not
        iif(isnull(FRELO,0)=0,'false','true') as relos,                         --Flag to know if the worker is trained to do relocation work or not
        iif(isnull(FFULL,0)=0,'false','true') as stage_move,                    --The training flag is the same than full work
        iif(isnull(FTUNNELING,0)=0,'false','true') as tunneling,                --Flag to know if the worker is trained to do tunneling work or not
        iif(isnull(FUNLOADING,0)=0,'false','true') as unloading,                --Flag to know if the worker is trained to do unloading work or not
        iif(isnull(ISMOLETRAINED,0)=0,'false','true') as ismoletrained,         --Flag to know if the worker is trained to do operate mole
        [ZONE] as location_id,                                                  --Worker's assigned zone wirh default value
        CONVERT(DATETIME,SQLDATETIME) at time zone @target_time_zone as created,--Creation date
        dateadd(hh, convert(int,left(right(GETDATE() at time zone @target_time_zone,6),3)), GETDATE()) at time zone @target_time_zone as updated
                                                                                --Current datetime as last seen datetime
    FROM
        " + $MSSQLDB + ".[dbo].[AUTHRIZE]   
COMMIT TRANSACTION; 
END"
            
$connectionString = 'Data Source={0};database={1};User ID={2};Password={3}' -f $SQLSERVER,$MSSQLDB,$Username,$Password

$sqlConnection = New-Object System.Data.SqlClient.SqlConnection $connectionString
$sqlConnection.Open()

$ResultSet = Invoke-Sqlcmd -Query $Query -ServerInstance $SQLSERVER

$multiInsert="";
ForEach($record in $ResultSet) {
  $source_id = $record.Item(0)
  $source_facility_id = $record.Item(1) -replace '(^\s+|\s+$)','' -replace '\s+',' ' -replace '''',''''''
  $source_worker_id = $record.Item(2) -replace '(^\s+|\s+$)','' -replace '\s+',' ' -replace '''',''''''
  $worker_role = $record.Item(4) -replace '(^\s+|\s+$)','' -replace '\s+',' ' -replace '''',''''''
  $full_name = $record.Item(5) -replace '(^\s+|\s+$)','' -replace '\s+',' ' -replace '''',''''''
  $casepick = $record.Item(7)
  $drops = $record.Item(8)
  $fulls = $record.Item(9)
  $loading = $record.Item(10)
  $putaway_a = $record.Item(11)
  $putaway_f = $record.Item(12)
  $receiving = $record.Item(13)
  $relos = $record.Item(14)
  $stage_move = $record.Item(15)
  $tunneling = $record.Item(16)
  $unloading = $record.Item(17)
  $ismoletrained = $record.Item(18)
  $created = $record.Item(20)
  $updated = $record.Item(21)

  $OFS = "`r`n"
  $multiInsert= $multiInsert + "INSERT INTO core.worker(source_id, source_worker_id, worker_role, full_name, 
                                casepick, drops, fulls, loading, putaway_a, putaway_f, receiving, relos, stage_move, tunneling, unloading, 
                                moletrained, source_facility_id, created, updated)
                        VALUES ($source_id,'$source_worker_id', '$worker_role', '$full_name', $casepick, 
                                $drops, $fulls, $loading, $putaway_a, $putaway_f, $receiving, $relos, $stage_move, $tunneling, $unloading, 
                                $ismoletrained, '$source_facility_id', '$created', '$updated');" + $OFS;            
}

$sqlConnection.Close()

$PostgreSQLServer = "localhost"
$PPort  = "5432"
$PosgreSQLDB = "destination"
$MyUid = "postgres"
$MyPass = "password"
$DBConnectionString = "Driver={PostgreSQL Unicode};Server=$PostgreSQLServer;Port=$PPort;Database=$PosgreSQLDB;Uid=$MyUid;Pwd=$MyPass;"
$DBConn = New-Object System.Data.Odbc.OdbcConnection
$DBConn.ConnectionString = $DBConnectionString
$DBConn.Open()
$DBCmd = $DBConn.CreateCommand()
$DBCmd.CommandText = $multiInsert
$DBCmd.ExecuteReader()
$DBConn.Close()

1 个答案:

答案 0 :(得分:0)

这是使用parameters的代码的简化版本。它将对每个结果行执行一次插入。同样,使用参数可以进行查询计划缓存,甚至可以使其更快一点。我将其更改为使用管道,因此结果将在到达时进行处理。

它可能还不完美,我当然不能真正休息。但我希望它能给您一个想法,并且您能够根据需要对其进行调整。

# ...open both your sqlserver and postgres connections here...

$DBCmd.CommandText = @"
INSERT INTO core.worker (source_id, source_worker_id, worker_role, full_name, 
        casepick, drops, fulls, loading, putaway_a, putaway_f, receiving, relos, stage_move, tunneling, unloading, 
        moletrained, source_facility_id, created, updated)
VALUES (@source_id, @source_worker_id, @worker_role, @full_name, @casepick, 
        @drops, @fulls, @loading, @putaway_a, @putaway_f, @receiving, @relos, @stage_move, @tunneling, @unloading, 
        @ismoletrained, @source_facility_id, @created, @updated)
"@

Invoke-Sqlcmd -Query $Query -ServerInstance $SQLSERVER | foreach {
    $DBCmd.Parameters.Clear()
    $DBCmd.Parameters.Add("source_id", $_.Item(0))
    $DBCmd.Parameters.Add("source_facility_id", $_.Item(1).ToString().Trim())
    $DBCmd.Parameters.Add("source_worker_id", $_.Item(2).ToString().Trim())
    $DBCmd.Parameters.Add("worker_role", $_.Item(4).ToString().Trim())
    $DBCmd.Parameters.Add("full_name", $_.Item(5).ToString().Trim)
    $DBCmd.Parameters.Add("casepick", $_.Item(7))
    $DBCmd.Parameters.Add("drops", $_.Item(8))
    $DBCmd.Parameters.Add("fulls", $_.Item(9))
    $DBCmd.Parameters.Add("loading", $_.Item(10))
    $DBCmd.Parameters.Add("putaway_a", $_.Item(11))
    $DBCmd.Parameters.Add("putaway_f", $_.Item(12))
    $DBCmd.Parameters.Add("receiving", $_.Item(13))
    $DBCmd.Parameters.Add("relos", $_.Item(14))
    $DBCmd.Parameters.Add("stage_move", $_.Item(15))
    $DBCmd.Parameters.Add("tunneling", $_.Item(16))
    $DBCmd.Parameters.Add("unloading", $_.Item(17))
    $DBCmd.Parameters.Add("ismoletrained", $_.Item(18))
    $DBCmd.Parameters.Add("created", $_.Item(20))
    $DBCmd.Parameters.Add("updated", $_.Item(21))
    $DBCmd.ExecuteNonQuery()
}

# ... close your sqlserver and postgress connections here ...