Question

我需要批量加载SqlDataReader中的DataTable。 SqlDataReader将返回数百万条记录，DataTable Load（）方法将耗尽可用内存。

这是我目前的代码：

[cmdletBinding( DefaultParameterSetName = 'Instance',
                    SupportsShouldProcess = $true,
                    ConfirmImpact = 'High' )]
Param (
   [string] $SrcServer     = "MySQLServer,12345",
   [string] $SrcDatabase   = "SourceDb",
   [string] $SrcTable      = "dbo.SourceTable",
   [string] $SrcQuery      = "SELECT TOP 100 HASHBYTES('SHA',stay_number) as stay_number_h, * FROM $SrcTable",
   [string] $TgtServer,
   [string] $TgtDatabase   = "TargetDb",
   [string] $TgtTable      = "tmp.TargetTable",
   [switch] $Truncate      = $true
)

Function ConnectionString([string] $ServerName, [string] $DbName)
{
   "Data Source=$ServerName;Initial Catalog=$DbName;Integrated Security=True;Connection Timeout=30"
}

########## Main body ############
Write-Host "Starting..."

If ($TgtServer.Length –eq 0) {
   $TgtServer = $SrcServer
}

If ($TgtDatabase.Length –eq 0) {
   $TgtDatabase = $SrcDatabase
}

If ($TgtTable.Length –eq 0) {
   $TgtTable = $SrcTable
}

If ($Truncate) {
   Write-Host "Truncating $TgtTable"
   $TruncateSql = "TRUNCATE TABLE " + $TgtTable
   Sqlcmd -S $TgtServer -d $TgtDatabase -Q $TruncateSql
}

$SrcConnStr = ConnectionString $SrcServer $SrcDatabase
$SrcConn  = New-Object System.Data.SqlClient.SQLConnection($SrcConnStr)
$CmdText = $SrcQuery
$SqlCommand = New-Object system.Data.SqlClient.SqlCommand($CmdText, $SrcConn)
$SrcConn.Open()
[System.Data.SqlClient.SqlDataReader] $SqlReader = $SqlCommand.ExecuteReader()

# Can we convert the SqlReader to a DataTable?
$dtSchema = $SqlReader.GetSchemaTable()
$dt = New-Object System.Data.DataTable

if ($dtSchema -ne $null)
{
    foreach ($drow in $dtSchema.Rows)
    {
        $columnName = $drow["ColumnName"]
        $column = New-Object System.Data.DataColumn($columnName, $drow["DataType"])
        $column.Unique = $drow["IsUnique"]
        $column.AllowDBNull = $drow["AllowDBNull"]
        $column.AutoIncrement = $drow["IsAutoIncrement"]
        $dt.Columns.Add($column)
    }
}

Write-Host "Now loading DataTable"
for ($i=0;$i -le 10; $i++) {
   $i
   $null = $dt.LoadDataRow($SqlReader,$true)
}
Write-Host "DataTable filled, how long and check memory consumption!"

sleep 30

$datatable.Clear()

Write-Host "Finished"

相关链接：

CSV到SQL Server：
https://gallery.technet.microsoft.com/scriptcenter/Import-Large-CSVs-into-SQL-216223d9 https://gallery.technet.microsoft.com/scriptcenter/Import-Large-CSVs-into-SQL-fa339046

SQL Server到SQL Server：
https://blogs.technet.microsoft.com/heyscriptingguy/2011/05/06/use-powershell-to-copy-a-table-between-two-sql-server-instances/ https://newsqlblog.com/2011/08/12/moving-data-between-sql-servers-with-powershell/ https://raw.githubusercontent.com/RamblingCookieMonster/PowerShell/master/Invoke-SQLBulkCopy.ps1

理想情况下，我的最终解决方案将支持SS和外部文件作为导入（实际上任何可以作为DataTable实现的内容）

Answer 1

不是Powershell，但这是我从db处理批处理的方式：

它需要一个具有与您的表匹配的字段的类，并处理您喜欢的任何浴室大小的数据USING SYSTEM.REFLECTION：

{{1}}

Answer 2

如果你想避免泛型类型并使用数据行：

public static void sql_Reader_To_DataTable_With_Buffer(Type t, SqlDataReader r)
    {

        DataTable dt = create_DataTable_From_Generic_Class(t);
        while (r.Read())
        {

            FieldInfo[] f = t.GetFields();
            object[] rowData = new object[dt.Columns.Count];
            for (int i = 0; i < f.Length; i++)
            {
                string thisType = f[i].FieldType.ToString();
                switch (thisType)
                {
                    case "System.String":

                       rowData[i] = Convert.ToString(r[f[i].Name]);
                        break;
                    case "System.Int16":
                       rowData[i] = Convert.ToInt16(r[f[i].Name]);
                        break;
                    case "System.Int32":
                       rowData[i] = Convert.ToInt32(r[f[i].Name]);
                        break;
                    case "System.Int64":
                       rowData[i] = Convert.ToInt64(r[f[i].Name]);
                        break;
                    case "System.Double":
                        // Console.WriteLine("converting " + f[i].Name + " to double");
                        rowData[i] = Convert.ToDouble(r[f[i].Name]);
                        break;
                    case "System.Boolean":
                       rowData[i] = Convert.ToInt32(r[f[i].Name]) == 1 ? true : false;
                        break;
                    case "System.DateTime":
                       rowData[i] = Convert.ToDateTime(r[f[i].Name]);
                        break;
                    default:
                        throw new Exception("Missed data type in sql select in getClassMembers class line 73");

                }
            }
            dt.Rows.Add(rowData);
            if (dt.Rows.Count == 50000)
            {
                //process table (dt);
                dt.Rows.Clear();
            }

        }
        if (dt.Rows.Count > 0)
        {
            //processTable (dt);

        }


    }
com.CommandText = "select top 10000000000000000 * from myHugeTable";
SqlDataReader = com.ExecuteDataReader();
sql_Reader_To_DataTable_With_Buffer(typeof(person),read);

使用Powershell批量加载来自SQLDataReader的DataTable

2 个答案: