Question

我正在将一个包含大约900000行的数据表插入MySQL数据库，现在13个小时后，我的程序已插入185000行。多数民众赞成我认为^^

这是我的代码如何将我的DataTable插入MYSQL DB

//data is typeof DataTable and filled with my data.

using (System.Data.SqlClient.SqlConnection srcConnection = new SqlConnection(Settings.Default.SourceConnection))
using (MySql.Data.MySqlClient.MySqlConnection dstConnection = new MySqlConnection(Settings.Default.TargetConnection))
{
    if (dstConnection.State != ConnectionState.Open)
        dstConnection.Open();
    if (srcConnection.State != ConnectionState.Open)
        srcConnection.Open();

    var builder = SqlBuilder(tableName, data)

    foreach (DataRow toCopy in data)
    {
        using (var cmdUpdate = builder.GenerateUpdateCommand(dstConnection, toCopy))
        {
            if (cmdUpdate.ExecuteNonQuery() == 0)
            {
                using (var cmdInsert = builder.GenerateInsertCommand(dstConnection, toCopy))
                {
                    cmdInsert.ExecuteNonQuery;
                }
            }
        }
    }
}

builder是我自己的类：

public class SqlBuilder
{
    private readonly string _tableName;
    private readonly DataColumnCollection _columns;
    private readonly List<string> _columnStrings = new List<string>();
    private DataColumn _primaryDataColumn;
    private readonly Logger _logger = LogManager.GetLogger(typeof(SqlBuilder).FullName);

    public SqlBuilder(string tableName, DataTable table)
    {
        if (tableName == null) throw new ArgumentNullException("tableName");
        if (table == null) throw new ArgumentNullException("table");
        if (table.PrimaryKey.Count() != 1) throw new ArgumentException("Only one primary key supported", "table");

        _tableName = tableName;
        _columns = table.Columns;
        _primaryDataColumn = table.PrimaryKey[0];

        foreach (DataColumn column in _columns)
            _columnStrings.Add(column.ColumnName);
    }

    public SqlBuilder()
    {
    }

    public string TableName
    {
        get { return _tableName; }
    }

    public String[] ColumnStrings
    {
        get { return _columnStrings.ToArray(); }
    }


    public IDbCommand GenerateCountCommand(IDbConnection con)
    {
        IDbCommand result = con.CreateCommand();
        result.CommandText = String.Format("SELECT COUNT(*) FROM {0}", TableName);
        return result;
    }

    public IDbCommand GenerateDeleteTableCommand(IDbConnection con)
    {
        IDbCommand result = con.CreateCommand();
        result.CommandText = String.Format("DELETE FROM {0}", TableName);
        return result;
    }

    public IDbCommand GenerateDeleteChangeLogTableCommand(IDbConnection con)
    {
        IDbCommand result = con.CreateCommand();
        result.CommandText = String.Format("DELETE FROM ChangeLog");
        return result;
    }

    public String GenerateInsertCommandText()
    {
        String result;

        result = String.Format(
            "INSERT INTO {0} ({1}) VALUES ({2})",
            TableName,
            String.Join(",", _columnStrings.ToArray()),
            String.Join(",", _columnStrings.Select(s => "@" + s).ToArray()));

        return result;
    }

    public string GenerateUpdateCommandText()
    {
        String result;

        result = String.Format(
            "UPDATE {0} SET {1} WHERE {2}",
            TableName,
            String.Join(", ", _columnStrings.Where((s =>!_primaryDataColumn.ColumnName.Equals(s))).Select(s => s+"=@" + s).ToArray()),
            _primaryDataColumn.ColumnName+"=@"+_primaryDataColumn.ColumnName
            );

        return result;
    }

    public string GenerateDeleteCommandText(int id)
    {
        string result = String.Format("DELETE FROM {0} WHERE {1} = {2}", _tableName , _columnStrings[0], id);
        return result;
    }

    public IDbCommand GenerateInsertCommand(IDbConnection con, DataRow row)
    {
        IDbCommand result = con.CreateCommand();
        result.CommandText = GenerateInsertCommandText();

        result = FillParameters(row, result);

        if (_logger.IsTraceEnabled) _logger.Trace(result.CommandText);

        return result;
    }

    public IDbCommand GenerateUpdateCommand(IDbConnection connection, DataRow dataRow)
    {
        IDbCommand result = connection.CreateCommand();
        result.CommandText = GenerateUpdateCommandText();

        result = FillParameters(dataRow, result);

        if (_logger.IsTraceEnabled) _logger.Trace(result.CommandText);

        return result;
    }

    public IDbCommand GenerateDeleteCommand(IDbConnection con, int id)
    {
        IDbCommand result = con.CreateCommand();

        result.CommandText = GenerateDeleteCommandText(id);

        return result;
    }

    private IDbCommand FillParameters(DataRow row, IDbCommand result)
    {
        foreach (var curColumn in _columnStrings)
        {
            IDbDataParameter parameter = result.CreateParameter();
            parameter.ParameterName = curColumn;
            parameter.Value = row[curColumn];
            result.Parameters.Add(parameter);
        }

        return result;
    }
}

任何人都知道如何加快这一点???

Answer 1

在一个语句中插入多行，如下所示：

INSERT INTO table (...) VALUES 
(values for first record),
(values for second record),
...

您可能会在这样的一个语句中插入几十行，具体取决于记录大小。

Answer 2

我在参数的类似构建过程中发现的是设置参数的“SourceColumn”...在“FillParameters”方法中尝试添加

parameter.SourceColumn = curColumn;

然后，在INSERT例程中，而不是

foreach( DataRow )
   Build The Insert Command
      Execute Insert

更改为

Build The Insert Command
  ForEach( DataRow )
      Execute Insert

这样，您不必一遍又一遍地构建命令。该参数将转到正在处理的数据行的源列，并将其拉入。

当我编写我的版本时，同样适用于更新和删除。我预先构建了命令和参数，并在我正在使用的DataTable期间保留它们。

因此，您可以在foreach（DataRow）之前实际构建insert和update命令，然后只使用它们。一遍又一遍。显然，测试只上传几条记录，以确认它在你假设完成之前就已经可以使用了。

MYSQL插入速度很慢

2 个答案: