Question

在我的代码中，用户可以上传一个excel文档，希望包含它的手机联系人列表。我作为开发人员应该读取excel文件将其转换为dataTable并将其插入数据库。问题是一些客户有大量的联系人，比如说5000和更多的联系人，当我试图将这些数据插入数据库时，它崩溃并给我一个超时异常。什么是避免这种异常的最佳方法，是什么代码可以减少insert语句的时间，以便用户不要等待太长时间？

代码

public SqlConnection connection = new SqlConnection(System.Configuration.ConfigurationManager.ConnectionStrings["ConnectionString"].ConnectionString);
public void Insert(string InsertQuery)
{
    SqlDataAdapter adp = new SqlDataAdapter();
    adp.InsertCommand = new SqlCommand(InsertQuery, connection);
    if (connection.State == System.Data.ConnectionState.Closed)
    {
        connection.Open();
    }
    adp.InsertCommand.ExecuteNonQuery();
    connection.Close();
}

protected void submit_Click(object sender, EventArgs e) 
{
    string UploadFolder = "Savedfiles/";
    if (Upload.HasFile) {
        string fileName = Upload.PostedFile.FileName;
        string path=Server.MapPath(UploadFolder+fileName);
        Upload.SaveAs(path);
        Msg.Text = "successfully uploaded";
        DataTable ValuesDt = new DataTable();
        ValuesDt = ConvertExcelFileToDataTable(path);
        Session["valuesdt"] = ValuesDt;
        Excel_grd.DataSource = ValuesDt;
        Excel_grd.DataBind();


    }
}

protected void SendToServer_Click(object sender, EventArgs e)
{
    DataTable Values = Session["valuesdt"] as DataTable ;
    if(Values.Rows.Count>0)
    {
        DataTable dv = Values.DefaultView.ToTable(true, "Mobile1", "Mobile2", "Tel", "Category");
        double Mobile1,Mobile2,Tel;string Category="";
        for (int i = 0; i < Values.Rows.Count; i++)
       {
            Mobile1 =Values.Rows[i]["Mobile1"].ToString()==""?0: double.Parse(Values.Rows[i]["Mobile1"].ToString());
            Mobile2 = Values.Rows[i]["Mobile2"].ToString() == "" ? 0 : double.Parse(Values.Rows[i]["Mobile2"].ToString());
            Tel = Values.Rows[i]["Tel"].ToString() == "" ? 0 : double.Parse(Values.Rows[i]["Tel"].ToString());

           Category = Values.Rows[i]["Category"].ToString();
           Insert("INSERT INTO client(Mobile1,Mobile2,Tel,Category) VALUES(" + Mobile1 + "," + Mobile2 + "," + Tel + ",'" + Category + "')");
           Msg.Text = "Submitied successfully to the server ";
       }



    }

}

Answer 1

您可以尝试SqlBulkCopy将数据表插入数据库表

像这样，

using (SqlBulkCopy bulkCopy = new SqlBulkCopy(sqlConnection, SqlBulkCopyOptions.KeepIdentity))
{
    bulkCopy.DestinationTableName = DestTableName;
    string[] DtColumnName = YourDataTableColumns;
    foreach (string dbcol in DbColumnName)//To map Column of Datatable to that of DataBase tabele
    {
        foreach (string dtcol in DtColumnName)
        {
            if (dbcol.ToLower() == dtcol.ToLower())
            {
                SqlBulkCopyColumnMapping mapID = new SqlBulkCopyColumnMapping(dtcol, dbcol);
                bulkCopy.ColumnMappings.Add(mapID);
                break;
            }
        }
    }
    bulkCopy.WriteToServer(YourDataTableName.CreateDataReader());
    bulkCopy.Close();
}

更多阅读http://msdn.microsoft.com/en-us/library/system.data.sqlclient.sqlbulkcopy.aspx

Answer 2

您一次插入一行，这对于此数据量非常昂贵

在这些情况下，您应该使用批量插入，因此如果您需要回滚，往返DB只会进行一次 - 所有都是同一个事务

Answer 3

您可以使用更多功能的SqlBulkCopy，也可以使用SqlAdpater的批量更新功能。而不是创建自己的insert语句，然后构建sqladapter，然后手动执行它，创建数据集，填充它，创建一个sqldataadpater，设置批处理中的插入数，然后执行一次适配器。

我可以重复这些代码，但本文将详细说明如何执行此操作：http://msdn.microsoft.com/en-us/library/kbbwt18a%28v=vs.80%29.aspx

protected void SendToServer_Click(object sender, EventArgs e)
{
    DataTable Values = Session["valuesdt"] as DataTable ;
    if(Values.Rows.Count>0)
    {
        DataTable dv = Values.DefaultView.ToTable(true, "Mobile1", "Mobile2", "Tel", "Category");
        //Fix up default values
        for (int i = 0; i < Values.Rows.Count; i++)
       {
            Values.Rows[i]["Mobile1"] =Values.Rows[i]["Mobile1"].ToString()==""?0: double.Parse(Values.Rows[i]["Mobile1"].ToString());
            Values.Rows[i]["Mobile2"] = Values.Rows[i]["Mobile2"].ToString() == "" ? 0 : double.Parse(Values.Rows[i]["Mobile2"].ToString());
            Values.Rows[i]["Tel"] = Values.Rows[i]["Tel"].ToString() == "" ? 0 : double.Parse(Values.Rows[i]["Tel"].ToString());

           Values.Rows[i]["Category"] = Values.Rows[i]["Category"].ToString();
       }
       BatchUpdate(dv,1000);


    }

}
public static void BatchUpdate(DataTable dataTable,Int32 batchSize)
{
    // Assumes GetConnectionString() returns a valid connection string.
    string connectionString = GetConnectionString();

    // Connect to the database.
    using (SqlConnection connection = new SqlConnection(connectionString))
    {

        // Create a SqlDataAdapter.
        SqlDataAdapter adapter = new SqlDataAdapter();

        // Set the INSERT command and parameter.
        adapter.InsertCommand = new SqlCommand(
            "INSERT INTO client(Mobile1,Mobile2,Tel,Category) VALUES(@Mobile1,@Mobile2,@Tel,@Category);", connection);
        adapter.InsertCommand.Parameters.Add("@Mobile1", 
          SqlDbType.Float);
        adapter.InsertCommand.Parameters.Add("@Mobile2", 
          SqlDbType.Float);
        adapter.InsertCommand.Parameters.Add("@Tel", 
          SqlDbType.Float);
        adapter.InsertCommand.Parameters.Add("@Category", 
          SqlDbType.NVarchar, 50);
        adapter.InsertCommand.UpdatedRowSource = UpdateRowSource.None;

        // Set the batch size.
        adapter.UpdateBatchSize = batchSize;

        // Execute the update.
        adapter.Update(dataTable);
    }
}

Answer 4

我知道这是一篇过时的文章，但是您无需使用现有答案中说明的批量操作（5000次插入）。由于关闭并重新打开每行插入的连接，您的性能受到很大影响。这是我过去使用过的一些代码，可保持一个连接打开并执行所需的命令以将所有数据推送到数据库：

public static class DataWorker
{
    public static Func<IEnumerable<T>, Task> GetStoredProcedureWorker<T>(Func<SqlConnection> connectionSource, string storedProcedureName, Func<T, IEnumerable<(string paramName, object paramValue)>> parameterizer)
    {

        if (connectionSource is null) throw new ArgumentNullException(nameof(connectionSource));

        SqlConnection openConnection()
        {
            var conn = connectionSource() ?? throw new ArgumentNullException(nameof(connectionSource), $"Connection from {nameof(connectionSource)} cannot be null");
            var connState = conn.State;

            if (connState != ConnectionState.Open)
            {
                conn.Open();
            }

            return conn;
        }

        async Task DoStoredProcedureWork(IEnumerable<T> workData)
        {
            using (var connection = openConnection())
            using (var command = connection.CreateCommand())
            {
                command.CommandType = CommandType.StoredProcedure;
                command.CommandText = storedProcedureName;

                command.Prepare();

                foreach (var thing in workData)
                {
                    command.Parameters.Clear();

                    foreach (var (paramName, paramValue) in parameterizer(thing))
                    {
                        command.Parameters.AddWithValue(paramName, paramValue ?? DBNull.Value);
                    }

                    await command.ExecuteNonQueryAsync().ConfigureAwait(false);
                }
            }
        }

        return DoStoredProcedureWork;
    }
}

这实际上来自我正在收集电子邮件以获取限制列表的项目，所以类似的示例显示了parameterizer参数的外观以及如何使用上面的代码：

        IEnumerable<(string,object)> RestrictionToParameter(EmailRestriction emailRestriction)
        {
            yield return ("@emailAddress", emailRestriction.Email);
            yield return ("@reason", emailRestriction.Reason);
            yield return ("@restrictionType", emailRestriction.RestrictionType);
            yield return ("@dateTime", emailRestriction.Date);
        }

        var worker = DataWorker.GetStoredProcedureWorker<EmailRestriction>(ConnectionFactory, @"[emaildata].[AddRestrictedEmail]", RestrictionToParameter);


        await worker(emailRestrictions).ConfigureAwait(false);

插入sql数据库时处理大量数据

4 个答案: