Question

我有一个c＃应用程序从一个数据库获取数据，进行必要的转换，并将数据插入另一个数据库的表中。我这样做是通过将我的源数据插入队列然后处理队列以将数据插入目标表。我有两个独立的线程来读取源数据并写入目标数据。读取线程的运行速度比写入线程快，因此我的队列很快就会填满。

正如您在阅读线程中所看到的，我正在使用SqlCommand.ExecuteReader（）来读取数据。然后我循环遍历队列并为每一行执行单独的INSERT语句。我想象的是，而不是逐行插入，做某种（可能是Linq语句）。有没有人有任何想法如何更快地进行插入？

队列定义：

private static readonly BlockingCollection<HistorianData> ValueQueue = new BlockingCollection<HistorianData>(new ConcurrentQueue<HistorianData>(), 1000000);

读：

    public static void EnqueueHistorianData(SqlConnection connection, int idToAdd, DateTime minDatetime, DateTime maxDateTime, string cluster, string dbName, string dataTable, string idTable, string mainIdColumn, string foreignIdColumn, string dateColumn, string nameColumn, string valueColumn)
    {
        StringBuilder select = new StringBuilder();
        HistorianData values;

        select.Append(String.Format("SELECT {0}.{1},", dataTable, dateColumn));
        select.Append(String.Format(" '{0}.' + {1}.{2},", cluster, idTable, nameColumn));
        select.Append(String.Format(" {0}.{1}", dataTable, valueColumn));
        select.Append(String.Format(" FROM {0}.{1}", dbName, dataTable));
        select.Append(String.Format(" JOIN {0}.{1}", dbName, idTable));
        select.Append(String.Format(" ON {0}.{1} = {2}.{3}", dataTable, foreignIdColumn, idTable, mainIdColumn));
        select.Append(String.Format(" INNER JOIN Runtime.dbo.Tag"));
        select.Append(String.Format(" ON Runtime.dbo.Tag.TagName = '{0}.' + {1}.{2}", cluster, idTable, nameColumn));
        select.Append(String.Format(" WHERE {0}.{1} >= '{2}'", dataTable, dateColumn, minDatetime.ToString()));
        select.Append(String.Format(" AND {0}.{1} = {2}", dataTable, foreignIdColumn, idToAdd.ToString()));
        select.Append(String.Format(" AND {0}.{1} >= '{2}'", dataTable, dateColumn, minDatetime.ToString()));
        select.Append(String.Format(" AND {0}.{1} < '{2}'", dataTable, dateColumn, maxDateTime.ToString()));

        using (var command = new SqlCommand(select.ToString(), connection))
        {
            command.CommandTimeout = 1000;

            using (var reader = command.ExecuteReader())
            {
                if (reader.HasRows)
                {
                    while (reader.Read())
                    {
                        values = new HistorianData();

                        values.SampleDate = reader.GetDateTime(0);
                        values.TagName = reader.GetString(1);
                        values.TagValue = reader.GetDouble(2);

                        ValueQueue.Add(values);
                    }

                    values = null;
                    reader.Close();
                }
            }
        }
    }

写作：

    public static void WriteQueueValuesToHistorian(string connectionString)
    {   
        HistorianData values;

        using (var connection = new SqlConnection(connectionString))
        {
            connection.Open();

            using (SqlCommand insertCommand = connection.CreateCommand())
            {
                insertCommand.CommandType = CommandType.Text;
                insertCommand.CommandText = "INSERT INTO Runtime.dbo.History (DateTime, TagName, Value, QualityDetail) VALUES (@P1, @P2, @P3, 192)";
                insertCommand.CommandTimeout = 1000;

                var param1 = new SqlParameter("@P1", SqlDbType.DateTime);
                insertCommand.Parameters.Add(param1);

                var param2 = new SqlParameter("@P2", SqlDbType.NVarChar, 512);
                insertCommand.Parameters.Add(param2);

                var param3 = new SqlParameter("@P3", SqlDbType.Float);
                insertCommand.Parameters.Add(param3);

                insertCommand.Prepare();

                while (!ValueQueue.IsCompleted && ValueQueue.TryTake(out values, System.Threading.Timeout.Infinite))
                {
                    int retries = 0;

                    while (retries < 3)
                    {
                        insertCommand.Parameters["@P1"].Value = values.SampleDate.ToLocalTime();
                        insertCommand.Parameters["@P2"].Value = values.TagName;
                        insertCommand.Parameters["@P3"].Value = values.TagValue;

                        try
                        {
                            insertCommand.ExecuteNonQuery();
                            retries = 4;
                        }
                        catch (SqlException)
                        {
                            retries += 1;
                            sw.WriteLine("SQLException - Values: " + insertCommand.Parameters["@P1"].Value + ", " + insertCommand.Parameters["@P2"].Value + ", " + insertCommand.Parameters["@P3"].Value);
                        }
                    }
                }
            }
        }
    }

Answer 1

您可以从要检索的数据创建DataTable并使用SqlBulkCopy方法。请检查以下网址;

http://msdn.microsoft.com/en-us/library/ex21zs8x.aspx

将ConcurrentQueue中的多行插入SQL Server表

1 个答案: