Question

我正在尝试读取块中的CSV文件，以避免导入大型CSV文件时出现内存不足问题。

我正在使用IEnumerable Collection方法从块中的csv读取数据并将其返回到另一个函数。

但是我将System.Collection.Generic.IEnumerable提供给System.Data.Datatable。不可能进行隐式转换。

以下是代码：

    public void ImportData(string targetPathwithName, System.Data.DataTable dt1, string targetName)
    {
        var con = ConfigurationManager.ConnectionStrings["con1"].ConnectionString.ToString();
        var connection = new SqlConnection(con);



        dt1 =  GetFileData(targetPathwithName, dt1);

        connection.Open();

        var bulkCopy = new SqlBulkCopy(connection);
        bulkCopy.DestinationTableName = "[" + targetName + "]";
        bulkCopy.WriteToServer(dt1);
        bulkCopy.Close();
        connection.Close();

    }

public static IEnumerable<System.Data.DataTable> GetFileData(string sourceFileFullName,  System.Data.DataTable dt1)
    {            
        var con = ConfigurationManager.ConnectionStrings["con1"].ConnectionString.ToString();
        var connection = new SqlConnection(con);

        int chunkRowCount = 0;
        int RowCount = 0;

        using (var sr = new StreamReader(sourceFileFullName))
        {
            if (RowCount != 0)
            {
                string Row = null;
                //Read and display lines from the file until the end of the file is reached.                
                while ((Row = sr.ReadLine()) != null)
                {
                    chunkRowCount++;
                    //var chunkDataTable = ; //Code for filling datatable or whatever  
                    dt1.Rows.Add();

                    if (chunkRowCount == 10000)
                    {
                        chunkRowCount = 0;
                        yield return dt1;
                        dt1 = null;
                    }

                    else
                    {
                        int i = 0;

                        foreach (string Cell in Row.Split(','))
                        {
                            if (String.IsNullOrEmpty(Cell))
                            {
                                dt1.Rows[dt1.Rows.Count - 1][i] = DBNull.Value;
                                i = i + 1;
                            }
                            else if (Cell == "00.00.0000")
                            {
                                dt1.Rows[dt1.Rows.Count - 1][i] = DBNull.Value;
                                i = i + 1;
                            }
                            else
                            {
                                dt1.Rows[dt1.Rows.Count - 1][i] = Cell;
                                i = i + 1;
                            }
                        }
                    }

                }
            }
            RowCount = RowCount + 1;
        }
            //return last set of data which less then chunk size
            if (null != dt1)                           
                yield return dt1;            
        }

如何在另一个函数中调用System.Data.Datatable类型的IEnumerable Collection？

Answer 1

实际上，您只返回一个数据表，而不是IEnumerable。

你能做的是 1）在函数GetFileData中创建IEnumerable的单个实例，如IEnumerable DTables = new IEnumerable（）; 2）每次为每个数据块创建一个新的datatable实例，并将该数据表添加到上面定义的IEnumerableCollection中，然后返回Collection。 3）用于函数ImportData中的每个循环来读取集合中的所有数据表。

Answer 2

您必须使用foreach循环对代码进行一些修改，如下所示：

public static void ImportData(string targetPathwithName, System.Data.DataTable dt1, string targetName)
{
    var con = ConfigurationManager.ConnectionStrings["con1"].ConnectionString.ToString();
    var connection = new SqlConnection(con);

    connection.Open();
    var bulkCopy = new SqlBulkCopy(connection);
    foreach (System.Data.DataTable dt in GetFileData(targetPathwithName, dt1))
    {
        bulkCopy.DestinationTableName = "[" + targetName + "]";
        bulkCopy.WriteToServer(dt);
    }
    bulkCopy.Close();
    connection.Close();
}

public static IEnumerable<System.Data.DataTable> GetFileData(string sourceFileFullName, System.Data.DataTable dt1)
{
    var con = ConfigurationManager.ConnectionStrings["con1"].ConnectionString.ToString();
    var connection = new SqlConnection(con);

    int chunkRowCount = 0;
    //int RowCount = 0;
    string Row;

    using (var sr = new StreamReader(sourceFileFullName))
    {
        //if (RowCount != 0) { //this is not meaningful here

        //Read and display lines from the file until the end of the file is reached.                
        while ((Row = sr.ReadLine()) != null)
        {
            chunkRowCount++;
            //var chunkDataTable = ; //Code for filling datatable or whatever  
            dt1.Rows.Add();

            int i = 0;

            foreach (string Cell in Row.Split(','))
            {
                if (String.IsNullOrEmpty(Cell))
                {
                    dt1.Rows[dt1.Rows.Count - 1][i] = DBNull.Value;
                    i = i + 1;
                }
                else if (Cell == "00.00.0000")
                {
                    dt1.Rows[dt1.Rows.Count - 1][i] = DBNull.Value;
                    i = i + 1;
                }
                else
                {
                    dt1.Rows[dt1.Rows.Count - 1][i] = Cell;
                    i = i + 1;
                }
            }

            if (chunkRowCount == 10000)
            {
                chunkRowCount = 0;
                yield return dt1;
                dt1.Clear(); // = null;
            }

        } //end while

        //}
        //RowCount = RowCount + 1;
    }

    //return last set of data which less then chunk size
    if (dt1.Rows.Count > 0)
        yield return dt1;
}

我测试了这个并且效果很好

如何在C＃中调用IEnumerable数据表集合

2 个答案: