Question

我使用EF6作为ETL工具的Load层。我承认有更好的工具（例如SSIS，直接SQL查询等），但是由于转换的复杂性，它需要在代码中完成，而目标DB是从EF模型构建的。插入批处理中的记录数可以超过100,000 记录。这不是非常慢（使用DBContext.AddRange()方法），但内存使用率非常高（超过1GB）

例如，我有以下数据类（在内存中创建）

    public class Foo
    {
        public long FooID { get; set; }
        public string SomeProperty { get; set; }
        public decimal AverageFlightSpeedOfUnladenSwallow { get; set; }
        public IEnumerable<Bar> Bars { get; set; }
    }

    public class Bar
    {
        public long BarID { get; set; }
        public Foo Foo { get; set; }
        public long FooID { get; set; }
        public string FavoriteColour { get; set; }
    }
dbContext.Foos.AddRange(ListOfFoos); //Pre constructed list of Foos
dbContext.Bars.AddRange(ListOfBars); //Pre constructed list of Bars (parent Foo items populated, FooID is not)
dbContext.SaveChanges();

我正在使用LINQ实体数据读取器来启用IList<Foo>到数据读取器的转换，以便我可以使用SQLBulkCopy（SqlBulkCopy and Entity Framework导入它， http://archive.msdn.microsoft.com/LinqEntityDataReader/Release/ProjectReleases.aspx?ReleaseId=389）。

要求
List<Bar>将不具有父Foo类的ID。实体框架处理这个很好，但我不知道如何在SqlBulkCopy中获得相同的功能。有没有办法完成它？

Answer 1

不，没有直接的方法可以使用SQL批量复制。

SQL Bulkcopy非常接近数据库，因此非常快。 ORM处理FK / PK关系，但缺点是速度慢。

根据您的数据模型，您可以执行类似以下问题：填充批量数据表

SQL Bulkcopy YYYYMMDD problem

Answer 2

所以，

如果您的EF用户能够更改数据库上的架构，您可以采用这种方法来解决问题：

向表格添加GUID列
通过GUID
通过批量插入插入值，包括标识GUID
选择它们并将它们映射到插入的GUID
删除GUID列

以下是一些代码。它有点脏，没有优化，但它将原始任务最小化到30MB内存和1分钟处理

public static class ForeignKeyBulkInsert
{
    private const string GUID_COLUMN_NAME = "GUID_SURROGATE_KEY";

    public static string GetTableName<T>(this ObjectContext context) where T : class
    {
        string sql = context.CreateObjectSet<T>().ToTraceString();
        Regex regex = new Regex("FROM (?<table>.*) AS");
        Match match = regex.Match(sql);

        string table = match.Groups["table"].Value;
        return table;
    }

    public static void AddRange<TEntity>(this DbContext db, IEnumerable<TEntity> range, bool importForeignKeyIDs = false)
        where TEntity : class
    {
        Dictionary<Guid, TEntity> lookup = new Dictionary<Guid, TEntity>();

        var objectContext = ((IObjectContextAdapter)db).ObjectContext;
        var os = objectContext.CreateObjectSet<TEntity>();
        bool hasAutoGeneratedKey = os.EntitySet.ElementType.KeyProperties.Any();
        Type entityType = typeof(TEntity);

        if (importForeignKeyIDs)
        {
            var foreignKeyProperties = os.EntitySet.ElementType.NavigationProperties.Where(x => x.ToEndMember.RelationshipMultiplicity == RelationshipMultiplicity.One);
            foreach (var foreignKeyProperty in foreignKeyProperties)
            {
                var foreignKeyIdProperty = foreignKeyProperty.GetDependentProperties().First();
                var parentKeyProperty = foreignKeyProperty.ToEndMember.GetEntityType().KeyMembers.First();
                PropertyInfo foreignKeyPropertyInfo = null;
                Type parentType = null;
                PropertyInfo parentKeyPropertyInfo = null;
                PropertyInfo foreignKeyIdPropertyInfo = null;
                foreach (var item in range)
                {
                    entityType.GetProperty(foreignKeyProperty.Name).GetValue(item);
                    if (foreignKeyPropertyInfo == null)
                        foreignKeyPropertyInfo = entityType.GetProperty(foreignKeyProperty.Name);
                    if (parentType == null)
                        parentType = foreignKeyPropertyInfo.GetValue(item).GetType();
                    if (parentKeyPropertyInfo == null)
                        parentKeyPropertyInfo = parentType.GetProperty(parentKeyProperty.Name);
                    if (foreignKeyIdPropertyInfo == null)
                        foreignKeyIdPropertyInfo = entityType.GetProperty(foreignKeyIdProperty.Name);

                    var foreignKey = foreignKeyPropertyInfo.GetValue(item);
                    if (foreignKey == null)
                        break;

                    var parentKey = parentKeyPropertyInfo.GetValue(foreignKey);
                    foreignKeyIdPropertyInfo.SetValue(item, parentKey);
                }
            }
        }

        string tableName = objectContext.GetTableName<TEntity>();
        var entityReader = range.AsDataReader(GUID_COLUMN_NAME, lookup);

        if (hasAutoGeneratedKey)
        {
            try
            {
                db.Database.ExecuteSqlCommand(string.Format("ALTER TABLE {0} ADD [{1}] uniqueidentifier null", tableName, GUID_COLUMN_NAME));
            }
            catch (Exception)
            {
                db.Database.ExecuteSqlCommand(string.Format("ALTER TABLE {0} DROP COLUMN [{1}]", tableName, GUID_COLUMN_NAME));
                db.Database.ExecuteSqlCommand(string.Format("ALTER TABLE {0} ADD [{1}] uniqueidentifier null", tableName, GUID_COLUMN_NAME));
            }
        }
        try
        {
            var connection = db.Database.Connection as SqlConnection;
            connection.Open();
            using (SqlBulkCopy cpy = new SqlBulkCopy(connection))
            {
                cpy.BulkCopyTimeout = 0;
                cpy.DestinationTableName = tableName;
                cpy.WriteToServer(entityReader);
                connection.Close();
            }

            if (hasAutoGeneratedKey)
            {
                db.Database.Connection.Open();
                var comm = db.Database.Connection.CreateCommand();
                comm.CommandText = string.Format("SELECT * FROM {0} WHERE [{1}] is not null", tableName, GUID_COLUMN_NAME);
                try
                {
                    using (var reader = comm.ExecuteReader())
                    {
                        while (reader.Read())
                        {
                            Guid surrogateKey = Guid.Parse(reader[GUID_COLUMN_NAME].ToString());
                            TEntity entity = lookup[surrogateKey];
                            var keyProperty = entityType.GetProperty(os.EntitySet.ElementType.KeyMembers.First().Name);
                            keyProperty.SetValue(entity, reader[keyProperty.Name]);
                        }
                    }
                }
                catch (Exception)
                {
                    throw;
                }
                finally
                {
                    //This should never occur
                    db.Database.Connection.Close();
                }
            }
        }
        catch (Exception)
        {
            throw;
        }
        finally
        {
            if (hasAutoGeneratedKey)
                db.Database.ExecuteSqlCommand(string.Format("ALTER TABLE {0} DROP COLUMN [{1}]", tableName, GUID_COLUMN_NAME));
        }

    }
}

SQLBulkCopy使用Entity Framework和Foreign Key对象进行插入

2 个答案: