Question

您好
我有自己的用C＃编写的数据结构，如：

public class ElectricScheme
{
    public List<Element> Elements { get; set; }
    public List<Net> Nets { get; set; }
}

public class Element
{       
    public string IdName { get; set; }
    public string Func { get; set; }
    public string Name { get; set; }
    public BaseElementType Type { get; set; }
    public List<Pin> Pins { get; set; }
}

public class Pin
{
    public string IdName { get; set; }
    public string Name { get; set; }
    public BasePinType PinType { get; set; }
    public BasePinDirection PinDirection { get; set; }
}

public class Net
{
    public string IdName { get; set; }
    public string Name { get; set; }
    public List<Tuple<Element,Pin>> ConnectionPoints { get; set; }
}

如果Elements计数~19000，则每个元素包含＆gt; = 3 Pin，
网数约为20000，每个网包含＆gt; = 3对（元素，Pin）

解析txt（文件大小~17mb）文件需要5分钟。默认序列化器的Serilization / Deserialization约3分钟。从DB加载20分钟而不加载... 我使用Entity Framework之类的


  public ElectricScheme LoadScheme(int schemeId)
  {
    var eScheme = (from s in container.ElectricSchemesSet
                   where s.IdElectricScheme.Equals(schemeId)
                   select s).FirstOrDefault();
    if (eScheme == null) return null;

    container.LoadProperty(eScheme, "Elements");
    container.LoadProperty(eScheme, "Nets");
    container.LoadProperty(eScheme, "Elements.Pins");
    return eScheme;
  }

问题是Element和Pin之间的依赖关系......（对于~19000个元素~95000个引脚）

有什么想法吗？

Answer 1

任务解决方案：
数据库结构（MS SQL CE）：


-- Creating table 'ElectricSchemesSet'
CREATE TABLE [ElectricSchemesSet] (
    [IdElectricScheme] int  NOT NULL,
    [Name] nvarchar(4000)  NOT NULL,
    [SourceFileHash] nvarchar(4000)  NOT NULL,
    [CreatedDate] datetime  NOT NULL,
    [LastUpdatedDate] datetime  NOT NULL
);
-- Creating table 'Nets'
CREATE TABLE [Nets] (
    [IdNet] int  NOT NULL,
    [IdName] nvarchar(4000)  NOT NULL,
    [Name] nvarchar(4000)  NOT NULL,
    [ElectricSchemes_IdElectricScheme] int  NOT NULL
);
-- Creating table 'Elements'
CREATE TABLE [Elements] (
    [IdElement] int  NOT NULL,
    [Func] nvarchar(4000)  NULL,
    [IdName] nvarchar(4000)  NOT NULL,
    [Name] nvarchar(4000)  NOT NULL,
    [Type] nvarchar(4000)  NOT NULL,
    [ElectricSchemes_IdElectricScheme] int  NOT NULL
);
-- Creating table 'Pins'
CREATE TABLE [Pins] (
    [IdPin] int  NOT NULL,
    [IdName] nvarchar(4000)  NOT NULL,
    [Name] nvarchar(4000)  NOT NULL,
    [Direction] nvarchar(4000)  NOT NULL,
    [Type] nvarchar(4000)  NOT NULL,
    [ElementsIdElement] int  NOT NULL,
    [ElectricSchemesIdElectricScheme] int  NOT NULL
);
-- Creating table 'NetConnections'
CREATE TABLE [NetConnections] (
    [IdNetConnections] int  NOT NULL,
    [NetsIdNet] int  NOT NULL,
    [ElementsIdElement] int  NOT NULL,
    [PinsIdPin] int  NOT NULL,
    [ElectricSchemesIdElectricScheme] int  NOT NULL
);

使用Entity Framework加载（单线程2.08分钟，双核1.48分钟）：


public ElectricScheme LoadScheme(int schemeId)
        {
            var eScheme = (from s in container.ElectricSchemesSet
                            where s.IdElectricScheme.Equals(schemeId)
                            select s).FirstOrDefault();

        if (eScheme == null)
        {
            return null;
        }
        TaskFactory tf = new TaskFactory();

        Elements[] elems = null;
        IGrouping<int, Pins>[] pins = null;
        List<Element> mElements = null;

        var loadElements = tf.StartNew(() =>
                                       elems = (from e in container.Elements
                                                where e.ElectricSchemes.IdElectricScheme.Equals(schemeId)
                                                select e).ToArray());

        var loadPins = tf.StartNew(() =>
                                   pins = (from p in container.Pins
                                           where p.ElectricSchemesIdElectricScheme.Equals(schemeId)
                                           select p)
                                           .GroupBy(x => x.ElementsIdElement).ToArray());

        var buildElements = tf.ContinueWhenAll(
            new Task[] {loadElements, loadPins},
            delegate { mElements = Builder.BuildElement(elems, pins); });

        Nets[] net = null;
        IGrouping<int, NetConnections>[] nConn = null;
        List<Net> mNet = null;
        var loadNet =tf.StartNew(() =>
                    net = (from n in container.Nets
                           where n.ElectricSchemes.IdElectricScheme.Equals(schemeId)
                           select n).ToArray());

        var loadConn = tf.StartNew(() =>
                    nConn = (from c in container.NetConnections
                             where c.ElectricScheme.IdElectricScheme.Equals(schemeId)
                             select c)
                             .GroupBy(x => x.NetsIdNet).ToArray());

        var buildNet = tf.ContinueWhenAll(
            new Task[] {loadNet, loadConn},
            delegate { mNet = Builder.BuildNet(net, nConn); });

        ElectricScheme scheme = null;
        var buildScheme = tf.ContinueWhenAll(new Task[] {buildElements, buildNet},
                           delegate { scheme = Builder.BuildScheme(mNet, mElements, eScheme.IdElectricScheme); });

        buildScheme.Wait();
        return scheme;

    }

保存（）（45秒）：

        if (eScheme == null)
        {
            return null;
        }
        TaskFactory tf = new TaskFactory();

        Elements[] elems = null;
        IGrouping<int, Pins>[] pins = null;
        List<Element> mElements = null;

        var loadElements = tf.StartNew(() =>
                                       elems = (from e in container.Elements
                                                where e.ElectricSchemes.IdElectricScheme.Equals(schemeId)
                                                select e).ToArray());

        var loadPins = tf.StartNew(() =>
                                   pins = (from p in container.Pins
                                           where p.ElectricSchemesIdElectricScheme.Equals(schemeId)
                                           select p)
                                           .GroupBy(x => x.ElementsIdElement).ToArray());

        var buildElements = tf.ContinueWhenAll(
            new Task[] {loadElements, loadPins},
            delegate { mElements = Builder.BuildElement(elems, pins); });

        Nets[] net = null;
        IGrouping<int, NetConnections>[] nConn = null;
        List<Net> mNet = null;
        var loadNet =tf.StartNew(() =>
                    net = (from n in container.Nets
                           where n.ElectricSchemes.IdElectricScheme.Equals(schemeId)
                           select n).ToArray());

        var loadConn = tf.StartNew(() =>
                    nConn = (from c in container.NetConnections
                             where c.ElectricScheme.IdElectricScheme.Equals(schemeId)
                             select c)
                             .GroupBy(x => x.NetsIdNet).ToArray());

        var buildNet = tf.ContinueWhenAll(
            new Task[] {loadNet, loadConn},
            delegate { mNet = Builder.BuildNet(net, nConn); });

        ElectricScheme scheme = null;
        var buildScheme = tf.ContinueWhenAll(new Task[] {buildElements, buildNet},
                           delegate { scheme = Builder.BuildScheme(mNet, mElements, eScheme.IdElectricScheme); });

        buildScheme.Wait();
        return scheme;

    }

Answer 2

我看到你的帖子有点老了，但我会为后代添加另一种方法;-)

它可能比你的提取更快，只会帮助你提取，而不是保存......

您可以创建一个返回多个结果集的存储过程，并使用EF自动加载不相关的集合（在我的情况下，我的sp是名称searchJobs）：

 var conn = Database.DefaultConnectionFactory.CreateConnection(GetConnectionString(context));
        try
        {
            conn.Open();
            using (var cmd = conn.CreateCommand())
            {
                cmd.CommandText = "SearchJobs";
                cmd.CommandType = CommandType.StoredProcedure;
                AddParameter(cmd, "serviceProviderId", filter.ServiceProviderId);
                AddParameter(cmd, "service", filter.Service);
                AddParameter(cmd, "statuses", string.Join(",", filter.Statuses));
                AddParameter(cmd, "deadlineFrom", filter.DeadlineFrom);
                AddParameter(cmd, "deadlineTo", filter.DeadlineTo);

                using (var rdr = cmd.ExecuteReader())
                {
                    //I just need a read-only list for viewing, so I won’t worry about change tracking
                    //the objects need to be read out of the DbReader before it is closed 
                    jobs = context.ObjectContext.Translate<Job>(rdr, GetEntitySetName<Job>(context), MergeOption.AppendOnly).ToList();
                    rdr.NextResult();

                    documentRequests = context.ObjectContext.Translate<DocumentRequest>(rdr, GetEntitySetName<DocumentRequest>(context), MergeOption.AppendOnly).ToList();
                    rdr.NextResult();

                    serviceorders = context.ObjectContext.Translate<ServiceOrder>(rdr, GetEntitySetName<ServiceOrder>(context), MergeOption.AppendOnly).ToList();
                }
                fetch.Stop();
            }
        }
        finally
        {
            conn.Close();
        }

然后迭代不相关的集合并使用简单的代码将它们连接起来，在这里你可以使用多线程（就像你的例子）来加快速度。

foreach (var job in jobs)
            {
                job.ServiceOrders = serviceorders.Where(s => s.Job_Id == job.Id).ToList();               
                job.Request = documentRequests.FirstOrDefault(d => d.Id == job.OfficerRequest_Id);
            }

从DB快速加载大对象图

2 个答案: