使用查找表和联结表选择许多列/字段。内存不足异常

时间:2014-06-18 20:17:45

标签: c# linq entity-framework out-of-memory entity-framework-6

我有一个高度规范化的房地产数据库,用于存储列表数据。主要数据分布在3个表(Listings,ListingDataCommons和ListingDataOthers)之间,因为有大量的字段,因此有几个连接表可用于功能,类型等。

应用程序的用户使用GUI来定义条件/字段,系统将使用WhereSelectOrderBy语句生成(使用动态linq)。问题是当Select语句使用许多联结和/或查找表时,我得到Out Of Memory Exception

下面是一个强类型(为了便于阅读)将抛出错误的查询示例。即使它只通过MLS号返回单个记录,它也会抛出OOM异常。

var ListingResult = context.Listings
    .Where(a => a.MLSNumber == "123456" || a.MLSNumber == "654321")
    .Select(a => new
    {
        //--- select some data from the Listings table
        a.MLSNumber,
        a.DateLastUpdated,
        a.DateLastImageUpdated,
        a.Address,
        a.ZipCode,
        a.DaysOnMarket,
        a.DisplayOnInternet,
        a.DisplayReviews,
        a.AuctionYN,
        a.ListPrice,
        a.LeasePrice,
        a.SystemID,
        a.DateLastPriceChange,
        a.DateLastStatusChange,
        a.DisplayAddressOnlineYN,
        a.ListingID,
        Status = a.Status.Name,
        PropertyType = a.PropertyType.Name,
        PropertyStyle = a.PropertyStyle.Name,
        Country = a.Country.Name,
        State = a.State.Name,
        County = a.County.Name,
        City = a.City.Name,
        SaleType = a.SaleType.Name,

        //--- select some data from the ListingDataCommons table
        BathsFull = a.ListingDataCommon.BathsFull,
        BathsHalf = a.ListingDataCommon.BathsHalf,
        Beds = a.ListingDataCommon.Beds,
        FireplaceYN = a.ListingDataCommon.FireplaceYN,
        GarageSpaces = a.ListingDataCommon.GarageSpaces,
        HOAYN = a.ListingDataCommon.HOAYN,
        LotAcres = a.ListingDataCommon.LotAcres,
        NewConstructionYN = a.ListingDataCommon.NewConstructionYN,
        PetsAllowedYN = a.ListingDataCommon.PetsAllowedYN,
        PetsMaxWeight = a.ListingDataCommon.PetsMaxWeight,
        PetsMaxNumber = a.ListingDataCommon.PetsMaxNumber,
        RemarksPublic = a.ListingDataCommon.RemarksPublic,
        SqftHeated = a.ListingDataCommon.SqftHeated,
        SubdivisionName = a.ListingDataCommon.SubdivisionName,
        Taxes = a.ListingDataCommon.Taxes,
        TaxYear = a.ListingDataCommon.TaxYear,
        YearBuilt = a.ListingDataCommon.YearBuilt,
        AirConditioning = a.ListingDataCommon.AirConditioning.Name,
        ConstructionStatus = a.ListingDataCommon.ConstructionStatus.Name,
        HousingForOlder = a.ListingDataCommon.HousingForOlder.Name,

        //--- select some data from the ListingDataOthers table
        CDDFee = a.ListingDataOther.CDDFee,
        CDDFeeYN = a.ListingDataOther.CDDFeeYN,
        CondoFee = a.ListingDataOther.CondoFee,
        HOAFee = a.ListingDataOther.HOAFee,
        HomesteadYN = a.ListingDataOther.HomesteadYN,
        LotDimensions = a.ListingDataOther.LotDimensions,
        LotSqft = a.ListingDataOther.LotSqft,
        NumberBays = a.ListingDataOther.NumberBays,
        NumberBuildings = a.ListingDataOther.NumberBuildings,
        NumberFloors = a.ListingDataOther.NumberFloors,
        NumberHotelRooms = a.ListingDataOther.NumberHotelRooms,
        NumberOffices = a.ListingDataOther.NumberOffices,
        NumberRestrooms = a.ListingDataOther.NumberRestrooms,
        ProjectedCompletionDate = a.ListingDataOther.ProjectedCompletionDate,
        SchoolElementary = a.ListingDataOther.SchoolElementary,
        SchoolMiddle = a.ListingDataOther.SchoolMiddle,
        SchoolHigh = a.ListingDataOther.SchoolHigh,
        SizePorch = a.ListingDataOther.SizePorch,
        SizeBedMaster = a.ListingDataOther.SizeBedMaster,
        SizeBed2 = a.ListingDataOther.SizeBed2,
        SizeBed3 = a.ListingDataOther.SizeBed3,
        SizeBed4 = a.ListingDataOther.SizeBed4,
        SizeBed5 = a.ListingDataOther.SizeBed5,
        SizeBonusRoom = a.ListingDataOther.SizeBonusRoom,
        SizeDinette = a.ListingDataOther.SizeDinette,
        SizeDiningRoom = a.ListingDataOther.SizeDiningRoom,
        SizeFamilyRoom = a.ListingDataOther.SizeFamilyRoom,
        SizeGreatRoom = a.ListingDataOther.SizeGreatRoom,
        SizeKitchen = a.ListingDataOther.SizeKitchen,
        SizeLivingRoom = a.ListingDataOther.SizeLivingRoom,
        SizeStudio = a.ListingDataOther.SizeStudio,
        SizeStudyDen = a.ListingDataOther.SizeStudyDen,
        SqftTotalBldg = a.ListingDataOther.SqftTotalBldg,
        TotalUnits = a.ListingDataOther.TotalUnits,
        VirtualTourLink = a.ListingDataOther.VirtualTourLink,
        WaterAccessYN = a.ListingDataOther.WaterAccessYN,
        WaterExtrasYN = a.ListingDataOther.WaterExtrasYN,
        WaterFrontageYN = a.ListingDataOther.WaterFrontageYN,
        WaterViewYN = a.ListingDataOther.WaterViewYN,
        ZipCodePlusFour = a.ListingDataOther.ZipCodePlusFour,
        Zoning = a.ListingDataOther.Zoning,
        AWCRemarks = a.ListingDataOther.AWCRemarks,
        ArchitecturalStyle = a.ListingDataOther.ArchitecturalStyle.Name,
        CondoFeeSchedule = a.ListingDataOther.TimeFrame.Name,
        FrontExposure = a.ListingDataOther.FrontExposure.Name,
        Foundation = a.ListingDataOther.Foundation.Name,
        Furnishing = a.ListingDataOther.Furnishing.Name,
        HOASchedule = a.ListingDataOther.TimeFrame.Name,
        MobileHomeWidth = a.ListingDataOther.MobileHomeWidth.Name,

        //--- select some data from the junction tables (which in turn use lookup tables)
        AdditionalRooms = a.ListingAdditionalRooms.Select(b => b.AdditionalRoom.Name),
        AppliancesIncluded = a.ListingAppliances.Select(b => b.Appliance.Name),
        CommunityFeatures = a.ListingCommunityFeatures.Select(b => b.CommunityFeature.Name),
        ExteriorConstructions = a.ListingExteriorConstructions.Select(b => b.ExteriorConstruction.Name),
        ExteriorFeatures = a.ListingExteriorFeatures.Select(b => b.ExteriorFeature.Name),
        Financings = a.ListingFinancings.Select(b => b.Financing.Name),
        FireplaceDescriptions = a.ListingFireplaceDescriptions.Select(b => b.FireplaceDescription.Name),
        FloorCoverings = a.ListingFloorCoverings.Select(b => b.FloorCovering.Name),
        FuelTypes = a.ListingFuelTypes.Select(b => b.FuelType.Name),
        GarageFeatures = a.ListingGarageFeatures.Select(b => b.GarageFeature.Name),
        GarageTypes = a.ListingGarageTypes.Select(b => b.GarageType.Name),
        HeatTypes = a.ListingHeatTypes.Select(b => b.HeatType.Name),
        InteriorFeatures = a.ListingInteriorFeatures.Select(b => b.InteriorFeature.Name),
        KitchenFeatures = a.ListingKitchenFeatures.Select(b => b.KitchenFeature.Name),
        LeaseIncludes = a.ListingLeaseIncludes.Select(b => b.LeaseInclude.Name),
        MasterBathFeatures = a.ListingMasterBathFeatures.Select(b => b.MasterBathFeature.Name),
        ParkingOptions = a.ListingParkingOptions.Select(b => b.ParkingOption.Name),
        PoolFeatures = a.ListingPoolFeatures.Select(b => b.PoolFeature.Name),
        PoolTypes = a.ListingPoolTypes.Select(b => b.PoolType.Name),
        PropertyUses = a.ListingPropertyUses.Select(b => b.PropertyUse.Name),
        RoofTypes = a.ListingRoofTypes.Select(b => b.RoofType.Name),
        WaterAccessTypes = a.ListingWaterAccessTypes.Select(b => b.WaterType.Name),
        WaterExtraTypes = a.ListingWaterExtraTypes.Select(b => b.WaterExtraType.Name),
        WaterFrontageTypes = a.ListingWaterFrontageTypes.Select(b => b.WaterType.Name),
        WaterViewTypes = a.ListingWaterViewTypes.Select(b => b.WaterType.Name),
    })
    .OrderBy(a => a.MLSNumber)
    .ToList();

这种结构有更好的方法吗?即使在查询上调用.ToString()来查看生成的SQL也会抛出OOM异常。

更新

在回应@Gert Arnold时,您能否进一步解释数据库未正常化的原因?我们以Status的字段为例,我有Status = a.Status.Name。数据库中有一个名为Statuses的表,其中包含2列StatusIDName,数据类似于1|Active2|Pending,{{1 }}。 3|Sold表上的字段为Listings,其中包含对StatusID表中StatusID字段的引用。为了获取实际名称而不是状态ID,我必须Statuses。这与PropertyType,PropertyStyle,Country,State,County,City,SaleType的结构完全相同。

然后,对于a.Status.NameListingDataCommons表,这些表与ListingDataOthers表的关系是1:1创建的。创建它们是因为列表中有数百个字段,而不是将它们转储到一个巨大的表中,它们根据每个字段的查询频率进行分割。在这些表中,有一些列引用了查找表的ID而不是重复的字符串值,如上面的状态所述。

然后有一些联结表,例如Listings,它具有1:多关系,其中1个列表可以有许多额外的房间。 ListingAdditionalRooms表(以及所有其他联结表)有2列(ListingID | AdditionalRoomID)引用ListingAdditionalRooms表和Listings表中的相应记录。

如果这是您已经看过的最糟糕的数据库设计之一,您如何建议对其进行改进?我应该有一个AdditionalRooms表,其中有近300列,通过记录重复存储字符串值吗?这似乎不是一个好的解决方案。请简要描述一下如何进行(Listings表有数百万条记录)。不要求图表,只是简单的解释。

对于建议,将其分解为更小的块并在2个请求中请求数据似乎确实解决了问题(一个请求中的联结表数据,另一个请求中的所有其他数据)。

关于此数据量从未在任何UI中显示,这是不正确的。虽然此查询仅测试限制,但绝对有必要向用户显示列表的完整详细信息。

我期待您对数据库结构的建议。

1 个答案:

答案 0 :(得分:2)

实体框架在它必须生成的所有连接中窒息。只有在a.<some property>部分,您才能获得 35 不同的导航属性!最重要的是,您可以在嵌套的Select语句中访问大量导航属性。

核心问题是,这是迄今为止我见过的最糟糕的数据库设计之一。这些表只是大量不相关和重复的数据。没有任何规范化。

你唯一的希望是对数据模型进行重大改革,这是一种新的设计,基本上。实体框架是一个ORM,对象关系映射器,所以应该有一些关系来开始使它成为一个有用的工具。

如果设计不在您的手中,您可以考虑两件事:

  • 获取内存中的数据逐个,并从这些构建块中构建客户端对象。

  • 必须可以使用较小的模型。我无法想象有一个UI视图会一次显示所有这些数据。为每个视图构建专用视图模型。