我正在研究控制台应用程序，它只负责从sql server中提取数据。为此，我使用sqlreader非常快速地读取数据。
我担心的是，当我运行控制台应用程序时，该应用程序占用了太多内存。它不断增加。我已经检查了任务管理器，但没有运气。我没有找到任何占用太多内存的进程。经过一些谷歌搜索后，我找到了Rammap工具。根据该工具，AWE占用了太多内存。它需要7 GB的8 GB内存。但是它没有释放内存，之后查询没有被sql server服务并且获得超时过期错误。
但是，我搜索了AWE，但没有找到任何与我的问题相关的有用信息。
那么，为什么要占用太多内存？
以下是我检索数据的示例代码。
/// <summary>
/// Get Products
/// </summary>
/// <param name="productIds">Product Ids</param>
/// <returns>Product List</returns>
public IList<Product> GetProducts(IList<int> productIds)
{
try
{
//pass product identifiers as comma-delimited string
string commaSeparatedProductIds = "";
if (productIds != null)
{
commaSeparatedProductIds = String.Join(",", productIds);
}
string query = "GetProducts";
List<Product> productList = new List<Product>();
//Open connection
connection = new SqlConnection(_connectionString);
if (connection.State == ConnectionState.Closed)
connection.Open();
//create a command object
using (var cmd = connection.CreateCommand())
{
//command to execute
cmd.CommandText = query;
cmd.CommandType = CommandType.StoredProcedure;
cmd.CommandTimeout = 120;
cmd.Parameters.Add("@ProductIds", commaSeparatedProductIds);
//database call
var reader = cmd.ExecuteReader();
if (reader.HasRows)
{
while (reader.Read())
{
//TODO Uncomment se name in sp
Product product = new Product();
product.Id = reader.GetValue<int>("Id");
product.Name = reader.GetValue<string>("Name");
product.ShortDescription = reader.GetValue<string>("ShortDescription");
product.FullDescription = reader.GetValue<string>("FullDescription");
product.ProductTypeId = reader.GetValue<int>("ProductTypeId");
product.CreatedOnUtc = reader.GetValue<DateTime>("CreatedOnUtc");
product.Sku = reader.GetValue<string>("Sku");
product.AllowCustomerReviews = reader.GetValue<bool>("AllowCustomerReviews"); Convert.ToBoolean(reader["AllowCustomerReviews"].ToString());
product.ApprovedRatingSum = reader.GetValue<int>("ApprovedRatingSum");
product.ApprovedTotalReviews = reader.GetValue<int>("ApprovedTotalReviews");
product.VendorId = reader.GetValue<int>("VendorId");
product.IsTaxExempt = reader.GetValue<bool>("IsTaxExempt"); Convert.ToBoolean(reader["IsTaxExempt"].ToString());
product.TaxCategoryId = reader.GetValue<int>("TaxCategoryId");
product.OldPrice = reader.GetValue<decimal>("OldPrice");
product.Price = reader.GetValue<decimal>("Price");
product.DisableBuyButton = reader.GetValue<bool>("DisableBuyButton"); Convert.ToBoolean(reader["DisableBuyButton"].ToString());
product.AvailableForPreOrder = reader.GetValue<bool>("AvailableForPreOrder"); Convert.ToBoolean(reader["AvailableForPreOrder"].ToString());
product.SpecialPrice = reader.GetValue<decimal>("SpecialPrice");
product.SpecialPriceStartDateTimeUtc = reader.GetValue<DateTime?>("SpecialPriceStartDateTimeUtc");
product.SpecialPriceEndDateTimeUtc = reader.GetValue<DateTime?>("SpecialPriceEndDateTimeUtc");
product.AvailableStartDateTimeUtc = reader.GetValue<DateTime?>("AvailableStartDateTimeUtc");
product.AvailableEndDateTimeUtc = reader.GetValue<DateTime?>("AvailableEndDateTimeUtc");
product.CallForPrice = reader.GetValue<bool>("CallForPrice"); Convert.ToBoolean(reader["CallForPrice"].ToString());
product.CustomerEntersPrice = reader.GetValue<bool>("CustomerEntersPrice"); Convert.ToBoolean(reader["CustomerEntersPrice"].ToString());
product.VendorId = reader.GetValue<int>("VendorId");
product.VendorName = reader.GetValue<string>("VendorName");
product.SeName = reader.GetValue<string>("SeName");
product.Category = reader.GetValue<string>("Category");
product.Manufacturer = reader.GetValue<string>("Manufacturer");
product.Tag = reader.GetValue<string>("Tag");
product.Picture = reader.GetValue<string>("Picture");
productList.Add(product);
}
}
else
{
Console.WriteLine("No rows found.");
}
//close up the reader, we're done saving results
reader.Close();
//close connection
connection.Close();
return productList;
}
}
catch (Exception ex)
{
HelperClass.CatchException(ex);
return new List<Product>();
}
finally
{
connection.Close();
}
}
/// <summary>
/// Helper class for SqlDataReader, which allows for the calling code to retrieve a value in a generic fashion.
/// </summary>
public static class SqlReaderHelper
{
private static bool IsNullableType(Type theValueType)
{
return (theValueType.IsGenericType && theValueType.GetGenericTypeDefinition().Equals(typeof(Nullable<>)));
}
/// <summary>
/// Returns the value, of type T, from the SqlDataReader, accounting for both generic and non-generic types.
/// </summary>
/// <typeparam name="T">T, type applied</typeparam>
/// <param name="theReader">The SqlDataReader object that queried the database</param>
/// <param name="theColumnName">The column of data to retrieve a value from</param>
/// <returns>T, type applied; default value of type if database value is null</returns>
public static T GetValue<T>(this SqlDataReader theReader, string theColumnName)
{
// Read the value out of the reader by string (column name); returns object
object theValue = theReader[theColumnName];
// Cast to the generic type applied to this method (i.e. int?)
Type theValueType = typeof(T);
// Check for null value from the database
if (DBNull.Value != theValue)
{
// We have a null, do we have a nullable type for T?
if (!IsNullableType(theValueType))
{
// No, this is not a nullable type so just change the value's type from object to T
return (T)Convert.ChangeType(theValue, theValueType);
}
else
{
// Yes, this is a nullable type so change the value's type from object to the underlying type of T
NullableConverter theNullableConverter = new NullableConverter(theValueType);
return (T)Convert.ChangeType(theValue, theNullableConverter.UnderlyingType);
}
}
// The value was null in the database, so return the default value for T; this will vary based on what T is (i.e. int has a default of 0)
return default(T);
}
}
这是存储过程
-- =============================================
-- Author: Dharmik
-- Create date: 29-01-2014
-- Description: Get products for indexing
-- =============================================
ALTER PROCEDURE [dbo].[GetProducts]
@ProductIds nvarchar(MAX) = NULL
AS
BEGIN
if(@ProductIds is not null)
BEGIN
CREATE TABLE #Product(
Id [int] ,
Name [nvarchar](400) NOT NULL,
ShortDescription [nvarchar](max) NULL,
FullDescription [nvarchar](max) NULL,
ProductTypeId [int] NOT NULL,
CreatedOnUtc [datetime] NOT NULL,
Sku [nvarchar](400) NULL,
AllowCustomerReviews [bit] NOT NULL,
ApprovedRatingSum [int] NOT NULL,
ApprovedTotalReviews [int] NOT NULL,
VendorId [int] NOT NULL,
IsTaxExempt [bit] NOT NULL,
TaxCategoryId [int] NOT NULL,
Price [decimal](18, 4) NOT NULL,
OldPrice [decimal](18, 4) NOT NULL,
DisableBuyButton [bit] NOT NULL,
AvailableForPreOrder [bit] NOT NULL,
SpecialPrice [decimal](18, 4) NULL,
SpecialPriceStartDateTimeUtc [datetime] NULL,
SpecialPriceEndDateTimeUtc [datetime] NULL,
AvailableStartDateTimeUtc [datetime] NULL,
AvailableEndDateTimeUtc [datetime] NULL,
CallForPrice [bit] NOT NULL,
CustomerEntersPrice [bit] NULL,
VendorName [nvarchar](max) NULL,
SeName [nvarchar](max) NULL,
Category [nvarchar](max) NULL,
Manufacturer [nvarchar](max) NULL,
Tag [nvarchar](max) NULL,
Picture [nvarchar](max) NULL)
DECLARE @ProductId INT
DECLARE mapping_cursor CURSOR
FOR
SELECT * FROM [nop_splitstring_to_table](@ProductIds, ',')
--SELECT TOP 80000 ProductId FROM Incremental_Solr_Product WHERE SolrStatus=1 AND IsDeleted=0 AND StoreId=1
OPEN mapping_cursor
FETCH NEXT
FROM mapping_cursor INTO @ProductId
WHILE @@FETCH_STATUS = 0
BEGIN
INSERT INTO #Product
(Id,
Name,
ShortDescription,
FullDescription,
ProductTypeId,
CreatedOnUtc,
Sku,
AllowCustomerReviews,
ApprovedRatingSum,
ApprovedTotalReviews,
VendorId,
IsTaxExempt,
TaxCategoryId,
Price,
OldPrice,
DisableBuyButton,
AvailableForPreOrder,
SpecialPrice,
SpecialPriceStartDateTimeUtc,
SpecialPriceEndDateTimeUtc,
AvailableStartDateTimeUtc,
AvailableEndDateTimeUtc,
CallForPrice,
CustomerEntersPrice,
VendorName,
SeName,
Category,
Manufacturer,
Tag,
Picture)
SELECT
p.Id,
p.Name,
p.ShortDescription,
p.FullDescription,
p.ProductTypeId,
p.CreatedOnUtc,
p.Sku,
p.AllowCustomerReviews,
p.ApprovedRatingSum,
p.ApprovedTotalReviews,
p.VendorId,
p.IsTaxExempt,
p.TaxCategoryId,
p.Price,
p.OldPrice,
p.DisableBuyButton,
p.AvailableForPreOrder,
p.SpecialPrice,
p.SpecialPriceStartDateTimeUtc,
p.SpecialPriceEndDateTimeUtc,
p.AvailableStartDateTimeUtc,
p.AvailableEndDateTimeUtc,
p.CallForPrice,
p.CustomerEntersPrice,
v.Name AS 'VendorName',
u.Slug AS 'SeName',
(SELECT pcm.Id,pcm.CategoryId,c.Name AS 'CategoryName',pcm.DisplayOrder AS 'CategoryDisplayOrder' FROM Product_Category_Mapping AS pcm JOIN Category AS c ON pcm.CategoryId=c.Id WHERE pcm.ProductId=@ProductId FOR XML RAW ,ROOT('Category')) AS 'Category',
(SELECT pmm.ManufacturerId ,m.Name,pmm.DisplayOrder FROM Product_Manufacturer_Mapping AS pmm JOIN Manufacturer AS m ON pmm.ManufacturerId=m.Id WHERE pmm.ProductId=@ProductId FOR XML RAW ,ROOT('Manufacturer')) AS 'Manufacturer',
(SELECT ptm.ProductTag_Id,t.Name FROM Product_ProductTag_Mapping AS ptm JOIN ProductTag AS t ON ptm.ProductTag_Id=t.Id WHERE ptm.Product_Id=@ProductId FOR XML RAW ,ROOT('Tag')) AS 'Tag',
(SELECT TOP 1 ppm.PictureId,p.MimeType,p.SeoFilename FROM Product_Picture_Mapping AS ppm LEFT JOIN Picture AS p ON ppm.PictureId=p.Id WHERE ProductId=@ProductId ORDER BY DisplayOrder FOR XML RAW ,ROOT('Picture')) AS 'Picture'
FROM Product as p LEFT OUTER JOIN Vendor AS v ON p.VendorId=v.Id
LEFT OUTER JOIN UrlRecord AS u ON p.Id=u.EntityId
WHERE p.Id=@ProductId AND u.EntityName='Product' AND u.LanguageId=0
FETCH NEXT
FROM mapping_cursor INTO @ProductId
END
CLOSE mapping_cursor
DEALLOCATE mapping_cursor
SELECT * FROM #Product
DROP TABLE #Product
END
ELSE
PRINT 'Provide product ids...'
END
答案 0 :(得分：2)
使用SqlReader无关紧要，您在数据库查询中创建了一个对象列表，这很可能是您的内存所在。
最佳做法是尽可能减少数据量。
我怀疑你的批处理逻辑是由于你用来获取每个批次的循环而导致内存问题。
添加一些跟踪，查看每个数据库函数的调用次数 您可能会发现一个或多个函数被调用多次，当您希望它只被调用一次时。然后这将帮助您缩小故障区域。
答案 1 :(得分：1)
当您将数据流式传输到您的应用中时，您正在使用它做什么？听起来你正在存储与每一行相关的东西，这就是你的内存消耗继续攀升的原因。
答案 2 :(得分：0)
如果你的应用程序将数据加载到内存中，那么....它会消耗内存。
如果SQL Server必须工作（为您的应用提供数据），那么SQL Server将消耗内存......而不是释放它（因为这是它的构建方式）。您可以尝试减少SQL Server上的最大内存来排除这种情况。 SQL Server使用AWE，如果您已将其配置为但它实际上仅用于32位安装，您必须以这种方式显式配置它。
许多非数据库开发人员没有意识到您可以经常使用更少的内存来更有效地在数据库中完成所需的工作。由于我们没有您的所有代码，如果是这种情况我们无法解决。
关于：存储数据库中所有内容的最佳做法是什么？最佳做法是不要存储数据库中的所有内容。把它留在数据库中，只得到你需要的东西。