Question

我想从数据库中重新处理350万个数据。我们正在使用oracle pl / sql 11g。我尝试每批获取1000行进行重新处理。但是在处理了42000行之后，它会抛出内存异常。这是我的代码

internal void dataReprocessor(int rowCount)
{

    DataTable dataTable = new DataTable();
    int MaxProcessid = AssetProcessorDbHandler.GetMaxProcessId();
    startCount=AssetProcessorDbHandler.GetErrorCount();
    ShowStatus("Max. processid:" + MaxProcessid);
    ShowStatus("Fetching " + rowCount + " rows....");
    do
    {

        dataTable.Clear();
        dataTable = AssetProcessorDbHandler.GetHundreadrows(rowCount, MaxProcessid);
         if (dataTable.Rows.Count > 0)
         {


             for (int i = 0; i < dataTable.Rows.Count; i++)
             {
                int error = 0;
                mainId = Convert.ToInt32(dataTable.Rows[i]["MAINID"]);
                itemId = Convert.ToInt32(dataTable.Rows[i]["ITEMID"]);
                siteid = Convert.ToInt32(dataTable.Rows[i]["SITE_ID"]);
                equipmentId = Convert.ToInt32(dataTable.Rows[i]["EQUIPMENT_TYPE"]);
                qrCode = dataTable.Rows[i]["QRCODE"].ToString();
                string equipmentname = AssetProcessorDbHandler.Getequipmentname(equipmentId);
                error = DbHandler.CheckQRCofEquipment(siteid, equipmentId, itemId, qrCode,ref assetStatus);
                if (error == 1)//Not available in iMapp.
                {
                    MESSAGE=string.Concat("QRCode ",qrCode," for ",equipmentname," is not available in iMapp!");
                    AssetProcessorDbHandler.UpdateSMSintoQRCtable(itemId, MESSAGE);
                }
                else if (error == 2)//Not available in that site.
                {
                    int mappedsite = AssetProcessorDbHandler.Getmappedsiteid(qrCode);
                    MESSAGE = string.Concat("QRCode ", qrCode, " for ", equipmentname, " is mapped with site IN-", mappedsite);
                    AssetProcessorDbHandler.UpdateSMSintoQRCtable(itemId,MESSAGE);
                }
                else if (assetStatus == 1)//Equipment was in Finance verifiaction pending
                {
                    MESSAGE = string.Concat("Finance verification pending of ", equipmentname, " for site IN-", siteid, " in iMapp. Please contact corporate finance team.");
                    AssetProcessorDbHandler.UpdateSMSintoQRCtable(itemId, MESSAGE);
                }
                 AssetProcessorDbHandler.UpdateReprocessed(MaxProcessid + 1, itemId);
            }
        }
    } while (dataTable.Rows.Count > 0);

    ShowStatus("--Completed.--");
    finalCount = AssetProcessorDbHandler.GetErrorCount();
    ShowStatus(startCount + " ROWS PROCESSED.");
    ShowStatus(finalCount + " ROWS ARE STILL ON ERROR.");

}

有人帮帮我吗？问题正在升级。

Answer 1

当我们每天处理数百万条记录时，我们遇到了同样的问题。

问题是我们有太多的连接打开oracle数据库并且没有关闭并妥善处理这些对象。

最好在与oracle数据库交互时使用波纹管片段。

using(var oracleConnection = new OracleConnection(connectionString))
{
      oracleConnection.Open()
      //do your stuff here.

}

Answer 2

您永远不会丢弃数据表。每个循环保留一个数据表并正确处理它。

这部分：

do
{

    dataTable.Clear();
    dataTable = AssetProcessorDbHandler.GetHundreadrows(rowCount, MaxProcessid);
     if (dataTable.Rows.Count > 0)
     {

应阅读：

while(true)
{
    using(var dataTable = AssetProcessorDbHandler.GetHundreadrows(rowCount, MaxProcessid))
    {
        if (dataTable.Rows.Count == 0) break;

从oracle DB获取数据时在C＃中获取System.OutOfMemoryException

2 个答案: