Question

我正在使用Open XML SDK创建带有自定义代码的XLSX文件。它适用于50000条记录，200列，最大13 GB RAM。

但是，当我尝试使用最多16 GB RAM的100000行和200列时，从未创建XLSX文件并继续增加和减少RAM内存，同时增加和减少CPU使用率。

我正在向Stream写入100000行和200列，并同时将流复制到Package Part Stream而不拆分XML文件。该XML文件大小为3 GB。

请您使用Open XML SDK解决此问题。

当我尝试使用Open XML时，它使用100个记录，其中200列用于单个用户。但是，当时正在为两个用户创建包含200列的100000条记录，服务器正在挂起。

我的自定义代码占用了更多内存，但没有挂起。

在下面的代码“CreateOpenXMLComWorkSheet_XMLWriter”中，方法占用的RAM大小更多。

我使用下面的代码供您参考。如果需要更改，请告诉我。

//Package method

    Package package = null;
    using (package = ZipPackage.Open(path, FileMode.Create))
    {

        packgPart = package.CreatePart(new Uri(relativePaths[relIndex], UriKind.Relative), contentTypes[6], CompressionOption.Maximum);
        XmlWriter xmlWriter;
        Stream stream = CreateOpenXMLComWorkSheet_XMLWriter(data, "", out xmlWriter);
        CopyStream(stream, packgPart.GetStream());
        xmlWriter.Flush();
        xmlWriter.Close();
        xmlWriter = null;
        package.Flush();
        packgPart = null;
        stream.Close();
        stream.Dispose();
        stream = null;
        relIndex++;
        GC.Collect();
        package.Close();

    }

    // CreateOpenXMLComWorkSheet method
    // Define other methods and classes here
private static Stream CreateOpenXMLComWorkSheet_XMLWriter(List<StringBuilder> rows, string sheet,out XmlWriter xmlWriter)

{

    string[]  cols;

    XmlWriterSettings xmlWriterSettings = new XmlWriterSettings();
             xmlWriterSettings.NewLineHandling = NewLineHandling.None;
             xmlWriterSettings.Indent = false;
             xmlWriter = null;
             MemoryStream stream = new MemoryStream();
             string nameSpace = "http://schemas.openxmlformats.org/spreadsheetml/2006/main";
             xmlWriter = XmlWriter.Create(stream,xmlWriterSettings);

    xmlWriter.WriteStartElement("x","worksheet",nameSpace);
    xmlWriter.WriteStartElement("x","sheetData",nameSpace);


    for (m = 0; m < rows.Count; m++)
    {

        xmlWriter.WriteStartElement("x","row",nameSpace);

        cols = rows[m].ToString().Split(new string[] { univDelimiter }, StringSplitOptions.None);


        for (int i = 1; i <= cols.Length; i++)
         {
             cellValue = cols[i - 1];
             if (double.TryParse(cellValue,out dVal))
             {
                 dataType = "n";
             }
             else
             {
                 dataType = "str";
             }
             xmlWriter.WriteStartElement("x","c",nameSpace);
             xmlWriter.WriteAttributeString("s", "13");
             xmlWriter.WriteAttributeString("t", dataType);
             xmlWriter.WriteStartElement("x", "v",nameSpace);
             xmlWriter.WriteValue(cellValue);
             xmlWriter.WriteEndElement();
             xmlWriter.WriteEndElement();
         }

        xmlWriter.WriteEndElement();

        rows[m] = null;

    }
    xmlWriter.WriteEndElement();
    xmlWriter.WriteEndElement();
    xmlWriter.Flush();
    stream.Position = 0;
    return stream;
}

//CopyStream method
private static void CopyStream(Stream source, Stream target)
{
     const int bufSize = 0x1000;
     byte[] buf = new byte[bufSize];
     int bytesRead = 0;
     while ((bytesRead = source.Read(buf, 0, bufSize)) > 0)
        target.Write(buf, 0, bytesRead);
}

Answer 1

看起来你在编写文件时采取了错误的方法，打开xml sdk是一个很好的工具来创建具有大量数据的excel。我认为你需要采用类似SAX的方法，它使用xmlreader和writer的组合而不会耗尽内存。

看看这个符合您特定要求的精彩博客。

https://blogs.msdn.microsoft.com/brian_jones/2010/06/22/writing-large-excel-files-with-the-open-xml-sdk/

Answer 2

为减少内存压力，请考虑不要在XmlWriter中使用MemoryStream。如果您使用基于磁盘的流，那么这将显着降低内存压力。

Answer 3

使用您在此处获取的流packgPart.GetStream()作为xml编写器的后端存储。

此外，我感觉您不需要在内存中加载整个CSV。

这是一个仅使用流的版本。

void Main()
{
    string inputFile = "D:\\_bigfile.csv";
    string path = "D:\\pack.zip";


    Package package = null;
    using (package = ZipPackage.Open(path, FileMode.Create))
    {

        var packgPart = package.CreatePart(new Uri("/test.xml", UriKind.Relative), System.Net.Mime.MediaTypeNames.Text.Xml, CompressionOption.Maximum);
        using (var inputStream = File.OpenRead(inputFile))
        {
            CreateOpenXMLComWorkSheet_XMLWriter(inputStream, "", packgPart.GetStream());
        }
    }
}
private const string univDelimiter = "|";

private static void CreateOpenXMLComWorkSheet_XMLWriter(Stream inputStream, string sheet, Stream packagePartStream)

{
    string cellValue = "";
    string dataType = "";
    double dVal = 0;
    string[]  cols;

    XmlWriterSettings xmlWriterSettings = new XmlWriterSettings();
     xmlWriterSettings.NewLineHandling = NewLineHandling.None;
     xmlWriterSettings.Indent = false;


     string nameSpace = "http://schemas.openxmlformats.org/spreadsheetml/2006/main";
     using (var xmlWriter = XmlWriter.Create(packagePartStream, xmlWriterSettings))
    {
        xmlWriter.WriteStartElement("x","worksheet",nameSpace);
        xmlWriter.WriteStartElement("x","sheetData",nameSpace);

        using (var sr = new StreamReader(inputStream))
        {
            string line = null;
            while ((line = sr.ReadLine()) != null)
            {
                xmlWriter.WriteStartElement("x","row",nameSpace);

                cols = line.Split(new string[] { univDelimiter }, StringSplitOptions.None);
                for (int i = 1; i <= cols.Length; i++)
                 {
                     cellValue = cols[i - 1];
                     if (double.TryParse(cellValue,out dVal))
                     {
                         dataType = "n";
                     }
                     else
                     {
                         dataType = "str";
                     }
                     xmlWriter.WriteStartElement("x","c",nameSpace);
                     xmlWriter.WriteAttributeString("s", "13");
                     xmlWriter.WriteAttributeString("t", dataType);
                     xmlWriter.WriteStartElement("x", "v",nameSpace);
                     xmlWriter.WriteValue(cellValue);
                     xmlWriter.WriteEndElement();
                     xmlWriter.WriteEndElement();
                 }

                xmlWriter.WriteEndElement();
            }
        }
        xmlWriter.WriteEndElement();
        xmlWriter.WriteEndElement();
    }
}

创建XLSX文件需要更多时间来处理100000条记录并占用最大RAM内存

3 个答案: