Question

我在解析大型XML文件时遇到内存问题。

文件看起来像（仅前几行）：

<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE raml SYSTEM 'raml20.dtd'>
<raml version="2.0" xmlns="raml20.xsd">
  <cmData type="actual">
    <header>
      <log dateTime="2019-02-05T19:00:18" action="created" appInfo="ActualExporter">InternalValues are used</log>
    </header>
    <managedObject class="MRBTS" version="MRBTS17A_1701_003" distName="PL/M-1" id="366">
      <p name="linkedMrsiteDN">PL/TE-2/p>
      <p name="name">Name of street</p>
      <list name="PiOptions">
        <p>0</p>
        <p>5</p>
        <p>2</p>
        <p>6</p>
        <p>7</p>
        <p>3</p>
        <p>9</p>
        <p>10</p>
      </list>
      <p name="btsName">4251</p>
      <p name="spareInUse">1</p>
    </managedObject>
    <managedObject class="MRBTS" version="MRBTS17A_1701_003" distName="PL/M10" id="958078">
      <p name="linkedMrsiteDN">PLMN-PLMN/MRSITE-138</p>
      <p name="name">Street 2</p>
      <p name="btsName">748</p>
      <p name="spareInUse">3</p>
    </managedObject>
    <managedObject class="MRBTS" version="MRBTS17A_1701_003" distName="PL/M21" id="1482118">
      <p name="name">Stree 3</p>
      <p name="btsName">529</p>
      <p name="spareInUse">4</p>
    </managedObject>
  </cmData>
</raml>

我正在使用xml eTree Element解析器，但是在机器上文件超过4GB且RAM超过32 GB时，我的内存不足。我正在使用的代码：

def parse_xml(data, string_in, string_out):
    """
    :param data: xml raw file that need to be processed and prased
    :param string_in: string that should exist in distinguish name
    :param string_out: string that should not exist in distinguish name
    string_in and string_out represent the way to filter level of parsing (site or cell)
    :return: dictionary with all unnecessary objects for selected technology
    """
    version_dict = {}
    for child in data:
        for grandchild in child:
            if isinstance(grandchild.get('distName'), str) and string_in in grandchild.get('distName') and string_out not in grandchild.get('distName'):
                inner_dict = {}
                inner_dict.update({'class': grandchild.get('class')})
                inner_dict.update({'version': grandchild.get('version')})
                for grandgrandchild in grandchild:
                    if grandgrandchild.tag == '{raml20.xsd}p':
                        inner_dict.update({grandgrandchild.get('name'): grandgrandchild.text})
                    elif grandgrandchild.tag == '{raml20.xsd}list':
                        p_lista = []
                        for gggchild in grandgrandchild:
                            if gggchild.tag == '{raml20.xsd}p':
                                p_lista.append(gggchild.text)
                            inner_dict.update({grandgrandchild.get('name'): p_lista})
                            if gggchild.tag == '{raml20.xsd}item':
                                for gdchild in gggchild:
                                    inner_dict.update({gdchild.get('name'): gdchild.text})
                    version_dict.update({grandchild.get('distName'): inner_dict})
    return version_dict

我已经尝试过使用iterparse和root.clear（）进行操作，但是没有任何帮助。我听说DOM解析器的速度较慢，但是SAX给我一个错误：

ValueError: unknown url type: '/development/data/raml20.dtd'

不知道为什么。如果有人对如何改进方式和性能有任何建议，我将非常感谢。我需要更大的XML示例，我愿意提供它。

先谢谢了。

编辑：

我在第一个答案之后尝试的代码：

import xml.etree.ElementTree as ET
def parse_item(d):
#     print(d)
#     print('---')

    a = '<root>'+ d + '</root>'
    tree = ET.fromstring(a)
    outer_dict_yield = {}
    for elem in tree:
        inner_dict_yield = {}
        for el in elem:
            if isinstance(el.get('name'), str):
                inner_dict_yield.update({el.get('name'): el.text})
            inner_dict.update({'version': elem.get('version')})
#                 print (inner_dict_yield)
    outer_dict_yield.update({elem.get('distName'): inner_dict_yield})
#     print(outer_dict_yield)
    return outer_dict_yield


def read_a_line(file_object):
    while True:
        data = file_object.readline()
        if not data:
            break
        yield data


min_data = ""
inside = False

f = open('/development/file.xml')
outer_main = {}
counter = 1
for line in read_a_line(f):
    if line.find('<managedObject') != -1:
        inside = True
    if inside:
        min_data += line
    if line.find('</managedObject') != -1:
        inside = False
        a = parse_item(min_data)
        counter = counter + 1
        outer_main.update({counter: a})
        min_data = ''

Answer 1

我可以问一个棘手的问题吗？文件平坦吗？似乎有几个父标记，然后所有其他标记都是managedObject项，也许您可以编写一个自定义解析器，通过该解析器解析每个标记，将其视为XML文档，然后将其丢弃。通过文件流式传输将使您能够交替读取，分析和丢弃项目，从而有效地节省了您的存储空间。

下面是一些示例代码，这些代码将流式传输文件并允许您逐个处理每个块。将parse_item替换为对您有用的东西。

def parse_item(d):
    print('---')
    print(d)
    print('---')


def read_a_line(file_object):
    while True:
        data = file_object.readline()
        if not data:
            break
        yield data


min_data = ""
inside = False

f = open('bigfile.xml')
for line in read_a_line(f):
    if line.find('<managedObject') != -1:
        inside = True
    if inside:
        min_data += line
    if line.find('</managedObject') != -1:
        inside = False
        parse_item(min_data)
        min_data = ''

我还应该提到我很懒，并且使用此处列出的生成器来读取文件（但是我做了一些修改）：Lazy Method for Reading Big File in Python?

Answer 2

如果您只需要从XML文件中提取数据并且不需要执行任何特定于XML的操作（例如XSL转换等），则内存占用量非常低的方法是定义自己的TreeBuilder 。示例：

import pathlib
from pprint import pprint
from xml.etree import ElementTree as ET


class ManagedObjectsCollector:
    def __init__(self):
        self.item_count = 0
        self.items = []
        self.curr_item = None
        self.attr_name = None
        self.list_name = None
        self.list_entry = False

    def start(self, tag, attr):
        if tag == '{raml20.xsd}managedObject':
            self.curr_item = dict()
            self.curr_item.update(**attr)
        elif tag == '{raml20.xsd}p':
            if self.list_name is None:
                self.attr_name = attr.get('name', None)
            self.list_entry = self.list_name is not None
        elif tag == '{raml20.xsd}list':
            self.list_name = attr.get('name', None)
            if self.list_name is not None:
                self.curr_item[self.list_name] = []

    def end(self, tag):
        if tag == '{raml20.xsd}managedObject':
            self.items.append(self.curr_item)
            self.curr_item = None
        elif tag == '{raml20.xsd}p':
            self.attr_name = None
            self.list_entry = False
        elif tag == '{raml20.xsd}list':
            self.list_name = None

    def data(self, data):
        if self.curr_item is None:
            return
        if self.attr_name is not None:
            self.curr_item[self.attr_name] = data
        elif self.list_entry:
            self.curr_item[self.list_name].append(data)

    def close(self):
        return self.items


if __name__ == '__main__':
    file = pathlib.Path('data.xml')
    with file.open(encoding='utf-8') as stream:
        collector = ManagedObjectsCollector()
        parser = ET.XMLParser(target=collector)
        ET.parse(stream, parser=parser)
    items = collector.items
    print('total:', len(items))
    pprint(items)

将上述代码与示例数据一起运行将输出：

total: 3
[{'PiOptions': ['0', '5', '2', '6', '7', '3', '9', '10'],
  'btsName': '4251',
  'class': 'MRBTS',
  'distName': 'PL/M-1',
  'id': '366',
  'linkedMrsiteDN': 'PL/TE-2',
  'name': 'Name of street',
  'spareInUse': '1',
  'version': 'MRBTS17A_1701_003'},
 {'btsName': '748',
  'class': 'MRBTS',
  'distName': 'PL/M10',
  'id': '958078',
  'linkedMrsiteDN': 'PLMN-PLMN/MRSITE-138',
  'name': 'Street 2',
  'spareInUse': '3',
  'version': 'MRBTS17A_1701_003'},
 {'btsName': '529',
  'class': 'MRBTS',
  'distName': 'PL/M21',
  'id': '1482118',
  'name': 'Stree 3',
  'spareInUse': '4',
  'version': 'MRBTS17A_1701_003'}]

因为我们没有在ManagedObjectsCollector中构造XML树，并且一次也没有在内存中保留超过当前文件行的数量，所以解析器的内存分配很小，并且内存使用率很大受collector.items列表的影响。上面的示例解析每个managedObject项目中的所有数据，因此列表可能会变得非常大。您可以通过注释self.items.append(self.curr_item)行来验证它-一旦列表不增加，内存使用就保持不变（大约20-30 MiB，具体取决于您的Python版本）。

如果仅需要部分数据，则将受益于TreeBuilder的更简单实现。例如，下面的TreeBuilder仅收集版本属性，而忽略其余标签：

class VersionCollector:
    def __init__(self):
        self.items = []

    def start(self, tag, attr):
        if tag == '{raml20.xsd}managedObject':
            self.items.append(attr['version'])

    def close(self):
        return self.items

奖金

这是一个自包含的脚本，该脚本通过内存使用率测量进行了扩展。您需要安装一些额外的软件包：

$ pip install humanize psutil tqdm

可选：使用lxml进行更快的解析：

$ pip install lxml

以文件名作为参数运行脚本。 40 MiB XML文件的示例输出：

$ python parse.py data_39M.xml
mem usage:   1%|▏    | 174641152/16483663872 [00:01<03:05, 87764892.80it/s, mem=174.6 MB]
total items memory size: 145.9 MB
total items count: 150603
[{'PiOptions': ['0', '5', '2', '6', '7', '3', '9', '10'],
  'btsName': '4251',
  'class': 'MRBTS',
  'distName': 'PL/M-1',
  'id': '366',
  'linkedMrsiteDN': 'PL/TE-2',
  'name': 'Name of street',
  'spareInUse': '1',
  'version': 'MRBTS17A_1701_003'},
  ...

请注意，对于40 MB的XML文件，峰值内存使用量约为174 MB，而items列表的内存分配约为146 MB；剩下的是Python开销，并且无论文件大小如何都保持不变。这样可以粗略估计读取较大文件所需的内存量。

源代码：

from collections import deque
import itertools
import pathlib
from pprint import pprint
import os
import sys
import humanize
import psutil
import tqdm

try:
    from lxml import etree as ET
except ImportError:
    from xml.etree import ElementTree as ET


def total_size(o, handlers={}, verbose=False):
    """https://code.activestate.com/recipes/577504/"""
    dict_handler = lambda d: itertools.chain.from_iterable(d.items())
    all_handlers = {
        tuple: iter,
        list: iter,
        deque: iter,
        dict: dict_handler,
        set: iter,
        frozenset: iter,
    }
    all_handlers.update(handlers)
    seen = set()
    default_size = sys.getsizeof(0)

    def sizeof(o):
        if id(o) in seen:
            return 0
        seen.add(id(o))
        s = sys.getsizeof(o, default_size)

        if verbose:
            print(s, type(o), repr(o), file=sys.stderr)

        for typ, handler in all_handlers.items():
            if isinstance(o, typ):
                s += sum(map(sizeof, handler(o)))
                break
        return s

    return sizeof(o)


class ManagedObjectsCollector:
    def __init__(self, mem_pbar):
        self.item_count = 0
        self.items = []
        self.curr_item = None
        self.attr_name = None
        self.list_name = None
        self.list_entry = False
        self.mem_pbar = mem_pbar
        self.mem_pbar.set_description('mem usage')

    def update_mem_usage(self):
        proc_mem = psutil.Process(os.getpid()).memory_info().rss
        self.mem_pbar.n = 0
        self.mem_pbar.update(proc_mem)
        self.mem_pbar.set_postfix(mem=humanize.naturalsize(proc_mem))

    def start(self, tag, attr):
        if tag == '{raml20.xsd}managedObject':
            self.curr_item = dict()
            self.curr_item.update(**attr)
        elif tag == '{raml20.xsd}p':
            if self.list_name is None:
                self.attr_name = attr.get('name', None)
            self.list_entry = self.list_name is not None
        elif tag == '{raml20.xsd}list':
            self.list_name = attr.get('name', None)
            if self.list_name is not None:
                self.curr_item[self.list_name] = []

    def end(self, tag):
        if tag == '{raml20.xsd}managedObject':
            self.items.append(self.curr_item)
            self.curr_item = None
        elif tag == '{raml20.xsd}p':
            self.attr_name = None
            self.list_entry = False
        elif tag == '{raml20.xsd}list':
            self.list_name = None

        # Updating progress bar costs resources, don't do it
        # on each item parsed or it will slow down the parsing
        self.item_count += 1
        if self.item_count % 10000 == 0:
            self.update_mem_usage()

    def data(self, data):
        if self.curr_item is None:
            return
        if self.attr_name is not None:
            self.curr_item[self.attr_name] = data
        elif self.list_entry:
            self.curr_item[self.list_name].append(data)

    def close(self):
        return self.items


if __name__ == '__main__':
    file = pathlib.Path(sys.argv[1])
    total_mem = psutil.virtual_memory().total
    with file.open(encoding='utf-8') as stream, tqdm.tqdm(total=total_mem, position=0) as pbar_total_mem:
        collector = ManagedObjectsCollector(pbar_total_mem)
        parser = ET.XMLParser(target=collector)
        ET.parse(stream, parser=parser)
    items = collector.items
    print('total:', len(items))
    print('total items memory size:', humanize.naturalsize(total_size(items)))
    pprint(items)

python elementTree XML解析器的性能问题

2 个答案:

奖金