使用嵌套的defaultdict

Question

此问题是上一个问题的扩展：rebuild python array based on common elements   - 但足够不同以保证一个新问题：

我现在一直在努力解决这个问题。我的数据是来自SQL查询的字典数组。数组中的每个元素代表一个货件，并且存在基于密钥的公共值。

data = [
    {"CustName":"customer1", "PartNum":"part1", "delKey":"0001", "qty":"10", "memo":"blah1"},
    {"CustName":"customer1", "PartNum":"part1", "delKey":"0002", "qty":"10", "memo":"blah2"},
    {"CustName":"customer1", "PartNum":"part1", "delKey":"0003", "qty":"10", "memo":"blah3"},
    {"CustName":"customer2", "PartNum":"part3", "delKey":"0004", "qty":"20", "memo":"blah4"},
    {"CustName":"customer2", "PartNum":"part3", "delKey":"0005", "qty":"20", "memo":"blah5"},
    {"CustName":"customer3", "PartNum":"partXYZ", "delKey":"0006", "qty":"50", "memo":"blah6"},
    {"CustName":"customer3", "PartNum":"partABC", "delKey":"0007", "qty":"100", "memo":"blah7"}]

我想要的输出根据特定键

分组

dataOut = [
   {"CustName":"customer1", "Parts":[
        {"PartNum":"part1", "deliveries":[
            {"delKey":"0001", "qty":"10", "memo":"blah1"},
            {"delKey":"0002", "qty":"10", "memo":"blah2"},
            {"delKey":"0003", "qty":"10", "memo":"blah3"}]}]},
   {"CustName":"customer2", "Parts":[
        {"PartNum":"part3", "deliveries":[
            {"delKey":"0004", "qty":"20", "memo":"blah4"},
            {"delKey":"0005", "qty":"20", "memo":"blah5"}]}]},
   {"CustName":"customer3", "Parts":[
        {"PartNum":"partXYZ", "deliveries":[
            {"delKey":"0006", "qty":"50", "memo":"blah6"}]},
        {"PartNum":"partABC", "deliveries":[
            {"delKey":"0007", "qty":"100", "memo":"blah7"}]}]}]

我可以使用上一个问题提供的defaultdict和list comprehension来获得单个级别的分组，并稍加修改

d = defaultdict(list)
for item in data:
    d[item['CustName']].append(item)
print([{'CustName': key, 'parts': value} for key, value in d.items()])

但我似乎无法在输出数组中获得第二级 - 分组b PartNum键。通过一些研究，我认为我需要做的是使用defaultdict作为外部`defaultdict'的类型，如下所示：

d = defaultdict(defaultdict(list))

抛出错误，因为defaultdict返回一个函数，所以我需要使用lambda（是吗？）

d = defaultdict(lambda:defaultdict(list))
for item in data:
    d[item['CustName']].append(item) <----this?

我的问题是如何在循环中“访问”第二级数组，并告诉“内部”defaultdict要分组的内容（PartNum）？数据来自数据库程序员并且项目不断发展以添加越来越多的数据（密钥），所以我希望这个解决方案尽可能通用，以防更多数据被我的方式抛出。我希望能够根据我需要的级别来“链接”默认值。我正在学习，因为我正在努力理解lambda以及defaultdict类型的基础知识以及从何处开始。

Answer 1

按照@Pynchia的建议使用groupby并使用sorted按照@hege_hegedus的建议使用无序数据：

from itertools import groupby
dataOut = []
dataSorted = sorted(data, key=lambda x: (x["CustName"], x["PartNum"]))
for cust_name, cust_group in groupby(dataSorted, lambda x: x["CustName"]):
    dataOut.append({
        "CustName": cust_name,
        "Parts": [],
    })
    for part_num, part_group in groupby(cust_group, lambda x: x["PartNum"]):
        dataOut[-1]["Parts"].append({
            "PartNum": part_num,
            "deliveries": [{
                "delKey": delivery["delKey"],
                "memo": delivery["memo"],
                "qty": delivery["qty"],
            } for delivery in part_group]
        })

如果你看第二个for循环，这有望回答你关于在循环中访问二级数组的问题。

Answer 2

您可以使用基于OrderedDefaultdict而非defaultdict(list)的树状数据结构。 （来自我的无关answer的定义。）

from collections import OrderedDict

class OrderedDefaultdict(OrderedDict):
    def __init__(self, *args, **kwargs):
        if not args:
            self.default_factory = None
        else:
            if not (args[0] is None or callable(args[0])):
                raise TypeError('first argument must be callable or None')
            self.default_factory = args[0]
            args = args[1:]
        super(OrderedDefaultdict, self).__init__(*args, **kwargs)

    def __missing__ (self, key):
        if self.default_factory is None:
            raise KeyError(key)
        self[key] = default = self.default_factory()
        return default

Tree = lambda: OrderedDefaultdict(Tree)

d = Tree()
for rec in data:
    custName, partNum, delKey = rec['CustName'], rec['PartNum'], rec['delKey']
    details = {"qty": rec["qty"], "memo": rec["memo"]}
    d[custName]['Parts'][partNum]['deliveries'][delKey] = details

因此，对于问题中显示的data，d最终会包含：

d = {
    "customer1": {
        "Parts": {
            "part1": {
                "deliveries": {"0001": {"memo": "blah1", "qty": "10"},
                               "0002": {"memo": "blah2", "qty": "10"},
                               "0003": {"memo": "blah3", "qty": "10"}}}}},
    "customer2": {
        "Parts": {
            "part3": {
                "deliveries": {"0004": {"memo": "blah4", "qty": "20"},
                               "0005": {"memo": "blah5", "qty": "20"}}}}},
    "customer3": {
        "Parts": {
            "partXYZ": {
                "deliveries": {"0006": {"memo": "blah6", "qty": "50"}}},
            "partABC": {
                "deliveries": {"0007": {"memo": "blah7", "qty": "100"}}}}}
}

这可以简单地打印出来，因为它现在按照你想要的方式分组。

Answer 3

这是我能做到的最漂亮的方式。它使用相同的defaultdict想法来实现正确的分组，因为python的内置groupby函数仅适用于有序数据。

请注意，此版本将改变输入数据集中的项目，因此结果中的叶项与输入的dict实例相同，但删除了"CustName"和"PartNum"条目的

from collections import defaultdict

def groupby_mutate(seq, key):
  d = defaultdict(list)
  for item in seq:
    d[item[key]].append(item)
    del item[key]
  return d

def your_operation(data):
  return [ {
    'CustName': CustName,
    'Parts': [ { 
      'PartNum': PartNum,
      'deliveries': deliveries
    } for PartNum,deliveries in groupby_mutate(custItems, 'PartNum').items() ]
  } for CustName,custItems in groupby_mutate(data, 'CustName').items() ]


# try it
from pprint import *
data = [
    {"CustName":"customer1", "PartNum":"part1", "delKey":"0001", "qty":"10", "memo":"blah1"},
    {"CustName":"customer1", "PartNum":"part1", "delKey":"0002", "qty":"10", "memo":"blah2"},
    {"CustName":"customer1", "PartNum":"part1", "delKey":"0003", "qty":"10", "memo":"blah3"},
    {"CustName":"customer2", "PartNum":"part3", "delKey":"0004", "qty":"20", "memo":"blah4"},
    {"CustName":"customer2", "PartNum":"part3", "delKey":"0005", "qty":"20", "memo":"blah5"},
    {"CustName":"customer3", "PartNum":"partXYZ", "delKey":"0006", "qty":"50", "memo":"blah6"},
    {"CustName":"customer3", "PartNum":"partABC", "delKey":"0007", "qty":"100", "memo":"blah7"}
]

pprint(your_operation(data))

修改

如果将来有人需要它，这里的版本不会改变原始数据：

from collections import defaultdict

def groupby_getitem(seq, key):
  d = defaultdict(list)
  for item in seq:
    d[item[key]].append(item)
  return d

def your_operation(data):
  return [ {
    'CustName': CustName,
    'Parts': [ { 
      'PartNum': PartNum,
      'deliveries': [ dict(
        (k,v) for k,v in delivery.items() if not k in ['CustName', 'PartNum']
      ) for delivery in deliveries ]
    } for PartNum,deliveries in groupby_getitem(custItems, 'PartNum').items() ]
  } for CustName,custItems in groupby_getitem(data, 'CustName').items() ]

Answer 4

按"CustName", "PartNum", "delKey"排序。为每个客户迭代每个零件的交货项目，并累计以匹配您的输出规格。

我喜欢使用operator.itemgetter - 对我而言，它让事情变得更加清晰。

import collections, itertools, operator

cust_name = operator.itemgetter('CustName')
part_num = operator.itemgetter('PartNum')
group_sort = operator.itemgetter('CustName', 'PartNum', 'delKey')
del_key = operator.itemgetter('delKey')
qty = operator.itemgetter('qty')
memo = operator.itemgetter('memo')


# sort on the relavent keys
data.sort(key = group_sort)
result = []

# iterate over customers
for custname, group1 in itertools.groupby(data, cust_name):
    cust_dict = {'CustName' : custname, 'Parts': []}
    # iterate over parts for this customer
    for partnum, group2 in itertools.groupby(group1, part_num):
        part_dict = {"PartNum" : partnum, 'deliveries' : []}
        # iterate over delivery items for this part
        for thing in group2:
            part_dict['deliveries'].append({'delKey':del_key(thing),
                                            'qty':qty(thing),
                                            'memo':memo(thing)})
        cust_dict['Parts'].append(part_dict)
    result.append(cust_dict)

这显然会多次迭代原始数据中的项目，这可能会影响性能 - 但我没有看到多次迭代的方法，因为您需要做什么。