Question

我正在尝试使用protobuf-net序列化大型对象图，大约250万个项目，但应用程序似乎陷入了无限循环。内存消耗在20-40 GB之间，并使用100％的一个CPU内核。

序列化大约200万个对象的图表可以正常工作，并产生850 MB的文件。

我有什么办法可以防止这个问题，或者在序列化之前我是否需要将树分成小块？

我很抱歉我的课程有点复杂，但我认为这可能是问题的一部分。我使用的是protobuf-net 2.0.0.668版。每个RoutingNode包含大约50个子节点。

[ProtoContract()]
public class Tree
{
    public Tree() { }

    [ProtoMember(1, DataFormat = DataFormat.Group)]
    public byte[] SomeValues { get; set; }

    [ProtoMember(2, DataFormat = DataFormat.Group)]
    public RoutingNode Root { get; set; }
}

[ProtoContract(SkipConstructor = true), ProtoInclude(101, typeof(RoutingNode)), ProtoInclude(102, typeof(LeafNode))]
public abstract class TreeNode
{
    [ProtoMember(1)]
    public byte[] Hash { get; set; }

    [ProtoMember(2)]
    public uint Id { get; set; }

    public TreeNode() { }
}

[ProtoContract(SkipConstructor = true)]
public class RoutingNode : TreeNode
{
    public RoutingNode() { }

    static Random r = new Random();

    [ProtoMember(1, DataFormat = DataFormat.Group)]
    private List<TreeNode> _subTree = new List<TreeNode>();

    public RoutingNode Parent { get; set; }

    public void AddNode(TreeNode node)
    {
        if (_subTree.Count < 50 || node is LeafNode)
        {
            _subTree.Add(node);
        }
        else
        {
            ((RoutingNode)_subTree[r.Next(_subTree.Count - 1)]).AddNode(node);
        }

    }

    [ProtoAfterDeserialization]
    protected void OnDeserialized()
    {
        if (_subTree == null) _subTree = new List<TreeNode>();
        foreach (TreeNode node in _subTree)
        {
            if (node is RoutingNode)
            {
                (node as RoutingNode).Parent = this;
            }
        }
    }
}

[ProtoContract(SkipConstructor = true)]
public class LeafNode : TreeNode
{
    public LeafNode() { }

    [ProtoMember(1)]
    public int[] SomeValues { get; set; }
}

示例程序

var tree = new Tree();

        tree.Root = new RoutingNode();

        var r = new Random();

        for (uint i = 0; i < 5000000; i++)
        {
            byte[] hash = new byte[200];
            r.NextBytes(hash);
            tree.Root.AddNode(new RoutingNode() { Hash = hash, Id = i });
        }

        for (uint j = 0; j < 5000; j++)
        {
            byte[] hash = new byte[200];
            r.NextBytes(hash);

            var node = new RoutingNode() { Hash = hash, Id = j };
            for (uint i = 0; i < 50; i++)
            {
                hash = new byte[200];
                r.NextBytes(hash);
                node.AddNode(new LeafNode() { Hash = hash, Id = i });                    
            }
            tree.Root.AddNode(node);
        }

        using (var stream = File.OpenWrite("test.tmp"))
            ProtoBuf.Serializer.Serialize(stream, tree);

序列化大对象图导致protobuf-net无限循环

示例程序

0 个答案: