无法使用pickle序列化包含SWIG的C ++类

时间:2019-05-15 23:44:30

标签: python c++ pickle swig built-in

出于日常工作的需要,我必须用SWIG包装DSO中的一些C ++类。在出于性能原因决定使用Python缓冲区协议之前,一切都很好,这使我使用了SWIG工具的-builtin标志。我设法使事情正常进行,但是当我尝试使用包装好的咸菜序列化/反序列化时,出现以下错误:

Traceback (most recent call last):
  File "test.py", line 23, in <module>
    s = pickle.dumps(a) # crash here
_pickle.PicklingError: Can't pickle <class 'mymodule.A'>: attribute lookup A on imp failed

这是揭露该问题的MWE。

首先,SWIG接口文件mymodule.i包装一个内联的C ++类A。

// File mymodule.i
%module mymodule

%feature("python:tp_str")  A "A_str" ;
%feature("python:tp_repr") A "A_str" ;
%{
#include <iostream>
#include <sstream>

// forward declarations
class A ;
static std::ostream & operator << (std::ostream &, const A &) ;
static int  A_getI(const A &) ;
static void A_setI(A &, int) ;

class A
{
  int i_ ;
public:
  A(int i) : i_(i) {}
  friend std::ostream & operator << (std::ostream &, const A &) ;
  friend int  A_getI(const A &) ;
  friend void A_setI(A &, int) ;
} ;

std::ostream & operator << (std::ostream & os, const A & a)
{
  return os << "A(" << a.i_ << ")" ;
}

int A_getI(const A & a)
{
  return a.i_ ;
}

void A_setI(A & a, int i)
{
  a.i_ = i ;
}

static PyObject * A_str(SwigPyObject * obj)
{
  A * p_a = reinterpret_cast<A *>(obj->ptr) ;
  std::ostringstream oss ;
  oss << *p_a ;
  return PyUnicode_FromString(oss.str().c_str()) ;
}

%}

class A
{
public:
  A(int i) ;
} ;

%extend A
{
  PyObject * __getstate__()
  {
    PyObject * tuple = PyTuple_New(1) ;
    PyTuple_SetItem(tuple, 0, PyLong_FromLong(A_getI(*self))) ;
    return tuple ;
  }
  void __setstate__(PyObject * state)
  {
    if (PyTuple_Check(state)) {
      PyObject * val = PyTuple_GetItem(state, 0) ;
      if (PyLong_Check(val)) {
        int i = (int) PyLong_AsLong(val) ;
        A_setI(*self, i) ;
      }
    }
  }
}

然后,我使用的test.py

#!/usr/bin/env python3
# File: test.py

from mymodule import *

# a first object a
a = A(10)
print("a=",a)

# we manually get the internal state of a
sa = a.__getstate__()
print("sa=",sa)

# a second object b with a different internal state
b = A(-1)
print("b=",b)

# we manually set a's internal state to b
b.__setstate__(sa)
print("b=",b)

# now we try to dump a's state using pickle...
import pickle
s = pickle.dumps(a) # crash here
print(s)

我编译模块:

swig3.0 -python -py3 -c++ -builtin -o mymodulePYTHON_wrap.cxx mymodule.i
g++ -D_mymodule_EXPORTS -DSWIG_REV="" -fPIC -I/usr/include/python3.4m -std=c++11 -o mymodulePYTHON_wrap.cxx.o -c mymodulePYTHON_wrap.cxx
g++ -fPIC -shared -Wl,-soname,_mymodule.so -o _mymodule.so mymodulePYTHON_wrap.cxx.o

我运行测试:

python3 test.py

我得到以下输出:

a= A(10)
sa= (10,)
b= A(-1)
b= A(10)
Traceback (most recent call last):
  File "test.py", line 23, in <module>
    s = pickle.dumps(a) # crash here
_pickle.PicklingError: Can't pickle <class 'mymodule.A'>: attribute lookup A on imp failed

错误原因是什么?

我该如何解决我的问题?

我也尝试过:

  • 阅读SWIG和Python文档
  • 搜索网络
  • 在SO上阅读了与该问题有关的大多数文章,但没有更好的知识(更接近我的问题似乎是Serialize SWIG extension with dill,只是Mark McKems的回答使我无望,What's the exact usage of __reduce__ in Pickler,{{ 3}})
  • 使用__reduce__方法返回预期的结果(有效),但会发生相同的错误
  • 使用莳萝代替泡菜
  • 将协议更改为最高协议

我的工具版本很旧,因为我必须使用Debian jessie:

  • Python 3.4.2
  • G ++ 4.9.2
  • SWIG 3.0.2

我被困住了。任何帮助将不胜感激。

编辑

SWIG生成的mymodule.py文件(带有-builtin标志):

# This file was automatically generated by SWIG (http://www.swig.org).
# Version 3.0.2
#
# Do not make changes to this file unless you know what you are doing--modify
# the SWIG interface file instead.





from sys import version_info
if version_info >= (2,6,0):
    def swig_import_helper():
        from os.path import dirname
        import imp
        fp = None
        try:
            fp, pathname, description = imp.find_module('_mymodule', [dirname(__file__)])
        except ImportError:
            import _mymodule
            return _mymodule
        if fp is not None:
            try:
                _mod = imp.load_module('_mymodule', fp, pathname, description)
            finally:
                fp.close()
            return _mod
    _mymodule = swig_import_helper()
    del swig_import_helper
else:
    import _mymodule
del version_info
from _mymodule import *
try:
    _swig_property = property
except NameError:
    pass # Python < 2.2 doesn't have 'property'.
def _swig_setattr_nondynamic(self,class_type,name,value,static=1):
    if (name == "thisown"): return self.this.own(value)
    if (name == "this"):
        if type(value).__name__ == 'SwigPyObject':
            self.__dict__[name] = value
            return
    method = class_type.__swig_setmethods__.get(name,None)
    if method: return method(self,value)
    if (not static):
        self.__dict__[name] = value
    else:
        raise AttributeError("You cannot add attributes to %s" % self)

def _swig_setattr(self,class_type,name,value):
    return _swig_setattr_nondynamic(self,class_type,name,value,0)

def _swig_getattr(self,class_type,name):
    if (name == "thisown"): return self.this.own()
    method = class_type.__swig_getmethods__.get(name,None)
    if method: return method(self)
    raise AttributeError(name)

def _swig_repr(self):
    try: strthis = "proxy of " + self.this.__repr__()
    except: strthis = ""
    return "<%s.%s; %s >" % (self.__class__.__module__, self.__class__.__name__, strthis,)

try:
    _object = object
    _newclass = 1
except AttributeError:
    class _object : pass
    _newclass = 0



# This file is compatible with both classic and new-style classes.

没有mymodule.py SWIG标志的-builtin文件:

# This file was automatically generated by SWIG (http://www.swig.org).
# Version 3.0.2
#
# Do not make changes to this file unless you know what you are doing--modify
# the SWIG interface file instead.





from sys import version_info
if version_info >= (2,6,0):
    def swig_import_helper():
        from os.path import dirname
        import imp
        fp = None
        try:
            fp, pathname, description = imp.find_module('_mymodule', [dirname(__file__)])
        except ImportError:
            import _mymodule
            return _mymodule
        if fp is not None:
            try:
                _mod = imp.load_module('_mymodule', fp, pathname, description)
            finally:
                fp.close()
            return _mod
    _mymodule = swig_import_helper()
    del swig_import_helper
else:
    import _mymodule
del version_info
try:
    _swig_property = property
except NameError:
    pass # Python < 2.2 doesn't have 'property'.
def _swig_setattr_nondynamic(self,class_type,name,value,static=1):
    if (name == "thisown"): return self.this.own(value)
    if (name == "this"):
        if type(value).__name__ == 'SwigPyObject':
            self.__dict__[name] = value
            return
    method = class_type.__swig_setmethods__.get(name,None)
    if method: return method(self,value)
    if (not static):
        self.__dict__[name] = value
    else:
        raise AttributeError("You cannot add attributes to %s" % self)

def _swig_setattr(self,class_type,name,value):
    return _swig_setattr_nondynamic(self,class_type,name,value,0)

def _swig_getattr(self,class_type,name):
    if (name == "thisown"): return self.this.own()
    method = class_type.__swig_getmethods__.get(name,None)
    if method: return method(self)
    raise AttributeError(name)

def _swig_repr(self):
    try: strthis = "proxy of " + self.this.__repr__()
    except: strthis = ""
    return "<%s.%s; %s >" % (self.__class__.__module__, self.__class__.__name__, strthis,)

try:
    _object = object
    _newclass = 1
except AttributeError:
    class _object : pass
    _newclass = 0


class A(_object):
    __swig_setmethods__ = {}
    __setattr__ = lambda self, name, value: _swig_setattr(self, A, name, value)
    __swig_getmethods__ = {}
    __getattr__ = lambda self, name: _swig_getattr(self, A, name)
    __repr__ = _swig_repr
    def __init__(self, *args): 
        this = _mymodule.new_A(*args)
        try: self.this.append(this)
        except: self.this = this
    def __getstate__(self) -> "PyObject *" : return _mymodule.A___getstate__(self)
    def __setstate__(self, *args) -> "void" : return _mymodule.A___setstate__(self, *args)
    __swig_destroy__ = _mymodule.delete_A
    __del__ = lambda self : None;
A_swigregister = _mymodule.A_swigregister
A_swigregister(A)

# This file is compatible with both classic and new-style classes.

0 个答案:

没有答案