我在代码的混乱配置中完全使用multiprocessing。我决定给我的代码命令并重新编写为类，然后我可以轻松更改输入，我的新代码如下：
class LikelihoodTest:
def __init__(self,Xgal,Ygal):
self.x=Xgal
self.y=Ygal
self.objPosition=gal_pos
self.beta_s=beta
self.RhoCrit_SigmaC=rho_c_over_sigma_c
self.AngularDiameter=DA
self.RhoCrit=rho_crit
self.Reducedshear=observed_g
self.ShearError=g_err
#The 2D function
def like2d(self,posx, posy):
stuff=[self.objPosition, self.beta_s, self.RhoCrit_SigmaC , self.AngularDiameter, self.RhoCrit]
m=4.447e14
c=7.16
param=[posx, posy, m, c]
return reduced_shear( param, stuff, self.Reducedshear, self.ShearError)
def ShearLikelihood(self):
n=len(self.x)
m=len(self.y)
shared_array_base = multiprocessing.Array(ctypes.c_double, n*m)
shared_array = np.ctypeslib.as_array(shared_array_base.get_obj())
shared_array = shared_array.reshape( n,m)
#Restructure the function before you create instance of Pool.
# Parallel processing
def my_func(self,i, def_param=shared_array):
shared_array[i,:] = np.array([float(self.like2d(self.x[j],self.y[i])) for j in range(len(self.x))])
while True:
try:
print "processing to estimate likelihood in 2D grids......!!!"
start = time.time()
pool = multiprocessing.Pool(processes=10)
pool.map(my_func, range(len(self.y)))
print shared_array
end = time.time()
print "process time:\n",end - start
pool.close()
except ValueError:
print "Oops! value error!"
return shared_array
def plotLikelihood(self,shared_array):
#plotting on a mesh the likelihood function in order to see whether you have defined the inputs correctly and you can observe the maximum likelihood in 2D
# Set up a regular grid of interpolation points
xi, yi = np.linspace(self.x.min(), self.x.max(), 100), np.linspace(self.y.min(), self.y.max(), 100)
# Interpolate
rbf = scipy.interpolate.interp2d(self.x, self.y,shared_array , kind='linear')
zi = rbf(xi, yi)
fig, ax = plt.subplots()
divider = make_axes_locatable(ax)
im = ax.imshow(zi, vmin=shared_array.min(), vmax=shared_array.max(), origin='lower',
extent=[self.x.min(), self.x.max(), self.y.min(),self.y.max()])
ax.set_xlabel(r"$Xpos$")
ax.set_ylabel(r"$Ypos$")
ax.xaxis.set_label_position('top')
ax.xaxis.set_tick_params(labeltop='on')
cax = divider.append_axes("right", size="5%", pad=0.05)
cbar = fig.colorbar(im,cax=cax, ticks=list(np.linspace(shared_array.max(), shared_array.min(),20)),format='$%.2f$')
cbar.ax.tick_params(labelsize=8)
plt.savefig('/users/Desktop/MassRecons/Likelihood2d_XY_Without_Shear_Uncertainty.pdf', transparent=True, bbox_inches='tight', pad_inches=0)
plt.close()
当我尝试使用类配置运行它时出现以下错误：
if __name__ == '__main__':
Xgal = np.linspace(Xgalaxy.min(), Xgalaxy.max(), 1000)
Ygal = np.linspace(Ygalaxy.min(), Ygalaxy.max(), 1000)
Test=LikelihoodTest(Xgal,Ygal)
Test.ShearLikelihood()
processing to estimate likelihood in 2D grids......!!!
ERROR: PicklingError: Can't pickle <type 'function'>: attribute lookup __builtin__.function failed [multiprocessing.pool]
PicklingError: Can't pickle <type 'function'>: attribute lookup __builtin__.function failed
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
File "<stdin>", line 34, in ShearLikelihood
File "/vol/1/anaconda/lib/python2.7/multiprocessing/pool.py", line 251, in map
return self.map_async(func, iterable, chunksize).get()
File "/vol/1/anaconda/lib/python2.7/multiprocessing/pool.py", line 558, in get
raise self._value
cPickle.PicklingError: Can't pickle <type 'function'>: attribute lookup __builtin__.function failed
无论如何要解决它？
答案 0 :(得分：2)
我终于弄明白了如何在班上使用multiprocessing。我使用pathos.multiprocessing并将代码更改为：
import numpy as np
import pathos.multiprocessing as multiprocessing
class LikelihoodTest:
def __init__(self,Xgal,Ygal):
self.x=Xgal
self.y=Ygal
self.objPosition=gal_pos
self.beta_s=beta
self.RhoCrit_SigmaC=rho_c_over_sigma_c
self.AngularDiameter=DA
self.RhoCrit=rho_crit
self.Reducedshear=observed_g
self.ShearError=g_err
#The 2D function
def like2d(self,posx, posy):
stuff=[self.objPosition, self.beta_s, self.RhoCrit_SigmaC , self.AngularDiameter, self.RhoCrit]
m=4.447e14
c=7.16
param=[posx, posy, m, c]
return reduced_shear( param, stuff, self.Reducedshear, self.ShearError)
def ShearLikelihood(self,r):
return [float(self.like2d(self.x[j],r)) for j in range(len(self.x))]
def run(self):
try:
print "processing to estimate likelihood in 2D grids......!!!"
start = time.time()
pool = multiprocessing.Pool(processes=10)
seq=[ self.y[i] for i in range( self.y.shape[0])]
results=np.array( pool.map(self.ShearLikelihood, seq ))
end = time.time()
print "process time:\n",end - start
pool.close()
except ValueError:
print "Oops! value error ....!"
return results
def plotLikelihood(self,shared_array):
#plotting on a mesh the likelihood function in order to see whether you have defined the inputs correctly and you can observe the maximum likelihood in 2D
# Set up a regular grid of interpolation points
xi, yi = np.linspace(self.x.min(), self.x.max(), 100), np.linspace(self.y.min(), self.y.max(), 100)
# Interpolate
rbf = scipy.interpolate.interp2d(self.x, self.y,shared_array , kind='linear')
zi = rbf(xi, yi)
fig, ax = plt.subplots()
divider = make_axes_locatable(ax)
im = ax.imshow(zi, vmin=shared_array.min(), vmax=shared_array.max(), origin='lower',
extent=[self.x.min(), self.x.max(), self.y.min(),self.y.max()])
ax.set_xlabel(r"$Xpos$")
ax.set_ylabel(r"$Ypos$")
ax.xaxis.set_label_position('top')
ax.xaxis.set_tick_params(labeltop='on')
cax = divider.append_axes("right", size="5%", pad=0.05)
cbar = fig.colorbar(im,cax=cax, ticks=list(np.linspace(shared_array.max(), shared_array.min(),20)),format='$%.2f$')
cbar.ax.tick_params(labelsize=8)
plt.savefig('/users/Desktop/MassRecons/Likelihood2d_XY_coordinate.pdf', transparent=True, bbox_inches='tight', pad_inches=0)
plt.close()
if __name__ == '__main__':
Xgal = np.linspace(Xgalaxy.min(), Xgalaxy.max(), 1000)
Ygal = np.linspace(Ygalaxy.min(), Ygalaxy.max(), 1000)
Test=LikelihoodTest(Xgal,Ygal)
x=Test.run()
Test.plotLikelihood(x)
现在它的工作就像一个魅力！ ：）
答案 1 :(得分：-3)
您无法使用Pickle将函数或方法传递给不同的进程，但您可以传递字符串。
您可以保留方法字典并通过字符串键引用方法。这不是很优雅，但解决了这个问题。
修改 当您使用多处理时，会有一个隐含的＆＃34; fork＆＃34;。这会创建多个没有共享资源的独立进程，因此，传递给另一个进程的每个进程都必须使用Pickle进行序列化。问题是，pickle不允许序列化可执行代码以将其发送到另一个进程。