不明白:ValueError:操作数无法与形状一起广播

时间:2017-08-20 12:19:23

标签: python numpy python-multiprocessing

我有一个代码,使用锁定方案为每个“线程”创建进程并使用两个自己的数组进行操作:

if __name__ == "__main__":
    geom_file = sys.argv[1]
    filename = sys.argv[2]
    event = None
    resolution = 5814.0 
    dist = 0.2
    N_3D = 501  # num of points

    output_array = []

    pixm = cg.pixel_maps_from_geometry_file(geom_file) #getting pairs of coordinates in 3D
    x_array = pixm.x.flatten()
    y_array = pixm.y.flatten()
    z_array = np.ones_like(x_array) * dist

    with open(filename,'r') as stream:
        #reading stream file ang getting the necessary info
        lines = stream.readlines()
        output_array = get_opt_patterns(lines)

    I_xyz_buf = RawArray(ct.c_double, N_3D * N_3D * N_3D)
    count_dot_buf = RawArray(ct.c_double, N_3D * N_3D * N_3D)


    mask_buf = RawArray(ct.c_int, len(x_array))

    result_lock = Lock()
    mask_lock = Lock()  

    pool = Pool(os.cpu_count(),
                initializer=init_worker,
                initargs=(result_lock, mask_lock,
                          I_xyz_buf, count_dot_buf, mask_buf,
                          x_array, y_array, z_array)) 

    pool.map(processing, output_array)

    I_xyz = np.frombuffer(I_xyz_buf, dtype=np.double)
    count_dot = np.frombuffer(count_dot_buf, dtype=np.double)

运行后,它显示以下错误:

  

追踪(最近一次呼叫最后一次):

File "/opt/anaconda/3/lib/python3.5/multiprocessing/process.py", line 254, in _bootstrap
self.run()
File "/opt/anaconda/3/lib/python3.5/multiprocessing/process.py", line 93, in run
self._target(*self._args, **self._kwargs)
File "/opt/anaconda/3/lib/python3.5/multiprocessing/pool.py", line 103, in worker
initializer(*initargs)
File "processing.py", line 340, in init_worker
mask_arr = np.frombuffer(mask_buf, dtype=np.int) 
ValueError: buffer size must be a multiple of element size

创建工人:

def init_worker(res_lck, mask_lck,
            I_xyz_buf, count_dot_buf, mask_buf,
            x_arr, y_arr, z_arr):

    global result_lock
    global mask_lock

    global I_xyz_arr
    global count_dot_arr
    global mask_arr

    global x_array
    global y_array
    global z_array

    I_xyz_arr = np.frombuffer(I_xyz_buf, dtype=np.double)
    count_dot_arr = np.frombuffer(count_dot_buf, dtype=np.double)
    mask_arr = np.frombuffer(mask_buf, dtype=np.int)

    result_lock = res_lck
    mask_lock = mask_lck

    x_array = x_arr
    y_array = y_arr
    z_array = z_arr

负责此获取的功能:

def NEW_sphere_Evald(x_array, y_array, z_array, mask, Int, k):
    global resolution
    global N_3D

    global mask_lock

    len_array = len(x_array)

    qx = x_array / 2 + N_3D / 5
    qy = y_array * 3 - N_3D / 100
    qz = z_array / resolution

    with mask_lock:
        indexes = np.where((qx < N_3D) &
                           (qy < N_3D) &
                           (qz < N_3D) &
                           (mask != MASK_BAD))[0]

    coords = np.array((qx, qy, qz), dtype=np.int32)[:, indexes]
    coords_intens = Int[indexes]
    intensity = np.histogramdd(coords.T,
                               bins=[np.arange(0, N_3D + 1, 1),
                                     np.arange(0, N_3D + 1, 1),
                                     np.arange(0, N_3D + 1, 1)],
                               normed=False,
                               weights=coords_intens)[0]

    click_count = np.histogramdd(coords.T,
                                 bins=[np.arange(0, N_3D + 1, 1),
                                       np.arange(0, N_3D + 1, 1),
                                       np.arange(0, N_3D + 1, 1)])[0]

    return intensity, click_count


def get_opt_patterns(lines):
    ....
    # metadata = [name_of_file,energy,len_of_camera,[a,b,c,alpha,betta,gamma],[[a1*,a2*,a3*],[b1*,b2*,b3*],[c1*,c2*,c3*]]]
    return metadata


def processing(i):
    global result_lock
    global mask_lock

    global I_xyz_arr
    global count_dot_arr
    global mask_arr

    global x_array
    global y_array
    global z_array

    len_array = len(x_array)

    name_of_file = i[0]
    # print(name_of_file)
    f = h5.File(name_of_file, 'r')  # h5.File(sys.argv[1],'r')
    Int = f['data/data']
    Int = np.reshape(Int, len_array)

    index = Int >= 0

    with mask_lock:
        mask_arr[index] = MASK_GOOD

    k = i[1] * (1e10) / 12.4

    intensity, click_count = NEW_sphere_Evald(x_array, y_array, z_array, mask_arr, Int, k)  # higher

    with result_lock:
        I_xyz_arr += intensity
        count_dot_arr += click_count.astype(count_dot_arr.dtype)

我应该注意和修理什么?这是我第一次面对多进程的必要使用。

  

尝试:dtype="int32"

     

它解决了一个问题。但看起来是另一个:

line 119, in worker 
result = (True, func(*args, **kwds))  
File "/opt/anaconda/3/lib/python3.5/multiprocessing/pool.py", line 44, in mapstar  
return list(map(*args)) 
File "processing.py", line 319, in processing 
I_xyz_arr += intensity  
ValueError: operands could not be broadcast together with shapes 
    (125751501,) (501,501,501) (125751501,)

0 个答案:

没有答案