pathos多进程无法写入共享内存

时间：2019-12-18 23:48:53

标签： python multiprocessing pathos

我正在尝试将代码（from this example）更改为使用pathos多进程而不是多进程。示例代码运行良好。用多进程替换多进程无法更改存储在共享内存中的数组，并引发断言错误。下面是经过修改的代码。

import ctypes
import logging
import multiprocess as mp

from contextlib import closing

import numpy as np

info = mp.get_logger().info

def main():
    logger = mp.log_to_stderr()
    logger.setLevel(logging.INFO)

    # create shared array
    N, M = 100, 11
    shared_arr = mp.Array(ctypes.c_double, N)
    arr = tonumpyarray(shared_arr)

    # fill with random values
    arr[:] = np.random.uniform(size=N)
    print(arr)
    arr_orig = arr.copy()

    # write to arr from different processes
    with closing(mp.Pool(initializer=init, initargs=(shared_arr,))) as p:
        # many processes access the same slice
        stop_f = N // 10
        p.map_async(f, [slice(stop_f)]*M)

        # many processes access different slices of the same array
        assert M % 2 # odd
        step = N // 10
        p.map_async(g, [slice(i, i + step) for i in range(stop_f, N, step)])
    p.join()
    print(arr)
    assert np.allclose(((-1)**M)*tonumpyarray(shared_arr), arr_orig)

def init(shared_arr_):
    global shared_arr
    shared_arr = shared_arr_ # must be inherited, not passed as an argument

def tonumpyarray(mp_arr):
    return np.frombuffer(mp_arr.get_obj())

def f(i):
    """synchronized."""
    with shared_arr.get_lock(): # synchronize access
        g(i)

def g(i):
    """no synchronization."""
    info("start %s" % (i,))
    arr = tonumpyarray(shared_arr)
    arr[i] = -1 * arr[i]
    info("end   %s" % (i,))

if __name__ == '__main__':
    mp.freeze_support()
    main()

以下是脚本的输出，该脚本演示了不良行为（共享数组未更改）。

[INFO/MainProcess] allocating a new mmap of length 4096
[7.53982236e-01 8.91817533e-01 3.42952411e-01 4.81562566e-01
 8.11524911e-01 9.39701276e-01 2.49035240e-01 9.81663723e-01
 8.08235711e-01 1.42681227e-01 3.45497140e-01 8.90562288e-01
 4.35241873e-01 4.60578813e-01 3.46264261e-01 4.93318426e-01
 3.22399114e-01 7.83147144e-01 9.47066783e-02 5.18534236e-01
 8.51086344e-01 6.27441844e-01 1.06383034e-01 2.38705882e-01
 7.97136179e-01 7.86004956e-01 8.66374084e-01 5.17888509e-01
 8.17624705e-01 9.49644794e-01 6.31860201e-01 5.36854341e-01
 3.04143232e-01 3.72530617e-01 2.30401250e-01 8.07846412e-02
 9.07925558e-01 2.83202034e-01 2.29512224e-01 1.10851201e-01
 7.72062029e-01 3.60300517e-01 5.54688922e-04 5.65488687e-01
 5.62561777e-01 6.34814941e-01 5.96339057e-01 9.79598924e-01
 2.09899979e-01 7.15104955e-01 6.55292670e-01 1.68393973e-01
 9.72467624e-01 8.35882816e-01 7.43016913e-01 4.94503316e-01
 4.64440104e-01 9.19258107e-01 8.77918280e-01 3.50650473e-02
 1.78977134e-02 8.37892866e-01 3.97096237e-01 3.98471290e-01
 1.50104963e-01 6.85465476e-01 8.44218457e-01 5.62325961e-01
 1.10385518e-01 3.60357701e-01 3.57917840e-01 8.63808991e-01
 8.39141018e-01 1.79879196e-01 8.02565416e-01 6.78710916e-01
 9.40750906e-01 6.95072638e-01 4.07945520e-01 8.04220531e-01
 9.36671866e-01 8.04467890e-01 2.71010169e-01 6.12931552e-02
 9.13496687e-01 9.94252838e-03 6.29349539e-01 7.82656394e-02
 6.58311027e-01 9.18628747e-01 9.21899148e-01 6.34954946e-03
 9.87928873e-01 2.35952899e-01 6.93470029e-01 9.41543773e-01
 2.27509027e-01 7.76581113e-01 5.76203661e-01 7.70004310e-01]
[INFO/MainProcess] child process calling self.run()
[INFO/MainProcess] child process calling self.run()
[INFO/MainProcess] child process calling self.run()
[INFO/MainProcess] child process calling self.run()
[INFO/MainProcess] child process calling self.run()
[INFO/MainProcess] child process calling self.run()
[INFO/MainProcess] child process calling self.run()
[INFO/MainProcess] child process calling self.run()
[INFO/MainProcess] child process calling self.run()
[INFO/MainProcess] child process calling self.run()
[INFO/MainProcess] child process calling self.run()
[INFO/MainProcess] process shutting down
[INFO/MainProcess] process exiting with exitcode 0
[INFO/MainProcess] process shutting down
[INFO/MainProcess] process exiting with exitcode 0
[INFO/MainProcess] process shutting down
[INFO/MainProcess] process shutting down
[INFO/MainProcess] process exiting with exitcode 0
[INFO/MainProcess] process shutting down
[INFO/MainProcess] process exiting with exitcode 0
[INFO/MainProcess] process shutting down
[INFO/MainProcess] process exiting with exitcode 0
[INFO/MainProcess] process shutting down
[INFO/MainProcess] process exiting with exitcode 0
[INFO/MainProcess] process exiting with exitcode 0
[INFO/MainProcess] process shutting down
[INFO/MainProcess] process exiting with exitcode 0
[INFO/MainProcess] process shutting down
[INFO/MainProcess] process exiting with exitcode 0
[INFO/MainProcess] process shutting down
[INFO/MainProcess] process exiting with exitcode 0
[INFO/MainProcess] process shutting down
[INFO/MainProcess] process exiting with exitcode 0
[INFO/MainProcess] child process calling self.run()
[INFO/MainProcess] process shutting down
[INFO/MainProcess] process exiting with exitcode 0
[7.53982236e-01 8.91817533e-01 3.42952411e-01 4.81562566e-01
 8.11524911e-01 9.39701276e-01 2.49035240e-01 9.81663723e-01
 8.08235711e-01 1.42681227e-01 3.45497140e-01 8.90562288e-01
 4.35241873e-01 4.60578813e-01 3.46264261e-01 4.93318426e-01
 3.22399114e-01 7.83147144e-01 9.47066783e-02 5.18534236e-01
 8.51086344e-01 6.27441844e-01 1.06383034e-01 2.38705882e-01
 7.97136179e-01 7.86004956e-01 8.66374084e-01 5.17888509e-01
 8.17624705e-01 9.49644794e-01 6.31860201e-01 5.36854341e-01
 3.04143232e-01 3.72530617e-01 2.30401250e-01 8.07846412e-02
 9.07925558e-01 2.83202034e-01 2.29512224e-01 1.10851201e-01
 7.72062029e-01 3.60300517e-01 5.54688922e-04 5.65488687e-01
 5.62561777e-01 6.34814941e-01 5.96339057e-01 9.79598924e-01
 2.09899979e-01 7.15104955e-01 6.55292670e-01 1.68393973e-01
 9.72467624e-01 8.35882816e-01 7.43016913e-01 4.94503316e-01
 4.64440104e-01 9.19258107e-01 8.77918280e-01 3.50650473e-02
 1.78977134e-02 8.37892866e-01 3.97096237e-01 3.98471290e-01
 1.50104963e-01 6.85465476e-01 8.44218457e-01 5.62325961e-01
 1.10385518e-01 3.60357701e-01 3.57917840e-01 8.63808991e-01
 8.39141018e-01 1.79879196e-01 8.02565416e-01 6.78710916e-01
 9.40750906e-01 6.95072638e-01 4.07945520e-01 8.04220531e-01
 9.36671866e-01 8.04467890e-01 2.71010169e-01 6.12931552e-02
 9.13496687e-01 9.94252838e-03 6.29349539e-01 7.82656394e-02
 6.58311027e-01 9.18628747e-01 9.21899148e-01 6.34954946e-03
 9.87928873e-01 2.35952899e-01 6.93470029e-01 9.41543773e-01
 2.27509027e-01 7.76581113e-01 5.76203661e-01 7.70004310e-01]

这是完整的追溯。

Traceback (most recent call last):
  File "C:\Users\Fernando\AppData\Local\Continuum\anaconda3\envs\fountaintx\lib\site-packages\IPython\core\interactiveshell.py", line 3326, in run_code
    exec(code_obj, self.user_global_ns, self.user_ns)
  File "<ipython-input-2-4f9235b548aa>", line 1, in <module>
    runfile('C:/Users/Fernando/.PyCharm2019.2/config/scratches/scratch_1.py', wdir='C:/Users/Fernando/.PyCharm2019.2/config/scratches')
  File "C:\Program Files\JetBrains\PyCharm Professional Edition with Anaconda plugin 2019.2.4\helpers\pydev\_pydev_bundle\pydev_umd.py", line 197, in runfile
    pydev_imports.execfile(filename, global_vars, local_vars)  # execute the script
  File "C:\Program Files\JetBrains\PyCharm Professional Edition with Anaconda plugin 2019.2.4\helpers\pydev\_pydev_imps\_pydev_execfile.py", line 18, in execfile
    exec(compile(contents+"\n", file, 'exec'), glob, loc)
  File "C:/Users/Fernando/.PyCharm2019.2/config/scratches/scratch_1.py", line 60, in <module>
    main()
  File "C:/Users/Fernando/.PyCharm2019.2/config/scratches/scratch_1.py", line 37, in main
    assert np.allclose(((-1)**M)*tonumpyarray(shared_arr), arr_orig)
AssertionError

0 个答案:

没有答案
相关问题
最新问题