Question

我一直在尝试找到一个简单的示例，其中在我的进程池中启动的每个进程共享一个常量。大多数示例向您展示了如何跨进程共享变量，这不是我想要的。

import multiprocessing
import time

data = (
    {"var":1, "shared": None}, {"var":2, "shared": None}, {"var":3, "shared": None}, {"var":4, "shared": None}
)

def mp_worker(input):
    print input
    # print " Processs %s\tWaiting %s seconds" % (inputs, the_time)
    # time.sleep(int(the_time))
    # print " Process %s\tDONE" % inputs

def mp_handler():
    p = multiprocessing.Pool(2)
    p.map(mp_worker, data)

if __name__ == '__main__':
    mp_handler()

例如，如果我运行此代码，则希望对每个进程初始化一次我的“共享”组件。

我想做这样的事情（这不起作用）：

from multiprocessing import Pool, Process

class Worker(Process):
    def __init__(self):
        print 'Worker started'
        # do some initialization here
        super(Worker, self).__init__()

    def compute(self, data):
        print 'Computing things!'
        return data * data


if __name__ == '__main__':
    # This works fine
    worker = Worker()
    #print worker.compute(3)

    # workers get initialized fine
    pool = Pool(processes = 4,
                initializer = Worker)
    data = range(10)
    # How to use my worker pool?
    # result = pool.map(Worker.compute, data)

    result = pool.map(Worker.compute, data)

Answer 1

使用共享的c_types：

from multiprocessing import Process, Lock
from multiprocessing.sharedctypes import Value
from ctypes import Structure, c_double

class Point(Structure):
    _fields_ = [('x', c_double), ('y', c_double)]

def modify(parmMap):
    parmMap['point'].x = parmMap['var']
    parmMap['point'].y = parmMap['var'] * 2


if __name__ == '__main__':
    lock = Lock()

    data = ( {'var' : 1, 'shared' : Value(Point, (0,0), lock=lock) },
             {'var' : 2, 'shared' : Value(Point, (0,0), lock=lock) },
             {'var' : 3, 'shared' : Value(Point, (0,0), lock=lock) },
             {'var' : 4, 'shared' : Value(Point, (0,0), lock=lock) }
    )

    p = multiprocessing.Pool(2)
    print p.map(mp_worker, data)
    print data

Answer 2

def init(args, num_gpu):
    pid = int(str(multiprocessing.current_process()).split(" ")[0].split("-")[-1].split(",")[0]) - 1
    gpu_id = pid % num_gpu
    global testModule
    testModule = TestModuleShared(args, gpu_id)

def worker(datum):
    pid = int(str(multiprocessing.current_process()).split(" ")[0].split("-")[-1].split(",")[0]) - 1
    params = datum["params"]
    # print str(datum["fc"]) + " " + str(pid) 
    # print testModule.openpose

    # Reset State
    testModule.run()

p = multiprocessing.Pool(per_gpu_threads*num_gpu, initializer=init, initargs=(params["test_module_param"],num_gpu,))

事实证明，您只可以使用global变量关键字以及初始化程序回调对其进行初始化。

Python多进程池-每个进程共享一个变量吗？

2 个答案: