Question

我正在尝试开发一个动态使用multiprocessing模块的包装器。我在各种模块中有许多功能需要妥善管理。我需要能够将一个源自任何模块的函数及其参数传递给我的包装器。数据和函数在运行时都不会知道，因为它取决于用户。

这是我想要做的一个例子：

import sys
from multiprocessing import Process, Queue, cpu_count

def dyn():
    pass

class mp():
    def __init__(self, data, func, n_procs = cpu_count()):
        self.data    = data
        self.q       = Queue()
        self.n_procs = n_procs

        # Replace module-level 'dyn' function with the provided function
        setattr(sys.modules[__name__], 'dyn', func)
        # Calling dyn(...) at this point will produce the same output as
        # calling func(...)

    def _worker(self, *items):
        data = []
        for item in items:
            data.append(dyn(item))
        self.q.put(data)

    def compute(self):
        for item in self.data:
            Process(target=getattr(self, '_worker'), args=item).start()

    def items(self):
        queue_count = self.n_procs
        while queue_count > 0:
            queue_count -= 1
            yield self.q.get()

if __name__ == '__main__':  
    def work(x):
        return x ** 2

    # Create workers
    data = [range(10)] * cpu_count()
    workers = mp(data, work)

    # Make the workers work
    workers.compute()

    # Get the data from the workers
    workers_data = []
    for item in workers.items():
        workers_data.append(item)
    print workers_data

对于此示例，输出应采用以下格式：

[[0, 1, 4, 9, 16, 25, 36, 49, 64, 81] * n_procs]

如果您尝试运行此代码，您将收到一个异常，说明有太多参数传递给dyn。我认为问题是dyn被覆盖了这个实例，但是当调用Process时，更改不再存在。

如何解决这个问题？

注意 - 此代码需要在Windows上运行。我使用的是Python 2.7。

更新

根据我得到的评论，我决定做一些“凌乱”的事情。以下是我的工作解决方案：

import sys, re, uuid, os
from cStringIO import StringIO
from multiprocessing import Process, Queue, cpu_count

class mp():
    def __init__(self, data, func, n_procs = cpu_count()):
        self.data    = data
        self.q       = Queue()
        self.n_procs = n_procs
        self.module  = 'm' + str(uuid.uuid1()).replace('-', '')
        self.file    = self.module + '.py'

        # Build external module
        self.__func_to_module(func)

    def __func_to_module(self, func):   
        with open(self.file, 'wb') as f:
            for line in StringIO(func):
                if 'def' in line:
                    f.write(re.sub(r'def .*\(', 'def work(', line))
                else:
                    f.write(line)

    def _worker(self, q, module, *items):
        exec('from %s import work' % module)
        data = []
        for item in items[0]:
            data.append(work(item))
        q.put(data)

    def compute(self):
        for item in self.data:
            Process(target=getattr(self, '_worker'),
                args=(self.q, self.module, item)).start()

    def items(self):
        queue_count = self.n_procs
        while queue_count > 0:
            queue_count -= 1
            yield self.q.get()
        os.remove(self.file)
        os.remove(self.file + 'c')

if __name__ == '__main__':  
    func = '''def func(x):
        return x ** 2'''

    # Create workers
    data = [range(10)] * cpu_count()
    workers = mp(data, func)

    # Make the workers work
    workers.compute()

    # Get the data from the workers
    workers_data = []
    for item in workers.items():
        workers_data.append(item)
    print workers_data

Answer 1

在Windows上，每个新进程启动时都会重新加载模块，因此dyn的定义会丢失。但是，您可以通过队列或通过参数传递进程的目标函数。

def _worker(*items, func=None, q=None):  
    #note that this had better be a function not a method
    data = []
    for item in items:
        data.append(func(item))
    q.put(data)

#...
Process(target=_worker, args=item, kwargs={'func':dyn, 'q':q})

Python - 多处理模块的动态使用

1 个答案: