Question

我在网上找到了一个非常复杂的函数。我确信该函数本身使用了python多处理软件包。

现在我有很多系列数据，我也想使用该功能进行多处理。

以下是该功能的简化版本：

import multiprocessing
import numpy as np
def main(number=10):
    print('ok')
    def segment():
        result=[]
        error='meet bad case'
        for i in range(number):
            result.append(i*np.random.rand())
            if result[-1] ==number/2:
                raise AssertionError(error)
        json.dump(result, codecs.open(os.getcwd()+'/data/result'+str(number), \
                'w', encoding='utf-8'), separators=(',', ':'), sort_keys=True, indent=4) 
    p = multiprocessing.Process(target=segment)
    p.daemon = True
    p.start()
    p.join()

这个主要功能代表我下载的复杂功能。现在，如何使用parellel多处理技术将此函数应用于10到100的可变数字范围？

Answer 1

#!/usr/bin/env python
# -*- coding: UTF-8 -*-

import multiprocessing
# We must import this explicitly, it is not imported by the top-level
# multiprocessing module.
import multiprocessing.pool
import time

from random import randint


class NoDaemonProcess(multiprocessing.Process):
    # make 'daemon' attribute always return False
    def _get_daemon(self):
        return False
    def _set_daemon(self, value):
        pass
    daemon = property(_get_daemon, _set_daemon)

# We sub-class multiprocessing.pool.Pool instead of multiprocessing.Pool
# because the latter is only a wrapper function, not a proper class.
class MyPool(multiprocessing.pool.Pool):
    Process = NoDaemonProcess
def main(number):

    def segment():
        result=[]
        error='meet bad case'
        for i in range(number):
            result.append(i*np.random.rand())
            if result[-1] ==number/2:
                raise AssertionError(error)
        json.dump(result, codecs.open(os.getcwd()+'/data/result'+str(number), \
                'w', encoding='utf-8'), separators=(',', ':'), sort_keys=True, indent=4) 
    p = multiprocessing.Process(target=segment)
    p.start()
    p.join() 
    print('ok')

def test():
    print("Creating 5 (non-daemon) workers and jobs in main process.")
    pool = MyPool(5)

    pool.map(main, [5,6,7])

    pool.close()
    pool.join()

if __name__ == '__main__':
    test()

如何对包含多处理的函数进行多处理？

1 个答案: