多处理需要更多时间,然后没有多处理 - Python

时间:2014-01-15 11:38:17

标签: python time multiprocessing

我在一个科学项目上工作,我有一个方法,需要花费很多时间来终止,并且调用超过20次。该方法也可以很容易地并行化。问题是并行化代码比非并行化代码花费的时间要多得多(在代码中注释)。

这是我的一段代码,只是为了展示我是如何做这件事的:

import copy_reg
import types
from itertools import product
import multiprocessing as mp

def _pickle_method(method):
    """
    Author: Steven Bethard (author of argparse)
    http://bytes.com/topic/python/answers/552476-why-cant-you-pickle-instancemethods
    """
    func_name = method.im_func.__name__
    obj = method.im_self
    cls = method.im_class
    cls_name = ''
    if func_name.startswith('__') and not func_name.endswith('__'):
        cls_name = cls.__name__.lstrip('_')
    if cls_name:
        func_name = '_' + cls_name + func_name
    return _unpickle_method, (func_name, obj, cls)


def _unpickle_method(func_name, obj, cls):
    """
    Author: Steven Bethard
    http://bytes.com/topic/python/answers/552476-why-cant-you-pickle-instancemethods
    """
    for cls in cls.mro():
        try:
            func = cls.__dict__[func_name]
        except KeyError:
            pass
        else:
            break
    return func.__get__(obj, cls)

copy_reg.pickle(types.MethodType, _pickle_method, _unpickle_method)

class ImageData(object):

    def __init__(self, width=60, height=60):
        self.width = width
        self.height = height
        self.data = []
        for i in range(width):
            self.data.append([0] * height)      

def parallel_orientation_uncertainty_calculus(x, y, mean_gradient, mean_gradient_direction, gradient_covariance,
                                               gradient_correlation, bins):
    v = mean_gradient_direction.data[x][y]
    theta_sigma = Uts.Utils.translate_to_polar_coordinates(v[0].item(0), v[1].item(0))
    sigma_theta = 0.0
    for i in range(bins):
        n1 = mt.pow(-mt.pi / 2 + mt.pi * i / bins, 2)
        n2 = VariabilityOfGradients.calculate_gradient_orientation_probability_density_function(
            mean_gradient, gradient_covariance, gradient_correlation, x, y,
            (theta_sigma - mt.pi / 2 + mt.pi * i / bins))
        sigma_theta += n1 * n2
    return [x, y, sigma_theta]

class VariabilityOfGradients(object):
    parallel_orientation_uncertainty_calculus = staticmethod(parallel_orientation_uncertainty_calculus)

    @staticmethod
    def calculate_orientation_uncertainty(mean_gradient, mean_gradient_direction, gradient_covariance, gradient_correlation, bins):
        output = ImD.ImageData(range_min=0, range_max=1)

        results = []
        pool = Pool()
        for x, y in product(range(1, output.width - 1), range(1, output.height - 1)):
            print "Iteration ", x, y
            result = pool.apply_async(VariabilityOfGradients.parallel_orientation_uncertainty_calculus,
                                  args=[x, y, mean_gradient, mean_gradient_direction, gradient_covariance,
                                        gradient_correlation, bins])
            results.append(result.get())
        pool.close()
        pool.join()        
        for i, result in enumerate(results):
            result = results[i]
            print result
            output.data[result[0], result[1]] = result[2]

        # for x, y in product(range(1, output.width - 1), range(1, output.height - 1)):
        #     print "Iteration ", x, y
        #     v = mean_gradient_direction.data[x][y]
        #     theta_sigma = Uts.Utils.translate_to_polar_coordinates(v[0].item(0), v[1].item(0))
        #     sigma_theta = 0.0
        #     for i in range(bins):
        #         n1 = mt.pow(-mt.pi / 2 + mt.pi * i / bins, 2)
        #         n2 = VariabilityOfGradients.calculate_gradient_orientation_probability_density_function(
                mean_gradient, gradient_covariance, gradient_correlation, x, y,
                (theta_sigma - mt.pi / 2 + mt.pi * i / bins))
        #         sigma_theta += n1 * n2
        #     output.data[x][y] = sigma_theta

        return output    

if __name__ == '__main__':  
    VariabilityOfGradients.calculate_orientation_uncertainty()  

我想知道自己做错了什么。我使用多处理错了吗?

提前谢谢。

0 个答案:

没有答案