Question

我正在编写一个简单的python脚本，需要将其扩展到许多线程。为简单起见，我将需要使用的实际函数替换为矩阵乘法。我在使代码与处理器数量成比例时遇到麻烦。任何帮助我获得正确加速的建议都将有所帮助！我的代码和结果如下：

import numpy as np
import time
import math
from multiprocessing.dummy import Pool

res = 4

#we must iterate over all of these values
wavektests = np.linspace(.1,2.5,res)
omegaratios = np.linspace(.1,2.5,res)
wavekmat,omegamat = np.meshgrid(wavektests,omegaratios)

def solve_for_omegaratio( ind ):
    #obtain the indices for this run
    x_ind = ind % res
    y_ind = math.floor(ind / res)
    #obtain the value for this run
    wavek = wavektests[x_ind]
    omega = omegaratios[y_ind]
    #do some work ( I have replaced the real function with this)
    randmat = np.random.rand(4000,4000)
    nop = np.linalg.matrix_power(randmat,3)
    #obtain a scalar value
    value = x_ind + y_ind**2.0
    return value

list_ind = range(res**2)

#Serial code execution

t0_proc = time.clock()
t0_wall = time.time()

threads = 0
dispersion = map( solve_for_omegaratio , list_ind)
displist = list(dispersion)

t1_proc = time.clock()
t1_wall = time.time()


print('serial execution')
print('wall clock time = ',t1_wall-t0_wall)
print('processor clock time = ',t1_proc-t0_proc)
print('------------------------------------------------')



#Using pool defaults
t0_proc = time.clock()
t0_wall = time.time()

if __name__ == '__main__':
    pool = Pool()
    dispersion = pool.map( solve_for_omegaratio , list_ind)

displist = list(dispersion)
t1_proc = time.clock()
t1_wall = time.time()

pool.close
print('num of threads  = default')
print('wall clock time = ',t1_wall-t0_wall)
print('processor clock time = ',t1_proc-t0_proc)
print('------------------------------------------------')




# Using 4 threads
t0_proc = time.clock()
t0_wall = time.time()

threads = 4
if __name__ == '__main__':
    pool = Pool(threads)
    dispersion = pool.map( solve_for_omegaratio , list_ind)

displist = list(dispersion)
t1_proc = time.clock()
t1_wall = time.time()

pool.close
print('num of threads  = ' + str(threads))
print('wall clock time = ',t1_wall-t0_wall)
print('processor clock time = ',t1_proc-t0_proc)
print('------------------------------------------------')

结果：

serial execution
wall clock time =  66.1561758518219
processor clock time =  129.16376499999998
------------------------------------------------
num of threads  = default
wall clock time =  81.86436200141907
processor clock time =  263.45369
------------------------------------------------
num of threads  = 4
wall clock time =  77.63390111923218
processor clock time =  260.66285300000004
------------------------------------------------

Answer 1

因为python具有GIL https://wiki.python.org/moin/GlobalInterpreterLock，所以“ python-native”线程无法真正并发运行，因此无法提高CPU绑定任务（如数学）的性能。它们可用于有效地并行化与IO绑定的任务（例如，几乎花费所有时间等待网络I / O的API调用）。使用multiprocessing而不是dummy的线程支持的实现来分叉单独的进程将创建多个进程，而不是线程，这些进程将能够并发运行（以大量内存开销为代价）。

python的多处理池缺乏扩展能力

1 个答案: