map(f,very long sequence)
比multiprocessing.Pool.map(f, very long sequence)
快?
x
是元组的字典。看起来像
[(0,{0:50}),(1,{0:44,67:58,76:8,93:91}),(3,{24:35,45:93})]
f
函数映射到x
上,并计算x
的所有值之和。
import multiprocessing
import random
random.seed(1234)
def ParallelMap(func,argList,threads=5):
pool = multiprocessing.Pool(processes = threads)
res=pool.map(func, argList)
pool.close()
pool.join()
return res
def f(mytuple):
mysum = 0
for k,v in mytuple[1].items():
mysum += v
return mysum
if __name__=="__main__":
step = 500000
mydict_dict = {}
for i in range(step):
temp_dict = {}
for i in range(random.randrange(1,5)):
temp_dict[random.randrange(100)] = random.randrange(100)
mydict_dict[random.randrange(step)] = temp_dict
x = mydict_dict.items()
#print(sum(map(f, x)))
print(sum(ParallelMap(f, x)))
sum(map(f, x))
的时间是:
真实0m3.160s
用户0m3.095s
sys 0m0.058s
sum(ParallelMap(f, x))
的时间是:
真实0m5.184s
用户0m5.679s
sys 0m0.323s
即使使用pool.map(func, argList)
或pool.imap(func, argList, chunksize=50000)
也无法提高速度。