我正在尝试将非常简单的功能与多处理并行化
from multiprocessing import Pool
import time
import multiprocessing as mp
import numpy as np
def main():
num_cpus = mp.cpu_count() - 1
pool = mp.Pool(processes=num_cpus)
data = list(range(0,20))
index_chunks = np.array_split(data, num_cpus)
print(index_chunks)
def f(x):
return sum(x)
data_list = [list(index_element) for index_element in index_chunks]
async_mapper = pool.map_async(f, data_list)
async_mapper.wait()
pool.close()
result_list = async_mapper.get()
print(result_list)
if __name__ == '__main__':
main()
我收到以下错误:
Traceback (most recent call last):
File "C:/Users/user/Documents/Projects/folder/multiprocessing_test/test.py", line 24, in <module>
main()
File "C:/Users/user/Documents/Projects/folder/multiprocessing_test/test.py", line 20, in main
result_list = async_mapper.get()
File "C:\Users\user\AppData\Local\Continuum\anaconda3\lib\multiprocessing\pool.py", line 644, in get
raise self._value
File "C:\Users\user\AppData\Local\Continuum\anaconda3\lib\multiprocessing\pool.py", line 424, in _handle_tasks
put(task)
File "C:\Users\user\AppData\Local\Continuum\anaconda3\lib\multiprocessing\connection.py", line 206, in send
self._send_bytes(_ForkingPickler.dumps(obj))
File "C:\Users\user\AppData\Local\Continuum\anaconda3\lib\multiprocessing\reduction.py", line 51, in dumps
cls(buf, protocol).dump(obj)
AttributeError: Can't pickle local object 'main.<locals>.f'
有人可以解释如何解决吗?我在Windows和Linux上都收到此错误,非常感谢!