我正在尝试对MNIST数据集并行执行某些操作。通过多处理,它会卡住并且永远不会结束。我尝试将close和join函数放在for循环的内部或外部。我也尝试将池实例化放在函数中,似乎也不起作用。我在这里做错了什么?不知道我应该在哪里解决这个问题。非常感谢您的提前帮助
class MnistPartialDataset(Dataset):
def __init__(self, data, transform=None):
self.data = data
def __getitem__(self, index):
return self.data[index]
def __len__(self):
return len(self.data)
def create_tasks(train_pool, test_pool):
train_thread_result = []
start = time.time()
for k, v in train_tasks.items():
tc_data = train_pool.map_async(test_function, v)
train_thread_result.append(tc_data)
train_pool.close()
train_pool.join()
for i in range(len(train_thread_result)):
train_dataset[i] = MnistPartialDataset(train_thread_result[i].get())
end = time.time()
print("tc time: " + str(end-start))
test_thread_result = []
start = time.time()
for k, v in test_tasks.items():
tc_data = test_pool.map_async(test_function, v)
test_thread_result.append(tc_data)
test_pool.close()
test_pool.join()
for i in range(len(test_thread_result)):
test_dataset[i] = MnistPartialDataset(test_thread_result[i].get())
end = time.time()
def main():
train_pool = multiprocessing.Pool(processes = multiprocessing.cpu_count()-1)
test_pool = multiprocessing.Pool(processes = multiprocessing.cpu_count()-1)
create_tasks(train_pool, test_pool)
pool.close()
pool.terminate()