Question

我尝试使用多处理从sklearn（在这种情况下为高斯混合模式）中对分类器进行并行训练，与顺序运行相比，我得到了更糟糕的分类器。此外，每次训练后结果都不同，就好像代码不是线程安全的。任何人都可以解释我发生了什么事吗？这是代码，最后是线程函数：

nrProc = 8
semaphore = Semaphore(nrProc)
m = Manager()
models = m.list()
modelsOut = m.list()
processes = []   

cnt = 0                
for event_label in data_positive:                        
    models.append(mixture.GMM(**classifier_params))  
    models.append(mixture.GMM(**classifier_params))

for event_label in data_positive:
    if classifier_method == 'gmm':                        
        processes.append(Process(target=trainProcess, args=(models[cnt], data_positive[event_label], semaphore, modelsOut)))
        cnt = cnt + 1                        
        processes.append(Process(target=trainProcess, args=(models[cnt], data_negative[event_label], semaphore, modelsOut)))
        cnt = cnt + 1
    else:
        raise ValueError("Unknown classifier method ["+classifier_method+"]")

for proc in processes:
    proc.start()

for proc in processes:
    proc.join()


cnt = 0                
for event_label in data_positive:
    model_container['models'][event_label] = {}
    model_container['models'][event_label]['positive'] = modelsOut[cnt]
    cnt = cnt + 1
    model_container['models'][event_label]['negative'] = modelsOut[cnt]
    cnt = cnt + 1

def trainProcess(model, data, semaphore, modelsOut):
    semaphore.acquire()    
    modelsOut.append(model.fit(data))
    semaphore.release()
    return 0

Answer 1

因此解决方案是使用sklearn中的clone函数，它执行估算器的深层复制。

Python sklearn和多处理

1 个答案: