Question

我正在设计一个Python多处理代码，以便在可能在处理过程中更新的队列中工作。以下代码有时会起作用，或者卡住，或者出现Empty错误。

import multiprocessing as mp

def worker(working_queue, output_queue):
    while True:
        if working_queue.empty() is True:
            break    
        else:
            picked = working_queue.get_nowait()
            if picked % 2 == 0: 
                    output_queue.put(picked)
            else:
                working_queue.put(picked+1)
    return

if __name__ == '__main__':
    manager = mp.Manager()
    static_input = xrange(100)    
    working_q = manager.Queue()
    output_q = mp.Queue()
    for i in static_input:
        working_q.put(i)
    processes = [mp.Process(target=worker,args=(working_q, output_q)) for i in range(mp.cpu_count())]
    for proc in processes:
        proc.start()
    for proc in processes:
        proc.join()
    results_bank = []
    while True:
       if output_q.empty() is True:
           break
       results_bank.append(output_q.get_nowait())
    print len(results_bank) # length of this list should be equal to static_input, which is the range used to populate the input queue. In other words, this tells whether all the items placed for processing were actually processed.
    results_bank.sort()
    print results_bank

我应该使用列表作为全局变量，并将其锁定，而不是manager.Queue（）吗？

Answer 1

我刚刚添加了try:和except Exception:来处理Empty错误。结果似乎现在一致。请告诉我如果您发现我在此解决方案中忽略了问题。

import multiprocessing as mp

def worker(working_queue, output_queue):
    while True:
        try:
            if working_queue.empty() is True:
                break  
            else:
                picked = working_queue.get_nowait()
                if picked % 2 == 0: 
                        output_queue.put(picked)
                else:
                    working_queue.put(picked+1)
        except Exception:
            continue

    return

if __name__ == '__main__':
    #Manager seem to be unnecessary.
    #manager = mp.Manager()
    #working_q = manager.Queue()

    working_q = mp.Queue()
    output_q = mp.Queue()
    static_input = xrange(100)     
    for i in static_input:
        working_q.put(i)
    processes = [mp.Process(target=worker,args=(working_q, output_q)) for i in range(mp.cpu_count())]
    for proc in processes:
        proc.start()
    for proc in processes:
        proc.join()
    results_bank = []
    while True:
       if output_q.empty() is True:
           break
       results_bank.append(output_q.get_nowait())
    print len(results_bank) # length of this list should be equal to static_input, which is the range used to populate the input queue. In other words, this tells whether all the items placed for processing were actually processed.
    results_bank.sort()
    print results_bank

Answer 2

只需使用锁来保护对共享数据的访问，它更安全（并且可以保护您免受该过程的奇怪行为）：

import multiprocessing as mp

def worker(working_queue, output_queue, lock):
    while True:
        shouldBeak = False
        lock.acquire()
        if working_queue.empty() is True:
            shouldBeak = True    
        else:

            picked = working_queue.get_nowait()
            if picked % 2 == 0: 
                output_queue.put(picked)
            else:
                working_queue.put(picked+1)
        lock.release()
        if shouldBeak:
            break
    return

if __name__ == '__main__':
    manager = mp.Manager()
    static_input = xrange(1000)    
    working_q = manager.Queue()
    output_q = mp.Queue()
    lock = mp.Lock()
    for i in static_input:
        working_q.put(i)
    processes = [mp.Process(target=worker,args=(working_q, output_q,lock)) for i in range(mp.cpu_count())]
    for proc in processes:
        proc.start()
    for proc in processes:
        proc.join()
    results_bank = []
    while True:
       if output_q.empty() is True:
           break
       results_bank.append(output_q.get_nowait())
    print len(results_bank) # length of this list should be equal to static_input, which is the range used to populate the input queue. In other words, this tells whether all the items placed for processing were actually processed.
    results_bank.sort()
    print results_bank

使用管理器在Python多进程中更新队列

2 个答案: