Question

我的代码：

def create_rods(folder="./", kappas=10, allowed_kappa_error=.3,
                radius_correction_ratio=0.1):
    """
    Create one rod for each rod_data and for each file
    returns [RodGroup1, RodGroup2, ...]
    """
    names, files = import_files(folder=folder)
    if len(files) == 0:
        print "No files to import."
        raise ValueError
    states = [None for dummy_ in range(len(files))]
    processes = []
    states_queue = mp.Queue()
    for index in range(len(files)):
        process = mp.Process(target=create_rods_process,
                        args=(kappas, allowed_kappa_error,
                        radius_correction_ratio, names,
                        files, index, states_queue))
        processes.append(process)
    run_processes(processes)        #This part seem to take a lot of time.
    try:
        while True:
            [index, state] = states_queue.get(False)
            states[index] = state
    except Queue.Empty:
        pass    
    return names, states

def create_rods_process(kappas, allowed_kappa_error,
                    radius_correction_ratio, names,
                    files, index, states_queue):
    """
    Process of method.
    """
    state = SystemState(kappas, allowed_kappa_error,
                radius_correction_ratio, names[index])
    data = import_data(files[index])
    for dataline in data:
        parameters = tuple(dataline)
        new_rod = Rod(parameters)
        state.put_rod(new_rod)
    state.check_rods()
    states_queue.put([index, state])

def run_processes(processes, time_out=None):
    """
        Runs all processes using all cores.
    """
    running = []
    cpus = mp.cpu_count()
    try:
        while True:
        #for cpu in range(cpus):
            next_process = processes.pop()
            running.append(next_process)
            next_process.start()
    except IndexError:
        pass
    if not time_out:
        try:
            while True:
                for process in running:
                    if not process.is_alive():
                        running.remove(process)
        except TypeError:
            pass
    else:
        for process in running:
            process.join(time_out)

我希望流程结束，但我得到了一个流程。我不知道run_processes（）方法或create_rods（）方法是否存在问题。随着连接cpus被释放，但程序没有继续。

Answer 1

来自Python的multiprocessing guidelines。

加入使用队列的进程

请记住，将项目放入队列的进程将在终止之前等待，直到所有缓冲的项目由“feeder”线程提供给底层管道。（子进程可以调用队列的Queue.cancel_join_thread方法来避免此行为。）

这意味着无论何时使用队列，您都需要确保在加入进程之前最终删除已放入队列的所有项目。否则，您无法确定已将项目放入队列的进程将终止。还要记住，非守护进程将自动加入。

在排空队列之前加入进程会导致死锁。在加入流程之前，您需要确保队列已清空。

使用python进行多处理时cpu被锁定

1 个答案: