Question

我修改了此链接https://pymotw.com/2/multiprocessing/communication.html上的Joinable队列上的示例，以运行我编写的函数而不是Task对象。修改后的代码如下所示。我得到的问题是消费者在没有在任务队列中放入None的情况下中毒。他们在完成任务之前退出。所以我从run函数中删除了None（如下所示）的检查，我发现了这个异常：

＆＃39; NoneType＆＃39;对象不可调用

我确信自从消息＆＃34; Poisoning Consumers＆＃34;还没有打印

import multiprocessing as mp
import MyLib

# Subclass of Process
class Consumer(mp.Process):

    def __init__(self, task_queue, result_queue):
        mp.Process.__init__(self)
        self.task_queue = task_queue
        self.result_queue = result_queue
        self.daemon = True

    # A method that defines the behavior of the process
    def run(self):
        proc_name = self.name
        while True:
            try:
                next_task = self.task_queue.get()
                # if next_task is None:
                #     # Poison pill means shutdown
                #     print('%s: Exiting' % proc_name)
                #     self.task_queue.task_done()
                #     break

                mxR, disC = next_task()
                self.task_queue.task_done()
                self.result_queue.put((mxR, disC))
            except Exception as e:
                print(e)

        return


if __name__ == '__main__':
    # Establish communication queues
    tasks = mp.JoinableQueue()
    results = mp.Queue()

    # Start consumers
    num_consumers = mp.cpu_count() * 2
    print('Creating %d consumers' % num_consumers)
    consumers = [ Consumer(tasks, results)
                  for i in range(num_consumers) ]
    for w in consumers:
        w.start()

    # Enqueue jobs

    trials = 10
    Tx_Range = 50
    prnts = 4
    for tx in list(range(30, 200, 20)):
        file_name = 'output_{}_{}.txt'.format(tx,prnts)
        output_file = open(file_name,'a')
        output_file.write('Nodes\tTx_Range\tAvg_Rings\tAvg_Disc\n')
        for n in list(range(50, 101, 50)):
            ring_sum, disc_sum = 0, 0
            for i in range (0, trials):
                tasks.put(MyLib.GBMR_mp(1000, 1000, n, prnts, tx, False, results))

            print('Done putting jobs')
            for i in range (0, trials):
                mxR, discN = results.get()
                ring_sum += mxR
                disc_sum += discN
            avg_ring = ring_sum/trials
            avg_disc = disc_sum/trials
            print('Done Collecting Results, avg_disc = ', avg_disc,' and avg_rings = ', avg_ring)
            s = '{}\t\t{}\t\t{}\t\t{}\n'.format(n,tx,avg_ring,avg_disc)
            print('Nodes', n, 'is Done for Tx_range', tx)
            output_file.write(s)
        output_file.close()



    # Add a poison pill for each consumer
    print('Poisoning Consumers')
    for i in range(num_consumers):
        tasks.put(None)

    # Wait for all of the tasks to finish
    tasks.join()

这个问题的原因是什么？可能是queue.get（）返回None？

提前致谢

Answer 1

由于我没有得到任何建议，我试图解决问题，我想出了一个解决方案（我相信一个糟糕的解决方案），我只是忘记了消费者并开始在流程中运行每个功能。我通过检查启动数量来保持并发进程的数量，如下所示。但我确信我做错了，因为这个解决方案的性能比不使用多进程要糟糕得多。使用多处理时，“n”上的内部for循环大约需要2分钟，但如果没有多处理，则需要几秒钟。我仍然是一个菜鸟，有人能指出我正确的方向吗？这是代码：

import multiprocessing as mp
import MyLib


if __name__ == '__main__':

    results = mp.Queue()


    num_consumers = mp.cpu_count()


    trials = 500
    prnts = 4
    num_of_proc = 0
    consumers = []
    joined = 0
    for tx in list(range(30, 200, 20)):
        file_name = 'Centered_BS_output_{}_{}.txt'.format(tx,prnts)
        output_file = open(file_name,'a')
        output_file.write('Nodes\tTx_Range\tAvg_Rings\tAvg_Disc\n')
        for n in list(range(30, 1030, 30)):
            consumers.clear()
            ring_sum, disc_sum, joined, i, num_of_proc = 0, 0, 0, 0, 0
            #for i in range (0, trials):
            while i < trials:
                if num_of_proc < num_consumers:
                    consumers.append(mp.Process(target=MyLib.GBMR_mp, args=(1000, 1000, n, prnts, tx, False, results)))
                    consumers[i].daemon = True
                    consumers[i].start()
                    num_of_proc += 1
                    i += 1

                else:

                    consumers[joined].join()
                    num_of_proc -= 1
                    joined += 1

            print('Done putting jobs')
            for i in range (0, trials):
                mxR, discN = results.get()
                ring_sum += mxR
                disc_sum += discN
            avg_ring = ring_sum/trials
            avg_disc = disc_sum/trials
            print('Done Collecting Results, avg_disc = ', avg_disc,' and avg_rings = ', avg_ring)
            s = '{}\t\t{}\t\t{}\t\t{}\n'.format(n,tx,avg_ring,avg_disc)
            print('Nodes', n, 'is Done for Tx_range', tx)
            output_file.write(s)
        output_file.close()

queue.get返回NoneType对象？

1 个答案: