并行处理带有flush = True的打印功能时脚本挂起

时间:2019-07-16 17:34:07

标签: python python-3.x

我制作了一个processParallel类,该类接受项目,函数的队列,并使用给定的函数并行处理每个项目。问题是,当我只处理带有flush = True的打印功能时,它会挂起。没有该选项,一切将无法顺利进行。我的python版本是3.6,我正在OS X El Capitan 10.11.6上运行脚本。

import time
import threading
import multiprocessing
from functools import partial


class myThread (threading.Thread):
    def __init__(self, function):
        threading.Thread.__init__(self)
        self.function = function

    def run(self):
        self.function()



class processParallel():

    def __init__(self, work_queue, processingFunction, no_threads = 1, verbose = 3):
        self.exit_flag = [0]
        self.processingFunction = processingFunction
        self.work_queue = work_queue
        self.threads = []
        self.verbose = verbose
        self.locks_dict = {"work_queue_lock": threading.Lock(), "data_lock": threading.Lock(), "print_lock": threading.Lock()}
        self.data = []
        self.params = []


        total_num_threads = multiprocessing.cpu_count()
        if(no_threads == 0 or no_threads > total_num_threads):
            no_threads = total_num_threads

        self.run_event = threading.Event()
        self.run_event.set()

        threadID = 1
        for i in range(no_threads):
            self.threads.append(myThread( partial(self.processingLoop, threadID) ))
            threadID += 1

    def processingLoop(self, threadID):

            while not self.exit_flag[0]:


                self.locks_dict["work_queue_lock"].acquire()

                if self.work_queue.empty():
                    self.exit_flag[0] = 1

                if not self.work_queue.empty():
                    work_piece = self.work_queue.get()
                    self.locks_dict["work_queue_lock"].release()

                    if(self.verbose >= 2):
                        self.locks_dict["print_lock"].acquire()
                        print("Thread %s processing %s" % (threading.current_thread(), work_piece), flush=True)
                        self.locks_dict["print_lock"].release()



                    self.processingFunction(self, work_piece)



                    if(self.verbose >=1):
                        self.locks_dict["print_lock"].acquire()
                        print("Thread %s finished processing %s" % (threading.current_thread(), work_piece), flush=True)
                        self.locks_dict["print_lock"].release()




                else:
                    self.locks_dict["work_queue_lock"].release()

                ## If processingFunction is fast the value here ought to be smaller.
#                time.sleep(0.01)




    def process(self):
        for thread in self.threads:
            thread.start()

        for thread in self.threads:
            thread.join()

    def join(self):
        self.run_event.clear()
        for thread in self.threads:
            thread.join()



import queue

list_to_process = [i for i in range(10000)]

days_queue = queue.Queue()
days_queue.queue = queue.deque(list_to_process)

def fun(self, work_piece):
    self.locks_dict["print_lock"].acquire()
    print(work_piece, flush = True)
    self.locks_dict["print_lock"].release()

parallel_processor = processParallel(days_queue, fun, no_threads = 8, verbose = 0)
parallel_processor.process()

问题是为什么?

0 个答案:

没有答案