Question

我想创建一些工作者，并根据套接字接收的数据给他们工作。与此同时，我希望在完成工作后返回工人的结果，并且不要等到所有工人都结束。

关闭游泳池之前可以返回吗？

main.py

from multiprocessing import Pool, current_process
import time
import random
import copy_reg
import types
import Extractor as e
import threading


class PageControler(object):
    def __init__(self):
        self.nProcess = 3
        self.pages = [1,2,3,4,5,6,7,8,9,10]
        self.__result = []
        self.manageWork()
        self.extractor = e.Extractor()


    def manageWork(self):

        worker_thread = threading.Thread(target=self.initPool)
        worker_thread.start()

        work_queue = threading.Thread(target=self.modifyQueue)
        work_queue.start()

    def initPool(self):
        pool = Pool(processes=self.nProcess)

        ti = time.time()
        while (time.time()-ti)<10:
            if self.pages != []:
                pag = self.pages.pop()
                pool.apply_async(self.BarcodeSearcher, args = (pag, ), callback = self.resultCollector)
        pool.close()
        pool.join()

    def modifyQueue(self):
    """ Simulate socket data receive """    
    ti = time.time()
        while (time.time()-ti)<10:
            time.sleep(4)
            self.pages.append(99)
            print self.pages

    def BarcodeSearcher(self, input_page):        
        print ("Process %s: handling %s" % (current_process(), input_page))  
        result = e.Extractor().BarcodeSearcher(input_page)
        return result

    def resultCollector(self, result):
        self.__result.append(result)


if __name__ == '__main__':

    def _pickle_method(m):
        if m.im_self is None:
            return getattr, (m.im_class, m.im_func.func_name)
        else:
            return getattr, (m.im_self, m.im_func.func_name)

    copy_reg.pickle(types.MethodType, _pickle_method)

    PageControler()

Extractor.py

class Extractor(object):
    """Simulate extractor"""
    def __init__(self):
        pass

    def BarcodeSearcher(self,x):
        ti = time.time()
        tf = random.randint(1,4)
        while time.time() - ti < tf:
            random.uniform(1., 2.)
        return tf

Answer 1

您的回调函数将在工作人员完成后立即返回结果，然后才能关闭池。

所以为了得到结果你可以只是轮询你的__result，看看你是否有新的。

确保你在没有等待的线程中执行它，在这种情况下这将是你的主线程。

last_results_len = 0
while True: # or whatever you choose to..
    results_len = len(__result)
    if results_len != last_results_len:
        last_results_len += 1  # add one so you get them all in case they come in quick
        print(__result[results_len - 1]) # whatever you want to do here

    time.sleep(1)

多处理队列和工作人员在关闭池之前返回

1 个答案: