Question

我正在尝试在管理进程下启动数据队列服务器（以便以后可以将其转换为服务），并且虽然数据队列服务器功能在主进程中正常工作，但它不能在使用multiprocessing.Process创建的进程。

dataQueueServer和dataQueueClient代码基于多处理模块文档here中的代码。

单独运行时， dataQueueServer 运行良好。但是，在 mpquueue 中使用multiprocessing.Process的{{1}}运行时，它不起作用（使用客户端测试时）。我正在使用 dataQueueClient 而不进行更改来测试这两种情况。

在这两种情况下，代码都会到达start()，所以我认为服务器正在运行，但有些东西阻止它在 mpqueue 情况下与客户端进行通信。

我已经在一个线程下放置了运行serve_forever部分的循环，因此它可以停止。

以下是代码：

mpqueue ＃这是尝试在子进程中生成服务器的“经理”进程

serve_forever()

dataQueueServer

import time
import multiprocessing
import threading
import dataQueueServer

class Printer():
    def __init__(self):
        self.lock = threading.Lock()
    def tsprint(self, text):
        with self.lock:
            print text

class QueueServer(multiprocessing.Process):
    def __init__(self, name = '', printer = None):
        multiprocessing.Process.__init__(self)
        self.name = name
        self.printer = printer
        self.ml = dataQueueServer.MainLoop(name = 'ml', printer = self.printer)

    def run(self):
        self.printer.tsprint(self.ml)
        self.ml.start()

    def stop(self):
        self.ml.stop()

if __name__ == '__main__':
    printer = Printer()
    qs = QueueServer(name = 'QueueServer', printer =  printer)
    printer.tsprint(qs)
    printer.tsprint('starting')
    qs.start()
    printer.tsprint('started.')
    printer.tsprint('Press Ctrl-C to quit')
    try:
        while True:
            time.sleep(60)
    except KeyboardInterrupt:
        printer.tsprint('\nTrying to exit cleanly...')
        qs.stop()

    printer.tsprint('stopped')

和客户：

dataQueueClient

import time
import threading

from multiprocessing.managers import BaseManager
from multiprocessing import Queue

HOST = ''
PORT = 50010
AUTHKEY = 'authkey'

## Define some helper functions for use by the main process loop
class Printer():
    def __init__(self):
        self.lock = threading.Lock()
    def tsprint(self, text):
        with self.lock:
            print text



class QueueManager(BaseManager): 
    pass


class MainLoop(threading.Thread):
    """A thread based loop manager, allowing termination signals to be sent
    to the thread"""
    def __init__(self, name = '', printer = None):
        threading.Thread.__init__(self)
        self._stopEvent = threading.Event()
        self.daemon = True
        self.name = name

        if printer is None:
            self.printer = Printer()
        else:
            self.printer = printer

        ## create the queue
        self.queue = Queue()
        ## Add a function to the handler to return the queue to clients
        self.QM = QueueManager

        self.QM.register('get_queue', callable=lambda:self.queue)
        self.queue_manager = self.QM(address=(HOST, PORT), authkey=AUTHKEY)
        self.queue_server = self.queue_manager.get_server()

    def __del__(self):
        self.printer.tsprint( 'closing...')


    def run(self):
        self.printer.tsprint( '{}: started serving'.format(self.name))
        self.queue_server.serve_forever()


    def stop(self):
        self.printer.tsprint ('{}: stopping'.format(self.name))
        self._stopEvent.set()

    def stopped(self):
        return self._stopEvent.isSet()

def start():
    printer = Printer() 
    ml = MainLoop(name = 'ml', printer = printer)
    ml.start()
    return ml

def stop(ml):
    ml.stop()

if __name__ == '__main__':
    ml = start()
    raw_input("\nhit return to stop")
    stop(ml)

Answer 1

所以似乎解决方案很简单：不要使用serve_forever()，而是使用manager.start()。

根据Eli Bendersky，BaseManager（以及它的扩展版本SyncManager）已在新进程中生成服务器（并查看多处理。管理器代码确认这个）。我遇到的问题源于示例中使用的表单，其中服务器在主进程下启动。

我仍然不明白为什么当前的例子在子进程下运行时不起作用，但这不再是问题。

这是管理多个队列服务器的工作（以及简化的OP）代码：

服务器：

from multiprocessing import Queue from multiprocessing.managers import SyncManager HOST = '' PORT0 = 5011 PORT1 = 5012 PORT2 = 5013 AUTHKEY = 'authkey' name0 = 'qm0' name1 = 'qm1' name2 = 'qm2' description = 'Queue Server' def CreateQueueServer(HOST, PORT, AUTHKEY, name = None, description = None): name = name description = description q = Queue() class QueueManager(SyncManager): pass QueueManager.register('get_queue', callable = lambda: q) QueueManager.register('get_name', callable = name) QueueManager.register('get_description', callable = description) manager = QueueManager(address = (HOST, PORT), authkey = AUTHKEY) manager.start() # This actually starts the server return manager # Start three queue servers qm0 = CreateQueueServer(HOST, PORT0, AUTHKEY, name0, description) qm1 = CreateQueueServer(HOST, PORT1, AUTHKEY, name1, description) qm2 = CreateQueueServer(HOST, PORT2, AUTHKEY, name2, description) raw_input("return to end")

<强>客户端：

from multiprocessing.managers import SyncManager HOST = '' PORT0 = 5011 PORT1 = 5012 PORT2 = 5013 AUTHKEY = 'authkey' def QueueServerClient(HOST, PORT, AUTHKEY): class QueueManager(SyncManager): pass QueueManager.register('get_queue') QueueManager.register('get_name') QueueManager.register('get_description') manager = QueueManager(address = (HOST, PORT), authkey = AUTHKEY) manager.connect() # This starts the connected client return manager # create three connected managers qc0 = QueueServerClient(HOST, PORT0, AUTHKEY) qc1 = QueueServerClient(HOST, PORT1, AUTHKEY) qc2 = QueueServerClient(HOST, PORT2, AUTHKEY) # Get the queue objects from the clients q0 = qc0.get_queue() q1 = qc1.get_queue() q2 = qc2.get_queue() # put stuff in the queues q0.put('some stuff') q1.put('other stuff') q2.put({1:123, 2:'abc'}) # check their sizes print 'q0 size', q0.qsize() print 'q1 size', q1.qsize() print 'q2 size', q2.qsize() # pull some stuff and print it print q0.get() print q1.get() print q2.get()

添加额外的服务器以与正在运行的队列服务器的信息共享字典，以便消费者可以使用该模型轻松地告诉可用的哪些内容很容易。但有一点需要注意的是，共享字典需要的语法与普通字典略有不同：dictionary[0] = something不起作用。您需要使用dictionary.update([(key, value), (otherkey, othervalue)])和dictionary.get(key)语法，该语法传播到连接到此词典的所有其他客户端。

在multiprocessing.Process下的Python多处理RemoteManager

1 个答案: