如何在python中使用多线程的ioloops?

时间:2014-08-08 09:53:40

标签: python multithreading tornado pyzmq

我有问题弄清楚如何将zmq与ioloops和多线程一起使用。无论我做什么,我都有一些例外。

    __author__ = 'michael'
    class ZmqLoopRunner(Thread):

        def __init__(self, callback):
            super(ZmqLoopRunner, self).__init__()
            self.loop = IOLoop.current()
            self.callback = callback

        def run(self):
            self.loop.start()
            print('loop have been stopped')
            self.callback()

        def stop(self):
            self.loop.stop()

class BaseZmqNode():
    __metaclass__ = ABCMeta

    def __init__(self, host, port,  bind, hwm):
        self.node = self.create_node()
        self.node.host = host
        self.port = port
        self.context = zmq.Context().instance()
        self.socket = self.create_socket()
        if bind:
            self.socket.bind(self.build_address(host, port))
        else:
            self.socket.connect(self.build_address(host, port))
        self.set_hwm(hwm)

    def set_hwm(self, hwm):
        self.socket.set_hwm(hwm)

    def send_multipart(self, message):
        self.socket.send_multipart(message)

    def send_json(self, json):
        self.socket.send_json(json)

    def create_node(self):
        return BaseMessagingNode

    def close(self):
        self.socket.close()

    @staticmethod
    def build_address(host, port):
        strings = [host, ':', str(port)]
        return ''.join(strings)

    @abstractmethod
    def create_socket(self):
        pass

class BaseZmqReceiver(BaseZmqNode):

    __metaclass__ = ABCMeta

    def __init__(self, host, port, hwm, bind, on_receive_callback):
        super(BaseZmqReceiver, self).__init__(host=host, port=port, bind=bind, hwm=hwm)
        self.node.on_message_callback = on_receive_callback
        self.stream = ZMQStream(self.socket)
        self.stream.on_recv(self.on_message_received)
        self.runner = ZmqLoopRunner(self.stream.close)
        self.runner.start()

    def on_message_received(self, message):
        return self.node.on_message_callback(message)

    def create_node(self):
        return ReceivingNode(None, None)


    def close(self):
        # super(BaseZmqReceiver, self).close()

        self.runner.stop()

        # self.socket.close()

以下是我的代码现在的样子。我在测试'已使用的地址中有例外情况'在运行我的测试时。

这是stacktrace

Traceback (most recent call last):
  File "/opt/leos/code/messaging_system/tests/ZmqTest.py", line 51, in test_send_json
    publisher = ZmqPublisher('tcp://*', 6000)
  File "/opt/leos/code/messaging_system/zeromq/ZmqPublisher.py", line 11, in __init__
    super(ZmqPublisher, self).__init__(host=host, port=port, bind=bind, hwm=hwm)
  File "/opt/leos/code/messaging_system/zeromq/BaseZmqNode.py", line 18, in __init__
    self.socket.bind(self.build_address(host, port))
  File "socket.pyx", line 434, in zmq.backend.cython.socket.Socket.bind (zmq/backend/cython/socket.c:3928)
  File "checkrc.pxd", line 21, in zmq.backend.cython.checkrc._check_rc (zmq/backend/cython/socket.c:6058)
ZMQError: Address already in use

class ZmqTest(AbstractMessagingTest):

    def setUp(self):
        super(ZmqTest, self).setUp()
        self.multipart_messages = self.create_multipart_messages(10)

    def tearDown(self):
        super(ZmqTest, self).tearDown()

    def test_request_reply(self):
        requester = ZmqReq(host='tcp://localhost', port=6000)
        self.request = 'Hello'
        self.reply = 'World!'
        replier = ZmqRep(host='tcp://*', port=6000, request_processor=self.on_request_received)
        self.assertEqual(self.reply, requester.execute(request=self.request))
        # requester.close()
        replier.close()
        requester.close()

    def test_address_creation(self):
        full_address = "tcp://localhost:5559"
        self.assertEqual(full_address, ZmqSubscriber.build_address("tcp://localhost", 5559))
        self.assertEqual('tcp://*:6000', ZmqPublisher.build_address("tcp://*", 6000))

    def test_publisher_subscriber(self):

        publisher = ZmqPublisher('tcp://*', 6000)
        subscriber = ZmqSubscriber('tcp://localhost', 6000, self.handle_message)
        self.send_messages(publisher, wait=False)
        sleep(0.5)
        self.assertSequenceEqual(self.test_messages, self.received_messages)
        publisher.close()
        subscriber.close()

    def handle_message(self, message):
        self.base_handle_message(message[0])

    def test_send_json(self):
        publisher = ZmqPublisher('tcp://*', 6000)
        subscriber = ZmqSubscriber('tcp://localhost', 6000, self.handle_json_message)
        md = {'1' : 1}
        publisher.send_json(md)
        publisher.close()
        subscriber.close()

    def create_multipart_messages(self, size):
        messages = []
        for i in range(size):
            messages.append(['Multipart test message', str(i)])
        return messages

    def send_multipart_messages(self, sender):
        for message in self.multipart_messages:
            sender.send_multipart(message)


    def test_multipart_messages(self):
        publisher = ZmqPublisher('tcp://*', 6000)
        subscriber = ZmqSubscriber('tcp://localhost', 6000, self.base_handle_message)
        self.send_multipart_messages(publisher)
        sleep(0.5)
        self.assertSequenceEqual(self.multipart_messages, self.received_messages)
        publisher.close()
        subscriber.close()

    def test_push_pull_multipart(self):
        ventilator = ZmqPush('tcp://*', 6000)
        worker = ZmqPull('tcp://localhost', 6000, self.base_handle_message)
        self.send_multipart_messages(ventilator)
        sleep(0.5)
        self.assertSequenceEqual(self.multipart_messages, self.received_messages)
        ventilator.close()
        worker.close()


    def handle_json_message(self, json):
        print(str(json))

    def test_push_pull(self):
        ventilator = ZmqPush('tcp://*', 6000)
        worker = ZmqPull('tcp://localhost', 6000, self.handle_message)
        self.send_messages(ventilator, wait=False)
        sleep(0.5)
        self.assertSequenceEqual(self.test_messages, self.received_messages)
        ventilator.close()
        worker.close()

    def on_request_received(self, message):
        if message[0] == self.request:
            return self.reply
        else:
            return 'ERROR'

我已尝试过此代码的多种变体。就像你现在看到的那样,我想在loop.start()方法返回后尝试关闭流。我已经尝试在停止方法后关闭流,但它不能正常工作。

1 个答案:

答案 0 :(得分:1)

因此,您收到套接字地址已打开的错误。这可能是因为你已经运行了程序,抛出异常并且你还没有关闭套接字。

我建议使用一些tryexceptfinally块:

try:
    requester = ZmqReq(host='tcp://localhost', port=6000)
    self.request = 'Hello'
    self.reply = 'World!'
    replier = ZmqRep(host='tcp://*', port=6000, request_processor=self.on_request_received)
    self.assertEqual(self.reply, requester.execute(request=self.request))
except Exception as e:
    # You can catch exceptions here
    pass
finally:
    # Once the code completes or exceptions are thrown, clean up
    replier.close()
    requester.close()