我正在尝试编写一个看似简单的经典制作人实现 - 消费者习语 在Python中。对于多个较慢的消费者，有一个相对较快的生产者。原则上，这个 很容易使用Queue模块和。{ 库文档有一个示例只产生几行代码。
但是，我还希望代码在异常发生时正常工作。制片人 如果发生以下任何事情，所有消费者都应该停止：
之后，整个过程应该失败，提出初始异常通知 打电话给我们说错了什么。
主要的挑战似乎是干净地终止消费者线程而不会结束 在阻塞连接（）中。设置Thread.deamon = True似乎很受欢迎，但对我的 理解这会导致资源泄漏，以防生产者因异常而失败。
我设法编写了一个满足我要求的实现（见下文）。然而 我发现代码比预期的要复杂得多。
是否有更简洁的方法来处理这些情况？
以下是一些示例调用以及我当前生成的最终日志消息 实现：
生产和消费10件物品：
$ python procon.py
INFO:root:processed all items
不产生任何物品：
$ python procon.py --items 0
INFO:root:processed all items
为10个消费者生产5个项目，因此只使用一些可用的消费者：
$ python procon.py --items 5 --consumers 10
INFO:root:processed all items
按Control-C中断：
$ python procon.py
^CWARNING:root:interrupted by user
未能生成第3项：
$ python procon.py --producer-fails-at 3
ERROR:root:cannot produce item 3
无法使用第3项：
$ python procon.py --consumer-fails-at 3
ERROR:root:cannot consume item 3
无法使用最后一项：
$ python procon.py --items 10 --consumer-fails-at 9
ERROR:root:cannot consume item 9
这可能是过于复杂的源代码：
"""
Consumer/producer to test exception handling in threads. Both the producer
and the consumer can be made to fail deliberately when processing a certain
item using command line options.
"""
import logging
import optparse
import Queue
import threading
import time
_PRODUCTION_DELAY = 0.1
_CONSUMPTION_DELAY = 0.3
# Delay for ugly hacks and polling loops.
_HACK_DELAY = 0.05
class _Consumer(threading.Thread):
"""
Thread to consume items from an item queue filled by a producer, which can
be told to terminate in two ways:
1. using `finish()`, which keeps processing the remaining items on the
queue until it is empty
2. using `cancel()`, which finishes consuming the current item and then
terminates
"""
def __init__(self, name, itemQueue, failedConsumers):
super(_Consumer, self).__init__(name=name)
self._log = logging.getLogger(name)
self._itemQueue = itemQueue
self._failedConsumers = failedConsumers
self.error = None
self.itemToFailAt = None
self._log.info(u"waiting for items to consume")
self._isFinishing = False
self._isCanceled = False
def finish(self):
self._isFinishing = True
def cancel(self):
self._isCanceled = True
def consume(self, item):
self._log.info(u"consume item %d", item)
if item == self.itemToFailAt:
raise ValueError("cannot consume item %d" % item)
time.sleep(_CONSUMPTION_DELAY)
def run(self):
try:
while not (self._isFinishing and self._itemQueue.empty()) \
and not self._isCanceled:
# HACK: Use a timeout when getting the item from the queue
# because between `empty()` and `get()` another consumer might
# have removed it.
try:
item = self._itemQueue.get(timeout=_HACK_DELAY)
self.consume(item)
except Queue.Empty:
pass
if self._isCanceled:
self._log.info(u"canceled")
if self._isFinishing:
self._log.info(u"finished")
except Exception, error:
self._log.error(u"cannot continue to consume: %s", error)
self.error = error
self._failedConsumers.put(self)
class Worker(object):
"""
Controller for interaction between producer and consumers.
"""
def __init__(self, itemsToProduceCount, itemProducerFailsAt,
itemConsumerFailsAt, consumerCount):
self._itemsToProduceCount = itemsToProduceCount
self._itemProducerFailsAt = itemProducerFailsAt
self._itemConsumerFailsAt = itemConsumerFailsAt
self._consumerCount = consumerCount
self._itemQueue = Queue.Queue()
self._failedConsumers = Queue.Queue()
self._log = logging.getLogger("producer")
self._consumers = []
def _possiblyRaiseConsumerError(self):
if not self._failedConsumers.empty():
failedConsumer = self._failedConsumers.get()
self._log.info(u"handling failed %s", failedConsumer.name)
raise failedConsumer.error
def _cancelAllConsumers(self):
self._log.info(u"canceling all consumers")
for consumerToCancel in self._consumers:
consumerToCancel.cancel()
self._log.info(u"waiting for consumers to be canceled")
for possiblyCanceledConsumer in self._consumers:
# In this case, we ignore possible consumer errors because there
# already is an error to report.
possiblyCanceledConsumer.join(_HACK_DELAY)
if possiblyCanceledConsumer.isAlive():
self._consumers.append(possiblyCanceledConsumer)
def work(self):
"""
Launch consumer thread and produce items. In case any consumer or the
producer raise an exception, fail by raising this exception
"""
self.consumers = []
for consumerId in range(self._consumerCount):
consumerToStart = _Consumer(u"consumer %d" % consumerId,
self._itemQueue, self._failedConsumers)
self._consumers.append(consumerToStart)
consumerToStart.start()
if self._itemConsumerFailsAt is not None:
consumerToStart.itemToFailAt = self._itemConsumerFailsAt
self._log = logging.getLogger("producer ")
self._log.info(u"producing %d items", self._itemsToProduceCount)
for itemNumber in range(self._itemsToProduceCount):
self._possiblyRaiseConsumerError()
self._log.info(u"produce item %d", itemNumber)
if itemNumber == self._itemProducerFailsAt:
raise ValueError("ucannot produce item %d" % itemNumber)
# Do the actual work.
time.sleep(_PRODUCTION_DELAY)
self._itemQueue.put(itemNumber)
self._log.info(u"telling consumers to finish the remaining items")
for consumerToFinish in self._consumers:
consumerToFinish.finish()
self._log.info(u"waiting for consumers to finish")
for possiblyFinishedConsumer in self._consumers:
self._possiblyRaiseConsumerError()
possiblyFinishedConsumer.join(_HACK_DELAY)
if possiblyFinishedConsumer.isAlive():
self._consumers.append(possiblyFinishedConsumer)
if __name__ == "__main__":
logging.basicConfig(level=logging.INFO)
parser = optparse.OptionParser()
parser.add_option("-c", "--consumer-fails-at", metavar="NUMBER",
type="long", help="number of items at which consumer fails (default: %default)")
parser.add_option("-i", "--items", metavar="NUMBER", type="long",
help="number of items to produce (default: %default)", default=10)
parser.add_option("-n", "--consumers", metavar="NUMBER", type="long",
help="number of consumers (default: %default)", default=2)
parser.add_option("-p", "--producer-fails-at", metavar="NUMBER",
type="long", help="number of items at which producer fails (default: %default)")
options, others = parser.parse_args()
worker = Worker(options.items, options.producer_fails_at,
options.consumer_fails_at, options.consumers)
try:
worker.work()
logging.info(u"processed all items")
except KeyboardInterrupt:
logging.warning(u"interrupted by user")
worker._cancelAllConsumers()
except Exception, error:
logging.error(u"%s", error)
worker._cancelAllConsumers()
答案 0 :(得分：2)
您需要一个带有cancel方法的队列，该方法清空内部队列，设置已取消的标志，然后唤醒所有人。工作人员将从join（）中醒来，检查队列中已取消的标志并采取适当的行动。消费者将从get（）中醒来并检查队列上的已取消标志并打印错误。然后您的消费者只需要在发生异常时调用cancel（）方法。
不幸的是，Python队列没有取消方法。我想到了一些选择：
答案 1 :(得分：0)
由于到目前为止答案提供了很好的提示但缺少工作代码，我从我的问题中取出代码并将其包装在一个库中，该库可以从http://pypi.python.org/pypi/proconex/获得。您可以在https://github.com/roskakori/proconex找到源代码。虽然界面感觉合理，但实现仍然使用轮询，所以欢迎贡献。
生产者或消费者线程中的任何异常都会在主线程中重新加载。只需确保使用with语句或finally:worker.close()即可确保所有线程都正常关闭。
以下是生产者的简短示例，其中包含两个使用整数的消费者：
import logging
import proconex
class IntegerProducer(proconex.Producer):
def items(self):
for item in xrange(10):
logging.info('produce %d', item)
yield item
class IntegerConsumer(proconex.Consumer):
def consume(self, item):
logging.info('consume %d with %s', item, self.name)
if __name__ == '__main__':
logging.basicConfig(level=logging.INFO)
producer = IntegerProducer()
consumer1 = IntegerConsumer('consumer1')
consumer2 = IntegerConsumer('consumer2')
with proconex.Worker(producer, [consumer1, consumer2]) as worker:
worker.work()