我正在使用python的多处理模块来启动多个进程,这些进程正在消耗来自Rabbitmq队列的消息
def consume():
from doc_parser.resumes.resume_server.resume_entities.premium_match import PremiumMatch
matching_object = PremiumMatch(logger)
rabbit_mq_hostname = config_obj.rabbitmq_hostname
connection = pika.BlockingConnection(pika.ConnectionParameters(host=rabbit_mq_hostname))
channel = connection.channel()
channel.queue_declare(queue=request_queue_name, arguments={'x-message-ttl': 30000})
channel.queue_declare(queue=resp_queue_name, arguments={'x-message-ttl': 30000})
channel.basic_qos(prefetch_count=1)
# channel.basic_consume(process_request, queue=request_queue_name)
channel.basic_consume(lambda ch, method, properties, body: process_request(
ch, method, properties, body, matching_obj=matching_object), queue=request_queue_name)
logger.info(" [x] Waiting for messages. To exit press CTRL+C")
try:
channel.start_consuming()
except KeyboardInterrupt:
pass
def main():
workers = 8
pool = multiprocessing.Pool(processes=workers)
for i in range(0, workers):
pool.apply_async(consume, error_callback=handle_error)
# Stay alive
try:
while True:
continue
except KeyboardInterrupt:
logger.error(' [*] Exiting...')
pool.terminate()
pool.join()
if __name__ == '__main__':
main()
此启动多个进程的脚本正在4个VM中运行。因此,在所有8 X 4 = 32的消费者中,他们从队列中读取消息。
一切似乎都正常,但是我已经看到第1条8条消息发送到第1个VM(理想的是第2、3rd和4th VM使用者),接下来的8条消息发送到了2nd VM(第1、3rd和4th VM消费者是理想的)等等。我认为这与Rabbit-mq的循环调度有关
是否可以像在将消息1、2、3和4发送到VM 1、2、3和4一样,一次在所有VM上分发队列消息。