我有一个任务经纪人，他使用内部消息队列（redis），工作者在其中轮询任务。 任务有超时，应由工作人员在到期之前完成。 因此，我在Broker中实现了一个队列，该队列在给定任务时添加任务，并根据最近添加的任务的最短等待时间检查该队列的线程。 从理论上讲，一旦完成任务的工作人员应将任务从此队列中删除，以使最短的等待时间到期，该等待时间不再存在于队列中。 如果是这样，则应进行某种回滚。
下面的代码不起作用。具体来说，我可以在日志中看到任务已成功完成，甚至由工作人员处理。 但是，有两个问题： （1）当工作人员调用“ remove_task_from_queue”时，找不到任务。 （2）调用add_task_to_queue时，也会调用task_signal.set（），但task_queue_monitor函数无法执行。
我不确定这是怎么回事。你能帮忙吗？
在代码下，还附有日志的简化版本。无论是来自Worker（在另一个容器中运行）还是来自带有Task Broker的主应用程序。
import threading
from logger import Logger
from task_broker import TaskBroker
from message_queue.message_queue_factory import MessageQueueFactory
from utils.general_utils import get_absolute_time_seconds
class CustomTaskBroker(TaskBroker):
task_queue = []
task_signal = None
def __init__(self):
self.logger = Logger("CustomTaskBroker")
self.message_queue = MessageQueueFactory().get_message_queue()
self.task_signal = threading.Event()
self.task_monitor_thread = threading.Thread(name = "Task Monitor Thread", target = self.task_queue_monitor)
self.task_monitor_thread.start()
def start(self, task_handler, subscribe_channels=[]):
self.logger.log("[CustomTaskBroker] Starting Broker: starting MQ")
return self.message_queue.start(task_handler, subscribe_channels)
def give_task(self, task_obj):
try:
self.add_task_to_queue(task_obj)
self.logger.log("[CustomTaskBroker] Task given. Task is '{0}'. Added to queue: '{1}'".format(task_obj.cast_type, self.task_queue))
if task_obj.cast_type is "anycast":
self.message_queue.queue.lpush('changes_' + task_obj.tenant + "_" + "common" +':process', task_obj.as_json_string())
else:
self.message_queue.publish(task_obj)
except Exception as e:
self.logger.error("[CustomTaskBroker] invalid task received. Error message '{0}'", e)
def add_task_to_queue(self, task_obj):
self.task_queue.append({"task": task_obj, "id": task_obj.id, "start_time": get_absolute_time_seconds(), "wait_time": task_obj.wait_time})
self.task_signal.set()
def remove_task_from_queue(self, task):
self.logger.error("[CustomTaskBroker] Trying to remove task with id '{0}' from task_queue '{1}'".format(task["id"], self.task_queue))
for i in range(len(self.task_queue)):
if self.task_queue[i]["id"] == task["id"]:
self.task_queue.pop(i)
return
self.logger.error("[CustomTaskBroker] Tried to remove task with id '{0}' but not found!".format(task["id"]))
def task_queue_monitor(self):
try:
while True:
earliest_task_timer = self.determine_wait_time()
self.logger.log("[CustomTaskBroker] Monitor_threads, len task_queue is '{0}'. Waiting until task added or wait_time '{1}'".format(len(self.task_queue), earliest_task_timer["wait_time"]))
self.task_signal.wait(earliest_task_timer["wait_time"])
self.task_signal.clear()
self.logger.log("[CustomTaskBroker] Wait time expired or new task added. Checking if tasks expired")
earliest_task_timer = self.determine_wait_time()
if earliest_task_timer["wait_time"] < 0:
self.logger.log("[CustomTaskBroker] Wait time expired. Checking if the waiting task is still in queue")
if earliest_task_timer["task_id"] in [ task["id"] for task in self.task_queue]:
raise Exception("We have an expired Task! Rollback")
#TODO implement rollback
except Exception as e:
self.logger.error("[CustomTaskBroker] Monitor threads encountered an issue. Exception: '{0}'".format(e))
def determine_wait_time(self):
if len(self.task_queue) == 0:
earliest_task_timer = { "id" : "None", "wait_time" : None} #block indefinitely
else:
list_of_wait_time_tasks = [(task["id"], task["wait_time"] - (get_absolute_time_seconds() - task["start_time"])) for task in self.task_queue]
self.logger.log("[CustomTaskBroker] list_of_wait_time_tasks: '{0}'".format(list_of_wait_time_tasks))
earliest_task_tuple = min(list_of_wait_time_tasks, key= lambda n: n[1])
earliest_task_timer = { "id" : earliest_task_tuple[0], "wait_time" : earliest_task_tuple[1]}
self.logger.log("[CustomTaskBroker] determine_wait_time determined earliest_task_timer: '{0}'".format(earliest_task_timer))
return earliest_task_timer
日志工作者
[[[16-07-19 07:05:09] Worker [info] # [Worker] Waiting for items to appear in queue
[[[16-07-19 07:05:09] RedisWrapper [debug] # [RedisWrapper] Doing block_rpop(). Name 'changes_nubera_common:process'
[[[16-07-19 07:05:16] Worker [info] # [Worker] Popped item. Type '<class 'bytes'>'. Item: 'b'{"trigger_invocation": false, "type": "document_task", "action": "unknown", "cast_type": "anycast", "tenant": "nubera", "return_channel": null,
"wait_time": 2, "tasks": ["put_document"], "channel": "worker", "id": "sroxkaqqle", "source": "a628cedb41b5", "doc": {"payload": {"name": "tst_sv_zone", "description": "Testing tst_sv_zone", "type": "zone"}, "category": "types", "subcate
gory": "zone", "name": "tst_sv_zone", "description": "Testing tst_sv_zone"}}''
[[[16-07-19 07:05:16] Document_Manager [debug] # [Document_manager] Succesfully handled document task. Notifying task broker.
[[[16-07-19 07:05:16] CustomTaskBroker [error] # [CustomTaskBroker] Tried to remove task with id 'sroxkaqqle' but not found!
[[[16-07-19 07:05:16] CustomTaskBroker NoneType: None
记录主应用程序/ Task_broker
[[[16-07-19 07:04:54] TaskBrokerFactory [info] # [TaskBrokerFactory] Creating Broker with type: Custom
[[[16-07-19 07:04:54] MessageQueueFactory [info] # [MessageQueueFactory] Creating MQ with type: Redis
[[[16-07-19 07:04:54] CustomTaskBroker [debug] # [CustomTaskBroker] determine_wait_time determined earliest_task_timer: '{'id': 'None', 'wait_time': None}'
[[[16-07-19 07:04:54] CustomTaskBroker [debug] # [CustomTaskBroker] Monitor_threads, len task_queue is '0'. Waiting until task added or wait_time 'None'
[[[16-07-19 07:04:54] DefaultTaskHandler [info] # [DefaultTaskHandler] Created with call_object 'None', tenants 'None', and stackl_type 'rest'
[[[16-07-19 07:04:54] CustomTaskBroker [debug] # [CustomTaskBroker] Starting Broker: starting MQ
[[[16-07-19 07:04:54] RedisQueue [info] # [RedisQueue] Broker connecting to redis
[[[16-07-19 07:05:16] CustomTaskBroker [debug] # [CustomTaskBroker] Task given. Task is 'anycast'. Added to queue: '[{'task': <task.document_task.DocumentTask object at 0x7ffa1c5a7e80>, 'id': 'sroxkaqqle', 'start_time': 1563260716.1085796, 'wait_time': 2}]'
[[[16-07-19 07:05:16] RedisWrapper [debug] # [RedisWrapper] Doing lpush(). Name changes_nubera_common:process and values {"trigger_invocation": false, "type": "document_task", "action": "unknown", "cast_type": "anycast", "tenant": "nuber
a", "return_channel": null, "wait_time": 2, "tasks": ["put_document"], "channel": "worker", "id": "sroxkaqqle", "source": "a628cedb41b5", "doc": {"payload": {"name": "tst_sv_zone", "description": "Testing tst_sv_zone", "type": "zone"}, "
category": "types", "subcategory": "zone", "name": "tst_sv_zone", "description": "Testing tst_sv_zone"}}