我的芹菜任务有手柄泄漏问题。这是我的伪代码:
import billiard as mp
from celery import shared_task
def ssh_op(ip, port, user, passwd, cmd, timeout=None, get_pty=False):
logger.info(f'run {cmd} on {ip}:{port}')
return True, "" # just for test, i didn't run the actual cmd
def parse_results(results, timeout=None):
logger.info("in parse_results")
rets = []
for res in results:
try:
ret, msg = res.get(timeout=timeout)
except Exception as e:
logger.error("return place 1")
return False, str(e)
if not ret:
logger.error("return place 2")
return False, msg
else:
rets.append(msg)
logger.info("return place 3")
return True, rets
@shared_task
def check_job_result(my_args):
...
with mp.Pool(N) as pool:
results = [pool.apply_async(ssh_op, args=(arg,)) for arg in my_args]
logger.info('place 1')
ret, msg= parse_results(results, timeout=10)
if ret:
return
else:
check_job_result.apply_async(args=(my_args,), dountdown=5)
### run check_job_result
如果我注释掉mp.Pool
周围的代码,并构造虚拟ret
和msg
,则不会发生句柄泄漏。像这样:
@shared_task
def check_job_result(my_args):
...
# with mp.Pool(N) as pool:
# results = [pool.apply_async(ssh_op, args=(arg,)) for arg in my_args]
# logger.info('place 1')
# ret, msg= parse_results(results, timeout=10)
return = False
msg = 'is still running'
if ret:
return
else:
check_job_result.apply_async(args=(my_args,), dountdown=5)
我不知道为什么以及如何解决它!
celery==4.3.0
django-celery-results==1.0.4
billiard==3.6.0.0
python==3.7.4