发送任务不会偶尔使用celery + rabbitmq

时间：2015-07-20 06:31:18

标签： python rabbitmq celery

我做了一个帖子网站使用烧瓶。当用户发帖时，我使用celery + rabbitmq告诉它的朋友。但是，芹菜偶尔不起作用，大部分时间都是正常的。

详情如下。

这是我的芹菜配置：

from celery import Celery
# from celery.schedules import crontab
from kombu import Queue, Exchange
from nowdo.config import setting
# from datetime import timedelta


class BaseConfig(object):
    CELERY_ACKS_LATE = True
    CELERY_TIMEZONE = 'Asia/Shanghai'
    CELERY_QUEUES = (
        Queue('nowdo_default', Exchange('nowdo_default'),  routing_key='nowdo_default'),
        Queue('nowdo_file_extract', Exchange('nowdo_file_extract'), routing_key='nowdo.file.extract'),
        Queue('nowdo_send_mail', Exchange("nowdo_send_mail"), routing_key='nowdo.sendmail'),
        Queue('nowdo_create_trend_showed', Exchange('nowdo_create_trend_showed'),
          routing_key='nowdo.trend_showed.create'),
        Queue('nowdo_refresh_some_trend_showed_by_user', Exchange('nowdo_refresh_some_trend_showed_by_user'),
          routing_key='nowdo.trend_showed.refresh_some_by_user'),
        Queue('nowdo_create_some_trend_showed_by_trend', Exchange('nowdo_create_some_trend_showed_by_trend'),
          routing_key='nowdo.trend_showed.create_some_by_trend'),
        Queue('nowdo_pull_web_file', Exchange('nowdo_pull_web_file'),
          routing_key='nowdo.pull.web_file'),
        Queue('nowdo_delete_some_trend_showed', Exchange('nowdo_delete_some_trend_showed'),
          routing_key='nowdo.trend_showed.delete_some_trend_showed'),
        Queue('nowdo_send_group_notice', Exchange('nowdo_send_group_notice'),
          routing_key='nowdo.notice.send_group_notice'),
        Queue('nowdo_send_group_new_task_notice', Exchange('nowdo_send_group_new_task_notice'),
          routing_key='nowdo.notice.send_group_new_task_notice'),
        Queue('nowdo_send_some_group_new_task_notice', Exchange('nowdo_send_some_group_new_task_notice'),
          routing_key='nowdo.notice.send_some_group_new_task_notice'),
    # not use
        Queue('nowdo_create_text_result_cache', Exchange('nowdo_create_text_result_cache'),
          routing_key='nowdo.task_cache.create_text_result_cache'),
)

CELERY_DEFAULT_ROUTING_KEY = "nowdo_default"
CELERY_DEFAULT_QUEUE = 'nowdo_default'
CELERY_DEFAULT_EXCHANGE = "nowdo_default"
CELERY_DEFAULT_EXCHANGE_TYPE = "direct"

CELERY_ROUTES = {
    "nowdo.tasks.translate.extract_worker": {
        "routing_key": "nowdo.file.extract",
        "queue": "nowdo_file_extract",
    },
    "nowdo.utils.mail.send_mail": {
        "queue": "nowdo_send_mail",
        "routing_key": "nowdo.sendmail",
    },
    "nowdo.tasks.trend_showed.create_trend_showed_worker": {
        "routing_key": "nowdo.trend_showed.create",
        "queue": "nowdo_create_trend_showed",
    },
    "nowdo.tasks.trend_showed.refresh_some_trend_showed_by_user_worker": {
        "routing_key": "nowdo.trend_showed.refresh_some_by_user",
        "queue": "nowdo_refresh_some_trend_showed_by_user",
    },
    "nowdo.tasks.trend_showed.create_some_trend_showed_by_trend_worker": {
        "routing_key": "nowdo.trend_showed.create_some_by_trend",
        "queue": "nowdo_create_some_trend_showed_by_trend",
    },
    "nowdo.tasks.pull_web_file.pull_web_file_worker": {
        "routing_key": "nowdo.pull.web_file",
        "queue": "nowdo_pull_web_file",
    },
    "nowdo.tasks.trend_showed.delete_some_trend_showed_worker": {
        "routing_key": "nowdo.trend_showed.delete_some_trend_showed",
        "queue": "nowdo_delete_some_trend_showed",
    },
    "nowdo.tasks.notice.send_group_notice_worker": {
        "routing_key": "nowdo.notice.send_group_notice",
        "queue": "nowdo_send_group_notice",
    },
    "nowdo.tasks.notice.send_group_new_task_notice_worker": {
        "routing_key": "nowdo.notice.send_group_new_task_notice",
        "queue": "nowdo_send_group_new_task_notice",
    },
    "nowdo.tasks.notice.send_some_group_new_task_notice_worker": {
        "routing_key": "nowdo.notice.send_some_group_new_task_notice",
        "queue": "nowdo_send_some_group_new_task_notice",
    },
    # not use
    "nowdo.tasks.task_cache.create_text_result_cache": {
        "routing_key": "nowdo.task_cache.create_text_result_cache",
        "queue": "nowdo_create_text_result_cache",
    },
}

CELERY_IMPORTS = (
    'nowdo.tasks.pull_web_file',
    'nowdo.tasks.translate',
    'nowdo.tasks.trend_showed',
    'nowdo.tasks.task_cache',
    'nowdo.utils.mail',
    'nowdo.tasks.notice',
)

#: Email settings
CELERY_SEND_TASK_ERROR_EMAILS = True
ADMINS = (
    ('zouyingjun', 'zouyingjun@xingcloud.com'),
    ('fuluheng', 'fuluheng@xingcloud.com'),
    ('wangweiwei', 'wangweiwei@xingcloud.com')
)
SERVER_EMAIL = 'DNFS stat info<xcmonitor01@163.com>'
EMAIL_HOST = 'smtp.163.com'
EMAIL_HOST_USER = 'xcmonitor01@163.com'
EMAIL_HOST_PASSWORD = 'xingcloud'
EMAIL_PORT = 25
EMAIL_USE_TLS = True
EMAIL_TIMEOUT = 10
BROKER_URL = setting.BROKER_URL

celery = Celery()
celery.config_from_object(BaseConfig)

这是我的任务：

import traceback
import time
from nowdo.config.celeryconfig import celery
from nowdo.controls.group import GroupUserRel
from nowdo.controls.notice import Notice
from nowdo.controls.session import session_cm
from nowdo.utils.nowdo_logger import nowdo_celery_logger as logger


@celery.task(ignore_result=True)
def send_group_notice_worker(sender_id, receiver_id_list, group_notice_id):
    try:
        with session_cm() as db_session:
            Notice.send_to_many(db_session, Notice.NOTICE_TYPE_GROUP_NOTICE, sender_id,
                                receiver_id_list, group_notice_id)
    except:
        logger.error(traceback.format_exc())


@celery.task(ignore_result=True, default_retry_delay=5, max_retries=10)
def send_group_new_task_notice_worker(sender_id, group_id, task_id):
    a = time.time()
    with session_cm() as db_session:
        b = time.time()

        user_ids = db_session.query(GroupUserRel.user_id).filter(GroupUserRel.group_id == group_id)\
            .filter(GroupUserRel.user_id != sender_id).all()
        c = time.time()
        user_id_list = [u[0] for u in user_ids]
        d = time.time()
        size = len(user_id_list)
        block_len = 20
        e = time.time()
        user_id_lists = [user_id_list[i:i+block_len] for i in range(0, size, block_len)]
        f = time.time()
        for u in user_id_lists:
            celery.send_task("nowdo.tasks.notice.send_some_group_new_task_notice_worker", [sender_id, u, task_id])
            # send_some_group_new_task_notice_worker(sender_id, u, task_id)
        g = time.time()
        load_session, count_user_ids, get_id_list, defined_variable, get_id_lists, send_every_task = b - a, c - b, d - c, e - d, f - e, g - f
        if load_session > 10:
            logger.warn('load_session problem')
            # print 'load_session problem'
        if count_user_ids > 10:
            logger.warn('count_user_ids problem')
            # print 'count_user_ids problem'
        if get_id_list > 10:
            logger.warn('get_id_list problem')
            # print 'get_id_list problem'
        if defined_variable > 10:
            logger.warn('defined_variable problem')
            # print 'defined_variable problem'
        if get_id_lists > 10:
            logger.warn('get_id_lists problem')
            # print 'get_id_lists problem'
        if send_every_task > 10:
            logger.warn('send_every_task problem')
            # print 'send_every_task problem'


@celery.task(ignore_result=True, default_retry_delay=5, max_retries=10)
def send_some_group_new_task_notice_worker(sender_id, user_id_list, task_id):
    with session_cm() as db_session:
        a = time.time()
        Notice.send_to_many(db_session, Notice.NOTICE_TYPE_CREATE_GROUP_TASK,
                            sender_id, user_id_list, task_id)
        if time.time() - a > 10:
            logger.warn('send_to_many problem')
            # print 'send_to_many problem'


if __name__ == '__main__':
    pass

这是代码的一部分; 这是视图中的句柄：

@task_bp.route('/create_task', methods=['POST'])
@ajax_login_required()
@talk_forbidden_check
def create_task():
            # this is create a task
            ......
            t1_1 = time.time()
            if group_id:
                group = db_session.query(Group).filter(Group.id == group_id).first()
            else:
                group = None
            t1_2 = time.time()
            new_task, new_trends = CrowdSourceTask.create_crowd_source_task(
                db_session,
                current_user,
                task_name,
                src_lang,
                status,
                tags,
                group=group,
                content=task_content,
                max_translate_count=50,
            )

            t1_3 = time.time()
            ......
            res.update({
                "status": "create",
                "data": FormattedTrends.get_formatted_trends_list([new_trends], current_user)[0]
            })
            t2_1, t2_2, t2_3, t2_4, t2_5 = t1_2 - t1_1, t1_3 - t1_2, t1_4 - t1_3, t1_5 - t1_4, time.time() - t1_5
            if t2_1 > 10:
                current_app.logger.error("group handle problem")

            if t2_2 > 10:
                current_app.logger.error("create_crowd_source_task handle problem")

            if t2_3 > 10:
                current_app.logger.error("add score handle problem")

            if t2_4 > 10:
                current_app.logger.error("remove_draft handle problem")

            if t2_5 > 10:
                current_app.logger.error("get_formatted_trends_list handle problem")

            return jsonify(res)

这是控件中的句柄：

    @classmethod
    def create_crowd_source_task(cls, session, current_user, name, src_lang, status, tags_str,
                             description=None,
                             group=None,
                             content=None,
                             uploaded_file=None,
                             # glossary_table_id=None,
                             max_translate_count=100,
                             start_date=None, end_date=None,
                             reserved_words=None,
                             ):
        t3_1 = time.time()
        ......
        t3_5 = time.time()
        # after comit
        if group:
            celery.send_task('nowdo.tasks.notice.send_group_new_task_notice_worker',
                             [current_user.id, group.id, crowd_task.id])
            # from nowdo.tasks.notice import send_group_new_task_notice_worker
            # send_group_new_task_notice_worker(current_user.id, group.id, crowd_task.id)
        t3_6 = time.time()
        if status:
            crowd_task.active(src_lang)
        t3_7 = time.time()
        group_id = group.id if group else None


        # crowd_task.init_empty_progress()

        trends = Trends.add_trends(current_user, TREND_ACTION_CREATE, TREND_TARGET_TYPE_TASK, crowd_task, group_id)

        t4_1, t4_2, t4_3, t4_4, t4_5, t4_6, t4_7 = t3_2 - t3_1, t3_3 - t3_2, t3_4 - t3_3, t3_5 - t3_4, t3_6 - t3_5,\
            t3_7 - t3_6, time.time() - t3_7

        if t4_1 > 10:
            logger.warn("create_orig_file handle problem")
            # print "create_orig_file handle problem"

        if t4_2 > 10:
            logger.warn("CrowdSourceTaskContent handle problem")
            # print "CrowdSourceTaskContent handle problem"

        if t4_3 > 10:
            logger.warn("update_tags handle problem")
            # print "update_tags handle problem"

        if t4_4 > 10:
            logger.warn("CrowdSourceTaskTagRel handle problem")
            # print "CrowdSourceTaskTagRel handle problem"

        if t4_5 > 10:
            logger.warn("send_task handle problem")
            # print "send_task handle problem"

        if t4_6 > 10:
            logger.warn("crowd_task.active handle problem")
            # print "crowd_task.active handle problem"

        if t4_7 > 10:
            logger.warn("add_trends handle problem")
            # print "add_trends handle problem"

        return crowd_task, trends

这是我的日志： enter image description here 我找到了代码

celery.send_task('nowdo.tasks.notice.send_group_new_task_notice_worker',
                                 [current_user.id, group.id, crowd_task.id])

完成不起作用，以便nginx超时。

我希望我的描述准确无误，请帮帮我。

0 个答案:

没有答案
相关问题
最新问题