Question

我正在运行heroku logs --ps celery --tail并看到以下内容：

Error R14 (Memory quota exceeded)
Process running mem=650M(127.1%)

DEBUG设置为False。

芹菜版本为3.1.9

我不知道如何追查这个内存泄漏。我采取的第一步是打印出所有与Celery相关的配置设置，这些设置粘贴在下面。我对其他有关如何调试此问题的想法感兴趣。我可以在Celery任务中查看正在使用的内存量吗？我是否必须明确告诉它释放内存？还有其他想法吗？

芹菜配置设置：

{
    "CELERY_RESULT_DB_TABLENAMES": null,
    "BROKER_TRANSPORT_OPTIONS": {},
    "BROKER_USE_SSL": false,
    "CELERY_BROADCAST_QUEUE": "celeryctl",
    "CELERY_STORE_ERRORS_EVEN_IF_IGNORED": false,
    "CELERY_CREATE_MISSING_QUEUES": true,
    "CELERY_DEFAULT_QUEUE": "celery",
    "CELERY_SEND_TASK_SENT_EVENT": false,
    "CELERYD_TASK_TIME_LIMIT": null,
    "CELERY_EVENT_QUEUE_EXPIRES": null,
    "CELERY_DEFAULT_EXCHANGE_TYPE": "direct",
    "CELERYBEAT_SCHEDULER": "djcelery.schedulers.DatabaseScheduler",
    "CELERY_MAX_CACHED_RESULTS": 5000,
    "CELERY_RESULT_PERSISTENT": null,
    "CELERYD_POOL": "prefork",
    "CELERYD_AGENT": null,
    "CELERY_CACHE_BACKEND_OPTIONS": {},
    "BROKER_HEARTBEAT": null,
    "CELERY_RESULT_ENGINE_OPTIONS": null,
    "CELERY_RESULT_SERIALIZER": "json",
    "CELERYBEAT_SCHEDULE_FILENAME": "celerybeat-schedule",
    "CELERY_REDIRECT_STDOUTS_LEVEL": "WARNING",
    "CELERY_IMPORTS": [],
    "CELERYD_TASK_LOG_FORMAT": "[%(asctime)s: %(levelname)s/%(processName)s] %(task_name)s[%(task_id)s]: %(message)s",
    "CELERY_SECURITY_CERTIFICATE": null,
    "CELERYD_LOG_COLOR": null,
    "CELERY_RESULT_EXCHANGE": "celeryresults",
    "CELERY_TRACK_STARTED": false,
    "CELERY_REDIS_PASSWORD": null,
    "BROKER_USER": null,
    "CELERY_COUCHBASE_BACKEND_SETTINGS": null,
    "CELERY_RESULT_EXCHANGE_TYPE": "direct",
    "CELERY_REDIS_DB": null,
    "CELERYD_TIMER_PRECISION": 1.0,
    "CELERY_REDIS_PORT": null,
    "BROKER_TRANSPORT": null,
    "CELERYMON_LOG_FILE": null,
    "CELERYD_CONCURRENCY": 0,
    "CELERYD_HIJACK_ROOT_LOGGER": true,
    "BROKER_VHOST": null,
    "CELERY_DEFAULT_EXCHANGE": "celery",
    "CELERY_DEFAULT_ROUTING_KEY": "celery",
    "CELERY_ALWAYS_EAGER": false,
    "CELERYD_TASK_SOFT_TIME_LIMIT": null,
    "CELERY_WORKER_DIRECT": false,
    "CELERY_REDIS_HOST": null,
    "CELERY_QUEUE_HA_POLICY": null,
    "BROKER_PORT": null,
    "CELERYD_AUTORELOADER": "celery.worker.autoreload:Autoreloader",
    "BROKER_CONNECTION_TIMEOUT": 4,
    "CELERY_ENABLE_REMOTE_CONTROL": true,
    "CELERY_RESULT_DB_SHORT_LIVED_SESSIONS": false,
    "CELERY_EVENT_SERIALIZER": "json",
    "CELERY_REDIS_MAX_CONNECTIONS": null,
    "CELERY_CACHE_BACKEND": null,
    "CELERYD_PREFETCH_MULTIPLIER": 4,
    "BROKER_PASSWORD": null,
    "CELERY_MONGODB_BACKEND_SETTINGS": null,
    "CELERY_EAGER_PROPAGATES_EXCEPTIONS": false,
    "CELERY_IGNORE_RESULT": false,
    "CELERYMON_LOG_LEVEL": "INFO",
    "CELERY_DISABLE_RATE_LIMITS": false,
    "CELERY_TASK_PUBLISH_RETRY_POLICY": {
        "interval_start": 0,
        "interval_max": 1,
        "max_retries": 3,
        "interval_step": 0.2
    },
    "CELERY_SECURITY_KEY": null,
    "CELERY_BROADCAST_EXCHANGE_TYPE": "fanout",
    "CELERY_DEFAULT_RATE_LIMIT": null,
    "CELERYD_POOL_PUTLOCKS": true,
    "CELERY_TASK_SERIALIZER": "json",
    "CELERYD_WORKER_LOST_WAIT": 10.0,
    "CELERYD_POOL_RESTARTS": false,
    "CELERY_TASK_PUBLISH_RETRY": true,
    "CELERY_ENABLE_UTC": true,
    "CELERY_SEND_EVENTS": false,
    "BROKER_CONNECTION_MAX_RETRIES": 100,
    "CELERYD_LOG_FILE": null,
    "CELERYD_FORCE_EXECV": false,
    "CELERY_CHORD_PROPAGATES": true,
    "CELERYD_AUTOSCALER": "celery.worker.autoscale:Autoscaler",
    "CELERYD_STATE_DB": null,
    "CELERY_ROUTES": null,
    "BROKER_HEARTBEAT_CHECKRATE": 3.0,
    "CELERY_ACCEPT_CONTENT": [
        "json"
    ],
    "BROKER_LOGIN_METHOD": null,
    "BROKER_CONNECTION_RETRY": true,
    "CELERY_TIMEZONE": null,
    "CELERYBEAT_MAX_LOOP_INTERVAL": 0,
    "CELERYD_LOG_LEVEL": "WARN",
    "CELERY_REDIRECT_STDOUTS": true,
    "BROKER_POOL_LIMIT": 10,
    "CELERY_SECURITY_CERT_STORE": null,
    "CELERYD_CONSUMER": "celery.worker.consumer:Consumer",
    "CELERY_INCLUDE": [],
    "CELERYD_MAX_TASKS_PER_CHILD": null,
    "CELERYD_LOG_FORMAT": "[%(asctime)s: %(levelname)s/%(processName)s] %(message)s",
    "CELERY_ANNOTATIONS": null,
    "CELERY_MESSAGE_COMPRESSION": null,
    "CELERY_SEND_TASK_ERROR_EMAILS": false,
    "CELERY_QUEUES": null,
    "CELERY_ACKS_LATE": false,
    "CELERYMON_LOG_FORMAT": "[%(asctime)s: %(levelname)s] %(message)s",
    "BROKER_HOST": null,
    "CELERY_EVENT_QUEUE_TTL": null,
    "BROKER_FAILOVER_STRATEGY": null,
    "CELERY_RESULT_BACKEND": "djcelery.backends.database:DatabaseBackend",
    "CELERY_BROADCAST_EXCHANGE": "celeryctl",
    "CELERYBEAT_LOG_FILE": null,
    "CELERYD_TIMER": null,
    "CELERY_RESULT_DBURI": null,
    "CELERY_DEFAULT_DELIVERY_MODE": 2,
    "CELERYBEAT_LOG_LEVEL": "INFO"
}

Answer 1

嗯，不确定这是否是一个答案，但这里有一些我已经学到的东西。

首先，这是我的Heroku procfile：

web: gunicorn my_app.wsgi
celery: celery worker -A my_app -l info -c 3 --beat -b amqp://K5r-y7B1:1xxxxxxxi44x6j6PgfOJcWUksQ-Qyw1@yellow-thistle-3.bigwig.lshift.net:10653/kbOAUITC91D

其次，Heroku实际上以多线程方式运行这个芹菜工人。这与我的开发设置不同。结果是开发工作一个接一个地运行，但在Heroku上，长期工作不会阻止更短的工作运行。基本上，Heroku将应用程序的多个实例加载到内存中，而在开发时，一次只加载一个实例。

第三，这可能不是内存泄漏，确切地说。根据我的理解，在内存使用超过400％之前不会发生任何不良事在任何情况下，我的应用程序已运行数月，此警告不断发生，但没有任何负面影响。

Answer 2

有一个名为objgraph的celery inspect命令，可以生成用于内存审核的图表。您需要先点安装objgraph，然后运行：

celery -A Project inspect objgraph

https://javier.gr/blog/2014/04/python-reference-leaks-and-objgraph/ https://github.com/celery/celery/issues/3436

Celery - Heroku上的Django Web应用程序与Celery工作进程 - 我怎样才能追逐这个内存泄漏？

2 个答案: