PyGelf＆amp;使Graypy的CPU达到99％

Question

我们只是从 Graylog v1.3.3 转移到 Graylog 2.0.3（f07c170），如果请求不大，它的效果会很好。

我们正在使用Flask uWSGI python 3.5，请求大约每分钟1000次，并且每分钟记录大约2,700 - 3,000条消息。

我们尝试了什么。

• Graypy library作为Graylog2的托运人，结果是CPU 99％
• PyGelf library作为Graylog2的托运人，结果是CPU 99％

logger_info.py

import logging
class LoggerInfo(logging.Filter):
    def __init__(self, message_id=None, full_message=None, feed_id=None):
        self.message_id = message_id
        self.full_message = full_message
        self.feed_id = feed_id

    def filter(self, record):
        record.message_id = self.message_id
        record.full_message = self.full_message
        record.feed_id = self.feed_id
        return True

log_graylog2_adapter.py（Pygelf）

from config import setting
from pygelf import GelfUdpHandler
import logging


class LogGraylog2Adapter():
    gelf_version = '1.1'

    def __init__(self, feed_id):
        self.feed_id = feed_id
        self.init_logger(feed_id)
    def init_logger(self, feed_id):
        try:
            logging.basicConfig(level=logging.INFO)
            self.my_logger = logging.getLogger(setting.LOG_NAME)
            self.my_logger.addHandler(
                GelfUdpHandler(host='127.0.0.1', port=12201,
                               compress=True, chunk_size=1350, include_extra_fields=True))
        except Exception as er:
            printd('graylog_error %s' % er)

    def log(self, logger_info):
        try:
            self.my_logger.addFilter(logger_info)
            self.my_logger.info(logger_info.full_message)
        except Exception as er:
            printd('graylog_error %s' % er)

log_graylog2_adapter.py（GrayPy）

import logging
import graypy


class LogGraylog2Adapter():
    gelf_version = '1.1'

    def __init__(self, feed_id):
        self.server = '127.0.0.1'
        self.port = 12201
        self.feed_id = feed_id
        self.init_logger(feed_id)

    def init_logger(self, feed_id):
        self.my_logger = logging.getLogger('logname')
        self.my_logger.setLevel(logging.INFO)
        self.init_handler()

    def init_handler(self):
        try:
            self.handler = graypy.GELFHandler(self.server, self.port, debugging_fields=False,
                                              localname='hostname')
            self.my_logger.addHandler(self.handler)
        except Exception as er:
            printd('graylog_error %s' % er)

    def log(self, logger_info):
        try:
            self.my_logger.addFilter(logger_info)
            self.my_logger.info(logger_info.full_message)
        except Exception as er:
            printd('graylog_error %s' % er)

每个请求都在脚本下运行。

from log_graylog2_adapter import LogGraylog2Adapter
from logger_info import LoggerInfo

logger = LogGraylog2Adapter('feed_id')
log_info = LoggerInfo(
   message_id='message_id',
   feed_id='feed_id',
   full_message='full_message'
)
logger.log(log_info)

关于此问题的任何想法。

Answer 1

我使用Fluentd代替它，效果很好。