Question

我正在尝试将structlog集成到使用multiprocessing的应用程序中。当我计划完全切换到structlog时，我也想捕获第三方库的stdlib logging调用。由于我们计划发出JSON日志文件和键值输出，因此我认为的正确方法是将实际格式委派给logging：

    logging   \
               structlog processors (worker) -> logging queuehandler -> logging handler (main process) -> stdlib logging output
    structlog /

根据https://docs.python.org/3/howto/logging-cookbook.html#logging-to-a-single-file-from-multiple-processes，我们使用QueueHandler将所有日志消息发送到主进程进行写入。

这是我当前的配置：

import logging.handlers
import multiprocessing
import sys
import traceback

import structlog
from structlog.processors import JSONRenderer, KeyValueRenderer


def proc_info(logger, method, event_dict):
    event_dict['worker-process'] = multiprocessing.current_process().name
    return event_dict

pre_chain = [
    structlog.stdlib.add_log_level,
    proc_info
]

def configure_root_logging():
    logging.config.dictConfig({
            "version": 1,
            "disable_existing_loggers": False,
            "formatters": {
                "plain": {
                    "()": structlog.stdlib.ProcessorFormatter,
                    "processor": KeyValueRenderer(),
                    "foreign_pre_chain": pre_chain,
                },
                "json": {
                    "()": structlog.stdlib.ProcessorFormatter,
                    "processor": JSONRenderer(),
                    "foreign_pre_chain": pre_chain,
                },
            },
            "handlers": {
                "console_kvp": {
                    "level": "DEBUG",
                    "class": "logging.StreamHandler",
                    "formatter": "json",
                },
                "console_json": {
                    "level": "DEBUG",
                    "class": "logging.StreamHandler",
                    "formatter": "plain",
                },
            },
            "loggers": {
                "": {
                    "handlers": ["console_kvp", "console_json"],
                    "level": "DEBUG",
                    "propagate": True,
                },
            }
    })


def configure_structlog():
    structlog.configure(
        processors=pre_chain + [structlog.stdlib.ProcessorFormatter.wrap_for_formatter],
        context_class=dict,
        logger_factory=structlog.stdlib.LoggerFactory(),
        wrapper_class=structlog.stdlib.BoundLogger,
        cache_logger_on_first_use=True,
    )


def setup_worker_logging(logging_queue):
    root = logging.getLogger()
    queue_handler = logging.handlers.QueueHandler(logging_queue)
    root.addHandler(queue_handler)
    root.setLevel(logging.INFO)


def worker_entrypoint(logging_queue):
    setup_worker_logging(logging_queue)
    configure_structlog()

    logger = logging.getLogger(__name__)
    logger.warning("worker_startup_std_logging")

    slogger = structlog.get_logger()
    slogger.warning("worker_startup_structlog")


def handle_logging_queue_messages(queue):
    while True:
        try:
            record = queue.get()
            logger = logging.getLogger(record.name)
            logger.handle(record)
        except KeyboardInterrupt:
            break
        except Exception:
            print('LOGGING FAILED!!! ', file=sys.stderr)
            traceback.print_exc(file=sys.stderr)


def main():
    logging_queue = multiprocessing.Queue()

    t = multiprocessing.Process(
        target=worker_entrypoint, args=[logging_queue], daemon=True,
        name='worker-9999')
    t.start()

    configure_root_logging()
    configure_structlog()

    handle_logging_queue_messages(logging_queue)

    slogger = structlog.get_logger()
    slogger.warning("Startup")


main()

注意：在调试过程中，我已经将StreamHandler用于两种输出格式，但是JSON格式的输出最终将到达FileHandler，因此介于两者之间。

输出为：

{"event": "worker_startup_std_logging", "level": "warning", "worker-process": "MainProcess"}
event='worker_startup_std_logging' level='warning' worker-process='MainProcess'
{"event": "{'event': 'worker_startup_structlog', 'level': 'warning', 'worker-process': 'worker-9999'}", "level": "warning", "worker-process": "MainProcess"}
event="{'event': 'worker_startup_structlog', 'level': 'warning', 'worker-process': 'worker-9999'}" level='warning' worker-process='MainProcess'

这里有多个问题：

worker-process字段具有主进程的值，而不是辅助进程的值
event记录器的structlog字段包含event_dict的序列化值，其中包含worker-process的正确值

我希望structlog在移交给logging的QueueHandler之前先评估处理器链。

有人可以解释在这种情况下structlog应该如何工作吗？

多处理中的structlog和stdlib日志记录配置

0 个答案: