Question

我想将nginx访问日志发送到远程syslog-ng服务器。我在每一侧安装了syslog-ng（服务器 - 客户端）。客户：10.10.10.2 服务器：10.10.10.1 一些日志文件（messages，syslog，mail.log）从客户端成功发送到服务器，但不发送nginx日志。服务器配置：

source s_net {
   tcp(ip(0.0.0.0) port(1999)
   tls( key_file("/etc/syslog-ng/key.d/privkey.pem")
        cert_file("/etc/syslog-ng/cert.d/cacert.pem")
        peer_verify(optional-untrusted)) ); };

destination d_net_nginx_access { file("/mnt/syslog_storage/HOSTS/$HOST
/nginx.access.log"); };

filter f_nginx_access { program("nginx") };

log { source(s_net); filter(f_nginx_access); destination(d_net_nginx_access); };

客户端配置：

    source s_src {
       system();
       internal();
};

destination tls_log {
tcp("10.10.10.1" port(1999)
tls( ca_dir("/etc/syslog-ng/ca.d")) );};

destination d_nginx_access { file("/var/log/nginx/nginx.access.log"); };

filter f_nginx { program("nginx"); };

log { source(s_src); filter(f_nginx); destination(d_nginx_access); };

# All messages send to a remote site
#
log { source(s_src); destination(tls_log); };

我现在不是什么问题。提前谢谢。

Answer 1

问题是在客户端上，您将Nginx日志作为目标访问。目的地 - 顾名思义 - 你放的东西。 syslog-ng不读取目的地。邮件日志和其他邮件通过网络的原因是因为它们直接登录到syslog()，因此syslog-ng不必读取单独的文件。 Nginx不使用syslog()服务（除非您配置它，最近版本有syslog插件），因此nginx日志不会到达syslog-ng，因此您设置的过滤器也不会捕获它们。

现在，要传输nginx日志，请将客户端上的nginx目标更改为以下内容：

source s_nginx_access { file("/var/log/nginx/nginx.access.log" flags(no-parse)); }; log { source(s_src); source(s_nginx_access); destination(tls_log); };

flags(no-parse)存在，因为nginx不会以正常的syslog格式登录。

Answer 2

为syslog-ng服务器构建docker

FROM ubuntu:xenial

ENV LD_LIBRARY_PATH=/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/server:$LD_LIBRARY_PATH
ENV PATH=$PATH:$LD_LIBRARY_PATH

RUN apt-get update --fix-missing && apt-get install -y wget \
  curl \
  tzdata \
  apt-transport-https \
  nano \
  gnupg
#RUN locale-gen ru_RU.UTF-8  
#ENV LANG ru_RU.UTF-8  
#ENV LANGUAGE ru_RU:en  
#ENV LC_ALL ru_RU.UTF-8  

RUN apt-get update && apt-get install -y openjdk-8-jdk



RUN wget -P . http://download.opensuse.org/repositories/home:/laszlo_budai:/syslog-ng/xUbuntu_17.04/Release.key; apt-key add Release.key
RUN echo "deb http://download.opensuse.org/repositories/home:/laszlo_budai:/syslog-ng/xUbuntu_16.10 ./" > /etc/apt/sources.list.d/syslog-ng-obs.list
RUN echo "deb http://download.opensuse.org/repositories/home:/laszlo_budai:/syslog-ng/xUbuntu_16.10 ./" > /etc/apt/sources.list.d/syslog-ng-obs.list
RUN cd /tmp && wget http://archive.ubuntu.com/ubuntu/pool/main/j/json-c/libjson-c3_0.12.1-1.2_amd64.deb && dpkg -i libjson-c3_0.12.1-1.2_amd64.deb
RUN apt-get update  && \
    apt-get install -y syslog-ng-core 
RUN apt-get update  && \
    apt-get install -y \
    #syslog-ng-mod-basicfuncs-plus \
    syslog-ng-mod-smtp \
    #syslog-ng-mod-trigger \
    syslog-ng-mod-pgsql \
    syslog-ng-core \
    syslog-ng-mod-kafka \
    #syslog-ng-mod-lua \
    syslog-ng-mod-amqp \
    syslog-ng-mod-snmptrapd-parser \
    syslog-ng-mod-sqlite3 \
    syslog-ng-mod-elastic \
    syslog-ng-mod-geoip \
    syslog-ng-mod-java-common-lib \
   # syslog-ng-mod-rss \
    syslog-ng-mod-add-contextual-data \
    syslog-ng-mod-sql \
    syslog-ng-mod-redis \
    syslog-ng-mod-http \
    syslog-ng-mod-stardate \
    syslog-ng-mod-java \
    syslog-ng-mod-geoip2 \
    syslog-ng-mod-mysql \
    #syslog-ng-mod-elasticsearch \
    syslog-ng-mod-graphite \
    #syslog-ng-mod-perl \
    syslog-ng-mod-stomp \
    syslog-ng-mod-json \
    syslog-ng-mod-freetds \
    syslog-ng-mod-mongodb \
    syslog-ng-mod-map-value-pairs \
    syslog-ng-mod-python \
    syslog-ng-mod-java-http \
    syslog-ng-mod-riemann \
    #syslog-ng-mod-getent \
    syslog-ng-mod-pacctformat \
    syslog-ng-mod-hdfs
RUN ldconfig -v 
RUN echo "LD_LIBRARY_PATH=/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/server:$LD_LIBRARY_PATH" >> /etc/default/syslog-ng
#STOPSIGNAL SIGTERM
ENTRYPOINT ["/usr/sbin/syslog-ng", "-F"]

使用docker-compose的swarm模式进行部署

---
version: '3.3'
services:

  syslogngserver:
    build: ./build/syslog-ng/.
    hostname: syslogng-server
    ports:
      - "514:514"
      - "514:514/udp" 
      - "25000-25020:25000-25020"
      - "25000-25020:25000-25020/udp"
    environment:
       - "TZ=Asia/Yekaterinburg" 
    networks:
      monitoring-net:
        aliases:
          - syslogng-server
          - syslogng
    deploy:
         #mode: replicated
         replicas: 1
         #endpoint mode: dnsrr
         # service resource management
         resources:
           # Hard limit - Docker does not allow to allocate more
           limits:
             cpus: '1'
             memory: 512M
           # Soft limit - Docker makes best effort to return to it
           reservations:
             memory: 32M
         # service restart policy
         restart_policy:
           condition: on-failure
           delay: 5s
           max_attempts: 3
           window: 20s
         # service update configuration
         update_config:
           parallelism: 1
           delay: '3s'
           failure_action: continue
           monitor: '3s'
           max_failure_ratio: 0.3
         # placement constraint - in this case on 'worker' nodes only
         placement:
           constraints: [node.role == manager]

##NETWORKS
networks:
  monitoring-net:
    driver: overlay

部署

export $(cat .env) && docker stack deploy --compose-file=docker-compose.yml monitoring

配置nginx日志格式并配置发送日志到syslog-ng服务器

log_format full_format '"$time_iso8601"|"$host"|"$http_host"|"$remote_addr"|"$http_x_forwarded_for"|"$request_method"|"$request"|"$status"|"$upstream_status"|"$body_bytes_sent"|"$http_referer"|"$request_time"|"$upstream_response_time"|"$upstream_http_x_cache"|"$uri"|"$upstream_addr"|"$upstream_response_length"|"$server_name"|"$upstream_cache_status"|"$http_user_agent"|"$scheme://$host$request_uri"|"$request_body"';   
access_log syslog:server=syslogng-server:25000 full_format;
error_log syslog:server=syslogng-server:25001 warn;

在syslog-ng服务器上 - 配置输入，解析并发送到elaticsearch

##Принимаем  nginx access логи на порт 25230 по tcp, можно использовать что-то одно
source udp-nginxaccessgeoip2 {
udp(port(25000)
log_iw_size(1000)
log_fetch_limit(1000000));
};

###Парсим принятые логи nginx, здесь мы указываем имена полей, куда будут попадать данные, укажем также разделитель полей символ "|"
parser p-nginx-acessfull-mapped {
    csv-parser(columns("nginx.time","nginx.host", "nginx.http_host", "nginx.remote_addr", "nginx.http_x_forwarded_for", "nginx.request_method", "nginx.request","nginx.status", "nginx.upstream_status","nginx.body_bytes_sent", "nginx.http_referer", "nginx.request_time","nginx.upstream_response_time", "nginx.upstream_http_x_cache", "nginx.uri", "nginx.upstream_addr",  "nginx.upstream_response_length", "nginx.server_name","nginx.upstream_cache_status", "nginx.user_agent","nginx.request_uri","nginx.cookie_bar" )
         flags(escape-double-char,strip-whitespace)
         delimiters("|")
         quote-pairs('""[]')
         );
};

###Здесь мы сверяем прилетающий ip из логов nginx -  в поле "nginx.remote_addr" с нашей базой данных maxmidn geoip2
parser p_geoip2 { geoip2( "${nginx.remote_addr}", prefix( "geoip2." ) database( "/etc/syslog-ng/GeoLite2-City.mmdb" ) ); };

##Rewrite необходим для того, чтобы информация о местоположении находилась в форме, ожидаемой Elasticsearch. Он выглядит несколько более сложным, чем для первой версии анализатора GeoIP, поскольку имеется больше информации и информация теперь структурирована.

rewrite r_geoip2 {
    set(
        "${geoip2.location.latitude},${geoip2.location.longitude}",
        value( "geoip2.location2" ),
        condition(not "${geoip2.location.latitude}" == "")
    );
};

###Направляем nginx_
destination d_elastic-nginx-acessfull {
  elasticsearch2(
    index("nginxaccess-${YEAR}.${MONTH}")
    type("test")
    time-zone("UTC")
    client_mode("http")
    flush-limit("100")
    cluster_url("http://elasticsearch:9200")
    custom_id("${UNIQID}")
    template("$(format-json --scope rfc5424 --scope nv-pairs --exclude DATE --key ISODATE)")
    type("netdata")
    persist-name(elasticsearch-netdata)

  );
};

log {source (udp-nginxaccessgeoip2);
parser(p-nginx-acessfull-mapped);
rewrite(r_geoip2);
parser(p_geoip2);
destination(d_elastic-nginx-acessfull);};



#Принимаем  nginx error логи на порт 25231 по udp
source udp-server_nginx_error {
udp(port(25001)
log_iw_size(1000)
log_fetch_limit(1000000));
};

destination d_elastic-nginxerror {
  elasticsearch2(
    index("nginxerror-${YEAR}.${MONTH}")
    type("nginxerror")
    time-zone("UTC")
    client_mode("http")
    flush-limit("100")
    cluster_url("http://elasticsearch:9200")
    custom_id("${UNIQID}")
    persist-name(elasticsearch-nginxerror)
    template("$(format_json --scope nv_pairs --key ISODATE @timestamp=${ISODATE})")
  );
};

###########фильтры  - исключим unlink

filter f_tail_nginx_error_unlink {not match ("unlink");};

log {source (udp-server_nginx_error);
filter(f_tail_nginx_error_unlink);
destination(d_elastic-nginxerror);};

Wget GeoLite2-City.mmdb

cd /etc/syslog-ng/
wget http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.tar.gz
tar -xvf GeoLite2-City.tar.gz

将2018年和03月的映射放入elasticsearch（这是针对第6版）

curl -H "Content-Type: application/json" -XPUT 'http://elasticsearch:9200/nginx-2018.03' -d \
'{
   "mappings" : {
      "_default_" : {
        "properties" : {
            "nginx" : {
               "properties" : {
                  "request_time": {"type": "integer","ignore_malformed": true},
                  "upstream_response_time": {"type": "integer","ignore_malformed": true},
                  "remote_addr": {"type": "ip"},
                  "status": {"type": "integer","ignore_malformed": true},
                  "body_bytes_sent": {"type": "integer","ignore_malformed": true},
                  "upstream_response_length": {"type": "integer","ignore_malformed": true},
                  "request_uri": {"type": "text","fields": {"keyword": {"type": "keyword"}
                                                           }
                                                                       }
                                                                                          }
                              },

            "geoip2" : {
               "properties" : {
                  "location2" : {"type" : "geo_point"}
                              }
      }
    }
  }
}
}
}'

nginx将日志发送到远程syslog服务器

2 个答案: