Logstash多次分析日志

时间：2019-02-19 11:20:55

标签： logstash kibana elastic-stack

我已经设置了ELK，我正在尝试解析多个日志文件。我正在将日志文件从生产服务器传输到使用RSYNC安装ELK的服务器。

现在，问题是logstash多次解析一个特定的日志（16次）。我希望此日志仅解析一次。

我没有使用filebeat。

我正在将logstash执行为“ ./logstash -f logstash.conf”。

logstash conf的内容如下：

input
{

  file
  {
                path => "/home/finassure/datadog/logs/node1/LISRVR_CDCI_SWIF_*.log"
                start_position => "beginning"
                sincedb_path => "/home/finassure/ELK/lastlog"  
                type => "cbclog1"
                codec => multiline
                {
                        pattern => "^(Pid:\s[0-9]{3}|[0-9]{4}|[0-9]{5}|[0-9]{6}\sReceived At:)|^(Pid:\s[0-9]{3}|[0-9]{4}|[0-9]{5}|[0-9]{6}\sSent At:)|^(MessageId:)|^(Field\s[0-9][0-9][0-9]:)"
                        negate => false
                        what => "next"
                }

  }
}

filter {
if [type] == "cbclog1" {
    if "Received At:" in [message]
        {
        mutate {
            add_field => { "cdci_msg_type" => "request" }
        }
    }

    if "Sent At:" in [message]
        {
        mutate {
            add_field => { "cdci_msg_type" => "response" }
        }
    }

    if [cdci_msg_type] not in ["request", "response"] {
        drop { }
    }

    mutate {
        gsub => ['message', "\n", " "]
        gsub => ['message', "\t", " "]
    }




    if [cdci_msg_type] in ["response","request"] {
        grok {
          match => {
              message => "(Pid:)\s+(%{INT:cdci_pid})\s+(Sent|Received)\s(At:)\s+(?<cdci_timestamp>([0-3][0-9]\/[0-1][0-9]\/[1|2][0|9][0-9]{2}\s[0-2][0-9](\:[0-5][0-9]){2})\.[0-9]{3})\s+(MessageId:)\s+(%{INT:cdci_msg_id})\s+(Field 002)\:\s+(?<cdci_field_002>(\S+))\s+(Field 003)\:\s+(?<cdci_field_003>(\S+))\s+(Field 004)\:\s+(?<cdci_field_004>(\S+))\s+(Field 011)\:\s+(?<cdci_field_011>(\S+))\s+(Field 012)\:\s+(?<cdci_field_012>(\S+))\s+(Field 017)\:\s+(?<cdci_field_017>(\S+))\s+(Field 024)\:\s+(?<cdci_field_024>(\S+))\s+(Field 032)\:\s+(?<cdci_field_032>(\S+))\s+(Field 033)\:\s+(?<cdci_field_033>(\S+))\s+(Field 037)\:\s+(?<cdci_field_037>(\S+))\s+(Field 041)\:\s+(?<cdci_field_041>(\S+))\s+(Field 042)\:\s+(?<cdci_field_042>(\S+))\s+(Field 043)\:\s+(?<cdci_field_043>(\S+))\s+(Field 049)\:\s+(?<cdci_field_049>(\S+))\s+(Field 059)\:\s+(?<cdci_field_059>(\S+))\s+(%{GREEDYDATA:custom})"
                  }
        overwrite => ["message"]
        add_field => { "msg_type" => "cbclogs" }
break_on_match => "true"
                }
    }



if "_grokparsefailure" in [tags]
 {
      grok
          {


        match => { message => "(Pid:)\s+(%{INT:cdci_pid})\s+(Sent|Received)\s(At:)\s+(?<cdci_timestamp>([0-3][0-9]\/[0-1][0-9]\/[1|2][0|9][0-9]{2}\s[0-2][0-9](\:[0-9][0-9]){2})\.[0-9]{3})\s+(MessageId:)\s+(%{INT:cdci_msg_id})\s+(Field 002)\:\s+(?<cdci_field_002>(\S+))\s+(Field 003)\:\s+(?<cdci_field_003>(\S+))\s+(Field 004)\:\s+(?<cdci_field_004>(\S+))\s+(Field 011)\:\s+(?<cdci_field_011>(\S+))\s+(Field 012)\:\s+(?<cdci_field_012>(\S+))\s+(Field 017)\:\s+(?<cdci_field_017>(\S+))\s+(Field 032)\:\s+(?<cdci_field_032>(\S+))\s+(Field 033)\:\s+(?<cdci_field_033>(\S+))\s+(Field 037)\:\s+(?<cdci_field_037>(\S+))\s+(Field 038)\:\s+(?<cdci_field_038>(\S+))\s+(Field 039)\:\s+(?<cdci_field_039>(\S+))\s+(%{GREEDYDATA:custom })" }
        remove_tag => ["_grokparsefailure"]
        add_field => { "msg_type" => "cbclogs" }
        break_on_match => "true"
      }
  }




if "_grokparsefailure" in [tags]
        {
        grok
                {

        match =>
                {

                message => "(Pid:)\s+(%{INT:cdci_pid})\s+(Sent|Received)\s(At:)\s+(?<cdci_timestamp>([0-3][0-9]\/[0-1][0-9]\/[1|2][0|9][0-9]{2}\s[0-2][0-9](\:[0-9][0-9]){2})\.[0-9]{3})\s+(MessageId:)\s+(%{INT:cdci_msg_id})\s+(Field 002)\:\s+(?<cdci_field_002>(\S+))\s+(Field 003)\:\s+(?<cdci_field_003>(\S+))\s+(Field 004)\:\s+(?<cdci_field_004>(\S+))\s+(Field 011)\:\s+(?<cdci_field_011>(\S+))\s+(Field 012)\:\s+(?<cdci_field_012>(\S+))\s+(Field 017)\:\s+(?<cdci_field_017>(\S+))\s+(Field 024)\:\s+(?<cdci_field_024>(\S+))\s+(Field 032)\:\s+(?<cdci_field_032>(\S+))\s+(Field 033)\:\s+(?<cdci_field_033>(\S+))\s+(Field 037)\:\s+(?<cdci_field_037>(\S+))\s+(Field 041)\:\s+(?<cdci_field_041>(\S+))\s+(Field 042)\:\s+(?<cdci_field_042>(\S+))\s+(Field 043)\:\s+(?<cdci_field_043>(\S+))\s+(Field 049)\:\s+(?<cdci_field_049>(\S+))\s+"
        }
        remove_tag => ["_grokparsefailure"]
        add_field => { "msg_type" => "cbclogs" }
        break_on_match => "true"
        }
    }




        if "_grokparsefailure" in [tags]
        {
      grok
          {

        match => { "message" => "(Pid:)\s+(%{INT:cdci_pid})\s+(Sent|Received)\s(At:)\s+(?<cdci_timestamp>([0-3][0-9]\/[0-1][0-9]\/[1|2][0|9][0-9]{2}\s[0-2][0-9](\:[0-9][0-9]){2})\.[0-9]{3})\s+(MessageId:)\s+(%{INT:cdci_msg_id})\s+(Field 002)\:\s+(?<cdci_field_002>(\S+))\s+(Field 003)\:\s+(?<cdci_field_003>(\S+))\s+(Field 004)\:\s+(?<cdci_field_004>(\S+))\s+(Field 011)\:\s+(?<cdci_field_011>(\S+))\s+(Field 012)\:\s+(?<cdci_field_012>(\S+))\s+(Field 017)\:\s+(?<cdci_field_017>(\S+))\s+(Field 024)\:\s+(?<cdci_field_024>(\S+))\s+(Field 032)\:\s+(?<cdci_field_032>(\S+))\s+(Field 037)\:\s+(?<cdci_field_037>(\S+))\s+(Field 041)\:\s+(?<cdci_field_041>(\S+))\s+(Field 042)\:\s+(?<cdci_field_042>(\S+))\s+(Field 049)\:\s+(?<cdci_field_049>(\S+))\s+(Field 056)\:\s+(?<cdci_field_056>(\S+))\s+(Field 059)\:\s+(?<cdci_field_059>(\S+))\s+" }
        remove_tag => ["_grokparsefailure"]
        add_field => { "msg_type" => "cbclogs" }
        break_on_match => "true"
      }
    }




        if "_grokparsefailure" in [tags]
                {
      grok
          {

        match =>
                {
                message => "(Pid:)\s+(%{INT:cdci_pid})\s+(Sent|Received)\s(At:)\s+(?<cdci_timestamp>([0-3][0-9]\/[0-1][0-9]\/[1|2][0|9][0-9]{2}\s[0-2][0-9](\:[0-9][0-9]){2})\.[0-9]{3})\s+(MessageId:)\s+(%{INT:cdci_msg_id})\s+(Field 002)\:\s+(?<cdci_field_002>(\S+))\s+(Field 003)\:\s+(?<cdci_field_003>(\S+))\s+(Field 004)\:\s+(?<cdci_field_004>(\S+))\s+(Field 011)\:\s+(?<cdci_field_011>(\S+))\s+(Field 012)\:\s+(?<cdci_field_012>(\S+))\s+(Field 017)\:\s+(?<cdci_field_017>(\S+))\s+(Field 032)\:\s+(?<cdci_field_032>(\S+))\s+(Field 037)\:\s+(?<cdci_field_037>(\S+))\s+(Field 038)\:\s+(?<cdci_field_038>(\S+))\s+(Field 039)\:\s+(?<cdci_field_039>(\S+))\s+(Field 041)\:\s+(?<cdci_field_041>(\S+))\s+(Field 042)\:\s+(?<cdci_field_042>(\S+))\s+(%{GREEDYDATA:custom })"
        }
        remove_tag => ["_grokparsefailure"]
        add_field => { "msg_type" => "cbclogs" }
        break_on_match => "true"
      }


}



if "_grokparsefailure" in [tags]
        {
        grok
                {

        match =>
                {

                        message => "(Pid:)\s+(%{INT:cdci_pid})\s+(Sent|Received)\s(At:)\s+(?<cdci_timestamp>([0-3][0-9]\/[0-1][0-9]\/[1|2][0|9][0-9]{2}\s[0-2][0-9](\:[0-9][0-9]){2})\.[0-9]{3})\s+(MessageId:)\s+(%{INT:cdci_msg_id})\s+(Field 002)\:\s+(?<cdci_field_002>(\S+))\s+(Field 003)\:\s+(?<cdci_field_003>(\S+))\s+(Field 004)\:\s+(?<cdci_field_004>(\S+))\s+(Field 011)\:\s+(?<cdci_field_011>(\S+))\s+(Field 012)\:\s+(?<cdci_field_012>(\S+))\s+(Field 017)\:\s+(?<cdci_field_017>(\S+))\s+(Field 024)\:\s+(?<cdci_field_024>(\S+))\s+(Field 032)\:\s+(?<cdci_field_032>(\S+))\s+(Field 037)\:\s+(?<cdci_field_037>(\S+))\s+(Field 041)\:\s+(?<cdci_field_041>(\S+))\s+"
        }
        remove_tag => ["_grokparsefailure"]
        add_field => { "msg_type" => "cbclogs" }
        break_on_match => "true"
        }
    }





if "_grokparsefailure" in [tags]
        {
        grok
                {

        match =>
                {

        message => "(Pid:)\s+(%{INT:cdci_pid})\s+(Sent|Received)\s(At:)\s+(?<cdci_timestamp>([0-3][0-9]\/[0-1][0-9]\/[1|2][0|9][0-9]{2}\s[0-2][0-9](\:[0-9][0-9]){2})\.[0-9]{3})\s+(MessageId:)\s+(%{INT:cdci_msg_id})\s+(Field 002)\:\s+(?<cdci_field_002>(\S+))\s+(Field 003)\:\s+(?<cdci_field_003>(\S+))\s+(Field 004)\:\s+(?<cdci_field_004>(\S+))\s+(Field 011)\:\s+(?<cdci_field_011>(\S+))\s+(Field 012)\:\s+(?<cdci_field_012>(\S+))\s+(Field 017)\:\s+(?<cdci_field_017>(\S+))\s+(Field 032)\:\s+(?<cdci_field_032>(\S+))\s+(Field 033)\:\s+(?<cdci_field_033>(\S+))\s+(Field 037)\:\s+(?<cdci_field_037>(\S+))\s+(Field 038)\:\s+(?<cdci_field_038>(\S+))\s+(Field 039)\:\s+(?<cdci_field_039>(\S+))\s+"

        }
        remove_tag => ["_grokparsefailure"]
        add_field => { "msg_type" => "cbclogs" }
        break_on_match => "true"
        }
    }

##############HANDLE CUSTOM ECHO MESSAGE MSCR REQUEST####################

if "_grokparsefailure" in [tags]
        {
        grok
                {

        match =>
                {

        message => "(Pid:)\s+(%{INT:cdci_pid})\s+(Sent|Received)\s(At:)\s+(?<cdci_timestamp>([0-3][0-9]\/[0-1][0-9]\/[1|2][0|9][0-9]{2}\s[0-2][0-9](\:[0-9][0-9]){2})\.[0-9]{3})\s+(MessageId:)\s+(%{INT:cdci_msg_id})\s+(Field 002)\:\s+(?<cdci_field_002>(\S+))\s+(Field 003)\:\s+(?<cdci_field_003>(\S+))\s+(Field 004)\:\s+(?<cdci_field_004>(\S+))\s+(Field 011)\:\s+(?<cdci_field_011>(\S+))\s+(Field 012)\:\s+(?<cdci_field_012>(\S+))\s+(Field 017)\:\s+(?<cdci_field_017>(\S+))\s+(Field 024)\:\s+(?<cdci_field_024>(\S+))\s+(Field 032)\:\s+(?<cdci_field_032>(\S+))\s+(Field 037)\:\s+(?<cdci_field_037>(\S+))\s+(Field 049)\:\s+(?<cdci_field_049>(\S+))\s+"

        }
        remove_tag => ["_grokparsefailure"]
        add_field => { "msg_type" => "cbclogs" }
        break_on_match => "true"
        }
    }




if "_grokparsefailure" in [tags]
        {
        grok
                {

        match =>
                {

        message => "(Pid:)\s+(%{INT:cdci_pid})\s+(Sent|Received)\s(At:)\s+(?<cdci_timestamp>([0-3][0-9]\/[0-1][0-9]\/[1|2][0|9][0-9]{2}\s[0-2][0-9](\:[0-9][0-9]){2})\.[0-9]{3})\s+(MessageId:)\s+(%{INT:cdci_msg_id})\s+(Field 002)\:\s+(?<cdci_field_002>(\S+))\s+(Field 003)\:\s+(?<cdci_field_003>(\S+))\s+(Field 004)\:\s+(?<cdci_field_004>(\S+))\s+(Field 011)\:\s+(?<cdci_field_011>(\S+))\s+(Field 012)\:\s+(?<cdci_field_012>(\S+))\s+(Field 017)\:\s+(?<cdci_field_017>(\S+))\s+(Field 032)\:\s+(?<cdci_field_032>(\S+))\s+(Field 037)\:\s+(?<cdci_field_037>(\S+))\s+(Field 038)\:\s+(?<cdci_field_038>(\S+))\s+(Field 039)\:\s+(?<cdci_field_039>(\S+))\s+"

        }
        remove_tag => ["_grokparsefailure"]
         add_field => { "msg_type" => "cbclogs" }
        break_on_match => "true"
        }
    }
}


}
output {

  stdout { codec => rubydebug }
if [type] == "cbclog1"
{
  elasticsearch {
      hosts => "10.0.100.167:9200"
      index => "cbclog1"
    }
}



}

0 个答案:

没有答案
相关问题
最新问题