Question

我只是在学习Elasticsearch，我需要知道如何正确地将配置文件分成多个文件。我正在使用官方logstash on docker，并将端口绑定在9600和5044上。最初，我有一个正常工作的logstash文件，没有这样的条件：

input {
    beats {
        port => '5044'
    }
}

filter
{
    grok{
        match => {
            "message" => "%{TIMESTAMP_ISO8601:timestamp} \[(?<event_source>[\w\s]+)\]:\[(?<log_type>[\w\s]+)\]:\[(?<id>\d+)\] %{GREEDYDATA:details}"
            "source" => "%{GREEDYDATA}\\%{GREEDYDATA:app}.log"
        }
    }
    mutate{
        convert => { "id" => "integer" }
    }
    date {
        match => [ "timestamp", "ISO8601" ]
        locale => en
        remove_field => "timestamp"
    }
}


output
{
    elasticsearch {
        hosts => ["http://elastic:9200"]
        index => "logstash-supportworks"
    }

}

当我想添加metricbeat时，我决定将该配置拆分为一个新文件。所以我最终得到了3个文件：

__ input.conf

input {
    beats {
        port => '5044'
    }
}

metric.conf

# for testing I'm adding no filters just to see what the data looks like

output {
  if ['@metadata']['beat'] == 'metricbeat' {
    elasticsearch {
        hosts => ["http://elastic:9200"]
        index => "%{[@metadata][beat]}-%{[@metadata][version]}" 
    }
  }
}

supportworks.conf

filter
{
    if ["source"] =~ /Supportwork Server/ {
        grok{
            match => {
                "message" => "%{TIMESTAMP_ISO8601:timestamp} \[(?<event_source>[\w\s]+)\]:\[(?<log_type>[\w\s]+)\]:\[(?<id>\d+)\] %{GREEDYDATA:details}"
                "source" => "%{GREEDYDATA}\\%{GREEDYDATA:app}.log"
            }
        }
        mutate{
            convert => { "id" => "integer" }
        }
        date {
            match => [ "timestamp", "ISO8601" ]
            locale => en
            remove_field => "timestamp"
        }
    }
}


output
{
    if ["source"] =~ /Supportwork Server/ {
        elasticsearch {
            hosts => ["http://elastic:9200"]
            index => "logstash-supportworks"
        }
    }

}

现在没有数据正在发送到ES实例。我已经验证了filebeat至少正在运行并发布消息，所以我希望至少能看到这么多关于ES的信息。这是我的服务器上运行filebeat的已发布消息

2019-03-06T09:16:44.634-0800    DEBUG   [publish]       pipeline/processor.go:308       Publish event: {
  "@timestamp": "2019-03-06T17:16:44.634Z",
  "@metadata": {
    "beat": "filebeat",
    "type": "doc",
    "version": "6.6.1"
  },
  "source": "C:\\Program Files (x86)\\Hornbill\\Supportworks Server\\log\\swserver.log",
  "offset": 4773212,
  "log": {
    "file": {
      "path": "C:\\Program Files (x86)\\Hornbill\\Supportworks Server\\log\\swserver.log"
    }
  },
  "message": "2019-03-06 09:16:42 [COMMS]:[INFO ]:[4924] Helpdesk API (5005) Socket error while idle - 10053",
  "prospector": {
    "type": "log"
  },
  "input": {
    "type": "log"
  },
  "beat": {
    "name": "WIN-22VRRIEO8LM",
    "hostname": "WIN-22VRRIEO8LM",
    "version": "6.6.1"
  },
  "host": {
    "name": "WIN-22VRRIEO8LM",
    "architecture": "x86_64",
    "os": {
      "platform": "windows",
      "version": "6.3",
      "family": "windows",
      "name": "Windows Server 2012 R2 Standard",
      "build": "9600.0"
    },
    "id": "e5887ac2-6fbf-45ef-998d-e40437066f56"
  }
}

Answer 1

通过在 __ input.conf 中添加变异过滤器以在源字段中将反斜杠替换为正斜杠来实现此目的

filter {
    mutate{
        gsub => [ "source", "[\\]", "/" ]
    }
}

然后在我的条件中从字段访问器中删除"

if ["source"] =~ /Supportwork Server/

蜜饯

if [source] =~ /Supportwork Server/

要使此配置正常工作，两项更改似乎都是必需的。

使用多个配置文件进行Logstash

1 个答案: