Question

我们正在使用Ansible并拥有Logstash。

如何编写一些测试来覆盖我们的自定义logstash过滤器？我想做的是：

给出带有过滤器的Logstash配置
传递日志行（或多行日志记录），
看到它成功解析成碎片。

我知道有这个 - https://github.com/elastic/logstash/wiki/Tips:Testing-your-filters，但我不明白它是如何有用的 - 它看起来已经过时了。

Answer 1

我找到了this，最后得到了以下工作测试代码：

# simple_filter_spec.rb
#
# run using:
#   bundle exec rspec simple_filter_spec.rb

require "logstash/devutils/rspec/spec_helper"

LogStash::Environment::LOGSTASH_HOME = `gem which logstash-core`
module LogStash::Environment
  unless self.method_defined?(:pattern_path)
    def pattern_path(path)
      ::File.join(LOGSTASH_HOME, "patterns", path)
    end
  end
end


require "logstash/filters/grok"

describe LogStash::Filters::Grok do
  config <<-CONFIG
  filter {
    grok {
      match => { "message" => "%{SYSLOGLINE}" }
      singles => true
      overwrite => [ "message" ]
    }
  }
  CONFIG

  sample "Mar 16 00:01:25 evita postfix/smtpd[1713]: connect from camomile.cloud9.net[168.100.1.3]" do
    insist { subject["tags"] }.nil?
    insist { subject["logsource"] } == "evita"
    insist { subject["timestamp"] } == "Mar 16 00:01:25"
    insist { subject["message"] } == "connect from camomile.cloud9.net[168.100.1.3]"
    insist { subject["program"] } == "postfix/smtpd"
    insist { subject["pid"] } == "1713"
  end
end

我的Gemfile看起来像这样：

source 'https://www.rubygems.org'

platform :jruby do
  gem 'pry'
  gem 'rspec'
  gem 'logstash-core'
  gem 'logstash-devutils'
  gem 'logstash-filter-grok'
end

Answer 2

gmile给出的答案适用于logstash 1.5版。从那时起，grok模式从核心中删除并置于logstash-core-patterns中。在logstash 2.2中对我有用的是：

# encoding: utf-8

require 'spec_helper'
require "logstash/patterns/core"

# solution based on https://github.com/logstash-plugins/logstash-filter-grok/blob/master/spec/filters/grok_spec.rb
module LogStash::Environment
  # running the grok code outside a logstash package means
  # LOGSTASH_HOME will not be defined, so let's set it here
  # before requiring the grok filter

  # the path that is set is the plugin root path
  unless self.const_defined?(:LOGSTASH_HOME)
    LOGSTASH_HOME = File.expand_path("../../../", __FILE__)
  end

  # also :pattern_path method must exist so we define it too

  # method is called by logstash-filter-grok to create patterns_path array
  #
  #   logstash-filter-grok/lib/logstash/filters/grok.rb(line ~230):
  #
  #   @@patterns_path += [
  #     LogStash::Patterns::Core.path,
  #     LogStash::Environment.pattern_path("*")
  #
  # patterns defined in spec/patterns/ will be joined to the array by grok 

  unless self.method_defined?(:pattern_path)
    def pattern_path(path)
      ::File.join(LOGSTASH_HOME, "spec", "patterns", path)
    end
  end
end

require "logstash/filters/grok"
require "logstash/filters/<tested-plugin>"

规范示例的其余部分仍然有效。

随着新的依赖性要求的到来，Gemfile也必须改变。我的gemspec依赖是这样的：

# Gem dependencies
s.add_runtime_dependency "logstash-core-plugin-api", "~> 1.0"
s.add_runtime_dependency "<tested-plugin>"
s.add_development_dependency 'logstash-devutils', '~> 0'
s.add_development_dependency 'logstash-filter-grok', '~> 3.2'
s.add_development_dependency 'logstash-patterns-core', '~> 4.0'

可以找到完整的工作原型here

Answer 3

您可以使用Logstash-Tester - 一个小工具来对您的logstash过滤器和模式进行单元测试。您在json中编写测试用例，logstash-tester使用docker容器在logstash上运行它们。（免责声明：我写了这个工具）

Answer 4

来自this blog：

$ git clone https://github.com/elastic/logstash
$ cd logstash
$ git checkout 2.1
$ rake bootstrap
$ rake test:install-core

您应该查看实际上正在运行的标记版本的logstash，而不是检查2.1分支。 v2.3.2（注意领先＆＃34; v＆＃34;）。

运行上述命令后，您可以在logstash repo中运行bin/rspec /some/path/your_filter_spec.rb。

重要：我发现编码行# encoding: utf-8是必需的，否则匹配将失败。

示例测试文件：

# encoding: utf-8

require "spec_helper"

describe "simple request log" do
  config (<<-CONFIG)
  filter {
    grok {
      match => { "message" => "%{IP:client} %{WORD:method} %{URIPATHPARAM:request} %{NUMBER:bytes} %{NUMBER:duration}" }
    }
  }
  CONFIG

  sample '55.3.244.1 GET /index.html 15824 0.043' do
    insist { subject['client']   } == '55.3.244.1'
    insist { subject['method']   } == 'GET'
    insist { subject['request']  } == '/index.html'
    insist { subject['bytes']    } == '15824'
    insist { subject['duration'] } == '0.043'
  end
end

测试自定义Logstash过滤器

4 个答案: