Logstash筛选器插件指纹因未知原因失败

时间：2018-08-29 08:58:27

标签： elasticsearch logstash

几天以来，我一直在尝试解决logstash失败的问题。

1。。我正在使用http_poller下载IP数据库。每次拉取大约有30k IP地址。

2。。我正在解析http_poller的输入以提取ipv4address。然后，我想从IP计算哈希并将其插入elasticsearch。如果下一次哈希已经存在，则我不会创建新条目，而是进行更新。通过配置，它可以（部分）运行。

3。。如果我没有打开Logstash的调试模式，则可以处理3k条记录。在调试模式下，我能够处理28k。怎么了？为什么配置仍然会失败？

我的配置文件： 

input {
      http_poller {
           urls => {
           blocklist_de_all => "http://lists.blocklist.de/lists/all.txt"
      }
      request_timeout => 30
      tags => ["blocklist"]
      codec => "line"
      validate_after_inactivity => 200
      schedule => { cron => "*/30 * * * *" }
      metadata_target => "feed_metadata"
      }
      }
filter {
      split {
      field => "[message]"
      }
     if ([message] =~ /^#/) {
        drop{}
     }
     else {
          grok {
          match => { "message" => "^%{GREEDYDATA:**ipv4address**}" }
          }
     }
     geoip {
           source => "ipv4address"
           add_tag => [ "ipv4enriched" ]
           add_field => [ "[geoip][coordinates]", "%{[geoip][longitude]}" ]
           add_field => [ "[geoip][coordinates]", "%{[geoip][latitude]}" ]
     }
     mutate {
            convert => [ "[geoip][coordinates]", "float"]
     }

     fingerprint {
                 id => "blocklist1"
                 **source => [ "ipv4address" ]**
                 method => [ "SHA512" ]
                 **add_tag => [ "fingerprinted" ]**
    }
   }

output {
       elasticsearch {
       hosts => ["10.0.50.51:9200"]
       index => "ipv4_to_block"
       **document_id => "%{fingerprint}"**
       document_type => "default"
       }
}

管道配置： 

pipeline.id: blocklist_ips
path.config: "/etc/logstash/conf.d/blocklist_de_all_low_confidence.conf"
pipeline.workers: 16

Kibana看到以下json输出： 

{
"_index": "ipv4_to_block",
"_type": "default",
**"_id": "1eda4277c8b054652a08b0f56f26656babbe8328"**,
"_version": 1,
"_score": 1,
"_source": {
**"fingerprint": "1eda4277c8b054652a08b0f56f26656babbe8328",**
"@version": "1",
"metadata": {
"host": "elk2",
"name": "blocklist_ips",
"request": {
"method": "get",
"url": "http://lists.blocklist.de/lists/all.txt"
},
"code": 200,
"response_message": "OK",
"runtime_seconds": 0.115168,
"times_retried": 0,
"response_headers": {
"connection": "keep-alive",
"content-type": "text/plain; charset=UTF-8",
"transfer-encoding": "chunked",
"date": "Tue, 28 Aug 2018 14:16:55 GMT",
"last-modified": "Tue, 28 Aug 2018 14:14:10 GMT",
"cache-control": "public",
"x-frame-options": "sameorigin",
"keep-alive": "timeout=20",
"server": "nginx/1.12.2",
"etag": "W/"6550b-5747f74a5da2f""
}
},
"tags": [
"blocklist",
"_geoip_lookup_failure",
**"fingerprinted"**
],
**"ipv4address": "103.115.180.188",**
"@timestamp": "2018-08-28T14:21:00.411Z",
"message": "103.115.180.188",
"geoip": {}
},
"fields": {
"@timestamp": [
"2018-08-28T14:21:00.411Z"
]
}
}

我从logstash-plain.log中得到的错误：

[2018-08-28T16:24:01,488][ERROR][logstash.pipeline ] Exception in pipelineworker, the pipeline stopped processing new events, please check your filter configuration and restart Logstash. {:pipeline_id=>"blocklist_ips", "exception"=>"8", "backtrace"=>["org.bouncycastle.crypto.digests.LongDigest.update(Unknown Source)", "org.bouncycastle.crypto.digests.LongDigest.finish(Unknown Source)", "org.bouncycastle.crypto.digests.SHA512Digest.doFinal(Unknown Source)", "org.bouncycastle.jcajce.provider.digest.BCMessageDigest.engineDigest(Unknown Source)", "java.security.MessageDigest.digest(MessageDigest.java:365)", "org.jruby.ext.openssl.Digest.finish(Digest.java:204)", "org.jruby.ext.openssl.Digest$INVOKER$i$0$0$finish.call(Digest$INVOKER$i$0$0$finish.gen)", "org.jruby.RubyClass.finvoke(RubyClass.java:557)", "org.jruby.runtime.Helpers.invoke(Helpers.java:399)", "org.jruby.RubyBasicObject.callMethod(RubyBasicObject.java:354)", "org.jruby.ext.digest.RubyDigest$DigestInstance.digest(RubyDigest.java:320)", "org.jruby.ext.digest.RubyDigest$DigestInstance.hexdigest(RubyDigest.java:339)", "org.jruby.ext.digest.RubyDigest$DigestInstance$INVOKER$s$0$1$hexdigest.call(RubyDigest$DigestInstance$INVOKER$s$0$1$hexdigest.gen)", "org.jruby.internal.runtime.methods.JavaMethod$JavaMethodN.call(JavaMethod.java:721)", "org.jruby.internal.runtime.methods.DynamicMethod.call(DynamicMethod.java:200)", "org.jruby.runtime.callsite.CachingCallSite.call(CachingCallSite.java:161)", "org.jruby.ir.interpreter.InterpreterEngine.processCall(InterpreterEngine.java:314)", "org.jruby.ir.interpreter.StartupInterpreterEngine.interpret(StartupInterpreterEngine.java:73)", "org.jruby.ir.interpreter.InterpreterEngine.interpret(InterpreterEngine.java:83)", "org.jruby.internal.runtime.methods.MixedModeIRMethod.INTERPRET_METHOD(MixedModeIRMethod.java:179)", "org.jruby.internal.runtime.methods.MixedModeIRMethod.call(MixedModeIRMethod.java:165)", "org.jruby.internal.runtime.methods.DynamicMethod.call(DynamicMethod.java:200)", "org.jruby.internal.runtime.methods.AliasMethod.call(AliasMethod.java:61)", "usr.share.logstash.vendor.bundle.jruby.$2_dot_3_dot_0.gems.logstash_minus_filter_minus_fingerprint_minus_3_dot_2_dot_0.lib.logstash.filters.fingerprint.RUBY$block$filter$4(/usr/share/logstash/vendor/bundle/jruby/2.3.0/gems/logstash-filter-fingerprint-3.2.0/lib/logstash/filters/fingerprint.rb:140)", "org.jruby.runtime.CompiledIRBlockBody.yieldDirect(CompiledIRBlockBody.java:156)", "org.jruby.runtime.BlockBody.yield(BlockBody.java:114)", "org.jruby.runtime.Block.yield(Block.java:165)", "org.jruby.RubyArray.each(RubyArray.java:1734)", "usr.share.logstash.vendor.bundle.jruby.$2_dot_3_dot_0.gems.logstash_minus_filter_minus_fingerprint_minus_3_dot_2_dot_0.lib.logstash.filters.fingerprint.RUBY$method$filter$0(/usr/share/logstash/vendor/bundle/jruby/2.3.0/gems/logstash-filter-fingerprint-3.2.0/lib/logstash/filters/fingerprint.rb:135)", "usr.share.logstash.logstash_minus_core.lib.logstash.filters.base.RUBY$method$do_filter$0(/usr/share/logstash/logstash-core/lib/logstash/filters/base.rb:143)", "usr.share.logstash.logstash_minus_core.lib.logstash.filters.base.RUBY$block$multi_filter$1(/usr/share/logstash/logstash-core/lib/logstash/filters/base.rb:162)", "org.jruby.runtime.CompiledIRBlockBody.yieldDirect(CompiledIRBlockBody.java:156)", "org.jruby.runtime.BlockBody.yield(BlockBody.java:114)", "org.jruby.runtime.Block.yield(Block.java:165)", "org.jruby.RubyArray.each(RubyArray.java:1734)", "usr.share.logstash.logstash_minus_core.lib.logstash.filters.base.RUBY$method$multi_filter$0(/usr/share/logstash/logstash-core/lib/logstash/filters/base.rb:159)", "usr.share.logstash.logstash_minus_core.lib.logstash.filter_delegator.RUBY$method$multi_filter$0(/usr/share/logstash/logstash-core/lib/logstash/filter_delegator.rb:44)", "org.jruby.internal.runtime.methods.CompiledIRMethod.call(CompiledIRMethod.java:103)", "org.jruby.internal.runtime.methods.MixedModeIRMethod.call(MixedModeIRMethod.java:163)", "org.jruby.internal.runtime.methods.DynamicMethod.call(DynamicMethod.java:200)", "org.jruby.runtime.callsite.CachingCallSite.call(CachingCallSite.java:161)", "org.jruby.ir.interpreter.InterpreterEngine.processCall(InterpreterEngine.java:314)", "org.jruby.ir.interpreter.StartupInterpreterEngine.interpret(StartupInterpreterEngine.java:73)", "org.jruby.ir.interpreter.Interpreter.INTERPRET_BLOCK(Interpreter.java:132)", "org.jruby.runtime.MixedModeIRBlockBody.commonYieldPath(MixedModeIRBlockBody.java:148)", "org.jruby.runtime.IRBlockBody.call(IRBlockBody.java:73)", "org.jruby.runtime.Block.call(Block.java:124)", "org.jruby.RubyProc.call(RubyProc.java:289)", "org.jruby.internal.runtime.methods.ProcMethod.call(ProcMethod.java:63)", "org.jruby.internal.runtime.methods.DynamicMethod.call(DynamicMethod.java:204)", "org.jruby.internal.runtime.methods.DynamicMethod.call(DynamicMethod.java:200)", "usr.share.logstash.logstash_minus_core.lib.logstash.pipeline.RUBY$method$filter_batch$0(/usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:340)", "usr.share.logstash.logstash_minus_core.lib.logstash.pipeline.RUBY$method$worker_loop$0(/usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:319)", "usr.share.logstash.logstash_minus_core.lib.logstash.pipeline.RUBY$block$start_workers$2(/usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:285)", "org.jruby.runtime.CompiledIRBlockBody.callDirect(CompiledIRBlockBody.java:145)", "org.jruby.runtime.IRBlockBody.call(IRBlockBody.java:71)", "org.jruby.runtime.Block.call(Block.java:124)", "org.jruby.RubyProc.call(RubyProc.java:289)", "org.jruby.RubyProc.call(RubyProc.java:246)", "org.jruby.internal.runtime.RubyRunnable.run(RubyRunnable.java:104)", "java.lang.Thread.run(Thread.java:748)"], :thread=>"#<Thread:0x812622b sleep>"}
[2018-08-28T16:24:01,490][ERROR][logstash.pipeline ] Exception in pipelineworker, the pipeline stopped processing new events, please check your filter configuration and restart Logstash. {:pipeline_id=>"blocklist_ips", "exception"=>"8", "backtrace"=>["org.bouncycastle.crypto.digests.LongDigest.update(Unknown Source)", "org.bouncycastle.crypto.digests.LongDigest.update(Unknown Source)", "org.bouncycastle.jcajce.provider.digest.BCMessageDigest.engineUpdate(Unknown Source)", "java.security.MessageDigest.update(MessageDigest.java:325)", "org.jruby.ext.openssl.Digest.update(Digest.java:192)", "org.jruby.ext.openssl.Digest$INVOKER$i$1$0$update.call(Digest$INVOKER$i$1$0$update.gen)",
...

打开Logstash调试：

curl -XPUT 'localhost:9600/_node/logging?pretty' -H 'Content-Type: application/json' -d'
{
"logger.logstash.agent" : "DEBUG",
"logger.logstash.api.service" : "DEBUG",
"logger.logstash.codecs.json" : "DEBUG",
"logger.logstash.codecs.line" : "DEBUG",
"logger.logstash.codecs.plain" : "DEBUG",
"logger.logstash.config.source.local.configpathloader" : "DEBUG",
"logger.logstash.config.source.multilocal" : "DEBUG",
"logger.logstash.config.sourceloader" : "DEBUG",
"logger.logstash.configmanagement.extension" : "DEBUG",
"logger.logstash.filters.drop" : "DEBUG",
"logger.logstash.filters.grok" : "DEBUG",
"logger.logstash.filters.split" : "DEBUG",
"logger.logstash.inputs.http_poller" : "DEBUG",
"logger.logstash.instrument.periodicpoller.deadletterqueue" : "DEBUG",
"logger.logstash.instrument.periodicpoller.jvm" : "INFO",
"logger.logstash.instrument.periodicpoller.os" : "DEBUG",
"logger.logstash.instrument.periodicpoller.persistentqueue" : "DEBUG",
"logger.logstash.modules.scaffold" : "DEBUG",
"logger.logstash.modules.xpackscaffold" : "DEBUG",
"logger.logstash.monitoringextension" : "DEBUG",
"logger.logstash.monitoringextension.pipelineregisterhook" : "DEBUG",
"logger.logstash.outputs.elasticsearch" : "DEBUG",
"logger.logstash.outputs.file" : "DEBUG",
"logger.logstash.pipeline" : "INFO",
"logger.logstash.plugins.registry" : "DEBUG",
"logger.logstash.runner" : "DEBUG",
"logger.org.logstash.Logstash" : "DEBUG",
"logger.org.logstash.common.DeadLetterQueueFactory" : "DEBUG",
"logger.org.logstash.common.io.DeadLetterQueueWriter" : "DEBUG",
"logger.org.logstash.config.ir.CompiledPipeline" : "DEBUG",
"logger.org.logstash.instrument.metrics.gauge.LazyDelegatingGauge" : "DEBUG",
"logger.org.logstash.plugins.pipeline.PipelineBus" : "DEBUG",
"logger.org.logstash.secret.store.SecretStoreFactory" : "DEBUG",
"logger.slowlog.logstash.codecs.json" : "DEBUG",
"logger.slowlog.logstash.aodecs.line" : "DEBUG",
"logger.slowlog.logstash.codecs.plain" : "DEBUG",
"logger.slowlog.logstash.filters.drop" : "DEBUG",
"logger.slowlog.logstash.filters.grok" : "DEBUG",
"logger.slowlog.logstash.filters.split" : "DEBUG",
"logger.slowlog.logstash.inputs.http_poller" : "DEBUG",
"logger.slowlog.logstash.outputs.elasticsearch" : "DEBUG",
"logger.slowlog.logstash.outputs.file" : "DEBUG"
}
'

如果有人知道如何纠正它，请尝试将点连接起来。 谢谢！

0 个答案:

没有答案
相关问题
最新问题