我正在尝试汇总我的陈述的一些结果,我在这里关注此示例:https://www.javacodegeeks.com/2017/10/aggregate-index-data-elasticsearch-using-logstash-jdbc.html
但是我遇到了一些错误
logstash配置在哪里
# file: simple-out.conf
input {
jdbc {
jdbc_connection_string => "jdbc:mysql://localhost:3306/mestradods"
jdbc_user => "root"
jdbc_password => ""
jdbc_validate_connection => true
jdbc_driver_library => "C:\Program Files (x86)\MySQL\Connector.J 5.1\mysql-connector-java-5.1.41-bin.jar"
jdbc_driver_class => "com.mysql.jdbc.Driver"
statement => "SELECT p.*, f.* FROM mestradods.projetos p join mestradods.fileuploaded f on p.projeto_id = f.projeto_id where p.projeto_id > :sql_last_value order by p.projeto_id;"
use_column_value => true
tracking_column => projeto_id
schedule => "*/5 * * * * *"
last_run_metadata_path => "G:\logstash\es_jdbc_last_run_metadata.log"
}
}
filter {
aggregate {
task_id => "%{projeto_id}"
code => "
map['projeto_id'] = event.get('projeto_id')
map['projeto_datacriacao'] = event.get('projeto_datacriacao')
map['projeto_sessao'] = event.get('projeto_sessao')
map['projeto_nome'] = event.get('projeto_nome')
map['projeto_path'] = event.get('projeto_path')
map['fileuploaded_list'] ||= []
map['fileuploaded'] ||= []
if(event.get('file_id') != nil)
if!(map['fileuploaded_list'].include? event.get('file_id'))
map['fileuploaded_list'] << event.get('file_id')
map['fileuploaded'] << {
'file_id' => event.get('file_id'),
'file_name' => event.get('file_name'),
'file_name_encode' => event.get('file_name_encode'),
'file_datacriacao' => event.get('file_datacriacao'),
'file_size' => event.get('file_size'),
'file_path' => event.get('file_path'),
'file_type' => event.get('file_type')
}
end
end
event.cancel()
"
push_previous_map_as_event => true
timeout => 150000
}
}
output {
#stdout { codec => json_lines }
elasticsearch {
index => "mestrado"
document_type => "infos"
document_id => "%{projeto_id}"
hosts => "localhost"
}
}
错误1:
> [2018-06-29T02:34:02,635][ERROR][org.logstash.Logstash ]
> java.lang.IllegalStateException: org.jruby.exceptions.RaiseException:
> (SyntaxError) (aggregate filter code):24: syntax error, unexpected
> kEND
> end
删除一端后,出现此错误:
> [2018-06-29T02:20:19,971][ERROR][logstash.filters.aggregate] Aggregate
> exception occurred {:error=>#<NoMethodError: undefined methodif!' for
> # LogStash::Filters::Aggregate:0x1caefe75>, :code=>" \n map['projeto_id'] = event.get('projeto_id')\n
> map['projeto_datacriacao'] = event.get('projeto_datacriacao')\n
> map['projeto_sessao'] = event.get('projeto_sessao')\n\t
> map['projeto_nome'] = event.get('projeto_nome')\n\t
> map['projeto_path'] = event.get('projeto_path')\n\t \n\t
> map['fileuploaded_list'] ||= []\n\t map['fileuploaded'] ||= []\t \n\t
> if(event.get('file_id') != nil)\n\t\t
> if!(map['fileuploaded_list'].include? event.get('file_id'))\n\t\t\t
> map['fileuploaded_list'] << event.get('file_id')\n\t\t\t \n\t\t\t
> map['fileuploaded'] << {\n\t\t\t\t 'file_id' =>
> event.get('file_id'),\n\t\t\t\t 'file_name' =>
> event.get('file_name'),\n\t\t\t\t 'file_name_encode' =>
> event.get('file_name_encode'),\n\t\t\t\t 'file_datacriacao' =>
> event.get('file_datacriacao'),\n\t\t\t\t 'file_size' =>
> event.get('file_size'),\n\t\t\t\t 'file_path' =>
> event.get('file_path'),\n\t\t\t\t 'file_type' =>
> event.get('file_type')\n\t\t\t }\n\t\t end\n\t\tevent.cancel()\n ",
> :map=>{"projeto_id"=>8,
> "projeto_datacriacao"=>2017-08-13T04:33:54.000Z,
> "projeto_sessao"=>"3QoTLWI-pXsXNDcvhIzuB3_WmxIugKPRykiuPAca",
> "projeto_nome"=>"Teste",
> "projeto_path"=>"C:\Users\Bruno\Documents\Arquivos\oVWeV2qCi5dMvytKXpY1709APsHAMnqCpeUN2yqd\8",
> "fileuploaded_list"=>[], "fileuploaded"=>[]},
> :event_data=>{"file_path"=>"C:\Users\Bruno\Documents\Arquivos\oVWeV2qCi5dMvytKXpY1709APsHAMnqCpeUN2yqd\8\APkvoLW5x",
> "projeto_path"=>"C:\Users\Bruno\Documents\Arquivos\oVWeV2qCi5dMvytKXpY1709APsHAMnqCpeUN2yqd\8",
> "file_name_encode"=>"APkvoLW5x", "file_name"=>"16_S16_L001_R1_001",
> "projeto_nome"=>"Teste", "file_datacriacao"=>2017-08-13T04:35:03.000Z,
> "file_size"=>"158,2 MB", "@timestamp"=>2018-06-29T05:20:16.860Z,
> "projeto_id"=>8,
> "projeto_sessao"=>"3QoTLWI-pXsXNDcvhIzuB3_WmxIugKPRykiuPAca",
> "file_type"=>"fastq", "file_id"=>191, "@version"=>"1",
> "projeto_datacriacao"=>2017-08-13T04:33:54.000Z}}
你们能帮我吗?