Logstash汇总错误

时间:2018-06-30 01:09:29

标签: elasticsearch logstash

我正在尝试汇总我的陈述的一些结果,我在这里关注此示例:https://www.javacodegeeks.com/2017/10/aggregate-index-data-elasticsearch-using-logstash-jdbc.html

但是我遇到了一些错误

logstash配置在哪里

# file: simple-out.conf
input {
    jdbc {

        jdbc_connection_string => "jdbc:mysql://localhost:3306/mestradods"
        jdbc_user => "root"
        jdbc_password => ""
        jdbc_validate_connection => true
        jdbc_driver_library => "C:\Program Files (x86)\MySQL\Connector.J 5.1\mysql-connector-java-5.1.41-bin.jar"       
        jdbc_driver_class => "com.mysql.jdbc.Driver"       
        statement => "SELECT p.*, f.* FROM mestradods.projetos p join mestradods.fileuploaded f on p.projeto_id = f.projeto_id where p.projeto_id > :sql_last_value order by p.projeto_id;"
        use_column_value => true
        tracking_column => projeto_id
        schedule => "*/5 * * * * *"
        last_run_metadata_path => "G:\logstash\es_jdbc_last_run_metadata.log"
    }
}
filter {    

  aggregate {
    task_id => "%{projeto_id}"

    code => "     
      map['projeto_id'] = event.get('projeto_id')
      map['projeto_datacriacao'] = event.get('projeto_datacriacao')
      map['projeto_sessao'] = event.get('projeto_sessao')
      map['projeto_nome'] = event.get('projeto_nome')
      map['projeto_path'] = event.get('projeto_path')

      map['fileuploaded_list'] ||= []
      map['fileuploaded'] ||= []      
      if(event.get('file_id') != nil)
          if!(map['fileuploaded_list'].include? event.get('file_id'))
              map['fileuploaded_list'] << event.get('file_id')

              map['fileuploaded'] << {
                  'file_id' => event.get('file_id'),
                  'file_name' => event.get('file_name'),
                  'file_name_encode' => event.get('file_name_encode'),
                  'file_datacriacao' => event.get('file_datacriacao'),
                  'file_size' => event.get('file_size'),
                  'file_path' => event.get('file_path'),
                  'file_type' => event.get('file_type')
              }
          end
          end
        event.cancel()
    "
    push_previous_map_as_event => true
    timeout => 150000
  }

}
output {
    #stdout { codec => json_lines }
    elasticsearch {
        index => "mestrado"
        document_type => "infos"
        document_id => "%{projeto_id}"
        hosts => "localhost"
    }

}

错误1:

> [2018-06-29T02:34:02,635][ERROR][org.logstash.Logstash    ]
> java.lang.IllegalStateException: org.jruby.exceptions.RaiseException:
> (SyntaxError) (aggregate filter code):24: syntax error, unexpected
> kEND
>                   end

删除一端后,出现此错误:

> [2018-06-29T02:20:19,971][ERROR][logstash.filters.aggregate] Aggregate
> exception occurred {:error=>#<NoMethodError: undefined methodif!' for
> # LogStash::Filters::Aggregate:0x1caefe75>, :code=>" \n map['projeto_id'] = event.get('projeto_id')\n
> map['projeto_datacriacao'] = event.get('projeto_datacriacao')\n
> map['projeto_sessao'] = event.get('projeto_sessao')\n\t
> map['projeto_nome'] = event.get('projeto_nome')\n\t
> map['projeto_path'] = event.get('projeto_path')\n\t \n\t
> map['fileuploaded_list'] ||= []\n\t map['fileuploaded'] ||= []\t \n\t
> if(event.get('file_id') != nil)\n\t\t
> if!(map['fileuploaded_list'].include? event.get('file_id'))\n\t\t\t
> map['fileuploaded_list'] << event.get('file_id')\n\t\t\t \n\t\t\t
> map['fileuploaded'] << {\n\t\t\t\t 'file_id' =>
> event.get('file_id'),\n\t\t\t\t 'file_name' =>
> event.get('file_name'),\n\t\t\t\t 'file_name_encode' =>
> event.get('file_name_encode'),\n\t\t\t\t 'file_datacriacao' =>
> event.get('file_datacriacao'),\n\t\t\t\t 'file_size' =>
> event.get('file_size'),\n\t\t\t\t 'file_path' =>
> event.get('file_path'),\n\t\t\t\t 'file_type' =>
> event.get('file_type')\n\t\t\t }\n\t\t end\n\t\tevent.cancel()\n ",
> :map=>{"projeto_id"=>8,
> "projeto_datacriacao"=>2017-08-13T04:33:54.000Z,
> "projeto_sessao"=>"3QoTLWI-pXsXNDcvhIzuB3_WmxIugKPRykiuPAca",
> "projeto_nome"=>"Teste",
> "projeto_path"=>"C:\Users\Bruno\Documents\Arquivos\oVWeV2qCi5dMvytKXpY1709APsHAMnqCpeUN2yqd\8",
> "fileuploaded_list"=>[], "fileuploaded"=>[]},
> :event_data=>{"file_path"=>"C:\Users\Bruno\Documents\Arquivos\oVWeV2qCi5dMvytKXpY1709APsHAMnqCpeUN2yqd\8\APkvoLW5x",
> "projeto_path"=>"C:\Users\Bruno\Documents\Arquivos\oVWeV2qCi5dMvytKXpY1709APsHAMnqCpeUN2yqd\8",
> "file_name_encode"=>"APkvoLW5x", "file_name"=>"16_S16_L001_R1_001",
> "projeto_nome"=>"Teste", "file_datacriacao"=>2017-08-13T04:35:03.000Z,
> "file_size"=>"158,2 MB", "@timestamp"=>2018-06-29T05:20:16.860Z,
> "projeto_id"=>8,
> "projeto_sessao"=>"3QoTLWI-pXsXNDcvhIzuB3_WmxIugKPRykiuPAca",
> "file_type"=>"fastq", "file_id"=>191, "@version"=>"1",
> "projeto_datacriacao"=>2017-08-13T04:33:54.000Z}}

你们能帮我吗?

0 个答案:

没有答案