我想问题可能与我的logstash.conf有关,但我不知道该怎么做。我找到了很好的教程,解释了如何仅使用ElasticSearch来完成它,但在我的情况下,所有数据都将通过LogStash来自NodeJs。
我搜索有关启用fieldData但我无法弄清楚如何在我的logstash.conf中执行此操作。我应该创建一个索引模板吗?如果是这样的话?
上下文是我想在每次用户访问我们的应用程序时记录,然后根据每月的访问号码向他/她收费。
logstash.conf
input {
tcp {
port => 5000
type => cpfTipo
}
}
filter {
json {
source => "message"
}
}
output {
elasticsearch { hosts => ["localhost:9200"] index => "mycostumer_indice" document_type => "cpfTipo"}
}
暂时过滤:
1)
curl -XGET http://127.0.0.1:9200/mycostumer_indice/cpfTipo/_search -d '{
"query": {
"filtered": {
"query": {
"match_all": {}
},
"filter":
{
"term": {
"username": "a"
}
}
]
}
}
}
}'
{"error":{"root_cause":[{"type":"parsing_exception","reason":"no [query] registered for [filtered]","line":3,"col":21}],"type":"parsing_exception","reason":"no [query] registered for [filtered]","line":3,"col":21},"status":400}demetrio@nodejs ~/tool
汇总的尝试:
1)
curl -XGET http://127.0.0.1:9200/mycostumer_indice/cpfTipo/_search -d '{
{
"aggs" : {
"message" : {
"terms" : {
"field" : "cpfTipo",
"size" : 5
}
}
}
}'
{"error":{"root_cause":[{"type":"json_parse_exception","reason":"Unexpected character ('{' (code 123)): was expecting double-quote to start field name\n at [Source: org.elasticsearch.transport.netty4.ByteBufStreamInput@3ce63313; line: 2, column: 2]"}],"type":"json_parse_exception","reason":"Unexpected character ('{' (code 123)): was expecting double-quote to start field name\n at [Source: org.elasticsearch.transport.netty4.ByteBufStreamInput@3ce63313; line: 2, column: 2]"},"status":500}
2)
curl -XPOST 'http://127.0.0.1:9200/mycostumer_indice/_search?pretty' -d '
{
"size": 0,
"aggs": {
"group_by_username": {
"terms": {
"field": "username"
}
}
}
}'
{
"error" : {
"root_cause" : [
{
"type" : "illegal_argument_exception",
"reason" : "Fielddata is disabled on text fields by default. Set fielddata=true on [username] in order to load fielddata in memory by uninverting the inverted index. Note that this can however use significant memory."
}
],
"type" : "search_phase_execution_exception",
"reason" : "all shards failed",
"phase" : "query",
"grouped" : true,
"failed_shards" : [
{
"shard" : 0,
"index" : "mycostumer_indice",
"node" : "-em7X-ssT3SL2JBtfs0VTQ",
"reason" : {
"type" : "illegal_argument_exception",
"reason" : "Fielddata is disabled on text fields by default. Set fielddata=true on [username] in order to load fielddata in memory by uninverting the inverted index. Note that this can however use significant memory."
}
}
],
"caused_by" : {
"type" : "illegal_argument_exception",
"reason" : "Fielddata is disabled on text fields by default. Set fielddata=true on [username] in order to load fielddata in memory by uninverting the inverted index. Note that this can however use significant memory."
}
},
"status" : 400
}
如何出现mycostumer索引:
curl http://127.0.0.1:9200/mycostumer_indice/cpfTipo/_search?pretty
{
"took" : 2,
"timed_out" : false,
"_shards" : {
"total" : 5,
"successful" : 5,
"failed" : 0
},
"hits" : {
"total" : 1,
"max_score" : 1.0,
"hits" : [
{
"_index" : "mycostumer_indice",
"_type" : "cpfTipo",
"_id" : "AVrxUi5cIZDJUBCguFI8",
"_score" : 1.0,
"_source" : {
"password" : "a",
"@timestamp" : "2017-03-21T14:42:54.466Z",
"port" : 56012,
"@version" : "1",
"host" : "127.0.0.1",
"message" : "{\"username\":\"a\",\"password\":\"a\"}",
"type" : "cpfTipo",
"username" : "a"
}
}
]
}
}
在nodeJs
中var express = require('express');
var bodyParser = require('body-parser');
var Client = require('node-rest-client').Client;
var expressWinston = require('express-winston');
var winston = require('winston');
require('winston-logstash');
var client = new Client();
var Logstash = require('logstash-client');
var app = express();
expressWinston.requestWhitelist.push('body');
expressWinston.responseWhitelist.push('body')
app.use(bodyParser.json());
app.use(bodyParser.urlencoded({
extended: true
}));
var port = process.env.PORT || 3000;
var router = express.Router();
var tokenRoute = router.route('/token');
tokenRoute.post(function (req, res) {
var user = {
username: req.body.username,
password: req.body.password
};
logstash.send(user);
答案 0 :(得分:2)
您的第一个搜索查询使用已弃用的filtered
查询,只需将其替换为bool
,您就可以了:
curl -XGET http://127.0.0.1:9200/mycostumer_indice/cpfTipo/_search -d '{
"query": {
"bool": {
"filter":
{
"term": {
"username": "a"
}
}
]
}
}
}
}'
你的第二个查询在开头有一个太多的开括号,而是使用这个。
curl -XGET http://127.0.0.1:9200/mycostumer_indice/cpfTipo/_search -d '{
"aggs" : {
"message" : {
"terms" : {
"field" : "cpfTipo",
"size" : 5
}
}
}
}'
您的第三个查询失败,因为您尝试在username
上汇总text
字段。您应该更改该字段的映射,以改为使用keyword
type。