我决定将kafka包含在我的ELK架构中，因为它读取mongodb oplog并推送更新，我不得不对数据进行编码，以LOGSTASH和BAM的json格式将其获取，但我将更新推送到es，但是作为新行在一个名为patch的字符串字段下，未更新的mongodb数据也被推送到一个称为after的字段下，该字段也被定义为一个字符串，我曾尝试使用我使用过的AVSC文件在kafka级别上将其映射schema_uri => logstash中的路径/到/文件，但没有任何变化，我也尝试在elasticsearch中将其映射，但仍然出现错误 这是我的logstash conf文件
input {
kafka {
bootstrap_servers => "localhost:9092"
topics => "mongo_conn.test.testCollection"
auto_offset_reset => "earliest"
codec => avro_schema_registry {
endpoint => "http://localhost:8081"
schema_uri =>"/home/jhon/Desktop/test.AVSC" }
value_deserializer_class => "org.apache.kafka.common.serialization.ByteArrayDeserializer"
}
}
filter{
mutate {
convert => { "number" => "integer" }
}
mutate {
rename => { "_id" => "id" }
}
mutate {
rename => { "$oid" => "oid" }
}
}
output {
amazon_es {
这是我的AVSC文件
{
"type" : "record",
"name" : "mongo_conn",
"namespace" : "test.testCollection",
"fields" : [ {
"name" : "_id",
"type" : {
"type" : "record",
"name" : "testEvent",
"fields" : [ {
"name" : "$oid",
"type" : "string"
}, {
"name" : "name",
"type" : "string"
}, {
"name" : "season",
"type" : "string"
}, {
"name" : "number",
"type" : "long"
} ]
}
} ]
}
这是Elasticsearch中的默认映射
{
"yoyo4y" : {
"mappings" : {
"doc" : {
"properties" : {
"@timestamp" : {
"type" : "date"
},
"@version" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"after" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"op" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"patch" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"source" : {
"properties" : {
"connector" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"h" : {
"type" : "long"
},
"initsync" : {
"type" : "boolean"
},
"name" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"ns" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"ord" : {
"type" : "long"
},
"rs" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"sec" : {
"type" : "long"
},
"version" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
}
}
},
"ts_ms" : {
"type" : "long"
}
}
}
}
}
}
这是一条插入行，被推送以使Elasticsearch注意到“之后”和“补丁”字段
{
"_index": "yoyo4y",
"_type": "doc",
"_id": "vzFU_moBQHn1BMrfqi3m",
"_version": 1,
"_score": null,
"_source": {
"patch": null,
"after": "{\"_id\" : {\"$oid\" : \"5cebf2d7575801a61a5aa171\"},\"name\" : \"mongo59\",\"season\" : \"colddd\",\"number\" : 999.0}",
"op": "r",
"@version": "1",
"source": {
"version": "0.9.5.Final",
"ord": 1,
"ns": "test.testCollection",
"initsync": true,
"sec": 1559044767,
"h": 1595360082745638400,
"rs": "rs0",
"name": "mongo_conn",
"connector": "mongodb"
},
"@timestamp": "2019-05-28T12:05:21.939Z",
"ts_ms": 1559044773272
},
"fields": {
"@timestamp": [
"2019-05-28T12:05:21.939Z"
]
},
"highlight": {
"op": [
"@kibana-highlighted-field@r@/kibana-highlighted-field@"
]
},
"sort": [
1559045121939
]
}
这是推送到elasticsearch的更新
{
"_index": "yoyo4y",
"_type": "doc",
"_id": "ujFb-WoBQHn1BMrfcy1W",
"_version": 1,
"_score": null,
"_source": {
"patch": "{\"$v\" : 1,\"$set\" : {\"number\" : 95.0}}",
"op": "u",
"@version": "1",
"@timestamp": "2019-05-27T12:54:40.966Z",
"after": null,
"source": {
"connector": "mongodb",
"version": "0.9.5.Final",
"name": "mongo_conn",
"ns": "test.testCollection",
"sec": 1558961680,
"rs": "rs0",
"h": -4743345325051307000,
"initsync": false,
"ord": 1
},
"ts_ms": 1558961680735
},
"fields": {
"@timestamp": [
"2019-05-27T12:54:40.966Z"
]
},
"highlight": {
"op": [
"@kibana-highlighted-field@u@/kibana-highlighted-field@"
]
},
"sort": [
1558961680966
]
}
这是我想在ES中定义的映射，但返回错误
PUT firstindex/_mapping/
{
"mappings" : {
"doc" : {
"properties" : {
"@timestamp" : {
"type" : "date"
},
"@version" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"after": {
"properties": {
"_id": {
"properties": {
"$oid" : { "type": "text" },
"name" : { "type": "text" },
"number": { "type": "int" },
"season": { "type": "text" }
},
"op" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"patch" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"source" : {
"properties" : {
"connector" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"h" : {
"type" : "long"
},
"initsync" : {
"type" : "boolean"
},
"name" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"ns" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"ord" : {
"type" : "long"
},
"rs" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
},
"sec" : {
"type" : "long"
},
"version" : {
"type" : "text",
"fields" : {
"keyword" : {
"type" : "keyword",
"ignore_above" : 256
}
}
}
}
},
"ts_ms" : {
"type" : "long"
}
}
}
}
}
}
}
}
如果一切正常，我如何管理更新与其行的链接或更新行本身，并使行中的另一个字段显示更新的字段，类似这样。