方案1：通过HTTP PUT创建弹性索引
第1步：通过http put将sl_template_v1加载到弹性DB中
PUT http://localhost:9200/_template/sl_template_v1
有效载荷：
{
"index_patterns": [
"sl_v1_*"
],
"settings": {
"index": {
"analysis": {
"filter": {
"custom_email_domain_filter": {
"pattern": ".*@(\\S*)",
"type": "pattern_replace",
"replacement": "$1"
},
"custom_postal_code_filter": {
"type": "pattern_capture",
"preserve_original": "true",
"patterns": [
"((\\d{5})-?(\\d{4}))",
"(([A-Z]{1}\\d{1}[A-Z]{1}) ?([A-Z]{1}\\d{1}[A-Z]{1}))"
]
}
},
"analyzer": {
"custom_email_domain_analyzer": {
"filter": [
"custom_email_domain_filter"
],
"type": "custom",
"tokenizer": "custom_email_domain_tokenizer"
},
"custom_lowercase_keyword_analyzer": {
"filter": [
"lowercase"
],
"type": "custom",
"tokenizer": "keyword"
},
"custom_postal_code_analyzer": {
"filter": [
"custom_postal_code_filter",
"lowercase"
],
"type": "custom",
"tokenizer": "standard"
}
},
"tokenizer": {
"custom_email_domain_tokenizer": {
"reverse": "true",
"type": "path_hierarchy",
"delimiter": "."
}
}
}
}
},
"mappings": {
"_default_": {
"properties": {
"listId": {
"type": "keyword"
},
"source": {
"type": "text",
"analyzer": "english"
},
"notes": {
"type": "text",
"analyzer": "english"
},
"clientIdentifiers": {
"type": "keyword"
},
"createdBy": {
"type": "keyword"
},
"createdOn": {
"type": "date",
"format": "date_optional_time"
},
"updatedBy": {
"type": "keyword"
},
"updatedOn": {
"type": "date",
"format": "date_optional_time"
},
"active": {
"type": "boolean"
}
}
}
},
"aliases": {
"all_sl": {}
}
}
第2步：通过http put在弹性数据库中创建弹性索引
PUT http://localhost:9200/sl_v1_realbad00102
有效载荷：
{
"aliases": {
"all_sl": {}
},
"mappings": {
"entry": {
"properties": {
"active": {
"type": "boolean"
},
"clientIdentifiers": {
"type": "keyword"
},
"content": {
"properties": {
"billingAddress": {
"properties": {
"city": {
"type": "text",
"fields": {
"raw": {
"type": "text",
"analyzer": "custom_lowercase_keyword_analyzer"
}
}
},
"country": {
"type": "text",
"fields": {
"raw": {
"type": "text",
"analyzer": "custom_lowercase_keyword_analyzer"
}
}
},
"line1": {
"type": "text",
"fields": {
"raw": {
"type": "text",
"analyzer": "custom_lowercase_keyword_analyzer"
}
}
},
"line2": {
"type": "text",
"fields": {
"raw": {
"type": "text",
"analyzer": "custom_lowercase_keyword_analyzer"
}
}
},
"line3": {
"type": "text",
"fields": {
"raw": {
"type": "text",
"analyzer": "custom_lowercase_keyword_analyzer"
}
}
},
"line4": {
"type": "text",
"fields": {
"raw": {
"type": "text",
"analyzer": "custom_lowercase_keyword_analyzer"
}
}
},
"mainDivision": {
"type": "text",
"fields": {
"raw": {
"type": "text",
"analyzer": "custom_lowercase_keyword_analyzer"
}
}
},
"postalCode": {
"type": "text",
"analyzer": "custom_postal_code_analyzer"
}
}
},
"email": {
"type": "text",
"fields": {
"domain": {
"type": "text",
"analyzer": "custom_email_domain_analyzer"
},
"raw": {
"type": "text",
"analyzer": "custom_lowercase_keyword_analyzer"
}
},
"analyzer": "simple"
},
"shippingAddress": {
"properties": {
"city": {
"type": "text",
"fields": {
"raw": {
"type": "text",
"analyzer": "custom_lowercase_keyword_analyzer"
}
}
},
"country": {
"type": "text",
"fields": {
"raw": {
"type": "text",
"analyzer": "custom_lowercase_keyword_analyzer"
}
}
},
"line1": {
"type": "text",
"fields": {
"raw": {
"type": "text",
"analyzer": "custom_lowercase_keyword_analyzer"
}
}
},
"line2": {
"type": "text",
"fields": {
"raw": {
"type": "text",
"analyzer": "custom_lowercase_keyword_analyzer"
}
}
},
"line3": {
"type": "text",
"fields": {
"raw": {
"type": "text",
"analyzer": "custom_lowercase_keyword_analyzer"
}
}
},
"line4": {
"type": "text",
"fields": {
"raw": {
"type": "text",
"analyzer": "custom_lowercase_keyword_analyzer"
}
}
},
"mainDivision": {
"type": "text",
"fields": {
"raw": {
"type": "text",
"analyzer": "custom_lowercase_keyword_analyzer"
}
}
},
"postalCode": {
"type": "text",
"analyzer": "custom_postal_code_analyzer"
}
}
}
}
},
"createdBy": {
"type": "keyword"
},
"createdOn": {
"type": "date",
"format": "date_optional_time"
},
"listId": {
"type": "keyword"
},
"notes": {
"type": "text",
"analyzer": "english"
},
"source": {
"type": "text",
"analyzer": "english"
},
"updatedBy": {
"type": "keyword"
},
"updatedOn": {
"type": "date",
"format": "date_optional_time"
}
}
}
}
}
弹性搜索数据库使用索引创建模板“ sl_template_v1”并以预期格式创建索引。创建的请求上的http get请求显示索引内容长为300行。
GET http://localhost:9200/sl_v1_realbad00102
方案2： 我正在尝试使用弹性搜索输出插件通过logstash创建弹性索引。 Logstash输出相同的映射json（与方案1中相同）文件，弹性搜索将其作为输入。 弹性搜索借助索引模板sl_template_v1（如方案1所示）爆炸了json映射，并将爆炸的json内容存储为es索引的内容。
输出到弹性DB的logstash如下所示。
{
"mappings" => {
"entry" => {
"properties" => {
"content" => {
"type" => "object",
"properties" => {
"shippingAddress" => {
"type" => "object",
"properties" => {
"country" => {
"type" => "text",
"fields" => {
"raw" => {
"analyzer" => "custom_lowercase_keyword_analyzer",
"type" => "text"
}
}
},
"line4" => {
"type" => "text",
"fields" => {
"raw" => {
"analyzer" => "custom_lowercase_keyword_analyzer",
"type" => "text"
}
}
},
"city" => {
"type" => "text",
"fields" => {
"raw" => {
"analyzer" => "custom_lowercase_keyword_analyzer",
"type" => "text"
}
}
},
"postalCode" => {
"analyzer" => "custom_postal_code_analyzer",
"type" => "text"
},
"line3" => {
"type" => "text",
"fields" => {
"raw" => {
"analyzer" => "custom_lowercase_keyword_analyzer",
"type" => "text"
}
}
},
"line2" => {
"type" => "text",
"fields" => {
"raw" => {
"analyzer" => "custom_lowercase_keyword_analyzer",
"type" => "text"
}
}
},
"line1" => {
"type" => "text",
"fields" => {
"raw" => {
"analyzer" => "custom_lowercase_keyword_analyzer",
"type" => "text"
}
}
},
"mainDivision" => {
"type" => "text",
"fields" => {
"raw" => {
"analyzer" => "custom_lowercase_keyword_analyzer",
"type" => "text"
}
}
}
}
},
"billingAddress" => {
"type" => "object",
"properties" => {
"country" => {
"type" => "text",
"fields" => {
"raw" => {
"analyzer" => "custom_lowercase_keyword_analyzer",
"type" => "text"
}
}
},
"line4" => {
"type" => "text",
"fields" => {
"raw" => {
"analyzer" => "custom_lowercase_keyword_analyzer",
"type" => "text"
}
}
},
"city" => {
"type" => "text",
"fields" => {
"raw" => {
"analyzer" => "custom_lowercase_keyword_analyzer",
"type" => "text"
}
}
},
"postalCode" => {
"analyzer" => "custom_postal_code_analyzer",
"type" => "text"
},
"line3" => {
"type" => "text",
"fields" => {
"raw" => {
"analyzer" => "custom_lowercase_keyword_analyzer",
"type" => "text"
}
}
},
"line2" => {
"type" => "text",
"fields" => {
"raw" => {
"analyzer" => "custom_lowercase_keyword_analyzer",
"type" => "text"
}
}
},
"line1" => {
"type" => "text",
"fields" => {
"raw" => {
"analyzer" => "custom_lowercase_keyword_analyzer",
"type" => "text"
}
}
},
"mainDivision" => {
"type" => "text",
"fields" => {
"raw" => {
"analyzer" => "custom_lowercase_keyword_analyzer",
"type" => "text"
}
}
}
}
},
"email" => {
"analyzer" => "simple",
"type" => "text",
"fields" => {
"domain" => {
"analyzer" => "custom_email_domain_analyzer",
"type" => "text"
},
"raw" => {
"analyzer" => "custom_lowercase_keyword_analyzer",
"type" => "text"
}
}
}
}
}
}
}
},
"@metadata" => {
"_action" => "index",
"_index" => "sl_v1_realbad00170"
},
"alias" => {
"entry" => {}
},
"tags" => [
[0] "esIndexCreate"
]
}
logstash conf文件如下所示：
logstash.conf
input {
kafka {
bootstrap_servers => "localhost:9092"
topics => [ "sl-topic1-lvs","sl-topic1-ash" ]
tags => []
}
}
filter {
json { source => "message" }
mutate { add_field => { "[@metadata][_action]" => "%{action}" } }
mutate { add_field => { "[@metadata][_index]" => "%{index}" } }
if "esIndexCreate" in [type] {
mutate { add_tag => [ "esIndexCreate" ] }
}else{
mutate { add_field => { "[@metadata][_document_type]" => "%{type}" } }
mutate { add_field => { "[@metadata][_document_id]" => "%{identifier}" } }
}
mutate { remove_field => [ "message" ] }
mutate { remove_field => [ "action" ] }
mutate { remove_field => [ "index" ] }
mutate { remove_field => [ "type" ] }
mutate { remove_field => [ "@timestamp" ] }
mutate { remove_field => [ "@version" ] }
mutate { remove_field => [ "identifier" ] }
}
output {
if "esIndexCreate" in [tags] {
elasticsearch {
hosts => ["localhost:9200"]
user => "elastic"
password => "changeme"
action =>"%{[@metadata][_action]}"
index => "%{[@metadata][_index]}"
}
} else {
elasticsearch {
hosts => ["localhost:9200"]
user => "elastic"
password => "changeme"
action =>"%{[@metadata][_action]}"
index => "%{[@metadata][_index]}"
document_id => "%{[@metadata][_document_id]}"
document_type =>"%{[@metadata][_document_type]}"
}
}
stdout { codec => rubydebug { metadata => true }}
}
在方案2中创建的索引的内容长900行，并且与在方案1中创建的索引的内容（300行）不匹配。 这两个索引的内容值不同的原因可能是什么？