I have created following mapping for in ElasticSearch and same is created. But when I add documents to this type, my nested field type变为正常。所以当我对嵌套对象字段执行查询时，它 显示意外的结果，而其余的字段结果为 预期。我不确定我做错了什么。一定有东西 我在创建嵌套对象映射时缺少。
request_body = {
"settings": {
"index": {
"analysis": {
"analyzer": {
"standard": {
"tokenizer": "standard",
"filter": ["lowercase", "stop"]
}
}
}
}
},
"mappings" : {
"userdetails" : {
"properties" : {
"accountType" : {
"type" : "long"
},
"addressLineOne" : {
"type" : "string"
},
"addressLineTwo" : {
"type" : "string"
},
"alternateEmailId" : {
"type" : "string"
},
"archiveId" : {
"type" : "string"
},
"blockedEmployerId" : {
"type" : "long"
},
"cityId" : {
"type" : "long"
},
"country" : {
"type" : "string"
},
"coverPicUrl" : {
"type" : "string"
},
"currentEmployer" : {
"type" : "string"
},
"currentPackageCurrencyId" : {
"type" : "long"
},
"currentPackageUnit" : {
"type" : "string"
},
"currentSalaryPackage" : {
"type" : "double"
},
"currentlocation" : {
"type" : "string"
},
"designation" : {
"type" : "string"
},
"emailId" : {
"type" : "string"
},
"employmentDetail" : {
"type":"nested",
"properties" : {
"companyName" : {
"type" : "string"
},
"designation" : {
"type" : "string"
},
"durationEndMonth" : {
"type" : "long"
},
"durationEndYear" : {
"type" : "long"
},
"durationStartMonth" : {
"type" : "long"
},
"durationStartYear" : {
"type" : "long"
},
"employmentStatus" : {
"type" : "string"
},
"id" : {
"type" : "long"
},
"isCurrent" : {
"type" : "long"
},
"jobProfile" : {
"type" : "string"
},
"location" : {
"type" : "string"
},
"timeDate" : {
"type" : "date",
"format" : "strict_date_optional_time||epoch_millis"
},
"userId" : {
"type" : "long"
}
}
},
"examscores" : {
"type":"nested",
"properties" : {
"attemptedCount" : {
"type" : "long"
},
"candidateId" : {
"type" : "long"
},
"examDate" : {
"type" : "date",
"format" : "strict_date_optional_time||epoch_millis"
},
"examId" : {
"type" : "long"
},
"examScoreDetailId" : {
"type" : "long"
},
"examTotalScore" : {
"type" : "double"
},
"examType" : {
"type" : "long"
},
"katExamEventId" : {
"type" : "long"
},
"rightAttempted" : {
"type" : "long"
},
"rptVisStatus" : {
"type" : "long"
},
"scorePercent" : {
"type" : "double"
},
"timeDate" : {
"type" : "date",
"format" : "strict_date_optional_time||epoch_millis"
},
"token" : {
"type" : "string"
},
"totalQuestion" : {
"type" : "long"
},
"totalScore" : {
"type" : "double"
}
}
},
"experienceInMonths" : {
"type" : "long"
},
"experienceInYears" : {
"type" : "long"
},
"firstName" : {
"type" : "string"
},
"gender" : {
"type" : "string"
},
"hQPS" : {
"type" : "long"
},
"higestQualInstitute" : {
"type" : "string"
},
"higestQualInstituteLoc" : {
"type" : "string"
},
"higestQualPassingYear" : {
"type" : "long"
},
"higestQualPercentCgpa" : {
"type" : "double"
},
"higestQualification" : {
"type" : "string"
},
"highestQualSpecialization" : {
"type" : "string"
},
"highestQualificationStream" : {
"type" : "string"
},
"industryDomain" : {
"type" : "string"
},
"keySkills" : {
"type" : "string"
},
"lastLoginDate" : {
"type" : "string"
},
"lastName" : {
"type" : "string"
},
"lastUpdateDate" : {
"type" : "string"
},
"locationName" : {
"type" : "string"
},
"maritalStatus" : {
"type" : "string"
},
"mobileNumber" : {
"type" : "long"
},
"password" : {
"type" : "string"
},
"paswdChangedDate" : {
"type" : "string"
},
"phoneNumber" : {
"type" : "long"
},
"preferredLocations" : {
"type" : "string"
},
"profileImageUrl" : {
"type" : "string"
},
"profileSummary" : {
"type" : "string"
},
"projectdetail" : {
"type":"nested",
"properties" : {
"clientName" : {
"type" : "string"
},
"designation" : {
"type" : "string"
},
"durationEndMonth" : {
"type" : "long"
},
"durationEndYear" : {
"type" : "long"
},
"durationStartMonth" : {
"type" : "long"
},
"durationStartYear" : {
"type" : "long"
},
"employmentNature" : {
"type" : "long"
},
"id" : {
"type" : "long"
},
"location" : {
"type" : "string"
},
"rolesResponsibilities" : {
"type" : "string"
},
"skillsUsed" : {
"type" : "string"
},
"teamSize" : {
"type" : "long"
},
"timeDate" : {
"type" : "date",
"format" : "strict_date_optional_time||epoch_millis"
},
"title" : {
"type" : "string"
},
"userId" : {
"type" : "long"
}
}
},
"qualification" : {
"type":"nested",
"properties" : {
"id" : {
"type" : "long"
},
"institute" : {
"type" : "string"
},
"isPrimary" : {
"type" : "long"
},
"passingYear" : {
"type" : "long"
},
"percentageCgpa" : {
"type" : "double"
},
"qualification" : {
"type" : "string"
},
"specialization" : {
"type" : "string"
},
"timeDate" : {
"type" : "date",
"format" : "strict_date_optional_time||epoch_millis"
},
"userId" : {
"type" : "long"
}
}
},
"registrationDate" : {
"type" : "string"
},
"resumeContentType" : {
"type" : "string"
},
"resumeTitle" : {
"type" : "string"
},
"s3DocKey" : {
"type" : "string"
},
"s3HtmlKey" : {
"type" : "string"
},
"sectionalscores" : {
"type":"nested",
"properties" : {
"attemptedCount" : {
"type" : "long"
},
"candidateId" : {
"type" : "long"
},
"examScoreId" : {
"type" : "long"
},
"examTime" : {
"type" : "long"
},
"examTitleId" : {
"type" : "long"
},
"katExamId" : {
"type" : "long"
},
"katScoreId" : {
"type" : "long"
},
"rightAttempted" : {
"type" : "long"
},
"score" : {
"type" : "double"
},
"scorePercent" : {
"type" : "double"
},
"timeDate" : {
"type" : "date",
"format" : "strict_date_optional_time||epoch_millis"
},
"totalQuestion" : {
"type" : "long"
},
"totalScore" : {
"type" : "double"
}
}
},
"stateId" : {
"type" : "long"
},
"status" : {
"type" : "long"
},
"timeDate" : {
"type" : "string"
},
"userId" : {
"type" : "long"
},
"uuid" : {
"type" : "string"
},
"videoIntroType" : {
"type" : "long"
},
"videoIntroUrl" : {
"type" : "string"
},
"videoProfileTimeDate" : {
"type" : "date",
"format" : "strict_date_optional_time||epoch_millis"
},
"videoProfileUrl" : {
"type" : "string"
},
"videoProfileVisibility" : {
"type" : "long"
},
"workStatusForOtherCountries" : {
"type" : "long"
},
"workStatusForUS" : {
"type" : "long"
},
"xIIPS" : {
"type" : "long"
},
"xIIPassingYear" : {
"type" : "long"
},
"xIIPercentageCgpa" : {
"type" : "double"
},
"xPS" : {
"type" : "long"
},
"xPassingYear" : {
"type" : "long"
},
"xPercentageCgpa" : {
"type" : "double"
}
}
}
}
}
print("creating 'example_index' index...")
res = es1.indices.create(index = 'no-one', body = request_body)
使用上面的代码创建索引后，我使用以下代码将数据添加到此索引中。
from elasticsearch import Elasticsearch
from elasticsearch import helpers
import MySQLdb
import json
db = MySQLdb.connect(host='localhost',
user='xxxx',
passwd='xxxx',
db='xxxxx'
)
cursor_mysql = db.cursor()
cursor_mysql.execute("select * from USER ")
userTable = cursor_mysql.fetchall()
cursor_mysql.execute("select * from qualificationmapping")
secondaryQualification = cursor_mysql.fetchall()
cursor_mysql.execute("select * from examscoredetail")
examscore = cursor_mysql.fetchall()
cursor_mysql.execute("select * from katsectionalscore")
sectionalScore = cursor_mysql.fetchall()
cursor_mysql.execute("select * from projectexperiencedetails")
proejctDetails = cursor_mysql.fetchall()
cursor_mysql.execute("select * from useremploymentdetails")
employmentDetails = cursor_mysql.fetchall()
data = []
listSecondaryQualification=[]
listExamScores=[]
listSectionalScores=[]
listProjectDetails=[]
listEmploymentDetails=[]
# jsonSecondaryQualification = ''
# jsonExamScores = ''
# jsonSectionalScores = ''
# jsonProjectDetails = ''
# jsonEmploymentDetails = ''
# proejctdetail = {}
# qualification = {}
sectionalscores = {}
indexdata={}
# examscores = {}
# employmentDetail = {}
es1 = Elasticsearch()
# es1.create(index="xxxxx", doc_type='xxxx', body=indexdata)
count = 0
for userid in userTable:
count+=1
indexdata = {
"_index": "xxxxx",
"_type": "xxxxx",
"_id": userid[0]
}
for item in userTable:
if userid[0]== item[0]:
userdata = {
'userId': item[0],
'accountType': item[1],
'addressLineOne': item[2],
'addressLineTwo': item[3],
'alternateEmailId': item[4],
'archiveId':item[5],
'blockedEmployerId': item[6],
'cityId': item[7],
'classXIISchool': item[8],
'classXSchool': item[9],
'country': item[10],
'coverPicUrl': item[11],
'currentEmployer': item[12],
'currentPackageCurrencyId': item[13],
'currentPackageUnit': item[14],
'currentSalaryPackage': item[15],
'currentlocation': item[16],
'designation': item[17],
'emailId': item[19],
'experienceInMonths': item[20],
'experienceInYears': item[21],
'firstName': item[22],
'gender': item[23],
'hQPS': item[24],
'higestQualInstitute': item[25],
'higestQualInstituteLoc': item[26],
'higestQualPassingYear': item[27],
'higestQualPercentCgpa': item[28],
'higestQualification': item[29],
'highestQualSpecialization': item[30],
'highestQualificationStream': item[31],
'industryDomain': item[32],
'keySkills': item[33],
'lastLoginDate': str(item[34]),
'lastName': item[35],
'lastUpdateDate': str(item[36]),
'locationName': item[37],
'maritalStatus': item[38],
'middleName': item[39],
'mobileNumber': item[40],
'password': item[41],
'paswdChangedDate': item[42],
'phoneNumber': item[43],
'preferredLocations': item[44],
'previousEmployer': item[45],
'profileImageUrl': item[46],
'profileSummary': item[47],
'registrationDate': str(item[48]),
'resumeContentType': item[49],
'resumeTitle': item[50],
's3DocKey': item[51],
's3HtmlKey': item[52],
'stateId': item[53],
'status': item[54],
'timeDate': str(item[55]),
'uuid': item[56],
'videoIntroType': item[57],
'videoIntroUrl': item[58],
'videoProfileTimeDate': item[59],
'videoProfileUrl': item[60],
'videoProfileVisibility': item[61],
'workStatusForOtherCountries': item[62],
'workStatusForUS': item[63],
'xIIPS': item[64],
'xIIPassingYear': item[65],
'xIIPercentageCgpa': item[66],
'xPS': item[67],
'xPassingYear': item[68],
'xPercentageCgpa': item[69]
}
for quid in secondaryQualification:
if userid[0]== quid[8]:
indQualification = {
'id': quid[0],
'institute': quid[1],
'isPrimary': quid[2],
'passingYear': quid[3],
'percentageCgpa': quid[4],
'qualification': quid[5],
'specialization': quid[6],
'timeDate': str(quid[7]),
'userId': quid[8]
}
listSecondaryQualification.append(indQualification)
# qualification[quid[0]] = indQualification
# print(qualification)
for esdid in examscore:
if userid[0]== esdid[2]:
indExamscores = {
'examScoreDetailId': esdid[0],
'attemptedCount': esdid[1],
'candidateId': esdid[2],
'examDate': esdid[3],
'examId': esdid[4],
'examTotalScore': esdid[5],
'examType': esdid[6],
'katExamEventId': esdid[7],
'rightAttempted': esdid[8],
'rptVisStatus': esdid[9],
'scorePercent': esdid[10],
'timeDate': str(esdid[11]),
'token': esdid[12],
'totalQuestion': esdid[13],
'totalScore': esdid[14]
}
listExamScores.append(indExamscores)
# examscores[esdid[0]]=indExamscores
# print(examscores)
for scid in sectionalScore:
if userid[0]== scid[2]:
indSectionalscores = {
'katScoreId': scid[0],
'attemptedCount': scid[1],
'candidateId': scid[2],
'examScoreId': scid[3],
'examTime': scid[4],
'examTitleId': scid[5],
'katExamId': scid[6],
'rightAttempted': scid[7],
'score': scid[8],
'scorePercent': scid[9],
'scoreS3Key': scid[10],
'timeDate': str(scid[11]),
'totalQuestion': scid[12],
'totalScore': scid[13]
}
listSectionalScores.append(indSectionalscores)
# sectionalscores[scid[0]]=indSectionalscores
# print(sectionalscores)
for prid in proejctDetails:
if userid[0] == prid[16]:
indvidualProjectdetail = {
'id': prid[0],
'clientName': prid[1],
'designation': prid[2],
'durationEndMonth': prid[3],
'durationEndYear': prid[4],
'durationStartMonth': prid[5],
'durationStartYear': prid[6],
'employmentNature': prid[7],
'location': prid[8],
'projectDetails': prid[9],
'rolesResponsibilities': prid[10],
'site': prid[11],
'skillsUsed': prid[12],
'teamSize': prid[13],
'timeDate': str(prid[14]),
'title': prid[15],
'userId': prid[16]
}
listProjectDetails.append(indvidualProjectdetail)
# proejctdetail[prid[0]]=indvidualProjectdetail
# print(proejctdetail)
for userempid in employmentDetails:
if userid[0]==userempid[12]:
indEmploymentDetail = {
'id': userempid[0],
'companyName': userempid[1],
'designation': userempid[2],
'durationEndMonth': userempid[3],
'durationEndYear': userempid[4],
'durationStartMonth': userempid[5],
'durationStartYear': userempid[6],
'employmentStatus': userempid[7],
'isCurrent': userempid[8],
'jobProfile': userempid[9],
'location': userempid[10],
'timeDate': str(userempid[11]),
'userId': userempid[12]
}
listEmploymentDetails.append(indEmploymentDetail)
# employmentDetail[userempid[0]]=indEmploymentDetail
# print(employmentDetail)
# for item in listSecondaryQualification:
# jsonSecondaryQualification += json.dumps(item) + ','
# jsonSecondaryQualification = jsonSecondaryQualification[:-1]
#
# for item in listEmploymentDetails:
# jsonEmploymentDetails+=json.dumps(item)+','
# jsonEmploymentDetails = jsonEmploymentDetails[:-1]
#
# for item in listExamScores:
# jsonExamScores+=json.dumps(item)+','
# jsonExamScores = jsonExamScores[:-1]
#
# for item in listProjectDetails:
# jsonProjectDetails+=json.dumps(item)+','
# jsonProjectDetails=jsonProjectDetails[:-1]
#
# for item in listSectionalScores:
# jsonSectionalScores+=json.dumps(item)+','
# jsonSectionalScores=jsonSectionalScores[:-1]
userdata['qualification'] = listSecondaryQualification
userdata['examscores'] = listExamScores
userdata['sectionalscores'] = listSectionalScores
userdata['projectdetail'] = listProjectDetails
userdata['employmentDetail'] = listEmploymentDetails
indexdata["_source"] = userdata
# doc['doc'] = indexdata
data.append(indexdata)
# print(data)
# break
#Flusing All lists
listSecondaryQualification = []
listExamScores = []
listSectionalScores = []
listProjectDetails = []
listEmploymentDetails = []
# jsonSecondaryQualification = ''
# jsonExamScores = ''
# jsonSectionalScores = ''
# jsonProjectDetails = ''
# jsonEmploymentDetails = ''
# proejctdetail = {}
# qualification = {}
# sectionalscores = {}
# examscores = {}
# employmentDetail = {}
# print("Document Inserted Successfully")
if (count==2000):
break
print('On my way')
helpers.bulk(es1, data)
print("bulk import done")