我正在尝试创建NIST数据库源的MySQL备份。我正在尝试使用线程来快速升级数据库，这样我就可以把它推到lambda中（花了10分钟，线程正在极大地帮助）。
当我没有线程时，我似乎不会遇到这个问题。
但是，我遇到了一个问题，经过一百次更新后，我收到以下错误。
CVE-2017-8829
CVE-2017-8830
CVE-2017-8831
CVE-2017-8832
CVE-2017-8833
CVE-2017-8842
CVE-2017-8843
CVE-2017-8844
CVE-2017-8845
CVE-2017-8846
CVE-2017-8847
CVE-2017-8848
Traceback (most recent call last):
File "./thread.py", line 146, in <module>
cveDB.commit()
File "/usr/local/lib/python2.7/site-packages/pymysql/connections.py", line 787, in commit
self._execute_command(COMMAND.COM_QUERY, "COMMIT")
File "/usr/local/lib/python2.7/site-packages/pymysql/connections.py", line 1071, in _execute_command
raise err.InterfaceError("(0, '')")
pymysql.err.InterfaceError: (0, '')
到目前为止，这是我的代码。
#!/usr/bin/env python
import threading
import sys
import json
import pymysql
import urllib
import gzip
import os,glob
import time
cveDB = pymysql.connect (host="remote-database",port=3306,user="database-user",passwd="database-password",db='database',use_unicode=True, charset="utf8")
threadLimiter = threading.BoundedSemaphore(10)
feed_url = 'https://static.nvd.nist.gov/feeds/json/cve/1.0/nvdcve-1.0-modified.json.gz'
print('Downloading Feed')
urllib.urlretrieve(feed_url, "/tmp/nvdcve-1.0-modified.json.gz")
print('Uncompressing File')
latest_file = gzip.open('/tmp/nvdcve-1.0-modified.json.gz', 'rb')
print('Opening Json')
cve_data = json.loads(latest_file.read())
print('Parsing Json')
#creates threading to speed up queries.
class threader(threading.Thread):
def __init__(self,query):
threadLimiter.acquire()
self.query = query
try:
threading.Thread.__init__(self)
finally:
threadLimiter.release()
def run(self):
try:
cve_cursor = cveDB.cursor()
cve_cursor.execute(self.query)
except:
sys.exit(1)
return
#creates the parent. This is where each individual CVE resides.
for data_list in cve_data["CVE_Items"]:
for cve_tag,cve_id in data_list["CVE_data_meta"].items():
cve = str(cve_id)
print(cve)
# This crawls down to get software impacted and its version.
for vendor_data in data_list["CVE_affects"]["CVE_vendor"]["CVE_vendor_data"]:
for product_data in vendor_data["CVE_product"]["CVE_product_data"]:
software_name = product_data["CVE_product_name"]
for software_version in product_data["CVE_version"]["CVE_version_data"]:
if software_version["CVE_version_value"] == '-':
application_version = software_name + ' ' + software_version["CVE_version_affected"] + ' version any'
else:
application_version = software_name + ' ' + software_version["CVE_version_affected"] + ' ' + software_version["CVE_version_value"]
# This gets the description of the vulnerability.
for cve_description_data in data_list["CVE_description"]["CVE_description_data"]:
cve_description = cve_description_data["value"]
#Gets the CVE Publication information such as the initial reports and publication dates.
for cve_reference_data in data_list["CVE_references"]["CVE_reference_data"]:
vuln_url = cve_reference_data["url"]
try:
publish_date = cve_reference_data["publish_date"]
except KeyError:
publish_date = ''
#Puts the above information into the MySQL database. This is under table CVE Details which has general CVE Information.
cve_details_query = '''REPLACE into cve_details (cve_id,cve_package,cve_description,cve_url,cve_published) VALUES ('%s', '%s', '%s', '%s', '%s')''' % (cve,application_version,cve_description,vuln_url,publish_date)
query_thread = threader(cve_details_query)
query_thread.start()
#This pulls each individual metric and stores it, and then also combines them into the CVSS v2 "string" which works with the NIST calculator.
#Also adds everything to the database.
if data_list["CVE_impact"]["CVE_impact_cvssv2"] != {}:
cvss_v2_av = data_list["CVE_impact"]["CVE_impact_cvssv2"]["bm"]["av"]
cvss_v2_ac = data_list["CVE_impact"]["CVE_impact_cvssv2"]["bm"]["ac"]
cvss_v2_au = data_list["CVE_impact"]["CVE_impact_cvssv2"]["bm"]["au"]
cvss_v2_c = data_list["CVE_impact"]["CVE_impact_cvssv2"]["bm"]["c"]
cvss_v2_i = data_list["CVE_impact"]["CVE_impact_cvssv2"]["bm"]["i"]
cvss_v2_a = data_list["CVE_impact"]["CVE_impact_cvssv2"]["bm"]["a"]
cvss_v2_score = data_list["CVE_impact"]["CVE_impact_cvssv2"]["bm"]["score"]
cvss_v2_total = 'AV:'+cvss_v2_av[0]+'/AC:'+cvss_v2_ac[0]+'/Au:'+cvss_v2_au[0]+'/C:'+cvss_v2_c[0]+'/I:'+cvss_v2_i[0]+'/A:'+cvss_v2_a[0]
cvssv2_query = '''REPLACE into cve_cvssv2 (cve_id, cvss_v2_av, cvss_v2_ac, cvss_v2_au, cvss_v2_c, cvss_v2_i, cvss_v2_a, cvss_v2_total, cvss_v2_score)
VALUES ('%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s')''' % (cve,cvss_v2_av,cvss_v2_ac,cvss_v2_au,cvss_v2_c,cvss_v2_i,cvss_v2_a,cvss_v2_total,cvss_v2_score)
query_thread = threader(cvssv2_query)
query_thread.start()
#Same as above but for CVSSv3.
if data_list["CVE_impact"]["CVE_impact_cvssv3"] != {}:
cvss_v3_av = data_list["CVE_impact"]["CVE_impact_cvssv3"]["bm"]["av"]
cvss_v3_ac = data_list["CVE_impact"]["CVE_impact_cvssv3"]["bm"]["ac"]
cvss_v3_pr = data_list["CVE_impact"]["CVE_impact_cvssv3"]["bm"]["pr"]
cvss_v3_ui = data_list["CVE_impact"]["CVE_impact_cvssv3"]["bm"]["ui"]
cvss_v3_scope = data_list["CVE_impact"]["CVE_impact_cvssv3"]["bm"]["scope"]
cvss_v3_c = data_list["CVE_impact"]["CVE_impact_cvssv3"]["bm"]["c"]
cvss_v3_i = data_list["CVE_impact"]["CVE_impact_cvssv3"]["bm"]["i"]
cvss_v3_a = data_list["CVE_impact"]["CVE_impact_cvssv3"]["bm"]["a"]
cvss_v3_score = data_list["CVE_impact"]["CVE_impact_cvssv3"]["bm"]["score"]
cvss_v3_total = 'AV:'+cvss_v3_av[0]+'/AC:'+cvss_v3_ac[0]+'/PR:'+cvss_v3_pr[0]+'/UI:'+cvss_v3_ui[0]+'/S:'+cvss_v3_scope[0]+'/C:'+cvss_v3_c[0]+'/I:'+cvss_v3_i[0]+'/A:'+cvss_v3_a[0]
cvssv3_query = '''REPLACE into cve_cvssv3 (cve_id,cvss_v3_av,cvss_v3_ac,cvss_v3_pr,cvss_v3_ui,cvss_v3_s,cvss_v3_c,cvss_v3_i,cvss_v3_a,cvss_v3_score,cvss_v3_total)
VALUES ('%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s')''' % (cve,cvss_v3_av,cvss_v3_ac,cvss_v3_pr,cvss_v3_ui,cvss_v3_scope,cvss_v3_c,cvss_v3_i,cvss_v3_a,cvss_v3_score,cvss_v3_total)
query_thread = threader(cvssv3_query)
query_thread.start()
#Gets the CPE for impacted OSes to compare against fleet.
#Because a CVE can have many CPE, easier to delete existing ones
#And add new ones because CVE cant run as a Primary Key.
for cve_configurations in data_list["CVE_configurations"]['CVE_configuration_data']:
delete_query = '''DELETE FROM cve_cpe WHERE cve_id LIKE '%s' ''' % (cve)
query_thread = threader(delete_query)
query_thread.start()
try:
for cve_cpe_data in cve_configurations['cpe']:
cve_cpe = cve_cpe_data['cpeMatchString']
cpe_query = '''INSERT into cve_cpe (cve_id,cpe_id) VALUES ('%s', '%s')''' % (cve,cve_cpe)
query_thread = threader(cpe_query)
query_thread.start()
except KeyError:
for cve_cpe_child_data in cve_configurations['children']:
for cve_cpe_data in cve_cpe_child_data['cpe']:
cve_cpe = str(cve_cpe_data['cpeMatchString'])
cpe_query = '''INSERT into cve_cpe (cve_id,cpe_id) VALUES ('%s', '%s')''' % (cve,cve_cpe)
query_thread = threader(cpe_query)
query_thread.start()
cveDB.commit()
print ('Database Updated')
cveDB.close()