需要使用python将消息从Kafka Client（windows）连接并发送到Hortonworks Platform上的Kerberized环境。
我们有客户端证书cacerts，该证书在通配符cert中带有条目*.localdomain。
尝试了以下可能性，但结果不成功。请帮我解决。
我们提到this link进行连接。
使用pykafka：
代码
import logging as log
log.basicConfig(level=log.DEBUG)
from pykafka import KafkaClient,SslConfig
config = SslConfig(cafile='D:\cacerts',password='password')
config._wrap_socket = config._legacy_wrap_socket()
client = KafkaClient(hosts='node1.localdomain:6668,node2.localdomain:6668',
ssl_config=config,
socket_timeout_ms=10000,
zookeeper_hosts="zknode.localdomain:2181")
client.topics
print (client.topics)
注意：它在非kerberized环境下工作正常。
错误
DEBUG:pykafka.cluster:Updating cluster, attempt 1/3
INFO:kazoo.client:Connecting to 192.168.xx.xxx:2181
DEBUG:kazoo.client:Sending request(xid=None): Connect(protocol_version=0, last_zxid_seen=0, time_out=10000, session_id=0, passwd=b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00', read_only=None)
INFO:kazoo.client:Zookeeper connection established, state: CONNECTED
DEBUG:kazoo.client:Sending request(xid=1): GetChildren(path='/brokers/ids', watcher=None)
DEBUG:kazoo.client:Received response(xid=1): ['1002']
DEBUG:kazoo.client:Sending request(xid=2): GetData(path='/brokers/ids/1002', watcher=None)
DEBUG:kazoo.client:Received response(xid=2): (b'{"jmx_port":-1,"timestamp":"1531102985486","endpoints":["SASL_PLAINTEXT://node1.localdomain:6667","SASL_SSL://node2.localdomain:6668"],"host":null,"version":3,"port":-1,"rack":"/default-rack"}', ZnodeStat(czxid=34360596628, mzxid=34360596628, ctime=1531102985489, mtime=1531102985489, version=0, cversion=0, aversion=0, ephemeralOwner=244410183054001653, dataLength=202, numChildren=0, pzxid=34360596628))
DEBUG:kazoo.client:Sending request(xid=3): Close()
INFO:kazoo.client:Closing connection to 192.168.xx.xxx:2181
INFO:kazoo.client:Zookeeper session lost, state: CLOSED
DEBUG:pykafka.connection:Connecting to None:-1
INFO:pykafka.connection:Failed to connect to None:-1
INFO:pykafka.connection:[WinError 10061] No connection could be made because the target machine actively refused it
WARNING:pykafka.broker:Failed to connect newly created broker for None:-1
ERROR:pykafka.cluster:Socket disconnected during request for broker None:-1. Continuing.
DEBUG:pykafka.connection:Connecting to None:-1
INFO:pykafka.connection:Failed to connect to None:-1
INFO:pykafka.connection:[WinError 10061] No connection could be made because the target machine actively refused it
WARNING:pykafka.broker:Failed to connect newly created broker for None:-1
ERROR:pykafka.cluster:Socket disconnected during request for broker None:-1. Continuing.
DEBUG:pykafka.connection:Connecting to None:-1
INFO:pykafka.connection:Failed to connect to None:-1
INFO:pykafka.connection:[WinError 10061] No connection could be made because the target machine actively refused it
WARNING:pykafka.broker:Failed to connect newly created broker for None:-1
ERROR:pykafka.cluster:Socket disconnected during request for broker None:-1. Continuing.
使用kafka-python：
代码
import logging as log
log.basicConfig(level=log.DEBUG)
from __future__ import print_function # python 2/3 compatibility
import sys # used to exit
from kafka import KafkaConsumer
import ssl
KAFKA_TOPIC = 'test'
KAFKA_BROKERS = 'node1.localdomain:6668,node2.localdomain:6668'
consumer = KafkaConsumer(KAFKA_TOPIC,
bootstrap_servers=KAFKA_BROKERS,
auto_offset_reset='earliest',
security_protocol='SSL',
ssl_certfile='D:\cacerts',
ssl_password='password',
ssl_check_hostname=True
)
try:
for message in consumer:
print(message.value)
except KeyboardInterrupt:
sys.exit()
错误
Traceback (most recent call last):
File "C:\Users\user\AppData\Local\Programs\Python\Python37\Lib\site-packages\kafkapython.py", line 64, in <module>
ssl_check_hostname=True
File "C:\Users\user\AppData\Local\Programs\Python\Python37\Lib\site-packages\kafka\consumer\group.py", line 340, in __init__
self._client = KafkaClient(metrics=self._metrics, **self.config)
File "C:\Users\user\AppData\Local\Programs\Python\Python37\Lib\site-packages\kafka\client_async.py", line 214, in __init__
self._bootstrap(collect_hosts(self.config['bootstrap_servers']))
File "C:\Users\user\AppData\Local\Programs\Python\Python37\Lib\site-packages\kafka\client_async.py", line 245, in _bootstrap
if not bootstrap.connect_blocking():
File "C:\Users\user\AppData\Local\Programs\Python\Python37\Lib\site-packages\kafka\conn.py", line 301, in connect_blocking
self.connect()
File "C:\Users\user\AppData\Local\Programs\Python\Python37\Lib\site-packages\kafka\conn.py", line 354, in connect
ret = self._sock.connect_ex(self._sock_addr)
File "C:\Users\user\AppData\Local\Programs\Python\Python37\lib\ssl.py", line 1146, in connect_ex
return self._real_connect(addr, True)
File "C:\Users\user\AppData\Local\Programs\Python\Python37\lib\ssl.py", line 1118, in _real_connect
raise ValueError("attempt to connect already-connected SSLSocket!")
ValueError: attempt to connect already-connected SSLSocket!
代码
import logging as log
log.basicConfig(level=log.DEBUG)
from __future__ import print_function # python 2/3 compatibility
import sys # used to exit
from kafka import KafkaConsumer
import ssl
KAFKA_TOPIC = 'test'
KAFKA_BROKERS = 'node1.localdomain:6668,node2.localdomain:6668'
consumer = KafkaConsumer(KAFKA_TOPIC,
bootstrap_servers=KAFKA_BROKERS,
auto_offset_reset='earliest',
security_protocol='SSL',
ssl_cafile='D:\cacerts',
ssl_certfile='D:\certificate.pem',
ssl_keyfile='D:\key.pem',
ssl_password='password',
ssl_check_hostname=True
)
try:
for message in consumer:
print(message.value)
except KeyboardInterrupt:
sys.exit()
仅供参考，这些pem文件是从cacerts（Alias：*.localdomain）生成的
错误
Traceback (most recent call last):
File "C:\Users\user\AppData\Local\Programs\Python\Python37\Lib\site-packages\kafkapython.py", line 40, in <module>
ssl_check_hostname=True
File "C:\Users\user\AppData\Local\Programs\Python\Python37\Lib\site-packages\kafka\consumer\group.py", line 340, in __init__
self._client = KafkaClient(metrics=self._metrics, **self.config)
File "C:\Users\user\AppData\Local\Programs\Python\Python37\Lib\site-packages\kafka\client_async.py", line 214, in __init__
self._bootstrap(collect_hosts(self.config['bootstrap_servers']))
File "C:\Users\user\AppData\Local\Programs\Python\Python37\Lib\site-packages\kafka\client_async.py", line 245, in _bootstrap
if not bootstrap.connect_blocking():
File "C:\Users\user\AppData\Local\Programs\Python\Python37\Lib\site-packages\kafka\conn.py", line 301, in connect_blocking
self.connect()
File "C:\Users\user\AppData\Local\Programs\Python\Python37\Lib\site-packages\kafka\conn.py", line 340, in connect
self._wrap_ssl()
File "C:\Users\user\AppData\Local\Programs\Python\Python37\Lib\site-packages\kafka\conn.py", line 426, in _wrap_ssl
self._ssl_context.load_verify_locations(self.config['ssl_cafile'])
OSError: [Errno 22] Invalid argument