我是kafka的新来者。当我使用kafka创建使用者时,遇到了一个问题。当我打开两个使用者程序并绑定相同的kafka组时,两个使用者使用了相同的数据。 Google有这个问题,很多答案是如果同一组绑定到多个使用者,那么这就是队列模式,不同的使用者应该获取主题下的数据之一,而不是相同。 Consumer_id设置了不同的值。 代码显示如下:
from pykafka import KafkaClient
class KafkaConsumer():
'''kafka consumer'''
def __init__(self, hosts, zookeeper, topic, consumer_group):
self.hosts = hosts
self.zookeeper = zookeeper
self.topic = topic
self.consumer_group = consumer_group
# self.consumer = KafkaConsumer(self.topic, group_id = 'test1',
# bootstrap_servers = self.hosts,
# auto_offset_reset='largest',
# enable_auto_commit=True)
# def consumer_message_1(self):
# for msg in self.consumer:
# print(msg)
def consumer_message(self):
dtime = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(time.time()))
client = KafkaClient(hosts=self.hosts)
topic = client.topics[self.topic]
consumer = topic.get_simple_consumer(
consumer_group = self.consumer_group,
consumer_id='test_client2',
auto_commit_enable=True,
auto_commit_interval_ms=1,
)
print(consumer.partitions)
for message in consumer:
print(consumer.held_offsets)
print(topic.partitions)
print(topic.latest_available_offsets())
print(message.value.decode())
consumer.commit_offsets()
return True
def main():
client = KafkaConsumer("xxx:8000", "test1", "test1")
client.consumer_message()
if __name__ == "__main__":
main()