Python代码卡在获取Kafka Broker Client的过程中

时间:2018-12-14 19:59:52

标签: python kafka-producer-api kafka-python

我有一段代码,它向Kafka客户端发送一些消息 但是很多时候,代码被困在获取经纪人的过程中。有时重新启动程序可以正常工作,否则程序会卡住,这是一个无限循环。

生产者= KafkaProducer(bootstrap_servers = ['host1:port','host2:port','host3:port','host4:port','host5:port'])

def KafkaProducerFromCSV(csv_file, topic):
    csv_file_reader = csv.reader(open(csv_file, 'rU'), quotechar='"',delimiter=',')
    data = {'schemaVersion': None, 'data': None, 'entityId': None, 'entityVersion': None, 'eventId': None,'eventTime': None, 'ingestedAt': None, 'parentId': None, 'parentVersion': None, 'updatedAt': None,'encodingType': None, 'payment_advice_created_at_date_key': None, 'payment_advice_id': None,'payment_advice_status_key': None}
    count = 0
    for i in range(1):
        next(csv_file_reader, 'END OF FILE')
        count = count + 1
    print("Yaha aaya")
    producer = KafkaProducer(bootstrap_servers=['host1:port','host2:port','host3:port','host4:port','host5:port'])#this is where the problem occur.!
    print(producer)
    try:
        for split_msg in csv_file_reader:
            print(split_msg)
            if split_msg == ['END OF FILE']:
                print("End of File encountered, ingestion completed for this slot.!")
                producer.close()
                csv_file_reader.close()
                break
            else:
                data['entityId'] = split_msg[3]
                data['entityVersion'] = split_msg[1]
                data['eventId'] = split_msg[2]
                data['eventTime'] = split_msg[4]
                data['ingestedAt'] = split_msg[5]
                data['parentId'] = split_msg[6]
                data['parentVersion'] = split_msg[7]
                data['updatedAt'] = split_msg[8]
                data['encodingType'] = split_msg[9]
                data['schemaVersion'] = split_msg[10]
                data['payment_advice_created_at_date_key'] = split_msg[11]
                data['payment_advice_id'] = split_msg[12]
                data['payment_advice_status_key'] = split_msg[13]
                data['data'] = split_msg[0].decode('string_escape')
                stringified_data = str(data['data'])
                json_data = json.loads(stringified_data[1:len(stringified_data) - 1])
                data['data'] = json_data
                producer.send(topic, json.dumps(data))
                count = count + 1
                if count % 1000 == 0:
                    producer.flush()
                    print "Successfully send " + str(count) + " messages"
                    continue
    except:
        pass

我该怎么办,我可以使用任何重试机制来获取代理客户端。

0 个答案:

没有答案