您好,我目前正在使用Docker设置Kafka。我已设法使用已发布的汇合图像设置Zookeeper和Kafka,请参阅以下docker-compose文件:
version: '2'
services:
zookeeper:
image: confluentinc/cp-zookeeper:3.2.0
container_name: zookeeper
hostname: zookeeper
ports:
- "2181:2181"
environment:
ZOOKEEPER_CLIENT_PORT: 2181
ZOOKEEPER_TICK_TIME: 2000
restart: always
kafka:
image: confluentinc/cp-kafka:3.2.0
hostname: kafka
container_name: kafka
depends_on:
- zookeeper
ports:
- '9092:9092'
environment:
KAFKA_BROKER_ID: 1
KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181'
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://192.168.99.100:9092
LISTENERS: PLAINTEXT://0.0.0.0:9092
restart: always
kafka-rest:
image: confluentinc/cp-kafka-rest:3.2.0
container_name: kafka-rest
depends_on:
- kafka
ports:
- '8082:8082'
environment:
KAFKA_REST_ZOOKEEPER_CONNECT: 'zookeeper:2181'
KAFKA_REST_LISTENERS: http://kafka-rest:8082
KAFKA_REST_SCHEMA_REGISTRY_URL: http://schema-registry:8081
KAFKA_REST_HOST_NAME: kafka-rest
restart: always
schema-registry:
image: confluentinc/cp-schema-registry:3.2.0
container_name: schema-registry
depends_on:
- kafka
ports:
- '8081'
environment:
SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: 'zookeeper:2181'
SCHEMA_REGISTRY_HOST_NAME: schema-registry
SCHEMA_REGISTRY_LISTENERS: http://schema-registry:8081
restart: always
connect:
image: confluentinc/cp-kafka-connect:3.2.0
container_name: kafka-connect
depends_on:
- zookeeper
- kafka
- schema-registry
ports:
- "8083:8083"
restart: always
environment:
CONNECT_BOOTSTRAP_SERVERS: 'kafka:9092'
CONNECT_REST_ADVERTISED_HOST_NAME: connect
CONNECT_REST_PORT: 8083
CONNECT_GROUP_ID: compose-connect-group
CONNECT_CONFIG_STORAGE_TOPIC: docker-connect-configs
CONNECT_OFFSET_STORAGE_TOPIC: docker-connect-offsets
CONNECT_STATUS_STORAGE_TOPIC: docker-connect-status
CONNECT_KEY_CONVERTER: io.confluent.connect.avro.AvroConverter
CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL: http://schema-registry:8081
CONNECT_VALUE_CONVERTER: io.confluent.connect.avro.AvroConverter
CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: http://schema-registry:8081
CONNECT_INTERNAL_KEY_CONVERTER: org.apache.kafka.connect.json.JsonConverter
CONNECT_INTERNAL_VALUE_CONVERTER: org.apache.kafka.connect.json.JsonConverter
CONNECT_ZOOKEEPER_CONNECT: "zookeeper:2181"
现在我已经设法通过正确地将advertised.listener属性设置为PLAINTEXT:// {DOCKER_MACHINE_IP}:9092来正确地将Kafka容器暴露给我的非dockerized应用程序,但是你可以看到我'我还添加了其他融合应用程序来扩展我的Kafka设置(Kafka REST,Schema-Registry)。由于advertised.listener属性,它们无法再连接到我的Kafka实例。
我可以将其更改为正确的容器主机名 - > PLAINTEXT:// kafka:9092然后我失去了使用我的其他应用程序再次访问kafka实例的能力。有没有简单的方法来解决这个问题?
答案 0 :(得分:4)
作为管理员,打开C:\Windows\System32\drivers\etc\hosts
并添加以下行以将kafka代理公开为localhost。
127.0.0.1 broker
我的docker-compose.yml
文件如下所示:
---
version: '2'
services:
zookeeper:
image: confluentinc/cp-zookeeper
hostname: zookeeper
extra_hosts:
- "moby:127.0.0.1"
ports:
- "2181:2181"
environment:
ZOOKEEPER_CLIENT_PORT: 2181
ZOOKEEPER_TICK_TIME: 2000
broker:
image: confluentinc/cp-kafka
hostname: broker
extra_hosts:
- "moby:127.0.0.1"
depends_on:
- zookeeper
ports:
- '9092:9092'
environment:
KAFKA_BROKER_ID: 1
KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181'
KAFKA_ADVERTISED_LISTENERS: 'PLAINTEXT://broker:9092'
KAFKA_DEFAULT_REPLICATION_FACTOR: 1
schema_registry:
image: confluentinc/cp-schema-registry
hostname: schema_registry
# extra_hosts:
# - "moby:127.0.0.1"
depends_on:
- zookeeper
- broker
ports:
- '8081:8081'
environment:
SCHEMA_REGISTRY_HOST_NAME: schema_registry
SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: 'zookeeper:2181'
kafka-rest:
image: confluentinc/cp-kafka-rest
container_name: kafka-rest
extra_hosts:
- "moby:127.0.0.1"
depends_on:
- zookeeper
- broker
ports:
- '8082:8082'
environment:
KAFKA_REST_ZOOKEEPER_CONNECT: 'zookeeper:2181'
KAFKA_REST_LISTENERS: http://kafka-rest:8082
KAFKA_REST_SCHEMA_REGISTRY_URL: http://schema-registry:8081
KAFKA_REST_HOST_NAME: kafka-rest
或者,暴露我的笔记本电脑的当前IP地址(使用ipconfig / all)也可以,但这样做的缺点是,每当我的网络发生变化时,我都必须更改docker-compose.yml
文件。
答案 1 :(得分:0)
假定这是针对您的本地开发环境的设置,那么我将在主机网络上而不是在Docker网络中运行所有容器。
这可以通过在容器配置中添加network_mode: host
来完成。这样会使添加端口映射或主机名变得过时。
version: '2'
services:
zookeeper:
image: confluentinc/cp-zookeeper:3.2.0
network_mode: host
environment:
ZOOKEEPER_CLIENT_PORT: 2181
ZOOKEEPER_TICK_TIME: 2000
restart: always
kafka:
image: confluentinc/cp-kafka:3.2.0
depends_on:
- zookeeper
network_mode: host
environment:
KAFKA_BROKER_ID: 1
KAFKA_ZOOKEEPER_CONNECT: 'localhost:2181'
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:9092
LISTENERS: PLAINTEXT://localhost:9092
restart: always
kafka-rest:
image: confluentinc/cp-kafka-rest:3.2.0
depends_on:
- kafka
network_mode: host
environment:
KAFKA_REST_ZOOKEEPER_CONNECT: 'localhost:2181'
KAFKA_REST_LISTENERS: http://localhost:8082
KAFKA_REST_SCHEMA_REGISTRY_URL: http://localhost:8081
KAFKA_REST_HOST_NAME: localhost
restart: always
schema-registry:
image: confluentinc/cp-schema-registry:3.2.0
depends_on:
- kafka
network_mode: host
environment:
SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: 'localhost:2181'
SCHEMA_REGISTRY_HOST_NAME: localhost
SCHEMA_REGISTRY_LISTENERS: http://localhost:8081
restart: always
connect:
image: confluentinc/cp-kafka-connect:3.2.0
depends_on:
- zookeeper
- kafka
- schema-registry
network_mode: host
restart: always
environment:
CONNECT_BOOTSTRAP_SERVERS: 'localhost:9092'
CONNECT_REST_ADVERTISED_HOST_NAME: connect
CONNECT_REST_PORT: 8083
CONNECT_GROUP_ID: compose-connect-group
CONNECT_CONFIG_STORAGE_TOPIC: docker-connect-configs
CONNECT_OFFSET_STORAGE_TOPIC: docker-connect-offsets
CONNECT_STATUS_STORAGE_TOPIC: docker-connect-status
CONNECT_KEY_CONVERTER: io.confluent.connect.avro.AvroConverter
CONNECT_KEY_CONVERTER_SCHEMA_REGISTRY_URL: http://localhost:8081
CONNECT_VALUE_CONVERTER: io.confluent.connect.avro.AvroConverter
CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: http://localhost:8081
CONNECT_INTERNAL_KEY_CONVERTER: org.apache.kafka.connect.json.JsonConverter
CONNECT_INTERNAL_VALUE_CONVERTER: org.apache.kafka.connect.json.JsonConverter
CONNECT_ZOOKEEPER_CONNECT: "localhost:2181"
Mac警告:由于Mac上的Docker在虚拟机上运行,因此不支持此功能。据我所知,mac用户别无选择,只能使用端口映射并编辑其/etc/hosts
。
针对Windows的警告:可能是同一个问题-毕竟所有Docker仅在Linux Kernels上本地运行。
答案 2 :(得分:0)
带有单个Zookepper的单个卡夫卡
version: '2.1'
services:
zookeeper:
image: wurstmeister/zookeeper
hostname: zookeeper
container_name: zookeeper
ports:
- "2181:2181"
environment:
ZOO_MY_ID: 1
ZOO_PORT: 2181
ZOO_SERVERS: server.1=zookeeper:2888:3888
restart: always
volumes:
- ./zk-single-kafka-single/zookeeper/data:/data
- ./zk-single-kafka-single/zookeeper/datalog:/datalog
kafka:
image: wurstmeister/kafka
hostname: kafka
container_name: kafka
ports:
- "9092:9092"
environment:
HOSTNAME_COMMAND: "docker info | grep ^Name: | cut -d' ' -f 2"
KAFKA_ADVERTISED_LISTENERS: LISTENER_DOCKER_INTERNAL://kafka:9092,LISTENER_DOCKER_EXTERNAL://_{HOSTNAME_COMMAND}:9094
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: LISTENER_DOCKER_INTERNAL:PLAINTEXT,LISTENER_DOCKER_EXTERNAL:PLAINTEXT
KAFKA_ZOOKEEPER_CONNECT: "zookeeper:2181"
KAFKA_LISTENERS: LISTENER_DOCKER_INTERNAL://:9092,LISTENER_DOCKER_EXTERNAL://:9094
KAFKA_INTER_BROKER_LISTENER_NAME: LISTENER_DOCKER_INTERNAL
KAFKA_ADVERTISED_HOST_NAME: 172.19.0.1
KAFKA_BROKER_ID: 1
KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO"
KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
restart: always
volumes:
- ./zk-single-kafka-single/kafka/data:/var/lib/kafka/data
depends_on:
- zookeeper