logo标签列表

Docker上的Apache Kafka集群无法连接到Zookeeper

dockerapache-kafkadocker-composeapache-zookeeper
3
我正在使用Docker创建一个包含3个Zookeeper节点和3个Kafka节点的集群。我需要将Kafka集群与Zookeeper集群连接起来。
以下是docker-compose.yml代码 -
version: '2'
services:
  zookeeper-1:
    image: zookeeper:latest
    environment:
      ZOOKEEPER_SERVER_ID: 1
      ZOOKEEPER_CLIENT_PORT: 22181
      ZOOKEEPER_TICK_TIME: 2000
      ZOOKEEPER_INIT_LIMIT: 5
      ZOOKEEPER_SYNC_LIMIT: 2
      ZOOKEEPER_SERVERS: localhost:22888:23888;localhost:32888:33888;localhost:42888:43888

  zookeeper-2:
    image: zookeeper:latest
    environment:
      ZOOKEEPER_SERVER_ID: 2
      ZOOKEEPER_CLIENT_PORT: 32181
      ZOOKEEPER_TICK_TIME: 2000
      ZOOKEEPER_INIT_LIMIT: 5
      ZOOKEEPER_SYNC_LIMIT: 2
      ZOOKEEPER_SERVERS: localhost:22888:23888;localhost:32888:33888;localhost:42888:43888  

  zookeeper-3:
    image: zookeeper:latest
    environment:
      ZOOKEEPER_SERVER_ID: 3
      ZOOKEEPER_CLIENT_PORT: 42181
      ZOOKEEPER_TICK_TIME: 2000
      ZOOKEEPER_INIT_LIMIT: 5
      ZOOKEEPER_SYNC_LIMIT: 2
      ZOOKEEPER_SERVERS: localhost:22888:23888;localhost:32888:33888;localhost:42888:43888    

  kafka1:
    image: abc/kafka:latest
    hostname: kafka1
    network_mode: host
    environment:
      KAFKA_BROKER_ID: 1
      KAFKA_ZOOKEEPER_CONNECT: localhost:22181,localhost:32181,localhost:42181
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:19092
    depends_on:
      - zookeeper-1
      - zookeeper-2
      - zookeeper-3

  kafka2:
    image: abc/kafka:latest
    hostname: kafka2
    network_mode: host
    environment:
      KAFKA_BROKER_ID: 2
      KAFKA_ZOOKEEPER_CONNECT: localhost:22181,localhost:32181,localhost:42181
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:29092
    depends_on:
      - zookeeper-1
      - zookeeper-2
      - zookeeper-3

  kafka3:
    image: abc/kafka:latest
    hostname: kafka3
    network_mode: host
    environment:
      KAFKA_BROKER_ID: 3
      KAFKA_ZOOKEEPER_CONNECT: localhost:22181,localhost:32181,localhost:42181
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:39092
    depends_on:
      - zookeeper-1
      - zookeeper-2
      - zookeeper-3

当我运行docker-compose up命令时，成功创建了一个由3个节点组成的zookeeper群集。但是Kafka群集没有被创建。命令行上只显示以下错误 -
kafka_kafka2_1 exited with code 0
kafka_kafka1_1 exited with code 0
kafka_kafka3_1 exited with code 0

我也添加了用于创建镜像的Dockerfile，如下所示 -
FROM centos:7
MAINTAINER abc@xyz.com

ENV KAFKA_BIN=http://www-eu.apache.org/dist/kafka/0.11.0.2/kafka_2.11-0.11.0.2.tgz

RUN yum install -y wget java-1.8.0-openjdk \
    && cd /tmp && wget -q $KAFKA_BIN \
    && export K_TAR=/tmp/$(ls kafka* | head -1) \
    && mkdir -p /opt/apache/kafka/ && tar -zxf $K_TAR -C /opt/apache/kafka/ \
    && cd /opt/apache/kafka && ln -s $(ls) current \
    && rm -rf $K_TAR

ENV KAFKA_HOME /opt/apache/kafka/current
ENV PATH $PATH:$KAFKA_HOME/bin

ADD resources /home/kafka

RUN groupadd -r kafka \
    && useradd -r -g kafka kafka \
    && mkdir -p /home/kafka \
    && chown -R kafka:kafka /home/kafka \
    && chmod -R +x /home/kafka/scripts \
    && mkdir -p /var/log/kafka \
    && chown -R kafka:kafka /var/log/kafka \
    && mkdir -p /etc/kafka \
    && chown -R kafka:kafka /etc/kafka

USER kafka

没有任何错误信息。请有人帮助我调试这个问题，为什么kafka集群无法启动。我无法调试它，特别是因为它只使用代码0退出，并且没有提供任何错误消息。

3个回答
4
首先，您忘记了暴露zookeeper端口。其次，“localhost”表示本地主机——即使对于docker容器也是如此。因此，您的Kafka容器将尝试在同一容器内连接到zookeeper，这是行不通的。
请参考以下链接，了解正确的操作方式：https://github.com/confluentinc/cp-docker-images/blob/4.0.x/examples/cp-all-in-one/docker-compose.yml 如果您不喜欢该方法，可以不暴露任何端口。为此，在compose文件中的zookeeper环境部分设置zookeeper客户端端口即可：
...
services:
  zookeeper:
    ...
    environment:
       ZOOKEEPER_CLIENT_PORT: 2181

第二步，将你的Kafka容器连接到它：
...
kafka:
  ...
  environment:
    ZOOKEEPER_SERVERS: zookeeper:2181

采用这种方法，所有操作都将在docker compose网络内部进行。

嗨，亨德里克，如果你看一下这个链接 - https://github.com/confluentinc/cp-docker-images/blob/4.0.x/examples/kafka-cluster/docker-compose.yml - 我所做的事情已经被解释了。即使是那个也似乎给我同样的错误。你能帮忙吗？ - Siddharth Satish
相比之下，在您的Zookeeper镜像中，缺少网络模式：主机。 - Hendrik M Halkow
即使使用网络模式：主机，它仍然会给出相同的错误 :/ - Siddharth Satish
你正在使用哪个主机操作系统？ - Hendrik M Halkow
1
可能还需要查看代理日志，而不仅仅是容器状态。使用docker-compose log命令可以获取该日志。有可能您现在已经没有与zookeeper的连接问题，但其他原因导致代理立即关闭。 - dawsaw
我也遇到了同样的问题，这个问题已经困扰我一段时间了。由于这个错误，我无法升级我的kafka集群。Zk集群从主机上完全可访问，我可以在zk节点上执行mntr命令而不会出现任何错误。但是Kafka就是无法连接。 - Anand Nadar
0
在 Docker 中，localhost 是指容器实例，而不是主机。这是 Kafka 安装过程中的一个主要混淆点。需要配置 Kafka 的监听器以广告不同的主机：端口组合 - 一个可以从容器内部访问的组合。

下面展示了如何为多个经纪人执行此操作的示例。 Credit: 该示例摘自伴随 Effective Kafka 的存储库。


version: "3.2"
services:
  zookeeper:
    image: bitnami/zookeeper:3
    ports:
      - 2181:2181
    environment:
      ALLOW_ANONYMOUS_LOGIN: "yes"
  kafka-0:
    image: bitnami/kafka:2
    ports:
      - 9092:9092
    environment:
      KAFKA_CFG_ZOOKEEPER_CONNECT: zookeeper:2181
      ALLOW_PLAINTEXT_LISTENER: "yes"
      KAFKA_LISTENERS: >-
          INTERNAL://:29092,EXTERNAL://:9092
      KAFKA_ADVERTISED_LISTENERS: >-
          INTERNAL://kafka-0:29092,EXTERNAL://localhost:9092
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: >-
          INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT
      KAFKA_INTER_BROKER_LISTENER_NAME: "INTERNAL"
    depends_on:
      - zookeeper
  kafka-1:
    image: bitnami/kafka:2
    ports:
      - 9093:9093
    environment:
      KAFKA_CFG_ZOOKEEPER_CONNECT: zookeeper:2181
      ALLOW_PLAINTEXT_LISTENER: "yes"
      KAFKA_LISTENERS: >-
          INTERNAL://:29092,EXTERNAL://:9093
      KAFKA_ADVERTISED_LISTENERS: >-
          INTERNAL://kafka-1:29092,EXTERNAL://localhost:9093
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: >-
          INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT
      KAFKA_INTER_BROKER_LISTENER_NAME: "INTERNAL"
    depends_on:
      - zookeeper
  kafka-2:
    image: bitnami/kafka:2
    ports:
      - 9094:9094
    environment:
      KAFKA_CFG_ZOOKEEPER_CONNECT: zookeeper:2181
      ALLOW_PLAINTEXT_LISTENER: "yes"
      KAFKA_LISTENERS: >-
          INTERNAL://:29092,EXTERNAL://:9094
      KAFKA_ADVERTISED_LISTENERS: >-
          INTERNAL://kafka-2:29092,EXTERNAL://localhost:9094
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: >-
          INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT
      KAFKA_INTER_BROKER_LISTENER_NAME: "INTERNAL"
    depends_on:
      - zookeeper
  kafdrop:
    image: obsidiandynamics/kafdrop:latest
    ports:
      - 9000:9000
    environment:
      KAFKA_BROKERCONNECT: >-
          kafka-0:29092,kafka-1:29092,kafka-2:29092
    depends_on:
      - kafka-0
      - kafka-1
      - kafka-2

这里，连接到代理的应用程序是Kafdrop，但您可以将其替换为自己的镜像。此外，为了简洁起见，示例仅使用一个带有3个代理的ZK节点；这应该是一个简单的更改。

0

你需要在/etc/hosts中设置正确的主机名，才能在Docker外部连接Kafka容器：

 cat /etc/hosts
# This file was automatically generated by WSL. To stop automatic generation of this file, add the following entry to /etc/wsl.conf:
# [network]
# generateHosts = false
127.0.0.1       localhost
127.0.0.1       kafka1
127.0.0.1       kafka2
127.0.0.1       kafka3

这是我的工作配置：

version: '2'
services:
  zookeeper1:
    image: confluentinc/cp-zookeeper:$CP_VERSION
    container_name: zookeeper1
    hostname: zookeeper1
    environment:
      ZOOKEEPER_CLIENT_PORT: 2181
      ZOOKEEPER_TICK_TIME: 2000
      ZOOKEEPER_SERVER_ID: 1
      ZOOKEEPER_SERVERS: "zookeeper1:22888:23888;zookeeper2:22888:23888;zookeeper3:22888:23888"
    ports:
      - 2181:2181
    volumes: 
      - ./data/zoo-1/data:/var/lib/zookeeper/data
      - ./data/zoo-1/log:/var/lib/zookeeper/log

  zookeeper2:
    image: confluentinc/cp-zookeeper:$CP_VERSION
    container_name: zookeeper2
    hostname: zookeeper2
    environment:
      ZOOKEEPER_CLIENT_PORT: 2181
      ZOOKEEPER_TICK_TIME: 2000
      ZOOKEEPER_SERVER_ID: 2
      ZOOKEEPER_SERVERS: "zookeeper1:22888:23888;zookeeper2:22888:23888;zookeeper3:22888:23888"
    ports:
      - 2182:2181
    volumes: 
      - ./data/zoo-2/data:/var/lib/zookeeper/data
      - ./data/zoo-2/log:/var/lib/zookeeper/log

  zookeeper3:
    image: confluentinc/cp-zookeeper:$CP_VERSION
    container_name: zookeeper3
    hostname: zookeeper3
    environment:
      ZOOKEEPER_CLIENT_PORT: 2181
      ZOOKEEPER_TICK_TIME: 2000
      ZOOKEEPER_SERVER_ID: 3
      ZOOKEEPER_SERVERS: "zookeeper1:22888:23888;zookeeper2:22888:23888;zookeeper3:22888:23888"
    ports:
      - 2183:2181
    volumes: 
      - ./data/zoo-3/data:/var/lib/zookeeper/data
      - ./data/zoo-3/log:/var/lib/zookeeper/log

  kafka1:
    image: confluentinc/cp-kafka:$CP_VERSION
    container_name: kafka1
    hostname: kafka1
    depends_on:
      - zookeeper1
      - zookeeper2
      - zookeeper3
    ports:
      - 9091:9091
    environment:
      KAFKA_BROKER_ID: 1
      KAFKA_ZOOKEEPER_CONNECT: zookeeper1:2181,zookeeper2:2181,zookeeper3:2181
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,OUTSIDE:PLAINTEXT
      KAFKA_LISTENERS: INTERNAL://:29092,OUTSIDE://:9091
      KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka1:29092,OUTSIDE://kafka1:9091
      KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_LOG_DIRS: /var/lib/kafka/logs


  kafka2:
    image: confluentinc/cp-kafka:$CP_VERSION
    container_name: kafka2
    hostname: kafka2
    depends_on:
      - zookeeper1
      - zookeeper2
      - zookeeper3
    ports:
      - 9092:9092
    environment:
      KAFKA_BROKER_ID: 2
      KAFKA_ZOOKEEPER_CONNECT: zookeeper1:2181,zookeeper2:2181,zookeeper3:2181
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,OUTSIDE:PLAINTEXT
      KAFKA_LISTENERS: INTERNAL://:29092,OUTSIDE://:9092
      KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka2:29092,OUTSIDE://kafka2:9092
      KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_LOG_DIRS: /var/lib/kafka/logs

  kafka3:
    image: confluentinc/cp-kafka:$CP_VERSION
    container_name: kafka3
    hostname: kafka3
    depends_on:
      - zookeeper1
      - zookeeper2
      - zookeeper3
    ports:
      - 9093:9093
    environment:
      KAFKA_BROKER_ID: 3
      KAFKA_ZOOKEEPER_CONNECT: zookeeper1:2181,zookeeper2:2181,zookeeper3:2181
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,OUTSIDE:PLAINTEXT
      KAFKA_LISTENERS: INTERNAL://:29092,OUTSIDE://:9093
      KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka3:29092,OUTSIDE://kafka3:9093
      KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_LOG_DIRS: /var/lib/kafka/logs
网页内容由stack overflow 提供, 点击上面的
可以查看英文原文，
原文链接