We need to implement a Kafka cluster with docker-compose. Facing Issue with Inter broker communication between machines. Can anyone help?

I have the following cluster architecture. But from machine 3 to machine 4 kafka inter broker communication is not happening. I am providing the docker-compose . please check the docker-compose?Do you find any wrong config?


machine 3


networks: 
  preprod-net: 
    driver: bridge

services:

  zookeeper-1:
    image: hub.fliplabs.net/cp-zookeeper
    container_name: zookeeper-1
    networks: 
      - preprod-net
    ports:
      - 2181:2181
      - 2888:2888
      - 3888:3888
    environment:
      ZOOKEEPER_CLIENT_PORT: 2181
      ZOOKEEPER_TICK_TIME: 2000
      ZOOKEEPER_SERVER_ID: 1
      ZOOKEEPER_SERVER_1: zookeeper-1:2888:3888
      ZOOKEEPER_SERVER_2: 192.168.39.96:2888:3888
      ZOOKEEPER_SERVER_3: 192.168.39.97:2888:3888


  kafka-1:
    image: hub.fliplabs.net/cp-kafka
    container_name: kafka-1
    networks: 
      - preprod-net
    ports:
      - 19092:19092
    depends_on:
      - zookeeper-1
    environment:
      KAFKA_BROKER_ID: 1
      KAFKA_ZOOKEEPER_CONNECT: zookeeper-1:2181,192.168.39.96:2181,192.168.39.97:2181
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT
      KAFKA_LISTENERS: INTERNAL://kafka-1:9092,EXTERNAL://0.0.0.0:19092
      KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka-1:9092,EXTERNAL://192.168.39.94:19092
      KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true"

  kafka-2:
    image: hub.fliplabs.net/cp-kafka
    container_name: kafka-2
    networks: 
      - preprod-net
    ports:
      - 29092:29092
    depends_on:
      - zookeeper-1
    environment:
      KAFKA_BROKER_ID: 2
      KAFKA_ZOOKEEPER_CONNECT: zookeeper-1:2181,192.168.39.96:2181,192.168.39.97:2181
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT
      KAFKA_LISTENERS: INTERNAL://kafka-2:9092,EXTERNAL://0.0.0.0:29092
      KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka-2:9092,EXTERNAL://192.168.39.94:29092
      KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true"

machine 4


networks: 
  preprod-net: 
    driver: bridge

services:

  zookeeper-2:
    image: hub.fliplabs.net/cp-zookeeper
    container_name: zookeeper-2
    networks: 
      - preprod-net
    ports:
      - 2181:2181
      - 2888:2888
      - 3888:3888
    environment:
      ZOOKEEPER_CLIENT_PORT: 2181
      ZOOKEEPER_TICK_TIME: 2000
      ZOOKEEPER_SERVER_ID: 2
      ZOOKEEPER_SERVER_1: zookeeper-2:2888:3888
      ZOOKEEPER_SERVER_2: 192.168.39.94:2888:3888
      ZOOKEEPER_SERVER_3: 192.168.39.97:2888:3888

  kafka-3:
    image: hub.fliplabs.net/cp-kafka
    container_name: kafka-3
    networks: 
      - preprod-net
    depends_on:
      - zookeeper-2
    ports:
      - 19092:19092
    environment:
      KAFKA_BROKER_ID: 3
      KAFKA_ZOOKEEPER_CONNECT: zookeeper-2:2181,192.168.39.94:2181,192.168.39.97:2181
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT
      KAFKA_LISTENERS: INTERNAL://kafka-3:9092,EXTERNAL://0.0.0.0:19092
      KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka-3:9092,EXTERNAL://192.168.39.96:19092
      KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true"

  kafka-4:
    image: hub.fliplabs.net/cp-kafka
    container_name: kafka-4
    networks: 
      - preprod-net
    depends_on:
      - zookeeper-2
    ports:
      - 29092:29092
    environment:
      KAFKA_BROKER_ID: 4
      KAFKA_ZOOKEEPER_CONNECT: zookeeper-2:2181,192.168.39.94:2181,192.168.39.97:2181
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT
      KAFKA_LISTENERS: INTERNAL://kafka-4:9092,EXTERNAL://0.0.0.0:29092
      KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka-4:9092,EXTERNAL://192.168.39.96:29092
      KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true"

machine 5

networks: 
  preprod-net: 
    driver: bridge

services:

  zookeeper-3:
    image: hub.fliplabs.net/cp-zookeeper
    container_name: zookeeper-3
    networks: 
      - preprod-net
    ports:
      - 2181:2181
      - 2888:2888
      - 3888:3888
    environment:
      ZOOKEEPER_CLIENT_PORT: 2181
      ZOOKEEPER_TICK_TIME: 2000
      ZOOKEEPER_SERVER_ID: 1
      ZOOKEEPER_SERVER_1: zookeeper-3:2888:3888
      ZOOKEEPER_SERVER_2: 192.168.39.94:2888:3888
      ZOOKEEPER_SERVER_3: 192.168.39.96:2888:3888

  kafdrop:
    image: hub.fliplabs.net/kafdrop
    container_name: kafdrop
    networks: 
      - preprod-net
    depends_on:
      - schema-registry
    ports:
      - 9000:9000
    environment:
      KAFKA_BROKERCONNECT: "192.168.39.94:19092,192.168.39.94:29092,192.168.39.96:19092,192.168.39.96:29092"
      JVM_OPTS: -Xms32M -Xmx64M
      SERVER_SERVLET_CONTEXTPATH: /

  schema-registry:
    image: hub.fliplabs.net/cp-schema-registry
    container_name: schema-registry
    networks: 
      - preprod-net
    ports:
      - 8081:8081
    environment:
      SCHEMA_REGISTRY_HOST_NAME: schema-registry
      SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: "192.168.39.94:19092,192.168.39.94:29092,192.168.39.96:19092,192.168.39.96:29092"
      SCHEMA_REGISTRY_ACCESS_CONTROL_ALLOW_METHODS: GET,POST,PUT,OPTIONS
      SCHEMA_REGISTRY_ACCESS_CONTROL_ALLOW_ORIGIN: '*'

  kafka-connect:
    image: hub.fliplabs.net/cp-kafka-connect
    container_name: kafka-connect
    volumes:
       - /home/kafka/plugin:/usr/local/share/kafka/plugins
    networks: 
      - preprod-net
    depends_on:
      - schema-registry
    ports:
      - 8083:8083
    environment:
      CONNECT_BOOTSTRAP_SERVERS: 192.168.39.94:19092,192.168.39.94:29092,192.168.39.96:19092,192.168.39.96:29092
      CONNECT_REST_PORT: 8083
      CONNECT_GROUP_ID: kafka-connect
      CONNECT_CONFIG_STORAGE_TOPIC: _connect-configs
      CONNECT_OFFSET_STORAGE_TOPIC: _connect-offsets
      CONNECT_STATUS_STORAGE_TOPIC: _connect-status
      CONNECT_KEY_CONVERTER: org.apache.kafka.connect.storage.StringConverter
      CONNECT_VALUE_CONVERTER: org.apache.kafka.connect.json.JsonConverter
      CONNECT_INTERNAL_KEY_CONVERTER: org.apache.kafka.connect.storage.StringConverter
      CONNECT_INTERNAL_VALUE_CONVERTER: org.apache.kafka.connect.json.JsonConverter
      CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: "http://schema-registry:8081"
      CONNECT_REST_ADVERTISED_HOST_NAME: "kafka-connect"
      CONNECT_LOG4J_APPENDER_STDOUT_LAYOUT_CONVERSIONPATTERN: "[%d] %p %X{connector.context}%m (%c:%L)%n"
      CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: "1"
      CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: "1"
      CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: "1"
      CONNECT_PLUGIN_PATH: /usr/share/java,/usr/share/confluent-hub-components,/data/connect-jars,/usr/share/java/kafka-connect-jdbc,/usr/local/share/kafka/plugins

hi @jithu

welcome :slight_smile:

did not yet check in detail though it me related to wrong listener and advertised listener settings.
check also @rmoff blog post about this

basic question:
is there a special reason for using docker?
it might be easier with a non-docker installation.

and what about distributing the brokers across all machines?

best,
michael