Error: Failed to get Kafka cluster ID

Schema Registry
1

Hi,

Good day!

I have a 3 Node Zookeeper, Kafka, and Schema Registry. I enabled SSL for Zookeeper, Kafka, and Schema Registry. However, upon running the Schema Registry I’m having this error.

io.confluent.kafka.schemaregistry.exceptions.SchemaRegistryException: Failed to get Kafka cluster ID
at io.confluent.kafka.schemaregistry.storage.KafkaSchemaRegistry.kafkaClusterId(KafkaSchemaRegistry.java:1261)
at io.confluent.kafka.schemaregistry.storage.KafkaSchemaRegistry.(KafkaSchemaRegistry.java:158)
at io.confluent.kafka.schemaregistry.rest.SchemaRegistryRestApplication.initSchemaRegistry(SchemaRegistryRestApplication.java:69)
at io.confluent.kafka.schemaregistry.rest.SchemaRegistryRestApplication.configureBaseApplication(SchemaRegistryRestApplication.java:88)
at io.confluent.rest.Application.configureHandler(Application.java:263)
at io.confluent.rest.ApplicationServer.doStart(ApplicationServer.java:196)
at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:73)
at io.confluent.kafka.schemaregistry.rest.SchemaRegistryMain.main(SchemaRegistryMain.java:44)
Caused by: java.util.concurrent.TimeoutException
at org.apache.kafka.common.internals.KafkaFutureImpl$SingleWaiter.await(KafkaFutureImpl.java:108)
at org.apache.kafka.common.internals.KafkaFutureImpl.get(KafkaFutureImpl.java:272)
at io.confluent.kafka.schemaregistry.storage.KafkaSchemaRegistry.kafkaClusterId(KafkaSchemaRegistry.java:1259)
… 7 more

2

Hi @hasmine.roldan ,I am also facing the same kind of issue ,Did you get the solution?

3

Hi @jithu, try adding this

inter.instance.protocol=https

4

@hasmine.roldan ,I have the following cluster architecture. But from machine 3 to machine 4 kafka inter broker communication is not happening. I am providing the docker-compose of machine 3 .Could you please check the docker-compose?Do you find any wrong config?

Kafka preprod
Kafka preprod229×542 33.7 KB

machine 3


networks: 
  preprod-net: 
    driver: bridge

services:

  zookeeper-1:
    image: hub.fliplabs.net/cp-zookeeper
    container_name: zookeeper-1
    networks: 
      - preprod-net
    ports:
      - 2181:2181
      - 2888:2888
      - 3888:3888
    environment:
      ZOOKEEPER_CLIENT_PORT: 2181
      ZOOKEEPER_TICK_TIME: 2000
      ZOOKEEPER_SERVER_ID: 1
      ZOOKEEPER_SERVER_1: zookeeper-1:2888:3888
      ZOOKEEPER_SERVER_2: 192.168.39.96:2888:3888
      ZOOKEEPER_SERVER_3: 192.168.39.97:2888:3888


  kafka-1:
    image: hub.fliplabs.net/cp-kafka
    container_name: kafka-1
    networks: 
      - preprod-net
    ports:
      - 19092:19092
    depends_on:
      - zookeeper-1
    environment:
      KAFKA_BROKER_ID: 1
      KAFKA_ZOOKEEPER_CONNECT: zookeeper-1:2181,192.168.39.96:2181,192.168.39.97:2181
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT
      KAFKA_LISTENERS: INTERNAL://kafka-1:9092,EXTERNAL://0.0.0.0:19092
      KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka-1:9092,EXTERNAL://192.168.39.94:19092
      KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true"

  kafka-2:
    image: hub.fliplabs.net/cp-kafka
    container_name: kafka-2
    networks: 
      - preprod-net
    ports:
      - 29092:29092
    depends_on:
      - zookeeper-1
    environment:
      KAFKA_BROKER_ID: 2
      KAFKA_ZOOKEEPER_CONNECT: zookeeper-1:2181,192.168.39.96:2181,192.168.39.97:2181
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT
      KAFKA_LISTENERS: INTERNAL://kafka-2:9092,EXTERNAL://0.0.0.0:29092
      KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka-2:9092,EXTERNAL://192.168.39.94:29092
      KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true"

machine 4


networks: 
  preprod-net: 
    driver: bridge

services:

  zookeeper-2:
    image: hub.fliplabs.net/cp-zookeeper
    container_name: zookeeper-2
    networks: 
      - preprod-net
    ports:
      - 2181:2181
      - 2888:2888
      - 3888:3888
    environment:
      ZOOKEEPER_CLIENT_PORT: 2181
      ZOOKEEPER_TICK_TIME: 2000
      ZOOKEEPER_SERVER_ID: 2
      ZOOKEEPER_SERVER_1: zookeeper-2:2888:3888
      ZOOKEEPER_SERVER_2: 192.168.39.94:2888:3888
      ZOOKEEPER_SERVER_3: 192.168.39.97:2888:3888

  kafka-3:
    image: hub.fliplabs.net/cp-kafka
    container_name: kafka-3
    networks: 
      - preprod-net
    depends_on:
      - zookeeper-2
    ports:
      - 19092:19092
    environment:
      KAFKA_BROKER_ID: 3
      KAFKA_ZOOKEEPER_CONNECT: zookeeper-2:2181,192.168.39.94:2181,192.168.39.97:2181
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT
      KAFKA_LISTENERS: INTERNAL://kafka-3:9092,EXTERNAL://0.0.0.0:19092
      KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka-3:9092,EXTERNAL://192.168.39.96:19092
      KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true"

  kafka-4:
    image: hub.fliplabs.net/cp-kafka
    container_name: kafka-4
    networks: 
      - preprod-net
    depends_on:
      - zookeeper-2
    ports:
      - 29092:29092
    environment:
      KAFKA_BROKER_ID: 4
      KAFKA_ZOOKEEPER_CONNECT: zookeeper-2:2181,192.168.39.94:2181,192.168.39.97:2181
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT
      KAFKA_LISTENERS: INTERNAL://kafka-4:9092,EXTERNAL://0.0.0.0:29092
      KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka-4:9092,EXTERNAL://192.168.39.96:29092
      KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true"

machine 5

networks: 
  preprod-net: 
    driver: bridge

services:

  zookeeper-3:
    image: hub.fliplabs.net/cp-zookeeper
    container_name: zookeeper-3
    networks: 
      - preprod-net
    ports:
      - 2181:2181
      - 2888:2888
      - 3888:3888
    environment:
      ZOOKEEPER_CLIENT_PORT: 2181
      ZOOKEEPER_TICK_TIME: 2000
      ZOOKEEPER_SERVER_ID: 1
      ZOOKEEPER_SERVER_1: zookeeper-3:2888:3888
      ZOOKEEPER_SERVER_2: 192.168.39.94:2888:3888
      ZOOKEEPER_SERVER_3: 192.168.39.96:2888:3888

  kafdrop:
    image: hub.fliplabs.net/kafdrop
    container_name: kafdrop
    networks: 
      - preprod-net
    depends_on:
      - schema-registry
    ports:
      - 9000:9000
    environment:
      KAFKA_BROKERCONNECT: "192.168.39.94:19092,192.168.39.94:29092,192.168.39.96:19092,192.168.39.96:29092"
      JVM_OPTS: -Xms32M -Xmx64M
      SERVER_SERVLET_CONTEXTPATH: /

  schema-registry:
    image: hub.fliplabs.net/cp-schema-registry
    container_name: schema-registry
    networks: 
      - preprod-net
    ports:
      - 8081:8081
    environment:
      SCHEMA_REGISTRY_HOST_NAME: schema-registry
      SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: "192.168.39.94:19092,192.168.39.94:29092,192.168.39.96:19092,192.168.39.96:29092"
      SCHEMA_REGISTRY_ACCESS_CONTROL_ALLOW_METHODS: GET,POST,PUT,OPTIONS
      SCHEMA_REGISTRY_ACCESS_CONTROL_ALLOW_ORIGIN: '*'

  kafka-connect:
    image: hub.fliplabs.net/cp-kafka-connect
    container_name: kafka-connect
    volumes:
       - /home/kafka/plugin:/usr/local/share/kafka/plugins
    networks: 
      - preprod-net
    depends_on:
      - schema-registry
    ports:
      - 8083:8083
    environment:
      CONNECT_BOOTSTRAP_SERVERS: 192.168.39.94:19092,192.168.39.94:29092,192.168.39.96:19092,192.168.39.96:29092
      CONNECT_REST_PORT: 8083
      CONNECT_GROUP_ID: kafka-connect
      CONNECT_CONFIG_STORAGE_TOPIC: _connect-configs
      CONNECT_OFFSET_STORAGE_TOPIC: _connect-offsets
      CONNECT_STATUS_STORAGE_TOPIC: _connect-status
      CONNECT_KEY_CONVERTER: org.apache.kafka.connect.storage.StringConverter
      CONNECT_VALUE_CONVERTER: org.apache.kafka.connect.json.JsonConverter
      CONNECT_INTERNAL_KEY_CONVERTER: org.apache.kafka.connect.storage.StringConverter
      CONNECT_INTERNAL_VALUE_CONVERTER: org.apache.kafka.connect.json.JsonConverter
      CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: "http://schema-registry:8081"
      CONNECT_REST_ADVERTISED_HOST_NAME: "kafka-connect"
      CONNECT_LOG4J_APPENDER_STDOUT_LAYOUT_CONVERSIONPATTERN: "[%d] %p %X{connector.context}%m (%c:%L)%n"
      CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: "1"
      CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: "1"
      CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: "1"
      CONNECT_PLUGIN_PATH: /usr/share/java,/usr/share/confluent-hub-components,/data/connect-jars,/usr/share/java/kafka-connect-jdbc,/usr/local/share/kafka/plugins
5

thanks for the awesome information.