Creating Kafka-connect with Azure-storage-sink in docker compose

Kafka Connect
1

So I start my zookeeper and kafka and they seem to work fine but the kafka connect is failing and I don’t know why:

version: '3'
services:
  zookeeper: 
    hostname: zookeeper
    image: confluentinc/cp-zookeeper:latest
    container_name: zookeeper
    environment:
      ZOOKEEPER_CLIENT_PORT: 2181
      ZOOKEEPER_TICK_TIME: 2000
    ports:
      - 2181:2181
    networks:
      - app-network

  kafka:
    hostname: kafka
    image: confluentinc/cp-kafka:latest
    container_name: kafka
    depends_on:
      - zookeeper
    ports:
      - 9092:9092
    environment:
      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
      KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka:29092,OUTSIDE://localhost:9092
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,OUTSIDE:PLAINTEXT
      KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL
      KAFKA_DELETE_TOPIC_ENABLE: "true"
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_DEFAULT_REPLICATION_FACTOR: 1
      KAFKA_AUTO_CREATE_TOPICS_ENABLE: false  # Disable auto topic creation

    networks:
      - app-network

 

  kafka-connect:
    hostname: kafka-connect
    image: confluentinc/cp-kafka-connect:latest
    container_name: kafka-connect
    environment:
      - CONNECT_BOOTSTRAP_SERVERS=kafka:29092
      - CONNECT_REST_PORT=8083
      - CONNECT_GROUP_ID=connect-cluster
      - CONNECT_CONFIG_STORAGE_TOPIC=connect-configs
      - CONNECT_OFFSET_STORAGE_TOPIC=connect-offsets
      - CONNECT_STATUS_STORAGE_TOPIC=connect-status
      - KAFKA_CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR=1
      - KAFKA_CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR=1
      - KAFKA_CONNECT_STATUS_STORAGE_REPLICATION_FACTOR=1
      - KAFKA_CONNECT_CREATE_TOPICS_ENABLE=false
      - CONNECT_KEY_CONVERTER=org.apache.kafka.connect.json.JsonConverter
      - CONNECT_VALUE_CONVERTER=org.apache.kafka.connect.json.JsonConverter
      - CONNECT_INTERNAL_KEY_CONVERTER=org.apache.kafka.connect.json.JsonConverter
      - CONNECT_INTERNAL_VALUE_CONVERTER=org.apache.kafka.connect.json.JsonConverter
      - CONNECT_LOG4J_ROOT_LOGLEVEL=INFO
      - CONNECT_PLUGIN_PATH=/usr/share/java,/etc/kafka-connect/jars
      - CONNECT_REST_ADVERTISED_HOST_NAME=kafka-connect  # Replace with your Kafka Connect host name
    ports:
      - "8083:8083"
    volumes:
      - ./jars:/etc/kafka-connect/jars
    depends_on:
      - kafka
    networks:
      - app-network

 

  azure-storage-sink:
    hostname: azure-storage-sink
    image: confluentinc/cp-kafka-connect:latest
    container_name: azure-storage-sink
    environment:
      - CONNECT_BOOTSTRAP_SERVERS=kafka:29092
      - CONNECT_REST_PORT=8083
      - CONNECT_GROUP_ID=connect-cluster
      - CONNECT_CONFIG_STORAGE_TOPIC=connect-configs
      - CONNECT_OFFSET_STORAGE_TOPIC=connect-offsets
      - CONNECT_STATUS_STORAGE_TOPIC=connect-status
      - CONNECT_PLUGIN_PATH=/usr/share/java,/etc/kafka-connect/jars
      - CONNECT_LOG4J_ROOT_LOGLEVEL=INFO
      - CONNECT_KEY_CONVERTER=org.apache.kafka.connect.json.JsonConverter
      - CONNECT_VALUE_CONVERTER=org.apache.kafka.connect.json.JsonConverter
      - CONNECT_INTERNAL_KEY_CONVERTER=org.apache.kafka.connect.json.JsonConverter
      - CONNECT_INTERNAL_VALUE_CONVERTER=org.apache.kafka.connect.json.JsonConverter
      - CONNECT_REST_ADVERTISED_HOST_NAME=kafka-connect  # Replace with your Kafka Connect host name
      - AZURE_STORAGE_CONNECTION_STRING=""
      - TOPICS=
      - TRANSFORMS=route
      - TRANSFORMS.route.type=org.apache.kafka.connect.transforms.RegexRouter
      - TRANSFORMS.route.regex=.*   # This regex matches all records
      - TRANSFORMS.route.replacement=destination-prefix/${topic}  # Replace with your desired prefix
    volumes:
      - ./jars:/etc/kafka-connect/jars
    depends_on:
      - kafka
      - kafka-connect
    networks:
      - app-network

 

networks:
  app-network:
    driver: bridge

And I get:
org.apache.kafka.connect.errors.ConnectException: Unable to initialize REST resources
2023-08-17 13:04:47 [2023-08-17 10:04:47,923] ERROR Failed to start Connect (org.apache.kafka.connect.cli.ConnectDistributed)
2023-08-17 13:04:47 org.apache.kafka.connect.errors.ConnectException: Unable to initialize REST resources
2023-08-17 13:04:47 at org.apache.kafka.connect.runtime.rest.RestServer.initializeResources(RestServer.java:297)
2023-08-17 13:04:47 at org.apache.kafka.connect.runtime.Connect.start(Connect.java:54)

2

Rather than mounting JARs, it’s recommended to create your own container image - Add Connectors or Software | Confluent Documentation

Regarding the error, you’ll need to provide more logs and clarify which of the two connect containers are failing

3

This topic was automatically closed 7 days after the last reply. New replies are no longer allowed.