How to mount volumes for Kafka docker images

Ops Containers
#1

Hello,

I’m using confluent docker images (see below) and I’m facing some issues.

In fact, the size of the docker virtual machine (directory Overlay2) is growing fastly even if there’s no new data in the project.

I saw that in this case, it’s better to use volumes but I don’t know the directories in the images/containers I have to mount as volumes.

Confluent images

  • confluentic/cp-zookeeper
  • confluentic/cp-server
  • confluentic/cp-schema-registry
  • confluentic/cp-kafka-connect
  • confluentic/cp-enterprise-control-center
  • confluentic/cp-ksqldb-server
  • confluentic/cp-ksqldb-cli
  • confluentic/cp-kafka-rest
1 Like
#2

Hi @miyapi,

Could you please mention the docker image repo or share the compose file? It will help me understand whats going on.

Thanks,
Rankesh

#3 
---

version: '2.2'

 

services:

  

  zookeeper:

    image: confluentinc/cp-zookeeper

    oom_kill_disable: true

    mem_limit: 2g

    memswap_limit: 3g

    hostname: zookeeper

    container_name: zookeeper

    ports:

      - "2181:2181"

    environment:

      ZOOKEEPER_CLIENT_PORT: 2181

      ZOOKEEPER_TICK_TIME: 2000

    volumes:

       - /C/SMARTPOSTE/SMARTPOSTES_VOLUMES_DOCKER/zookeeper-data:/var/lib/zookeeper/data

       - /C/SMARTPOSTE/SMARTPOSTES_VOLUMES_DOCKER/zookeeper-logs:/var/lib/zookeeper/log

  broker:

    image: confluentinc/cp-server

    oom_kill_disable: true

    mem_limit: 12g

    memswap_limit: 16g

    cpus: 0.000

    hostname: broker

    container_name: broker

    depends_on:

      - zookeeper

    ports:

      - "9092:9092"

      - "9101:9101"

    environment:

      KAFKA_BROKER_ID: 1

      KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181'

      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT

      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://broker:29092,PLAINTEXT_HOST://localhost:9092

      KAFKA_METRIC_REPORTERS: io.confluent.metrics.reporter.ConfluentMetricsReporter

      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1

      KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0

      KAFKA_CONFLUENT_LICENSE_TOPIC_REPLICATION_FACTOR: 1

      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1

      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1

      KAFKA_JMX_PORT: 9101

      CONFLUENT_METRICS_REPORTER_BOOTSTRAP_SERVERS: broker:29092

      CONFLUENT_METRICS_REPORTER_ZOOKEEPER_CONNECT: zookeeper:2181

      CONFLUENT_METRICS_REPORTER_TOPIC_REPLICAS: 1

      CONFLUENT_METRICS_ENABLE: 'true'

      CONFLUENT_SUPPORT_CUSTOMER_ID: 'anonymous'

  schema-registry:

    image: confluentinc/cp-schema-registry

    oom_kill_disable: true

    mem_limit: 8g

    memswap_limit: 12g

    cpus: 0.000

    hostname: schema-registry

    container_name: schema-registry

    depends_on:

      - zookeeper

      - broker

    ports:

      - "8081:8081"

    environment:

      SCHEMA_REGISTRY_HOST_NAME: schema-registry

      SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: 'zookeeper:2181'

  connect:

    image: confluentinc/cp-kafka-connect

    oom_kill_disable: true

    mem_limit: 16g

    memswap_limit: 20g

    cpus: 0.000

    hostname: connect

    container_name: connect

    depends_on:

      - zookeeper

      - broker

      - schema-registry

    ports:

      - "8083:8083"

    environment:

      CONNECT_BOOTSTRAP_SERVERS: 'broker:29092'

      CONNECT_REST_ADVERTISED_HOST_NAME: connect

      CONNECT_REST_PORT: 8083

      CONNECT_GROUP_ID: compose-connect-group

      CONNECT_CONFIG_STORAGE_TOPIC: docker-connect-configs

      CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: 1

      CONNECT_OFFSET_FLUSH_INTERVAL_MS: 10000

      CONNECT_OFFSET_STORAGE_TOPIC: docker-connect-offsets

      CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: 1

      CONNECT_STATUS_STORAGE_TOPIC: docker-connect-status

      CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: 1

      CONNECT_KEY_CONVERTER: org.apache.kafka.connect.storage.StringConverter

      CONNECT_VALUE_CONVERTER: io.confluent.connect.avro.AvroConverter

      CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: http://schema-registry:8081

      CONNECT_INTERNAL_KEY_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"

      CONNECT_INTERNAL_VALUE_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"

      CONNECT_ZOOKEEPER_CONNECT: 'zookeeper:2181'

      # CLASSPATH required due to CC-2422

      CLASSPATH: /usr/share/java/monitoring-interceptors/monitoring-interceptors-5.5.0.jar

      CONNECT_PRODUCER_INTERCEPTOR_CLASSES: "io.confluent.monitoring.clients.interceptor.MonitoringProducerInterceptor"

      CONNECT_CONSUMER_INTERCEPTOR_CLASSES: "io.confluent.monitoring.clients.interceptor.MonitoringConsumerInterceptor"

      CONNECT_PLUGIN_PATH: "/usr/share/java,/usr/share/confluent-hub-components"

      CONNECT_LOG4J_LOGGERS: org.apache.zookeeper=ERROR,org.I0Itec.zkclient=ERROR,org.reflections=ERROR

  control-center:

    image: confluentinc/cp-enterprise-control-center

    hostname: control-center

    container_name: control-center

    depends_on:

      - zookeeper

      - broker

      - schema-registry

      - connect

      - ksqldb-server

    ports:

      - "9021:9021"

    environment:

      CONTROL_CENTER_BOOTSTRAP_SERVERS: 'broker:29092'

      CONTROL_CENTER_ZOOKEEPER_CONNECT: 'zookeeper:2181'

      CONTROL_CENTER_CONNECT_CLUSTER: 'connect:8083'

      CONTROL_CENTER_KSQL_KSQLDB1_URL: "http://ksqldb-server:8088"

      CONTROL_CENTER_KSQL_KSQLDB1_ADVERTISED_URL: "http://localhost:8088"

      CONTROL_CENTER_SCHEMA_REGISTRY_URL: "http://schema-registry:8081"

      CONTROL_CENTER_REPLICATION_FACTOR: 1

      CONTROL_CENTER_INTERNAL_TOPICS_PARTITIONS: 1

      CONTROL_CENTER_MONITORING_INTERCEPTOR_TOPIC_PARTITIONS: 1

      CONFLUENT_METRICS_TOPIC_REPLICATION: 1

      PORT: 9021

  ksqldb-server:

    oom_kill_disable: true

    mem_limit: 12g

    memswap_limit: 16g

    cpus: 0.000

    image: confluentinc/cp-ksqldb-server

    hostname: ksqldb-server

    container_name: ksqldb-server

    depends_on:

      - broker

      - connect

    ports:

      - "8088:8088"

    environment:

      KSQL_KSQL_EXTENSION_DIR: "/etc/ksqldb/ext/"

      KSQL_CONFIG_DIR: "/etc/ksqldb"

      KSQL_BOOTSTRAP_SERVERS: "broker:29092"

      KSQL_HOST_NAME: ksqldb-server

      KSQL_LISTENERS: "http://0.0.0.0:8088"

      KSQL_CACHE_MAX_BYTES_BUFFERING: 0

      KSQL_KSQL_SCHEMA_REGISTRY_URL: "http://schema-registry:8081"

      KSQL_PRODUCER_INTERCEPTOR_CLASSES: "io.confluent.monitoring.clients.interceptor.MonitoringProducerInterceptor"

      KSQL_CONSUMER_INTERCEPTOR_CLASSES: "io.confluent.monitoring.clients.interceptor.MonitoringConsumerInterceptor"

      KSQL_KSQL_CONNECT_URL: "http://connect:8083"

    volumes:

       - /C/SMARTPOSTE/SMARTPOSTES_VOLUMES_DOCKER/ksqldb-server:/etc/ksqldb

  ksqldb-cli:

    image: confluentinc/cp-ksqldb-cli

    oom_kill_disable: true

    mem_limit: 2g

    memswap_limit: 3g

    cpus: 0.000

    container_name: ksqldb-cli

    depends_on:

      - broker

      - connect

      - ksqldb-server

    entrypoint: /bin/sh

    tty: true

    volumes:

       - /C/SMARTPOSTE/SMARTPOSTES_VOLUMES_DOCKER/ksql-cli-scripts:/data/scripts/

  #elasticsearch:

  #  oom_kill_disable: true

  #  mem_limit: 12g

  #  memswap_limit: 16g

  #  cpus: 0.000

  #  image: docker.elastic.co/elasticsearch/elasticsearch:7.6.2

  #  container_name: elasticsearch

  #  ports:

  #    - 9200:9200

  #  environment:

  #    ES_JAVA_OPTS: "-Xms1g -Xmx1g"

  #    discovery.type: "single-node"

  #    node.store.allow_mmap: "false"

  mongodb:

    image: mongo

    oom_kill_disable: true

    mem_limit: 12g

    memswap_limit: 16g

    cpus: 0.000

    ports:

      - "27017:27017"

    volumes:

       - /C/SMARTPOSTE/SMARTPOSTES_VOLUMES_DOCKER/mongo-data:/data/db

  rest-proxy:

    image: confluentinc/cp-kafka-rest

    oom_kill_disable: true

    mem_limit: 2g

    memswap_limit: 3g

    cpus: 0.000

    depends_on:

      - zookeeper

      - broker

      - schema-registry

    ports:

      - 8082:8082

    hostname: rest-proxy

    container_name: rest-proxy

    environment:

      KAFKA_REST_HOST_NAME: rest-proxy

      KAFKA_REST_BOOTSTRAP_SERVERS: 'broker:29092'

      KAFKA_REST_LISTENERS: "http://0.0.0.0:8082"

      KAFKA_REST_SCHEMA_REGISTRY_URL: 'http://schema-registry:8081'

  nifi:

    image: apache/nifi

    oom_kill_disable: true

    mem_limit: 16g

    memswap_limit: 20g

    cpus: 0.000

    ports:

      - 8080:8080

    volumes:

       - /C/SMARTPOSTE/SMARTPOSTES_VOLUMES_DOCKER/nifi-conf:/opt/conf

       - /C/SMARTPOSTE/SMARTPOSTES_VOLUMES_DOCKER/nifi-data:/data/nifi/state
#4

There’s some useful reference in the docs, as well as an explanation in this answer that I posted on StackOverflow.

2 Likes
#5

This topic was automatically closed after 30 days. New replies are no longer allowed.