Hello,
I’m using confluent docker images (see below) and I’m facing some issues.
In fact, the size of the docker virtual machine (directory Overlay2) is growing fastly even if there’s no new data in the project.
I saw that in this case, it’s better to use volumes but I don’t know the directories in the images/containers I have to mount as volumes.
Confluent images
- confluentic/cp-zookeeper
- confluentic/cp-server
- confluentic/cp-schema-registry
- confluentic/cp-kafka-connect
- confluentic/cp-enterprise-control-center
- confluentic/cp-ksqldb-server
- confluentic/cp-ksqldb-cli
- confluentic/cp-kafka-rest
1 Like
Hi @miyapi,
Could you please mention the docker image repo or share the compose file? It will help me understand whats going on.
Thanks,
Rankesh
---
version: '2.2'
services:
zookeeper:
image: confluentinc/cp-zookeeper
oom_kill_disable: true
mem_limit: 2g
memswap_limit: 3g
hostname: zookeeper
container_name: zookeeper
ports:
- "2181:2181"
environment:
ZOOKEEPER_CLIENT_PORT: 2181
ZOOKEEPER_TICK_TIME: 2000
volumes:
- /C/SMARTPOSTE/SMARTPOSTES_VOLUMES_DOCKER/zookeeper-data:/var/lib/zookeeper/data
- /C/SMARTPOSTE/SMARTPOSTES_VOLUMES_DOCKER/zookeeper-logs:/var/lib/zookeeper/log
broker:
image: confluentinc/cp-server
oom_kill_disable: true
mem_limit: 12g
memswap_limit: 16g
cpus: 0.000
hostname: broker
container_name: broker
depends_on:
- zookeeper
ports:
- "9092:9092"
- "9101:9101"
environment:
KAFKA_BROKER_ID: 1
KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181'
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://broker:29092,PLAINTEXT_HOST://localhost:9092
KAFKA_METRIC_REPORTERS: io.confluent.metrics.reporter.ConfluentMetricsReporter
KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
KAFKA_CONFLUENT_LICENSE_TOPIC_REPLICATION_FACTOR: 1
KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
KAFKA_JMX_PORT: 9101
CONFLUENT_METRICS_REPORTER_BOOTSTRAP_SERVERS: broker:29092
CONFLUENT_METRICS_REPORTER_ZOOKEEPER_CONNECT: zookeeper:2181
CONFLUENT_METRICS_REPORTER_TOPIC_REPLICAS: 1
CONFLUENT_METRICS_ENABLE: 'true'
CONFLUENT_SUPPORT_CUSTOMER_ID: 'anonymous'
schema-registry:
image: confluentinc/cp-schema-registry
oom_kill_disable: true
mem_limit: 8g
memswap_limit: 12g
cpus: 0.000
hostname: schema-registry
container_name: schema-registry
depends_on:
- zookeeper
- broker
ports:
- "8081:8081"
environment:
SCHEMA_REGISTRY_HOST_NAME: schema-registry
SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: 'zookeeper:2181'
connect:
image: confluentinc/cp-kafka-connect
oom_kill_disable: true
mem_limit: 16g
memswap_limit: 20g
cpus: 0.000
hostname: connect
container_name: connect
depends_on:
- zookeeper
- broker
- schema-registry
ports:
- "8083:8083"
environment:
CONNECT_BOOTSTRAP_SERVERS: 'broker:29092'
CONNECT_REST_ADVERTISED_HOST_NAME: connect
CONNECT_REST_PORT: 8083
CONNECT_GROUP_ID: compose-connect-group
CONNECT_CONFIG_STORAGE_TOPIC: docker-connect-configs
CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: 1
CONNECT_OFFSET_FLUSH_INTERVAL_MS: 10000
CONNECT_OFFSET_STORAGE_TOPIC: docker-connect-offsets
CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: 1
CONNECT_STATUS_STORAGE_TOPIC: docker-connect-status
CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: 1
CONNECT_KEY_CONVERTER: org.apache.kafka.connect.storage.StringConverter
CONNECT_VALUE_CONVERTER: io.confluent.connect.avro.AvroConverter
CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: http://schema-registry:8081
CONNECT_INTERNAL_KEY_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
CONNECT_INTERNAL_VALUE_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
CONNECT_ZOOKEEPER_CONNECT: 'zookeeper:2181'
# CLASSPATH required due to CC-2422
CLASSPATH: /usr/share/java/monitoring-interceptors/monitoring-interceptors-5.5.0.jar
CONNECT_PRODUCER_INTERCEPTOR_CLASSES: "io.confluent.monitoring.clients.interceptor.MonitoringProducerInterceptor"
CONNECT_CONSUMER_INTERCEPTOR_CLASSES: "io.confluent.monitoring.clients.interceptor.MonitoringConsumerInterceptor"
CONNECT_PLUGIN_PATH: "/usr/share/java,/usr/share/confluent-hub-components"
CONNECT_LOG4J_LOGGERS: org.apache.zookeeper=ERROR,org.I0Itec.zkclient=ERROR,org.reflections=ERROR
control-center:
image: confluentinc/cp-enterprise-control-center
hostname: control-center
container_name: control-center
depends_on:
- zookeeper
- broker
- schema-registry
- connect
- ksqldb-server
ports:
- "9021:9021"
environment:
CONTROL_CENTER_BOOTSTRAP_SERVERS: 'broker:29092'
CONTROL_CENTER_ZOOKEEPER_CONNECT: 'zookeeper:2181'
CONTROL_CENTER_CONNECT_CLUSTER: 'connect:8083'
CONTROL_CENTER_KSQL_KSQLDB1_URL: "http://ksqldb-server:8088"
CONTROL_CENTER_KSQL_KSQLDB1_ADVERTISED_URL: "http://localhost:8088"
CONTROL_CENTER_SCHEMA_REGISTRY_URL: "http://schema-registry:8081"
CONTROL_CENTER_REPLICATION_FACTOR: 1
CONTROL_CENTER_INTERNAL_TOPICS_PARTITIONS: 1
CONTROL_CENTER_MONITORING_INTERCEPTOR_TOPIC_PARTITIONS: 1
CONFLUENT_METRICS_TOPIC_REPLICATION: 1
PORT: 9021
ksqldb-server:
oom_kill_disable: true
mem_limit: 12g
memswap_limit: 16g
cpus: 0.000
image: confluentinc/cp-ksqldb-server
hostname: ksqldb-server
container_name: ksqldb-server
depends_on:
- broker
- connect
ports:
- "8088:8088"
environment:
KSQL_KSQL_EXTENSION_DIR: "/etc/ksqldb/ext/"
KSQL_CONFIG_DIR: "/etc/ksqldb"
KSQL_BOOTSTRAP_SERVERS: "broker:29092"
KSQL_HOST_NAME: ksqldb-server
KSQL_LISTENERS: "http://0.0.0.0:8088"
KSQL_CACHE_MAX_BYTES_BUFFERING: 0
KSQL_KSQL_SCHEMA_REGISTRY_URL: "http://schema-registry:8081"
KSQL_PRODUCER_INTERCEPTOR_CLASSES: "io.confluent.monitoring.clients.interceptor.MonitoringProducerInterceptor"
KSQL_CONSUMER_INTERCEPTOR_CLASSES: "io.confluent.monitoring.clients.interceptor.MonitoringConsumerInterceptor"
KSQL_KSQL_CONNECT_URL: "http://connect:8083"
volumes:
- /C/SMARTPOSTE/SMARTPOSTES_VOLUMES_DOCKER/ksqldb-server:/etc/ksqldb
ksqldb-cli:
image: confluentinc/cp-ksqldb-cli
oom_kill_disable: true
mem_limit: 2g
memswap_limit: 3g
cpus: 0.000
container_name: ksqldb-cli
depends_on:
- broker
- connect
- ksqldb-server
entrypoint: /bin/sh
tty: true
volumes:
- /C/SMARTPOSTE/SMARTPOSTES_VOLUMES_DOCKER/ksql-cli-scripts:/data/scripts/
#elasticsearch:
# oom_kill_disable: true
# mem_limit: 12g
# memswap_limit: 16g
# cpus: 0.000
# image: docker.elastic.co/elasticsearch/elasticsearch:7.6.2
# container_name: elasticsearch
# ports:
# - 9200:9200
# environment:
# ES_JAVA_OPTS: "-Xms1g -Xmx1g"
# discovery.type: "single-node"
# node.store.allow_mmap: "false"
mongodb:
image: mongo
oom_kill_disable: true
mem_limit: 12g
memswap_limit: 16g
cpus: 0.000
ports:
- "27017:27017"
volumes:
- /C/SMARTPOSTE/SMARTPOSTES_VOLUMES_DOCKER/mongo-data:/data/db
rest-proxy:
image: confluentinc/cp-kafka-rest
oom_kill_disable: true
mem_limit: 2g
memswap_limit: 3g
cpus: 0.000
depends_on:
- zookeeper
- broker
- schema-registry
ports:
- 8082:8082
hostname: rest-proxy
container_name: rest-proxy
environment:
KAFKA_REST_HOST_NAME: rest-proxy
KAFKA_REST_BOOTSTRAP_SERVERS: 'broker:29092'
KAFKA_REST_LISTENERS: "http://0.0.0.0:8082"
KAFKA_REST_SCHEMA_REGISTRY_URL: 'http://schema-registry:8081'
nifi:
image: apache/nifi
oom_kill_disable: true
mem_limit: 16g
memswap_limit: 20g
cpus: 0.000
ports:
- 8080:8080
volumes:
- /C/SMARTPOSTE/SMARTPOSTES_VOLUMES_DOCKER/nifi-conf:/opt/conf
- /C/SMARTPOSTE/SMARTPOSTES_VOLUMES_DOCKER/nifi-data:/data/nifi/state
rmoff
4
There’s some useful reference in the docs, as well as an explanation in this answer that I posted on StackOverflow.
2 Likes
system
Closed
5
This topic was automatically closed after 30 days. New replies are no longer allowed.