Hi, im trying to setup MDS with Kraft and 3 nodes and faced error.
there is config:
KAFKA_PROCESS_ROLES: 'controller,broker'
KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER'
KAFKA_LISTENERS: 'CONTROLLER://:9090,BROKER://:9091,PLAINTEXT://:9092,TOKEN://:9093,SSL://:9094'
KAFKA_CONTROLLER_QUORUM_VOTERS: '0@mds-0.mds.confluent.svc:9090, 1@mds-1.mds.confluent.svc:9090,2@mds-2.mds.confluent.svc:9090'
CLUSTER_ID: 'MkU3OEVBNTcwNTJENDM2Qk'
KAFKA_CONFLUENT_METADATA_SERVER_ADVERTISED_LISTENERS: 'https://${POD_NAME}.mds.confluent.svc:8090'
KAFKA_CONFLUENT_METADATA_SERVER_LISTENERS: 'https://${POD_NAME}.mds.confluent.svc:8090'
KAFKA_KAFKA_REST_BOOTSTRAP_SERVERS: 'mds.confluent.svc:9091'
KAFKA_KAFKA_REST_ADVERTISED_LISTENERS: 'https://${POD_NAME}.mds.confluent.svc:8090'
KAFKA_KAFKA_REST_CONFLUENT_METADATA_BOOTSTRAP_SERVER_URLS: 'https://mds.confluent.svc:8090'
KAFKA_CONFLUENT_HTTP_SERVER_LISTENERS: 'https://${POD_NAME}.mds.confluent.svc:8090'
CONFLUENT_METRICS_ENABLE: 'true'
CONFLUENT_METRICS_REPORTER_PUBLISH_MS: '30000'
CONFLUENT_METRICS_REPORTER_TOPIC_CREATES: 'true'
CONFLUENT_METRICS_REPORTER_TOPIC_REPLICAS: '1'
KAFKA_ALLOW_EVERYONE_IF_NO_ACL_FOUND: 'true'
KAFKA_AUTHORIZER_CLASS_NAME: io.confluent.kafka.security.authorizer.ConfluentServerAuthorizer
KAFKA_CONFLUENT_AUTHORIZER_ACCESS_RULE_PROVIDERS: CONFLUENT,KRAFT_ACL
KAFKA_CONFLUENT_BALANCER_TOPIC_REPLICATION_FACTOR: '1'
KAFKA_CONFLUENT_CLUSTER_LINK_METADATA_TOPIC_MIN_ISR: '1'
KAFKA_CONFLUENT_CLUSTER_LINK_METADATA_TOPIC_REPLICATION_FACTOR: '1'
KAFKA_CONFLUENT_DURABILITY_TOPIC_REPLICATION_FACTOR: '1'
KAFKA_CONFLUENT_HTTP_SERVER_SSL_KEYSTORE_LOCATION: /etc/kafka/secrets/keystore.jks
KAFKA_CONFLUENT_HTTP_SERVER_SSL_TRUSTSTORE_LOCATION: /etc/kafka/secrets/truststore.jks
KAFKA_CONFLUENT_INTERNAL_METRICS_ENABLE: 'true'
KAFKA_CONFLUENT_LICENSE_TOPIC_REPLICATION_FACTOR: '1'
KAFKA_CONFLUENT_METADATA_HTTP_AUTH_CREDENTIALS_PROVIDER: BASIC
KAFKA_CONFLUENT_METADATA_SASL_JAAS_CONFIG: >-
org.apache.kafka.common.security.plain.PlainLoginModule required
username="admin" password="admin-secret";
KAFKA_CONFLUENT_METADATA_SASL_MECHANISM: PLAIN
KAFKA_CONFLUENT_METADATA_SECURITY_PROTOCOL: SSL
KAFKA_CONFLUENT_METADATA_SERVER_AUTHENTICATION_METHOD: BEARER
KAFKA_CONFLUENT_METADATA_SERVER_ENABLE: 'true'
KAFKA_CONFLUENT_METADATA_SERVER_KRAFT_CONTROLLER_ENABLED: 'true'
KAFKA_CONFLUENT_METADATA_SERVER_OAUTHBEARER_EXPECTED_AUDIENCE: Confluent,api://default,https://sso.example.com
KAFKA_CONFLUENT_METADATA_SERVER_OAUTHBEARER_EXPECTED_ISSUER: Confluent
KAFKA_CONFLUENT_METADATA_SERVER_OAUTHBEARER_GROUPS_CLAIM_NAME: groups
KAFKA_CONFLUENT_METADATA_SERVER_OAUTHBEARER_JWKS_ENDPOINT_URL: https://sso.example.com/oauth2/keys
KAFKA_CONFLUENT_METADATA_SERVER_OAUTHBEARER_SUB_CLAIM_NAME: sub
KAFKA_CONFLUENT_METADATA_SERVER_OPENAPI_ENABLE: 'true'
KAFKA_CONFLUENT_METADATA_SERVER_SSL_KEYSTORE_LOCATION: /etc/kafka/secrets/keystore.jks
KAFKA_CONFLUENT_METADATA_SERVER_SSL_TRUSTSTORE_LOCATION: /etc/kafka/secrets/truststore.jks
KAFKA_CONFLUENT_METADATA_SERVER_TOKEN_KEY_PATH: /etc/kafka/secrets/token.key
KAFKA_CONFLUENT_METADATA_SERVER_TOKEN_MAX_LIFETIME_MS: '3600000'
KAFKA_CONFLUENT_METADATA_SERVER_TOKEN_SIGNATURE_ALGORITHM: RS256
KAFKA_CONFLUENT_METADATA_SERVER_USER_STORE: OAUTH
KAFKA_CONFLUENT_OAUTH_GROUPS_CLAIM_NAME: groups
KAFKA_CONFLUENT_SSL_KEYSTORE_LOCATION: /etc/kafka/secrets/keystore.jks
KAFKA_CONFLUENT_SSL_PROTOCOL: SSL
KAFKA_CONFLUENT_SSL_TRUSTSTORE_LOCATION: /etc/kafka/secrets/truststore.jks
KAFKA_CONFLUENT_TIER_METADATA_REPLICATION_FACTOR: '1'
KAFKA_DEFAULT_REPLICATION_FACTOR: '1'
KAFKA_EARLY_START_LISTENERS: CONTROLLER,PLAINTEXT,BROKER
KAFKA_INTER_BROKER_LISTENER_NAME: BROKER
KAFKA_KAFKA_REST_CLIENT_CONFLUENT_METADATA_SERVER_URLS_MAX_AGE_MS: '60000'
KAFKA_KAFKA_REST_CLIENT_SECURITY_PROTOCOL: SSL
KAFKA_KAFKA_REST_CLIENT_SSL_KEYSTORE_LOCATION: /etc/kafka/secrets/keystore.jks
KAFKA_KAFKA_REST_CLIENT_SSL_TRUSTSTORE_LOCATION: /etc/kafka/secrets/truststore.jks
KAFKA_KAFKA_REST_CONFLUENT_METADATA_BASIC_AUTH_USER_INFO: restAdmin:restAdmin
KAFKA_KAFKA_REST_CONFLUENT_METADATA_HTTP_AUTH_CREDENTIALS_PROVIDER: BASIC
KAFKA_KAFKA_REST_CONFLUENT_METADATA_SERVER_URLS_MAX_AGE_MS: '60000'
KAFKA_KAFKA_REST_KAFKA_REST_RESOURCE_EXTENSION_CLASS: io.confluent.kafkarest.security.KafkaRestSecurityResourceExtension
KAFKA_KAFKA_REST_PUBLIC_KEY_PATH: /etc/kafka/secrets/token.pub
KAFKA_KAFKA_REST_SSL_TRUSTSTORE_LOCATION: /etc/kafka/secrets/truststore.jks
KAFKA_LISTENER_NAME_BROKER_SASL_ENABLED_MECHANISMS: PLAIN
KAFKA_LISTENER_NAME_CONTROLLER_PLAIN_SASL_JAAS_CONFIG: 'org.apache.kafka.common.security.plain.PlainLoginModule required
username="admin" password="admin-secret" user_admin="admin-secret"
user_mds="mds-secret";'
KAFKA_LISTENER_NAME_CONTROLLER_SASL_ENABLED_MECHANISMS: PLAIN
KAFKA_LISTENER_NAME_SSL_SSL_PRINCIPAL_MAPPING_RULES: RULE:^CN=([a-zA-Z0-9.]*).*$$/$$1/ , DEFAULT
KAFKA_LISTENER_NAME_TOKEN_OAUTHBEARER_EXPECTED_AUDIENCE: api://default
KAFKA_LISTENER_NAME_TOKEN_OAUTHBEARER_JWKS_ENDPOINT_URL: https://sso.example.com/oauth2/keys
KAFKA_LISTENER_NAME_TOKEN_OAUTHBEARER_SASL_JAAS_CONFIG: 'org.apache.kafka.common.security.oauthbearer.OAuthBearerLoginModule required
publicKeyPath="/etc/kafka/secrets/token.pub";'
KAFKA_LISTENER_NAME_TOKEN_OAUTHBEARER_SASL_LOGIN_CALLBACK_HANDLER_CLASS: 'io.confluent.kafka.server.plugins.auth.token.TokenBearerServerLoginCallbackHandler'
KAFKA_LISTENER_NAME_TOKEN_OAUTHBEARER_SASL_SERVER_CALLBACK_HANDLER_CLASS: 'io.confluent.kafka.server.plugins.auth.token.TokenBearerValidatorCallbackHandler'
KAFKA_LISTENER_NAME_TOKEN_PRINCIPAL_BULDER_CLASS: 'io.confluent.kafka.security.authenticator.OAuthKafkaPrincipalBuilder'
KAFKA_LISTENER_NAME_TOKEN_SASL_ENABLED_MECHANISMS: 'OAUTHBEARER'
KAFKA_LISTENER_NAME_TOKEN_SSL_PRINCIPAL_MAPPING_RULES: 'RULE:^CN=([a-zA-Z0-9.]*).*$$/$$1/ , DEFAULT'
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,SSL:SSL,BROKER:SSL,TOKEN:SASL_SSL'
KAFKA_LOG4J_LOGGERS: kafka.authorizer.logger=DEBUG
KAFKA_LOG4J_ROOT_LOGLEVEL: DEBUG
KAFKA_METRIC_REPORTERS: 'io.confluent.metrics.reporter.ConfluentMetricsReporter'
KAFKA_MIN_INSYNC_REPLICAS: '1'
KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: '1'
KAFKA_QUOTAS_TOPIC_REPLICATION_FACTOR: '1'
KAFKA_SASL_ENABLED_MECHANISMS: PLAIN
KAFKA_SASL_MECHANISM: PLAIN, OAUTHBEARER
KAFKA_SASL_MECHANISM_CONTROLLER_PROTOCOL: PLAIN
KAFKA_SASL_MECHANISM_INTER_BROKER_PROTOCOL: PLAIN
KAFKA_SECURITY_PROTOCOL: SSL
KAFKA_SSL_CLIENT_AUTH: requested
KAFKA_SSL_KEYSTORE_CREDENTIALS: keystore.creds
KAFKA_SSL_KEYSTORE_FILENAME: keystore.jks
KAFKA_SSL_KEYSTORE_LOCATION: /etc/kafka/secrets/keystore.jks
KAFKA_SSL_KEY_CREDENTIALS: key.creds
KAFKA_SSL_TRUSTSTORE_CREDENTIALS: truststore.creds
KAFKA_SSL_TRUSTSTORE_FILENAME: truststore.jks
KAFKA_SSL_TRUSTSTORE_LOCATION: /etc/kafka/secrets/truststore.jks
KAFKA_SUPER_USERS: User:admin;User:mds;User:superUser;User:ANONYMOUS
KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: '1'
KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: '1'
Logs:
[2024-07-08 22:36:52,944] ERROR [StandardAuthorizer 0] Failed to complete initial ACL load process. (org.apache.kafka.metadata.authorizer.StandardAuthorizerData)
java.util.concurrent.TimeoutException
at kafka.server.metadata.AclPublisher.close(AclPublisher.scala:93)
at org.apache.kafka.image.loader.MetadataLoader.closePublisher(MetadataLoader.java:612)
at org.apache.kafka.image.loader.MetadataLoader.lambda$removeAndClosePublisher$7(MetadataLoader.java:572)
at org.apache.kafka.queue.KafkaEventQueue$EventContext.run(KafkaEventQueue.java:127)
at org.apache.kafka.queue.KafkaEventQueue$EventHandler.handleEvents(KafkaEventQueue.java:210)
at org.apache.kafka.queue.KafkaEventQueue$EventHandler.run(KafkaEventQueue.java:181)
at java.base/java.lang.Thread.run(Thread.java:829)
at org.apache.kafka.common.utils.KafkaThread.run(KafkaThread.java:66)