Source Connector fails to start in ansible environment with license problem.
This was never happend in docker environment, but it seems a specific problem in Kafka Connect in ansible.I manage Kafka Connect by Ansible Playbook.
My environment is here.
$ python3 --version
Python 3.9.16
$ ansible --version
ansible [core 2.14.5]
$ cat /etc/os-release
NAME="Red Hat Enterprise Linux"
VERSION="9.2 (Plow)"
Error is here.
INFO [s3-source|worker] Kafka version: 7.4.0-ce (org.apache.kafka.common.utils.AppInfoParser:119)
INFO [s3-source|worker] Kafka commitId: 214b55fd0ba24de4 (org.apache.kafka.common.utils.AppInfoParser:120)
INFO [s3-source|worker] Kafka startTimeMs: 1684481123293 (org.apache.kafka.common.utils.AppInfoParser:121)
INFO [s3-source|worker] App info kafka.admin.client for s3-source-license-manager unregistered (org.apache.kafka.common.utils.AppInfoParser:83)
INFO [s3-source|worker] Metrics scheduler closed (org.apache.kafka.common.metrics.Metrics:710)
INFO [s3-source|worker] Closing reporter org.apache.kafka.common.metrics.JmxReporter (org.apache.kafka.common.metrics.Metrics:714)
INFO [s3-source|worker] Metrics reporters closed (org.apache.kafka.common.metrics.Metrics:720)
INFO [s3-source|worker] Closing License Store (io.confluent.license.LicenseStore:251)
INFO [s3-source|worker] Stopping KafkaBasedLog for topic _confluent-command (org.apache.kafka.connect.util.KafkaBasedLog:339)
INFO [s3-source|worker] [Producer clientId=s3-source-license-manager] Closing the Kafka producer with timeoutMillis = 9223372036854775807 ms. (org.apache.kafka.clients.producer.KafkaProducer:1310)
INFO [s3-source|worker] Metrics scheduler closed (org.apache.kafka.common.metrics.Metrics:710)
INFO [s3-source|worker] Closing reporter org.apache.kafka.common.metrics.JmxReporter (org.apache.kafka.common.metrics.Metrics:714)
INFO [s3-source|worker] Metrics reporters closed (org.apache.kafka.common.metrics.Metrics:720)
INFO [s3-source|worker] App info kafka.producer for s3-source-license-manager unregistered (org.apache.kafka.common.utils.AppInfoParser:83)
INFO [s3-source|worker] Metrics scheduler closed (org.apache.kafka.common.metrics.Metrics:710)
INFO [s3-source|worker] Closing reporter org.apache.kafka.common.metrics.JmxReporter (org.apache.kafka.common.metrics.Metrics:714)
INFO [s3-source|worker] Metrics reporters closed (org.apache.kafka.common.metrics.Metrics:720)
INFO [s3-source|worker] App info kafka.consumer for s3-source-license-manager unregistered (org.apache.kafka.common.utils.AppInfoParser:83)
INFO [s3-source|worker] Stopped KafkaBasedLog for topic _confluent-command (org.apache.kafka.connect.util.KafkaBasedLog:371)
INFO [s3-source|worker] Closed License Store (io.confluent.license.LicenseStore:253)
ERROR [s3-source|worker] WorkerConnector{id=s3-source} Error while starting connector (org.apache.kafka.connect.runtime.WorkerConnector:199)
java.lang.NoSuchMethodError: 'void org.apache.kafka.connect.util.KafkaBasedLog.send(java.lang.Object, java.lang.Object, org.apache.kafka.clients.producer.Callback)'
at io.confluent.license.LicenseStore.registerLicense(LicenseStore.java:283)
at io.confluent.license.LicenseStore.registerLicense(LicenseStore.java:276)
at io.confluent.license.LicenseManager.registerOrValidateLicense(LicenseManager.java:438)
at io.confluent.connect.utils.licensing.ConnectLicenseManager.registerOrValidateLicense(ConnectLicenseManager.java:257)
at io.confluent.connect.cloud.storage.source.GenericStorageSourceConnector.doStart(GenericStorageSourceConnector.java:132)
at io.confluent.connect.cloud.storage.source.GenericStorageSourceConnector.start(GenericStorageSourceConnector.java:124)
at io.confluent.connect.cloud.storage.source.CompositeSourceConnector.start(CompositeSourceConnector.java:72)
at org.apache.kafka.connect.runtime.WorkerConnector.doStart(WorkerConnector.java:191)
at org.apache.kafka.connect.runtime.WorkerConnector.start(WorkerConnector.java:216)
at org.apache.kafka.connect.runtime.WorkerConnector.doTransitionTo(WorkerConnector.java:361)
at org.apache.kafka.connect.runtime.WorkerConnector.doTransitionTo(WorkerConnector.java:344)
at org.apache.kafka.connect.runtime.WorkerConnector.doRun(WorkerConnector.java:144)
at org.apache.kafka.connect.runtime.WorkerConnector.run(WorkerConnector.java:122)
at org.apache.kafka.connect.runtime.isolation.Plugins.lambda$withClassLoader$1(Plugins.java:177)
at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
at java.base/java.lang.Thread.run(Thread.java:833)
ERROR [s3-source|worker] [Worker clientId=connect-1, groupId=connect-cluster] Failed to start connector 's3-source' (org.apache.kafka.connect.runtime.distributed.DistributedHerder:1850)
org.apache.kafka.connect.errors.ConnectException: Failed to start connector: s3-source
at org.apache.kafka.connect.runtime.distributed.DistributedHerder.lambda$startConnector$35(DistributedHerder.java:1821)
at org.apache.kafka.connect.runtime.WorkerConnector.doTransitionTo(WorkerConnector.java:347)
at org.apache.kafka.connect.runtime.WorkerConnector.doRun(WorkerConnector.java:144)
at org.apache.kafka.connect.runtime.WorkerConnector.run(WorkerConnector.java:122)
at org.apache.kafka.connect.runtime.isolation.Plugins.lambda$withClassLoader$1(Plugins.java:177)
at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
at java.base/java.lang.Thread.run(Thread.java:833)
Caused by: org.apache.kafka.connect.errors.ConnectException: Failed to transition connector s3-source to state STARTED
... 9 more
Caused by: java.lang.NoSuchMethodError: 'void org.apache.kafka.connect.util.KafkaBasedLog.send(java.lang.Object, java.lang.Object, org.apache.kafka.clients.producer.Callback)'
at io.confluent.license.LicenseStore.registerLicense(LicenseStore.java:283)
at io.confluent.license.LicenseStore.registerLicense(LicenseStore.java:276)
at io.confluent.license.LicenseManager.registerOrValidateLicense(LicenseManager.java:438)
at io.confluent.connect.utils.licensing.ConnectLicenseManager.registerOrValidateLicense(ConnectLicenseManager.java:257)
at io.confluent.connect.cloud.storage.source.GenericStorageSourceConnector.doStart(GenericStorageSourceConnector.java:132)
at io.confluent.connect.cloud.storage.source.GenericStorageSourceConnector.start(GenericStorageSourceConnector.java:124)
at io.confluent.connect.cloud.storage.source.CompositeSourceConnector.start(CompositeSourceConnector.java:72)
at org.apache.kafka.connect.runtime.WorkerConnector.doStart(WorkerConnector.java:191)
at org.apache.kafka.connect.runtime.WorkerConnector.start(WorkerConnector.java:216)
at org.apache.kafka.connect.runtime.WorkerConnector.doTransitionTo(WorkerConnector.java:361)
at org.apache.kafka.connect.runtime.WorkerConnector.doTransitionTo(WorkerConnector.java:344)
... 8 more