Hi, we sometimes get a Socket Exception when trying to get a schema from the registry while we are producing a kafka message. We are using the KafkaProtobufSerializer (version 7.5.1) to serialize the message in a Spring Boot service and the schema registry has been deployed using the bitnami helm chart on a k8s cluster. The issue is intermittent and sometimes fixes by itself, but there are no network issues in communicating with any other service. Could there be an issue with the request itself or maybe with how the server is setup?
Error log
org.apache.kafka.common.errors.SerializationException: Error serializing Protobuf message
at io.confluent.kafka.serializers.protobuf.KafkaProtobufSerializer.serialize(KafkaProtobufSerializer.java:103) ~[kafka-protobuf-serializer-7.5.1.jar!/:?]
at io.confluent.kafka.serializers.protobuf.KafkaProtobufSerializer.serialize(KafkaProtobufSerializer.java:37) ~[kafka-protobuf-serializer-7.5.1.jar!/:?]
at org.apache.kafka.clients.producer.KafkaProducer.doSend(KafkaProducer.java:1015) ~[kafka-clients-3.4.1.jar!/:?]
at org.apache.kafka.clients.producer.KafkaProducer.send(KafkaProducer.java:962) ~[kafka-clients-3.4.1.jar!/:?]
at org.springframework.kafka.core.DefaultKafkaProducerFactory$CloseSafeProducer.send(DefaultKafkaProducerFactory.java:1062) ~[spring-kafka-3.0.12.jar!/:3.0.12]
at org.springframework.kafka.core.KafkaTemplate.doSend(KafkaTemplate.java:786) ~[spring-kafka-3.0.12.jar!/:3.0.12]
at org.springframework.kafka.core.KafkaTemplate.observeSend(KafkaTemplate.java:754) ~[spring-kafka-3.0.12.jar!/:3.0.12]
at org.springframework.kafka.core.KafkaTemplate.send(KafkaTemplate.java:544) ~[spring-kafka-3.0.12.jar!/:3.0.12]
at com.sadapay.invoicing.service.producers.PaymentEventDispatcher.dispatchPaymentUnderReview(PaymentEventDispatcher.kt:101) ~[classes!/:1.158.0]
at com.sadapay.invoicing.service.service.RiskDecisionService.handleTransactionHeldDecision(RiskDecisionService.kt:48) ~[classes!/:1.158.0]
at com.sadapay.invoicing.service.consumers.RiskTransactionDecisionEventHandler.consumeRiskTransactionDecisionEvent(RiskTransactionDecisionEventHandler.kt:35) ~[classes!/:1.158.0]
at jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:103) ~[?:?]
at java.lang.reflect.Method.invoke(Method.java:580) ~[?:?]
at org.springframework.messaging.handler.invocation.InvocableHandlerMethod.doInvoke(InvocableHandlerMethod.java:169) ~[spring-messaging-6.0.13.jar!/:6.0.13]
at org.springframework.messaging.handler.invocation.InvocableHandlerMethod.invoke(InvocableHandlerMethod.java:119) ~[spring-messaging-6.0.13.jar!/:6.0.13]
at org.springframework.kafka.listener.adapter.HandlerAdapter.invoke(HandlerAdapter.java:56) ~[spring-kafka-3.0.12.jar!/:3.0.12]
at org.springframework.kafka.listener.adapter.MessagingMessageListenerAdapter.invokeHandler(MessagingMessageListenerAdapter.java:375) ~[spring-kafka-3.0.12.jar!/:3.0.12]
at org.springframework.kafka.listener.adapter.RecordMessagingMessageListenerAdapter.onMessage(RecordMessagingMessageListenerAdapter.java:92) ~[spring-kafka-3.0.12.jar!/:3.0.12]
at org.springframework.kafka.listener.adapter.RecordMessagingMessageListenerAdapter.onMessage(RecordMessagingMessageListenerAdapter.java:53) ~[spring-kafka-3.0.12.jar!/:3.0.12]
at org.springframework.kafka.listener.KafkaMessageListenerContainer$ListenerConsumer.doInvokeOnMessage(KafkaMessageListenerContainer.java:2873) ~[spring-kafka-3.0.12.jar!/:3.0.12]
at org.springframework.kafka.listener.KafkaMessageListenerContainer$ListenerConsumer.invokeOnMessage(KafkaMessageListenerContainer.java:2854) ~[spring-kafka-3.0.12.jar!/:3.0.12]
at org.springframework.kafka.listener.KafkaMessageListenerContainer$ListenerConsumer.lambda$doInvokeRecordListener$57(KafkaMessageListenerContainer.java:2772) ~[spring-kafka-3.0.12.jar!/:3.0.12]
... 12 more
Caused by: java.net.SocketException: Unexpected end of file from server
at sun.net.www.http.HttpClient.parseHTTPHeader(HttpClient.java:955) ~[?:?]
at sun.net.www.http.HttpClient.parseHTTP(HttpClient.java:759) ~[?:?]
at sun.net.www.http.HttpClient.parseHTTPHeader(HttpClient.java:952) ~[?:?]
at sun.net.www.http.HttpClient.parseHTTP(HttpClient.java:759) ~[?:?]
at sun.net.www.protocol.http.HttpURLConnection.getInputStream0(HttpURLConnection.java:1690) ~[?:?]
at sun.net.www.protocol.http.HttpURLConnection.getInputStream(HttpURLConnection.java:1599) ~[?:?]
at java.net.HttpURLConnection.getResponseCode(HttpURLConnection.java:531) ~[?:?]
at sun.net.www.protocol.https.HttpsURLConnectionImpl.getResponseCode(HttpsURLConnectionImpl.java:307) ~[?:?]
at io.confluent.kafka.schemaregistry.client.rest.RestService.sendHttpRequest(RestService.java:312) ~[kafka-schema-registry-client-7.5.1.jar!/:?]
at io.confluent.kafka.schemaregistry.client.rest.RestService.httpRequest(RestService.java:408) ~[kafka-schema-registry-client-7.5.1.jar!/:?]
at io.confluent.kafka.schemaregistry.client.rest.RestService.getLatestVersion(RestService.java:979) ~[kafka-schema-registry-client-7.5.1.jar!/:?]
at io.confluent.kafka.schemaregistry.client.rest.RestService.getLatestVersion(RestService.java:970) ~[kafka-schema-registry-client-7.5.1.jar!/:?]
at io.confluent.kafka.schemaregistry.client.CachedSchemaRegistryClient.getLatestSchemaMetadata(CachedSchemaRegistryClient.java:573) ~[kafka-schema-registry-client-7.5.1.jar!/:?]
at io.confluent.kafka.serializers.AbstractKafkaSchemaSerDe.lookupLatestVersion(AbstractKafkaSchemaSerDe.java:571) ~[kafka-schema-serializer-7.5.1.jar!/:?]
at io.confluent.kafka.serializers.protobuf.AbstractKafkaProtobufSerializer.resolveDependencies(AbstractKafkaProtobufSerializer.java:382) ~[kafka-protobuf-serializer-7.5.1.jar!/:?]
at io.confluent.kafka.serializers.protobuf.AbstractKafkaProtobufSerializer.resolveDependencies(AbstractKafkaProtobufSerializer.java:327) ~[kafka-protobuf-serializer-7.5.1.jar!/:?]
at io.confluent.kafka.serializers.protobuf.AbstractKafkaProtobufSerializer.resolveDependencies(AbstractKafkaProtobufSerializer.java:290) ~[kafka-protobuf-serializer-7.5.1.jar!/:?]
at io.confluent.kafka.serializers.protobuf.KafkaProtobufSerializer.serialize(KafkaProtobufSerializer.java:97) ~[kafka-protobuf-serializer-7.5.1.jar!/:?]
at io.confluent.kafka.serializers.protobuf.KafkaProtobufSerializer.serialize(KafkaProtobufSerializer.java:37) ~[kafka-protobuf-serializer-7.5.1.jar!/:?]
at org.apache.kafka.clients.producer.KafkaProducer.doSend(KafkaProducer.java:1015) ~[kafka-clients-3.4.1.jar!/:?]
at org.apache.kafka.clients.producer.KafkaProducer.send(KafkaProducer.java:962) ~[kafka-clients-3.4.1.jar!/:?]
at org.springframework.kafka.core.DefaultKafkaProducerFactory$CloseSafeProducer.send(DefaultKafkaProducerFactory.java:1062) ~[spring-kafka-3.0.12.jar!/:3.0.12]
at org.springframework.kafka.core.KafkaTemplate.doSend(KafkaTemplate.java:786) ~[spring-kafka-3.0.12.jar!/:3.0.12]
at org.springframework.kafka.core.KafkaTemplate.observeSend(KafkaTemplate.java:754) ~[spring-kafka-3.0.12.jar!/:3.0.12]
at org.springframework.kafka.core.KafkaTemplate.send(KafkaTemplate.java:544) ~[spring-kafka-3.0.12.jar!/:3.0.12]
at com.sadapay.invoicing.service.producers.PaymentEventDispatcher.dispatchPaymentUnderReview(PaymentEventDispatcher.kt:101) ~[classes!/:1.158.0]
at com.sadapay.invoicing.service.service.RiskDecisionService.handleTransactionHeldDecision(RiskDecisionService.kt:48) ~[classes!/:1.158.0]
at com.sadapay.invoicing.service.consumers.RiskTransactionDecisionEventHandler.consumeRiskTransactionDecisionEvent(RiskTransactionDecisionEventHandler.kt:35) ~[classes!/:1.158.0]
at jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:103) ~[?:?]
at java.lang.reflect.Method.invoke(Method.java:580) ~[?:?]
at org.springframework.messaging.handler.invocation.InvocableHandlerMethod.doInvoke(InvocableHandlerMethod.java:169) ~[spring-messaging-6.0.13.jar!/:6.0.13]
at org.springframework.messaging.handler.invocation.InvocableHandlerMethod.invoke(InvocableHandlerMethod.java:119) ~[spring-messaging-6.0.13.jar!/:6.0.13]
at org.springframework.kafka.listener.adapter.HandlerAdapter.invoke(HandlerAdapter.java:56) ~[spring-kafka-3.0.12.jar!/:3.0.12]
at org.springframework.kafka.listener.adapter.MessagingMessageListenerAdapter.invokeHandler(MessagingMessageListenerAdapter.java:375) ~[spring-kafka-3.0.12.jar!/:3.0.12]
at org.springframework.kafka.listener.adapter.RecordMessagingMessageListenerAdapter.onMessage(RecordMessagingMessageListenerAdapter.java:92) ~[spring-kafka-3.0.12.jar!/:3.0.12]
at org.springframework.kafka.listener.adapter.RecordMessagingMessageListenerAdapter.onMessage(RecordMessagingMessageListenerAdapter.java:53) ~[spring-kafka-3.0.12.jar!/:3.0.12]
at org.springframework.kafka.listener.KafkaMessageListenerContainer$ListenerConsumer.doInvokeOnMessage(KafkaMessageListenerContainer.java:2873) ~[spring-kafka-3.0.12.jar!/:3.0.12]
at org.springframework.kafka.listener.KafkaMessageListenerContainer$ListenerConsumer.invokeOnMessage(KafkaMessageListenerContainer.java:2854) ~[spring-kafka-3.0.12.jar!/:3.0.12]
at org.springframework.kafka.listener.KafkaMessageListenerContainer$ListenerConsumer.lambda$doInvokeRecordListener$57(KafkaMessageListenerContainer.java:2772) ~[spring-kafka-3.0.12.jar!/:3.0.12]
... 12 more