Hi All
We have 5 node cluster and post upgrade from 1.7.1 to 1.14.0.
We observed that below warnings are continuously writing on primary node logs.
2022-01-20 05:32:49,820 WARN [kafka-producer-network-thread | producer-1] o.a.k.common.network.SslTransportLayer Failed to send SSL Close message
java.io.IOException: Connection reset by peer
at sun.nio.ch.FileDispatcherImpl.write0(Native Method)
at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:47)
at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93)
at sun.nio.ch.IOUtil.write(IOUtil.java:65)
at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:471)
at org.apache.kafka.common.network.SslTransportLayer.flush(SslTransportLayer.java:213)
at org.apache.kafka.common.network.SslTransportLayer.close(SslTransportLayer.java:176)
at org.apache.kafka.common.utils.Utils.closeAll(Utils.java:703)
at org.apache.kafka.common.network.KafkaChannel.close(KafkaChannel.java:61)
at org.apache.kafka.common.network.Selector.doClose(Selector.java:741)
at org.apache.kafka.common.network.Selector.close(Selector.java:729)
at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:522)
at org.apache.kafka.common.network.Selector.poll(Selector.java:412)
at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:460)
at org.apache.kafka.clients.NetworkClientUtils.awaitReady(NetworkClientUtils.java:71)
at org.apache.kafka.clients.producer.internals.Sender.awaitLeastLoadedNodeReady(Sender.java:415)
at org.apache.kafka.clients.producer.internals.Sender.maybeSendTransactionalRequest(Sender.java:347)
at org.apache.kafka.clients.producer.internals.Sender.run(Sender.java:214)
at org.apache.kafka.clients.producer.internals.Sender.run(Sender.java:163)
at java.lang.Thread.run(Thread.java:748)
2022-01-20 05:32:49,857 WARN [kafka-producer-network-thread | producer-2] org.apache.kafka.clients.NetworkClient [Producer clientId=producer-2, transactionalId=d801193a-15ab-4823-be82-caddbcae65b0] Connection to node -9 could not be established. Broker may not be available.
2022-01-20 05:32:49,906 WARN [kafka-producer-network-thread | producer-83] org.apache.kafka.clients.NetworkClient [Producer clientId=producer-83, transactionalId=6fe4555f-5f6e-4fb3-82c5-a7ddaa1e8788] Connection to node -9 could not be established. Broker may not be available.
2022-01-20 05:32:49,949 WARN [kafka-producer-network-thread | producer-92] org.apache.kafka.clients.NetworkClient [Producer clientId=producer-92, transactionalId=3fe7b7c8-336b-4c29-addb-ab5cdb8425dc] Connection to node -9 could not be established. Broker may not be available.
2022-01-20 05:32:49,973 INFO [Cleanup Archive for default] o.a.n.c.repository.FileSystemRepository Successfully deleted 0 files (0 bytes) from archive
2022-01-20 05:32:50,073 WARN [kafka-producer-network-thread | producer-2] o.a.k.common.network.SslTransportLayer Failed to send SSL Close message
java.io.IOException: Connection reset by peer
at sun.nio.ch.FileDispatcherImpl.write0(Native Method)
at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:47)
at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93)
at sun.nio.ch.IOUtil.write(IOUtil.java:65)
at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:471)
at org.apache.kafka.common.network.SslTransportLayer.flush(SslTransportLayer.java:213)
at org.apache.kafka.common.network.SslTransportLayer.close(SslTransportLayer.java:176)
at org.apache.kafka.common.utils.Utils.closeAll(Utils.java:703)
at org.apache.kafka.common.network.KafkaChannel.close(KafkaChannel.java:61)
at org.apache.kafka.common.network.Selector.doClose(Selector.java:741)
at org.apache.kafka.common.network.Selector.close(Selector.java:729)
at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:522)
at org.apache.kafka.common.network.Selector.poll(Selector.java:412)
at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:460)
at org.apache.kafka.clients.NetworkClientUtils.awaitReady(NetworkClientUtils.java:71)
at org.apache.kafka.clients.producer.internals.Sender.awaitLeastLoadedNodeReady(Sender.java:415)
at org.apache.kafka.clients.producer.internals.Sender.maybeSendTransactionalRequest(Sender.java:347)
at org.apache.kafka.clients.producer.internals.Sender.run(Sender.java:214)
at org.apache.kafka.clients.producer.internals.Sender.run(Sender.java:163)
at java.lang.Thread.run(Thread.java:748)
2022-01-20 05:32:50,236 WARN [kafka-producer-network-thread | producer-83] o.a.k.common.network.SslTransportLayer Failed to send SSL Close message
java.io.IOException: Broken pipe
at sun.nio.ch.FileDispatcherImpl.write0(Native Method)
at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:47)
at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93)
at sun.nio.ch.IOUtil.write(IOUtil.java:65)
at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:471)
at org.apache.kafka.common.network.SslTransportLayer.flush(SslTransportLayer.java:213)
at org.apache.kafka.common.network.SslTransportLayer.close(SslTransportLayer.java:176)
at org.apache.kafka.common.utils.Utils.closeAll(Utils.java:703)
at org.apache.kafka.common.network.KafkaChannel.close(KafkaChannel.java:61)
at org.apache.kafka.common.network.Selector.doClose(Selector.java:741)
at org.apache.kafka.common.network.Selector.close(Selector.java:729)
at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:522)
at org.apache.kafka.common.network.Selector.poll(Selector.java:412)
at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:460)
at org.apache.kafka.clients.NetworkClientUtils.awaitReady(NetworkClientUtils.java:71)
at org.apache.kafka.clients.producer.internals.Sender.awaitLeastLoadedNodeReady(Sender.java:415)
at org.apache.kafka.clients.producer.internals.Sender.maybeSendTransactionalRequest(Sender.java:347)
at org.apache.kafka.clients.producer.internals.Sender.run(Sender.java:214)
at org.apache.kafka.clients.producer.internals.Sender.run(Sender.java:163)
at java.lang.Thread.run(Thread.java:748)
2022-01-20 05:32:50,304 WARN [kafka-producer-network-thread | producer-37] org.apache.kafka.clients.NetworkClient [Producer clientId=producer-37, transactionalId=d229f1f0-31f2-4b58-9099-29b6df1db2dc] Connection to node -9 could not be established. Broker may not be available.
2022-01-20 05:32:50,398 WARN [kafka-producer-network-thread | producer-2] o.a.k.common.network.SslTransportLayer Failed to send SSL Close message
java.io.IOException: Broken pipe
at sun.nio.ch.FileDispatcherImpl.write0(Native Method)
at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:47)
at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93)
at sun.nio.ch.IOUtil.write(IOUtil.java:65)
at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:471)
at org.apache.kafka.common.network.SslTransportLayer.flush(SslTransportLayer.java:213)
at org.apache.kafka.common.network.SslTransportLayer.close(SslTransportLayer.java:176)
at org.apache.kafka.common.utils.Utils.closeAll(Utils.java:703)
at org.apache.kafka.common.network.KafkaChannel.close(KafkaChannel.java:61)
at org.apache.kafka.common.network.Selector.doClose(Selector.java:741)
at org.apache.kafka.common.network.Selector.close(Selector.java:729)
at org.apache.kafka.common.network.Selector.pollSelectionKeys(Selector.java:522)
at org.apache.kafka.common.network.Selector.poll(Selector.java:412)
at org.apache.kafka.clients.NetworkClient.poll(NetworkClient.java:460)
at org.apache.kafka.clients.NetworkClientUtils.awaitReady(NetworkClientUtils.java:71)
at org.apache.kafka.clients.producer.internals.Sender.awaitLeastLoadedNodeReady(Sender.java:415)
at org.apache.kafka.clients.producer.internals.Sender.maybeSendTransactionalRequest(Sender.java:347)
at org.apache.kafka.clients.producer.internals.Sender.run(Sender.java:214)
at org.apache.kafka.clients.producer.internals.Sender.run(Sender.java:163)
at java.lang.Thread.run(Thread.java:748)
@MattWho
