2019-04-13 23:49:25,547 WARN New I/O worker #19:com.cloudera.server.cmf.HeartbeatRequester: Error requesting heartbeat of host id 010a1e08-ccb1-44c3-8346-2940070d5b9f java.net.ConnectException: https://xxxxxx.bdg112:9000 at com.ning.http.client.providers.netty.request.NettyConnectListener.onFutureFailure(NettyConnectListener.java:133) at com.ning.http.client.providers.netty.request.NettyConnectListener.access$200(NettyConnectListener.java:37) at com.ning.http.client.providers.netty.request.NettyConnectListener$1.operationComplete(NettyConnectListener.java:104) at org.jboss.netty.channel.DefaultChannelFuture.notifyListener(DefaultChannelFuture.java:409) at org.jboss.netty.channel.DefaultChannelFuture.notifyListeners(DefaultChannelFuture.java:395) at org.jboss.netty.channel.DefaultChannelFuture.setFailure(DefaultChannelFuture.java:362) at org.jboss.netty.handler.ssl.SslHandler.channelDisconnected(SslHandler.java:575) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:102) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireChannelDisconnected(Channels.java:396) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.close(AbstractNioWorker.java:360) at org.jboss.netty.channel.socket.nio.NioClientSocketPipelineSink.eventSunk(NioClientSocketPipelineSink.java:58) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendDownstream(DefaultChannelPipeline.java:779) at org.jboss.netty.channel.Channels.close(Channels.java:828) at org.jboss.netty.handler.ssl.SslHandler$ClosingChannelFutureListener.operationComplete(SslHandler.java:1542) at org.jboss.netty.channel.DefaultChannelFuture.notifyListener(DefaultChannelFuture.java:409) at org.jboss.netty.channel.DefaultChannelFuture.addListener(DefaultChannelFuture.java:145) at org.jboss.netty.handler.ssl.SslHandler.closeOutboundAndChannel(SslHandler.java:1502) at org.jboss.netty.handler.ssl.SslHandler.handleDownstream(SslHandler.java:514) at org.jboss.netty.channel.DefaultChannelPipeline.sendDownstream(DefaultChannelPipeline.java:591) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendDownstream(DefaultChannelPipeline.java:784) at org.jboss.netty.handler.codec.oneone.OneToOneEncoder.handleDownstream(OneToOneEncoder.java:54) at org.jboss.netty.handler.codec.http.HttpClientCodec.handleDownstream(HttpClientCodec.java:97) at org.jboss.netty.channel.DefaultChannelPipeline.sendDownstream(DefaultChannelPipeline.java:591) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendDownstream(DefaultChannelPipeline.java:784) at org.jboss.netty.handler.stream.ChunkedWriteHandler.handleDownstream(ChunkedWriteHandler.java:109) at org.jboss.netty.channel.DefaultChannelPipeline.sendDownstream(DefaultChannelPipeline.java:591) at org.jboss.netty.channel.DefaultChannelPipeline.sendDownstream(DefaultChannelPipeline.java:582) at org.jboss.netty.channel.Channels.close(Channels.java:812) at org.jboss.netty.channel.AbstractChannel.close(AbstractChannel.java:205) at com.ning.http.client.providers.netty.channel.Channels.silentlyCloseChannel(Channels.java:48) at com.ning.http.client.providers.netty.channel.ChannelManager.closeChannel(ChannelManager.java:376) at com.ning.http.client.providers.netty.handler.Processor.exceptionCaught(Processor.java:184) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:112) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.stream.ChunkedWriteHandler.handleUpstream(ChunkedWriteHandler.java:142) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.exceptionCaught(SimpleChannelUpstreamHandler.java:153) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:112) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.handler.codec.frame.FrameDecoder.exceptionCaught(FrameDecoder.java:377) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:112) at org.jboss.netty.handler.codec.http.HttpClientCodec.handleUpstream(HttpClientCodec.java:92) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline$DefaultChannelHandlerContext.sendUpstream(DefaultChannelPipeline.java:791) at org.jboss.netty.channel.Channels.fireExceptionCaught(Channels.java:536) at org.jboss.netty.handler.ssl.SslHandler.decode(SslHandler.java:862) at org.jboss.netty.handler.codec.frame.FrameDecoder.callDecode(FrameDecoder.java:425) at org.jboss.netty.handler.codec.frame.FrameDecoder.messageReceived(FrameDecoder.java:303) at org.jboss.netty.channel.SimpleChannelUpstreamHandler.handleUpstream(SimpleChannelUpstreamHandler.java:70) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:564) at org.jboss.netty.channel.DefaultChannelPipeline.sendUpstream(DefaultChannelPipeline.java:559) at org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:268) at org.jboss.netty.channel.Channels.fireMessageReceived(Channels.java:255) at org.jboss.netty.channel.socket.nio.NioWorker.read(NioWorker.java:88) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.process(AbstractNioWorker.java:108) at org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:337) at org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:89) at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) at org.jboss.netty.util.ThreadRenamingRunnable.run(ThreadRenamingRunnable.java:108) at org.jboss.netty.util.internal.DeadLockProofWorker$1.run(DeadLockProofWorker.java:42) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Caused by: java.nio.channels.ClosedChannelException ... 62 more 2019-04-13 23:49:25,648 INFO scm-web-110:com.cloudera.enterprise.JavaMelodyFacade: Exiting HTTP Operation: Method:POST, Path:/v31/clusters/BigData/commands/perfInspector, Status:200 2019-04-13 23:49:25,677 WARN avro-servlet-hb-processor-1:com.cloudera.server.cmf.AgentProtocolImpl: Received Process Heartbeat for unknown (or duplicate) process. Ignoring. This is expected to happen once after old process eviction or process deletion (as happens in restarts). id=72 name=null host=36353fc5-c4e3-4e4a-93b3-f4883ccda7d0/xxxxxx.bdg110 2019-04-13 23:49:25,685 WARN avro-servlet-hb-processor-0:com.cloudera.server.cmf.AgentProtocolImpl: Received Process Heartbeat for unknown (or duplicate) process. Ignoring. This is expected to happen once after old process eviction or process deletion (as happens in restarts). id=69 name=null host=e123aacf-fc47-43d8-a285-9c03cad95e12/xxxxxx.bdg113 2019-04-13 23:49:25,712 WARN avro-servlet-hb-processor-1:com.cloudera.server.cmf.AgentProtocolImpl: Received Process Heartbeat for unknown (or duplicate) process. Ignoring. This is expected to happen once after old process eviction or process deletion (as happens in restarts). id=70 name=null host=da8b59cc-68ca-4cd9-97c7-f62ee1db44e4/xxxxxx.bdg111 2019-04-13 23:49:25,993 WARN avro-servlet-hb-processor-0:com.cloudera.server.cmf.AgentProtocolImpl: Received Process Heartbeat for unknown (or duplicate) process. Ignoring. This is expected to happen once after old process eviction or process deletion (as happens in restarts). id=71 name=null host=010a1e08-ccb1-44c3-8346-2940070d5b9f/xxxxxx.bdg112 2019-04-13 23:49:34,546 WARN AgentResultFetcher-0:com.cloudera.cmf.service.AgentResultFetcher: Error fetching result from agent. javax.net.ssl.SSLException: Unrecognized SSL message, plaintext connection? at sun.security.ssl.InputRecord.handleUnknownRecord(InputRecord.java:710) at sun.security.ssl.InputRecord.read(InputRecord.java:527) at sun.security.ssl.SSLSocketImpl.readRecord(SSLSocketImpl.java:983) at sun.security.ssl.SSLSocketImpl.performInitialHandshake(SSLSocketImpl.java:1385) at sun.security.ssl.SSLSocketImpl.startHandshake(SSLSocketImpl.java:1413) at sun.security.ssl.SSLSocketImpl.startHandshake(SSLSocketImpl.java:1397) at sun.net.www.protocol.https.HttpsClient.afterConnect(HttpsClient.java:559) at sun.net.www.protocol.https.AbstractDelegateHttpsURLConnection.connect(AbstractDelegateHttpsURLConnection.java:185) at sun.net.www.protocol.http.HttpURLConnection.getInputStream0(HttpURLConnection.java:1564) at sun.net.www.protocol.http.HttpURLConnection.getInputStream(HttpURLConnection.java:1492) at sun.net.www.protocol.https.HttpsURLConnectionImpl.getInputStream(HttpsURLConnectionImpl.java:263) at com.cloudera.server.common.ConnectionUtils.readAgentUrlWithTimeouts(ConnectionUtils.java:75) at com.cloudera.server.common.ConnectionUtils.readAgentUrlWithTimeouts(ConnectionUtils.java:61) at com.cloudera.cmf.service.AgentResultFetcher.read(AgentResultFetcher.java:172) at com.cloudera.cmf.service.AgentResultFetcher$ResultFetcher.run(AgentResultFetcher.java:146) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 2019-04-13 23:49:34,638 WARN CommandPusher:com.cloudera.cmf.command.flow.SingleRequestAgentResultFetcherWorkOutput: Unable to fetch result from agent. javax.net.ssl.SSLException: Unrecognized SSL message, plaintext connection? at sun.security.ssl.InputRecord.handleUnknownRecord(InputRecord.java:710) at sun.security.ssl.InputRecord.read(InputRecord.java:527) at sun.security.ssl.SSLSocketImpl.readRecord(SSLSocketImpl.java:983) at sun.security.ssl.SSLSocketImpl.performInitialHandshake(SSLSocketImpl.java:1385) at sun.security.ssl.SSLSocketImpl.startHandshake(SSLSocketImpl.java:1413) at sun.security.ssl.SSLSocketImpl.startHandshake(SSLSocketImpl.java:1397) at sun.net.www.protocol.https.HttpsClient.afterConnect(HttpsClient.java:559) at sun.net.www.protocol.https.AbstractDelegateHttpsURLConnection.connect(AbstractDelegateHttpsURLConnection.java:185) at sun.net.www.protocol.http.HttpURLConnection.getInputStream0(HttpURLConnection.java:1564) at sun.net.www.protocol.http.HttpURLConnection.getInputStream(HttpURLConnection.java:1492) at sun.net.www.protocol.https.HttpsURLConnectionImpl.getInputStream(HttpsURLConnectionImpl.java:263) at com.cloudera.server.common.ConnectionUtils.readAgentUrlWithTimeouts(ConnectionUtils.java:75) at com.cloudera.server.common.ConnectionUtils.readAgentUrlWithTimeouts(ConnectionUtils.java:61) at com.cloudera.cmf.service.AgentResultFetcher.read(AgentResultFetcher.java:172) at com.cloudera.cmf.service.AgentResultFetcher$ResultFetcher.run(AgentResultFetcher.java:146) at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) at java.util.concurrent.FutureTask.run(FutureTask.java:266) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) ... View more