Cloudera Community

Support Questions

Find answers, ask questions, and share your expertise
cancel
Turn on suggestions
Auto-suggest helps you quickly narrow down your search results by suggesting possible matches as you type.
Showing results for 
Show  only  | Search instead for 
Did you mean: 
Advanced Search

Canary issue: Connection failure: Failed to connect

avatar
author-rank stale
Explorer

Created ‎07-14-2022 02:07 AM

Hello,

could anyone help with this warning? What does it mean? HDFS is not usable at the moment:

 

2022-07-13 13:54:24,140 WARN org.apache.hadoop.hdfs.DFSClient: Connection failure: Failed to connect to /X.X.X.223:9866 for file /tmp/.cloudera_health_monitoring_canary_files/.canary_file_2022_07_13-13_5

4_24.7b7b1aba16dd0018 for block BP-1398826736-X.X.X.220-1656342421752:blk_1073760105_19281:com.google.protobuf.InvalidProtocolBufferException: Message missing required fields: seqno, lastPacketInBlock, d

ataLen

com.google.protobuf.InvalidProtocolBufferException: Message missing required fields: seqno, lastPacketInBlock, dataLen

        at com.google.protobuf.UninitializedMessageException.asInvalidProtocolBufferException(UninitializedMessageException.java:79)

        at com.google.protobuf.AbstractParser.checkMessageInitialized(AbstractParser.java:68)

        at com.google.protobuf.AbstractParser.parseFrom(AbstractParser.java:191)

        at com.google.protobuf.AbstractParser.parseFrom(AbstractParser.java:203)

        at com.google.protobuf.AbstractParser.parseFrom(AbstractParser.java:208)

        at com.google.protobuf.AbstractParser.parseFrom(AbstractParser.java:48)

        at org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos$PacketHeaderProto.parseFrom(DataTransferProtos.java:20951)

        at org.apache.hadoop.hdfs.protocol.datatransfer.PacketHeader.setFieldsFromData(PacketHeader.java:130)

        at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:179)

        at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:102)

        at org.apache.hadoop.hdfs.client.impl.BlockReaderRemote.readTrailingEmptyPacket(BlockReaderRemote.java:268)

        at org.apache.hadoop.hdfs.client.impl.BlockReaderRemote.readNextPacket(BlockReaderRemote.java:233)

        at org.apache.hadoop.hdfs.client.impl.BlockReaderRemote.read(BlockReaderRemote.java:169)

        at org.apache.hadoop.hdfs.DFSInputStream.actualGetFromOneDataNode(DFSInputStream.java:1072)

        at org.apache.hadoop.hdfs.DFSInputStream.fetchBlockByteRange(DFSInputStream.java:1014)

        at org.apache.hadoop.hdfs.DFSInputStream.pread(DFSInputStream.java:1373)

        at org.apache.hadoop.hdfs.DFSInputStream.read(DFSInputStream.java:1337)

        at org.apache.hadoop.fs.FSInputStream.readFully(FSInputStream.java:124)

        at org.apache.hadoop.fs.FSDataInputStream.readFully(FSDataInputStream.java:125)

        at com.cloudera.cmf.cdh7client.hdfs.FSDataInputStreamImpl.readFully(FSDataInputStreamImpl.java:24)

        at com.cloudera.cmon.firehose.polling.hdfs.HdfsCanary.readFile(HdfsCanary.java:205)

        at com.cloudera.cmon.firehose.polling.hdfs.HdfsCanary.doWork(HdfsCanary.java:105)

        at com.cloudera.cmon.firehose.polling.hdfs.HdfsCanary.doWork(HdfsCanary.java:47)

        at com.cloudera.cmon.firehose.polling.AbstractFileSystemClientTask.doWorkWithClientConfig(AbstractFileSystemClientTask.java:55)

        at com.cloudera.cmon.firehose.polling.AbstractCdhWorkUsingClientConfigs.doWork(AbstractCdhWorkUsingClientConfigs.java:45)

        at com.cloudera.cmon.firehose.polling.CdhTask$InstrumentedWork.doWork(CdhTask.java:231)

        at com.cloudera.cmf.cdhclient.util.ImpersonatingTaskWrapper.runTask(ImpersonatingTaskWrapper.java:72)

        at com.cloudera.cmf.cdhclient.util.ImpersonatingTaskWrapper.access$000(ImpersonatingTaskWrapper.java:21)

        at com.cloudera.cmf.cdhclient.util.ImpersonatingTaskWrapper$1.run(ImpersonatingTaskWrapper.java:107)

        at java.base/java.security.AccessController.doPrivileged(Native Method)

        at java.base/javax.security.auth.Subject.doAs(Subject.java:423)

        at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1898)

        at com.cloudera.cmf.cdh7client.security.UserGroupInformationImpl.doAs(UserGroupInformationImpl.java:42)

        at com.cloudera.cmf.cdhclient.util.ImpersonatingTaskWrapper.doWork(ImpersonatingTaskWrapper.java:104)

        at com.cloudera.cmf.cdhclient.CdhExecutor$SecurityWrapper$1.run(CdhExecutor.java:189)

        at java.base/java.security.AccessController.doPrivileged(Native Method)

        at java.base/javax.security.auth.Subject.doAs(Subject.java:423)

        at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1898)

        at com.cloudera.cmf.cdh7client.security.UserGroupInformationImpl.doAs(UserGroupInformationImpl.java:42)

        at com.cloudera.cmf.cdhclient.CdhExecutor$SecurityWrapper.doWork(CdhExecutor.java:186)

        at com.cloudera.cmf.cdhclient.CdhExecutor$1.call(CdhExecutor.java:125)

        at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)

        at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)

        at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)

        at java.base/java.lang.Thread.run(Thread.java:829)

2022-07-13 13:54:24,142 WARN org.apache.hadoop.hdfs.DFSClient: Connection failure: Failed to connect to /X.X.X.225:9866 for file /tmp/.cloudera_health_monitoring_canary_files/.canary_file_2022_07_13-13_54_24.7b7b1aba16dd0018 for block BP-1398826736-X.X.X.220-1656342421752:blk_1073760105_19281:com.google.protobuf.InvalidProtocolBufferException$InvalidWireTypeException: Protocol message tag had invalid wire type.

com.google.protobuf.InvalidProtocolBufferException$InvalidWireTypeException: Protocol message tag had invalid wire type.

        at com.google.protobuf.InvalidProtocolBufferException.invalidWireType(InvalidProtocolBufferException.java:111)

        at com.google.protobuf.UnknownFieldSet$Builder.mergeFieldFrom(UnknownFieldSet.java:557)

        at com.google.protobuf.GeneratedMessage.parseUnknownField(GeneratedMessage.java:275)

        at org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos$PacketHeaderProto.<init>(DataTransferProtos.java:20614)

        at org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos$PacketHeaderProto.<init>(DataTransferProtos.java:20572)

        at org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos$PacketHeaderProto$1.parsePartialFrom(DataTransferProtos.java:20675)

        at org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos$PacketHeaderProto$1.parsePartialFrom(DataTransferProtos.java:20670)

        at com.google.protobuf.AbstractParser.parsePartialFrom(AbstractParser.java:158)

        at com.google.protobuf.AbstractParser.parseFrom(AbstractParser.java:191)

        at com.google.protobuf.AbstractParser.parseFrom(AbstractParser.java:203)

        at com.google.protobuf.AbstractParser.parseFrom(AbstractParser.java:208)

        at com.google.protobuf.AbstractParser.parseFrom(AbstractParser.java:48)

        at org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos$PacketHeaderProto.parseFrom(DataTransferProtos.java:20951)

        at org.apache.hadoop.hdfs.protocol.datatransfer.PacketHeader.setFieldsFromData(PacketHeader.java:130)

        at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:179)

        at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:102)

        at org.apache.hadoop.hdfs.client.impl.BlockReaderRemote.readTrailingEmptyPacket(BlockReaderRemote.java:268)

        at org.apache.hadoop.hdfs.client.impl.BlockReaderRemote.readNextPacket(BlockReaderRemote.java:233)

        at org.apache.hadoop.hdfs.client.impl.BlockReaderRemote.read(BlockReaderRemote.java:169)

        at org.apache.hadoop.hdfs.DFSInputStream.actualGetFromOneDataNode(DFSInputStream.java:1072)

        at org.apache.hadoop.hdfs.DFSInputStream.fetchBlockByteRange(DFSInputStream.java:1014)

        at org.apache.hadoop.hdfs.DFSInputStream.pread(DFSInputStream.java:1373)

        at org.apache.hadoop.hdfs.DFSInputStream.read(DFSInputStream.java:1337)

        at org.apache.hadoop.fs.FSInputStream.readFully(FSInputStream.java:124)

        at org.apache.hadoop.fs.FSDataInputStream.readFully(FSDataInputStream.java:125)

        at com.cloudera.cmf.cdh7client.hdfs.FSDataInputStreamImpl.readFully(FSDataInputStreamImpl.java:24)

        at com.cloudera.cmon.firehose.polling.hdfs.HdfsCanary.readFile(HdfsCanary.java:205)

        at com.cloudera.cmon.firehose.polling.hdfs.HdfsCanary.doWork(HdfsCanary.java:105)

        at com.cloudera.cmon.firehose.polling.hdfs.HdfsCanary.doWork(HdfsCanary.java:47)

        at com.cloudera.cmon.firehose.polling.AbstractFileSystemClientTask.doWorkWithClientConfig(AbstractFileSystemClientTask.java:55)

        at com.cloudera.cmon.firehose.polling.AbstractCdhWorkUsingClientConfigs.doWork(AbstractCdhWorkUsingClientConfigs.java:45)

        at com.cloudera.cmon.firehose.polling.CdhTask$InstrumentedWork.doWork(CdhTask.java:231)

        at com.cloudera.cmf.cdhclient.util.ImpersonatingTaskWrapper.runTask(ImpersonatingTaskWrapper.java:72)

        at com.cloudera.cmf.cdhclient.util.ImpersonatingTaskWrapper.access$000(ImpersonatingTaskWrapper.java:21)

        at com.cloudera.cmf.cdhclient.util.ImpersonatingTaskWrapper$1.run(ImpersonatingTaskWrapper.java:107)

        at java.base/java.security.AccessController.doPrivileged(Native Method)

        at java.base/javax.security.auth.Subject.doAs(Subject.java:423)

        at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1898)

        at com.cloudera.cmf.cdh7client.security.UserGroupInformationImpl.doAs(UserGroupInformationImpl.java:42)

        at com.cloudera.cmf.cdhclient.util.ImpersonatingTaskWrapper.doWork(ImpersonatingTaskWrapper.java:104)

        at com.cloudera.cmf.cdhclient.CdhExecutor$SecurityWrapper$1.run(CdhExecutor.java:189)

        at java.base/java.security.AccessController.doPrivileged(Native Method)

        at java.base/javax.security.auth.Subject.doAs(Subject.java:423)

        at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1898)

        at com.cloudera.cmf.cdh7client.security.UserGroupInformationImpl.doAs(UserGroupInformationImpl.java:42)

        at com.cloudera.cmf.cdhclient.CdhExecutor$SecurityWrapper.doWork(CdhExecutor.java:186)

        at com.cloudera.cmf.cdhclient.CdhExecutor$1.call(CdhExecutor.java:125)

        at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)

        at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)

        at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)

        at java.base/java.lang.Thread.run(Thread.java:829)

2022-07-13 13:54:24,143 WARN org.apache.hadoop.hdfs.DFSClient: Connection failure: Failed to connect to /X.X.X.228:9866 for file /tmp/.cloudera_health_monitoring_canary_files/.canary_file_2022_07_13-13_54_24.7b7b1aba16dd0018 for block BP-1398826736-X.X.X.220-1656342421752:blk_1073760105_19281:com.google.protobuf.InvalidProtocolBufferException: Protocol message contained an invalid tag (zero).

com.google.protobuf.InvalidProtocolBufferException: Protocol message contained an invalid tag (zero).

        at com.google.protobuf.InvalidProtocolBufferException.invalidTag(InvalidProtocolBufferException.java:102)

        at com.google.protobuf.CodedInputStream$ArrayDecoder.readTag(CodedInputStream.java:627)

        at org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos$PacketHeaderProto.<init>(DataTransferProtos.java:20608)

        at org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos$PacketHeaderProto.<init>(DataTransferProtos.java:20572)

        at org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos$PacketHeaderProto$1.parsePartialFrom(DataTransferProtos.java:20675)

        at org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos$PacketHeaderProto$1.parsePartialFrom(DataTransferProtos.java:20670)

        at com.google.protobuf.AbstractParser.parsePartialFrom(AbstractParser.java:158)

        at com.google.protobuf.AbstractParser.parseFrom(AbstractParser.java:191)

        at com.google.protobuf.AbstractParser.parseFrom(AbstractParser.java:203)

        at com.google.protobuf.AbstractParser.parseFrom(AbstractParser.java:208)

        at com.google.protobuf.AbstractParser.parseFrom(AbstractParser.java:48)

        at org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos$PacketHeaderProto.parseFrom(DataTransferProtos.java:20951)

        at org.apache.hadoop.hdfs.protocol.datatransfer.PacketHeader.setFieldsFromData(PacketHeader.java:130)

        at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:179)

        at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:102)

        at org.apache.hadoop.hdfs.client.impl.BlockReaderRemote.readTrailingEmptyPacket(BlockReaderRemote.java:268)

        at org.apache.hadoop.hdfs.client.impl.BlockReaderRemote.readNextPacket(BlockReaderRemote.java:233)

        at org.apache.hadoop.hdfs.client.impl.BlockReaderRemote.read(BlockReaderRemote.java:169)

        at org.apache.hadoop.hdfs.DFSInputStream.actualGetFromOneDataNode(DFSInputStream.java:1072)

        at org.apache.hadoop.hdfs.DFSInputStream.fetchBlockByteRange(DFSInputStream.java:1014)

        at org.apache.hadoop.hdfs.DFSInputStream.pread(DFSInputStream.java:1373)

        at org.apache.hadoop.hdfs.DFSInputStream.read(DFSInputStream.java:1337)

        at org.apache.hadoop.fs.FSInputStream.readFully(FSInputStream.java:124)

        at org.apache.hadoop.fs.FSDataInputStream.readFully(FSDataInputStream.java:125)

        at com.cloudera.cmf.cdh7client.hdfs.FSDataInputStreamImpl.readFully(FSDataInputStreamImpl.java:24)

        at com.cloudera.cmon.firehose.polling.hdfs.HdfsCanary.readFile(HdfsCanary.java:205)

        at com.cloudera.cmon.firehose.polling.hdfs.HdfsCanary.doWork(HdfsCanary.java:105)

        at com.cloudera.cmon.firehose.polling.hdfs.HdfsCanary.doWork(HdfsCanary.java:47)

        at com.cloudera.cmon.firehose.polling.AbstractFileSystemClientTask.doWorkWithClientConfig(AbstractFileSystemClientTask.java:55)

        at com.cloudera.cmon.firehose.polling.AbstractCdhWorkUsingClientConfigs.doWork(AbstractCdhWorkUsingClientConfigs.java:45)

        at com.cloudera.cmon.firehose.polling.CdhTask$InstrumentedWork.doWork(CdhTask.java:231)

        at com.cloudera.cmf.cdhclient.util.ImpersonatingTaskWrapper.runTask(ImpersonatingTaskWrapper.java:72)

        at com.cloudera.cmf.cdhclient.util.ImpersonatingTaskWrapper.access$000(ImpersonatingTaskWrapper.java:21)

        at com.cloudera.cmf.cdhclient.util.ImpersonatingTaskWrapper$1.run(ImpersonatingTaskWrapper.java:107)

        at java.base/java.security.AccessController.doPrivileged(Native Method)

        at java.base/javax.security.auth.Subject.doAs(Subject.java:423)

        at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1898)

        at com.cloudera.cmf.cdh7client.security.UserGroupInformationImpl.doAs(UserGroupInformationImpl.java:42)

        at com.cloudera.cmf.cdhclient.util.ImpersonatingTaskWrapper.doWork(ImpersonatingTaskWrapper.java:104)

        at com.cloudera.cmf.cdhclient.CdhExecutor$SecurityWrapper$1.run(CdhExecutor.java:189)

        at java.base/java.security.AccessController.doPrivileged(Native Method)

        at java.base/javax.security.auth.Subject.doAs(Subject.java:423)

        at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1898)

        at com.cloudera.cmf.cdh7client.security.UserGroupInformationImpl.doAs(UserGroupInformationImpl.java:42)

        at com.cloudera.cmf.cdhclient.CdhExecutor$SecurityWrapper.doWork(CdhExecutor.java:186)

        at com.cloudera.cmf.cdhclient.CdhExecutor$1.call(CdhExecutor.java:125)

        at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)

        at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)

        at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)

        at java.base/java.lang.Thread.run(Thread.java:829)

 

 

583 Views
0 Kudos
0 REPLIES 0
Announcements
What's New @ Cloudera
Cloudera Data Engineering 1.23: Access Spark from Your Favor...
What's New @ Cloudera
HBase REST server scaling support is Generally Available
What's New @ Cloudera
New CLI option in the update-database command
What's New @ Cloudera
New Action menu item in the Cloudera Operational Database UI
What's New @ Cloudera
Get the list of supported instance types in the Cloudera Ope...
View More Announcements