java.io.IOException: Connection reset by peer at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:47) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:471) at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:63) at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:142) at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:159) at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:117) at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82) at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:140) at java.io.DataOutputStream.flush(DataOutputStream.java:123) at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1590) at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1525) at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1438) at java.lang.Thread.run(Thread.java:745) 2017-10-30 19:04:52,235 INFO datanode.DataNode (BlockReceiver.java:run(1449)) - PacketResponder: BP-2139487625-10.11.12.11-1447775100056:blk_1073910780_170751, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=2:[10.11.12.11:50010, 10.11.12.12:50010] java.io.IOException: Connection reset by peer at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:47) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:471) at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:63) at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:142) at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:159) at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:117) at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82) at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:140) at java.io.DataOutputStream.flush(DataOutputStream.java:123) at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1590) at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1525) at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1438) at java.lang.Thread.run(Thread.java:745) 2017-10-30 19:04:52,236 INFO datanode.DataNode (BlockReceiver.java:run(1463)) - PacketResponder: BP-2139487625-10.11.12.11-1447775100056:blk_1073910780_170751, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=2:[10.11.12.11:50010, 10.11.12.12:50010] terminating 2017-10-30 19:04:52,242 INFO datanode.DataNode (DataXceiver.java:writeBlock(669)) - Receiving BP-2139487625-10.11.12.11-1447775100056:blk_1073910780_170751 src: /10.11.12.15:48177 dest: /10.11.12.15:50010 2017-10-30 19:04:52,243 INFO impl.FsDatasetImpl (FsDatasetImpl.java:recoverClose(1306)) - Recover failed close BP-2139487625-10.11.12.11-1447775100056:blk_1073910780_170751 2017-10-30 19:05:52,248 ERROR datanode.DataNode (DataXceiver.java:writeBlock(787)) - DataNode{data=FSDataset{dirpath='[/hadoop/hdfs/data/current, /hdfs/current]'}, localName='comu5.baidu.cn:50010', datanodeUuid='2d2bf8fa-6617-43b5-a379-5d236e6c0987', xmitsInProgress=0}:Exception transfering block BP-2139487625-10.11.12.11-1447775100056:blk_1073910780_170751 to mirror 10.11.12.11:50010: java.io.EOFException: Premature EOF: no length prefix available 2017-10-30 19:05:52,249 INFO datanode.DataNode (DataXceiver.java:writeBlock(850)) - opWriteBlock BP-2139487625-10.11.12.11-1447775100056:blk_1073910780_170751 received exception java.io.EOFException: Premature EOF: no length prefix available 2017-10-30 19:05:52,249 ERROR datanode.DataNode (DataXceiver.java:run(278)) - comu5.baidu.cn:50010:DataXceiver error processing WRITE_BLOCK operation src: /10.11.12.15:48177 dst: /10.11.12.15:50010 java.io.EOFException: Premature EOF: no length prefix available at org.apache.hadoop.hdfs.protocolPB.PBHelper.vintPrefixed(PBHelper.java:2464) at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:758) at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:137) at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:74) at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:251) at java.lang.Thread.run(Thread.java:745) 2017-10-30 19:05:52,255 INFO datanode.DataNode (DataXceiver.java:writeBlock(669)) - Receiving BP-2139487625-10.11.12.11-1447775100056:blk_1073910780_170751 src: /10.11.12.15:48206 dest: /10.11.12.15:50010 2017-10-30 19:05:52,255 INFO impl.FsDatasetImpl (FsDatasetImpl.java:recoverClose(1306)) - Recover failed close BP-2139487625-10.11.12.11-1447775100056:blk_1073910780_170751 2017-10-30 19:05:52,256 INFO datanode.DataNode (DataXceiver.java:writeBlock(842)) - Received BP-2139487625-10.11.12.11-1447775100056:blk_1073910780_170753 src: /10.11.12.15:48206 dest: /10.11.12.15:50010 of size 134217728 2017-10-30 19:07:02,357 INFO datanode.DataNode (DataXceiver.java:writeBlock(669)) - Receiving BP-2139487625-10.11.12.11-1447775100056:blk_1073910782_170755 src: /10.11.12.15:48243 dest: /10.11.12.15:50010 2017-10-30 19:12:43,895 INFO datanode.DataNode (DataXceiver.java:writeBlock(669)) - Receiving BP-2139487625-10.11.12.11-1447775100056:blk_1073910797_170770 src: /10.11.12.12:40557 dest: /10.11.12.15:50010 2017-10-30 19:12:55,991 INFO datanode.DataNode (DataXceiver.java:writeBlock(669)) - Receiving BP-2139487625-10.11.12.11-1447775100056:blk_1073910799_170772 src: /10.11.12.11:60253 dest: /10.11.12.15:50010 2017-10-30 19:13:01,805 INFO datanode.DataNode (DataXceiver.java:writeBlock(669)) - Receiving BP-2139487625-10.11.12.11-1447775100056:blk_1073910802_170775 src: /10.11.12.15:48416 dest: /10.11.12.15:50010 2017-10-30 19:13:28,414 INFO datanode.DataNode (DataXceiver.java:writeBlock(669)) - Receiving BP-2139487625-10.11.12.11-1447775100056:blk_1073910803_170776 src: /10.11.12.12:40574 dest: /10.11.12.15:50010 ^C [root@comu5 ~]# > /var/log/hadoop/hdfs/hadoop-hdfs-datanode-comu5.baidu.cn.log [root@comu5 ~]# tail -f -n 2000 /var/log/hadoop/hdfs/hadoop-hdfs-datanode-comu5.baidu.cn.log [root@comu5 ~]# tail -f -n 2000 /var/log/hadoop/hdfs/hadoop-hdfs-datanode-comu5.baidu.cn.log 2017-10-30 19:14:11,644 INFO datanode.DataNode (DataXceiver.java:writeBlock(669)) - Receiving BP-2139487625-10.11.12.11-1447775100056:blk_1073910807_170780 src: /10.11.12.14:38023 dest: /10.11.12.15:50010 2017-10-30 19:16:10,099 WARN nio.NioEventLoop (Slf4JLogger.java:warn(136)) - Selector.select() returned prematurely 512 times in a row; rebuilding selector. 2017-10-30 19:16:10,100 INFO nio.NioEventLoop (Slf4JLogger.java:info(101)) - Migrated 0 channel(s) to the new Selector. 2017-10-30 19:16:18,969 INFO datanode.DataNode (BlockReceiver.java:run(1411)) - PacketResponder: BP-2139487625-10.11.12.11-1447775100056:blk_1073910774_170744, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[10.11.12.13:50010] java.net.SocketTimeoutException: 65000 millis timeout while waiting for channel to be ready for read. ch : java.nio.channels.SocketChannel[connected local=/10.11.12.15:40422 remote=/10.11.12.13:50010] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:164) at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:118) at java.io.FilterInputStream.read(FilterInputStream.java:83) at java.io.FilterInputStream.read(FilterInputStream.java:83) at org.apache.hadoop.hdfs.protocolPB.PBHelper.vintPrefixed(PBHelper.java:2462) at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1341) at java.lang.Thread.run(Thread.java:745) 2017-10-30 19:16:18,992 WARN datanode.DataNode (BlockReceiver.java:run(1446)) - IOException in BlockReceiver.run(): java.io.IOException: Connection reset by peer at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:47) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:471) at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:63) at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:142) at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:159) at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:117) at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82) at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:140) at java.io.DataOutputStream.flush(DataOutputStream.java:123) at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1590) at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1525) at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1438) at java.lang.Thread.run(Thread.java:745) 2017-10-30 19:16:18,993 INFO datanode.DataNode (BlockReceiver.java:run(1449)) - PacketResponder: BP-2139487625-10.11.12.11-1447775100056:blk_1073910774_170744, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[10.11.12.13:50010] java.io.IOException: Connection reset by peer at sun.nio.ch.FileDispatcherImpl.write0(Native Method) at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:47) at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93) at sun.nio.ch.IOUtil.write(IOUtil.java:65) at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:471) at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:63) at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:142) at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:159) at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:117) at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:82) at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:140) at java.io.DataOutputStream.flush(DataOutputStream.java:123) at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1590) at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1525) at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1438) at java.lang.Thread.run(Thread.java:745) 2017-10-30 19:16:18,994 INFO datanode.DataNode (BlockReceiver.java:run(1463)) - PacketResponder: BP-2139487625-10.11.12.11-1447775100056:blk_1073910774_170744, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[10.11.12.13:50010] terminating