[root@chtcuxhd08 current]# tail -200 /var/log/hadoop/hdfs/hadoop-hdfs-namenode-chtcuxhd08.log 2017-08-25 16:02:46,566 WARN client.QuorumJournalManager (QuorumCall.java:waitFor(134)) - Waited 18015 ms (timeout=20000 ms) for a response for selectInputStreams. No responses yet. 2017-08-25 16:02:47,566 WARN client.QuorumJournalManager (QuorumCall.java:waitFor(134)) - Waited 19016 ms (timeout=20000 ms) for a response for selectInputStreams. No responses yet. 2017-08-25 16:02:48,552 WARN namenode.FSEditLog (JournalSet.java:selectInputStreams(280)) - Unable to determine input streams from QJM to [172.16.0.76:8485, 172.16.0.77:8485, 172.16.0.75:8485]. Skipping. java.io.IOException: Timed out waiting 20000ms for a quorum of nodes to respond. at org.apache.hadoop.hdfs.qjournal.client.AsyncLoggerSet.waitForWriteQuorum(AsyncLoggerSet.java:137) at org.apache.hadoop.hdfs.qjournal.client.QuorumJournalManager.selectInputStreams(QuorumJournalManager.java:471) at org.apache.hadoop.hdfs.server.namenode.JournalSet.selectInputStreams(JournalSet.java:278) at org.apache.hadoop.hdfs.server.namenode.FSEditLog.selectInputStreams(FSEditLog.java:1528) at org.apache.hadoop.hdfs.server.namenode.FSEditLog.selectInputStreams(FSEditLog.java:1552) at org.apache.hadoop.hdfs.server.namenode.FSImage.loadFSImage(FSImage.java:652) at org.apache.hadoop.hdfs.server.namenode.FSImage.recoverTransitionRead(FSImage.java:294) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.loadFSImage(FSNamesystem.java:1015) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.loadFromDisk(FSNamesystem.java:690) at org.apache.hadoop.hdfs.server.namenode.NameNode.loadNamesystem(NameNode.java:688) at org.apache.hadoop.hdfs.server.namenode.NameNode.initialize(NameNode.java:752) at org.apache.hadoop.hdfs.server.namenode.NameNode.(NameNode.java:992) at org.apache.hadoop.hdfs.server.namenode.NameNode.(NameNode.java:976) at org.apache.hadoop.hdfs.server.namenode.NameNode.createNameNode(NameNode.java:1686) at org.apache.hadoop.hdfs.server.namenode.NameNode.main(NameNode.java:1754) 2017-08-25 16:02:48,553 INFO namenode.FSImage (FSImage.java:loadFSImage(669)) - No edit log streams selected. 2017-08-25 16:02:48,554 INFO namenode.FSImage (FSImage.java:loadFSImageFile(736)) - Planning to load image: FSImageFile(file=/data/hadoop/hdfs/namenode/current/fsimage_0000000000010550957, cpktTxId=0000000000010550957) 2017-08-25 16:02:48,596 INFO namenode.FSImageFormatPBINode (FSImageFormatPBINode.java:loadINodeSection(255)) - Loading 19849 INodes. 2017-08-25 16:02:48,846 INFO namenode.FSImageFormatProtobuf (FSImageFormatProtobuf.java:load(184)) - Loaded FSImage in 0 seconds. 2017-08-25 16:02:48,846 INFO namenode.FSImage (FSImage.java:loadFSImage(976)) - Loaded image for txid 10550957 from /data/hadoop/hdfs/namenode/current/fsimage_0000000000010550957 2017-08-25 16:02:48,880 INFO namenode.FSNamesystem (FSNamesystem.java:loadFSImage(1022)) - Need to save fs image? false (staleImage=true, haEnabled=true, isRollingUpgrade=false) 2017-08-25 16:02:48,881 INFO namenode.NameCache (NameCache.java:initialized(143)) - initialized with 282 entries 3976 lookups 2017-08-25 16:02:48,881 INFO namenode.FSNamesystem (FSNamesystem.java:loadFromDisk(697)) - Finished loading FSImage in 20532 msecs 2017-08-25 16:02:49,046 INFO namenode.NameNode (NameNodeRpcServer.java:(422)) - RPC server is binding to chtcuxhd08:8020 2017-08-25 16:02:49,051 INFO ipc.CallQueueManager (CallQueueManager.java:(75)) - Using callQueue: class java.util.concurrent.LinkedBlockingQueue scheduler: class org.apache.hadoop.ipc.DefaultRpcScheduler 2017-08-25 16:02:49,063 INFO ipc.Server (Server.java:run(811)) - Starting Socket Reader #1 for port 8020 2017-08-25 16:02:49,087 INFO namenode.FSNamesystem (FSNamesystem.java:registerMBean(6289)) - Registered FSNamesystemState MBean 2017-08-25 16:02:49,088 WARN common.Util (Util.java:stringAsURI(56)) - Path /data/hadoop/hdfs/namenode should be specified as a URI in configuration files. Please update hdfs configuration. 2017-08-25 16:02:49,098 INFO namenode.LeaseManager (LeaseManager.java:getNumUnderConstructionBlocks(138)) - Number of blocks under construction: 3 2017-08-25 16:02:49,098 INFO hdfs.StateChange (FSNamesystem.java:reportStatus(5724)) - STATE* Safe mode ON. The reported blocks 0 needs additional 25957 blocks to reach the threshold 0.9900 of total blocks 26219. The number of live datanodes 0 has reached the minimum number 0. Safe mode will be turned off automatically once the thresholds have been reached. 2017-08-25 16:02:49,105 INFO blockmanagement.DatanodeDescriptor (DatanodeDescriptor.java:updateHeartbeatState(451)) - Number of failed storage changes from 0 to 0 2017-08-25 16:02:49,106 INFO block.BlockTokenSecretManager (BlockTokenSecretManager.java:updateKeys(222)) - Updating block keys 2017-08-25 16:02:49,132 INFO ipc.Server (Server.java:run(1045)) - IPC Server Responder: starting 2017-08-25 16:02:49,132 INFO ipc.Server (Server.java:run(881)) - IPC Server listener on 8020: starting 2017-08-25 16:02:49,248 INFO namenode.NameNode (NameNode.java:startCommonServices(876)) - NameNode RPC up at: chtcuxhd08/172.16.0.77:8020 2017-08-25 16:02:49,249 INFO namenode.FSNamesystem (FSNamesystem.java:startStandbyServices(1286)) - Starting services required for standby state 2017-08-25 16:02:49,252 INFO ha.EditLogTailer (EditLogTailer.java:(117)) - Will roll logs on active node at chtcuxhd07/172.16.0.76:8020 every 120 seconds. 2017-08-25 16:02:49,256 INFO ha.StandbyCheckpointer (StandbyCheckpointer.java:start(129)) - Starting standby checkpoint thread... Checkpointing active NN at http://chtcuxhd07:50070 Serving checkpoints at http://chtcuxhd08:50070 2017-08-25 16:02:55,259 INFO client.QuorumJournalManager (QuorumCall.java:waitFor(136)) - Waited 6001 ms (timeout=20000 ms) for a response for selectInputStreams. No responses yet. 2017-08-25 16:02:56,260 INFO client.QuorumJournalManager (QuorumCall.java:waitFor(136)) - Waited 7003 ms (timeout=20000 ms) for a response for selectInputStreams. No responses yet. 2017-08-25 16:02:57,262 INFO client.QuorumJournalManager (QuorumCall.java:waitFor(136)) - Waited 8004 ms (timeout=20000 ms) for a response for selectInputStreams. No responses yet. 2017-08-25 16:02:58,263 INFO client.QuorumJournalManager (QuorumCall.java:waitFor(136)) - Waited 9005 ms (timeout=20000 ms) for a response for selectInputStreams. No responses yet. 2017-08-25 16:02:59,264 INFO client.QuorumJournalManager (QuorumCall.java:waitFor(136)) - Waited 10006 ms (timeout=20000 ms) for a response for selectInputStreams. No responses yet. 2017-08-25 16:03:00,265 INFO client.QuorumJournalManager (QuorumCall.java:waitFor(136)) - Waited 11008 ms (timeout=20000 ms) for a response for selectInputStreams. No responses yet. 2017-08-25 16:03:01,267 INFO client.QuorumJournalManager (QuorumCall.java:waitFor(136)) - Waited 12009 ms (timeout=20000 ms) for a response for selectInputStreams. No responses yet. 2017-08-25 16:03:02,267 INFO client.QuorumJournalManager (QuorumCall.java:waitFor(136)) - Waited 13010 ms (timeout=20000 ms) for a response for selectInputStreams. No responses yet. 2017-08-25 16:03:03,269 WARN client.QuorumJournalManager (QuorumCall.java:waitFor(134)) - Waited 14011 ms (timeout=20000 ms) for a response for selectInputStreams. No responses yet. 2017-08-25 16:03:04,269 WARN client.QuorumJournalManager (QuorumCall.java:waitFor(134)) - Waited 15012 ms (timeout=20000 ms) for a response for selectInputStreams. No responses yet. 2017-08-25 16:03:05,272 WARN client.QuorumJournalManager (QuorumCall.java:waitFor(134)) - Waited 16014 ms (timeout=20000 ms) for a response for selectInputStreams. No responses yet. 2017-08-25 16:03:06,273 WARN client.QuorumJournalManager (QuorumCall.java:waitFor(134)) - Waited 17015 ms (timeout=20000 ms) for a response for selectInputStreams. No responses yet. 2017-08-25 16:03:07,274 WARN client.QuorumJournalManager (QuorumCall.java:waitFor(134)) - Waited 18016 ms (timeout=20000 ms) for a response for selectInputStreams. No responses yet. 2017-08-25 16:03:08,275 WARN client.QuorumJournalManager (QuorumCall.java:waitFor(134)) - Waited 19017 ms (timeout=20000 ms) for a response for selectInputStreams. No responses yet. 2017-08-25 16:03:09,259 WARN namenode.FSEditLog (JournalSet.java:selectInputStreams(280)) - Unable to determine input streams from QJM to [172.16.0.76:8485, 172.16.0.77:8485, 172.16.0.75:8485]. Skipping. java.io.IOException: Timed out waiting 20000ms for a quorum of nodes to respond. at org.apache.hadoop.hdfs.qjournal.client.AsyncLoggerSet.waitForWriteQuorum(AsyncLoggerSet.java:137) at org.apache.hadoop.hdfs.qjournal.client.QuorumJournalManager.selectInputStreams(QuorumJournalManager.java:471) at org.apache.hadoop.hdfs.server.namenode.JournalSet.selectInputStreams(JournalSet.java:278) at org.apache.hadoop.hdfs.server.namenode.FSEditLog.selectInputStreams(FSEditLog.java:1528) at org.apache.hadoop.hdfs.server.namenode.FSEditLog.selectInputStreams(FSEditLog.java:1552) at org.apache.hadoop.hdfs.server.namenode.ha.EditLogTailer.doTailEdits(EditLogTailer.java:214) at org.apache.hadoop.hdfs.server.namenode.ha.EditLogTailer$EditLogTailerThread.doWork(EditLogTailer.java:331) at org.apache.hadoop.hdfs.server.namenode.ha.EditLogTailer$EditLogTailerThread.access$200(EditLogTailer.java:284) at org.apache.hadoop.hdfs.server.namenode.ha.EditLogTailer$EditLogTailerThread$1.run(EditLogTailer.java:301) at org.apache.hadoop.security.SecurityUtil.doAsLoginUserOrFatal(SecurityUtil.java:449) at org.apache.hadoop.hdfs.server.namenode.ha.EditLogTailer$EditLogTailerThread.run(EditLogTailer.java:297) 2017-08-25 16:03:09,273 INFO namenode.FSNamesystem (FSNamesystem.java:stopStandbyServices(1329)) - Stopping services started for standby state 2017-08-25 16:03:09,275 WARN ha.EditLogTailer (EditLogTailer.java:doWork(349)) - Edit log tailer interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) at org.apache.hadoop.hdfs.server.namenode.ha.EditLogTailer$EditLogTailerThread.doWork(EditLogTailer.java:347) at org.apache.hadoop.hdfs.server.namenode.ha.EditLogTailer$EditLogTailerThread.access$200(EditLogTailer.java:284) at org.apache.hadoop.hdfs.server.namenode.ha.EditLogTailer$EditLogTailerThread$1.run(EditLogTailer.java:301) at org.apache.hadoop.security.SecurityUtil.doAsLoginUserOrFatal(SecurityUtil.java:449) at org.apache.hadoop.hdfs.server.namenode.ha.EditLogTailer$EditLogTailerThread.run(EditLogTailer.java:297) 2017-08-25 16:03:09,380 INFO namenode.FSNamesystem (FSNamesystem.java:startActiveServices(1130)) - Starting services required for active state 2017-08-25 16:03:09,389 INFO client.QuorumJournalManager (QuorumJournalManager.java:recoverUnfinalizedSegments(435)) - Starting recovery process for unclosed journal segments... 2017-08-25 16:03:20,490 INFO logs (Slf4jLog.java:info(67)) - Aliases are enabled 2017-08-25 16:03:45,392 INFO client.QuorumJournalManager (QuorumCall.java:waitFor(136)) - Waited 36001 ms (timeout=120000 ms) for a response for getJournalState(). No responses yet. 2017-08-25 16:03:46,393 INFO client.QuorumJournalManager (QuorumCall.java:waitFor(136)) - Waited 37002 ms (timeout=120000 ms) for a response for getJournalState(). No responses yet. 2017-08-25 16:03:47,394 INFO client.QuorumJournalManager (QuorumCall.java:waitFor(136)) - Waited 38003 ms (timeout=120000 ms) for a response for getJournalState(). No responses yet. 2017-08-25 16:03:48,395 INFO client.QuorumJournalManager (QuorumCall.java:waitFor(136)) - Waited 39004 ms (timeout=120000 ms) for a response for getJournalState(). No responses yet. 2017-08-25 16:03:49,396 INFO client.QuorumJournalManager (QuorumCall.java:waitFor(136)) - Waited 40005 ms (timeout=120000 ms) for a response for getJournalState(). No responses yet. 2017-08-25 16:03:50,397 INFO client.QuorumJournalManager (QuorumCall.java:waitFor(136)) - Waited 41006 ms (timeout=120000 ms) for a response for getJournalState(). No responses yet. 2017-08-25 16:03:51,398 INFO client.QuorumJournalManager (QuorumCall.java:waitFor(136)) - Waited 42007 ms (timeout=120000 ms) for a response for getJournalState(). No responses yet. 2017-08-25 16:03:52,399 INFO client.QuorumJournalManager (QuorumCall.java:waitFor(136)) - Waited 43008 ms (timeout=120000 ms) for a response for getJournalState(). No responses yet. 2017-08-25 16:03:53,400 INFO client.QuorumJournalManager (QuorumCall.java:waitFor(136)) - Waited 44010 ms (timeout=120000 ms) for a response for getJournalState(). No responses yet. 2017-08-25 16:03:54,402 INFO client.QuorumJournalManager (QuorumCall.java:waitFor(136)) - Waited 45011 ms (timeout=120000 ms) for a response for getJournalState(). No responses yet. 2017-08-25 16:03:55,403 INFO client.QuorumJournalManager (QuorumCall.java:waitFor(136)) - Waited 46012 ms (timeout=120000 ms) for a response for getJournalState(). No responses yet. 2017-08-25 16:03:56,404 INFO client.QuorumJournalManager (QuorumCall.java:waitFor(136)) - Waited 47013 ms (timeout=120000 ms) for a response for getJournalState(). No responses yet. 2017-08-25 16:03:57,405 INFO client.QuorumJournalManager (QuorumCall.java:waitFor(136)) - Waited 48014 ms (timeout=120000 ms) for a response for getJournalState(). No responses yet. 2017-08-25 16:03:58,406 INFO client.QuorumJournalManager (QuorumCall.java:waitFor(136)) - Waited 49015 ms (timeout=120000 ms) for a response for getJournalState(). No responses yet. 2017-08-25 16:03:59,407 INFO client.QuorumJournalManager (QuorumCall.java:waitFor(136)) - Waited 50016 ms (timeout=120000 ms) for a response for getJournalState(). No responses yet. 2017-08-25 16:04:00,408 INFO client.QuorumJournalManager (QuorumCall.java:waitFor(136)) - Waited 51017 ms (timeout=120000 ms) for a response for getJournalState(). No responses yet. 2017-08-25 16:04:01,408 INFO client.QuorumJournalManager (QuorumCall.java:waitFor(136)) - Waited 52018 ms (timeout=120000 ms) for a response for getJournalState(). No responses yet. 2017-08-25 16:04:02,410 INFO client.QuorumJournalManager (QuorumCall.java:waitFor(136)) - Waited 53019 ms (timeout=120000 ms) for a response for getJournalState(). No responses yet. 2017-08-25 16:04:03,411 INFO client.QuorumJournalManager (QuorumCall.java:waitFor(136)) - Waited 54020 ms (timeout=120000 ms) for a response for getJournalState(). No responses yet. 2017-08-25 16:04:04,412 INFO client.QuorumJournalManager (QuorumCall.java:waitFor(136)) - Waited 55021 ms (timeout=120000 ms) for a response for getJournalState(). No responses yet. 2017-08-25 16:04:05,413 INFO client.QuorumJournalManager (QuorumCall.java:waitFor(136)) - Waited 56022 ms (timeout=120000 ms) for a response for getJournalState(). No responses yet. 2017-08-25 16:04:06,414 INFO client.QuorumJournalManager (QuorumCall.java:waitFor(136)) - Waited 57023 ms (timeout=120000 ms) for a response for getJournalState(). No responses yet. 2017-08-25 16:04:07,415 INFO client.QuorumJournalManager (QuorumCall.java:waitFor(136)) - Waited 58024 ms (timeout=120000 ms) for a response for getJournalState(). No responses yet. 2017-08-25 16:04:08,416 INFO client.QuorumJournalManager (QuorumCall.java:waitFor(136)) - Waited 59025 ms (timeout=120000 ms) for a response for getJournalState(). No responses yet. 2017-08-25 16:04:09,417 INFO client.QuorumJournalManager (QuorumCall.java:waitFor(136)) - Waited 60026 ms (timeout=120000 ms) for a response for getJournalState(). No responses yet. 2017-08-25 16:04:10,418 INFO client.QuorumJournalManager (QuorumCall.java:waitFor(136)) - Waited 61027 ms (timeout=120000 ms) for a response for getJournalState(). No responses yet. 2017-08-25 16:04:11,419 INFO client.QuorumJournalManager (QuorumCall.java:waitFor(136)) - Waited 62028 ms (timeout=120000 ms) for a response for getJournalState(). No responses yet. 2017-08-25 16:04:12,420 INFO client.QuorumJournalManager (QuorumCall.java:waitFor(136)) - Waited 63029 ms (timeout=120000 ms) for a response for getJournalState(). No responses yet. 2017-08-25 16:04:13,420 INFO client.QuorumJournalManager (QuorumCall.java:waitFor(136)) - Waited 64030 ms (timeout=120000 ms) for a response for getJournalState(). No responses yet. 2017-08-25 16:04:14,422 INFO client.QuorumJournalManager (QuorumCall.java:waitFor(136)) - Waited 65031 ms (timeout=120000 ms) for a response for getJournalState(). No responses yet. 2017-08-25 16:04:15,423 INFO client.QuorumJournalManager (QuorumCall.java:waitFor(136)) - Waited 66032 ms (timeout=120000 ms) for a response for getJournalState(). No responses yet. 2017-08-25 16:04:16,424 INFO client.QuorumJournalManager (QuorumCall.java:waitFor(136)) - Waited 67034 ms (timeout=120000 ms) for a response for getJournalState(). No responses yet. 2017-08-25 16:04:17,426 INFO client.QuorumJournalManager (QuorumCall.java:waitFor(136)) - Waited 68035 ms (timeout=120000 ms) for a response for getJournalState(). No responses yet. 2017-08-25 16:04:18,427 INFO client.QuorumJournalManager (QuorumCall.java:waitFor(136)) - Waited 69036 ms (timeout=120000 ms) for a response for getJournalState(). No responses yet. 2017-08-25 16:04:19,428 INFO client.QuorumJournalManager (QuorumCall.java:waitFor(136)) - Waited 70038 ms (timeout=120000 ms) for a response for getJournalState(). No responses yet. 2017-08-25 16:04:20,430 INFO client.QuorumJournalManager (QuorumCall.java:waitFor(136)) - Waited 71039 ms (timeout=120000 ms) for a response for getJournalState(). No responses yet. 2017-08-25 16:04:21,431 INFO client.QuorumJournalManager (QuorumCall.java:waitFor(136)) - Waited 72040 ms (timeout=120000 ms) for a response for getJournalState(). No responses yet. 2017-08-25 16:04:22,432 INFO client.QuorumJournalManager (QuorumCall.java:waitFor(136)) - Waited 73041 ms (timeout=120000 ms) for a response for getJournalState(). No responses yet. 2017-08-25 16:04:23,433 INFO client.QuorumJournalManager (QuorumCall.java:waitFor(136)) - Waited 74042 ms (timeout=120000 ms) for a response for getJournalState(). No responses yet. 2017-08-25 16:04:24,434 INFO client.QuorumJournalManager (QuorumCall.java:waitFor(136)) - Waited 75044 ms (timeout=120000 ms) for a response for getJournalState(). No responses yet. 2017-08-25 16:04:25,436 INFO client.QuorumJournalManager (QuorumCall.java:waitFor(136)) - Waited 76045 ms (timeout=120000 ms) for a response for getJournalState(). No responses yet. 2017-08-25 16:04:26,437 INFO client.QuorumJournalManager (QuorumCall.java:waitFor(136)) - Waited 77046 ms (timeout=120000 ms) for a response for getJournalState(). No responses yet. 2017-08-25 16:04:27,438 INFO client.QuorumJournalManager (QuorumCall.java:waitFor(136)) - Waited 78047 ms (timeout=120000 ms) for a response for getJournalState(). No responses yet. 2017-08-25 16:04:28,439 INFO client.QuorumJournalManager (QuorumCall.java:waitFor(136)) - Waited 79048 ms (timeout=120000 ms) for a response for getJournalState(). No responses yet. 2017-08-25 16:04:29,440 INFO client.QuorumJournalManager (QuorumCall.java:waitFor(136)) - Waited 80049 ms (timeout=120000 ms) for a response for getJournalState(). No responses yet. 2017-08-25 16:04:30,441 INFO client.QuorumJournalManager (QuorumCall.java:waitFor(136)) - Waited 81050 ms (timeout=120000 ms) for a response for getJournalState(). No responses yet. 2017-08-25 16:04:31,441 INFO client.QuorumJournalManager (QuorumCall.java:waitFor(136)) - Waited 82051 ms (timeout=120000 ms) for a response for getJournalState(). No responses yet. 2017-08-25 16:04:32,443 INFO client.QuorumJournalManager (QuorumCall.java:waitFor(136)) - Waited 83052 ms (timeout=120000 ms) for a response for getJournalState(). No responses yet. 2017-08-25 16:04:33,444 WARN client.QuorumJournalManager (QuorumCall.java:waitFor(134)) - Waited 84054 ms (timeout=120000 ms) for a response for getJournalState(). No responses yet. 2017-08-25 16:04:34,446 WARN client.QuorumJournalManager (QuorumCall.java:waitFor(134)) - Waited 85055 ms (timeout=120000 ms) for a response for getJournalState(). No responses yet. 2017-08-25 16:04:35,447 WARN client.QuorumJournalManager (QuorumCall.java:waitFor(134)) - Waited 86056 ms (timeout=120000 ms) for a response for getJournalState(). No responses yet. 2017-08-25 16:04:36,448 WARN client.QuorumJournalManager (QuorumCall.java:waitFor(134)) - Waited 87057 ms (timeout=120000 ms) for a response for getJournalState(). No responses yet. 2017-08-25 16:04:37,449 WARN client.QuorumJournalManager (QuorumCall.java:waitFor(134)) - Waited 88058 ms (timeout=120000 ms) for a response for getJournalState(). No responses yet. 2017-08-25 16:04:38,449 WARN client.QuorumJournalManager (QuorumCall.java:waitFor(134)) - Waited 89059 ms (timeout=120000 ms) for a response for getJournalState(). No responses yet. 2017-08-25 16:04:39,451 WARN client.QuorumJournalManager (QuorumCall.java:waitFor(134)) - Waited 90060 ms (timeout=120000 ms) for a response for getJournalState(). No responses yet. 2017-08-25 16:04:40,452 WARN client.QuorumJournalManager (QuorumCall.java:waitFor(134)) - Waited 91061 ms (timeout=120000 ms) for a response for getJournalState(). No responses yet. 2017-08-25 16:04:41,453 WARN client.QuorumJournalManager (QuorumCall.java:waitFor(134)) - Waited 92062 ms (timeout=120000 ms) for a response for getJournalState(). No responses yet. 2017-08-25 16:04:42,453 WARN client.QuorumJournalManager (QuorumCall.java:waitFor(134)) - Waited 93063 ms (timeout=120000 ms) for a response for getJournalState(). No responses yet. 2017-08-25 16:04:43,455 WARN client.QuorumJournalManager (QuorumCall.java:waitFor(134)) - Waited 94064 ms (timeout=120000 ms) for a response for getJournalState(). No responses yet. 2017-08-25 16:04:44,455 WARN client.QuorumJournalManager (QuorumCall.java:waitFor(134)) - Waited 95065 ms (timeout=120000 ms) for a response for getJournalState(). No responses yet. 2017-08-25 16:04:45,457 WARN client.QuorumJournalManager (QuorumCall.java:waitFor(134)) - Waited 96066 ms (timeout=120000 ms) for a response for getJournalState(). No responses yet. 2017-08-25 16:04:46,458 WARN client.QuorumJournalManager (QuorumCall.java:waitFor(134)) - Waited 97067 ms (timeout=120000 ms) for a response for getJournalState(). No responses yet. 2017-08-25 16:04:47,459 WARN client.QuorumJournalManager (QuorumCall.java:waitFor(134)) - Waited 98068 ms (timeout=120000 ms) for a response for getJournalState(). No responses yet. 2017-08-25 16:04:48,460 WARN client.QuorumJournalManager (QuorumCall.java:waitFor(134)) - Waited 99069 ms (timeout=120000 ms) for a response for getJournalState(). No responses yet. 2017-08-25 16:04:49,461 WARN client.QuorumJournalManager (QuorumCall.java:waitFor(134)) - Waited 100070 ms (timeout=120000 ms) for a response for getJournalState(). No responses yet. 2017-08-25 16:04:50,462 WARN client.QuorumJournalManager (QuorumCall.java:waitFor(134)) - Waited 101071 ms (timeout=120000 ms) for a response for getJournalState(). No responses yet. 2017-08-25 16:04:51,462 WARN client.QuorumJournalManager (QuorumCall.java:waitFor(134)) - Waited 102072 ms (timeout=120000 ms) for a response for getJournalState(). No responses yet. 2017-08-25 16:04:52,464 WARN client.QuorumJournalManager (QuorumCall.java:waitFor(134)) - Waited 103073 ms (timeout=120000 ms) for a response for getJournalState(). No responses yet. 2017-08-25 16:04:53,465 WARN client.QuorumJournalManager (QuorumCall.java:waitFor(134)) - Waited 104074 ms (timeout=120000 ms) for a response for getJournalState(). No responses yet. 2017-08-25 16:04:54,466 WARN client.QuorumJournalManager (QuorumCall.java:waitFor(134)) - Waited 105075 ms (timeout=120000 ms) for a response for getJournalState(). No responses yet. 2017-08-25 16:04:55,467 WARN client.QuorumJournalManager (QuorumCall.java:waitFor(134)) - Waited 106076 ms (timeout=120000 ms) for a response for getJournalState(). No responses yet. 2017-08-25 16:04:56,468 WARN client.QuorumJournalManager (QuorumCall.java:waitFor(134)) - Waited 107078 ms (timeout=120000 ms) for a response for getJournalState(). No responses yet. 2017-08-25 16:04:57,470 WARN client.QuorumJournalManager (QuorumCall.java:waitFor(134)) - Waited 108079 ms (timeout=120000 ms) for a response for getJournalState(). No responses yet. 2017-08-25 16:04:58,471 WARN client.QuorumJournalManager (QuorumCall.java:waitFor(134)) - Waited 109080 ms (timeout=120000 ms) for a response for getJournalState(). No responses yet. 2017-08-25 16:04:59,472 WARN client.QuorumJournalManager (QuorumCall.java:waitFor(134)) - Waited 110081 ms (timeout=120000 ms) for a response for getJournalState(). No responses yet. 2017-08-25 16:05:00,473 WARN client.QuorumJournalManager (QuorumCall.java:waitFor(134)) - Waited 111082 ms (timeout=120000 ms) for a response for getJournalState(). No responses yet. 2017-08-25 16:05:01,474 WARN client.QuorumJournalManager (QuorumCall.java:waitFor(134)) - Waited 112083 ms (timeout=120000 ms) for a response for getJournalState(). No responses yet. 2017-08-25 16:05:02,475 WARN client.QuorumJournalManager (QuorumCall.java:waitFor(134)) - Waited 113084 ms (timeout=120000 ms) for a response for getJournalState(). No responses yet. 2017-08-25 16:05:03,476 WARN client.QuorumJournalManager (QuorumCall.java:waitFor(134)) - Waited 114085 ms (timeout=120000 ms) for a response for getJournalState(). No responses yet. 2017-08-25 16:05:04,477 WARN client.QuorumJournalManager (QuorumCall.java:waitFor(134)) - Waited 115086 ms (timeout=120000 ms) for a response for getJournalState(). No responses yet. 2017-08-25 16:05:05,478 WARN client.QuorumJournalManager (QuorumCall.java:waitFor(134)) - Waited 116087 ms (timeout=120000 ms) for a response for getJournalState(). No responses yet. 2017-08-25 16:05:06,479 WARN client.QuorumJournalManager (QuorumCall.java:waitFor(134)) - Waited 117088 ms (timeout=120000 ms) for a response for getJournalState(). No responses yet. 2017-08-25 16:05:07,480 WARN client.QuorumJournalManager (QuorumCall.java:waitFor(134)) - Waited 118089 ms (timeout=120000 ms) for a response for getJournalState(). No responses yet. 2017-08-25 16:05:08,481 WARN client.QuorumJournalManager (QuorumCall.java:waitFor(134)) - Waited 119090 ms (timeout=120000 ms) for a response for getJournalState(). No responses yet. 2017-08-25 16:05:09,392 FATAL namenode.FSEditLog (JournalSet.java:mapJournalsAndReportErrors(398)) - Error: recoverUnfinalizedSegments failed for required journal (JournalAndStream(mgr=QJM to [172.16.0.76:8485, 172.16.0.77:8485, 172.16.0.75:8485], stream=null)) java.io.IOException: Timed out waiting 120000ms for a quorum of nodes to respond. at org.apache.hadoop.hdfs.qjournal.client.AsyncLoggerSet.waitForWriteQuorum(AsyncLoggerSet.java:137) at org.apache.hadoop.hdfs.qjournal.client.QuorumJournalManager.createNewUniqueEpoch(QuorumJournalManager.java:182) at org.apache.hadoop.hdfs.qjournal.client.QuorumJournalManager.recoverUnfinalizedSegments(QuorumJournalManager.java:436) at org.apache.hadoop.hdfs.server.namenode.JournalSet$8.apply(JournalSet.java:624) at org.apache.hadoop.hdfs.server.namenode.JournalSet.mapJournalsAndReportErrors(JournalSet.java:393) at org.apache.hadoop.hdfs.server.namenode.JournalSet.recoverUnfinalizedSegments(JournalSet.java:621) at org.apache.hadoop.hdfs.server.namenode.FSEditLog.recoverUnclosedStreams(FSEditLog.java:1459) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.startActiveServices(FSNamesystem.java:1139) at org.apache.hadoop.hdfs.server.namenode.NameNode$NameNodeHAContext.startActiveServices(NameNode.java:1915) at org.apache.hadoop.hdfs.server.namenode.ha.ActiveState.enterState(ActiveState.java:61) at org.apache.hadoop.hdfs.server.namenode.ha.HAState.setStateInternal(HAState.java:64) at org.apache.hadoop.hdfs.server.namenode.ha.StandbyState.setState(StandbyState.java:49) at org.apache.hadoop.hdfs.server.namenode.NameNode.transitionToActive(NameNode.java:1783) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.transitionToActive(NameNodeRpcServer.java:1631) at org.apache.hadoop.ha.protocolPB.HAServiceProtocolServerSideTranslatorPB.transitionToActive(HAServiceProtocolServerSideTranslatorPB.java:107) at org.apache.hadoop.ha.proto.HAServiceProtocolProtos$HAServiceProtocolService$2.callBlockingMethod(HAServiceProtocolProtos.java:4460) at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:640) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:982) at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:2313) at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:2309) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1724) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2307) 2017-08-25 16:05:09,394 INFO util.ExitUtil (ExitUtil.java:terminate(124)) - Exiting with status 1 2017-08-25 16:05:09,396 INFO namenode.NameNode (LogAdapter.java:info(47)) - SHUTDOWN_MSG: /************************************************************ SHUTDOWN_MSG: Shutting down NameNode at chtcuxhd08/172.16.0.77 ************************************************************/