Created 07-24-2019 01:25 PM
We are running HDP 3.0. Out of 4 datanodes, one Datanode failed to start from ambari. Rebooting server and restarting all components did not help. Any help will be greatly appreciated.
/tmp folder has 777 permissions.
ls -ld /tmp
drwxrwxrwt. 91 root root 8192 Jul 22 12:59 /tmp
Thank you
Suresh
2019-07-22 12:44:59,912 INFO server.AuthenticationFilter (AuthenticationFilter.java:constructSecretProvider(240)) - Unable to initialize FileSignerSecretProvider, falling back to use random secrets.
2019-07-22 12:44:59,914 INFO handler.ContextHandler (ContextHandler.java:doStart(781)) - Started o.e.j.s.ServletContextHandler@2a79d4b1{/logs,file:///var/log/hadoop/hdfs/,AVAILABLE}
2019-07-22 12:44:59,915 INFO handler.ContextHandler (ContextHandler.java:doStart(781)) - Started o.e.j.s.ServletContextHandler@17cdf2d0{/static,file:///usr/hdp/3.0.1.0-187/hadoop-hdfs/webapps/static/,AVAILABLE}
2019-07-22 12:44:59,927 WARN webapp.WebAppContext (WebAppContext.java:doStart(531)) - Failed startup of context o.e.j.w.WebAppContext@cb191ca{/,null,null}{/datanode}
java.lang.IllegalStateException: Temp dir /tmp/jetty-localhost-35953-datanode-_-any-4263088992112773049.dir not useable: writeable=false, dir=true
at org.eclipse.jetty.webapp.WebInfConfiguration.configureTempDirectory(WebInfConfiguration.java:388)
at org.eclipse.jetty.webapp.WebInfConfiguration.makeTempDirectory(WebInfConfiguration.java:360)
at org.eclipse.jetty.webapp.WebInfConfiguration.resolveTempDirectory(WebInfConfiguration.java:308)
at org.eclipse.jetty.webapp.WebInfConfiguration.preConfigure(WebInfConfiguration.java:69)
at org.eclipse.jetty.webapp.WebAppContext.preConfigure(WebAppContext.java:485)
at org.eclipse.jetty.webapp.WebAppContext.doStart(WebAppContext.java:521)
at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:68)
at org.eclipse.jetty.util.component.ContainerLifeCycle.start(ContainerLifeCycle.java:131)
at org.eclipse.jetty.util.component.ContainerLifeCycle.doStart(ContainerLifeCycle.java:113)
at org.eclipse.jetty.server.handler.AbstractHandler.doStart(AbstractHandler.java:61)
at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:68)
at org.eclipse.jetty.util.component.ContainerLifeCycle.start(ContainerLifeCycle.java:131)
at org.eclipse.jetty.server.Server.start(Server.java:422)
at org.eclipse.jetty.util.component.ContainerLifeCycle.doStart(ContainerLifeCycle.java:105)
at org.eclipse.jetty.server.handler.AbstractHandler.doStart(AbstractHandler.java:61)
at org.eclipse.jetty.server.Server.doStart(Server.java:389)
at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:68)
at org.apache.hadoop.http.HttpServer2.start(HttpServer2.java:1134)
at org.apache.hadoop.hdfs.server.datanode.web.DatanodeHttpServer.<init>(DatanodeHttpServer.java:141)
at org.apache.hadoop.hdfs.server.datanode.DataNode.startInfoServer(DataNode.java:957)
at org.apache.hadoop.hdfs.server.datanode.DataNode.startDataNode(DataNode.java:1417)
at org.apache.hadoop.hdfs.server.datanode.DataNode.<init>(DataNode.java:500)
at org.apache.hadoop.hdfs.server.datanode.DataNode.makeInstance(DataNode.java:2782)
at org.apache.hadoop.hdfs.server.datanode.DataNode.instantiateDataNode(DataNode.java:2690)
at org.apache.hadoop.hdfs.server.datanode.DataNode.createDataNode(DataNode.java:2732)
at org.apache.hadoop.hdfs.server.datanode.DataNode.secureMain(DataNode.java:2876)
at org.apache.hadoop.hdfs.server.datanode.DataNode.main(DataNode.java:2900)
2019-07-22 12:44:59,935 INFO server.AbstractConnector (AbstractConnector.java:doStart(278)) - Started ServerConnector@53812a9b{HTTP/1.1,[http/1.1]}{localhost:35953}
2019-07-22 12:44:59,935 INFO server.Server (Server.java:doStart(414)) - Started @2100ms
2019-07-22 12:44:59,939 INFO server.AbstractConnector (AbstractConnector.java:doStop(318)) - Stopped ServerConnector@53812a9b{HTTP/1.1,[http/1.1]}{localhost:0}
2019-07-22 12:44:59,939 INFO handler.ContextHandler (ContextHandler.java:doStop(910)) - Stopped o.e.j.w.WebAppContext@cb191ca{/,null,UNAVAILABLE}{/datanode}
2019-07-22 12:44:59,944 INFO datanode.DataNode (DataNode.java:shutdown(2134)) - Shutdown complete.
2019-07-22 12:44:59,944 ERROR datanode.DataNode (DataNode.java:secureMain(2883)) - Exception in secureMain
java.io.IOException: Problem starting http server
at org.apache.hadoop.http.HttpServer2.start(HttpServer2.java:1165)
at org.apache.hadoop.hdfs.server.datanode.web.DatanodeHttpServer.<init>(DatanodeHttpServer.java:141)
at org.apache.hadoop.hdfs.server.datanode.DataNode.startInfoServer(DataNode.java:957)
at org.apache.hadoop.hdfs.server.datanode.DataNode.startDataNode(DataNode.java:1417)
at org.apache.hadoop.hdfs.server.datanode.DataNode.<init>(DataNode.java:500)
at org.apache.hadoop.hdfs.server.datanode.DataNode.makeInstance(DataNode.java:2782)
at org.apache.hadoop.hdfs.server.datanode.DataNode.instantiateDataNode(DataNode.java:2690)
at org.apache.hadoop.hdfs.server.datanode.DataNode.createDataNode(DataNode.java:2732)
at org.apache.hadoop.hdfs.server.datanode.DataNode.secureMain(DataNode.java:2876)
at org.apache.hadoop.hdfs.server.datanode.DataNode.main(DataNode.java:2900)
Caused by: java.lang.NullPointerException
at org.eclipse.jetty.util.IO.delete(IO.java:344)
at org.eclipse.jetty.webapp.WebInfConfiguration.deconfigure(WebInfConfiguration.java:195)
at org.eclipse.jetty.webapp.WebAppContext.stopContext(WebAppContext.java:1380)
at org.eclipse.jetty.server.handler.ContextHandler.doStop(ContextHandler.java:880)
at org.eclipse.jetty.servlet.ServletContextHandler.doStop(ServletContextHandler.java:272)
at org.eclipse.jetty.webapp.WebAppContext.doStop(WebAppContext.java:546)
at org.eclipse.jetty.util.component.AbstractLifeCycle.stop(AbstractLifeCycle.java:89)
at org.eclipse.jetty.util.component.ContainerLifeCycle.stop(ContainerLifeCycle.java:142)
at org.eclipse.jetty.util.component.ContainerLifeCycle.doStop(ContainerLifeCycle.java:160)
at org.eclipse.jetty.server.handler.AbstractHandler.doStop(AbstractHandler.java:73)
at org.eclipse.jetty.util.component.AbstractLifeCycle.stop(AbstractLifeCycle.java:89)
at org.eclipse.jetty.util.component.ContainerLifeCycle.stop(ContainerLifeCycle.java:142)
at org.eclipse.jetty.util.component.ContainerLifeCycle.doStop(ContainerLifeCycle.java:160)
at org.eclipse.jetty.server.handler.AbstractHandler.doStop(AbstractHandler.java:73)
at org.eclipse.jetty.server.Server.doStop(Server.java:493)
at org.eclipse.jetty.util.component.AbstractLifeCycle.stop(AbstractLifeCycle.java:89)
at org.apache.hadoop.http.HttpServer2.start(HttpServer2.java:1155)
... 9 more
2019-07-22 12:44:59,946 INFO util.ExitUtil (ExitUtil.java:terminate(210)) - Exiting with status 1: java.io.IOException: Problem starting http server
2019-07-22 12:44:59,949 INFO datanode.DataNode (LogAdapter.java:info(51)) - SHUTDOWN_MSG:
/************************************************************
SHUTDOWN_MSG: Shutting down DataNode at datanode05/10.49.194.175
************************************************************/
Created on 02-20-2020 03:35 AM - edited 02-20-2020 03:38 AM
Hello @suresh_ctr ,