Support Questions
Find answers, ask questions, and share your expertise

http hiveserver2 is failing after 5 minutes

Solved Go to solution

http hiveserver2 is failing after 5 minutes

Contributor
 ERROR [main]: metastore.HiveMetaStore (HiveMetaStore.java:init(535)) - error in Metrics init: java.lang.reflect.InvocationTargetException null
java.lang.reflect.InvocationTargetException
        at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
        at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
        at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
        at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
        at org.apache.hadoop.hive.common.metrics.common.MetricsFactory.init(MetricsFactory.java:42)
        at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.init(HiveMetaStore.java:532)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at org.apache.hadoop.hive.metastore.RetryingHMSHandler.invokeInternal(RetryingHMSHandler.java:147)
        at org.apache.hadoop.hive.metastore.RetryingHMSHandler.invoke(RetryingHMSHandler.java:105)
        at org.apache.hadoop.hive.metastore.RetryingHMSHandler.<init>(RetryingHMSHandler.java:78)
        at org.apache.hadoop.hive.metastore.RetryingHMSHandler.getProxy(RetryingHMSHandler.java:91)
        at org.apache.hadoop.hive.metastore.HiveMetaStore.newRetryingHMSHandler(HiveMetaStore.java:6364)
        at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.<init>(HiveMetaStoreClient.java:205)
        at org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.<init>(SessionHiveMetaStoreClient.java:76)
        at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
        at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
        at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
        at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
        at org.apache.hadoop.hive.metastore.MetaStoreUtils.newInstance(MetaStoreUtils.java:1564)
        at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.<init>(RetryingMetaStoreClient.java:92)
        at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:138)
        at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:110)
        at org.apache.hadoop.hive.ql.metadata.Hive.createMetaStoreClient(Hive.java:3488)
        at org.apache.hadoop.hive.ql.metadata.Hive.getMSC(Hive.java:3520)
        at org.apache.hadoop.hive.ql.session.SessionState.start(SessionState.java:528)
        at org.apache.hive.service.cli.CLIService.applyAuthorizationConfigPolicy(CLIService.java:130)
        at org.apache.hive.service.cli.CLIService.init(CLIService.java:115)
        at org.apache.hive.service.CompositeService.init(CompositeService.java:59)
        at org.apache.hive.service.server.HiveServer2.init(HiveServer2.java:122)
        at org.apache.hive.service.server.HiveServer2.startHiveServer2(HiveServer2.java:474)
        at org.apache.hive.service.server.HiveServer2.access$700(HiveServer2.java:87)
        at org.apache.hive.service.server.HiveServer2$StartOptionExecutor.execute(HiveServer2.java:720)
        at org.apache.hive.service.server.HiveServer2.main(HiveServer2.java:593)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
Caused by: java.lang.AbstractMethodError: org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink.init(Lorg/apache/phoenix/shaded/org/apache/commons/configuration/SubsetConfiguration;)V
        at org.apache.hadoop.metrics2.impl.MetricsConfig.getPlugin(MetricsConfig.java:199)
        at org.apache.hadoop.metrics2.impl.MetricsSystemImpl.newSink(MetricsSystemImpl.java:529)
        at org.apache.hadoop.metrics2.impl.MetricsSystemImpl.configureSinks(MetricsSystemImpl.java:501)
        at org.apache.hadoop.metrics2.impl.MetricsSystemImpl.configure(MetricsSystemImpl.java:480)
        at org.apache.hadoop.metrics2.impl.MetricsSystemImpl.start(MetricsSystemImpl.java:189)
        at org.apache.hadoop.metrics2.impl.MetricsSystemImpl.init(MetricsSystemImpl.java:164)
        at org.apache.hadoop.metrics2.lib.DefaultMetricsSystem.init(DefaultMetricsSystem.java:54)
        at org.apache.hadoop.metrics2.lib.DefaultMetricsSystem.initialize(DefaultMetricsSystem.java:50)
        at org.apache.hadoop.hive.common.metrics.metrics2.CodahaleMetrics.initReporting(CodahaleMetrics.java:377)
        at org.apache.hadoop.hive.common.metrics.metrics2.CodahaleMetrics.<init>(CodahaleMetrics.java:199)
        ... 42 more


 WARN  [main]: server.HiveServer2 (HiveServer2.java:startHiveServer2(508)) - Error starting HiveServer2 on attempt 2, will retry in 60 seconds
java.lang.NoClassDefFoundError: org/apache/hadoop/yarn/webapp/YarnJacksonJaxbJsonProvider
        at org.apache.hadoop.yarn.client.api.impl.TimelineClientImpl.serviceInit(TimelineClientImpl.java:268)
        at org.apache.hadoop.service.AbstractService.init(AbstractService.java:163)
        at org.apache.hadoop.yarn.client.api.impl.YarnClientImpl.serviceInit(YarnClientImpl.java:169)
        at org.apache.hadoop.service.AbstractService.init(AbstractService.java:163)
        at org.apache.tez.client.TezYarnClient.init(TezYarnClient.java:46)
        at org.apache.tez.client.TezClient.start(TezClient.java:325)
        at org.apache.hadoop.hive.ql.exec.tez.TezSessionState.open(TezSessionState.java:197)
        at org.apache.hadoop.hive.ql.exec.tez.TezSessionState.open(TezSessionState.java:116)
        at org.apache.hadoop.hive.ql.exec.tez.TezSessionPoolManager.startPool(TezSessionPoolManager.java:76)
        at org.apache.hive.service.server.HiveServer2.startHiveServer2(HiveServer2.java:488)
        at org.apache.hive.service.server.HiveServer2.access$700(HiveServer2.java:87)
        at org.apache.hive.service.server.HiveServer2$StartOptionExecutor.execute(HiveServer2.java:720)
        at org.apache.hive.service.server.HiveServer2.main(HiveServer2.java:593)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at org.apache.hadoop.util.RunJar.run(RunJar.java:221)
        at org.apache.hadoop.util.RunJar.main(RunJar.java:136)






1 ACCEPTED SOLUTION

Accepted Solutions
Highlighted

Re: http hiveserver2 is failing after 5 minutes

Contributor

Found some custom jar files in the phoenix lib folder.. Deleting those jar files from there fixed the issue. Thanks @jay

View solution in original post

4 REPLIES 4
Highlighted

Re: http hiveserver2 is failing after 5 minutes

Super Mentor

@Sudheer Velagapudi

We see the error:

Caused by: java.lang.AbstractMethodError: org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink.init(Lorg/apache/phoenix/shaded/org/apache/commons/configuration/SubsetConfiguration;)V
  at org.apache.hadoop.metrics2.impl.MetricsConfig.getPlugin(MetricsConfig.java:199)
  at org.apache.hadoop.metrics2.impl.MetricsSystemImpl.newSink(MetricsSystemImpl.java:529)

.


As the error seems to be coming from the "HadoopTimelineMetricsSink" so can you pelase chekc the HiveServer2 host has the correct sink version installed?

# rpm -qa | grep ambari-metrics-hadoop-sink
# rpm -qa | grep ambari


.


Ideally the ambari-metrics-hadoop-sink version and the ambari version should be same. If it is not same then please check if there was any recent Ambari upgrade was performed or not? Because afetr ambari server upgrade it is must to perform the Post Upgrade steps which involves the "metrics-hadoop-sink" upgrade.

Please see: https://docs.hortonworks.com/HDPDocuments/Ambari-2.6.0.0/bk_ambari-upgrade/content/post_ambari_upgra...

https://docs.hortonworks.com/HDPDocuments/Ambari-2.6.0.0/bk_ambari-upgrade/content/upgrade_ambari_me...

.

Highlighted

Re: http hiveserver2 is failing after 5 minutes

Contributor

It looks they both are in the same version.. And we haven't upgraded anything. I got this error just after restarting hiverserver2.

 rpm -qa | grep ambari-metrics-hadoop-sink
ambari-metrics-hadoop-sink-2.5.1.0-159.x86_64
rpm -qa | grep ambari
ambari-agent-2.5.1.0-159.x86_64
ambari-metrics-monitor-2.5.1.0-159.x86_64
ambari-infra-solr-client-2.5.1.0-159.noarch
ambari-metrics-hadoop-sink-2.5.1.0-159.x86_64
Highlighted

Re: http hiveserver2 is failing after 5 minutes

Contributor

Found some custom jar files in the phoenix lib folder.. Deleting those jar files from there fixed the issue. Thanks @jay

View solution in original post

Highlighted

Re: http hiveserver2 is failing after 5 minutes

New Contributor

HI Team,

Even i was getting the following error But the issue is not related to Jar it seems. Please find th error log

2019-01-11 00:00:14,205 INFO [HiveServer2-HttpHandler-Pool: Thread-385]: thrift.ThriftHttpServlet (ThriftHttpServlet.java:doPost(145)) - Could not validate cookie sent, will try to generate a new cookie 2019-01-11 00:00:14,211 INFO [HiveServer2-HttpHandler-Pool: Thread-385]: thrift.ThriftHttpServlet (ThriftHttpServlet.java:doKerberosAuth(398)) - Failed to authenticate with http/_HOST kerberos principal, trying with hive/_HOST kerberos principal 2019-01-11 00:00:14,234 INFO [HiveServer2-HttpHandler-Pool: Thread-385]: thrift.ThriftHttpServlet (ThriftHttpServlet.java:doPost(204)) - Cookie added for clientUserName ambari-qa 2019-01-11 00:00:14,235 INFO [HiveServer2-HttpHandler-Pool: Thread-385]: thrift.ThriftCLIService (ThriftCLIService.java:OpenSession(316)) - Client protocol version: HIVE_CLI_SERVICE_PROTOCOL_V8 2019-01-11 00:00:14,255 WARN [HiveServer2-HttpHandler-Pool: Thread-385]: impl.MetricsSystemImpl (MetricsSystemImpl.java:init(152)) - hiveserver2 metrics system already initialized! 2019-01-11 00:00:14,260 ERROR [HiveServer2-HttpHandler-Pool: Thread-385]: metastore.HiveMetaStore (HiveMetaStore.java:init(446)) - error in Metrics init: java.lang.reflect.InvocationTargetException null java.lang.reflect.InvocationTargetException at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hive.common.metrics.common.MetricsFactory.init(MetricsFactory.java:42) at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.init(HiveMetaStore.java:443) at sun.reflect.GeneratedMethodAccessor43.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.hadoop.hive.metastore.RetryingHMSHandler.invokeInternal(RetryingHMSHandler.java:147) at org.apache.hadoop.hive.metastore.RetryingHMSHandler.invoke(RetryingHMSHandler.java:105) at org.apache.hadoop.hive.metastore.RetryingHMSHandler.<init>(RetryingHMSHandler.java:78) at org.apache.hadoop.hive.metastore.RetryingHMSHandler.getProxy(RetryingHMSHandler.java:91) at org.apache.hadoop.hive.metastore.HiveMetaStore.newRetryingHMSHandler(HiveMetaStore.java:6402) at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.<init>(HiveMetaStoreClient.java:206) at org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.<init>(SessionHiveMetaStoreClient.java:76) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hive.metastore.MetaStoreUtils.newInstance(MetaStoreUtils.java:1564) at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.<init>(RetryingMetaStoreClient.java:92) at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:138) at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:110) at org.apache.hadoop.hive.ql.metadata.Hive.createMetaStoreClient(Hive.java:3526) at org.apache.hadoop.hive.ql.metadata.Hive.getMSC(Hive.java:3558) at org.apache.hadoop.hive.ql.metadata.Hive.getDelegationToken(Hive.java:3684) at org.apache.hive.service.cli.CLIService.getDelegationTokenFromMetaStore(CLIService.java:539) at org.apache.hive.service.cli.thrift.ThriftCLIService.getDelegationToken(ThriftCLIService.java:442) at org.apache.hive.service.cli.thrift.ThriftCLIService.getSessionHandle(ThriftCLIService.java:423) at org.apache.hive.service.cli.thrift.ThriftCLIService.OpenSession(ThriftCLIService.java:319) at org.apache.hive.service.cli.thrift.TCLIService$Processor$OpenSession.getResult(TCLIService.java:1317) at org.apache.hive.service.cli.thrift.TCLIService$Processor$OpenSession.getResult(TCLIService.java:1302) at org.apache.thrift.ProcessFunction.process(ProcessFunction.java:39) at org.apache.thrift.TBaseProcessor.process(TBaseProcessor.java:39) at org.apache.thrift.server.TServlet.doPost(TServlet.java:83) at org.apache.hive.service.cli.thrift.ThriftHttpServlet.doPost(ThriftHttpServlet.java:206) at javax.servlet.http.HttpServlet.service(HttpServlet.java:727) at javax.servlet.http.HttpServlet.service(HttpServlet.java:820) at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:565) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:479) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:225) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1031) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:406) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:186) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:965) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:117) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:111) at org.eclipse.jetty.server.Server.handle(Server.java:349) at org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:449) at org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.content(AbstractHttpConnection.java:925) at org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:857)

Don't have an account?