Support Questions
Find answers, ask questions, and share your expertise
cancel
Turn on suggestions
Auto-suggest helps you quickly narrow down your search results by suggesting possible matches as you type.
Showing results for 
Search instead for 
Did you mean: 
Advanced Search

http hiveserver2 is failing after 5 minutes

Solved Go to solution

http hiveserver2 is failing after 5 minutes

Labels:
itsssudheer
Contributor

Created ‎01-04-2018 09:42 PM 

 ERROR [main]: metastore.HiveMetaStore (HiveMetaStore.java:init(535)) - error in Metrics init: java.lang.reflect.InvocationTargetException null
java.lang.reflect.InvocationTargetException
        at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
        at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
        at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
        at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
        at org.apache.hadoop.hive.common.metrics.common.MetricsFactory.init(MetricsFactory.java:42)
        at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.init(HiveMetaStore.java:532)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at org.apache.hadoop.hive.metastore.RetryingHMSHandler.invokeInternal(RetryingHMSHandler.java:147)
        at org.apache.hadoop.hive.metastore.RetryingHMSHandler.invoke(RetryingHMSHandler.java:105)
        at org.apache.hadoop.hive.metastore.RetryingHMSHandler.<init>(RetryingHMSHandler.java:78)
        at org.apache.hadoop.hive.metastore.RetryingHMSHandler.getProxy(RetryingHMSHandler.java:91)
        at org.apache.hadoop.hive.metastore.HiveMetaStore.newRetryingHMSHandler(HiveMetaStore.java:6364)
        at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.<init>(HiveMetaStoreClient.java:205)
        at org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.<init>(SessionHiveMetaStoreClient.java:76)
        at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
        at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
        at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
        at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
        at org.apache.hadoop.hive.metastore.MetaStoreUtils.newInstance(MetaStoreUtils.java:1564)
        at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.<init>(RetryingMetaStoreClient.java:92)
        at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:138)
        at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:110)
        at org.apache.hadoop.hive.ql.metadata.Hive.createMetaStoreClient(Hive.java:3488)
        at org.apache.hadoop.hive.ql.metadata.Hive.getMSC(Hive.java:3520)
        at org.apache.hadoop.hive.ql.session.SessionState.start(SessionState.java:528)
        at org.apache.hive.service.cli.CLIService.applyAuthorizationConfigPolicy(CLIService.java:130)
        at org.apache.hive.service.cli.CLIService.init(CLIService.java:115)
        at org.apache.hive.service.CompositeService.init(CompositeService.java:59)
        at org.apache.hive.service.server.HiveServer2.init(HiveServer2.java:122)
        at org.apache.hive.service.server.HiveServer2.startHiveServer2(HiveServer2.java:474)
        at org.apache.hive.service.server.HiveServer2.access$700(HiveServer2.java:87)
        at org.apache.hive.service.server.HiveServer2$StartOptionExecutor.execute(HiveServer2.java:720)
        at org.apache.hive.service.server.HiveServer2.main(HiveServer2.java:593)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
Caused by: java.lang.AbstractMethodError: org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink.init(Lorg/apache/phoenix/shaded/org/apache/commons/configuration/SubsetConfiguration;)V
        at org.apache.hadoop.metrics2.impl.MetricsConfig.getPlugin(MetricsConfig.java:199)
        at org.apache.hadoop.metrics2.impl.MetricsSystemImpl.newSink(MetricsSystemImpl.java:529)
        at org.apache.hadoop.metrics2.impl.MetricsSystemImpl.configureSinks(MetricsSystemImpl.java:501)
        at org.apache.hadoop.metrics2.impl.MetricsSystemImpl.configure(MetricsSystemImpl.java:480)
        at org.apache.hadoop.metrics2.impl.MetricsSystemImpl.start(MetricsSystemImpl.java:189)
        at org.apache.hadoop.metrics2.impl.MetricsSystemImpl.init(MetricsSystemImpl.java:164)
        at org.apache.hadoop.metrics2.lib.DefaultMetricsSystem.init(DefaultMetricsSystem.java:54)
        at org.apache.hadoop.metrics2.lib.DefaultMetricsSystem.initialize(DefaultMetricsSystem.java:50)
        at org.apache.hadoop.hive.common.metrics.metrics2.CodahaleMetrics.initReporting(CodahaleMetrics.java:377)
        at org.apache.hadoop.hive.common.metrics.metrics2.CodahaleMetrics.<init>(CodahaleMetrics.java:199)
        ... 42 more


 WARN  [main]: server.HiveServer2 (HiveServer2.java:startHiveServer2(508)) - Error starting HiveServer2 on attempt 2, will retry in 60 seconds
java.lang.NoClassDefFoundError: org/apache/hadoop/yarn/webapp/YarnJacksonJaxbJsonProvider
        at org.apache.hadoop.yarn.client.api.impl.TimelineClientImpl.serviceInit(TimelineClientImpl.java:268)
        at org.apache.hadoop.service.AbstractService.init(AbstractService.java:163)
        at org.apache.hadoop.yarn.client.api.impl.YarnClientImpl.serviceInit(YarnClientImpl.java:169)
        at org.apache.hadoop.service.AbstractService.init(AbstractService.java:163)
        at org.apache.tez.client.TezYarnClient.init(TezYarnClient.java:46)
        at org.apache.tez.client.TezClient.start(TezClient.java:325)
        at org.apache.hadoop.hive.ql.exec.tez.TezSessionState.open(TezSessionState.java:197)
        at org.apache.hadoop.hive.ql.exec.tez.TezSessionState.open(TezSessionState.java:116)
        at org.apache.hadoop.hive.ql.exec.tez.TezSessionPoolManager.startPool(TezSessionPoolManager.java:76)
        at org.apache.hive.service.server.HiveServer2.startHiveServer2(HiveServer2.java:488)
        at org.apache.hive.service.server.HiveServer2.access$700(HiveServer2.java:87)
        at org.apache.hive.service.server.HiveServer2$StartOptionExecutor.execute(HiveServer2.java:720)
        at org.apache.hive.service.server.HiveServer2.main(HiveServer2.java:593)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:498)
        at org.apache.hadoop.util.RunJar.run(RunJar.java:221)
        at org.apache.hadoop.util.RunJar.main(RunJar.java:136)
Reply
2,197 Views
0 Kudos
1 ACCEPTED SOLUTION

Accepted Solutions
Highlighted

Re: http hiveserver2 is failing after 5 minutes

itsssudheer
Contributor

Created ‎01-05-2018 02:34 PM

Found some custom jar files in the phoenix lib folder.. Deleting those jar files from there fixed the issue. Thanks @jay

View solution in original post

Reply
1,449 Views
0 Kudos
4 REPLIES 4
Highlighted

Re: http hiveserver2 is failing after 5 minutes

jsensharma
Super Mentor

Created ‎01-04-2018 10:48 PM

@Sudheer Velagapudi

We see the error:

Caused by: java.lang.AbstractMethodError: org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink.init(Lorg/apache/phoenix/shaded/org/apache/commons/configuration/SubsetConfiguration;)V
  at org.apache.hadoop.metrics2.impl.MetricsConfig.getPlugin(MetricsConfig.java:199)
  at org.apache.hadoop.metrics2.impl.MetricsSystemImpl.newSink(MetricsSystemImpl.java:529)

.


As the error seems to be coming from the "HadoopTimelineMetricsSink" so can you pelase chekc the HiveServer2 host has the correct sink version installed?

# rpm -qa | grep ambari-metrics-hadoop-sink
# rpm -qa | grep ambari


.


Ideally the ambari-metrics-hadoop-sink version and the ambari version should be same. If it is not same then please check if there was any recent Ambari upgrade was performed or not? Because afetr ambari server upgrade it is must to perform the Post Upgrade steps which involves the "metrics-hadoop-sink" upgrade.

Please see: https://docs.hortonworks.com/HDPDocuments/Ambari-2.6.0.0/bk_ambari-upgrade/content/post_ambari_upgra...

https://docs.hortonworks.com/HDPDocuments/Ambari-2.6.0.0/bk_ambari-upgrade/content/upgrade_ambari_me...

.

Reply
1,449 Views
0 Kudos
Highlighted

Re: http hiveserver2 is failing after 5 minutes

itsssudheer
Contributor

Created ‎01-05-2018 01:59 PM

It looks they both are in the same version.. And we haven't upgraded anything. I got this error just after restarting hiverserver2.

 rpm -qa | grep ambari-metrics-hadoop-sink
ambari-metrics-hadoop-sink-2.5.1.0-159.x86_64
rpm -qa | grep ambari
ambari-agent-2.5.1.0-159.x86_64
ambari-metrics-monitor-2.5.1.0-159.x86_64
ambari-infra-solr-client-2.5.1.0-159.noarch
ambari-metrics-hadoop-sink-2.5.1.0-159.x86_64
Reply
1,449 Views
0 Kudos
Highlighted

Re: http hiveserver2 is failing after 5 minutes

itsssudheer
Contributor

Created ‎01-05-2018 02:34 PM

Found some custom jar files in the phoenix lib folder.. Deleting those jar files from there fixed the issue. Thanks @jay

View solution in original post

Reply
1,450 Views
0 Kudos
Highlighted

Re: http hiveserver2 is failing after 5 minutes

venkatesh_ramis
New Contributor

Created ‎01-11-2019 02:48 AM

HI Team,

Even i was getting the following error But the issue is not related to Jar it seems. Please find th error log

2019-01-11 00:00:14,205 INFO [HiveServer2-HttpHandler-Pool: Thread-385]: thrift.ThriftHttpServlet (ThriftHttpServlet.java:doPost(145)) - Could not validate cookie sent, will try to generate a new cookie 2019-01-11 00:00:14,211 INFO [HiveServer2-HttpHandler-Pool: Thread-385]: thrift.ThriftHttpServlet (ThriftHttpServlet.java:doKerberosAuth(398)) - Failed to authenticate with http/_HOST kerberos principal, trying with hive/_HOST kerberos principal 2019-01-11 00:00:14,234 INFO [HiveServer2-HttpHandler-Pool: Thread-385]: thrift.ThriftHttpServlet (ThriftHttpServlet.java:doPost(204)) - Cookie added for clientUserName ambari-qa 2019-01-11 00:00:14,235 INFO [HiveServer2-HttpHandler-Pool: Thread-385]: thrift.ThriftCLIService (ThriftCLIService.java:OpenSession(316)) - Client protocol version: HIVE_CLI_SERVICE_PROTOCOL_V8 2019-01-11 00:00:14,255 WARN [HiveServer2-HttpHandler-Pool: Thread-385]: impl.MetricsSystemImpl (MetricsSystemImpl.java:init(152)) - hiveserver2 metrics system already initialized! 2019-01-11 00:00:14,260 ERROR [HiveServer2-HttpHandler-Pool: Thread-385]: metastore.HiveMetaStore (HiveMetaStore.java:init(446)) - error in Metrics init: java.lang.reflect.InvocationTargetException null java.lang.reflect.InvocationTargetException at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hive.common.metrics.common.MetricsFactory.init(MetricsFactory.java:42) at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.init(HiveMetaStore.java:443) at sun.reflect.GeneratedMethodAccessor43.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.hadoop.hive.metastore.RetryingHMSHandler.invokeInternal(RetryingHMSHandler.java:147) at org.apache.hadoop.hive.metastore.RetryingHMSHandler.invoke(RetryingHMSHandler.java:105) at org.apache.hadoop.hive.metastore.RetryingHMSHandler.<init>(RetryingHMSHandler.java:78) at org.apache.hadoop.hive.metastore.RetryingHMSHandler.getProxy(RetryingHMSHandler.java:91) at org.apache.hadoop.hive.metastore.HiveMetaStore.newRetryingHMSHandler(HiveMetaStore.java:6402) at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.<init>(HiveMetaStoreClient.java:206) at org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.<init>(SessionHiveMetaStoreClient.java:76) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hive.metastore.MetaStoreUtils.newInstance(MetaStoreUtils.java:1564) at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.<init>(RetryingMetaStoreClient.java:92) at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:138) at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:110) at org.apache.hadoop.hive.ql.metadata.Hive.createMetaStoreClient(Hive.java:3526) at org.apache.hadoop.hive.ql.metadata.Hive.getMSC(Hive.java:3558) at org.apache.hadoop.hive.ql.metadata.Hive.getDelegationToken(Hive.java:3684) at org.apache.hive.service.cli.CLIService.getDelegationTokenFromMetaStore(CLIService.java:539) at org.apache.hive.service.cli.thrift.ThriftCLIService.getDelegationToken(ThriftCLIService.java:442) at org.apache.hive.service.cli.thrift.ThriftCLIService.getSessionHandle(ThriftCLIService.java:423) at org.apache.hive.service.cli.thrift.ThriftCLIService.OpenSession(ThriftCLIService.java:319) at org.apache.hive.service.cli.thrift.TCLIService$Processor$OpenSession.getResult(TCLIService.java:1317) at org.apache.hive.service.cli.thrift.TCLIService$Processor$OpenSession.getResult(TCLIService.java:1302) at org.apache.thrift.ProcessFunction.process(ProcessFunction.java:39) at org.apache.thrift.TBaseProcessor.process(TBaseProcessor.java:39) at org.apache.thrift.server.TServlet.doPost(TServlet.java:83) at org.apache.hive.service.cli.thrift.ThriftHttpServlet.doPost(ThriftHttpServlet.java:206) at javax.servlet.http.HttpServlet.service(HttpServlet.java:727) at javax.servlet.http.HttpServlet.service(HttpServlet.java:820) at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:565) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:479) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:225) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1031) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:406) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:186) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:965) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:117) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:111) at org.eclipse.jetty.server.Server.handle(Server.java:349) at org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:449) at org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.content(AbstractHttpConnection.java:925) at org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:857)

Reply
1,449 Views
0 Kudos
Don't have an account?
Register