Created 01-04-2018 09:42 PM
ERROR [main]: metastore.HiveMetaStore (HiveMetaStore.java:init(535)) - error in Metrics init: java.lang.reflect.InvocationTargetException null java.lang.reflect.InvocationTargetException at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hive.common.metrics.common.MetricsFactory.init(MetricsFactory.java:42) at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.init(HiveMetaStore.java:532) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.hadoop.hive.metastore.RetryingHMSHandler.invokeInternal(RetryingHMSHandler.java:147) at org.apache.hadoop.hive.metastore.RetryingHMSHandler.invoke(RetryingHMSHandler.java:105) at org.apache.hadoop.hive.metastore.RetryingHMSHandler.<init>(RetryingHMSHandler.java:78) at org.apache.hadoop.hive.metastore.RetryingHMSHandler.getProxy(RetryingHMSHandler.java:91) at org.apache.hadoop.hive.metastore.HiveMetaStore.newRetryingHMSHandler(HiveMetaStore.java:6364) at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.<init>(HiveMetaStoreClient.java:205) at org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.<init>(SessionHiveMetaStoreClient.java:76) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hive.metastore.MetaStoreUtils.newInstance(MetaStoreUtils.java:1564) at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.<init>(RetryingMetaStoreClient.java:92) at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:138) at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:110) at org.apache.hadoop.hive.ql.metadata.Hive.createMetaStoreClient(Hive.java:3488) at org.apache.hadoop.hive.ql.metadata.Hive.getMSC(Hive.java:3520) at org.apache.hadoop.hive.ql.session.SessionState.start(SessionState.java:528) at org.apache.hive.service.cli.CLIService.applyAuthorizationConfigPolicy(CLIService.java:130) at org.apache.hive.service.cli.CLIService.init(CLIService.java:115) at org.apache.hive.service.CompositeService.init(CompositeService.java:59) at org.apache.hive.service.server.HiveServer2.init(HiveServer2.java:122) at org.apache.hive.service.server.HiveServer2.startHiveServer2(HiveServer2.java:474) at org.apache.hive.service.server.HiveServer2.access$700(HiveServer2.java:87) at org.apache.hive.service.server.HiveServer2$StartOptionExecutor.execute(HiveServer2.java:720) at org.apache.hive.service.server.HiveServer2.main(HiveServer2.java:593) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) Caused by: java.lang.AbstractMethodError: org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink.init(Lorg/apache/phoenix/shaded/org/apache/commons/configuration/SubsetConfiguration;)V at org.apache.hadoop.metrics2.impl.MetricsConfig.getPlugin(MetricsConfig.java:199) at org.apache.hadoop.metrics2.impl.MetricsSystemImpl.newSink(MetricsSystemImpl.java:529) at org.apache.hadoop.metrics2.impl.MetricsSystemImpl.configureSinks(MetricsSystemImpl.java:501) at org.apache.hadoop.metrics2.impl.MetricsSystemImpl.configure(MetricsSystemImpl.java:480) at org.apache.hadoop.metrics2.impl.MetricsSystemImpl.start(MetricsSystemImpl.java:189) at org.apache.hadoop.metrics2.impl.MetricsSystemImpl.init(MetricsSystemImpl.java:164) at org.apache.hadoop.metrics2.lib.DefaultMetricsSystem.init(DefaultMetricsSystem.java:54) at org.apache.hadoop.metrics2.lib.DefaultMetricsSystem.initialize(DefaultMetricsSystem.java:50) at org.apache.hadoop.hive.common.metrics.metrics2.CodahaleMetrics.initReporting(CodahaleMetrics.java:377) at org.apache.hadoop.hive.common.metrics.metrics2.CodahaleMetrics.<init>(CodahaleMetrics.java:199) ... 42 more WARN [main]: server.HiveServer2 (HiveServer2.java:startHiveServer2(508)) - Error starting HiveServer2 on attempt 2, will retry in 60 seconds java.lang.NoClassDefFoundError: org/apache/hadoop/yarn/webapp/YarnJacksonJaxbJsonProvider at org.apache.hadoop.yarn.client.api.impl.TimelineClientImpl.serviceInit(TimelineClientImpl.java:268) at org.apache.hadoop.service.AbstractService.init(AbstractService.java:163) at org.apache.hadoop.yarn.client.api.impl.YarnClientImpl.serviceInit(YarnClientImpl.java:169) at org.apache.hadoop.service.AbstractService.init(AbstractService.java:163) at org.apache.tez.client.TezYarnClient.init(TezYarnClient.java:46) at org.apache.tez.client.TezClient.start(TezClient.java:325) at org.apache.hadoop.hive.ql.exec.tez.TezSessionState.open(TezSessionState.java:197) at org.apache.hadoop.hive.ql.exec.tez.TezSessionState.open(TezSessionState.java:116) at org.apache.hadoop.hive.ql.exec.tez.TezSessionPoolManager.startPool(TezSessionPoolManager.java:76) at org.apache.hive.service.server.HiveServer2.startHiveServer2(HiveServer2.java:488) at org.apache.hive.service.server.HiveServer2.access$700(HiveServer2.java:87) at org.apache.hive.service.server.HiveServer2$StartOptionExecutor.execute(HiveServer2.java:720) at org.apache.hive.service.server.HiveServer2.main(HiveServer2.java:593) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.hadoop.util.RunJar.run(RunJar.java:221) at org.apache.hadoop.util.RunJar.main(RunJar.java:136)
Created 01-05-2018 02:34 PM
Found some custom jar files in the phoenix lib folder.. Deleting those jar files from there fixed the issue. Thanks @jay
Created 01-04-2018 10:48 PM
We see the error:
Caused by: java.lang.AbstractMethodError: org.apache.hadoop.metrics2.sink.timeline.HadoopTimelineMetricsSink.init(Lorg/apache/phoenix/shaded/org/apache/commons/configuration/SubsetConfiguration;)V at org.apache.hadoop.metrics2.impl.MetricsConfig.getPlugin(MetricsConfig.java:199) at org.apache.hadoop.metrics2.impl.MetricsSystemImpl.newSink(MetricsSystemImpl.java:529)
.
As the error seems to be coming from the "HadoopTimelineMetricsSink" so can you pelase chekc the HiveServer2 host has the correct sink version installed?
# rpm -qa | grep ambari-metrics-hadoop-sink # rpm -qa | grep ambari
.
Ideally the ambari-metrics-hadoop-sink version and the ambari version should be same. If it is not same then please check if there was any recent Ambari upgrade was performed or not? Because afetr ambari server upgrade it is must to perform the Post Upgrade steps which involves the "metrics-hadoop-sink" upgrade.
Please see: https://docs.hortonworks.com/HDPDocuments/Ambari-2.6.0.0/bk_ambari-upgrade/content/post_ambari_upgra...
https://docs.hortonworks.com/HDPDocuments/Ambari-2.6.0.0/bk_ambari-upgrade/content/upgrade_ambari_me...
.
Created 01-05-2018 01:59 PM
It looks they both are in the same version.. And we haven't upgraded anything. I got this error just after restarting hiverserver2.
rpm -qa | grep ambari-metrics-hadoop-sink ambari-metrics-hadoop-sink-2.5.1.0-159.x86_64 rpm -qa | grep ambari ambari-agent-2.5.1.0-159.x86_64 ambari-metrics-monitor-2.5.1.0-159.x86_64 ambari-infra-solr-client-2.5.1.0-159.noarch ambari-metrics-hadoop-sink-2.5.1.0-159.x86_64
Created 01-05-2018 02:34 PM
Found some custom jar files in the phoenix lib folder.. Deleting those jar files from there fixed the issue. Thanks @jay
Created 01-11-2019 02:48 AM
HI Team,
Even i was getting the following error But the issue is not related to Jar it seems. Please find th error log
2019-01-11 00:00:14,205 INFO [HiveServer2-HttpHandler-Pool: Thread-385]: thrift.ThriftHttpServlet (ThriftHttpServlet.java:doPost(145)) - Could not validate cookie sent, will try to generate a new cookie 2019-01-11 00:00:14,211 INFO [HiveServer2-HttpHandler-Pool: Thread-385]: thrift.ThriftHttpServlet (ThriftHttpServlet.java:doKerberosAuth(398)) - Failed to authenticate with http/_HOST kerberos principal, trying with hive/_HOST kerberos principal 2019-01-11 00:00:14,234 INFO [HiveServer2-HttpHandler-Pool: Thread-385]: thrift.ThriftHttpServlet (ThriftHttpServlet.java:doPost(204)) - Cookie added for clientUserName ambari-qa 2019-01-11 00:00:14,235 INFO [HiveServer2-HttpHandler-Pool: Thread-385]: thrift.ThriftCLIService (ThriftCLIService.java:OpenSession(316)) - Client protocol version: HIVE_CLI_SERVICE_PROTOCOL_V8 2019-01-11 00:00:14,255 WARN [HiveServer2-HttpHandler-Pool: Thread-385]: impl.MetricsSystemImpl (MetricsSystemImpl.java:init(152)) - hiveserver2 metrics system already initialized! 2019-01-11 00:00:14,260 ERROR [HiveServer2-HttpHandler-Pool: Thread-385]: metastore.HiveMetaStore (HiveMetaStore.java:init(446)) - error in Metrics init: java.lang.reflect.InvocationTargetException null java.lang.reflect.InvocationTargetException at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hive.common.metrics.common.MetricsFactory.init(MetricsFactory.java:42) at org.apache.hadoop.hive.metastore.HiveMetaStore$HMSHandler.init(HiveMetaStore.java:443) at sun.reflect.GeneratedMethodAccessor43.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.hadoop.hive.metastore.RetryingHMSHandler.invokeInternal(RetryingHMSHandler.java:147) at org.apache.hadoop.hive.metastore.RetryingHMSHandler.invoke(RetryingHMSHandler.java:105) at org.apache.hadoop.hive.metastore.RetryingHMSHandler.<init>(RetryingHMSHandler.java:78) at org.apache.hadoop.hive.metastore.RetryingHMSHandler.getProxy(RetryingHMSHandler.java:91) at org.apache.hadoop.hive.metastore.HiveMetaStore.newRetryingHMSHandler(HiveMetaStore.java:6402) at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.<init>(HiveMetaStoreClient.java:206) at org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient.<init>(SessionHiveMetaStoreClient.java:76) at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.hive.metastore.MetaStoreUtils.newInstance(MetaStoreUtils.java:1564) at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.<init>(RetryingMetaStoreClient.java:92) at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:138) at org.apache.hadoop.hive.metastore.RetryingMetaStoreClient.getProxy(RetryingMetaStoreClient.java:110) at org.apache.hadoop.hive.ql.metadata.Hive.createMetaStoreClient(Hive.java:3526) at org.apache.hadoop.hive.ql.metadata.Hive.getMSC(Hive.java:3558) at org.apache.hadoop.hive.ql.metadata.Hive.getDelegationToken(Hive.java:3684) at org.apache.hive.service.cli.CLIService.getDelegationTokenFromMetaStore(CLIService.java:539) at org.apache.hive.service.cli.thrift.ThriftCLIService.getDelegationToken(ThriftCLIService.java:442) at org.apache.hive.service.cli.thrift.ThriftCLIService.getSessionHandle(ThriftCLIService.java:423) at org.apache.hive.service.cli.thrift.ThriftCLIService.OpenSession(ThriftCLIService.java:319) at org.apache.hive.service.cli.thrift.TCLIService$Processor$OpenSession.getResult(TCLIService.java:1317) at org.apache.hive.service.cli.thrift.TCLIService$Processor$OpenSession.getResult(TCLIService.java:1302) at org.apache.thrift.ProcessFunction.process(ProcessFunction.java:39) at org.apache.thrift.TBaseProcessor.process(TBaseProcessor.java:39) at org.apache.thrift.server.TServlet.doPost(TServlet.java:83) at org.apache.hive.service.cli.thrift.ThriftHttpServlet.doPost(ThriftHttpServlet.java:206) at javax.servlet.http.HttpServlet.service(HttpServlet.java:727) at javax.servlet.http.HttpServlet.service(HttpServlet.java:820) at org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:565) at org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:479) at org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:225) at org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1031) at org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:406) at org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:186) at org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:965) at org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:117) at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:111) at org.eclipse.jetty.server.Server.handle(Server.java:349) at org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:449) at org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.content(AbstractHttpConnection.java:925) at org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:857)