Phoenix issue : org.apache.phoenix.exception.PhoenixIOException: org.apache.hadoop.hbase.MasterNotRunningException: Can't get connection to ZooKeeper

[#1]

This program we are submitting from Eclipse local and Phoenix store in remote server.
---------------------------------------------------------------------------------------

Java code: 


public static String driverName = "org.apache.phoenix.jdbc.PhoenixDriver";
try {
Class.forName(driverName); 
System.out.println("In try");
 conn = DriverManager.getConnection("jdbc:phoenix:10.40.17.183,10.40.17.155,10.40.17.129:2181:/hbase","","");
System.out.println("Connected");
conn.createStatement().execute("create table IF NOT EXISTS TEST0702(mykey integer not null primary key, asset_name varchar(50),item_name varchar(50),country_org_name varchar(100),branch_org_name varchar(100),dc_name varchar(100),building_name varchar(100),floor_desc varchar(500),cubicle_id varchar(500),ws_no varchar(500),iou_name varchar(500),monthly_count integer,month_year varchar(200),project_no varchar(200),project_name varchar(500),project_id varchar(200),project_status varchar(200),completion_date varchar(200),start_date varchar(200))"); conn.commit(); 
System.out.println("Table Created!!");
 }

Console Message :

In try log4j:WARN No appenders could be found for logger (org.apache.hadoop.conf.Configuration.deprecation). log4j:WARN Please initialize the log4j system properly. log4j:WARN See http://logging.apache.org/log4j/1.2/faq.html#noconfig for more info.

org.apache.phoenix.exception.PhoenixIOException: org.apache.hadoop.hbase.MasterNotRunningException: Can't get connection to ZooKeeper: KeeperErrorCode = OperationTimeout

at org.apache.phoenix.util.ServerUtil.parseServerException(ServerUtil.java:108)

at org.apache.phoenix.query.ConnectionQueryServicesImpl.ensureTableCreated(ConnectionQueryServicesImpl.java:890)

at org.apache.phoenix.query.ConnectionQueryServicesImpl.createTable(ConnectionQueryServicesImpl.java:1224)

at org.apache.phoenix.query.DelegateConnectionQueryServices.createTable(DelegateConnectionQueryServices.java:113)

at org.apache.phoenix.schema.MetaDataClient.createTableInternal(MetaDataClient.java:1937)

at org.apache.phoenix.schema.MetaDataClient.createTable(MetaDataClient.java:751)

at org.apache.phoenix.compile.CreateTableCompiler$2.execute(CreateTableCompiler.java:186)

at org.apache.phoenix.jdbc.PhoenixStatement$2.call(PhoenixStatement.java:314)

at org.apache.phoenix.jdbc.PhoenixStatement$2.call(PhoenixStatement.java:306)

at org.apache.phoenix.call.CallRunner.run(CallRunner.java:53)

at org.apache.phoenix.jdbc.PhoenixStatement.executeMutation(PhoenixStatement.java:304)

at org.apache.phoenix.jdbc.PhoenixStatement.executeUpdate(PhoenixStatement.java:1363)

at org.apache.phoenix.query.ConnectionQueryServicesImpl$12.call(ConnectionQueryServicesImpl.java:1911)

at org.apache.phoenix.query.ConnectionQueryServicesImpl$12.call(ConnectionQueryServicesImpl.java:1880)

at org.apache.phoenix.util.PhoenixContextExecutor.call(PhoenixContextExecutor.java:77)

at org.apache.phoenix.query.ConnectionQueryServicesImpl.init(ConnectionQueryServicesImpl.java:1880)

at org.apache.phoenix.jdbc.PhoenixDriver.getConnectionQueryServices(PhoenixDriver.java:180)

at org.apache.phoenix.jdbc.PhoenixEmbeddedDriver.connect(PhoenixEmbeddedDriver.java:132)

at org.apache.phoenix.jdbc.PhoenixDriver.connect(PhoenixDriver.java:151)

at java.sql.DriverManager.getConnection(DriverManager.java:571)

at java.sql.DriverManager.getConnection(DriverManager.java:215)

at PhoenixConnection.getConnection(PhoenixConnection.java:38)

at PhoenixConnection.main(PhoenixConnection.java:17)

Caused by: org.apache.hadoop.hbase.MasterNotRunningException: org.apache.hadoop.hbase.MasterNotRunningException: Can't get connection to ZooKeeper: KeeperErrorCode = OperationTimeout

at org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation$StubMaker.makeStub(HConnectionManager.java:1698)

at org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation$MasterServiceStubMaker.makeStub(HConnectionManager.java:1724)

at org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getKeepAliveMasterService(HConnectionManager.java:1931)

at org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.getHTableDescriptor(HConnectionManager.java:2732)

at org.apache.hadoop.hbase.client.HBaseAdmin.getTableDescriptor(HBaseAdmin.java:426)

at org.apache.hadoop.hbase.client.HBaseAdmin.getTableDescriptor(HBaseAdmin.java:431)

at org.apache.phoenix.query.ConnectionQueryServicesImpl.ensureTableCreated(ConnectionQueryServicesImpl.java:824)

... 21 more

Caused by: org.apache.hadoop.hbase.MasterNotRunningException: Can't get connection to ZooKeeper: KeeperErrorCode = OperationTimeout

at org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.checkIfBaseNodeAvailable(HConnectionManager.java:934)

at org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.access$600(HConnectionManager.java:597)

at org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation$StubMaker.makeStubNoRetries(HConnectionManager.java:1624)

at org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation$StubMaker.makeStub(HConnectionManager.java:1670)

... 27 more

Caused by: org.apache.zookeeper.KeeperException$OperationTimeoutException: KeeperErrorCode = OperationTimeout

at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.checkZk(RecoverableZooKeeper.java:145)

at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:222)

at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:481)

at org.apache.hadoop.hbase.client.HConnectionManager$HConnectionImplementation.checkIfBaseNodeAvailable(HConnectionManager.java:923)

... 30 more

[#2]

Josh Elser Please guide us

[#3]

Check that HBase master is up, is there Satan's by master available? If not, add one. Then make sure all ports for HBase are open, if you are in cloud cluster, open endpoints and use public IPs.

The doc below is from an old HDP release, I'm looking for equivalent in HDP 2.x. all you have to make sure is that 60000 ports in HBase switched to 16000. Zookeeper ports are fine.

http://docs.hortonworks.com/HDPDocuments/HDP1/HDP-1.2.0/bk_reference/content/reference_chap2_4.html

[#4]

As per jps HMaster running .. Please advice

Server 1 :

[hadoop@CDCUDHDPDB3 ~]$ sudo jps 5992 RunJar 23297 QuorumPeerMain 23781 Jps 13428 RunJar 13086 JobHistoryServer 2137 ResourceManager 22604 HMaster 7266 Master

Server 2 :

[hadoop@CDCUDHDPDB2 ~]$ sudo jps 19481 Worker 26187 HRegionServer 22091 NodeManager 30755 Jps 21979 DataNode 21797 NameNode 30017 QuorumPeerMain 30680 -- process information unavailable 21896 DFSZKFailoverController 26312 HMaster

Server 3:

[hadoop@CDCUDHDPDB1 ~]$ sudo jps 11820 Jps 8936 HRegionServer 18514 QuorumPeerMain 8365 NodeManager 6455 Worker 11695 -- process information unavailable 8254 DataNode

[#5]

Do we also need to open port 2888,3888 which is for peer and lead?

[#6]

ZooKeeper clients (your code and HBase) only communicate over 2181. ZooKeeper servers communicate among themselves on those ports.

[#7 · Accepted solution]

Did you include the proper hbase-site.xml on the classpath of your application?

[#8]

A quick check would be to ensure that you've used the right parent znode (as you've used /hbase) in the connection string, defined by 'zookeeper.znode.parent' inside hbase-site.xml. It usually looks like /hbase-secure or /hbase-unsecure, based on whether or not kerberos is enabled in the cluster.

And if that looks good, validate the connection string by using sqlline.py:

/usr/hdp/current/phoenix-client/bin/sqlline.py 10.40.17.183,10.40.17.155,10.40.17.129:2181:/hbase

[#9]

When we run from terminal its running fine . We are getting issue while submitting program from local (using eclipse) and phoenix is in these servers(10.40.17.183,10.40.17.155,10.40.17.129).

Can you please validate hbase-site.xml ( Do I need to place this file to hbase/conf and Phoenix/bin for all three servers/nodes)

<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<configuration>
<property>
<name>hbase.rootdir</name>
<value>hdfs://mycluster:8020/hbase</value>
</property>
<property>
<name>hbase.cluster.distributed</name>
<value>true</value>
</property>
<property>
<name>hbase.config.read.zookeeper.config</name>
<value>true</value>
</property>
<property>
<name>hbase.zookeeper.quorum</name>
<value>namenode1,namenode2,datanode2</value>
</property>
<property>
<name>hbase.zookeeper.property.dataDir</name>
<value>/DBHADOOP/installations/zookeeper-3.4.5/zookeeper/zk1</value>
</property>
<property>
<name>hbase.client.keyvalue.maxsize</name>
<value>0</value>
</property>
<property>
<name>hbase.client.scanner.timeout.period</name>
<value>600000</value>
</property>
<property>
<name>hbase.master</name>
<value>mycluster:60000</value>
<description>The host and port the HBase master runs at.
</description>
</property>
<property>
<name>hbase.regionserver.port</name>
<value>60020</value>
<description>The host and port the HBase master runs at.
</description>
</property>


<property>
<name>zookeeper.znode.parent</name>
<value>/hbase</value>
</property>
	
<property>
<name>hbase.zookeeper.peerport</name>
<value>2888</value>
</property>


<property>
<name>hbase.zookeeper.leaderport</name>
<value>3888</value>
</property>


<property>
<name>zookeeper.znode.clusterId</name>
<value>12345</value>
</property>


<property>
<name>hbase.zookeeper.property.clientPort</name>
<value>2181</value>
<description>Property from ZooKeeper's config zoo.cfg.
 The port at which the clients will connect.
</description>
</property>
<property>
  <name>hbase.rpc.timeout</name>
  <value>120000</value>
  <source>hbase-site.xml</source>
</property>
</configuration>