Support Questions

Find answers, ask questions, and share your expertise
Announcements
Celebrating as our community reaches 100,000 members! Thank you!

HCatalog and kerberos

avatar

I have a java application which reads hive metadata using hcatalog apis.

public static void main(String[] args) {		HCatClient hcatClient = null;try {			HiveConf hcatConf = new HiveConf();hcatConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://192.168.42.154:9083");hcatConf.set(HCatConstants.HCAT_HIVE_CLIENT_DISABLE_CACHE, "true");hcatClient = HCatClient.create(new Configuration(hcatConf));			List<String> dbs = hcatClient.listDatabaseNamesByPattern("*");for (String string : dbs) {				System.out.println(string);			}		} catch (Throwable t) {t.printStackTrace();		} finally {if (hcatClient != null)try {hcatClient.close();				} catch (HCatException e) {				}		}	}

I get the following exception on a cluster with Kerberos

org.apache.hive.hcatalog.common.HCatException : 9001 : Exception occurred while
processing HCat request : MetaException while listing db names. Cause :
MetaException(message:Got exception: org.apache.thrift.transport.TTransportException
java.net.SocketTimeoutException: Read timed out)org.apache.hive.hcatalog.common.HCatException
: 9001 : Exception occurred while processing HCat request : MetaException while
listing db names. Cause : MetaException(message:Got exception:
org.apache.thrift.transport.TTransportException java.net.SocketTimeoutException:
Read timed out)  at
org.apache.hive.hcatalog.api.HCatClientHMSImpl.listDatabaseNamesByPattern(HCatClientHMSImpl.java:68)
1 ACCEPTED SOLUTION

avatar

The below code worked.

@Shishir Saxena

package hadoop.test;

import java.util.List;

import org.apache.hadoop.conf.Configuration;

import org.apache.hadoop.hive.conf.HiveConf;

import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hive.hcatalog.api.HCatClient;
import org.apache.hive.hcatalog.api.HCatTable;
import org.apache.hive.hcatalog.common.HCatConstants;
import org.apache.hive.hcatalog.common.HCatException;
import org.apache.hive.hcatalog.data.schema.HCatFieldSchema;
import org.apache.hive.hcatalog.data.schema.HCatSchema;

public class ListDBs1 {

publicstaticvoid main(String[] args) {
		HCatClient hcatClient = null;
try {
			String principal ="hive/quickstart.cloudera@XXX.COM"; 
			String keytab = "E:\\apps\\metacenter_home\\hadoop\\hive.keytab";
			System.setProperty("sun.security.krb5.debug", "true");
			System.setProperty("java.security.krb5.conf", "E:\\apps\\hadoop\\krb5.conf");
			System.setProperty("java.security.auth.login.config", "E:\\apps\\hadoop\\jaas.conf");
			HiveConf hcatConf = new HiveConf();
hcatConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://server:9083");
hcatConf.set("hadoop.security.authentication", "kerberos"); 
hcatConf.set(HCatConstants.HCAT_HIVE_CLIENT_DISABLE_CACHE, "true");
hcatConf.setVar(HiveConf.ConfVars.METASTORE_KERBEROS_PRINCIPAL, principal);
hcatConf.setVar(HiveConf.ConfVars.METASTORE_KERBEROS_KEYTAB_FILE, keytab);
hcatConf.setVar(HiveConf.ConfVars.METASTORE_USE_THRIFT_SASL, "true");
			UserGroupInformation.setConfiguration(hcatConf); 
			UserGroupInformation.loginUserFromKeytab(principal, keytab);
hcatClient = HCatClient.create(new Configuration(hcatConf));
			HiveMetaStoreClient hiveMetastoreClient = new HiveMetaStoreClient(hcatConf);
			list(hcatClient,hiveMetastoreClient);
		} catch (Throwable t) {
t.printStackTrace();
		} finally {
if (hcatClient != null)
try {
hcatClient.close();
				} catch (HCatException e) {
				}
		}
	}
privatestaticvoid list(HCatClient hcatClient, HiveMetaStoreClient hiveMetastoreClient) throws Exception {
		List<String> dbs = hcatClient.listDatabaseNamesByPattern("*");
for (String db : dbs) {
			System.out.println(db);
			List<String> tables = hcatClient.listTableNamesByPattern(db, "*");
for (String tableString: tables) {
				HCatTable tbl = hcatClient.getTable(db, tableString);
				String tableType = tbl.getTabletype();
				String tableName = tbl.getTableName();
				System.out.println(tableType + " - " + tableName);
				System.out.println("Table Name is: " + tableName);
                System.out.println("Table Type is: " + tbl.getTabletype());
                System.out.println("Table Props are: " + tbl.getTblProps());
                List<HCatFieldSchema> fields = tbl.getCols();
for (HCatFieldSchema f: fields) {
                      System.out.println("Field Name is: " + f.getName());
                      System.out.println("Field Type String is: " + f.getTypeString());
                      System.out.println("Field Type Category is: " + f.getTypeString());
if (f.getCategory().equals(HCatFieldSchema.Category.STRUCT)) {
                            HCatSchema schema = f.getStructSubSchema();
                            List<String> structFields = schema.getFieldNames();
for (String fieldName: structFields) {
                                  System.out.println("Struct Field Name is: " + fieldName);                                               
                            }
                      }
                }  
if (tableType.equalsIgnoreCase("View") || tableType.equalsIgnoreCase("VIRTUAL_VIEW")) {
					org.apache.hadoop.hive.metastore.api.Table viewMetastoreObject = hiveMetastoreClient.getTable(db, tableName);
					String sql = viewMetastoreObject.getViewOriginalText();
					System.out.println(sql);
				}
			}
		}
	}
}

View solution in original post

14 REPLIES 14

avatar

This is the error I see in the hivemetastore.log

cmd=get_all_databases 2016-03-14 06:25:47,041 INFO [pool-5-thread-197]: metastore.HiveMetaStore (HiveMetaStore.java:newRawStore(590)) - 195: Opening raw store with implemenation class:org.apache.hadoop.hive.metastore.ObjectStore 2016-03-14 06:25:47,041 INFO [pool-5-thread-197]: metastore.ObjectStore (ObjectStore.java:initialize(290)) - ObjectStore, initialize called 2016-03-14 06:25:47,042 WARN [pool-5-thread-197]: metastore.MetaStoreDirectSql (MetaStoreDirectSql.java:determineDbType(160)) - DB Product name[PostgreSQL] obtained, but not used to determine db type. Falling back to using SQL to determine which db we're using 2016-03-14 06:25:47,044 INFO [pool-5-thread-197]: metastore.MetaStoreDirectSql (MetaStoreDirectSql.java:<init>(140)) - Using direct SQL, underlying DB is OTHER 2016-03-14 06:25:47,045 INFO [pool-5-thread-197]: metastore.ObjectStore (ObjectStore.java:setConf(273)) - Initialized ObjectStore 2016-03-14 06:26:03,614 ERROR [pool-5-thread-197]: server.TThreadPoolServer (TThreadPoolServer.java:run(296)) - Error occurred during processing of message. java.lang.RuntimeException: org.apache.thrift.transport.TTransportException: Peer indicated failure: GSS initiate failed at org.apache.thrift.transport.TSaslServerTransport$Factory.getTransport(TSaslServerTransport.java:219) at org.apache.hadoop.hive.thrift.HadoopThriftAuthBridge$Server$TUGIAssumingTransportFactory$1.run(HadoopThriftAuthBridge.java:739) at org.apache.hadoop.hive.thrift.HadoopThriftAuthBridge$Server$TUGIAssumingTransportFactory$1.run(HadoopThriftAuthBridge.java:736) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:360) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1637) at org.apache.hadoop.hive.thrift.HadoopThriftAuthBridge$Server$TUGIAssumingTransportFactory.getTransport(HadoopThriftAuthBridge.java:736) at org.apache.thrift.server.TThreadPoolServer$WorkerProcess.run(TThreadPoolServer.java:268) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Caused by: org.apache.thrift.transport.TTransportException: Peer indicated failure: GSS initiate failed at org.apache.thrift.transport.TSaslTransport.receiveSaslMessage(TSaslTransport.java:199) at org.apache.thrift.transport.TSaslServerTransport.handleSaslStartMessage(TSaslServerTransport.java:125) at org.apache.thrift.transport.TSaslTransport.open(TSaslTransport.java:271) at org.apache.thrift.transport.TSaslServerTransport.open(TSaslServerTransport.java:41) at org.apache.thrift.transport.TSaslServerTransport$Factory.getTransport(TSaslServerTransport.java:216) ... 10 more

avatar
@Rachna Bakhru

Please see this. https://community.hortonworks.com/content/kbentry/17648/access-kerberos-cluster-from-java-using-cach...

Do you kave keytab file for user that will be accessing cluster ? If yes, then you can use alternate approach of passing keytab and jaas file.

avatar

Yes we do have the keytab file.

hcatConf.setVar(HiveConf.ConfVars.METASTORE_KERBEROS_KEYTAB_FILE, keytab);

Now we get this error.

2016-03-14 13:32:35,223 ERROR [pool-5-thread-2]: server.TThreadPoolServer (TThreadPoolServer.java:run(296)) - Error occurred during processing of message.

java.lang.RuntimeException: org.apache.thrift.transport.TTransportException: Invalid status -128

at org.apache.thrift.transport.TSaslServerTransport$Factory.getTransport(TSaslServerTransport.java:219)

at org.apache.hadoop.hive.thrift.HadoopThriftAuthBridge$Server$TUGIAssumingTransportFactory$1.run(HadoopThriftAuthBridge.java:739)

at org.apache.hadoop.hive.thrift.HadoopThriftAuthBridge$Server$TUGIAssumingTransportFactory$1.run(HadoopThriftAuthBridge.java:736)

at java.security.AccessController.doPrivileged(Native Method)

at javax.security.auth.Subject.doAs(Subject.java:360)

at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1637)

at org.apache.hadoop.hive.thrift.HadoopThriftAuthBridge$Server$TUGIAssumingTransportFactory.getTransport(HadoopThriftAuthBridge.java:736)

at org.apache.thrift.server.TThreadPoolServer$WorkerProcess.run(TThreadPoolServer.java:268)

at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)

at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)

at java.lang.Thread.run(Thread.java:745)

avatar

when I set

hcatConf.setVar(HiveConf.ConfVars.METASTORE_USE_THRIFT_SASL, "true");

I get this error

SEVERE: org/apache/commons/configuration/Configuration java.lang.NoClassDefFoundError: org/apache/commons/configuration/Configuration at org.apache.hadoop.metrics2.lib.DefaultMetricsSystem.<init>(DefaultMetricsSystem.java:38) at org.apache.hadoop.metrics2.lib.DefaultMetricsSystem.<clinit>(DefaultMetricsSystem.java:36) at org.apache.hadoop.security.UserGroupInformation$UgiMetrics.create(UserGroupInformation.java:97) at org.apache.hadoop.security.UserGroupInformation.<clinit>(UserGroupInformation.java:190) at org.apache.hadoop.hive.shims.HadoopShimsSecure.getTokenStrForm(HadoopShimsSecure.java:455) at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.open(HiveMetaStoreClient.java:313) at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.<init>(HiveMetaStoreClient.java:214) at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.<init>(HiveMetaStoreClient.java:154)

shouldn't it use org.apache.hadoop.conf.Configuration?

avatar

This error was resolved by adding the commons-configuration-.x.x.jar

avatar

@Rachna Bakhru Is your problem fully resolved now ? As I understand, you made 2 changes in code

hcatConf.setVar(HiveConf.ConfVars.METASTORE_KERBEROS_KEYTAB_FILE, keytab);
hcatConf.setVar(HiveConf.ConfVars.METASTORE_USE_THRIFT_SASL, "true");

and added commons-configuration-.x.x.jar to your classes. Can you confirm, so this question can be closed.

avatar

No the problem isn't resolved yet.

avatar

We are currently getting this error...

16:28:11,820 INFO metastore:297 - Trying to connect to metastore with URI thrift://192.168.42.154:9083 16:28:11,851 ERROR TSaslTransport:296 - SASL negotiation failure javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] at com.sun.security.sasl.gsskerb.GssKrb5Client.evaluateChallenge(GssKrb5Client.java:212) at org.apache.thrift.transport.TSaslClientTransport.handleSaslStartMessage(TSaslClientTransport.java:94) at org.apache.thrift.transport.TSaslTransport.open(TSaslTransport.java:253) at org.apache.thrift.transport.TSaslClientTransport.open(TSaslClientTransport.java:37) at org.apache.hadoop.hive.thrift.client.TUGIAssumingTransport$1.run(TUGIAssumingTransport.java:52) at org.apache.hadoop.hive.thrift.client.TUGIAssumingTransport$1.run(TUGIAssumingTransport.java:49) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:415) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1408) at org.apache.hadoop.hive.thrift.client.TUGIAssumingTransport.open(TUGIAssumingTransport.java:49) at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.open(HiveMetaStoreClient.java:336) at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.<init>(HiveMetaStoreClient.java:214) at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.<init>(HiveMetaStoreClient.java:154) ...... Caused by: GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt) at sun.security.jgss.krb5.Krb5InitCredential.getInstance(Krb5InitCredential.java:147) at sun.security.jgss.krb5.Krb5MechFactory.getCredentialElement(Krb5MechFactory.java:121) at sun.security.jgss.krb5.Krb5MechFactory.getMechanismContext(Krb5MechFactory.java:187) at sun.security.jgss.GSSManagerImpl.getMechanismContext(GSSManagerImpl.java:223) at sun.security.jgss.GSSContextImpl.initSecContext(GSSContextImpl.java:212) at sun.security.jgss.GSSContextImpl.initSecContext(GSSContextImpl.java:179) at com.sun.security.sasl.gsskerb.GssKrb5Client.evaluateChallenge(GssKrb5Client.java:193)

avatar

Try these additional things. Create a jaas file with following configuration. And launch your java program with these additional options.

Client { 
com.sun.security.auth.module.Krb5LoginModule required 
useKeyTab=true 
useTicketCache=false 
renewTicket=true };
-Djava.security.auth.login.config="path-to-jaas-file" -Djava.security.krb5.conf="path-to-krb5.conf"