Our Community is getting an upgrade! To get everything ready for the relaunch, we’ll be placing the site in read-only mode starting September 21st.
We really appreciate your understanding while we get things set up behind the scenes. Catch up on all the exciting details about the move here.
Need help or have questions? Drop us a line at [email protected]
Created 03-08-2016 08:03 PM
I have a java application which reads hive metadata using hcatalog apis.
public static void main(String[] args) { HCatClient hcatClient = null;try { HiveConf hcatConf = new HiveConf();hcatConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://192.168.42.154:9083");hcatConf.set(HCatConstants.HCAT_HIVE_CLIENT_DISABLE_CACHE, "true");hcatClient = HCatClient.create(new Configuration(hcatConf)); List<String> dbs = hcatClient.listDatabaseNamesByPattern("*");for (String string : dbs) { System.out.println(string); } } catch (Throwable t) {t.printStackTrace(); } finally {if (hcatClient != null)try {hcatClient.close(); } catch (HCatException e) { } } }I get the following exception on a cluster with Kerberos
org.apache.hive.hcatalog.common.HCatException : 9001 : Exception occurred while processing HCat request : MetaException while listing db names. Cause : MetaException(message:Got exception: org.apache.thrift.transport.TTransportException java.net.SocketTimeoutException: Read timed out)org.apache.hive.hcatalog.common.HCatException : 9001 : Exception occurred while processing HCat request : MetaException while listing db names. Cause : MetaException(message:Got exception: org.apache.thrift.transport.TTransportException java.net.SocketTimeoutException: Read timed out) at org.apache.hive.hcatalog.api.HCatClientHMSImpl.listDatabaseNamesByPattern(HCatClientHMSImpl.java:68)
Created 07-05-2016 07:47 PM
The below code worked.
package hadoop.test;
import java.util.List;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hive.hcatalog.api.HCatClient;
import org.apache.hive.hcatalog.api.HCatTable;
import org.apache.hive.hcatalog.common.HCatConstants;
import org.apache.hive.hcatalog.common.HCatException;
import org.apache.hive.hcatalog.data.schema.HCatFieldSchema;
import org.apache.hive.hcatalog.data.schema.HCatSchema;
public class ListDBs1 {
publicstaticvoid main(String[] args) {HCatClient hcatClient = null;
try {String principal ="hive/[email protected]";
String keytab = "E:\\apps\\metacenter_home\\hadoop\\hive.keytab";
System.setProperty("sun.security.krb5.debug", "true"); System.setProperty("java.security.krb5.conf", "E:\\apps\\hadoop\\krb5.conf"); System.setProperty("java.security.auth.login.config", "E:\\apps\\hadoop\\jaas.conf");HiveConf hcatConf = new HiveConf();
hcatConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://server:9083");
hcatConf.set("hadoop.security.authentication", "kerberos"); hcatConf.set(HCatConstants.HCAT_HIVE_CLIENT_DISABLE_CACHE, "true");
hcatConf.setVar(HiveConf.ConfVars.METASTORE_KERBEROS_PRINCIPAL, principal);
hcatConf.setVar(HiveConf.ConfVars.METASTORE_KERBEROS_KEYTAB_FILE, keytab);
hcatConf.setVar(HiveConf.ConfVars.METASTORE_USE_THRIFT_SASL, "true");
UserGroupInformation.setConfiguration(hcatConf);
UserGroupInformation.loginUserFromKeytab(principal, keytab);
hcatClient = HCatClient.create(new Configuration(hcatConf));
HiveMetaStoreClient hiveMetastoreClient = new HiveMetaStoreClient(hcatConf);
list(hcatClient,hiveMetastoreClient);
} catch (Throwable t) {t.printStackTrace();
} finally {if (hcatClient != null)
try {hcatClient.close();
} catch (HCatException e) {}
}
}
privatestaticvoid list(HCatClient hcatClient, HiveMetaStoreClient hiveMetastoreClient) throws Exception { List<String> dbs = hcatClient.listDatabaseNamesByPattern("*");for (String db : dbs) {System.out.println(db);
List<String> tables = hcatClient.listTableNamesByPattern(db, "*");
for (String tableString: tables) {HCatTable tbl = hcatClient.getTable(db, tableString);
String tableType = tbl.getTabletype();
String tableName = tbl.getTableName();
System.out.println(tableType + " - " + tableName);
System.out.println("Table Name is: " + tableName); System.out.println("Table Type is: " + tbl.getTabletype()); System.out.println("Table Props are: " + tbl.getTblProps());List<HCatFieldSchema> fields = tbl.getCols();
for (HCatFieldSchema f: fields) { System.out.println("Field Name is: " + f.getName()); System.out.println("Field Type String is: " + f.getTypeString()); System.out.println("Field Type Category is: " + f.getTypeString());if (f.getCategory().equals(HCatFieldSchema.Category.STRUCT)) {HCatSchema schema = f.getStructSubSchema();
List<String> structFields = schema.getFieldNames();
for (String fieldName: structFields) { System.out.println("Struct Field Name is: " + fieldName); }
}
}
if (tableType.equalsIgnoreCase("View") || tableType.equalsIgnoreCase("VIRTUAL_VIEW")) {org.apache.hadoop.hive.metastore.api.Table viewMetastoreObject = hiveMetastoreClient.getTable(db, tableName);
String sql = viewMetastoreObject.getViewOriginalText();
System.out.println(sql);
}
}
}
}
}
Created 03-16-2016 07:42 PM
I am running my program from a windows machine.
I used
-Djava.security.auth.login.config="path-to-jaas-file" -Djava.security.krb5.conf="path-to-krb5.ini"
SEVERE: Error creating Hive objects: Could not connect to meta store using any of the URIs provided. Most recent failure: org.apache.thrift.transport.TTransportException: GSS initiate failed at org.apache.thrift.transport.TSaslTransport.sendAndThrowMessage(TSaslTransport.java:221) at org.apache.thrift.transport.TSaslTransport.open(TSaslTransport.java:297) at org.apache.thrift.transport.TSaslClientTransport.open(TSaslClientTransport.java:37) at org.apache.hadoop.hive.thrift.client.TUGIAssumingTransport$1.run(TUGIAssumingTransport.java:52) at org.apache.hadoop.hive.thrift.client.TUGIAssumingTransport$1.run(TUGIAssumingTransport.java:49) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:415) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1408) at org.apache.hadoop.hive.thrift.client.TUGIAssumingTransport.open(TUGIAssumingTransport.java:49) at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.open(HiveMetaStoreClient.java:336) at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.<init>(HiveMetaStoreClient.java:214) at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.<init>(HiveMetaStoreClient.java:154)
Error in hivemetastore.log
2016-03-16 13:31:09,808 ERROR [pool-5-thread-200]: server.TThreadPoolServer (TThreadPoolServer.java:run(296)) - Error occurred during processing of message. java.lang.RuntimeException: org.apache.thrift.transport.TTransportException: Peer indicated failure: GSS initiate failed at org.apache.thrift.transport.TSaslServerTransport$Factory.getTransport(TSaslServerTransport.java:219) at org.apache.hadoop.hive.thrift.HadoopThriftAuthBridge$Server$TUGIAssumingTransportFactory$1.run(HadoopThriftAuthBridge.java:739) at org.apache.hadoop.hive.thrift.HadoopThriftAuthBridge$Server$TUGIAssumingTransportFactory$1.run(HadoopThriftAuthBridge.java:736) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:360) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1637) at org.apache.hadoop.hive.thrift.HadoopThriftAuthBridge$Server$TUGIAssumingTransportFactory.getTransport(HadoopThriftAuthBridge.java:736) at org.apache.thrift.server.TThreadPoolServer$WorkerProcess.run(TThreadPoolServer.java:268) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:745) Caused by: org.apache.thrift.transport.TTransportException: Peer indicated failure: GSS initiate failed at org.apache.thrift.transport.TSaslTransport.receiveSaslMessage(TSaslTransport.java:199) at org.apache.thrift.transport.TSaslServerTransport.handleSaslStartMessage(TSaslServerTransport.java:125) at org.apache.thrift.transport.TSaslTransport.open(TSaslTransport.java:271) at org.apache.thrift.transport.TSaslServerTransport.open(TSaslServerTransport.java:41) at org.apache.thrift.transport.TSaslServerTransport$Factory.getTransport(TSaslServerTransport.java:216)
Created 03-17-2016 12:59 AM
Please reply in comments if it is not a new answer.
Here is some sample code to connect to a Kerberized cluster from JAVA program on Windows machine. Your steps should be similar.
1. Copy krb5.conf file to your Windows machine.
2. Copy different resource files to Windows. (core-site.xml,yarn-site.xml, hdfs-site.xml, hive-site.xml )
3. create a . jaas file with following configuration.
Client {
com.sun.security.auth.module.Krb5LoginModule required
useKeyTab=true
useTicketCache=false
renewTicket=true };
Change your login code as follows.
conf = new org.apache.hadoop.conf.Configuration();
try{
String principal = "<principal>";
String keytab = "<keytab location>";
conf.set("hadoop.security.authentication", "Kerberos");
conf.addResource(new Path("./core-site.xml"));
conf.addResource(new Path("./yarn-site.xml"));
conf.addResource(new Path("./hdfs-site.xml"));
UserGroupInformation.setConfiguration(conf);
UserGroupInformation.loginUserFromKeytab(principal, keytab);5. Then launch JAVA program with following parameters and specify paths for krb5.conf and .jaas file.
-Djava.security.auth.login.config="path-to-jaas-file" -Djava.security.krb5.conf="path-to-krb5.conf"
Created 03-21-2016 08:54 PM
Do I keep the original properties?
package com.dag.mc.biz.activelinx.emf.snapshot.hadoop;
//import javax.jdo.JDOException;
import java.util.List;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hive.hcatalog.api.HCatClient;
import org.apache.hive.hcatalog.api.HCatTable;
import org.apache.hive.hcatalog.common.HCatConstants;
import org.apache.hive.hcatalog.common.HCatException;
public class ListDBs {
/**
* @param args
*/
public static void main(String[] args) {
HCatClient hcatClient = null;
try {
String principal = "hive/[email protected]";
String keytab = "<keytab location>";
HiveConf hcatConf = new HiveConf();
hcatConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://192.168.42.154:9083");
hcatConf.set("hadoop.security.authentication", "Kerberos");
hcatConf.set(HCatConstants.HCAT_HIVE_CLIENT_DISABLE_CACHE, "true");
hcatConf.addResource(new Path("c:/temp/hive-site.xml"));
hcatConf.setVar(HiveConf.ConfVars.METASTORE_KERBEROS_PRINCIPAL, principal);
hcatConf.setVar(HiveConf.ConfVars.METASTORE_KERBEROS_KEYTAB_FILE, keytab);
hcatConf.setVar(HiveConf.ConfVars.METASTORE_USE_THRIFT_SASL, "true");
hcatClient = HCatClient.create(new Configuration(hcatConf));
UserGroupInformation.setConfiguration(hcatConf);
UserGroupInformation.loginUserFromKeytab(principal, keytab);
HiveMetaStoreClient hiveMetastoreClient = new HiveMetaStoreClient(hcatConf);
List<String> dbs = hcatClient.listDatabaseNamesByPattern("*");
for (String db : dbs) {
System.out.println(db);
List<String> tables = hcatClient.listTableNamesByPattern(db, "*");
for (String tableString: tables) {
HCatTable tbl = hcatClient.getTable(db, tableString);
String tableType = tbl.getTabletype();
String tableName = tbl.getTableName();
if (tableType.equalsIgnoreCase("View")) {
org.apache.hadoop.hive.metastore.api.Table viewMetastoreObject = hiveMetastoreClient.getTable(db, tableName);
String sql = viewMetastoreObject.getViewOriginalText();
System.out.println(sql);
}
}
}
} catch (Throwable t) {
t.printStackTrace();
} finally {
if (hcatClient != null)
try {
hcatClient.close();
} catch (HCatException e) {
}
}
}
}
Created 03-22-2016 04:38 PM
Current Error:
12:14:39,073 ERROR TSaslTransport:296 - SASL negotiation failure javax.security.sasl.SaslException: GSS initiate failed [Caused by GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt)] at com.sun.security.sasl.gsskerb.GssKrb5Client.evaluateChallenge(GssKrb5Client.java:212) at org.apache.thrift.transport.TSaslClientTransport.handleSaslStartMessage(TSaslClientTransport.java:94) at org.apache.thrift.transport.TSaslTransport.open(TSaslTransport.java:253) at org.apache.thrift.transport.TSaslClientTransport.open(TSaslClientTransport.java:37) at org.apache.hadoop.hive.thrift.client.TUGIAssumingTransport$1.run(TUGIAssumingTransport.java:52) at org.apache.hadoop.hive.thrift.client.TUGIAssumingTransport$1.run(TUGIAssumingTransport.java:49) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:415) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1408) at org.apache.hadoop.hive.thrift.client.TUGIAssumingTransport.open(TUGIAssumingTransport.java:49) at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.open(HiveMetaStoreClient.java:336) at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.<init>(HiveMetaStoreClient.java:214) at org.apache.hadoop.hive.metastore.HiveMetaStoreClient.<init>(HiveMetaStoreClient.java:154) at org.apache.hive.hcatalog.common.HiveClientCache.getNonCachedHiveClient(HiveClientCache.java:80) at org.apache.hive.hcatalog.common.HCatUtil.getHiveClient(HCatUtil.java:557) at org.apache.hive.hcatalog.api.HCatClientHMSImpl.initialize(HCatClientHMSImpl.java:595) at org.apache.hive.hcatalog.api.HCatClient.create(HCatClient.java:66) at ..... Caused by: GSSException: No valid credentials provided (Mechanism level: Failed to find any Kerberos tgt) at sun.security.jgss.krb5.Krb5InitCredential.getInstance(Krb5InitCredential.java:147) at sun.security.jgss.krb5.Krb5MechFactory.getCredentialElement(Krb5MechFactory.java:121) at sun.security.jgss.krb5.Krb5MechFactory.getMechanismContext(Krb5MechFactory.java:187) at sun.security.jgss.GSSManagerImpl.getMechanismContext(GSSManagerImpl.java:223) at sun.security.jgss.GSSContextImpl.initSecContext(GSSContextImpl.java:212) at sun.security.jgss.GSSContextImpl.initSecContext(GSSContextImpl.java:179) at com.sun.security.sasl.gsskerb.GssKrb5Client.evaluateChallenge(GssKrb5Client.java:193) ... 23 more
Created 07-05-2016 07:47 PM
The below code worked.
package hadoop.test;
import java.util.List;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hive.hcatalog.api.HCatClient;
import org.apache.hive.hcatalog.api.HCatTable;
import org.apache.hive.hcatalog.common.HCatConstants;
import org.apache.hive.hcatalog.common.HCatException;
import org.apache.hive.hcatalog.data.schema.HCatFieldSchema;
import org.apache.hive.hcatalog.data.schema.HCatSchema;
public class ListDBs1 {
publicstaticvoid main(String[] args) {HCatClient hcatClient = null;
try {String principal ="hive/[email protected]";
String keytab = "E:\\apps\\metacenter_home\\hadoop\\hive.keytab";
System.setProperty("sun.security.krb5.debug", "true"); System.setProperty("java.security.krb5.conf", "E:\\apps\\hadoop\\krb5.conf"); System.setProperty("java.security.auth.login.config", "E:\\apps\\hadoop\\jaas.conf");HiveConf hcatConf = new HiveConf();
hcatConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://server:9083");
hcatConf.set("hadoop.security.authentication", "kerberos"); hcatConf.set(HCatConstants.HCAT_HIVE_CLIENT_DISABLE_CACHE, "true");
hcatConf.setVar(HiveConf.ConfVars.METASTORE_KERBEROS_PRINCIPAL, principal);
hcatConf.setVar(HiveConf.ConfVars.METASTORE_KERBEROS_KEYTAB_FILE, keytab);
hcatConf.setVar(HiveConf.ConfVars.METASTORE_USE_THRIFT_SASL, "true");
UserGroupInformation.setConfiguration(hcatConf);
UserGroupInformation.loginUserFromKeytab(principal, keytab);
hcatClient = HCatClient.create(new Configuration(hcatConf));
HiveMetaStoreClient hiveMetastoreClient = new HiveMetaStoreClient(hcatConf);
list(hcatClient,hiveMetastoreClient);
} catch (Throwable t) {t.printStackTrace();
} finally {if (hcatClient != null)
try {hcatClient.close();
} catch (HCatException e) {}
}
}
privatestaticvoid list(HCatClient hcatClient, HiveMetaStoreClient hiveMetastoreClient) throws Exception { List<String> dbs = hcatClient.listDatabaseNamesByPattern("*");for (String db : dbs) {System.out.println(db);
List<String> tables = hcatClient.listTableNamesByPattern(db, "*");
for (String tableString: tables) {HCatTable tbl = hcatClient.getTable(db, tableString);
String tableType = tbl.getTabletype();
String tableName = tbl.getTableName();
System.out.println(tableType + " - " + tableName);
System.out.println("Table Name is: " + tableName); System.out.println("Table Type is: " + tbl.getTabletype()); System.out.println("Table Props are: " + tbl.getTblProps());List<HCatFieldSchema> fields = tbl.getCols();
for (HCatFieldSchema f: fields) { System.out.println("Field Name is: " + f.getName()); System.out.println("Field Type String is: " + f.getTypeString()); System.out.println("Field Type Category is: " + f.getTypeString());if (f.getCategory().equals(HCatFieldSchema.Category.STRUCT)) {HCatSchema schema = f.getStructSubSchema();
List<String> structFields = schema.getFieldNames();
for (String fieldName: structFields) { System.out.println("Struct Field Name is: " + fieldName); }
}
}
if (tableType.equalsIgnoreCase("View") || tableType.equalsIgnoreCase("VIRTUAL_VIEW")) {org.apache.hadoop.hive.metastore.api.Table viewMetastoreObject = hiveMetastoreClient.getTable(db, tableName);
String sql = viewMetastoreObject.getViewOriginalText();
System.out.println(sql);
}
}
}
}
}