Support Questions
Find answers, ask questions, and share your expertise
Announcements
Alert: Welcome to the Unified Cloudera Community. Former HCC members be sure to read and learn how to activate your account here.

HIVE-6427 Hive Server2 should reopen Metastore client in case of any Thrift exceptions

HIVE-6427 Hive Server2 should reopen Metastore client in case of any Thrift exceptions

New Contributor

Hi, I found that if metastore somehow losts connection (restart or haproxy disconnect), hive server2 is unable to function properly. That is because of unnecessary masking of exceptions and because of no retry logic used.

 

I make a patch against cloudera branch cdh5-0.12.0_5.0.0b2 wich somehow solves that problem. 

It is copy&paste of RetryingMetaStoreClient initialization. Also, for properly working proxy code 

I removed unnecessary exception mangling.

 

With this patch connection reesablished and hive server works properly (looks like it works :) )

 

From e5b6d3bc2afe69df1f4ce7323f079aa183f8c864 Mon Sep 17 00:00:00 2001
From: Andrey Stepachev <octo@yandex-team.ru>
Date: Fri, 14 Feb 2014 01:48:46 +0400
Subject: [PATCH] HIVE-6427 Hive Server2 should reopen Metastore client in case
 of any Thrift exceptions

---
 .../hadoop/hive/metastore/HiveMetaStoreClient.java | 36 +++++--------------
 .../hive/service/cli/session/HiveSessionImpl.java  | 41 ++++++++++++++++++++--
 2 files changed, 46 insertions(+), 31 deletions(-)

diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
index ed2cb5a..05ff5f5 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java
@@ -722,23 +722,13 @@ public boolean dropType(String type) throws NoSuchObjectException, MetaException

   /** {@inheritDoc} */
   public List<String> getDatabases(String databasePattern)
-    throws MetaException {
-    try {
-      return client.get_databases(databasePattern);
-    } catch (Exception e) {
-      MetaStoreUtils.logAndThrowMetaException(e);
-    }
-    return null;
+    throws MetaException, TException {
+    return client.get_databases(databasePattern);
   }

   /** {@inheritDoc} */
-  public List<String> getAllDatabases() throws MetaException {
-    try {
-      return client.get_all_databases();
-    } catch (Exception e) {
-      MetaStoreUtils.logAndThrowMetaException(e);
-    }
-    return null;
+  public List<String> getAllDatabases() throws MetaException, TException  {
+    return client.get_all_databases();
   }

   /**
@@ -922,23 +912,13 @@ public Type getType(String name) throws NoSuchObjectException, MetaException, TE
   }

   /** {@inheritDoc} */
-  public List<String> getTables(String dbname, String tablePattern) throws MetaException {
-    try {
-      return client.get_tables(dbname, tablePattern);
-    } catch (Exception e) {
-      MetaStoreUtils.logAndThrowMetaException(e);
-    }
-    return null;
+  public List<String> getTables(String dbname, String tablePattern) throws MetaException, TException  {
+    return client.get_tables(dbname, tablePattern);
   }

   /** {@inheritDoc} */
-  public List<String> getAllTables(String dbname) throws MetaException {
-    try {
-      return client.get_all_tables(dbname);
-    } catch (Exception e) {
-      MetaStoreUtils.logAndThrowMetaException(e);
-    }
-    return null;
+  public List<String> getAllTables(String dbname) throws MetaException, TException  {
+    return client.get_all_tables(dbname);
   }

   public boolean tableExists(String databaseName, String tableName) throws MetaException,
diff --git a/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java b/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java
index 66439b2..7c3b576 100644
--- a/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java
+++ b/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java
@@ -29,11 +29,19 @@
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
+import org.apache.hadoop.hive.metastore.HiveMetaHook;
+import org.apache.hadoop.hive.metastore.HiveMetaHookLoader;
 import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
 import org.apache.hadoop.hive.metastore.IMetaStoreClient;
+import org.apache.hadoop.hive.metastore.MetaStoreUtils;
+import org.apache.hadoop.hive.metastore.RetryingMetaStoreClient;
 import org.apache.hadoop.hive.metastore.api.MetaException;
 import org.apache.hadoop.hive.ql.history.HiveHistory;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.metadata.HiveStorageHandler;
+import org.apache.hadoop.hive.ql.metadata.HiveUtils;
 import org.apache.hadoop.hive.ql.session.SessionState;
+import org.apache.hadoop.util.StringUtils;
 import org.apache.hive.common.util.HiveVersionInfo;
 import org.apache.hive.service.auth.HiveAuthFactory;
 import org.apache.hive.service.cli.FetchOrientation;
@@ -56,6 +64,8 @@
 import org.apache.hive.service.cli.operation.MetadataOperation;
 import org.apache.hive.service.cli.operation.OperationManager;

+import static org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_STORAGE;
+
 /**
  * HiveSession
  *
@@ -153,7 +163,32 @@ public void setLogManager(LogManager logManager) {
   public IMetaStoreClient getMetaStoreClient() throws HiveSQLException {
     if (metastoreClient == null) {
       try {
-        metastoreClient = new HiveMetaStoreClient(getHiveConf());
+        final HiveConf hiveConf = getHiveConf();
+        HiveMetaHookLoader hookLoader = new HiveMetaHookLoader() {
+          public HiveMetaHook getHook(
+                  org.apache.hadoop.hive.metastore.api.Table tbl)
+                  throws MetaException {
+
+            try {
+              if (tbl == null) {
+                return null;
+              }
+              HiveStorageHandler storageHandler =
+                      HiveUtils.getStorageHandler(hiveConf,
+                              tbl.getParameters().get(META_TABLE_STORAGE));
+              if (storageHandler == null) {
+                return null;
+              }
+              return storageHandler.getMetaHook();
+            } catch (HiveException ex) {
+              LOG.error(StringUtils.stringifyException(ex));
+              throw new MetaException(
+                      "Failed to load storage handler:  " + ex.getMessage());
+            }
+          }
+        };
+        metastoreClient = RetryingMetaStoreClient.getProxy(hiveConf, hookLoader,
+                HiveMetaStoreClient.class.getName());
       } catch (MetaException e) {
         throw new HiveSQLException(e);
       }
@@ -485,8 +520,8 @@ public void renewDelegationToken(HiveAuthFactory authFactory, String tokenStr)
   }

   //Log capture
-  private void runOperationWithLogCapture(Operation operation,
-      OperationHandle opHandle) throws HiveSQLException {
+  private void runOperationWithLogCapture(Operation operation,
+      OperationHandle opHandle) throws HiveSQLException {
     getLogManager().unregisterCurrentThread();
     getLogManager().registerCurrentThread(opHandle);
     operation.run();
--
1.8.3.4 (Apple Git-47)