Hi All,
please help me out on this scenario, Im trying to connect from my local machine to my Hadoop Cluster. i have a following spark code.
im getting the following Error:
Exception in thread "main" java.lang.IllegalArgumentException: Error while instantiating 'org.apache.spark.sql.hive.HiveSessionStateBuilder':
at org.apache.spark.sql.SparkSession$.org$apache$spark$sql$SparkSession$instantiateSessionState(SparkSession.scala:1062)
at org.apache.spark.sql.SparkSession$anonfun$sessionState$2.apply(SparkSession.scala:137)
at org.apache.spark.sql.SparkSession$anonfun$sessionState$2.apply(SparkSession.scala:136)
at scala.Option.getOrElse(Option.scala:121)
at org.apache.spark.sql.SparkSession.sessionState$lzycompute(SparkSession.scala:136)
at org.apache.spark.sql.SparkSession.sessionState(SparkSession.scala:133)
at org.apache.spark.sql.SparkSession.sql(SparkSession.scala:632)
at org.apache.spark.sql.SQLContext.sql(SQLContext.scala:691)
at HiveTest$.main(HiveTest.scala:16)
at HiveTest.main(HiveTest.scala)
Caused by: org.apache.spark.sql.AnalysisException: java.lang.RuntimeException: java.lang.RuntimeException: Unable to instantiate org.apache.hadoop.hive.ql.metadata.SessionHiveMetaStoreClient;
at org.apache.spark.sql.hive.HiveExternalCatalog.withClient(HiveExternalCatalog.scala:106)
at org.apache.spark.sql.hive.HiveExternalCatalog.databaseExists(HiveExternalCatalog.scala:194)
at org.apache.spark.sql.internal.SharedState.externalCatalog$lzycompute(SharedState.scala:105)
at org.apache.spark.sql.internal.SharedState.externalCatalog(SharedState.scala:93)
at org.apache.spark.sql.hive.HiveSessionStateBuilder.externalCatalog(HiveSessionStateBuilder.scala:39)
at org.apache.spark.sql.hive.HiveSessionStateBuilder.catalog$lzycompute(HiveSessionStateBuilder.scala:54)
at org.apache.spark.sql.hive.HiveSessionStateBuilder.catalog(HiveSessionStateBuilder.scala:52)
at org.apache.spark.sql.hive.HiveSessionStateBuilder.catalog(HiveSessionStateBuilder.scala:35)
at org.apache.spark.sql.internal.BaseSessionStateBuilder.build(BaseSessionStateBuilder.scala:289)
at org.apache.spark.sql.SparkSession$.org$apache$spark$sql$SparkSession$instantiateSessionState(SparkSession.scala:1059)
... 9 more
the same working fine in My Command like from spark-shell
the only problem is from Intellij even after placed all my needed conf files in all the required Folders.
and following are my Environment Details
In MY Local:
-- Spark 2.2.1 with Hadoop 2.7
-- Sbt 0.13.15
-- Scala 2.10.6
and following are my Build.sbt details
name := "XXXXXX"
version := "0.1"
scalaVersion := "2.11.8"
val sparkVersion = "2.2.1"
libraryDependencies ++= Seq(
"org.apache.spark" %% "spark-core" % sparkVersion,
"org.apache.spark" %% "spark-sql" % sparkVersion ,
"org.apache.spark" %% "spark-hive" % sparkVersion,
"org.apache.hadoop" % "hadoop-client" % "2.7.1",
"org.apache.hadoop" % "hadoop-hdfs" % "2.7.1",
"org.apache.hive" % "hive-jdbc" % "2.1.0",
"org.apache.logging.log4j" % "log4j-api" % "2.8.2",
"org.apache.logging.log4j" % "log4j-core" % "2.8.2")
resolvers ++= Seq(
"Typesafe repository" at "https://repo.typesafe.com/typesafe/releases/",
"Sonatype snapshots" at "https://oss.sonatype.org/content/repositories/snapshots/",
"Maven Central" at "https://repo1.maven.org/maven2/",
Resolver.sonatypeRepo("releases")
)
import org.apache.spark.sql.{SQLContext, SparkSession}
object HiveTest {
def main(args:Array[String]) {
val sparkSession = SparkSession
.builder.master("local[*]")
.appName("Hive - Connect")
.enableHiveSupport()
.getOrCreate()
val sc = sparkSession.sparkContext
val sqlContext = new SQLContext(sc)
System.out.println("INFO: ****************** Starting Connection HIVE ******************")
sqlContext.sql("Show databases").show()
//sparkSession.sqlContext.sql("show databases").show()}
}
