@Divya Gehlot If you want the table to be accessible from hive as well, you cannot use saveAsTable. If you use saveAsTable only spark sql will be able to use it. You have two ways to create orc tables from spark (compatible with hive). I tested codes below with hdp 2.3.2 sandbox and spark 1.4.1 1- Saving orc file from spark and create table directly on hive, see this code: spark-shell --master yarn-client --driver-memory 512m --executor-memory 512m import org.apache.spark.sql._
import org.apache.spark.sql.types._
val people = sc.textFile("/tmp/people.txt")
val schemaString = "name age"
val schema =
StructType(
schemaString.split(" ").map(fieldName => StructField(fieldName, StringType, true)))
val rowRDD = people.map(_.split(",")).map(p => Row(p(0), p(1).trim))
val df = sqlContext.createDataFrame(rowRDD, schema);
sqlContext.sql("drop table if exists personhivetable")
sqlContext.sql("create external table personhivetable (name string, age string) stored as orc location '/tmp/personhivetable/'")
df.write.format("orc").mode("overwrite").save("/tmp/personhivetable")
sqlContext.sql("show tables").collect().foreach(println);
sqlContext.sql("select * from personhivetable").collect().foreach(println);
2- Registering your data frame as temporary table and performing a create table as select spark-shell --master yarn-client --driver-memory 512m --executor-memory 512m import org.apache.spark.sql._
import org.apache.spark.sql.types._
val people = sc.textFile("/tmp/people.txt")
val schemaString = "name age"
val schema =
StructType(
schemaString.split(" ").map(fieldName => StructField(fieldName, StringType, true)))
val rowRDD = people.map(_.split(",")).map(p => Row(p(0), p(1).trim))
val df = sqlContext.createDataFrame(rowRDD, schema);
df.registerTempTable("personhivetable_tmp")
sqlContext.sql("drop table if exists personhivetable2")
sqlContext.sql("CREATE TABLE personhivetable2 STORED AS ORC AS SELECT * from personhivetable_tmp")
sqlContext.sql("show tables").collect().foreach(println);
sqlContext.sql("select * from personhivetable2").collect().foreach(println);
Also, check this question with more discussion about orc + spark. https://community.hortonworks.com/questions/4292/how-do-i-create-an-orc-hive-table-from-spark.html
... View more
