@Joe Widen am unable to do select query on my input_file_temp, at the same time i can do it on gsam_temp which is the DF i made it from SQL table. If I could do query on both the DF then it would be much easier for me to finish it off. Here is complete code import sqlContext.implicits._
import org.apache.spark.sql.hive.HiveContext
import org.apache.spark.{SparkContext, SparkConf}
import org.apache.spark.sql.functions.broadcast
import org.apache.spark.sql.types._
import org.apache.spark.sql._
import org.apache.spark.sql.functions._
val hiveContext = new HiveContext(sc);
val sqlContext = new org.apache.spark.sql.SQLContext(sc);
// Loading DB table in to dataframe
val gsam = hiveContext.read.format("jdbc").option("driver","oracle.jdbc.driver.OracleDriver").option("url","jdbc:oracle:thin:NPIDWDEV/sysdwnpi@scan-nsgnp.ebiz.verizon.com:1521/nsgdev").option("dbtable", "GSAM_REF").load();
gsam.registerTempTable("gsam_temp")
// Create case class to load input file from local or hdfs
case class f1(
ckt_id:String,
location:String,
usage:String,
port:String,
machine:String
)
val input_file = sc.textFile("file:///data04/dev/v994292/spark/input_file.txt").map(_.split("\\|")).map(x => f1(x(0).toString,x(1).toString,x(2).toString,x(3).toString,x(4).toString)).toDF
input_file.registerTempTable("input_file_tmp")
... View more
