I don't know of any other way of comparing two dataframe other than joining first. Here is your action item. 1. Join tab1 and tab2 using broadcast hash by column key "locaton". 2. Filter column by pole = N Here is my code from your sample data. Paste it in a spark shell and see.
import org.apache.spark.{ SparkConf, SparkContext }
import org.apache.spark.sql.functions.broadcast
import org.apache.spark.sql.types._
import org.apache.spark.sql._
import org.apache.spark.sql.functions._
import org.apache.spark.sql.functions.udf
val sqlContext = new org.apache.spark.sql.hive.HiveContext(sc)
import sqlContext.implicits._
case class tab1x(id:Int,phno:Long,location:String,address:String,name:String,temp:String)
case class tab2x(id:Int,phno:Long,location:String,pole:String)
val tab1 = Seq(
tab1x(1,656,"IND","Street no1","X","30F"),
tab1x(2,657,"USA","RHGD no 23","Y","23F"),
tab1x(3,658,"RUS","YWKY 58","Z", "20F")
).toDF
val tab2 = Seq(
tab2x(1,656,"IND","S"),
tab2x(2,657,"USA","N"),
tab2x(3,658,"RUS","N")
).toDF
val joined_df = tab1.join(broadcast(tab2), "location")
val z = joined_df.filter($"pole" === "N")
z.show()
... View more
