I don't think that was the problem, I changed the code as below and it worked. The issue was in toDF method: import org.apache.spark.SparkContext import org.apache.spark.SparkContext._ import org.apache.spark.sql._ import org.apache.spark.sql.types._ import org.apache.spark.SparkConf import sys.process._ class cc extends Runnable{ val conf = new SparkConf().setAppName("LoadDW") val sc = new SparkContext(conf) val sqlContext= new org.apache.spark.sql.SQLContext(sc) import sqlContext.implicits._ override def run(): Unit = { var fileName = "DimCustomer.txt" val fDimCustomer = sc.textFile("DimCustomer.txt") val schemaString = "ID Name City EffectiveFrom EffectiveTo" val schema = StructType(List( StructField("ID", IntegerType, true), StructField("Name", StringType, true), StructField("City", StringType, true), StructField("EffectiveFrom", IntegerType, true), StructField("EffectiveTo", IntegerType, true) ) ) println("----->>>>>>sdsdsd2222\n") var dimCustomerRDD = fDimCustomer.map(_.split(',')).map(r=>Row(r(0).toInt,r(1),r(2),r(3).toInt,r(4).toInt)) var customerDataFrame = sqlContext.createDataFrame(dimCustomerRDD, schema) customerDataFrame.registerTempTable("Cust_1") val customers = sqlContext.sql("select * from Cust_1") customers.show() println("+") } } object pp extends App { val cp = new cc() val rThread = new Thread (cp) rThread.start() }
... View more
