I think this is a duplicate of the question for which I already posted an answer. https://community.hortonworks.com/questions/84507/sql-query-to-sparkdataframe-to-get-date-add-interv.html#answer-85407
This is the working code again, you can convert to dataframe or do operations in Rdd
import org.apache.spark.{ SparkConf, SparkContext }
import org.apache.spark.sql.functions.broadcast
import org.apache.spark.sql.types._
import org.apache.spark.sql._
import org.apache.spark.sql.functions._
import scala.collection.mutable.ListBuffer
import java.util.{GregorianCalendar, Date}
import java.util.Calendar
def generateDates(startdate: Date, enddate: Date): ListBuffer[String] = {
var dateList = new ListBuffer[String]()
var calendar = new GregorianCalendar()
calendar.setTime(startdate)
while (calendar.getTime().before(enddate)) {
dateList += calendar.getTime().toString().substring(0, 10) + "," + (calendar.get(Calendar.DAY_OF_MONTH)) + "," + calendar.get(Calendar.MONTH)
calendar.add(Calendar.DATE, 1)
}
dateList += calendar.getTime().toString()
println("\n" + dateList + "\n")
dateList
}
def getRddList(a :String) : ListBuffer[(String,String,String,String,String)] = {
var allDates = new ListBuffer[(String,String,String,String,String)]()
for (x <- generateDates(format.parse(a.split(",")(1)),format.parse(a.split(",")(2)))) {
allDates += (( a.split(",")(0).toString(), x , a.split(",")(3).toString(),
a.split(",")(4).toString(), a.split(",")(5).toString() ))
}
allDates
}
var fileRdd = sc.textFile("/data_1/date1");
var myRdd = fileRdd.map{x=>getRddList(x)}.flatMap(y=>y)
myRdd.collect()
... View more
