Hi @Felix Albani, Still the same issue Exists, please find my attached build.sbt and sample code attached. import sbt._
import sbt.Keys._
name := "BackupSnippets"
version := "1.0"
scalaVersion := "2.11.8"
val sparkVersion = "2.2.1"
val hadoopVersion = "2.7.1"
val poiVersion = "3.9"
val avroVersion = "1.7.6"
val hortonworksVersion = "2.2.0.2.6.3.79-2"
conflictManager := ConflictManager.latestRevision
/*libraryDependencies ++= Seq(
"org.apache.spark" %% "spark-core" % sparkVersion,
"org.apache.spark" %% "spark-sql" % sparkVersion,
"org.apache.spark" %% "spark-hive" % sparkVersion,
"org.apache.poi" % "poi-ooxml" % poiVersion,
"org.apache.poi" % "poi" % poiVersion,
"org.apache.avro" % "avro" % avroVersion
)*/
/*resolvers ++= Seq(
"Typesafe repository" at "https://repo.typesafe.com/typesafe/releases/",
"Typesafe Ivyrepository" at "https://repo.typesafe.com/typesafe/ivy-releases/",
"Maven Central" at "https://repo1.maven.org/maven2/",
"Sonatype snapshots" at "https://oss.sonatype.org/content/repositories/snapshots/",
Resolver.sonatypeRepo("releases")
) */
libraryDependencies ++= Seq(
"org.apache.spark" %% "spark-core" % hortonworksVersion,
"org.apache.spark" %% "spark-sql" % hortonworksVersion,
"org.apache.spark" %% "spark-hive" % hortonworksVersion)
resolvers ++= Seq("Hortonworks Releases" at "http://repo.hortonworks.com/content/repositories/releases/",
"Jetty Releases" at "http://repo.hortonworks.com/content/repositories/jetty-hadoop/") ************************************************************************************ package BigData101.ORC
import ScalaUtils.SchemaUtils
import org.apache.spark.sql.{Encoders, Row, SparkSession}
import org.apache.spark.sql.catalyst.ScalaReflection
import org.apache.spark.sql.types.StructType
object ORCTesting {
def main(args: Array[String]): Unit = {
val spark: SparkSession = SparkSession.builder()
.master("local[*]")
.appName("ORC Testing")
.enableHiveSupport()
.getOrCreate()
case class Airlines(Airline_id: Integer, Name: String, Alias: String, IATA: String, ICAO: String, Callsign: String,
Country: String, Active: String)
//val AirlineSchema = ScalaReflection.schemaFor[Airlines].dataType.asInstanceOf[StructType]
Encoders.product[Airlines].schema
sys.exit(1)
}
}
... View more
