Hi
After upgrading to Spark 2.0 recently I cannot seem to get something that relies on Spark SQL to run in Intelli J Idea in Windows. My code is provided below.
import org.apache.spark.sql.SparkSession
object PropertyInvestmentCalcs {
def main(args: Array[String]) {
val spark = SparkSession
.builder()
.appName("Spark PropertyInvestmentCalcs")
.master("local[6]")
.config("spark.sql.warehouse.dir", "\\\\TJVRLAPTOP\\Users\\tjoha\\Google Drive\\Programming\\IntelliJ\\PropertyInvestmentCalcs\\spark-warehouse")
//.config("fs.hdfs.impl", classOf[org.apache.hadoop.hdfs.DistributedFileSystem].getName)
//.config("fs.file.impl", classOf[org.apache.hadoop.fs.LocalFileSystem].getName)
.getOrCreate()
//val sqlContext = new org.apache.spark.sql.SparkSession(spark)
// Get number of data records in the table
val nrRecordsDF = spark.read.format("jdbc")
.option("url", "jdbc:mysql://localhost:3306/test")
.option("driver", "com.mysql.jdbc.Driver")
.option("dbtable", "(SELECT COUNT(*) AS nrRecords FROM test.propertydb) AS nrRecords_tmp")
.option("user", "tjohannvr")
.option("password", "[5010083]").load()
nrRecordsDF.show()
nrRecordsDF.printSchema()
val nrRecords = nrRecordsDF.head().getLong(0)
println("nrRecords = " + nrRecords)
// Select data from MySQL, with a specific number of records at a time
val NrRecordsAtATime = nrRecords * 0 + 100000
println("NrRecordsAtATime = " + NrRecordsAtATime)
spark.stop()
}
}
The error which occurs when trying to set nrRecordsDF is provided below:
Exception in thread "main" java.io.IOException: No FileSystem for scheme: null
Is it perhaps unable to load hadoop classes, specifically those related to HDFS. Why would this be the case?
Below is the contents of my build.sbt file.
import sbt._
name := "PropertyInvestmentCalcs"
version := "0.1.0-SNAPSHOT"
organization := "TJVR"
scalaVersion := "2.11.8"
libraryDependencies += "org.apache.spark" %% "spark-core" % "2.0.0" //% "provided"
libraryDependencies += "org.apache.spark" %% "spark-sql" % "2.0.0" //% "provided"
libraryDependencies += "org.apache.spark" %% "spark-hive" % "2.0.0" //% "provided"
libraryDependencies += "mysql" % "mysql-connector-java" % "5.1.39"
lazy val commonSettings = Seq(
version := "0.1-SNAPSHOT",
organization := "TJVR",
scalaVersion := "2.11.8"
)
lazy val app = (project in file("app")).
settings(commonSettings: _*).
settings(
// your settings here
)
artifact in (Compile, assembly) := {
val art = (artifact in (Compile, assembly)).value
art.copy(`classifier` = Some("assembly"))
}
addArtifact(artifact in (Compile, assembly), assembly)
Thanks in advance for any help.
