I am new to scala based sbt build. I am not sure whether it is correct build.sbt for given scala code. Please correct me as well
When i am trying to connect Hbase with spark. using scala code. Unable to do so because of sbt file error.
I stucked at sbt only. Aim is to add data using scala code programtic manner ( for CDH 5.13 version)
Version
Hbase - 1.2.0-cdh5.13.0
Cloudera - cdh5.13.0
Getting error :
assemblyMergeStrategy Portion is unreolvable. Pathlist as well
Build.sbt
name := "ScalaHBase" version := "1.0" scalaVersion := "2.11.8" resolvers ++= Seq( "Hadoop Releases" at "https://repository.cloudera.com/content/repositories/releases/" ) libraryDependencies ++= Seq( "com.google.guava" % "guava" % "15.0", "org.apache.hadoop" % "hadoop-common" % "2.6.0", "org.apache.hadoop" % "hadoop-mapred" % "0.22.0", "org.apache.hbase" % "hbase-common" % "1.0.0", "org.apache.hbase" % "hbase-client" % "1.0.0" ) dependencyOverrides += "com.google.guava" % "guava" % "15.0" assemblyMergeStrategy in assembly := { case PathList("javax", "servlet", xs @ _*) => MergeStrategy.last case PathList("javax", "activation", xs @ _*) => MergeStrategy.last case PathList("org", "apache", xs @ _*) => MergeStrategy.last case PathList("com", "google", xs @ _*) => MergeStrategy.first case PathList("com", "yammer", xs @ _*) => MergeStrategy.last case "about.html" => MergeStrategy.rename case "plugin.properties" => MergeStrategy.last case "log4j.properties" => MergeStrategy.last case x => val oldStrategy = (assemblyMergeStrategy in assembly).value oldStrategy(x) }
Scala code
import org.apache.hadoop.hbase.client._ import org.apache.hadoop.hbase.util.Bytes import org.apache.hadoop.hbase.{CellUtil, HBaseConfiguration, TableName} import org.apache.hadoop.conf.Configuration import scala.collection.JavaConverters._ object ScalaHBaseExample extends App{ def printRow(result : Result) = { val cells = result.rawCells(); print( Bytes.toString(result.getRow) + " : " ) for(cell <- cells){ val col_name = Bytes.toString(CellUtil.cloneQualifier(cell)) val col_value = Bytes.toString(CellUtil.cloneValue(cell)) print("(%s,%s) ".format(col_name, col_value)) } println() } val conf : Configuration = HBaseConfiguration.create() val ZOOKEEPER_QUORUM = "WRITE THE ZOOKEEPER CLUSTER THAT HBASE SHOULD USE" conf.set("hbase.zookeeper.quorum", ZOOKEEPER_QUORUM); val connection = ConnectionFactory.createConnection(conf) val table = connection.getTable(TableName.valueOf( Bytes.toBytes("emostafa:test_table") ) ) // Put example var put = new Put(Bytes.toBytes("row1")) put.addColumn(Bytes.toBytes("d"), Bytes.toBytes("test_column_name"), Bytes.toBytes("test_value")) put.addColumn(Bytes.toBytes("d"), Bytes.toBytes("test_column_name2"), Bytes.toBytes("test_value2")) table.put(put) // Get example println("Get Example:") var get = new Get(Bytes.toBytes("row1")) var result = table.get(get) printRow(result) //Scan example println("\nScan Example:") var scan = table.getScanner(new Scan()) scan.asScala.foreach(result => { printRow(result) }) table.close() connection.close() }
Also I tried this Build.sbt file
name := "HbaseExample"
version := "0.1"
scalaVersion := "2.12.5"
libraryDependencies ++= Seq(
//"org.apache.hbase" % "hbase" % "0.98.8-hadoop2",
"org.apache.hbase" % "hbase" % "1.2.0-cdh5.13.0" pomOnly(),
//"org.apache.hbase" % "hbase-client" % "0.98.8-hadoop2",
"org.apache.hbase" % "hbase-client" % "1.2.0-cdh5.13.0",
// "org.apache.hbase" % "hbase-common" % "0.98.8-hadoop2",
"org.apache.hbase" % "hbase-common" % "1.2.0-cdh5.13.0",
//"org.apache.hbase" % "hbase-server" % "0.98.8-hadoop2",
"org.apache.hbase" % "hbase-server" % "1.2.0-cdh5.13.0"
)
mergeStrategy in assembly := {
case m if m.toLowerCase.endsWith("manifest.mf") => MergeStrategy.discard
case m if m.toLowerCase.matches("meta-inf.*\\.sf$") => MergeStrategy.discard
case "META-INF/jersey-module-version" => MergeStrategy.first
case _ => MergeStrategy.first
}
when trying to import this build.sbt again this portion : saying cannot resolve symbol
Any help on it would be much appreciated ?