I'am Running Spark Job that does hbase scan. However I get an error java.lang.NoSuchMethodError: org.apache.hadoop.hbase.client.Scan.setCacheBlocks(Z)V
As I looked it up, it is caused by version mismatch between the hbase-client.jar to hbase version. However I used only hdp compiled jars.
My HDP version is 2.4.3.0
I run the sumbit the flowwing way:
export HADOOP_CONF_DIR=/etc/hadoop/conf/
export SPARK_CONF_DIR=/etc/spark/conf
/usr/hdp/current/spark-client/bin/spark-submit
--class MyClass
--master yarn-cluster
--num-executors 4
--driver-memory 1g
--executor-memory 4g
--executor-cores 6
--conf spark.driver.cores=6
--conf spark.storage.memoryFraction=0.8
--conf spark.shuffle.memoryFraction=0.1
--conf spark.yarn.jar=/usr/hdp/current/spark-client/lib/spark-hdp-assembly.jar
--conf spark.yarn.executor.memoryOverhead=2048
--conf spark.akka.frameSize=100
--conf spark.driver.extraJavaOptions="-Xss10m -XX:MaxPermSize=512M -XX:+CMSPermGenSweepingEnabled -XX:+CMSClassUnloadingEnabled -XX:+UseConcMarkSweepGC "
--conf spark.executor.extraJavaOptions="-Xss10m -XX:MaxPermSize=512M -XX:+CMSPermGenSweepingEnabled -XX:+CMSClassUnloadingEnabled -XX:+UseConcMarkSweepGC "
--jars /usr/hdp/current/hive-client/lib/hive-common.jar,
/usr/hdp/current/hive-client/lib/hive-hbase-handler.jar,
/usr/hdp/current/hbase-client/lib/hbase-common.jar,
/usr/hdp/current/hbase-client/lib/hbase-server.jar,
/usr/hdp/current/hbase-client/lib/hbase-client.jar,
/usr/hdp/current/hbase-client/lib/hbase-procedure.jar,
/usr/hdp/current/hbase-client/lib/htrace-core-3.1.0-incubating.jar,
/usr/hdp/current/spark-client/lib/datanucleus-api-jdo-3.2.6.jar,
/usr/hdp/current/spark-client/lib/datanucleus-core-3.2.10.jar,
/usr/hdp/current/spark-client/lib/datanucleus-rdbms-3.2.9.jar,
hdfs://mycluster:8020/lib/java/dependencies/mysql-connector-java-5.0.8-bin.jar
hdfs://mycluster:8020/lib/scala/myjar.jar
