error:
17/06/06 10:39:54 INFO zookeeper.ZooKeeper: Client environment:java.io.tmpdir=/tmp
17/06/06 10:39:54 INFO zookeeper.ZooKeeper: Client environment:java.compiler=<NA>
17/06/06 10:39:54 INFO zookeeper.ZooKeeper: Client environment:os.name=Linux
17/06/06 10:39:54 INFO zookeeper.ZooKeeper: Client environment:os.arch=amd64
17/06/06 10:39:54 INFO zookeeper.ZooKeeper: Client environment:os.version=4.1.6-1.el6.x86_64
17/06/06 10:39:54 INFO zookeeper.ZooKeeper: Client environment:user.name=root
17/06/06 10:39:54 INFO zookeeper.ZooKeeper: Client environment:user.home=/root
17/06/06 10:39:54 INFO zookeeper.ZooKeeper: Client environment:user.dir=/home/spark
17/06/06 10:39:54 INFO zookeeper.ZooKeeper: Initiating client connection, connectString=bg-js-sz1-ib1:2181 sessionTimeout=90000 watcher=hconnection-0x157d3a450x0, quorum=bg-js-sz1-ib1:2181, baseZNode=/hbase
17/06/06 10:39:54 INFO zookeeper.ClientCnxn: Opening socket connection to server bg-js-sz1-ib1/120.132.23.12:2181. Will not attempt to authenticate using SASL (unknown error)
17/06/06 10:39:54 INFO zookeeper.ClientCnxn: Socket connection established, initiating session, client: /120.132.23.12:55725, server: bg-js-sz1-ib1/120.132.23.12:2181
17/06/06 10:39:54 INFO zookeeper.ClientCnxn: Session establishment complete on server bg-js-sz1-ib1/120.132.23.12:2181, sessionid = 0x25b5cf27b987009, negotiated timeout = 60000
17/06/06 10:39:54 WARN ipc.RpcControllerFactory: Cannot load configured "hbase.rpc.controllerfactory.class" (org.apache.hadoop.hbase.ipc.controller.ClientRpcControllerFactory) from hbase-site.xml, falling back to use default RpcControllerFactory
17/06/06 10:39:54 INFO metrics.Metrics: Initializing metrics system: phoenix
17/06/06 10:39:54 WARN impl.MetricsConfig: Cannot locate configuration: tried hadoop-metrics2-phoenix.properties,hadoop-metrics2.properties
17/06/06 10:39:54 INFO impl.MetricsSystemImpl: Scheduled snapshot period at 10 second(s).
17/06/06 10:39:54 INFO impl.MetricsSystemImpl: phoenix metrics system started
17/06/06 10:39:54 INFO Configuration.deprecation: hadoop.native.lib is deprecated. Instead, use io.native.lib.available
17/06/06 10:40:43 INFO client.RpcRetryingCaller: Call exception, tries=10, retries=35, started=48474 ms ago, cancelled=false, msg=
17/06/06 10:41:03 INFO client.RpcRetryingCaller: Call exception, tries=11, retries=35, started=68544 ms ago, cancelled=false, msg=
code:
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.hbase.HBaseConfiguration
import org.apache.hadoop.security.UserGroupInformation
import org.apache.spark.{SparkConf, SparkContext}
import org.apache.phoenix.spark._
import org.slf4j.LoggerFactory
/**
* Created by root on 20170606.
* Update date:
* Time: 9:40 AM
* Project: ${PROJECT_HOME}
* Package: ${PACKAGE_HOME}
* Describle : save data to phoenix
*
* Result of Test: local test ok,remote test error
* Command:
**
*
spark-submit \
--master yarn-client \
--driver-memory 2g \
--num-executors 3 \
--driver-cores 2 \
--executor-cores 2 \
--executor-memory 2g \
--class com.yjf.phoenix.kerberos.SaveAsPhoenixApp \
--jars /home/spark/lib/phoenix-spark-4.7.0-HBase-0.98.jar,\
/home/spark/lib/hbase-common-1.2.0-cdh5.8.0.jar,\
/home/spark/lib/hbase-client-1.2.0-cdh5.8.0.jar,\
/home/spark/lib/hbase-server-1.2.0-cdh5.8.0.jar,\
/home/spark/lib/hbase-protocol-1.2.0-cdh5.8.0.jar,\
/home/spark/lib/htrace-core-3.2.0-incubating.jar,\
/home/spark/lib/guava-12.0.1.jar,\
/home/spark/lib/phoenix-core-4.7.0-HBase-0.98.jar \
--principal log2hadoop \
--keytab /home/spark/conf/log2hadoop_bgp.keytab \
/home/spark/lib/test-phonenix.jar yarn-client
*
*
*
*
* Email: jifei.yang@ngaa.com.cn
* Status：线上正在使用
* Machine ip:
* ^^集群----->192.168.1.1（001业务统计）
* ^^集群----->127.0.0.1 （002业务统计）
*
* Attention：
*
*
*/
object SaveAsPhoenixApp {
private val logger=LoggerFactory.getLogger(SaveAsPhoenixApp.getClass)
private val quorum="bg-js-sz1-ib1,bg-js-sz2-ib3,bg-js-sz1-ib2"
private val krb5 = "/home/spark/conf/krb5.conf"
private val user = "log2hadoop@EEENN.NET" //用户名称
private val keyPath = "/home/spark/conf/log2hadoop.keytab" //文件
def saveAsphoenixProcess(master:String):Unit={
System.setProperty("java.security.krb5.conf",krb5);
val conf=new SparkConf().setAppName("PhoenixSparkRDDApp")
.set("spark.app.id","PhoenixSparkRDDApp_id")
.setMaster(master)
val sc=new SparkContext(conf)
//save error
val dataSet = List(
("00059","www.tudou.com","134.0.0.22","2017-06-23 12:00:12",75,"2017-08-25"),
("00069","www.youku.com","197.1.0.23","2017-06-03 13:00:12",96,"2017-07-16"),
("00079","www.nggaa.com","187.0.6.32","2017-06-13 13:00:12",88,"2017-05-06"),
("00089","www.wec.com","138.0.3.62","2017-06-12 15:00:12",129,"2017-12-19")
)
val endData=sc.parallelize(dataSet)
val configuration = new Configuration()
//hbase
configuration.set("hbase.zookeeper.quorum", quorum)
configuration.set("hbase.rootdir", "/hbase")
configuration.set("hadoop.security.authentication", "kerberos")
configuration.set("hbase.security.authentication", "kerberos")
configuration.set("hbase.security.authorization", "true")
configuration.set("hbase.master.kerberos.principal", "hbase/_HOST@EEENN.NET")
configuration.set("hbase.thrift.kerberos.principal", "hbase/_HOST@EEENN.NET")
configuration.set("hbase.regionserver.kerberos.principal", "hbase/_HOST@EEENN.NET")
configuration.set("hbase.zookeeper.property.clientPort", "2181")
configuration.set("hbase.rpc.timeout", "10000")
configuration.set("hbase.client.retries.number", "5")
configuration.set("hbase.client.pause", "5000")
configuration.set("hbase.client.scanner.timeout.period", "50000")
logger.warn("开始认证")
try{
UserGroupInformation.setConfiguration(configuration)
UserGroupInformation.loginUserFromKeytab(user, keyPath)
logger.warn("认证成功!")
}catch {
case ex:Exception=>logger.error(s"auth login error :${ex.toString}")
}
logger.warn("save as phoenix")
val hbConf: Configuration = HBaseConfiguration.create(configuration)
endData.saveToPhoenix(
"PH_HOT_IP",
Array("CREATETIME","DOMAIN","IP","IPFIRSTTIME","IPACTIVENESS","DAY"),
conf=hbConf
)
logger.warn("save as phoenix ok !!!")
sc.stop()
}
def main(args: Array[String]): Unit = {
val Array(master)=args
saveAsphoenixProcess(master)
}
}
... View more
