@Sunile Manjee Yes. Here is what I did. Let me know if you have any questions. try{
UserGroupInformation ugi = UserGroupInformation.loginUserFromKeytabAndReturnUGI(kerberos_principal, kerberos_keytab);
objectOfMyType = ugi.doAs(new PrivilegedExceptionAction<MyType>(){
@Override
public MyType run() throws Exception{
System.setProperty("spark.serializer","org.apache.spark.serializer.KryoSerializer");
System.setProperty("spark.kryo.registrator","fire.util.spark.Registrator");
System.setProperty("spark.akka.timeout","900");
System.setProperty("spark.worker.timeout","900");
System.setProperty("spark.storage.blockManagerSlaveTimeoutMs","3200000");
// create spark context
SparkConf sparkConf = new SparkConf().setAppName("MyApp");
sparkConf.setMaster("local");
sparkConf.set("spark.broadcast.compress", "false");
sparkConf.set("spark.shuffle.compress", "false");
JavaSparkContext ctx = new JavaSparkContext(sparkConf);
DataFrame tdf = ctx.sqlctx().read().format("com.databricks.spark.csv")
.option("header", String.valueOf(header)) // Use first line of all files as header
.option("inferSchema", "true") // Automatically infer data types
.option("delimiter", delimiter)
.load(path);
//some more application specific code here
return objectOfMyType;
}
});
}
catch (Exception exception){
exception.printStackTrace();
}
... View more
