I am trying to load data into hdfs from local file system using Java SDK. I have created the jar(with all the dependencies[hadoop-mapreduce-client-common, hadoop-common, hadoop-client ver 2.7.3] Java 1.7) and when I ran it as a hadoop command it is working fine, but when I run it as a java jar it is throwing an error. Keytab is working fine and I can run kinit & klist successfully. Any help on this would be appreciated. hadoop jar ./AppHadoop-1.0-SNAPSHOT.jar org.apache.hadoop.examples.ProxyCreateSecureHDFSFile/user/hdfs_java user@XXXXX.XXXXX.COM keytabs/user.keytab /apache/hadoop/conf/core-site.xml /apache/hadoop/conf/hdfs-site.xml /apache/hadoop/conf/mapred-site.xml Working as expected. Success. java -cp ./AppHadoop-1.0-SNAPSHOT.jar org.apache.hadoop.examples.ProxyCreateSecureHDFSFile/user/hdfs_java user@XXXXX.XXXXX.COM keytabs/user.keytab /apache/hadoop/conf/core-site.xml /apache/hadoop/conf/hdfs-site.xml /apache/hadoop/conf/mapred-site.xml Output Home Directory : hdfs://namenode:8020/user/ Working Directory : hdfs://namenode:8020/user/ --------EXCEPTION________________ java.io.IOException:Failed on local exception: java.io.IOException: org.apache.hadoop.security.AccessControlException:Client cannot authenticate via:[TOKEN, KERBEROS];HostDetails: local host is:*"clientnode"*; destination host is:*"namenode"*:8020; at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:776) at org.apache.hadoop.ipc.Client.call(Client.java:1479) at org.apache.hadoop.ipc.Client.call(Client.java:1412) at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:229) at com.sun.proxy.$Proxy10.getFileInfo(UnknownSource) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getFileInfo(ClientNamenodeProtocolTranslatorPB.java:771) at sun.reflect.NativeMethodAccessorImpl.invoke0(NativeMethod) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:191) at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:102) at com.sun.proxy.$Proxy11.getFileInfo(UnknownSource) at org.apache.hadoop.hdfs.DFSClient.getFileInfo(DFSClient.java:2108) at org.apache.hadoop.hdfs.DistributedFileSystem$22.doCall(DistributedFileSystem.java:1305) at org.apache.hadoop.hdfs.DistributedFileSystem$22.doCall(DistributedFileSystem.java:1301) at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) at org.apache.hadoop.hdfs.DistributedFileSystem.getFileStatus(DistributedFileSystem.java:1317) at org.apache.hadoop.fs.FileSystem.exists(FileSystem.java:1426) at org.apache.hadoop.examples.ProxyCreateSecureHDFSFile$1.run(ProxyCreateSecureHDFSFile.java:133) at org.apache.hadoop.examples.ProxyCreateSecureHDFSFile$1.run(ProxyCreateSecureHDFSFile.java:123) at java.security.AccessController.doPrivileged(NativeMethod) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1698) at org.apache.hadoop.examples.ProxyCreateSecureHDFSFile.main(ProxyCreateSecureHDFSFile.java:123)Caused by: java.io.IOException: org.apache.hadoop.security.AccessControlException:Client cannot authenticate via:[TOKEN, KERBEROS] at org.apache.hadoop.ipc.Client$Connection$1.run(Client.java:687) at java.security.AccessController.doPrivileged(NativeMethod) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1698) at org.apache.hadoop.ipc.Client$Connection.handleSaslConnectionFailure(Client.java:650) at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:737) at org.apache.hadoop.ipc.Client$Connection.access$2900(Client.java:375) at org.apache.hadoop.ipc.Client.getConnection(Client.java:1528) at org.apache.hadoop.ipc.Client.call(Client.java:1451)...23 more Caused by: org.apache.hadoop.security.AccessControlException:Client cannot authenticate via:[TOKEN, KERBEROS] at org.apache.hadoop.security.SaslRpcClient.selectSaslClient(SaslRpcClient.java:172) at org.apache.hadoop.security.SaslRpcClient.saslConnect(SaslRpcClient.java:396) at org.apache.hadoop.ipc.Client$Connection.setupSaslConnection(Client.java:560) at org.apache.hadoop.ipc.Client$Connection.access$1900(Client.java:375) at org.apache.hadoop.ipc.Client$Connection$2.run(Client.java:729) at org.apache.hadoop.ipc.Client$Connection$2.run(Client.java:725) at java.security.AccessController.doPrivileged(NativeMethod) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1698) at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:725)...26 more This is my source code. package org.apache.hadoop.examples; import java.io.*; import java.security.PrivilegedExceptionAction; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.security.UserGroupInformation; public class ProxyCreateSecureHDFSFile { static String principal; static String keytab; public static void writeFileToHDFS(String in, String out, FileSystem fs) throws IOException { //read local file line by line and write to hdfs FileSystem hdfs =fs; BufferedReader br = new BufferedReader(new FileReader(in)); String line = ""; StringBuilder aux = new StringBuilder(); while ((line = br.readLine()) != null) { aux.append(line).append("\n"); } byte[] byt1=aux.toString().getBytes(); FSDataOutputStream fsOutStream3 = hdfs.create(new Path(out)); fsOutStream3.write(byt1); fsOutStream3.close(); br.close(); } public static void main(String[] args) throws IOException { if (args.length < 6){ System.out.println("Usage : <output-path> <principal> <keytab> <core-site.xml> <hdfs-site.xml> <mapred-site.xml>"); System.exit(1); } final String localInputPath = args[0]; final String outputArg = args[1]; final Path outputPath = new Path(outputArg); final String principal = args[2]; final String keytab = args[3]; final Configuration conf = new Configuration(); try { conf.set("hadoop.security.authentication", "Kerberos"); conf.set("debug", "true"); conf.set("sun.security.krb5.debug", "true"); conf.set("sun.security.spnego.debug", "true"); conf.set("hadoop.rpc.protection","authentication,privacy"); conf.addResource(new Path(args[4])); conf.addResource(new Path(args[5])); conf.addResource(new Path(args[6])); UserGroupInformation.getLoginUser().checkTGTAndReloginFromKeytab(); UserGroupInformation proxy = UserGroupInformation.createProxyUser("user@XXXXX.XXXXX.COM", UserGroupInformation.getLoginUser()); System.out.println(proxy.getUserName()); System.out.println(UserGroupInformation.getLoginUser().toString()); System.out.println(UserGroupInformation.getCurrentUser().toString()); proxy.doAs(new PrivilegedExceptionAction<Void>() { public Void run() throws Exception { FileSystem hdfs = FileSystem.get(conf); Path homeDir=hdfs.getHomeDirectory(); System.out.println("Home Directory : " +homeDir); Path workingDir=hdfs.getWorkingDirectory(); System.out.println("Working Directory : " +workingDir); if(hdfs.exists(outputPath)){ hdfs.delete(outputPath, true); } writeFileToHDFS("/home/user/file.txt", outputArg+"/file.txt", hdfs); return null; } }); } catch (Exception e) { System.out.println("--------EXCEPTION________________"); e.printStackTrace(); }}} I have to make sure this is working as a standard java application because the client machines won't have hadoop libraries installed. ... View more