Member since
05-12-2024
3
Posts
1
Kudos Received
0
Solutions
05-21-2024
09:14 PM
thanks for the solution. can you please share the documentation for hadoop delegation token only resource available is : https://blog.cloudera.com/hadoop-delegation-tokens-explained/
... View more
05-13-2024
11:18 PM
1 Kudo
package ptytest;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.security.UserGroupInformation;
import java.security.PrivilegedExceptionAction;
public class HdfsExample {
public static void main(String[] args) {
try {
Configuration conf = new Configuration();
conf.set("hadoop.security.authentication", "Kerberos");
conf.addResource(new Path("/etc/hadoop/conf.cloudera.hdfs/core-site.xml"));
conf.addResource(new Path("/etc/hadoop/conf.cloudera.hdfs/hdfs-site.xml"));
UserGroupInformation.setConfiguration(conf);
UserGroupInformation.loginUserFromKeytab("user1@EXAMPLE.COM", "/tmp/automation.keytab");
UserGroupInformation user1 = UserGroupInformation.getCurrentUser();
user1.doAs((PrivilegedExceptionAction<Void>) () -> {
FileSystem fs = FileSystem.get(conf);
Path root = new Path("/user/user1");
System.out.println("Listing root directory:");
fs.listStatus(root);
fs.close();
return null;
});
} catch (Exception e) {
e.printStackTrace();
}
}
} I am trying to run the above code
... View more
05-12-2024
08:31 PM
I am trying to access the hadoop filesystem but I am facing below error. How to provide the access right to the user. even getfacl command also giving same error. I have tried the kinit command user has valid TGT. My main objective is to use the HadoopDelegation token. but code is failing even when tried to just list the hdfs files. FileSystem fs = FileSystem.get(conf); Path root = new Path("/"); System.out.println((root)); fs.listStatus(root); ----- code failing on this line ERROR message : java -jar HadoopDelegation-1.0-SNAPSHOT.jar log4j:WARN No appenders could be found for logger (org.apache.hadoop.metrics2.lib.MutableMetricsFactory). log4j:WARN Please initialize the log4j system properly. log4j:WARN See http://logging.apache.org/log4j/1.2/faq.html#noconfig for more info. java.io.IOException: DestHost:destPort master.localdomain.com:8020 , LocalHost:localPort master.localdomain.com/10.49.0.149:0. Failed on local exception: java.io.IOException: org.apache.hadoop.security.AccessControlException: Client cannot authenticate via:[TOKEN, KERBEROS] at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:423) at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:831) at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:806) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1501) at org.apache.hadoop.ipc.Client.call(Client.java:1443) at org.apache.hadoop.ipc.Client.call(Client.java:1353) at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:228) at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:116) at com.sun.proxy.$Proxy11.getFileInfo(Unknown Source) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getFileInfo(ClientNamenodeProtocolTranslatorPB.java:900) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:422) at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:165) at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:157) at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:95) at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:359) at com.sun.proxy.$Proxy12.getFileInfo(Unknown Source) at org.apache.hadoop.hdfs.DFSClient.getFileInfo(DFSClient.java:1654) at org.apache.hadoop.hdfs.DistributedFileSystem$29.doCall(DistributedFileSystem.java:1579) at org.apache.hadoop.hdfs.DistributedFileSystem$29.doCall(DistributedFileSystem.java:1576) at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) at org.apache.hadoop.hdfs.DistributedFileSystem.getFileStatus(DistributedFileSystem.java:1591) at org.apache.hadoop.fs.FileSystem.exists(FileSystem.java:1734) at ptytest.HdfsExample.main(HdfsExample.java:31) Caused by: java.io.IOException: org.apache.hadoop.security.AccessControlException: Client cannot authenticate via:[TOKEN, KERBEROS] at org.apache.hadoop.ipc.Client$Connection$1.run(Client.java:757) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1729) at org.apache.hadoop.ipc.Client$Connection.handleSaslConnectionFailure(Client.java:720) at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:813) at org.apache.hadoop.ipc.Client$Connection.access$3600(Client.java:410) at org.apache.hadoop.ipc.Client.getConnection(Client.java:1558) at org.apache.hadoop.ipc.Client.call(Client.java:1389) ... 22 more Caused by: org.apache.hadoop.security.AccessControlException: Client cannot authenticate via:[TOKEN, KERBEROS] at org.apache.hadoop.security.SaslRpcClient.selectSaslClient(SaslRpcClient.java:173) at org.apache.hadoop.security.SaslRpcClient.saslConnect(SaslRpcClient.java:390) at org.apache.hadoop.ipc.Client$Connection.setupSaslConnection(Client.java:614) at org.apache.hadoop.ipc.Client$Connection.access$2300(Client.java:410) at org.apache.hadoop.ipc.Client$Connection$2.run(Client.java:800) at org.apache.hadoop.ipc.Client$Connection$2.run(Client.java:796) at java.security.AccessController.doPrivileged(Native Method) at javax.security.auth.Subject.doAs(Subject.java:422) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1729) at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:796) ... 25 more core-site.xml <?xml version="1.0" encoding="UTF-8"?> <!--Autogenerated by Cloudera Manager--> <configuration> <property> <name>hadoop.security.authentication</name> <value>kerberos</value> </property> <property> <name>hadoop.security.authorization</name> <value>true</value> </property> <property> <name>hadoop.rpc.protection</name> <value>authentication</value> </property> <property> <name>hadoop.http.header.Strict_Transport_Security</name> <value>max-age=0; includeSubDomains</value> </property> <property> <name>hadoop.ssl.enabled</name> <value>false</value> </property> <property> <name>hadoop.ssl.require.client.cert</name> <value>false</value> <final>true</final> </property> <property> <name>hadoop.ssl.keystores.factory.class</name> <value>org.apache.hadoop.security.ssl.FileBasedKeyStoresFactory</value> <final>true</final> </property> <property> <name>hadoop.ssl.server.conf</name> <value>ssl-server.xml</value> <final>true</final> </property> <property> <name>hadoop.ssl.client.conf</name> <value>ssl-client.xml</value> <final>true</final> </property> <property> <name>hadoop.security.auth_to_local</name> <value>RULE:[2:$1@$0](rangeradmin@EXAMPLE.COM)s/(.*)@EXAMPLE.COM/ranger/ RULE:[2:$1@$0](rangertagsync@EXAMPLE.COM)s/(.*)@EXAMPLE.COM/rangertagsync/ RULE:[2:$1@$0](rangerusersync@EXAMPLE.COM)s/(.*)@EXAMPLE.COM/rangerusersync/ DEFAULT</value> </property> <property> <name>hadoop.proxyuser.oozie.hosts</name> <value>*</value> </property> <property> <name>hadoop.proxyuser.oozie.groups</name> <value>*</value> </property> <property> <name>hadoop.proxyuser.HTTP.hosts</name> <value>*</value> </property> <property> <name>hadoop.proxyuser.HTTP.groups</name> <value>*</value> </property> <property> <name>hadoop.proxyuser.hive.hosts</name> <value>*</value> </property> <property> <name>hadoop.proxyuser.hive.groups</name> <value>*</value> </property> <property> <name>hadoop.proxyuser.hue.hosts</name> <value>*</value> </property> <property> <name>hadoop.proxyuser.hue.groups</name> <value>*</value> </property> <property> <name>hadoop.proxyuser.httpfs.hosts</name> <value>*</value> </property> <property> <name>hadoop.proxyuser.httpfs.groups</name> <value>*</value> </property> <property> <name>hadoop.proxyuser.knox.hosts</name> <value>*</value> </property> <property> <name>hadoop.proxyuser.knox.groups</name> <value>*</value> </property> <property> <name>hadoop.proxyuser.livy.hosts</name> <value>*</value> </property> <property> <name>hadoop.proxyuser.livy.groups</name> <value>*</value> </property> <property> <name>hadoop.proxyuser.impala.hosts</name> <value>*</value> </property> <property> <name>hadoop.proxyuser.impala.groups</name> <value>*</value> </property> <property> <name>hadoop.proxyuser.hdfs.groups</name> <value>*</value> </property> <property> <name>hadoop.proxyuser.hdfs.hosts</name> <value>*</value> </property> <property> <name>hadoop.proxyuser.yarn.hosts</name> <value>*</value> </property> <property> <name>hadoop.proxyuser.yarn.groups</name> <value>*</value> </property> <property> <name>hadoop.proxyuser.phoenix.hosts</name> <value>*</value> </property> <property> <name>hadoop.proxyuser.phoenix.groups</name> <value>*</value> </property> <property> <name>hadoop.proxyuser.kudu.hosts</name> <value>*</value> </property> <property> <name>hadoop.proxyuser.kudu.groups</name> <value>*</value> </property> <property> <name>hadoop.proxyuser.user2.hosts</name> <value>*</value> </property> <property> <name>hadoop.proxyuser.user2.groups</name> <value>*</value> </property> <property> <name>fs.defaultFS</name> <value>hdfs://master.localdomain.com:8020</value> </property> <property> <name>ipc.client.connection.maxidletime</name> <value>30000</value> </property> <property> <name>ipc.client.connect.max.retries</name> <value>50</value> </property> <property> <name>fs.trash.interval</name> <value>1</value> </property> <property> <name>io.compression.codecs</name> <value>org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.BZip2Codec,org.apache.hadoop.io.compress.DeflateCodec,org.apache.hadoop.io.compress.SnappyCodec,org.apache.hadoop.io.compress.Lz4Codec</value> </property> <property> <name>hadoop.security.group.mapping</name> <value>org.apache.hadoop.security.ShellBasedUnixGroupsMapping</value> </property> <property> <name>hadoop.security.instrumentation.requires.admin</name> <value>false</value> </property> </configuration> hdfs-site.xml: <?xml version="1.0" encoding="UTF-8"?> <!--Autogenerated by Cloudera Manager--> <configuration> <property> <name>dfs.namenode.name.dir</name> <value>file:///dfs/nn</value> </property> <property> <name>dfs.namenode.servicerpc-address</name> <value>master.localdomain.com:8022</value> </property> <property> <name>dfs.https.address</name> <value>master.localdomain.com:9871</value> </property> <property> <name>dfs.https.port</name> <value>9871</value> </property> <property> <name>dfs.namenode.http-address</name> <value>master.localdomain.com:9870</value> </property> <property> <name>dfs.replication</name> <value>3</value> </property> <property> <name>dfs.blocksize</name> <value>134217728</value> </property> <property> <name>dfs.client.use.datanode.hostname</name> <value>false</value> </property> <property> <name>fs.permissions.umask-mode</name> <value>022</value> </property> <property> <name>dfs.client.block.write.locateFollowingBlock.retries</name> <value>7</value> </property> <property> <name>dfs.encrypt.data.transfer.algorithm</name> <value>3des</value> </property> <property> <name>dfs.encrypt.data.transfer.cipher.suites</name> <value>AES/CTR/NoPadding</value> </property> <property> <name>dfs.encrypt.data.transfer.cipher.key.bitlength</name> <value>256</value> </property> <property> <name>dfs.namenode.acls.enabled</name> <value>true</value> </property> <property> <name>dfs.client.read.shortcircuit</name> <value>true</value> </property> <property> <name>dfs.client.read.shortcircuit.streams.cache.size</name> <value>4096</value> </property> <property> <name>dfs.domain.socket.path</name> <value>/var/run/hdfs-sockets/dn</value> </property> <property> <name>dfs.client.read.shortcircuit.skip.checksum</name> <value>false</value> </property> <property> <name>dfs.client.domain.socket.data.traffic</name> <value>false</value> </property> <property> <name>dfs.datanode.hdfs-blocks-metadata.enabled</name> <value>true</value> </property> <property> <name>dfs.block.access.token.enable</name> <value>true</value> </property> <property> <name>dfs.namenode.kerberos.principal</name> <value>hdfs/_HOST@EXAMPLE.COM</value> </property> <property> <name>dfs.namenode.kerberos.internal.spnego.principal</name> <value>HTTP/_HOST@EXAMPLE.COM</value> </property> <property> <name>dfs.datanode.kerberos.principal</name> <value>hdfs/_HOST@EXAMPLE.COM</value> </property> <property> <name>fs.defaultFS</name> <value>hdfs://master.localdomain.com:8020</value> </property> </configuration>
... View more