Using the same stack in dev we successfully Kerberized the cluster. Now in production the process is being blocked at testing the KDC. The service check user is being created with a lower case realm;
prod_hdp-121415@abc.def.com instead of prod_hdp-121415@ABC.DEF.COM
When Ambari tries to kinit with this principal it fails;
resource_management.core.exceptions.Fail: Execution of '/usr/bin/kinit -c /var/lib/ambari-agent/tmp/kerberos_service_check_cc_25fc6ba87c6a6872eb2c3b3167344f73 -kt /etc/security/keytabs/kerberos.service_check.121415.keytab hdp_prod-121415@abc.def.com' returned 1. kinit: Cannot find KDC for requested realm while getting initial credentials
To rule out a problem with the prod admin account and the prod container, the dev container and dev admin account were used in the production Ambari with the same result. So this is a configuration problem specific to the production environment.
Where should I look for something that would be lower casing the realm? It is correct in Ambari and the generated krb5.conf file.
The krb5.conf file;
[libdefaults]
renew_lifetime = 7d
forwardable = true
default_realm = ABC.DEF.COM
ticket_lifetime = 24h
dns_lookup_realm = false
dns_lookup_kdc = false
#default_tgs_enctypes = aes des3-cbc-sha1 rc4 des-cbc-md5
#default_tkt_enctypes = aes des3-cbc-sha1 rc4 des-cbc-md5
[domain_realm]
abc.def.com = ABC.DEF.COM
.abc.def.com = ABC.DEF.COM
[logging]
default = FILE:/var/log/krb5kdc.log
admin_server = FILE:/var/log/kadmind.log
kdc = FILE:/var/log/krb5kdc.log
[realms]
ABC.DEF.COM =
{
admin_server = ldap.abc.def.com
kdc = ldap.abc.def.com
}
########## Performing 'GET' on (Site:krb5-conf, Tag:version1450197141581)
"properties" : {
"conf_dir" : "/etc",
"content" : "\n[libdefaults]\n renew_lifetime = 7d\n forwardable = true\n default_realm = {{realm}}\n ticket_lifetime = 24h\n dns_lookup_realm = false\n dns_lookup_kdc = false\n #default_tgs_enctypes = {{encryption_types}}\n #default_tkt_enctypes = {{encryption_types}}\n\n{% if domains %}\n[domain_realm]\n{% for domain in domains.split(',') %}\n {{domain}} = {{realm}}\n{% endfor %}\n{% endif %}\n\n[logging]\n default = FILE:/var/log/krb5kdc.log\n admin_server = FILE:/var/log/kadmind.log\n kdc = FILE:/var/log/krb5kdc.log\n\n[realms]\n {{realm}} = {\n admin_server = {{admin_server_host|default(kdc_host, True)}}\n kdc = {{kdc_host}}\n }\n\n{# Append additional realm declarations below #}",
"domains" : "abc.def.com,.abc.def.com",
"manage_krb5_conf" : "true"
}
########## Performing 'GET' on (Site:kerberos-env, Tag:version1450197141581)
"properties" : {
"ad_create_attributes_template" : "\n{\n \"objectClass\": [\"top\", \"person\", \"organizationalPerson\", \"user\"],\n \"cn\": \"$principal_name\",\n #if( $is_service )\n \"servicePrincipalName\": \"$principal_name\",\n #end\n \"userPrincipalName\": \"$normalized_principal\",\n \"unicodePwd\": \"$password\",\n \"accountExpires\": \"0\",\n \"userAccountControl\": \"66048\"\n}",
"admin_server_host" : "ldap.abc.def.com",
"case_insensitive_username_rules" : "false",
"container_dn" : "OU=Hadoop,OU=Users,DC=abc,DC=def,DC=com",
"encryption_types" : "aes des3-cbc-sha1 rc4 des-cbc-md5",
"executable_search_paths" : "/usr/bin, /usr/kerberos/bin, /usr/sbin, /usr/lib/mit/bin, /usr/lib/mit/sbin",
"install_packages" : "true",
"kdc_create_attributes" : "",
"kdc_host" : "ldap.abc.def.com",
"kdc_type" : "active-directory",
"ldap_url" : "ldaps://ldap.abc.def.com:636",
"manage_identities" : "true",
"password_length" : "20",
"password_min_digits" : "3",
"password_min_lowercase_letters" : "1",
"password_min_punctuation" : "1",
"password_min_uppercase_letters" : "1",
"password_min_whitespace" : "0",
"realm" : "ABC.DEF.COM",
"service_check_principal_name" : "${cluster_name}-${short_date}"
}
