Enabling Keberos for cluster fails when importing KDC account manager

[#1]

Hi,

I am trying to enable kerberos for my cloudera cluster. I have setup the keberos configuration file on the server and added principal for cloudera-scm but when importing the account manager credentials, I am getting following error. I tried to find solutions from already posted solutions, but all looks fine and still getting error.

Can anyone help.  

Here are my configurations and versions of Cloudera

CDH 5.12.2

Java Version: 1.7.0_75

priclusedge.a.15192.internal

 cat /etc/krb5.conf


[logging]
 default = FILE:/var/log/krb5libs.log
 kdc = FILE:/var/log/krb5kdc.log
 admin_server = FILE:/var/log/kadmind.log

[libdefaults]
 default_realm = PRICLUSTER.COM
 dns_lookup_realm = false
 dns_lookup_kdc = false
 ticket_lifetime = 24h
 renew_lifetime = 7d
 forwardable = true
 udp_preference_limit = 1000000
 default_tkt_enctypes = des-cbc-md5 des-cbc-crc des3-cbc-sha1
 default_tgs_enctypes = des-cbc-md5 des-cbc-crc des3-cbc-sha1
 permitted_enctypes = des-cbc-md5 des-cbc-crc des3-cbc-sha1

[realms]
 PRICLUSTER.COM = {
  kdc = priclusedge.a.15192.internal:88
  admin_server = priclusedge.a.15192.internal:749
  default_domain = pricluster.com
 }

[domain_realm]
  .pricluster.com = PRICLUSTER.COM
  pricluster.com = PRICLUSTER.COM

 cat kdc.conf



[kdcdefaults]
 kdc_ports = 88
 kdc_tcp_ports = 88
 v4_mode = nopreauth

[realms]
 PRICLUSTER.COM = {
  #master_key_type = aes256-cts
  acl_file = /var/kerberos/krb5kdc/kadm5.acl
  key_stash_file = /var/kerberos/krb5kdc/stash
  dict_file = /usr/share/dict/words
  admin_keytab = /var/kerberos/krb5kdc/kadm5.keytab
  database_name = /var/kerberos/krb5kdc/principal
  max_life = 1d
  max_renewable_life = 7d
  master_key_type = des3-hmac-sha1
  default_principal_flags = +preauth
  supported_enctypes = aes256-cts:normal aes128-cts:normal des3-hmac-sha1:normal des-hmac-sha1:normal des-cbc-crc:normal
 }

default_realm = PRICLUSTER.COM

[root@priclusedge krb5kdc]# kadmin.local

Authenticating as principal root/admin@PRICLUSTER.COM with password.
kadmin.local:  get_principals
K/M@PRICLUSTER.COM
cloudera-scm/admin@PRICLUSTER.COM
kadmin/admin@PRICLUSTER.COM
kadmin/changepw@PRICLUSTER.COM
kadmin/priclusedge.a.15192.internal@PRICLUSTER.COM
krbtgt/PRICLUSTER.COM@PRICLUSTER.COM

[root@priclusedge krb5kdc]# service krb5kdc status
krb5kdc (pid  6096) is running...
[root@priclusedge krb5kdc]# service kadmin status
kadmind (pid  6129) is running...

Error Message while importing accout manager credentials

/usr/share/cmf/bin/import_credentials.sh failed with exit code 1 and output of <<
+ export PATH=/usr/kerberos/bin:/usr/kerberos/sbin:/usr/lib/mit/sbin:/usr/sbin:/usr/lib/mit/bin:/usr/bin:/sbin:/usr/sbin:/bin:/usr/bin
+ PATH=/usr/kerberos/bin:/usr/kerberos/sbin:/usr/lib/mit/sbin:/usr/sbin:/usr/lib/mit/bin:/usr/bin:/sbin:/usr/sbin:/bin:/usr/bin
+ KEYTAB_OUT=/var/run/cloudera-scm-server/cmf8091152271730902012.keytab
+ USER=cloudera-scm/REDACTED@PRICLUSTER.COM
+ PASSWD=REDACTED
+ KVNO=1
+ SLEEP=0
+ RHEL_FILE=/etc/redhat-release
+ '[' -f /etc/redhat-release ']'
+ set +e
+ grep Tikanga /etc/redhat-release
+ '[' 1 -eq 0 ']'
+ '[' 0 -eq 0 ']'
+ grep 'CentOS release 5' /etc/redhat-release
+ '[' 1 -eq 0 ']'
+ '[' 0 -eq 0 ']'
+ grep 'Scientific Linux release 5' /etc/redhat-release
+ '[' 1 -eq 0 ']'
+ set -e
+ '[' -z /etc/krb5.conf ']'
+ echo 'Using custom config path '\''/etc/krb5.conf'\'', contents below:'
+ cat /etc/krb5.conf
+ IFS=' '
+ read -a ENC_ARR
+ ktutil
+ for ENC in '"${ENC_ARR[@]}"'
+ echo 'addent -password -p cloudera-scm/REDACTED@PRICLUSTER.COM -k 1 -e aes256-cts:normal'
+ '[' 0 -eq 1 ']'
+ echo REDACTED
+ for ENC in '"${ENC_ARR[@]}"'
+ echo 'addent -password -p cloudera-scm/REDACTED@PRICLUSTER.COM -k 1 -e aes128-cts:normal'
+ '[' 0 -eq 1 ']'
+ echo REDACTED
+ for ENC in '"${ENC_ARR[@]}"'
+ echo 'addent -password -p cloudera-scm/REDACTED@PRICLUSTER.COM -k 1 -e des3-hmac-sha1:normal'
+ '[' 0 -eq 1 ']'
+ echo REDACTED
+ for ENC in '"${ENC_ARR[@]}"'
+ echo 'addent -password -p cloudera-scm/REDACTED@PRICLUSTER.COM -k 1 -e des-hmac-sha1:normal'
+ '[' 0 -eq 1 ']'
+ echo REDACTED
+ for ENC in '"${ENC_ARR[@]}"'
+ echo 'addent -password -p cloudera-scm/REDACTED@PRICLUSTER.COM -k 1 -e des-cbc-crc:normal'
+ '[' 0 -eq 1 ']'
+ echo REDACTED
+ echo 'wkt /var/run/cloudera-scm-server/cmf8091152271730902012.keytab'
addent: Bad encryption type while adding new entry
ktutil: Unknown request "REDACTED".  Type "?" for a request list.
addent: Bad encryption type while adding new entry
ktutil: Unknown request "REDACTED".  Type "?" for a request list.
addent: Bad encryption type while adding new entry
ktutil: Unknown request "REDACTED".  Type "?" for a request list.
addent: Bad encryption type while adding new entry
ktutil: Unknown request "REDACTED".  Type "?" for a request list.
addent: Bad encryption type while adding new entry
ktutil: Unknown request "REDACTED".  Type "?" for a request list.
+ chmod 600 /var/run/cloudera-scm-server/cmf8091152271730902012.keytab
chmod: cannot access `/var/run/cloudera-scm-server/cmf8091152271730902012.keytab': No such file or directory

>>

[#2]

Hi,

anybody has any idea.. ? 

[#3]

@sandy05,

This is a tricky one, but, in the past, this sort of issue was resolved by adding a 1 second sleep to the import script.

(1)

Back up the following file:

/usr/share/cmf/bin/import_credentials.sh file on your Cloudera Manager host.

(2)

Edit /usr/share/cmf/bin/import_credentials.sh on your Cloudera Manager host

Locate this text near the top of the file:

# Determine if sleep is needed before echoing password.
# This is needed on Centos/RHEL 5 where ktutil doesn't
# accept password from stdin.
SLEEP=0

(3)

Change:

SLEEP=0

to:

SLEEP=1

(4)

Try using Cloudera Manager to import credentials again.

We have observed from time to time that timing in the "addent" commands in the script will lead to this sort of issue.  Adding some sleep has resovled it in the past.

Regards,

Ben

[#4]

Thanks.
I tried sleep=1 in the past and it didn't work. But let me try again. 'll
keep u updated.

[#5]

But even for centos or rhel6 needed it ?

[#6]

@sandy05,

No, not usually and that is why we didn't code to add sleep there.  To be honest, I don't know the history of the need for the "sleep" in some OSes and not others.  Indeed, it has not been needed for el6 as far as I know.

Based on your report of the issue, though, the situation usually ends up being resolved (in Cloudera internal cases) by inserting a sleep of 1 second.

If that doesn't help, let us know and share with us the edited file so we can verify the change.

Ben

[#7]

@bgooley

Thanks a lot for reverting back to my queries .   I tried with the solution you said by changing sleep=0 to sleep=1 but still  get the same error message . 

/usr/share/cmf/bin/import_credentials.sh failed with exit code 1 and output of <<
+ export PATH=/usr/kerberos/bin:/usr/kerberos/sbin:/usr/lib/mit/sbin:/usr/sbin:/usr/lib/mit/bin:/usr/bin:/sbin:/usr/sbin:/bin:/usr/bin
+ PATH=/usr/kerberos/bin:/usr/kerberos/sbin:/usr/lib/mit/sbin:/usr/sbin:/usr/lib/mit/bin:/usr/bin:/sbin:/usr/sbin:/bin:/usr/bin
+ KEYTAB_OUT=/var/run/cloudera-scm-server/cmf4656589489540061286.keytab
+ USER=cloudera-scm/REDACTED@PRICLUSTER.COM
+ PASSWD=REDACTED
+ KVNO=1
+ SLEEP=1
+ RHEL_FILE=/etc/redhat-release
+ '[' -f /etc/redhat-release ']'
+ set +e
+ grep Tikanga /etc/redhat-release
+ '[' 1 -eq 0 ']'
+ '[' 1 -eq 0 ']'
+ '[' 1 -eq 0 ']'
+ set -e
+ '[' -z /etc/krb5.conf ']'
+ echo 'Using custom config path '\''/etc/krb5.conf'\'', contents below:'
+ cat /etc/krb5.conf
+ IFS=' '
+ read -a ENC_ARR
+ for ENC in '"${ENC_ARR[@]}"'
+ echo 'addent -password -p cloudera-scm/REDACTED@PRICLUSTER.COM -k 1 -e aes256-cts:normal'
+ '[' 1 -eq 1 ']'
+ sleep 1
+ ktutil
addent: Bad encryption type while adding new entry
+ echo REDACTED
+ for ENC in '"${ENC_ARR[@]}"'
+ echo 'addent -password -p cloudera-scm/REDACTED@PRICLUSTER.COM -k 1 -e aes128-cts:normal'
+ '[' 1 -eq 1 ']'
+ sleep 1
ktutil: Unknown request "REDACTED".  Type "?" for a request list.
addent: Bad encryption type while adding new entry
+ echo REDACTED
+ for ENC in '"${ENC_ARR[@]}"'
+ echo 'addent -password -p cloudera-scm/REDACTED@PRICLUSTER.COM -k 1 -e des3-hmac-sha1:normal'
+ '[' 1 -eq 1 ']'
+ sleep 1
ktutil: Unknown request "REDACTED".  Type "?" for a request list.
addent: Bad encryption type while adding new entry
+ echo REDACTED
+ for ENC in '"${ENC_ARR[@]}"'
+ echo 'addent -password -p cloudera-scm/REDACTED@PRICLUSTER.COM -k 1 -e des-hmac-sha1:normal'
+ '[' 1 -eq 1 ']'
+ sleep 1
ktutil: Unknown request "REDACTED".  Type "?" for a request list.
addent: Bad encryption type while adding new entry
+ echo REDACTED
+ for ENC in '"${ENC_ARR[@]}"'
+ echo 'addent -password -p cloudera-scm/REDACTED@PRICLUSTER.COM -k 1 -e des-cbc-crc:normal'
+ '[' 1 -eq 1 ']'
+ sleep 1
ktutil: Unknown request "REDACTED".  Type "?" for a request list.
addent: Bad encryption type while adding new entry
+ echo REDACTED
+ echo 'wkt /var/run/cloudera-scm-server/cmf4656589489540061286.keytab'
ktutil: Unknown request "REDACTED".  Type "?" for a request list.
+ chmod 600 /var/run/cloudera-scm-server/cmf4656589489540061286.keytab
chmod: cannot access `/var/run/cloudera-scm-server/cmf4656589489540061286.keytab': No such file or directory

I have also shared the modified import_credentials.sh.  

cat /etc/redhat-release
CentOS release 6.9 (Final)

#!/usr/bin/env bash

# Copyright (c) 2014 Cloudera, Inc. All rights reserved.

set -e
set -x

# Explicitly add RHEL5/6 and SLES11/12 locations to path
export PATH=/usr/kerberos/bin:/usr/kerberos/sbin:/usr/lib/mit/sbin:/usr/sbin:/usr/lib/mit/bin:/usr/bin:$PATH

KEYTAB_OUT=$1
USER=$2
PASSWD=$3
KVNO=$4

# Determine if sleep is needed before echoing password.
# This is needed on Centos/RHEL 5 where ktutil doesn't
# accept password from stdin.
SLEEP=1
RHEL_FILE=/etc/redhat-release
if [ -f $RHEL_FILE ]; then
  set +e # Ignore errors in grep
  grep Tikanga $RHEL_FILE
  if [ $? -eq 0 ]; then
    SLEEP=1
  fi
  if [ $SLEEP -eq 0 ]; then
    grep 'CentOS release 5' $RHEL_FILE
    if [ $? -eq 0 ]; then
      SLEEP=1
    fi
  fi
  if [ $SLEEP -eq 0 ]; then
    grep 'Scientific Linux release 5' $RHEL_FILE
    if [ $? -eq 0 ]; then
      SLEEP=1
    fi
  fi
  set -e
fi

if [ -z "$KRB5_CONFIG" ]; then
  echo "Using system default krb5.conf path."
else
  echo "Using custom config path '$KRB5_CONFIG', contents below:"
  cat $KRB5_CONFIG
fi
# Export password to keytab
IFS=' ' read -a ENC_ARR <<< "$ENC_TYPES"
{
  for ENC in "${ENC_ARR[@]}"
  do
    echo "addent -password -p $USER -k $KVNO -e $ENC"
    if [ $SLEEP -eq 1 ]; then
      sleep 1
    fi
    echo "$PASSWD"
  done
  echo "wkt $KEYTAB_OUT"
} | ktutil

chmod 600 $KEYTAB_OUT

# Do a kinit to validate that everything works
kinit -k -t $KEYTAB_OUT $USER

# If this is not AD admin account, return from here
if [ "$AD_ADMIN" != "true" ]; then
  exit 0
fi

# With AD do a simple search to make sure everything works.
# Set properties needed for ldapsearch to work.
# Tell GSSAPI not to negotiate a security or privacy layer since
# AD doesn't support nested security or privacy layers
LDAP_CONF=`mktemp /tmp/cm_ldap.XXXXXXXX`
echo "TLS_REQCERT     never" >> $LDAP_CONF
echo "sasl_secprops   minssf=0,maxssf=0" >> $LDAP_CONF

export LDAPCONF=$LDAP_CONF

set +e # Allow failures to SASL so we can see if simple auth works
ldapsearch -LLL -H "$AD_SERVER" -b "$DOMAIN" "userPrincipalName=$USER"
if [ $? -ne 0 ]; then
  echo "ldapsearch did not work with SASL authentication. Trying with simple authentication"
  ldapsearch -LLL -H "$AD_SERVER" -b "$DOMAIN" -x -D $USER -w $PASSWD "userPrincipalName=$USER"
  if [ $? -ne 0 ]; then
    echo "Failed to do ldapsearch."
    echo "Please make sure Active Directory configuration is correctly specified and LDAP over SSL is enabled."
    exit 1
  fi
  # Simple authentication worked. Store the password in output file.
  echo -n $PASSWD > $KEYTAB_OUT
fi
set -e
rm -f $LDAP_CONF

[#8]

Hi, any solution pls..

[#9]

Any solution pls.. 

[#10]

@bgooley I have updated the script, can you please check as its still failing