All,   We can set the parameters if we do the following and it was a tough lesson to learn..   There is a config session for Service wide parameters and then for specific areas, do not list or mix which parameters exist for which session as it will throw an error, but not an intelligent one.    Additionally, I would recommend before setting parameters to run the script to dump the entire configuration into a file and then to review the specific parameters and the naming of them.  I was using the GUI and seeing items like HBASE.replication and thought that this could be used, which was not the case.  Instead, it was HBASE_enable_replication.   I also found what I believe was a naming issue in that the name of the parameter did not align with the value, it wasn't spelled correctly like "..ion" and was "..on".  The problem was I was spelling according to english vs reviewing the parameter name.     ... View more

Below is my script that I got running.. but sanitized AWS credentials, network, security.. etc    In shows how to attach multiple EBS volumes (2), set delete on terminate so that they clean up, tag the volumes/resources     ################################################################################ #   begin #   name: INSERT-NAME-OF-CLUSTER  #Ultimately should have comments on each line for understanding.   provider {    type: aws    region: us-west-2    subnetId: INSERT AWS Subnet ID    securityGroupsIds: INSERT SG-ID  or create IAM Profile    instanceNamePrefix: INSERT-NAME-OF-CLUSTER      rootVolumeSizeGB: 100    rootVolumeType: gp2    # associatePublicIpAddresses: true    # spotBimi-e3d81583dUSDPerHr: 0.50 }   ssh {    username: ec2-user # for RHEL image    privateKey: keypair.pem # with an absolute path to .pem file }   instances {   m42x {    type: m4.2xlarge   # requires an HVM AMI    image: INSERT AMI    rootVolumeSizeGB: 100      # iamProfileName: iam-profile-REPLACE-ME      tags {        owner: ${?USER}        Project: "INSERT Project name"        Vertical: "Insert Project vertical"    }      bootstrapScript: """#!/bin/sh # # This is an embedded bootstrap script that runs as root and can be used to customize # the instances immediately after boot and before any other Cloudera Director action export AWS_ACCESS_KEY_ID=INSERT AWS Access Key export AWS_SECRET_ACCESS_KEY=INSERT AWS Secret Key export INSTANCE_ID=$(curl http://instance-data/latest/meta-data/instance-id) export AVAILABILITY_ZONE=$(curl http://instance-data/latest/meta-data/placement/availability-zone)   export VOL1_ID=$(aws ec2 create-volume --size 15 --region us-west-2 --availability-zone ${AVAILABILITY_ZONE} --volume-type gp2 --query "VolumeId" | tr -d '"') export VOL2_ID=$(aws ec2 create-volume --size 15 --region us-west-2 --availability-zone ${AVAILABILITY_ZONE} --volume-type gp2 --query "VolumeId" | tr -d '"')     echo 'Sleeping 30' sleep 30 echo 'Create additional EBS volumes' aws ec2 attach-volume --region us-west-2 --volume-id ${VOL1_ID} --instance-id ${INSTANCE_ID} --device /dev/sdf aws ec2 attach-volume --region us-west-2 --volume-id ${VOL2_ID} --instance-id ${INSTANCE_ID} --device /dev/sdg     # Create filesystems on the devices echo 'Create filesystems' mkfs.ext4 /dev/xvdf mkfs.ext4 /dev/xvdg     # Create directories for the mount points mkdir /data1 mkdir /data2     # Add the mount points to /etc/fstab echo "/dev/xvdf /data1 ext4 defaults 0 0" >> /etc/fstab echo "/dev/xvdg /data2 ext4 defaults 0 0" >> /etc/fstab   sleep 10   # Mount all the devices echo 'Mount drives' mount -a # # modify attributes of new volumes to delete on terminate echo 'Modify volume attributes' aws ec2 modify-instance-attribute --instance-id ${INSTANCE_ID} --region us-west-2 --block-device-mappings "[{\"DeviceName\": \"/dev/sdf\",\"Ebs\":{\"DeleteOnTermination\":true}}]" aws ec2 modify-instance-attribute --instance-id ${INSTANCE_ID} --region us-west-2 --block-device-mappings "[{\"DeviceName\": \"/dev/sdg\",\"Ebs\":{\"DeleteOnTermination\":true}}]"     # # tag the volumes echo 'Tag resources'   aws ec2 create-tags --region us-west-2 --resources ${INSTANCE_ID} --tags Key=Application,Value=BIGDATA   Key=Vertical,Value=VerticalName Key=Project,Value="Project name" aws ec2 create-tags --region us-west-2 --resources ${VOL1_ID} --tags Key=Application,Value=BIGDATA   Key=Vertical,Value=VerticalName Key=Project,Value="Project name" aws ec2 create-tags --region us-west-2 --resources ${VOL2_ID} --tags Key=Application,Value=BIGDATA   Key=Vertical,Value=VerticalName Key=Project,Value="Project name"   #       # If the exit code is not zero Cloudera Director will automatically retry echo 'This is the last line of the script' exit 0   """      #    # Flag indicating whether to normalize the instance. Not setting normalization here implies that your    # bootstrap script will take care of normalization. This is an advanced configuration that will require    # assistance from Cloudera support.    #    # Normalization includes:    #   downloading and installing packages    #   minimizing swappiness    #   increasing the maximun number of open files    #   mounting ephemeral disks    #   resizing the root partition.    #    # Defaults to true    #   normalizeInstance: true   }   m44x {    type: m4.4xlarge    image: ami      tags {      owner: ${?USER}    } }   c34x {    type: c3.4xlarge    image: ami      tags {      owner: ${?USER}    } }   i2x {    type: i2.xlarge    image: ami      tags {      owner: ${?USER}    } }   i22x {    type: i2.2xlarge    image: ami      tags {      owner: ${?USER}    } }   r32x {    type: r3.2xlarge    image: ami      tags {      owner: ${?USER}    } }   t2l {   # only suitable as a gateway    type: t2.large    image: ami      tags {      owner: ${?USER}    } } }   cloudera-manager {      instance: ${instances.m42x} {        tags {            # add any additional tags as needed        Application: "Cloudera Manager 5"        Project: "place project name here"        Vertical: "place vertical name here"            }    }    repository: "http://archive.cloudera.com/cm5/redhat/6/x86_64/cm/5.8/"    repositoryKeyUrl: "http://archive.cloudera.com/cm5/redhat/6/x86_64/cm/RPM-GPG-KEY-cloudera"      enableEnterpriseTrial: true }   cluster {      products {      CDH: 5    }      configs {          # HDFS fencing should be set to true for HA configurations          HDFS {              dfs_ha_fencing_methods: "shell(true)"          }          }          services: [HDFS, ZOOKEEPER, HBASE ]      hdfsmaster-1 {      count: 2        instance: ${instances.m44x} {        tags {          group: hdfsmaster-1        }      }        roles {        HDFS: [NAMENODE, JOURNALNODE, FAILOVERCONTROLLER ]        ZOOKEEPER: [SERVER]      }   # NameNode nameservice, autofailover, and quorum journal name must be configured for high availability        configs {                HDFS {                NAMENODE {                        dfs_federation_namenode_nameservice: hanameservice                        autofailover_enabled: true                        dfs_namenode_quorum_journal_name: hanameservice                        }                }        }        }      hdfsmaster-2 {      count: 1        instance: ${instances.m44x} {        tags {          group: hdfsmaster-2        }      }        roles {        HDFS: [ JOURNALNODE, BALANCER]        ZOOKEEPER: [SERVER]        HBASE: [MASTER]      }      }      master-1 {      count: 1        instance: ${instances.m44x} {        tags {          group: master        }      }        roles {        HBASE: [MASTER]        ZOOKEEPER: [SERVER]      }        }      workers {      count: 3      minCount: 1        instance: ${instances.m42x} {            # placementGroup: REPLACE-ME          tags {          group: worker        }      }        roles {        HDFS: [DATANODE]        HBASE: [REGIONSERVER]      }    }        postCreateScripts: ["""#!/bin/sh echo 'Hello World!' exit 0    """,    """#!/usr/bin/python   print 'Hello again!'    """]      preTerminateScripts: ["""#!/bin/sh   echo 'Goodbye World!' exit 0    """,    """#!/usr/bin/python   print 'Goodbye again!'        """]   }   ... View more