Created on 05-09-2018 12:51 PM - edited 08-18-2019 12:53 AM
Hello I have the following files and I want to deploy an HA ResourceManager:
{ "Blueprints": { "stack_name": "HDP", "stack_version": "2.6" }, "host_groups": [ { "name": "services", "cardinality" : "1", "components": [ { "name" : "HST_AGENT" }, { "name" : "MAPREDUCE2_CLIENT" }, { "name" : "YARN_CLIENT" }, { "name" : "HDFS_CLIENT" }, { "name" : "HBASE_CLIENT" }, { "name" : "METRICS_MONITOR" }, { "name" : "ZOOKEEPER_CLIENT" } ] }, { "name": "datanodes", "cardinality" : "3", "components": [ { "name" : "NODEMANAGER" }, { "name" : "HST_AGENT" }, { "name" : "MAPREDUCE2_CLIENT" }, { "name" : "YARN_CLIENT" }, { "name" : "HDFS_CLIENT" }, { "name" : "HBASE_REGIONSERVER" }, { "name" : "DATANODE" }, { "name" : "METRICS_MONITOR" }, { "name" : "ZOOKEEPER_CLIENT" }, { "name" : "HBASE_CLIENT" } ] }, { "name": "namenode1", "cardinality" : "1", "components": [ { "name" : "HST_AGENT" }, { "name" : "HDFS_CLIENT" }, { "name" : "ZKFC" }, { "name" : "ZOOKEEPER_SERVER" }, { "name" : "HST_SERVER" }, { "name" : "HBASE_CLIENT" }, { "name" : "METRICS_MONITOR" }, { "name" : "JOURNALNODE" }, { "name" : "HBASE_MASTER" }, { "name" : "NAMENODE" }, { "name" : "APP_TIMELINE_SERVER" }, { "name" : "METRICS_GRAFANA" } ] }, { "name": "namenode2", "cardinality" : "1", "components": [ { "name" : "HDFS_CLIENT" }, { "name" : "ZKFC" }, { "name" : "HISTORYSERVER" }, { "name" : "METRICS_MONITOR" }, { "name" : "HBASE_MASTER" }, { "name" : "NAMENODE" }, { "name" : "ACTIVITY_EXPLORER" }, { "name" : "HST_AGENT" }, { "name" : "ZOOKEEPER_SERVER" }, { "name" : "HBASE_CLIENT" }, { "name" : "JOURNALNODE" }, { "name" : "METRICS_COLLECTOR" }, { "name" : "RESOURCEMANAGER" } ] }, { "name": "namenode3", "cardinality" : "1", "components": [ { "name" : "ACTIVITY_ANALYZER" }, { "name" : "HST_AGENT" }, { "name" : "MAPREDUCE2_CLIENT" }, { "name" : "YARN_CLIENT" }, { "name" : "HDFS_CLIENT" }, { "name" : "ZOOKEEPER_SERVER" }, { "name" : "HBASE_CLIENT" }, { "name" : "METRICS_MONITOR" }, { "name" : "JOURNALNODE" }, { "name" : "RESOURCEMANAGER" } ] } ], "configurations": [ { "core-site": { "properties" : { "fs.defaultFS" : "hdfs://HACluster", "ha.zookeeper.quorum": "%HOSTGROUP::namenode1%:2181,%HOSTGROUP::namenode3%:2181", "hadoop.proxyuser.yarn.hosts": "%HOSTGROUP::namenode2%,%HOSTGROUP::namenode3%" } } }, { "hdfs-site": { "properties" : { "dfs.client.failover.proxy.provider.HACluster" : "org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider", "dfs.ha.automatic-failover.enabled" : "true", "dfs.ha.fencing.methods" : "shell(/bin/true)", "dfs.ha.namenodes.HACluster" : "nn1,nn2", "dfs.namenode.http-address" : "%HOSTGROUP::namenode1%:50070", "dfs.namenode.http-address.HACluster.nn1" : "%HOSTGROUP::namenode1%:50070", "dfs.namenode.http-address.HACluster.nn2" : "%HOSTGROUP::namenode2%:50070", "dfs.namenode.https-address" : "%HOSTGROUP::namenode1%:50470", "dfs.namenode.https-address.HACluster.nn1" : "%HOSTGROUP::namenode1%:50470", "dfs.namenode.https-address.HACluster.nn2" : "%HOSTGROUP::namenode2%:50470", "dfs.namenode.rpc-address.HACluster.nn1" : "%HOSTGROUP::namenode1%:8020", "dfs.namenode.rpc-address.HACluster.nn2" : "%HOSTGROUP::namenode2%:8020", "dfs.namenode.shared.edits.dir" : "qjournal://%HOSTGROUP::namenode1%:8485;%HOSTGROUP::namenode2%:8485;%HOSTGROUP::namenode3%:8485/HACluster", "dfs.nameservices" : "HACluster" } } }, { "yarn-site": { "properties": { "yarn.resourcemanager.ha.enabled": "true", "yarn.resourcemanager.ha.rm-ids": "rm1,rm2", "yarn.resourcemanager.hostname.rm1": "%HOSTGROUP::namenode3%", "yarn.resourcemanager.hostname.rm2": "%HOSTGROUP::namenode2%", "yarn.resourcemanager.webapp.address.rm1": "%HOSTGROUP::namenode3%:8088", "yarn.resourcemanager.webapp.address.rm2": "%HOSTGROUP::namenode2%:8088", "yarn.resourcemanager.webapp.https.address.rm1": "%HOSTGROUP::namenode3%:8090", "yarn.resourcemanager.webapp.https.address.rm2": "%HOSTGROUP::namenode2%:8090", "yarn.resourcemanager.recovery.enabled": "true", "yarn.resourcemanager.store.class": "org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore", "yarn.resourcemanager.zk-address": "%HOSTGROUP::namenode1%:2181,%HOSTGROUP::namenode3%:2181,%HOSTGROUP::namenode2%:2181", "yarn.client.failover-proxy-provider": "org.apache.hadoop.yarn.client.ConfiguredRMFailoverProxyProvider", "yarn.resourcemanager.cluster-id": "yarn-cluster", "yarn.resourcemanager.ha.automatic-failover.zk-base-path": "/yarn-leader-election" } } }, { "hdfs-site" : { "properties_attributes" : { }, "properties" : { "dfs.datanode.data.dir" : "/mnt/secondary1,/mnt/secondary2" } } }, { "hadoop-env" : { "properties_attributes" : { }, "properties" : { "namenode_heapsize" : "2048m" } } }, { "activity-zeppelin-shiro": { "properties": { "users.admin": "admin" } } }, { "hbase-site" : { "properties" : { "hbase.rootdir" : "hdfs://HACluster/apps/hbase/data" } } }, { "hbase-env" : { "properties" : { "hbase_regionserver_heapsize" : "1024" } } } ] }
{ "Repositories":{ "base_url":"http://public-repo-1.hortonworks.com/HDP-UTILS-1.1.0.22/repos/ubuntu16", "verify_base_url":true } }
{ "blueprint":"HACluster", "default_password":"admin", "host_groups": [ { "name": "services", "hosts": [ { "fqdn": "services01" } ] }, { "name": "datanodes", "hosts": [ { "fqdn": "datanode1" }, { "fqdn": "datanode2" }, { "fqdn": "datanode3" } ] }, { "name": "namenode1", "hosts": [ { "fqdn": "namenode1" } ] }, { "name": "namenode2", "hosts": [ { "fqdn": "namenode2" } ] }, { "name": "namenode3", "hosts": [ { "fqdn": "namenode3" } ] } ] }
{ "Repositories":{ "base_url":"http://public-repo-1.hortonworks.com/ambari/ubuntu16/2.x/updates/2.6.1.5", "verify_base_url":true } }
Then I launch this blueprint this way:
$ curl -i -H "X-Requested-By: ambari" -X POST -u admin:admin http://ambariserver:8080/api/v1/blueprints/HACluster -d @cluster_configuration.json $ curl -i -H "X-Requested-By: ambari" -X PUT -u admin:admin http://ambariserver:8080/api/v1/stacks/HDP/versions/2.6/operating_systems/ubuntu16/repositories/HDP-... -d @repo.json $ curl -i -H "X-Requested-By: ambari" -X PUT -u admin:admin http://ambariserver:8080/api/v1/stacks/HDP/versions/2.6/operating_systems/ubuntu16/repositories/HDP-... -d @hdputils-repo.json $ curl -i -H "X-Requested-By: ambari" -X POST -u admin:admin http://ambariserver:8080/api/v1/clusters/HACluster -d @hostmap.json
When everything is launched I get the following image... host are not registered and nothing works... If I delete RESOURCEMANAGER service from one host groups in cluster_configuration.json, everything works properly but "Resource Manager HA" is not deployed as HA...