{ "configurations": [ { "hdfs-site": { "properties": { "dfs.namenode.http-address": "localhost:50070", "dfs.namenode.https-address": "localhost:50470", "dfs.namenode.rpc-address": "localhost:8020", "dfs.namenode.secondary.http-address": "localhost:50090", "dfs.nameservices": "tc-mr", "dfs.internal.nameservices": "tc-mr", "dfs.ha.namenodes.tc-mr": "nn1,nn2", "dfs.client.failover.proxy.provider.tc-mr": "org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider", "dfs.ha.fencing.methods": "shell(/bin/true)", "dfs.ha.automatic-failover.enabled": "true", "dfs.namenode.shared.edits.dir": "qjournal://localhost:8485/tc-mr", "dfs.namenode.https-address.tc-mr.nn1": "%HOSTGROUP::master1%:50470", "dfs.namenode.http-address.tc-mr.nn1": "%HOSTGROUP::master1%:50070", "dfs.namenode.rpc-address.tc-mr.nn1": "%HOSTGROUP::master1%:8020", "dfs.namenode.https-address.tc-mr.nn2": "%HOSTGROUP::master2%:50470", "dfs.namenode.http-address.tc-mr.nn2": "%HOSTGROUP::master2%:50070", "dfs.namenode.rpc-address.tc-mr.nn2": "%HOSTGROUP::master2%:8020" } } }, { "core-site": { "properties": { "fs.defaultFS": "hdfs://tc-mr", "ha.zookeeper.quorum": "localhost:2181" } } }, { "hbase-site": { "properties": { "hbase.rootdir": "hdfs://tc-mr/apps/hbase/data" } } }, { "oozie-site": { "properties_attributes": {}, "properties": { "oozie.email.from.address": "hdp-prod-cloudbreak", "oozie.email.smtp.host": "mx.common.wr.truecarcorp.com" } } }, { "hbase-env": { "properties_attributes": {}, "properties": { "phoenix_sql_enabled": "true" } } }, { "capacity-scheduler": { "properties_attributes": {}, "properties": { "yarn.scheduler.capacity.root.queues": "default", "yarn.scheduler.capacity.root.default.acl_administer_jobs": "*", "yarn.scheduler.capacity.resource-calculator": "org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator", "yarn.scheduler.capacity.root.default.maximum-capacity": "100", "yarn.scheduler.capacity.node-locality-delay": "40", "yarn.scheduler.capacity.maximum-am-resource-percent": "0.3", "yarn.scheduler.capacity.root.default.acl_submit_applications": "*", "yarn.scheduler.capacity.root.default.state": "RUNNING", "yarn.scheduler.capacity.root.accessible-node-labels": "*", "yarn.scheduler.capacity.root.unfunded.capacity": "50", "yarn.scheduler.capacity.root.acl_administer_queue": "*", "yarn.scheduler.capacity.maximum-applications": "10000", "yarn.scheduler.capacity.root.default-node-label-expression": "", "yarn.scheduler.capacity.root.default.capacity": "100", "yarn.scheduler.capacity.root.default.user-limit-factor": "1", "yarn.scheduler.capacity.root.capacity": "100", "yarn.scheduler.capacity.default.minimum-user-limit-percent": "100" } } }, { "mapred-site": { "properties": { "mapreduce.map.log.level": "INFO", "mapreduce.job.counters.max": "300", "mapreduce.map.memory.mb": "4096", "mapreduce.map.speculative": "false", "mapreduce.reduce.memory.mb": "4096", "mapreduce.reduce.java.opts": "-Xmx3072m", "mapreduce.map.java.opts": "-Xmx3072m", "mapreduce.task.timeout": "600000", "mapreduce.application.classpath": "/etc/hadoop/conf:/etc/hbase/conf:$PWD/mr-framework/hadoop/share/hadoop/mapreduce/*:$PWD/mr-framework/hadoop/share/hadoop/mapreduce/lib/*:$PWD/mr-framework/hadoop/share/hadoop/common/*:$PWD/mr-framework/hadoop/share/hadoop/common/lib/*:$PWD/mr-framework/hadoop/share/hadoop/yarn/*:$PWD/mr-framework/hadoop/share/hadoop/yarn/lib/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/*:$PWD/mr-framework/hadoop/share/hadoop/hdfs/lib/*:$PWD/mr-framework/hadoop/share/hadoop/tools/lib/*:/usr/hdp/current/hadoop-client/lib/hadoop-lzo-0.6.0.${hdp.version}.jar:/etc/hadoop/conf/secure:/usr/hdp/current/phoenix-client/*:/usr/hdp/current/hbase-client/lib/*:/usr/hdp/current/phoenix-client/lib/*", "mapreduce.reduce.shuffle.fetch.retry.timeout-ms": "30000", "mapreduce.reduce.shuffle.fetch.retry.interval-ms": "1000", "mapreduce.reduce.shuffle.merge.percent": "0.66", "mapreduce.admin.reduce.child.java.opts": "-server -XX:NewRatio=8 -Djava.net.preferIPv4Stack=true -Dhdp.version=${hdp.version}", "mapreduce.admin.map.child.java.opts": "-server -XX:NewRatio=8 -Djava.net.preferIPv4Stack=true -Dhdp.version=${hdp.version}", "mapreduce.job.speculative.speculative-cap-running-tasks": "0.1", "io.sort.mb": "1000", "yarn.app.mapreduce.am.resource.mb": "4096" } } } ], "host_groups": [ { "name": "master1", "configurations": [], "components": [ { "name": "ZOOKEEPER_SERVER" }, { "name": "ZOOKEEPER_CLIENT" }, { "name": "NAMENODE" }, { "name": "JOURNALNODE" }, { "name": "ZKFC" }, { "name": "HDFS_CLIENT" }, { "name": "METRICS_MONITOR" }, { "name": "YARN_CLIENT" }, { "name": "MAPREDUCE2_CLIENT" }, { "name": "HISTORYSERVER" }, { "name": "SPARK2_JOBHISTORYSERVER" } ], "cardinality": "1" }, { "name": "master2", "configurations": [], "components": [ { "name": "ZOOKEEPER_SERVER" }, { "name": "ZOOKEEPER_CLIENT" }, { "name": "NAMENODE" }, { "name": "ZKFC" }, { "name": "JOURNALNODE" }, { "name": "HDFS_CLIENT" }, { "name": "HBASE_MASTER" }, { "name": "METRICS_MONITOR" }, { "name": "YARN_CLIENT" }, { "name": "MAPREDUCE2_CLIENT" }, { "name": "RESOURCEMANAGER" } ], "cardinality": "1" }, { "name": "master3", "configurations": [], "components": [ { "name": "ZOOKEEPER_SERVER" }, { "name": "ZOOKEEPER_CLIENT" }, { "name": "JOURNALNODE" }, { "name": "HDFS_CLIENT" }, { "name": "HBASE_MASTER" }, { "name": "METRICS_COLLECTOR" }, { "name": "APP_TIMELINE_SERVER" }, { "name": "WEBHCAT_SERVER" }, { "name": "HIVE_SERVER" }, { "name": "HIVE_METASTORE" }, { "name": "MYSQL_SERVER" }, { "name": "OOZIE_SERVER" }, { "name": "SPARK2_CLIENT" } ], "cardinality": "1" }, { "name": "worker", "configurations": [], "components": [ { "name": "DATANODE" }, { "name": "HDFS_CLIENT" }, { "name": "HBASE_REGIONSERVER" }, { "name": "HBASE_CLIENT" }, { "name": "ZOOKEEPER_CLIENT" }, { "name": "METRICS_MONITOR" }, { "name": "SQOOP" }, { "name": "PIG" }, { "name": "NODEMANAGER" }, { "name": "SPARK2_CLIENT" } ], "cardinality": "1" }, { "name": "client", "configurations": [], "components": [ { "name": "ZOOKEEPER_CLIENT" }, { "name": "PIG" }, { "name": "OOZIE_CLIENT" }, { "name": "HBASE_CLIENT" }, { "name": "METRICS_MONITOR" }, { "name": "SQOOP" }, { "name": "HDFS_CLIENT" }, { "name": "HIVE_CLIENT" }, { "name": "YARN_CLIENT" }, { "name": "MAPREDUCE2_CLIENT" }, { "name": "METRICS_GRAFANA" }, { "name": "SPARK2_CLIENT" } ], "cardinality": "1" }, { "name": "task", "configurations": [], "components": [ { "name": "HDFS_CLIENT" }, { "name": "HBASE_CLIENT" }, { "name": "ZOOKEEPER_CLIENT" }, { "name": "METRICS_MONITOR" }, { "name": "SQOOP" }, { "name": "PIG" }, { "name": "NODEMANAGER" }, { "name": "SPARK2_CLIENT" } ], "cardinality": "1" } ], "Blueprints": { "blueprint_name": "prod-mr-dataeng", "stack_name": "HDP", "stack_version": "2.6" } }