Use case There are 2 groups Analytics and DW. We want to split the cluster resources between these 2 groups. User - neeraj belongs to Analytics group. User - dwuser belongs to DW group User neeraj is not allowed to use Default and dwuser queue. Be default, all the jobs submitted by user neeraj must go to it's assigned queue. User dwuser is not allowed to use Default and Analytics queue. By default, all the jobs submitted by user dwuser must go to it's assigned queue. Environment HDP 2.3 (Hortonworks Data Platform) and Ambari 2.1 This tutorial completely independent of Hadoop distribution. Yarn is must i,e Hadoop 2.x I will be using Capacity Scheduler view to configure queues. yarn.scheduler.capacity.maximum-am-resource-percent=0.2
yarn.scheduler.capacity.maximum-applications=10000
yarn.scheduler.capacity.node-locality-delay=40
yarn.scheduler.capacity.queue-mappings=u:neeraj:Analytics,u:dwuser:DW
yarn.scheduler.capacity.queue-mappings-override.enable=true
yarn.scheduler.capacity.root.accessible-node-labels=*
yarn.scheduler.capacity.root.acl_administer_queue=yarn
yarn.scheduler.capacity.root.acl_submit_applications=yarn
yarn.scheduler.capacity.root.Analytics.acl_administer_queue=yarn
yarn.scheduler.capacity.root.Analytics.acl_submit_applications=neeraj
yarn.scheduler.capacity.root.Analytics.capacity=60
yarn.scheduler.capacity.root.Analytics.maximum-capacity=60
yarn.scheduler.capacity.root.Analytics.minimum-user-limit-percent=100
yarn.scheduler.capacity.root.Analytics.ordering-policy=fifo
yarn.scheduler.capacity.root.Analytics.state=RUNNING
yarn.scheduler.capacity.root.Analytics.user-limit-factor=1
yarn.scheduler.capacity.root.capacity=100
yarn.scheduler.capacity.root.default.acl_administer_queue=yarn
yarn.scheduler.capacity.root.default.acl_submit_applications=yarn
yarn.scheduler.capacity.root.default.capacity=10
yarn.scheduler.capacity.root.default.maximum-capacity=100
yarn.scheduler.capacity.root.default.state=RUNNING
yarn.scheduler.capacity.root.default.user-limit-factor=1
yarn.scheduler.capacity.root.DW.acl_administer_queue=yarn
yarn.scheduler.capacity.root.DW.acl_submit_applications=dwuser
yarn.scheduler.capacity.root.DW.capacity=30
yarn.scheduler.capacity.root.DW.maximum-capacity=30
yarn.scheduler.capacity.root.DW.minimum-user-limit-percent=100
yarn.scheduler.capacity.root.DW.ordering-policy=fifo
yarn.scheduler.capacity.root.DW.state=RUNNING
yarn.scheduler.capacity.root.DW.user-limit-factor=1
yarn.scheduler.capacity.root.maximum-capacity=100
yarn.scheduler.capacity.root.queues=Analytics,DW,default [root@nsfed01 ~]# su - neeraj [neeraj@nsfed01 ~]$ mapred queue -showacls 15/08/18 14:45:03 INFO impl.TimelineClientImpl: Timeline service address: http://nsfed03.cloud.hortonworks.com:8188/ws/v1/timeline/ 15/08/18 14:45:03 INFO client.RMProxy: Connecting to ResourceManager at nsfed03.cloud.hortonworks.com/172.24.64.22:8050 Queue acls for user : neeraj Queue Operations ===================== root Analytics SUBMIT_APPLICATIONS DW default [neeraj@nsfed01 ~]$ [root@nsfed01 ~]# su - neeraj [neeraj@nsfed01 ~]$ yarn jar /usr/hdp/2.3.0.0-2557/hadoop-mapreduce/hadoop-mapreduce-examples.jar pi 20 1000000009 Number of Maps = 20 Samples [root@nsfed03 yarn]# su - dwuser [dwuser@nsfed03 ~]$ yarn jar /usr/hdp/2.3.0.0-2557/hadoop-mapreduce/hadoop-mapreduce-examples.jar pi 20 1000000009 Number of Maps = 20 CS view
... View more
