Reply
New Contributor
Posts: 3
Registered: ‎02-18-2018

Why application master killing container at mid of the mapper progress?

[ Edited ]
  1. Application Master killing container and some time container is not starting can any one help
    
        #=================map red site============
    
        <!--
          Licensed under the Apache License, Version 2.0 (the "License");
          you may not use this file except in compliance with the License.
          You may obtain a copy of the License at
    
            http://www.apache.org/licenses/LICENSE-2.0
    
          Unless required by applicable law or agreed to in writing, software
          distributed under the License is distributed on an "AS IS" BASIS,
          WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
          See the License for the specific language governing permissions and
          limitations under the License. See accompanying LICENSE file.
        -->
    
        <!-- Put site-specific property overrides in this file. -->
    
        <configuration>
    
        <property>
        <name>yarn.App.map reduce.am.resource.Mb</name>
        <value>6144</value>
        </property>
        <property>
        <name>yarn.App.map reduce.am.command-opts</name>
        <value>-Xmx4096m</value>
        </property>
        <property>
        <name>map reduce.framework.name</name>
        <value>yarn</value>
        <description>Execution framework.</description>
        </property>
        <property>
        <name>map reduce.map.CPU.v cores</name>
        <value>2</value>
        <description>The number of virtual cores required for each map task.</description>
        </property>
        <property>
        <name>map reduce.reduce.CPU.v cores</name>
        <value>2</value>
        <description>The number of virtual cores required for each map task.</description>
        </property>
        <property>
        <name>map reduce.map.memory.Mb</name>
        <value>2048</value>
        <description>Larger resource limit for maps.</description>
        </property>
        <property>
        <name>map reduce.map.Java.opts</name>
        <value>-Xmx1638m</value>
        <description>Heap-size for child jvms of maps.</description>
        </property>
        <property>
        <name>map reduce.reduce.memory.Mb</name>
        <value>6144</value>
        <description>Larger resource limit for reduces.</description>
        </property>
        <property>
        <name>map reduce.reduce.Java.opts</name>
        <value>-Xmx4096m</value>
        <description>Heap-size for child jvms of reduces.</description>
        </property>
        <property>
        <name>map reduce.job-tracker.address</name>
        <value>Ip:0.0.0.0</value>
        </property>
        <property>
        <name>map reduce.task.timeout</name>
        <value>1200000</value>
        </property>
    
        </configuration>        enter code here
    
        #======================yarn site======================
    
        <!--
          Licensed under the Apache License, Version 2.0 (the "License");
          you may not use this file except in compliance with the License.
          You may obtain a copy of the License at
    
            http://www.apache.org/licenses/LICENSE-2.0
    
          Unless required by applicable law or agreed to in writing, software
          distributed under the License is distributed on an "AS IS" BASIS,
          WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
          See the License for the specific language governing permissions and
          limitations under the License. See accompanying LICENSE file.
        -->
        <configuration>
    
        <!-- Site specific YARN configuration properties -->
        <property>
        <name>yarn.resource manager.host name</name>
        <value>Ip:0.0.0.0</value> 
        </property>
        <property>
        <name>yarn.node manager.resource.memory-Mb</name>
        <value>25600</value>
        <description>Physical memory, in MB, to be made available to running containers</description>
    
            </property>
    
        <property>
        <name>yarn.scheduler.minimum-allocation-Mb</name>
        <value>2048</value>
        <description>Minimum limit of memory to allocate to each container request at the Resource Manager.</description>
        </property>
    
        <property>
        <name>yarn.scheduler.maximum-allocation-Mb</name>
        <value>25600</value>
        <description>Maximum limit of memory to allocate to each container request at the Resource Manager.</description>
        </property>
    
        <property>
        <name>yarn.scheduler.minimum-allocation-v cores</name>
        <value>1</value>
        <description>The minimum allocation for every container request at the RM, in terms of virtual CPU cores. Requests lower than this      won't take effect, and the specified value will get allocated the minimum.</description>
        </property>
    
        <property>
        <name>yarn.scheduler.maximum-allocation-v cores</name>
        <value>6</value>
        <description>The maximum allocation for every container request at the RM, in terms of virtual CPU cores. Requests higher than this         won't take effect, and will get capped to this value.</description>
        </property>
    
        <property>
        <name>yarn.node-manager.resource.CPU-vcores</name>
        <value>6</value>
        <description>Number of CPU cores that can be allocated for containers.</description>
        </property>
        <property>
        <name>yarn.node manager.aux-services</name>
        <value>map reduce_shuffle</value>
        <description>shuffle service that needs to be set for Map Reduce to run </description>
        </property>
    
        <property>
        <name>yarn.node-manager.VMS-check-enabled</name>
        <value>true</value>
        </property>
        <property>
        <name>yarn.node-manager.VMS-pmem-ratio</name>
        <value>4</value>
        </property>
        <property>
        <name>yarn.node-manager.disk-health-checker.min-healthy-disks</name>
        <value>0.25</value>
        </property>
    
        <property>
        <name>yarn.node-manager.pm-em-check-enabled</name>
        <value>true</value>
        </property>
        </configuration>2018-02-12 16:10:07,174 ERROR org.apache.hadoop.yarn.server.resource-manager.scheduler.Scheduler Application Attempt: Error trying to assign container token and NM token to an allocated container container_1518451240697_0001_01_000003
    java.lang.Illegal Argument Exception: java.net.Unknown Host Exception: coea18
Champion
Posts: 776
Registered: ‎05-16-2016

Re: Why application master killing container at mid of the mapper progress?

Could you give us a yarn log stack  error trace ? 

New Contributor
Posts: 3
Registered: ‎02-18-2018

Re: Why application master killing container at mid of the mapper progress?

ERROR org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt: Error trying to assign container token and NM token to an allocated container container_1518005003946_0001_01_000010
java.lang.IllegalArgumentException: java.net.UnknownHostException: coea19
at org.apache.hadoop.security.SecurityUtil.buildTokenService(SecurityUtil.java:377)
at org.apache.hadoop.yarn.server.utils.BuilderUtils.newContainerToken(BuilderUtils.java:258)
at org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager.createContainerToken(RMContainerTokenSecretManager.java:220)
at org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt.pullNewlyAllocatedContainersAndNMTokens(SchedulerApplicationAttempt.java:455)
at org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp.getAllocation(FiCaSchedulerApp.java:269)
at org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler.allocate(CapacityScheduler.java:988)
at org.apache.hadoop.yarn.server.resourcemanager.ApplicationMasterService.allocate(ApplicationMasterService.java:533)
at org.apache.hadoop.yarn.api.impl.pb.service.ApplicationMasterProtocolPBServiceImpl.allocate(ApplicationMasterProtocolPBServiceImpl.java:60)
at org.apache.hadoop.yarn.proto.ApplicationMasterProtocol$ApplicationMasterProtocolService$2.callBlockingMethod(ApplicationMasterProtocol.java:99)
at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:616)
at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:982)
at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:2049)
at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:2045)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:422)
at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1698)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2043)
Caused by: java.net.UnknownHostException: coea19
... 17 more

Highlighted
New Contributor
Posts: 3
Registered: ‎02-18-2018

Re: Why application master killing container at mid of the mapper progress?

Bro i have to machines with the 32GB RAM, 8 VCORES of each, 1Tb HD and i used only 25GB RAM,6 Vcores with the above configuration. i have given that configuration in both system and when i runed mahout LIb in single machine it creating upto 10 containers.if i run in master with slave at the two to three container i'm getting memory exception 

 

testuser@coea19:~/mahout/bin$ ./mahout trainnb -i /resultdata/20trainvectors -o /resultdata/20model -li /resultdata/labelindex -ow -c Running on hadoop, using /home/testuser/hadoop/bin/hadoop and HADOOP_CONF_DIR= MAHOUT-JOB: /home/testuser/mahout/mahout-examples-0.13.0-job.jar 18/02/12 16:09:53 WARN MahoutDriver: No trainnb.props found on classpath, will use command-line arguments only 18/02/12 16:09:53 INFO AbstractJob: Command line arguments: {--alphaI=[1.0], --endPhase=[2147483647], --input=[/resultdata/20trainvectors], --labelIndex=[/resultdata/labelindex], --output=[/resultdata/20model], --overwrite=null, --startPhase=[0], --tempDir=[temp], --trainComplementary=null} 18/02/12 16:09:53 INFO ZlibFactory: Successfully loaded & initialized native-zlib library 18/02/12 16:09:53 INFO CodecPool: Got brand-new compressor [.deflate] 18/02/12 16:09:53 INFO CodecPool: Got brand-new decompressor [.deflate] 18/02/12 16:09:54 INFO deprecation: mapred.input.dir is deprecated. Instead, use mapreduce.input.fileinputformat.inputdir 18/02/12 16:09:54 INFO deprecation: mapred.compress.map.output is deprecated. Instead, use mapreduce.map.output.compress 18/02/12 16:09:54 INFO deprecation: mapred.output.dir is deprecated. Instead, use mapreduce.output.fileoutputformat.outputdir 18/02/12 16:09:54 INFO RMProxy: Connecting to ResourceManager at /172.27.35.79:8032 18/02/12 16:09:55 INFO FileInputFormat: Total input paths to process : 1 18/02/12 16:09:55 INFO JobSubmitter: number of splits:1 18/02/12 16:09:55 INFO JobSubmitter: Submitting tokens for job: job_1518451240697_0001 18/02/12 16:09:55 INFO YarnClientImpl: Submitted application application_1518451240697_0001 18/02/12 16:09:55 INFO Job: The url to track the job: http://master:8088/proxy/application_1518451240697_0001/ 18/02/12 16:09:55 INFO Job: Running job: job_1518451240697_0001 18/02/12 16:10:00 INFO Job: Job job_1518451240697_0001 running in uber mode : false 18/02/12 16:10:00 INFO Job: map 0% reduce 0% 18/02/12 16:10:04 INFO Job: map 100% reduce 0% ^Z

Announcements