[spark@sandbox spark-client]$ ./bin/spark-submit --class org.apache.spark.examples.SparkPi --master yarn-client --num-executors 3 --driver-memory 512m --executor-memory 512m --executor-cores 1 lib/spark-examples*.jar 10
16/03/13 14:07:51 INFO SparkContext: Running Spark version 1.6.0
16/03/13 14:07:51 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
16/03/13 14:07:52 INFO SecurityManager: Changing view acls to: spark
16/03/13 14:07:52 INFO SecurityManager: Changing modify acls to: spark
16/03/13 14:07:52 INFO SecurityManager: SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(spark); users with modify permissions: Set(spark)
16/03/13 14:07:52 INFO Utils: Successfully started service 'sparkDriver' on port 39409.
16/03/13 14:07:53 INFO Slf4jLogger: Slf4jLogger started
16/03/13 14:07:53 INFO Remoting: Starting remoting
16/03/13 14:07:53 INFO Remoting: Remoting started; listening on addresses :[akka.tcp://sparkDriverActorSystem@192.168.1.106:36036]
16/03/13 14:07:53 INFO Utils: Successfully started service 'sparkDriverActorSystem' on port 36036.
16/03/13 14:07:53 INFO SparkEnv: Registering MapOutputTracker
16/03/13 14:07:53 INFO SparkEnv: Registering BlockManagerMaster
16/03/13 14:07:53 INFO DiskBlockManager: Created local directory at /tmp/blockmgr-16b99650-9632-41d3-8c2e-7d67f4e1d506
16/03/13 14:07:53 INFO MemoryStore: MemoryStore started with capacity 143.6 MB
16/03/13 14:07:53 INFO SparkEnv: Registering OutputCommitCoordinator
16/03/13 14:07:53 INFO Server: jetty-8.y.z-SNAPSHOT
16/03/13 14:07:54 WARN AbstractLifeCycle: FAILED SelectChannelConnector@0.0.0.0:4040: java.net.BindException: Address already in use
java.net.BindException: Address already in use
        at sun.nio.ch.Net.bind0(Native Method)
        at sun.nio.ch.Net.bind(Net.java:463)
        at sun.nio.ch.Net.bind(Net.java:455)
        at sun.nio.ch.ServerSocketChannelImpl.bind(ServerSocketChannelImpl.java:223)
        at sun.nio.ch.ServerSocketAdaptor.bind(ServerSocketAdaptor.java:74)
        at org.spark-project.jetty.server.nio.SelectChannelConnector.open(SelectChannelConnector.java:187)
        at org.spark-project.jetty.server.AbstractConnector.doStart(AbstractConnector.java:316)
        at org.spark-project.jetty.server.nio.SelectChannelConnector.doStart(SelectChannelConnector.java:265)
        at org.spark-project.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:64)
        at org.spark-project.jetty.server.Server.doStart(Server.java:293)
        at org.spark-project.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:64)
        at org.apache.spark.ui.JettyUtils$.org$apache$spark$ui$JettyUtils$$connect$1(JettyUtils.scala:252)
        at org.apache.spark.ui.JettyUtils$$anonfun$5.apply(JettyUtils.scala:262)
        at org.apache.spark.ui.JettyUtils$$anonfun$5.apply(JettyUtils.scala:262)
        at org.apache.spark.util.Utils$$anonfun$startServiceOnPort$1.apply$mcVI$sp(Utils.scala:1964)
        at scala.collection.immutable.Range.foreach$mVc$sp(Range.scala:141)
        at org.apache.spark.util.Utils$.startServiceOnPort(Utils.scala:1955)
        at org.apache.spark.ui.JettyUtils$.startJettyServer(JettyUtils.scala:262)
        at org.apache.spark.ui.WebUI.bind(WebUI.scala:136)
        at org.apache.spark.SparkContext$$anonfun$13.apply(SparkContext.scala:481)
        at org.apache.spark.SparkContext$$anonfun$13.apply(SparkContext.scala:481)
        at scala.Option.foreach(Option.scala:236)
        at org.apache.spark.SparkContext.<init>(SparkContext.scala:481)
        at org.apache.spark.examples.SparkPi$.main(SparkPi.scala:29)
        at org.apache.spark.examples.SparkPi.main(SparkPi.scala)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:606)
        at org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:731)
        at org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:181)
        at org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:206)
        at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:121)
        at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
16/03/13 14:07:54 WARN AbstractLifeCycle: FAILED org.spark-project.jetty.server.Server@707eaa3: java.net.BindException: Address already in use
java.net.BindException: Address already in use
        at sun.nio.ch.Net.bind0(Native Method)
        at sun.nio.ch.Net.bind(Net.java:463)
        at sun.nio.ch.Net.bind(Net.java:455)
        at sun.nio.ch.ServerSocketChannelImpl.bind(ServerSocketChannelImpl.java:223)
        at sun.nio.ch.ServerSocketAdaptor.bind(ServerSocketAdaptor.java:74)
        at org.spark-project.jetty.server.nio.SelectChannelConnector.open(SelectChannelConnector.java:187)
        at org.spark-project.jetty.server.AbstractConnector.doStart(AbstractConnector.java:316)
        at org.spark-project.jetty.server.nio.SelectChannelConnector.doStart(SelectChannelConnector.java:265)
        at org.spark-project.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:64)
        at org.spark-project.jetty.server.Server.doStart(Server.java:293)
        at org.spark-project.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:64)
        at org.apache.spark.ui.JettyUtils$.org$apache$spark$ui$JettyUtils$$connect$1(JettyUtils.scala:252)
        at org.apache.spark.ui.JettyUtils$$anonfun$5.apply(JettyUtils.scala:262)
        at org.apache.spark.ui.JettyUtils$$anonfun$5.apply(JettyUtils.scala:262)
        at org.apache.spark.util.Utils$$anonfun$startServiceOnPort$1.apply$mcVI$sp(Utils.scala:1964)
        at scala.collection.immutable.Range.foreach$mVc$sp(Range.scala:141)
        at org.apache.spark.util.Utils$.startServiceOnPort(Utils.scala:1955)
        at org.apache.spark.ui.JettyUtils$.startJettyServer(JettyUtils.scala:262)
        at org.apache.spark.ui.WebUI.bind(WebUI.scala:136)
        at org.apache.spark.SparkContext$$anonfun$13.apply(SparkContext.scala:481)
        at org.apache.spark.SparkContext$$anonfun$13.apply(SparkContext.scala:481)
        at scala.Option.foreach(Option.scala:236)
        at org.apache.spark.SparkContext.<init>(SparkContext.scala:481)
        at org.apache.spark.examples.SparkPi$.main(SparkPi.scala:29)
        at org.apache.spark.examples.SparkPi.main(SparkPi.scala)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:606)
        at org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:731)
        at org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:181)
        at org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:206)
        at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:121)
        at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
16/03/13 14:07:54 INFO ContextHandler: stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null}
16/03/13 14:07:54 INFO ContextHandler: stopped o.s.j.s.ServletContextHandler{/api,null}
16/03/13 14:07:54 INFO ContextHandler: stopped o.s.j.s.ServletContextHandler{/,null}
16/03/13 14:07:54 INFO ContextHandler: stopped o.s.j.s.ServletContextHandler{/static,null}
16/03/13 14:07:54 INFO ContextHandler: stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null}
16/03/13 14:07:54 INFO ContextHandler: stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null}
16/03/13 14:07:54 INFO ContextHandler: stopped o.s.j.s.ServletContextHandler{/executors/json,null}
16/03/13 14:07:54 INFO ContextHandler: stopped o.s.j.s.ServletContextHandler{/executors,null}
16/03/13 14:07:54 INFO ContextHandler: stopped o.s.j.s.ServletContextHandler{/environment/json,null}
16/03/13 14:07:54 INFO ContextHandler: stopped o.s.j.s.ServletContextHandler{/environment,null}
16/03/13 14:07:54 INFO ContextHandler: stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null}
16/03/13 14:07:54 INFO ContextHandler: stopped o.s.j.s.ServletContextHandler{/storage/rdd,null}
16/03/13 14:07:54 INFO ContextHandler: stopped o.s.j.s.ServletContextHandler{/storage/json,null}
16/03/13 14:07:54 INFO ContextHandler: stopped o.s.j.s.ServletContextHandler{/storage,null}
16/03/13 14:07:54 INFO ContextHandler: stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null}
16/03/13 14:07:54 INFO ContextHandler: stopped o.s.j.s.ServletContextHandler{/stages/pool,null}
16/03/13 14:07:54 INFO ContextHandler: stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null}
16/03/13 14:07:54 INFO ContextHandler: stopped o.s.j.s.ServletContextHandler{/stages/stage,null}
16/03/13 14:07:54 INFO ContextHandler: stopped o.s.j.s.ServletContextHandler{/stages/json,null}
16/03/13 14:07:54 INFO ContextHandler: stopped o.s.j.s.ServletContextHandler{/stages,null}
16/03/13 14:07:54 INFO ContextHandler: stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null}
16/03/13 14:07:54 INFO ContextHandler: stopped o.s.j.s.ServletContextHandler{/jobs/job,null}
16/03/13 14:07:54 INFO ContextHandler: stopped o.s.j.s.ServletContextHandler{/jobs/json,null}
16/03/13 14:07:54 INFO ContextHandler: stopped o.s.j.s.ServletContextHandler{/jobs,null}
16/03/13 14:07:54 WARN Utils: Service 'SparkUI' could not bind on port 4040. Attempting port 4041.
16/03/13 14:07:54 INFO Server: jetty-8.y.z-SNAPSHOT
16/03/13 14:07:54 INFO AbstractConnector: Started SelectChannelConnector@0.0.0.0:4041
16/03/13 14:07:54 INFO Utils: Successfully started service 'SparkUI' on port 4041.
16/03/13 14:07:54 INFO SparkUI: Started SparkUI at http://192.168.1.106:4041
16/03/13 14:07:54 INFO HttpFileServer: HTTP File server directory is /tmp/spark-3187f9c3-801c-471a-95b8-60164eae7993/httpd-33d2aef0-0489-4904-b775-86c971004810
16/03/13 14:07:54 INFO HttpServer: Starting HTTP Server
16/03/13 14:07:54 INFO Server: jetty-8.y.z-SNAPSHOT
16/03/13 14:07:54 INFO AbstractConnector: Started SocketConnector@0.0.0.0:43214
16/03/13 14:07:54 INFO Utils: Successfully started service 'HTTP file server' on port 43214.
16/03/13 14:07:54 INFO SparkContext: Added JAR file:/usr/hdp/2.4.0.0-169/spark/lib/spark-examples-1.6.0.2.4.0.0-169-hadoop2.7.1.2.4.0.0-169.jar at http://192.168.1.106:43214/jars/spark-examples-1.6.0.2.4.0.0-169-hadoop2.7.1.2.4.0.0-169.jar with timestamp 1457878074426
spark.yarn.driver.memoryOverhead is set but does not apply in client mode.
16/03/13 14:07:55 INFO TimelineClientImpl: Timeline service address: http://sandbox.hortonworks.com:8188/ws/v1/timeline/
16/03/13 14:07:55 INFO RMProxy: Connecting to ResourceManager at sandbox.hortonworks.com/192.168.1.106:8050
16/03/13 14:07:55 WARN DomainSocketFactory: The short-circuit local reads feature cannot be used because libhadoop cannot be loaded.
16/03/13 14:07:56 INFO Client: Requesting a new application from cluster with 1 NodeManagers
16/03/13 14:07:56 INFO Client: Verifying our application has not requested more than the maximum memory capability of the cluster (2250 MB per container)
16/03/13 14:07:56 INFO Client: Will allocate AM container, with 896 MB memory including 384 MB overhead
16/03/13 14:07:56 INFO Client: Setting up container launch context for our AM
16/03/13 14:07:56 INFO Client: Setting up the launch environment for our AM container
16/03/13 14:07:56 INFO Client: Using the spark assembly jar on HDFS because you are using HDP, defaultSparkAssembly:hdfs://sandbox.hortonworks.com:8020/hdp/apps/2.4.0.0-169/spark/spark-hdp-assembly.jar
16/03/13 14:07:56 INFO Client: Preparing resources for our AM container
16/03/13 14:07:56 INFO Client: Using the spark assembly jar on HDFS because you are using HDP, defaultSparkAssembly:hdfs://sandbox.hortonworks.com:8020/hdp/apps/2.4.0.0-169/spark/spark-hdp-assembly.jar
16/03/13 14:07:56 INFO Client: Source and destination file systems are the same. Not copying hdfs://sandbox.hortonworks.com:8020/hdp/apps/2.4.0.0-169/spark/spark-hdp-assembly.jar
16/03/13 14:07:56 INFO Client: Uploading resource file:/tmp/spark-3187f9c3-801c-471a-95b8-60164eae7993/__spark_conf__5020627426332142017.zip -> hdfs://sandbox.hortonworks.com:8020/user/spark/.sparkStaging/application_1457875462527_0009/__spark_conf__5020627426332142017.zip
16/03/13 14:07:56 INFO SecurityManager: Changing view acls to: spark
16/03/13 14:07:56 INFO SecurityManager: Changing modify acls to: spark
16/03/13 14:07:56 INFO SecurityManager: SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(spark); users with modify permissions: Set(spark)
16/03/13 14:07:56 INFO Client: Submitting application 9 to ResourceManager
16/03/13 14:07:56 INFO YarnClientImpl: Submitted application application_1457875462527_0009
16/03/13 14:07:56 INFO SchedulerExtensionServices: Starting Yarn extension services with app application_1457875462527_0009 and attemptId None
16/03/13 14:07:57 INFO Client: Application report for application_1457875462527_0009 (state: ACCEPTED)
16/03/13 14:07:57 INFO Client:
         client token: N/A
         diagnostics: N/A
         ApplicationMaster host: N/A
         ApplicationMaster RPC port: -1
         queue: default
         start time: 1457878076864
         final status: UNDEFINED
         tracking URL: http://sandbox.hortonworks.com:8088/proxy/application_1457875462527_0009/
         user: spark
16/03/13 14:07:58 INFO Client: Application report for application_1457875462527_0009 (state: ACCEPTED)
16/03/13 14:08:00 INFO Client: Application report for application_1457875462527_0009 (state: ACCEPTED)
16/03/13 14:08:01 INFO Client: Application report for application_1457875462527_0009 (state: ACCEPTED)
16/03/13 14:08:02 INFO Client: Application report for application_1457875462527_0009 (state: ACCEPTED)
16/03/13 14:08:03 INFO Client: Application report for application_1457875462527_0009 (state: ACCEPTED)
16/03/13 14:08:04 INFO Client: Application report for application_1457875462527_0009 (state: ACCEPTED)
^C16/03/13 14:08:04 INFO DiskBlockManager: Shutdown hook called
16/03/13 14:08:04 INFO ShutdownHookManager: Shutdown hook called
16/03/13 14:08:04 INFO ShutdownHookManager: Deleting directory /tmp/spark-3187f9c3-801c-471a-95b8-60164eae7993
16/03/13 14:08:04 INFO ShutdownHookManager: Deleting directory /tmp/spark-3187f9c3-801c-471a-95b8-60164eae7993/httpd-33d2aef0-0489-4904-b775-86c971004810
16/03/13 14:08:04 INFO ShutdownHookManager: Deleting directory /tmp/spark-3187f9c3-801c-471a-95b8-60164eae7993/userFiles-91baef28-6a61-4db1-b696-932bf19019c3
[spark@sandbox spark-client]$