only on one job having this issue     22/04/19 03:30:18 INFO executor.CoarseGrainedExecutorBackend: Driver commanded a shutdown 22/04/19 03:30:18 WARN server.TransportChannelHandler: Exception in connection from hostname018.corp.xyz/148.156.146.27:33617 java.util.concurrent.RejectedExecutionException: event executor terminated at io.netty.util.concurrent.SingleThreadEventExecutor.reject(SingleThreadEventExecutor.java:926) at io.netty.util.concurrent.SingleThreadEventExecutor.offerTask(SingleThreadEventExecutor.java:353) at io.netty.util.concurrent.SingleThreadEventExecutor.addTask(SingleThreadEventExecutor.java:346) at io.netty.util.concurrent.SingleThreadEventExecutor.execute(SingleThreadEventExecutor.java:828) at io.netty.util.concurrent.SingleThreadEventExecutor.execute(SingleThreadEventExecutor.java:818) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:250) at io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:241) at io.netty.channel.ChannelInboundHandlerAdapter.channelInactive(ChannelInboundHandlerAdapter.java:81) at org.apache.spark.network.server.TransportChannelHandler.channelInactive(TransportChannelHandler.java:118) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:262) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:248) at io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:241) at io.netty.channel.ChannelInboundHandlerAdapter.channelInactive(ChannelInboundHandlerAdapter.java:81) at io.netty.handler.timeout.IdleStateHandler.channelInactive(IdleStateHandler.java:277) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:262) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:248) at io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:241) at io.netty.channel.ChannelInboundHandlerAdapter.channelInactive(ChannelInboundHandlerAdapter.java:81) at org.apache.spark.network.util.TransportFrameDecoder.channelInactive(TransportFrameDecoder.java:182) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:262) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:248) at io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:241) at io.netty.channel.DefaultChannelPipeline$HeadContext.channelInactive(DefaultChannelPipeline.java:1405) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:262) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:248) at io.netty.channel.DefaultChannelPipeline.fireChannelInactive(DefaultChannelPipeline.java:901) at io.netty.channel.AbstractChannel$AbstractUnsafe$8.run(AbstractChannel.java:818) at io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:164) at io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472) at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:500) at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:748) 22/04/19 03:30:18 INFO memory.MemoryStore: MemoryStore cleared 22/04/19 03:30:18 WARN server.TransportChannelHandler: Exception in connection from hostname018.corp.xyz/148.156.146.27:33617 java.util.concurrent.RejectedExecutionException: event executor terminated at io.netty.util.concurrent.SingleThreadEventExecutor.reject(SingleThreadEventExecutor.java:926) at io.netty.util.concurrent.SingleThreadEventExecutor.offerTask(SingleThreadEventExecutor.java:353) at io.netty.util.concurrent.SingleThreadEventExecutor.addTask(SingleThreadEventExecutor.java:346) at io.netty.util.concurrent.SingleThreadEventExecutor.execute(SingleThreadEventExecutor.java:828) at io.netty.util.concurrent.SingleThreadEventExecutor.execute(SingleThreadEventExecutor.java:818) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelUnregistered(AbstractChannelHandlerContext.java:186) at io.netty.channel.AbstractChannelHandlerContext.fireChannelUnregistered(AbstractChannelHandlerContext.java:177) at io.netty.channel.ChannelInboundHandlerAdapter.channelUnregistered(ChannelInboundHandlerAdapter.java:57) at org.apache.spark.network.server.TransportChannelHandler.channelUnregistered(TransportChannelHandler.java:192) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelUnregistered(AbstractChannelHandlerContext.java:198) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelUnregistered(AbstractChannelHandlerContext.java:184) at io.netty.channel.AbstractChannelHandlerContext.fireChannelUnregistered(AbstractChannelHandlerContext.java:177) at io.netty.channel.DefaultChannelPipeline$HeadContext.channelUnregistered(DefaultChannelPipeline.java:1388) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelUnregistered(AbstractChannelHandlerContext.java:198) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelUnregistered(AbstractChannelHandlerContext.java:184) at io.netty.channel.DefaultChannelPipeline.fireChannelUnregistered(DefaultChannelPipeline.java:821) at io.netty.channel.AbstractChannel$AbstractUnsafe$8.run(AbstractChannel.java:826) at io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:164) at io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472) at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:500) at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:748) 22/04/19 03:30:18 INFO storage.BlockManager: BlockManager stopped 22/04/19 03:30:18 WARN server.TransportChannelHandler: Exception in connection from hostname018.corp.xyz/148.156.146.27:33617 java.util.concurrent.RejectedExecutionException: event executor terminated at io.netty.util.concurrent.SingleThreadEventExecutor.reject(SingleThreadEventExecutor.java:926) at io.netty.util.concurrent.SingleThreadEventExecutor.offerTask(SingleThreadEventExecutor.java:353) at io.netty.util.concurrent.SingleThreadEventExecutor.addTask(SingleThreadEventExecutor.java:346) at io.netty.util.concurrent.SingleThreadEventExecutor.execute(SingleThreadEventExecutor.java:828) at io.netty.util.concurrent.SingleThreadEventExecutor.execute(SingleThreadEventExecutor.java:818) at io.netty.channel.DefaultChannelPipeline.destroyUp(DefaultChannelPipeline.java:851) at io.netty.channel.DefaultChannelPipeline.destroy(DefaultChannelPipeline.java:836) at io.netty.channel.DefaultChannelPipeline.access$700(DefaultChannelPipeline.java:46) at io.netty.channel.DefaultChannelPipeline$HeadContext.channelUnregistered(DefaultChannelPipeline.java:1392) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelUnregistered(AbstractChannelHandlerContext.java:198) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelUnregistered(AbstractChannelHandlerContext.java:184) at io.netty.channel.DefaultChannelPipeline.fireChannelUnregistered(DefaultChannelPipeline.java:821) at io.netty.channel.AbstractChannel$AbstractUnsafe$8.run(AbstractChannel.java:826) at io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:164) at io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472) at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:500) at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:748) 22/04/19 03:30:18 INFO util.ShutdownHookManager: Shutdown hook called End of LogType:stderr *********************************************************************** End of LogType:stdout *********************************************************************** Container: container_e27_1646855837967_2018614_01_000003 on hostname033.corp.xyz_8041 LogAggregationType: AGGREGATED =========================================================================================== LogType:container-localizer-syslog LogLastModifiedTime:Tue Apr 19 03:31:11 -0500 2022 LogLength:184 LogContents: 2022-04-19 03:29:41,814 INFO [main] org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer: Disk Validator: yarn.nodemanager.disk-validator is loaded. End of LogType:container-localizer-syslog ******************************************************************************************* End of LogType:prelaunch.err ****************************************************************************** Container: container_e27_1646855837967_2018614_01_000003 on hostname033.corp.xyz_8041 LogAggregationType: AGGREGATED =========================================================================================== LogType:prelaunch.out LogLastModifiedTime:Tue Apr 19 03:31:11 -0500 2022 LogLength:70 LogContents: Setting up env variables Setting up job resources Launching container End of LogType:prelaunch.out ****************************************************************************** Container: container_e27_1646855837967_2018614_01_000003 on hostname033.corp.xyz_8041 LogAggregationType: AGGREGATED =========================================================================================== LogType:stderr LogLastModifiedTime:Tue Apr 19 03:31:11 -0500 2022 LogLength:20325 LogContents: [main] INFO ResourceCollector - Unravel Sensor 4.7.2.3-dev/2.0.3 initializing. 22/04/19 03:29:46 INFO executor.CoarseGrainedExecutorBackend: Started daemon with process name: 55954@hostname033.corp.xyz 22/04/19 03:29:46 INFO util.SignalUtils: Registered signal handler for TERM 22/04/19 03:29:46 INFO util.SignalUtils: Registered signal handler for HUP 22/04/19 03:29:46 INFO util.SignalUtils: Registered signal handler for INT 22/04/19 03:29:46 INFO spark.SecurityManager: Changing view acls to: cdlapp 22/04/19 03:29:46 INFO spark.SecurityManager: Changing modify acls to: cdlapp 22/04/19 03:29:46 INFO spark.SecurityManager: Changing view acls groups to: 22/04/19 03:29:46 INFO spark.SecurityManager: Changing modify acls groups to: 22/04/19 03:29:46 INFO spark.SecurityManager: SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(cdlapp); groups with view permissions: Set(); users with modify permissions: Set(cdlapp); groups with modify permissions: Set() 22/04/19 03:29:47 INFO client.TransportClientFactory: Successfully created connection to hostname018.corp.xyz/148.156.146.27:46519 after 116 ms (0 ms spent in bootstraps) 22/04/19 03:29:47 WARN spark.SparkConf: The configuration key 'spark.yarn.executor.memoryOverhead' has been deprecated as of Spark 2.3 and may be removed in the future. Please use the new key 'spark.executor.memoryOverhead' instead. 22/04/19 03:29:47 INFO deploy.SparkHadoopUtil: Updating delegation tokens for current user. 22/04/19 03:29:47 INFO spark.SecurityManager: Changing view acls to: cdlapp 22/04/19 03:29:47 INFO spark.SecurityManager: Changing modify acls to: cdlapp 22/04/19 03:29:47 INFO spark.SecurityManager: Changing view acls groups to: 22/04/19 03:29:47 INFO spark.SecurityManager: Changing modify acls groups to: 22/04/19 03:29:47 INFO spark.SecurityManager: SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(cdlapp); groups with view permissions: Set(); users with modify permissions: Set(cdlapp); groups with modify permissions: Set() 22/04/19 03:29:47 INFO client.TransportClientFactory: Successfully created connection to hostname018.corp.xyz/148.156.146.27:46519 after 4 ms (0 ms spent in bootstraps) 22/04/19 03:29:48 INFO storage.DiskBlockManager: Created local directory at /cdh/N/yarn/nm/usercache/cdlapp/appcache/application_1646855837967_2018614/blockmgr-74ab06df-9751-4cc7-8005-35500d6a5ec7 22/04/19 03:29:48 INFO storage.DiskBlockManager: Created local directory at /cdh/A/yarn/nm/usercache/cdlapp/appcache/application_1646855837967_2018614/blockmgr-6e5b2a32-623e-4030-a2f6-1c349ecfed0b 22/04/19 03:29:48 INFO storage.DiskBlockManager: Created local directory at /cdh/8/yarn/nm/usercache/cdlapp/appcache/application_1646855837967_2018614/blockmgr-9fb68b3a-e1ba-44ea-bd4e-0297e79538a9 22/04/19 03:29:48 INFO storage.DiskBlockManager: Created local directory at /cdh/F/yarn/nm/usercache/cdlapp/appcache/application_1646855837967_2018614/blockmgr-94ecec95-c92e-41f0-96f0-7787b561a33b 22/04/19 03:29:48 INFO storage.DiskBlockManager: Created local directory at /cdh/7/yarn/nm/usercache/cdlapp/appcache/application_1646855837967_2018614/blockmgr-cc55ba76-3b95-4726-98b6-d9e2cfa91b24 22/04/19 03:29:48 INFO storage.DiskBlockManager: Created local directory at /cdh/0/yarn/nm/usercache/cdlapp/appcache/application_1646855837967_2018614/blockmgr-488224ba-d746-4b67-ad15-5a3b5dfe56f7 22/04/19 03:29:48 INFO storage.DiskBlockManager: Created local directory at /cdh/I/yarn/nm/usercache/cdlapp/appcache/application_1646855837967_2018614/blockmgr-16d2c4e1-754e-4f62-8520-77a5e89a0b7e 22/04/19 03:29:48 INFO storage.DiskBlockManager: Created local directory at /cdh/K/yarn/nm/usercache/cdlapp/appcache/application_1646855837967_2018614/blockmgr-2b30ea24-0844-4126-bead-5f6283cc2641 22/04/19 03:29:48 INFO storage.DiskBlockManager: Created local directory at /cdh/D/yarn/nm/usercache/cdlapp/appcache/application_1646855837967_2018614/blockmgr-a4707124-2139-4797-8aa5-a7db2bce6cc2 22/04/19 03:29:48 INFO storage.DiskBlockManager: Created local directory at /cdh/C/yarn/nm/usercache/cdlapp/appcache/application_1646855837967_2018614/blockmgr-a298e760-d3d6-4fe7-b50f-84528c87c0e9 22/04/19 03:29:48 INFO storage.DiskBlockManager: Created local directory at /cdh/9/yarn/nm/usercache/cdlapp/appcache/application_1646855837967_2018614/blockmgr-81ab33ed-5f14-4cc4-912b-acee2453bfcb 22/04/19 03:29:48 INFO storage.DiskBlockManager: Created local directory at /cdh/H/yarn/nm/usercache/cdlapp/appcache/application_1646855837967_2018614/blockmgr-44ccf071-404f-4917-8d1b-3dd1f600c216 22/04/19 03:29:48 INFO storage.DiskBlockManager: Created local directory at /cdh/5/yarn/nm/usercache/cdlapp/appcache/application_1646855837967_2018614/blockmgr-8fea212e-0dd7-4a49-b9c8-f79dac207870 22/04/19 03:29:48 INFO storage.DiskBlockManager: Created local directory at /cdh/1/yarn/nm/usercache/cdlapp/appcache/application_1646855837967_2018614/blockmgr-0408d8a3-6af0-40db-8eee-8af8f772c037 22/04/19 03:29:48 INFO storage.DiskBlockManager: Created local directory at /cdh/3/yarn/nm/usercache/cdlapp/appcache/application_1646855837967_2018614/blockmgr-953671e2-1f39-455e-a4f5-987357b8b692 22/04/19 03:29:48 INFO storage.DiskBlockManager: Created local directory at /cdh/4/yarn/nm/usercache/cdlapp/appcache/application_1646855837967_2018614/blockmgr-ab5c20ea-be99-4689-b088-820ffca27558 22/04/19 03:29:48 INFO storage.DiskBlockManager: Created local directory at /cdh/2/yarn/nm/usercache/cdlapp/appcache/application_1646855837967_2018614/blockmgr-8613cafe-175d-45b9-83b0-507109c5bc6b 22/04/19 03:29:48 INFO storage.DiskBlockManager: Created local directory at /cdh/M/yarn/nm/usercache/cdlapp/appcache/application_1646855837967_2018614/blockmgr-aa2195b1-63d8-451e-b9d5-913ea73bddf2 22/04/19 03:29:48 INFO storage.DiskBlockManager: Created local directory at /cdh/G/yarn/nm/usercache/cdlapp/appcache/application_1646855837967_2018614/blockmgr-f8492843-09e1-4889-9049-a13b176a10ef 22/04/19 03:29:48 INFO storage.DiskBlockManager: Created local directory at /cdh/J/yarn/nm/usercache/cdlapp/appcache/application_1646855837967_2018614/blockmgr-9791ca81-36f5-4b12-b20f-923355d29f45 22/04/19 03:29:48 INFO storage.DiskBlockManager: Created local directory at /cdh/B/yarn/nm/usercache/cdlapp/appcache/application_1646855837967_2018614/blockmgr-b08195e7-1637-48ae-bff8-255972e4988f 22/04/19 03:29:48 INFO storage.DiskBlockManager: Created local directory at /cdh/L/yarn/nm/usercache/cdlapp/appcache/application_1646855837967_2018614/blockmgr-a5a1d38f-55d1-4bb0-ade7-771db0b50d89 22/04/19 03:29:48 INFO storage.DiskBlockManager: Created local directory at /cdh/E/yarn/nm/usercache/cdlapp/appcache/application_1646855837967_2018614/blockmgr-0c2799ad-d4ed-48bc-8f7c-4aeffd8c1733 22/04/19 03:29:48 INFO storage.DiskBlockManager: Created local directory at /cdh/6/yarn/nm/usercache/cdlapp/appcache/application_1646855837967_2018614/blockmgr-a8b48023-d64d-4373-a713-587c66e04cbf 22/04/19 03:29:48 INFO memory.MemoryStore: MemoryStore started with capacity 912.3 MB 22/04/19 03:29:48 INFO executor.CoarseGrainedExecutorBackend: Connecting to driver: spark://CoarseGrainedScheduler@hostname018.corp.xyz:46519 22/04/19 03:29:48 INFO executor.CoarseGrainedExecutorBackend: Successfully registered with driver 22/04/19 03:29:48 INFO executor.Executor: Starting executor ID 1 on host hostname033.corp.xyz 22/04/19 03:29:48 INFO util.Utils: Successfully started service 'org.apache.spark.network.netty.NettyBlockTransferService' on port 40300. 22/04/19 03:29:48 INFO netty.NettyBlockTransferService: Server created on hostname033.corp.xyz:40300 22/04/19 03:29:48 INFO storage.BlockManager: Using org.apache.spark.storage.RandomBlockReplicationPolicy for block replication policy 22/04/19 03:29:48 INFO storage.BlockManagerMaster: Registering BlockManager BlockManagerId(1, hostname033.corp.xyz, 40300, None) 22/04/19 03:29:48 INFO storage.BlockManagerMaster: Registered BlockManager BlockManagerId(1, hostname033.corp.xyz, 40300, None) 22/04/19 03:29:48 INFO storage.BlockManager: external shuffle service port = 7337 22/04/19 03:29:48 INFO storage.BlockManager: Registering executor with local external shuffle service. 22/04/19 03:29:48 INFO client.TransportClientFactory: Successfully created connection to hostname033.corp.xyz/148.156.146.42:7337 after 2 ms (0 ms spent in bootstraps) 22/04/19 03:29:48 INFO storage.BlockManager: Initialized BlockManager: BlockManagerId(1, hostname033.corp.xyz, 40300, None) 22/04/19 03:29:48 INFO util.JsonImpl: HttpClient configuration: maxClients=8192, messageQueueSize=512 22/04/19 03:29:50 INFO util.JsonImpl: HttpClient configuration: maxClients=8192, messageQueueSize=512 22/04/19 03:29:50 INFO util.JsonImpl: Low priority HttpClient Options{host='polpcdhen008.corp.xyz', port=4043, useSsl=false, connectTimeout=Period{time=200, timeUnit=MILLISECONDS}, readTimeout=Period{time=3500, timeUnit=MILLISECONDS}, dnsTtl=Period{time=21600000, timeUnit=MILLISECONDS}} 22/04/19 03:29:50 INFO jdk.JdkHttpClient: connect timeout: Period{time=200, timeUnit=MILLISECONDS}, read timeout: Period{time=3500, timeUnit=MILLISECONDS}, shutdown timeout: Period{time=50, timeUnit=MILLISECONDS} 22/04/19 03:29:50 INFO util.JsonImpl: HttpClient Options{host='polpcdhen008.corp.xyz', port=4043, useSsl=false, connectTimeout=Period{time=200, timeUnit=MILLISECONDS}, readTimeout=Period{time=3500, timeUnit=MILLISECONDS}, dnsTtl=Period{time=21600000, timeUnit=MILLISECONDS}} 22/04/19 03:29:50 INFO jdk.JdkHttpClient: connect timeout: Period{time=200, timeUnit=MILLISECONDS}, read timeout: Period{time=3500, timeUnit=MILLISECONDS}, shutdown timeout: Period{time=50, timeUnit=MILLISECONDS} 22/04/19 03:29:59 INFO executor.CoarseGrainedExecutorBackend: Got assigned task 3 22/04/19 03:30:00 INFO executor.Executor: Running task 0.0 in stage 3.0 (TID 3) 22/04/19 03:30:00 INFO broadcast.TorrentBroadcast: Started reading broadcast variable 3 22/04/19 03:30:00 INFO client.TransportClientFactory: Successfully created connection to hostname018.corp.xyz/148.156.146.27:33617 after 7 ms (0 ms spent in bootstraps) 22/04/19 03:30:00 INFO memory.MemoryStore: Block broadcast_3_piece0 stored as bytes in memory (estimated size 5.6 KB, free 912.3 MB) 22/04/19 03:30:00 INFO broadcast.TorrentBroadcast: Reading broadcast variable 3 took 129 ms 22/04/19 03:30:00 INFO memory.MemoryStore: Block broadcast_3 stored as values in memory (estimated size 12.3 KB, free 912.3 MB) 22/04/19 03:30:02 INFO codegen.CodeGenerator: Code generated in 254.56597 ms 22/04/19 03:30:02 INFO jdbc.JDBCRDD: closed connection 22/04/19 03:30:02 INFO executor.Executor: Finished task 0.0 in stage 3.0 (TID 3). 1828 bytes result sent to driver 22/04/19 03:30:05 INFO executor.CoarseGrainedExecutorBackend: Got assigned task 6 22/04/19 03:30:05 INFO executor.Executor: Running task 0.0 in stage 6.0 (TID 6) 22/04/19 03:30:05 INFO spark.MapOutputTrackerWorker: Updating epoch to 1 and clearing cache 22/04/19 03:30:05 INFO broadcast.TorrentBroadcast: Started reading broadcast variable 6 22/04/19 03:30:05 INFO memory.MemoryStore: Block broadcast_6_piece0 stored as bytes in memory (estimated size 4.5 KB, free 912.3 MB) 22/04/19 03:30:05 INFO broadcast.TorrentBroadcast: Reading broadcast variable 6 took 20 ms 22/04/19 03:30:05 INFO memory.MemoryStore: Block broadcast_6 stored as values in memory (estimated size 7.9 KB, free 912.3 MB) 22/04/19 03:30:05 INFO codegen.CodeGenerator: Code generated in 23.654799 ms 22/04/19 03:30:05 INFO jdbc.JDBCRDD: closed connection 22/04/19 03:30:05 INFO executor.Executor: Finished task 0.0 in stage 6.0 (TID 6). 1384 bytes result sent to driver 22/04/19 03:30:18 INFO executor.CoarseGrainedExecutorBackend: Driver commanded a shutdown 22/04/19 03:30:18 WARN server.TransportChannelHandler: Exception in connection from hostname018.corp.xyz/148.156.146.27:33617 java.util.concurrent.RejectedExecutionException: event executor terminated at io.netty.util.concurrent.SingleThreadEventExecutor.reject(SingleThreadEventExecutor.java:926) at io.netty.util.concurrent.SingleThreadEventExecutor.offerTask(SingleThreadEventExecutor.java:353) at io.netty.util.concurrent.SingleThreadEventExecutor.addTask(SingleThreadEventExecutor.java:346) at io.netty.util.concurrent.SingleThreadEventExecutor.execute(SingleThreadEventExecutor.java:828) at io.netty.util.concurrent.SingleThreadEventExecutor.execute(SingleThreadEventExecutor.java:818) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:250) at io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:241) at io.netty.channel.ChannelInboundHandlerAdapter.channelInactive(ChannelInboundHandlerAdapter.java:81) at org.apache.spark.network.server.TransportChannelHandler.channelInactive(TransportChannelHandler.java:118) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:262) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:248) at io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:241) at io.netty.channel.ChannelInboundHandlerAdapter.channelInactive(ChannelInboundHandlerAdapter.java:81) at io.netty.handler.timeout.IdleStateHandler.channelInactive(IdleStateHandler.java:277) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:262) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:248) at io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:241) at io.netty.channel.ChannelInboundHandlerAdapter.channelInactive(ChannelInboundHandlerAdapter.java:81) at org.apache.spark.network.util.TransportFrameDecoder.channelInactive(TransportFrameDecoder.java:182) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:262) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:248) at io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:241) at io.netty.channel.DefaultChannelPipeline$HeadContext.channelInactive(DefaultChannelPipeline.java:1405) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:262) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:248) at io.netty.channel.DefaultChannelPipeline.fireChannelInactive(DefaultChannelPipeline.java:901) at io.netty.channel.AbstractChannel$AbstractUnsafe$8.run(AbstractChannel.java:818) at io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:164) at io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472) at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:500) at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:748) 22/04/19 03:30:18 INFO memory.MemoryStore: MemoryStore cleared 22/04/19 03:30:18 WARN server.TransportChannelHandler: Exception in connection from hostname018.corp.xyz/148.156.146.27:33617 java.util.concurrent.RejectedExecutionException: event executor terminated at io.netty.util.concurrent.SingleThreadEventExecutor.reject(SingleThreadEventExecutor.java:926) at io.netty.util.concurrent.SingleThreadEventExecutor.offerTask(SingleThreadEventExecutor.java:353) at io.netty.util.concurrent.SingleThreadEventExecutor.addTask(SingleThreadEventExecutor.java:346) at io.netty.util.concurrent.SingleThreadEventExecutor.execute(SingleThreadEventExecutor.java:828) at io.netty.util.concurrent.SingleThreadEventExecutor.execute(SingleThreadEventExecutor.java:818) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelUnregistered(AbstractChannelHandlerContext.java:186) at io.netty.channel.AbstractChannelHandlerContext.fireChannelUnregistered(AbstractChannelHandlerContext.java:177) at io.netty.channel.ChannelInboundHandlerAdapter.channelUnregistered(ChannelInboundHandlerAdapter.java:57) at org.apache.spark.network.server.TransportChannelHandler.channelUnregistered(TransportChannelHandler.java:192) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelUnregistered(AbstractChannelHandlerContext.java:198) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelUnregistered(AbstractChannelHandlerContext.java:184) at io.netty.channel.AbstractChannelHandlerContext.fireChannelUnregistered(AbstractChannelHandlerContext.java:177) at io.netty.channel.DefaultChannelPipeline$HeadContext.channelUnregistered(DefaultChannelPipeline.java:1388) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelUnregistered(AbstractChannelHandlerContext.java:198) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelUnregistered(AbstractChannelHandlerContext.java:184) at io.netty.channel.DefaultChannelPipeline.fireChannelUnregistered(DefaultChannelPipeline.java:821) at io.netty.channel.AbstractChannel$AbstractUnsafe$8.run(AbstractChannel.java:826) at io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:164) at io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472) at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:500) at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:748) 22/04/19 03:30:18 INFO storage.BlockManager: BlockManager stopped 22/04/19 03:30:18 WARN server.TransportChannelHandler: Exception in connection from hostname018.corp.xyz/148.156.146.27:33617 java.util.concurrent.RejectedExecutionException: event executor terminated at io.netty.util.concurrent.SingleThreadEventExecutor.reject(SingleThreadEventExecutor.java:926) at io.netty.util.concurrent.SingleThreadEventExecutor.offerTask(SingleThreadEventExecutor.java:353) at io.netty.util.concurrent.SingleThreadEventExecutor.addTask(SingleThreadEventExecutor.java:346) at io.netty.util.concurrent.SingleThreadEventExecutor.execute(SingleThreadEventExecutor.java:828) at io.netty.util.concurrent.SingleThreadEventExecutor.execute(SingleThreadEventExecutor.java:818) at io.netty.channel.DefaultChannelPipeline.destroyUp(DefaultChannelPipeline.java:851) at io.netty.channel.DefaultChannelPipeline.destroy(DefaultChannelPipeline.java:836) at io.netty.channel.DefaultChannelPipeline.access$700(DefaultChannelPipeline.java:46) at io.netty.channel.DefaultChannelPipeline$HeadContext.channelUnregistered(DefaultChannelPipeline.java:1392) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelUnregistered(AbstractChannelHandlerContext.java:198) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelUnregistered(AbstractChannelHandlerContext.java:184) at io.netty.channel.DefaultChannelPipeline.fireChannelUnregistered(DefaultChannelPipeline.java:821) at io.netty.channel.AbstractChannel$AbstractUnsafe$8.run(AbstractChannel.java:826) at io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:164) at io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472) at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:500) at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.lang.Thread.run(Thread.java:748) 22/04/19 03:30:18 INFO util.ShutdownHookManager: Shutdown hook called End of LogType:stderr *********************************************************************** End of LogType:stdout ***********************************************************************   ... View more