https://community.cloudera.com/t5/Support-Questions/Livy-Job-failing-after-upgrading-cluster-to-HDP-2-6-3/m-p/199399#M161438 <P>I have upgraded my cluster to HDP 2.6.3 recently with having spark 2.2.0 and livy 0.4.0. My livy job is every failing every time with Rsc driver exception. I analysed logs and found that host not able to add the Jar file from the given location that why it is failing.</P><BLOCKQUOTE><P>Error: WARN DefaultPromise: An exception was thrown by org.apache.livy.rsc.Utils$2.operationComplete() java.util.concurrent.RejectedExecutionException: event executor terminated at io.netty.util.concurrent.SingleThreadEventExecutor.reject(SingleThreadEventExecutor.java:796) at io.netty.util.concurrent.SingleThreadEventExecutor.offerTask(SingleThreadEventExecutor.java:336) at io.netty.util.concurrent.SingleThreadEventExecutor.addTask(SingleThreadEventExecutor.java:329) at io.netty.util.concurrent.SingleThreadEventExecutor.execute(SingleThreadEventExecutor.java:739) at io.netty.util.concurrent.AbstractScheduledEventExecutor.schedule(AbstractScheduledEventExecutor.java:190) at io.netty.util.concurrent.AbstractScheduledEventExecutor.schedule(AbstractScheduledEventExecutor.java:134) at io.netty.util.concurrent.AbstractEventExecutorGroup.schedule(AbstractEventExecutorGroup.java:49) at org.apache.livy.rsc.driver.RSCDriver.setupIdleTimeout(RSCDriver.java:238) at org.apache.livy.rsc.driver.RSCDriver.access$100(RSCDriver.java:70) at org.apache.livy.rsc.driver.RSCDriver$2.onSuccess(RSCDriver.java:220) at org.apache.livy.rsc.driver.RSCDriver$2.onSuccess(RSCDriver.java:216) at org.apache.livy.rsc.Utils$2.operationComplete(Utils.java:108) at io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:507) at io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:481) at io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:420) at io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:104) at io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:82) at io.netty.channel.AbstractChannel$CloseFuture.setClosed(AbstractChannel.java:1004) at io.netty.channel.AbstractChannel$AbstractUnsafe.doClose0(AbstractChannel.java:633) at io.netty.channel.AbstractChannel$AbstractUnsafe.close(AbstractChannel.java:611) at io.netty.channel.AbstractChannel$AbstractUnsafe.close(AbstractChannel.java:554) at io.netty.channel.DefaultChannelPipeline$HeadContext.close(DefaultChannelPipeline.java:1236) at io.netty.channel.AbstractChannelHandlerContext.invokeClose(AbstractChannelHandlerContext.java:619) at io.netty.channel.AbstractChannelHandlerContext.close(AbstractChannelHandlerContext.java:603) at io.netty.channel.ChannelDuplexHandler.close(ChannelDuplexHandler.java:73) at io.netty.channel.AbstractChannelHandlerContext.invokeClose(AbstractChannelHandlerContext.java:619) at io.netty.channel.AbstractChannelHandlerContext.close(AbstractChannelHandlerContext.java:603) at io.netty.channel.AbstractChannelHandlerContext.close(AbstractChannelHandlerContext.java:460) at io.netty.channel.DefaultChannelPipeline.close(DefaultChannelPipeline.java:949) at io.netty.channel.AbstractChannel.close(AbstractChannel.java:194) at org.apache.livy.rsc.rpc.Rpc.close(Rpc.java:307) at org.apache.livy.rsc.driver.RSCDriver.shutdownServer(RSCDriver.java:309) at org.apache.livy.rsc.driver.RSCDriver.shutdown(RSCDriver.java:133) at org.apache.livy.rsc.driver.RSCDriver.handle(RSCDriver.java:396) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:497) at org.apache.livy.rsc.rpc.RpcDispatcher.handleCall(RpcDispatcher.java:130) at org.apache.livy.rsc.rpc.RpcDispatcher.channelRead0(RpcDispatcher.java:77) at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:105) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:357) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:343) at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:336) at io.netty.channel.ChannelInboundHandlerAdapter.channelRead(ChannelInboundHandlerAdapter.java:86) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:357) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:343) at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:336) at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:293) at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:267) at io.netty.handler.codec.ByteToMessageCodec.channelRead(ByteToMessageCodec.java:103) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:357) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:343) at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:336) at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1294) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:357) at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:343) at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:911) at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:131) at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:643) at io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:566) at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:480) at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:442) at io.netty.util.concurrent.SingleThreadEventExecutor$2.run(SingleThreadEventExecutor.java:131) at java.lang.Thread.run(Thread.java:745)</P></BLOCKQUOTE> Fri, 17 Nov 2017 02:42:18 GMT Shiva29 2017-11-17T02:42:18Z