still facing error in HDP2.6.4 & Livy 0.4.0, Steps: 1. <em>Livy session was created as spark.</em> 2.<em> Added scala jar(my function)</em> 3.<em> livyClient.run(livyJob()) </em>gives error: <em> 18/02/20 14:53:44 INFO InteractiveSession: Interactive session 12 created [appid: application_1518698685392_0040, owner: null, proxyUser: Some(hive), state: idle, kind: spark, info: {driverLogUrl=http://hdp04d03.fuzzyl.com:8042/node/containerlogs/container_e04_1518698685392_0040_01_000001/hive, sparkUiUrl=http://ambari04.fuzzyl.com:8088/proxy/application_1518698685392_0040/}] 18/02/20 14:53:44 INFO RSCClient: Received result for 51f4a85f-a499-4dc4-ad9f-9a0e7a42ea64 18/02/20 14:53:44 ERROR SessionServlet$: internal error java.util.concurrent.ExecutionException: java.lang.RuntimeException: py4j.Py4JException: Error while obtaining a new communication channel py4j.CallbackClient.getConnectionLock(CallbackClient.java:218) py4j.CallbackClient.sendCommand(CallbackClient.java:337) py4j.CallbackClient.sendCommand(CallbackClient.java:316) py4j.reflection.PythonProxyHandler.invoke(PythonProxyHandler.java:103) com.sun.proxy.$Proxy24.getLocalTmpDirPath(Unknown Source) org.apache.livy.repl.PythonInterpreter.addPyFile(PythonInterpreter.scala:264) org.apache.livy.repl.ReplDriver$anonfun$addJarOrPyFile$1.apply(ReplDriver.scala:110) org.apache.livy.repl.ReplDriver$anonfun$addJarOrPyFile$1.apply(ReplDriver.scala:110) scala.Option.foreach(Option.scala:257) org.apache.livy.repl.ReplDriver.addJarOrPyFile(ReplDriver.scala:110) org.apache.livy.rsc.driver.JobContextImpl.addJarOrPyFile(JobContextImpl.java:100) org.apache.livy.rsc.driver.AddJarJob.call(AddJarJob.java:39) org.apache.livy.rsc.driver.JobWrapper.call(JobWrapper.java:57) org.apache.livy.rsc.driver.JobWrapper.call(JobWrapper.java:34) java.util.concurrent.FutureTask.run(FutureTask.java:266) java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) java.lang.Thread.run(Thread.java:745) at io.netty.util.concurrent.AbstractFuture.get(AbstractFuture.java:37) at org.apache.livy.rsc.JobHandleImpl.get(JobHandleImpl.java:60) at org.apache.livy.server.interactive.InteractiveSession.addJar(InteractiveSession.scala:542) at org.apache.livy.server.interactive.InteractiveSessionServlet.org$apache$livy$server$interactive$InteractiveSessionServlet$addJarOrPyFile(InteractiveSessionServlet.scala:241) at org.apache.livy.server.interactive.InteractiveSessionServlet$anonfun$19$anonfun$apply$16.apply(InteractiveSessionServlet.scala:208) at org.apache.livy.server.interactive.InteractiveSessionServlet$anonfun$19$anonfun$apply$16.apply(InteractiveSessionServlet.scala:207) at org.apache.livy.server.interactive.SessionHeartbeatNotifier$anonfun$withModifyAccessSession$1.apply(SessionHeartbeat.scala:76) at org.apache.livy.server.interactive.SessionHeartbeatNotifier$anonfun$withModifyAccessSession$1.apply(SessionHeartbeat.scala:74) at org.apache.livy.server.SessionServlet.doWithSession(SessionServlet.scala:221) at org.apache.livy.server.SessionServlet.withModifyAccessSession(SessionServlet.scala:212) at org.apache.livy.server.interactive.InteractiveSessionServlet.org$apache$livy$server$interactive$SessionHeartbeatNotifier$super$withModifyAccessSession(InteractiveSessionServlet.scala:40) at org.apache.livy.server.interactive.SessionHeartbeatNotifier$class.withModifyAccessSession(SessionHeartbeat.scala:74) at org.apache.livy.server.interactive.InteractiveSessionServlet.withModifyAccessSession(InteractiveSessionServlet.scala:40) at org.apache.livy.server.interactive.InteractiveSessionServlet$anonfun$19.apply(InteractiveSessionServlet.scala:207) at org.apache.livy.server.interactive.InteractiveSessionServlet$anonfun$19.apply(InteractiveSessionServlet.scala:206) at org.apache.livy.server.JsonServlet.org$apache$livy$server$JsonServlet$doAction(JsonServlet.scala:113) at org.apache.livy.server.JsonServlet$anonfun$jpost$1.apply(JsonServlet.scala:75) at org.scalatra.ScalatraBase$class.org$scalatra$ScalatraBase$liftAction(ScalatraBase.scala:270) at org.scalatra.ScalatraBase$anonfun$invoke$1.apply(ScalatraBase.scala:265) at org.scalatra.ScalatraBase$anonfun$invoke$1.apply(ScalatraBase.scala:265) at org.scalatra.ApiFormats$class.withRouteMultiParams(ApiFormats.scala:178) at org.apache.livy.server.JsonServlet.withRouteMultiParams(JsonServlet.scala:39) at org.scalatra.ScalatraBase$class.invoke(ScalatraBase.scala:264) at org.scalatra.ScalatraServlet.invoke(ScalatraServlet.scala:49) at org.scalatra.ScalatraBase$anonfun$runRoutes$1$anonfun$apply$8.apply(ScalatraBase.scala:240) at org.scalatra.ScalatraBase$anonfun$runRoutes$1$anonfun$apply$8.apply(ScalatraBase.scala:238) at scala.Option.flatMap(Option.scala:170) at org.scalatra.ScalatraBase$anonfun$runRoutes$1.apply(ScalatraBase.scala:238) at org.scalatra.ScalatraBase$anonfun$runRoutes$1.apply(ScalatraBase.scala:237) at scala.collection.immutable.Stream.flatMap(Stream.scala:446)</em> ... View more