Hiveserver2 stops working after being up for about 6-7 minutes.

Question

1:25:46.144 AMINFOTezClient[main]: Failed to retrieve AM Status via proxycom.google.protobuf.ServiceException: java.io.EOFException: End of File Exception between local host is: "b7-40.lab.archivas.com/172.18.105.90"; destination host is: "b7-35.lab.archivas.com":41485; : java.io.EOFException; For more details see:  http://wiki.apache.org/hadoop/EOFException	at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:247) ~[hadoop-common-3.1.1.7.1.7.1000-141.jar:?]	at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-3.1.1.7.1.7.1000-141.jar:?]	at com.sun.proxy.$Proxy44.getAMStatus(Unknown Source) ~[?:?]	at org.apache.tez.client.FrameworkClient.getAMStatus(FrameworkClient.java:115) ~[tez-api-0.9.1.7.1.7.1000-141.jar:0.9.1.7.1.7.1000-141]	at org.apache.tez.client.TezClient.getAppMasterStatus(TezClient.java:843) [tez-api-0.9.1.7.1.7.1000-141.jar:0.9.1.7.1.7.1000-141]	at org.apache.tez.client.TezClient.waitTillReady(TezClient.java:977) [tez-api-0.9.1.7.1.7.1000-141.jar:0.9.1.7.1.7.1000-141]	at org.apache.tez.client.TezClient.waitTillReady(TezClient.java:948) [tez-api-0.9.1.7.1.7.1000-141.jar:0.9.1.7.1.7.1000-141]	at org.apache.hadoop.hive.ql.exec.tez.TezSessionState.startSessionAndContainers(TezSessionState.java:567) [hive-exec-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141]	at org.apache.hadoop.hive.ql.exec.tez.TezSessionState.openInternal(TezSessionState.java:385) [hive-exec-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141]	at org.apache.hadoop.hive.ql.exec.tez.TezSessionState.open(TezSessionState.java:314) [hive-exec-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141]	at org.apache.hadoop.hive.ql.exec.tez.TezSessionPoolSession.open(TezSessionPoolSession.java:118) [hive-exec-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141]	at org.apache.hadoop.hive.ql.exec.tez.TezSessionPool.startInitialSession(TezSessionPool.java:359) [hive-exec-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141]	at org.apache.hadoop.hive.ql.exec.tez.TezSessionPool.startUnderInitLock(TezSessionPool.java:171) [hive-exec-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141]	at org.apache.hadoop.hive.ql.exec.tez.TezSessionPool.start(TezSessionPool.java:123) [hive-exec-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141]	at org.apache.hadoop.hive.ql.exec.tez.TezSessionPoolManager.startPool(TezSessionPoolManager.java:115) [hive-exec-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141]	at org.apache.hive.service.server.HiveServer2.initAndStartTezSessionPoolManager(HiveServer2.java:822) [hive-service-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141]	at org.apache.hive.service.server.HiveServer2.startOrReconnectTezSessions(HiveServer2.java:795) [hive-service-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141]	at org.apache.hive.service.server.HiveServer2.start(HiveServer2.java:718) [hive-service-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141]	at org.apache.hive.service.server.HiveServer2.startHiveServer2(HiveServer2.java:1059) [hive-service-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141]	at org.apache.hive.service.server.HiveServer2.access$1400(HiveServer2.java:138) [hive-service-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141]	at org.apache.hive.service.server.HiveServer2$StartOptionExecutor.execute(HiveServer2.java:1333) [hive-service-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141]	at org.apache.hive.service.server.HiveServer2.main(HiveServer2.java:1177) [hive-service-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141]	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_322]	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_322]	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_322]	at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_322]	at org.apache.hadoop.util.RunJar.run(RunJar.java:318) [hadoop-common-3.1.1.7.1.7.1000-141.jar:?]	at org.apache.hadoop.util.RunJar.main(RunJar.java:232) [hadoop-common-3.1.1.7.1.7.1000-141.jar:?]Caused by: java.io.EOFException: End of File Exception between local host is: "b7-40.lab.archivas.com/172.18.105.90"; destination host is: "b7-35.lab.archivas.com":41485; : java.io.EOFException; For more details see:  http://wiki.apache.org/hadoop/EOFException	at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_322]	at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_322]	at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_322]	at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_322]	at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:892) ~[hadoop-common-3.1.1.7.1.7.1000-141.jar:?]	at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:846) ~[hadoop-common-3.1.1.7.1.7.1000-141.jar:?]	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1566) ~[hadoop-common-3.1.1.7.1.7.1000-141.jar:?]	at org.apache.hadoop.ipc.Client.call(Client.java:1508) ~[hadoop-common-3.1.1.7.1.7.1000-141.jar:?]	at org.apache.hadoop.ipc.Client.call(Client.java:1405) ~[hadoop-common-3.1.1.7.1.7.1000-141.jar:?]	at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:233) ~[hadoop-common-3.1.1.7.1.7.1000-141.jar:?]	... 27 moreCaused by: java.io.EOFException	at java.io.DataInputStream.readInt(DataInputStream.java:392) ~[?:1.8.0_322]	at org.apache.hadoop.ipc.Client$IpcStreams.readResponse(Client.java:1880) ~[hadoop-common-3.1.1.7.1.7.1000-141.jar:?]	at org.apache.hadoop.ipc.Client$Connection.receiveRpcResponse(Client.java:1191) ~[hadoop-common-3.1.1.7.1.7.1000-141.jar:?]	at org.apache.hadoop.ipc.Client$Connection.run(Client.java:1087) ~[hadoop-common-3.1.1.7.1.7.1000-141.jar:?]1:25:51.874 AMINFOTezClient[main]: Failed to retrieve AM Status via proxycom.google.protobuf.ServiceException: java.io.EOFException: End of File Exception between local host is: "b7-40.lab.archivas.com/172.18.105.90"; destination host is: "b7-37.lab.archivas.com":32988; : java.io.EOFException; For more details see:  http://wiki.apache.org/hadoop/EOFException	at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:247) ~[hadoop-common-3.1.1.7.1.7.1000-141.jar:?]	at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-3.1.1.7.1.7.1000-141.jar:?]	at com.sun.proxy.$Proxy44.getAMStatus(Unknown Source) ~[?:?]	at org.apache.tez.client.FrameworkClient.getAMStatus(FrameworkClient.java:115) ~[tez-api-0.9.1.7.1.7.1000-141.jar:0.9.1.7.1.7.1000-141]	at org.apache.tez.client.TezClient.getAppMasterStatus(TezClient.java:843) [tez-api-0.9.1.7.1.7.1000-141.jar:0.9.1.7.1.7.1000-141]	at org.apache.tez.client.TezClient.waitTillReady(TezClient.java:977) [tez-api-0.9.1.7.1.7.1000-141.jar:0.9.1.7.1.7.1000-141]	at org.apache.tez.client.TezClient.waitTillReady(TezClient.java:948) [tez-api-0.9.1.7.1.7.1000-141.jar:0.9.1.7.1.7.1000-141]	at org.apache.hadoop.hive.ql.exec.tez.TezSessionState.startSessionAndContainers(TezSessionState.java:567) [hive-exec-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141]	at org.apache.hadoop.hive.ql.exec.tez.TezSessionState.openInternal(TezSessionState.java:385) [hive-exec-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141]	at org.apache.hadoop.hive.ql.exec.tez.TezSessionState.open(TezSessionState.java:314) [hive-exec-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141]	at org.apache.hadoop.hive.ql.exec.tez.TezSessionPoolSession.open(TezSessionPoolSession.java:118) [hive-exec-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141]	at org.apache.hadoop.hive.ql.exec.tez.TezSessionPool.startInitialSession(TezSessionPool.java:359) [hive-exec-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141]	at org.apache.hadoop.hive.ql.exec.tez.TezSessionPool.startUnderInitLock(TezSessionPool.java:171) [hive-exec-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141]	at org.apache.hadoop.hive.ql.exec.tez.TezSessionPool.start(TezSessionPool.java:123) [hive-exec-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141]	at org.apache.hadoop.hive.ql.exec.tez.TezSessionPoolManager.startPool(TezSessionPoolManager.java:115) [hive-exec-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141]	at org.apache.hive.service.server.HiveServer2.initAndStartTezSessionPoolManager(HiveServer2.java:822) [hive-service-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141]	at org.apache.hive.service.server.HiveServer2.startOrReconnectTezSessions(HiveServer2.java:795) [hive-service-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141]	at org.apache.hive.service.server.HiveServer2.start(HiveServer2.java:718) [hive-service-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141]	at org.apache.hive.service.server.HiveServer2.startHiveServer2(HiveServer2.java:1059) [hive-service-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141]	at org.apache.hive.service.server.HiveServer2.access$1400(HiveServer2.java:138) [hive-service-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141]	at org.apache.hive.service.server.HiveServer2$StartOptionExecutor.execute(HiveServer2.java:1333) [hive-service-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141]	at org.apache.hive.service.server.HiveServer2.main(HiveServer2.java:1177) [hive-service-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141]	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_322]	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_322]	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_322]	at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_322]	at org.apache.hadoop.util.RunJar.run(RunJar.java:318) [hadoop-common-3.1.1.7.1.7.1000-141.jar:?]	at org.apache.hadoop.util.RunJar.main(RunJar.java:232) [hadoop-common-3.1.1.7.1.7.1000-141.jar:?]Caused by: java.io.EOFException: End of File Exception between local host is: "b7-40.lab.archivas.com/172.18.105.90"; destination host is: "b7-37.lab.archivas.com":32988; : java.io.EOFException; For more details see:  http://wiki.apache.org/hadoop/EOFException	at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_322]	at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_322]	at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_322]	at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_322]	at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:892) ~[hadoop-common-3.1.1.7.1.7.1000-141.jar:?]	at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:846) ~[hadoop-common-3.1.1.7.1.7.1000-141.jar:?]	at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1566) ~[hadoop-common-3.1.1.7.1.7.1000-141.jar:?]	at org.apache.hadoop.ipc.Client.call(Client.java:1508) ~[hadoop-common-3.1.1.7.1.7.1000-141.jar:?]	at org.apache.hadoop.ipc.Client.call(Client.java:1405) ~[hadoop-common-3.1.1.7.1.7.1000-141.jar:?]	at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:233) ~[hadoop-common-3.1.1.7.1.7.1000-141.jar:?]	... 27 moreCaused by: java.io.EOFException	at java.io.DataInputStream.readInt(DataInputStream.java:392) ~[?:1.8.0_322]	at org.apache.hadoop.ipc.Client$IpcStreams.readResponse(Client.java:1880) ~[hadoop-common-3.1.1.7.1.7.1000-141.jar:?]	at org.apache.hadoop.ipc.Client$Connection.receiveRpcResponse(Client.java:1191) ~[hadoop-common-3.1.1.7.1.7.1000-141.jar:?]	at org.apache.hadoop.ipc.Client$Connection.run(Client.java:1087) ~[hadoop-common-3.1.1.7.1.7.1000-141.jar:?]1:25:52.379 AMINFOTezClient[main]: App did not succeed. Diagnostics: Application application_1650968366405_0009 failed 2 times due to AM Container for appattempt_1650968366405_0009_000002 exited with  exitCode: 1Failing this attempt.Diagnostics: [2022-04-27 01:25:51.885]Exception from container-launch.Container id: container_e02_1650968366405_0009_02_000001Exit code: 1[2022-04-27 01:25:51.890]Container exited with a non-zero exit code 1. Error file: prelaunch.err.Last 4096 bytes of prelaunch.err :Last 4096 bytes of stderr :[2022-04-27 01:25:51.892]Container exited with a non-zero exit code 1. Error file: prelaunch.err.Last 4096 bytes of prelaunch.err :Last 4096 bytes of stderr :For more detailed output, check the application tracking page: https://b7-40.lab.archivas.com:8090/cluster/app/application_1650968366405_0009 Then click on links to logs of each attempt.. Failing the application.1:25:52.380 AMERRORHiveServer2[main]: Error starting  Tez sessions: org.apache.hive.service.ServiceException: Unable to setup tez session pool	at org.apache.hive.service.server.HiveServer2.initAndStartTezSessionPoolManager(HiveServer2.java:825) ~[hive-service-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141]	at org.apache.hive.service.server.HiveServer2.startOrReconnectTezSessions(HiveServer2.java:795) ~[hive-service-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141]	at org.apache.hive.service.server.HiveServer2.start(HiveServer2.java:718) [hive-service-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141]	at org.apache.hive.service.server.HiveServer2.startHiveServer2(HiveServer2.java:1059) [hive-service-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141]	at org.apache.hive.service.server.HiveServer2.access$1400(HiveServer2.java:138) [hive-service-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141]	at org.apache.hive.service.server.HiveServer2$StartOptionExecutor.execute(HiveServer2.java:1333) [hive-service-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141]	at org.apache.hive.service.server.HiveServer2.main(HiveServer2.java:1177) [hive-service-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141]	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_322]	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_322]	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_322]	at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_322]	at org.apache.hadoop.util.RunJar.run(RunJar.java:318) [hadoop-common-3.1.1.7.1.7.1000-141.jar:?]	at org.apache.hadoop.util.RunJar.main(RunJar.java:232) [hadoop-common-3.1.1.7.1.7.1000-141.jar:?]Caused by: org.apache.tez.dag.api.SessionNotRunning: TezSession has already shutdown. Application application_1650968366405_0009 failed 2 times due to AM Container for appattempt_1650968366405_0009_000002 exited with  exitCode: 1Failing this attempt.Diagnostics: [2022-04-27 01:25:51.885]Exception from container-launch.Container id: container_e02_1650968366405_0009_02_000001Exit code: 1[2022-04-27 01:25:51.890]Container exited with a non-zero exit code 1. Error file: prelaunch.err.Last 4096 bytes of prelaunch.err :Last 4096 bytes of stderr :[2022-04-27 01:25:51.892]Container exited with a non-zero exit code 1. Error file: prelaunch.err.Last 4096 bytes of prelaunch.err :Last 4096 bytes of stderr :For more detailed output, check the application tracking page: https://b7-40.lab.archivas.com:8090/cluster/app/application_1650968366405_0009 Then click on links to logs of each attempt.. Failing the application.	at org.apache.tez.client.TezClient.waitTillReady(TezClient.java:979) ~[tez-api-0.9.1.7.1.7.1000-141.jar:0.9.1.7.1.7.1000-141]	at org.apache.tez.client.TezClient.waitTillReady(TezClient.java:948) ~[tez-api-0.9.1.7.1.7.1000-141.jar:0.9.1.7.1.7.1000-141]	at org.apache.hadoop.hive.ql.exec.tez.TezSessionState.startSessionAndContainers(TezSessionState.java:567) ~[hive-exec-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141]	at org.apache.hadoop.hive.ql.exec.tez.TezSessionState.openInternal(TezSessionState.java:385) ~[hive-exec-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141]	at org.apache.hadoop.hive.ql.exec.tez.TezSessionState.open(TezSessionState.java:314) ~[hive-exec-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141]	at org.apache.hadoop.hive.ql.exec.tez.TezSessionPoolSession.open(TezSessionPoolSession.java:118) ~[hive-exec-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141]	at org.apache.hadoop.hive.ql.exec.tez.TezSessionPool.startInitialSession(TezSessionPool.java:359) ~[hive-exec-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141]	at org.apache.hadoop.hive.ql.exec.tez.TezSessionPool.startUnderInitLock(TezSessionPool.java:171) ~[hive-exec-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141]	at org.apache.hadoop.hive.ql.exec.tez.TezSessionPool.start(TezSessionPool.java:123) ~[hive-exec-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141]	at org.apache.hadoop.hive.ql.exec.tez.TezSessionPoolManager.startPool(TezSessionPoolManager.java:115) ~[hive-exec-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141]	at org.apache.hive.service.server.HiveServer2.initAndStartTezSessionPoolManager(HiveServer2.java:822) ~[hive-service-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141]	... 12 more1:25:52.382 AMINFOHiveServer2[main]: Shutting down HiveServer21:25:52.382 AMINFOThriftCLIService[main]: Thrift server has stopped1:25:52.382 AMINFOAbstractService[main]: Service:ThriftBinaryCLIService is stopped.1:25:52.382 AMINFOAbstractService[main]: Service:OperationManager is stopped.1:25:52.382 AMINFOAbstractService[main]: Service:SessionManager is stopped.1:25:52.384 AMINFOAbstractService[main]: Service:CLIService is stopped.1:25:52.385 AMINFOHiveMetaStoreClient[main]: Closed a connection to metastore, current connections: 161:25:52.385 AMINFOAbstractService[main]: Service:HiveServer2 is stopped.1:25:53.066 AMINFOScheduledQueryExecutionService[Scheduled Query Poller]: Thread finished; renaming back to: Scheduled Query Thread 01:25:53.386 AMWARNScheduledQueryExecutionService[Scheduled Query Progress Reporter]: interrupt discarded1:25:53.386 AMINFOScheduledQueryExecutionService[Scheduled Query Progress Reporter]: Thread finished; renaming back to: Scheduled Query Thread 11:25:53.388 AMINFOAbstractConnector[main]: Stopped ServerConnector@7df4709e{SSL, (ssl, http/1.1)}{0.0.0.0:10002}1:25:53.389 AMINFOsession[main]: node0 Stopped scavenging1:25:53.390 AMINFOContextHandler[main]: Stopped o.e.j.s.ServletContextHandler@7ab2ba47{static,/static,jar:file:/opt/cloudera/parcels/CDH-7.1.7-1.cdh7.1.7.p1000.24102687/jars/hive-service-3.1.3000.7.1.7.1000-141.jar!/hive-webapps/static,STOPPED}1:25:53.392 AMINFOContextHandler[main]: Stopped o.e.j.w.WebAppContext@559fd5ec{hiveserver2,/,null,STOPPED}{jar:file:/opt/cloudera/parcels/CDH-7.1.7-1.cdh7.1.7.p1000.24102687/jars/hive-service-3.1.3000.7.1.7.1000-141.jar!/hive-webapps/hiveserver2}1:25:53.394 AMINFOHiveServer2[main]: Web UI has stopped1:25:53.401 AMWARNZooKeeperHiveHelper[main-EventThread]: This server instance with path null is now de-registered from ZooKeeper. 1:25:53.401 AMWARNHiveServer2[main-EventThread]: This instance of HiveServer2 has been removed from the list of server instances available for dynamic service discovery. The last client session has ended - will shutdown now.1:25:53.401 AMINFOCuratorFrameworkImpl[Curator-Framework-0]: backgroundOperationsLoop exiting1:25:53.507 AMINFOZooKeeper[main]: Session: 0x10023db368405e2 closed1:25:53.507 AMINFOZooKeeperHiveHelper[main]: Server instance removed from ZooKeeper.1:25:53.507 AMINFOHiveServer2[main]: Stopping/Disconnecting tez sessions.1:25:53.507 AMINFOHiveServer2[main]: Stopped tez session pool manager.1:25:53.508 AMINFOHiveServer2[main-EventThread]: Shutting down HiveServer2   HiveServer2[main]: Error starting HiveServer2 on attempt 2, will retry in 60000msorg.apache.hive.service.ServiceException: org.apache.hive.service.ServiceException: Unable to setup tez session pool	at org.apache.hive.service.server.HiveServer2.start(HiveServer2.java:721) ~[hive-service-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141]	at org.apache.hive.service.server.HiveServer2.startHiveServer2(HiveServer2.java:1059) [hive-service-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141]	at org.apache.hive.service.server.HiveServer2.access$1400(HiveServer2.java:138) [hive-service-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141]	at org.apache.hive.service.server.HiveServer2$StartOptionExecutor.execute(HiveServer2.java:1333) [hive-service-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141]	at org.apache.hive.service.server.HiveServer2.main(HiveServer2.java:1177) [hive-service-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141]	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_322]	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_322]	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_322]	at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_322]	at org.apache.hadoop.util.RunJar.run(RunJar.java:318) [hadoop-common-3.1.1.7.1.7.1000-141.jar:?]	at org.apache.hadoop.util.RunJar.main(RunJar.java:232) [hadoop-common-3.1.1.7.1.7.1000-141.jar:?]Caused by: org.apache.hive.service.ServiceException: Unable to setup tez session pool	at org.apache.hive.service.server.HiveServer2.initAndStartTezSessionPoolManager(HiveServer2.java:825) ~[hive-service-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141]	at org.apache.hive.service.server.HiveServer2.startOrReconnectTezSessions(HiveServer2.java:795) ~[hive-service-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141]	at org.apache.hive.service.server.HiveServer2.start(HiveServer2.java:718) ~[hive-service-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141]	... 10 moreCaused by: org.apache.tez.dag.api.SessionNotRunning: TezSession has already shutdown. Application application_1650968366405_0009 failed 2 times due to AM Container for appattempt_1650968366405_0009_000002 exited with  exitCode: 1Failing this attempt.Diagnostics: [2022-04-27 01:25:51.885]Exception from container-launch.Container id: container_e02_1650968366405_0009_02_000001Exit code: 1[2022-04-27 01:25:51.890]Container exited with a non-zero exit code 1. Error file: prelaunch.err.Last 4096 bytes of prelaunch.err :Last 4096 bytes of stderr :[2022-04-27 01:25:51.892]Container exited with a non-zero exit code 1. Error file: prelaunch.err.Last 4096 bytes of prelaunch.err :Last 4096 bytes of stderr :For more detailed output, check the application tracking page: https://b7-40.lab.archivas.com:8090/cluster/app/application_1650968366405_0009 Then click on links to logs of each attempt.. Failing the application.	at org.apache.tez.client.TezClient.waitTillReady(TezClient.java:979) ~[tez-api-0.9.1.7.1.7.1000-141.jar:0.9.1.7.1.7.1000-141]	at org.apache.tez.client.TezClient.waitTillReady(TezClient.java:948) ~[tez-api-0.9.1.7.1.7.1000-141.jar:0.9.1.7.1.7.1000-141]	at org.apache.hadoop.hive.ql.exec.tez.TezSessionState.startSessionAndContainers(TezSessionState.java:567) ~[hive-exec-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141]	at org.apache.hadoop.hive.ql.exec.tez.TezSessionState.openInternal(TezSessionState.java:385) ~[hive-exec-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141]	at org.apache.hadoop.hive.ql.exec.tez.TezSessionState.open(TezSessionState.java:314) ~[hive-exec-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141]	at org.apache.hadoop.hive.ql.exec.tez.TezSessionPoolSession.open(TezSessionPoolSession.java:118) ~[hive-exec-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141]	at org.apache.hadoop.hive.ql.exec.tez.TezSessionPool.startInitialSession(TezSessionPool.java:359) ~[hive-exec-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141]	at org.apache.hadoop.hive.ql.exec.tez.TezSessionPool.startUnderInitLock(TezSessionPool.java:171) ~[hive-exec-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141]	at org.apache.hadoop.hive.ql.exec.tez.TezSessionPool.start(TezSessionPool.java:123) ~[hive-exec-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141]	at org.apache.hadoop.hive.ql.exec.tez.TezSessionPoolManager.startPool(TezSessionPoolManager.java:115) ~[hive-exec-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141]	at org.apache.hive.service.server.HiveServer2.initAndStartTezSessionPoolManager(HiveServer2.java:822) ~[hive-service-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141]	at org.apache.hive.service.server.HiveServer2.startOrReconnectTezSessions(HiveServer2.java:795) ~[hive-service-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141]	at org.apache.hive.service.server.HiveServer2.start(HiveServer2.java:718) ~[hive-service-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141]	... 10 more1:25:53.508 AMERRORHiveServer2[main-EventThread]: Error stopping schqjava.lang.IllegalStateException: The current ScheduledQueryExecutionService INSTANCE is invalid	at org.apache.hadoop.hive.ql.scheduled.ScheduledQueryExecutionService.close(ScheduledQueryExecutionService.java:312) ~[hive-exec-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141]	at org.apache.hive.service.server.HiveServer2.stop(HiveServer2.java:892) [hive-service-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141]	at org.apache.hive.service.server.HiveServer2$DeRegisterWatcher.process(HiveServer2.java:617) [hive-service-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141]	at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:77) [curator-framework-4.3.0.7.1.7.1000-141.jar:4.3.0.7.1.7.1000-141]	at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:532) [zookeeper-3.5.5.7.1.7.1000-141.jar:3.5.5.7.1.7.1000-141]	at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:507) [zookeeper-3.5.5.7.1.7.1000-141.jar:3.5.5.7.1.7.1000-141]

smruti · Answer

@jass @OmerSl Try disabling hive.server2.tez.initialize.default.sessions.

Go to CM UI > Hive on Tez > Configuration > HiveServer2 Advanced Configuration Snippet (Safety Valve) for hive-site.xml

Add:

Name: hive.server2.tez.initialize.default.sessions

Value: false

Save and restart the service. This should help resolve the issue.

More information on this property can be found here : https://docs.cloudera.com/cdp-private-cloud-base/7.1.7/configuring-apache-hive/topics/hive-on-tez-configurations.html

1:25:46.144 AM	INFO	TezClient	[main]: Failed to retrieve AM Status via proxy com.google.protobuf.ServiceException: java.io.EOFException: End of File Exception between local host is: "b7-40.lab.archivas.com/172.18.105.90"; destination host is: "b7-35.lab.archivas.com":41485; : java.io.EOFException; For more details see: http://wiki.apache.org/hadoop/EOFException at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:247) ~[hadoop-common-3.1.1.7.1.7.1000-141.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-3.1.1.7.1.7.1000-141.jar:?] at com.sun.proxy.$Proxy44.getAMStatus(Unknown Source) ~[?:?] at org.apache.tez.client.FrameworkClient.getAMStatus(FrameworkClient.java:115) ~[tez-api-0.9.1.7.1.7.1000-141.jar:0.9.1.7.1.7.1000-141] at org.apache.tez.client.TezClient.getAppMasterStatus(TezClient.java:843) [tez-api-0.9.1.7.1.7.1000-141.jar:0.9.1.7.1.7.1000-141] at org.apache.tez.client.TezClient.waitTillReady(TezClient.java:977) [tez-api-0.9.1.7.1.7.1000-141.jar:0.9.1.7.1.7.1000-141] at org.apache.tez.client.TezClient.waitTillReady(TezClient.java:948) [tez-api-0.9.1.7.1.7.1000-141.jar:0.9.1.7.1.7.1000-141] at org.apache.hadoop.hive.ql.exec.tez.TezSessionState.startSessionAndContainers(TezSessionState.java:567) [hive-exec-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141] at org.apache.hadoop.hive.ql.exec.tez.TezSessionState.openInternal(TezSessionState.java:385) [hive-exec-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141] at org.apache.hadoop.hive.ql.exec.tez.TezSessionState.open(TezSessionState.java:314) [hive-exec-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141] at org.apache.hadoop.hive.ql.exec.tez.TezSessionPoolSession.open(TezSessionPoolSession.java:118) [hive-exec-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141] at org.apache.hadoop.hive.ql.exec.tez.TezSessionPool.startInitialSession(TezSessionPool.java:359) [hive-exec-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141] at org.apache.hadoop.hive.ql.exec.tez.TezSessionPool.startUnderInitLock(TezSessionPool.java:171) [hive-exec-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141] at org.apache.hadoop.hive.ql.exec.tez.TezSessionPool.start(TezSessionPool.java:123) [hive-exec-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141] at org.apache.hadoop.hive.ql.exec.tez.TezSessionPoolManager.startPool(TezSessionPoolManager.java:115) [hive-exec-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141] at org.apache.hive.service.server.HiveServer2.initAndStartTezSessionPoolManager(HiveServer2.java:822) [hive-service-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141] at org.apache.hive.service.server.HiveServer2.startOrReconnectTezSessions(HiveServer2.java:795) [hive-service-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141] at org.apache.hive.service.server.HiveServer2.start(HiveServer2.java:718) [hive-service-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141] at org.apache.hive.service.server.HiveServer2.startHiveServer2(HiveServer2.java:1059) [hive-service-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141] at org.apache.hive.service.server.HiveServer2.access$1400(HiveServer2.java:138) [hive-service-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141] at org.apache.hive.service.server.HiveServer2$StartOptionExecutor.execute(HiveServer2.java:1333) [hive-service-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141] at org.apache.hive.service.server.HiveServer2.main(HiveServer2.java:1177) [hive-service-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_322] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_322] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_322] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_322] at org.apache.hadoop.util.RunJar.run(RunJar.java:318) [hadoop-common-3.1.1.7.1.7.1000-141.jar:?] at org.apache.hadoop.util.RunJar.main(RunJar.java:232) [hadoop-common-3.1.1.7.1.7.1000-141.jar:?] Caused by: java.io.EOFException: End of File Exception between local host is: "b7-40.lab.archivas.com/172.18.105.90"; destination host is: "b7-35.lab.archivas.com":41485; : java.io.EOFException; For more details see: http://wiki.apache.org/hadoop/EOFException at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_322] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_322] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_322] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_322] at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:892) ~[hadoop-common-3.1.1.7.1.7.1000-141.jar:?] at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:846) ~[hadoop-common-3.1.1.7.1.7.1000-141.jar:?] at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1566) ~[hadoop-common-3.1.1.7.1.7.1000-141.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1508) ~[hadoop-common-3.1.1.7.1.7.1000-141.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1405) ~[hadoop-common-3.1.1.7.1.7.1000-141.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:233) ~[hadoop-common-3.1.1.7.1.7.1000-141.jar:?] ... 27 more Caused by: java.io.EOFException at java.io.DataInputStream.readInt(DataInputStream.java:392) ~[?:1.8.0_322] at org.apache.hadoop.ipc.Client$IpcStreams.readResponse(Client.java:1880) ~[hadoop-common-3.1.1.7.1.7.1000-141.jar:?] at org.apache.hadoop.ipc.Client$Connection.receiveRpcResponse(Client.java:1191) ~[hadoop-common-3.1.1.7.1.7.1000-141.jar:?] at org.apache.hadoop.ipc.Client$Connection.run(Client.java:1087) ~[hadoop-common-3.1.1.7.1.7.1000-141.jar:?]
1:25:51.874 AM	INFO	TezClient	[main]: Failed to retrieve AM Status via proxy com.google.protobuf.ServiceException: java.io.EOFException: End of File Exception between local host is: "b7-40.lab.archivas.com/172.18.105.90"; destination host is: "b7-37.lab.archivas.com":32988; : java.io.EOFException; For more details see: http://wiki.apache.org/hadoop/EOFException at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:247) ~[hadoop-common-3.1.1.7.1.7.1000-141.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:118) ~[hadoop-common-3.1.1.7.1.7.1000-141.jar:?] at com.sun.proxy.$Proxy44.getAMStatus(Unknown Source) ~[?:?] at org.apache.tez.client.FrameworkClient.getAMStatus(FrameworkClient.java:115) ~[tez-api-0.9.1.7.1.7.1000-141.jar:0.9.1.7.1.7.1000-141] at org.apache.tez.client.TezClient.getAppMasterStatus(TezClient.java:843) [tez-api-0.9.1.7.1.7.1000-141.jar:0.9.1.7.1.7.1000-141] at org.apache.tez.client.TezClient.waitTillReady(TezClient.java:977) [tez-api-0.9.1.7.1.7.1000-141.jar:0.9.1.7.1.7.1000-141] at org.apache.tez.client.TezClient.waitTillReady(TezClient.java:948) [tez-api-0.9.1.7.1.7.1000-141.jar:0.9.1.7.1.7.1000-141] at org.apache.hadoop.hive.ql.exec.tez.TezSessionState.startSessionAndContainers(TezSessionState.java:567) [hive-exec-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141] at org.apache.hadoop.hive.ql.exec.tez.TezSessionState.openInternal(TezSessionState.java:385) [hive-exec-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141] at org.apache.hadoop.hive.ql.exec.tez.TezSessionState.open(TezSessionState.java:314) [hive-exec-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141] at org.apache.hadoop.hive.ql.exec.tez.TezSessionPoolSession.open(TezSessionPoolSession.java:118) [hive-exec-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141] at org.apache.hadoop.hive.ql.exec.tez.TezSessionPool.startInitialSession(TezSessionPool.java:359) [hive-exec-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141] at org.apache.hadoop.hive.ql.exec.tez.TezSessionPool.startUnderInitLock(TezSessionPool.java:171) [hive-exec-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141] at org.apache.hadoop.hive.ql.exec.tez.TezSessionPool.start(TezSessionPool.java:123) [hive-exec-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141] at org.apache.hadoop.hive.ql.exec.tez.TezSessionPoolManager.startPool(TezSessionPoolManager.java:115) [hive-exec-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141] at org.apache.hive.service.server.HiveServer2.initAndStartTezSessionPoolManager(HiveServer2.java:822) [hive-service-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141] at org.apache.hive.service.server.HiveServer2.startOrReconnectTezSessions(HiveServer2.java:795) [hive-service-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141] at org.apache.hive.service.server.HiveServer2.start(HiveServer2.java:718) [hive-service-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141] at org.apache.hive.service.server.HiveServer2.startHiveServer2(HiveServer2.java:1059) [hive-service-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141] at org.apache.hive.service.server.HiveServer2.access$1400(HiveServer2.java:138) [hive-service-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141] at org.apache.hive.service.server.HiveServer2$StartOptionExecutor.execute(HiveServer2.java:1333) [hive-service-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141] at org.apache.hive.service.server.HiveServer2.main(HiveServer2.java:1177) [hive-service-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_322] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_322] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_322] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_322] at org.apache.hadoop.util.RunJar.run(RunJar.java:318) [hadoop-common-3.1.1.7.1.7.1000-141.jar:?] at org.apache.hadoop.util.RunJar.main(RunJar.java:232) [hadoop-common-3.1.1.7.1.7.1000-141.jar:?] Caused by: java.io.EOFException: End of File Exception between local host is: "b7-40.lab.archivas.com/172.18.105.90"; destination host is: "b7-37.lab.archivas.com":32988; : java.io.EOFException; For more details see: http://wiki.apache.org/hadoop/EOFException at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:1.8.0_322] at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62) ~[?:1.8.0_322] at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:1.8.0_322] at java.lang.reflect.Constructor.newInstance(Constructor.java:423) ~[?:1.8.0_322] at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:892) ~[hadoop-common-3.1.1.7.1.7.1000-141.jar:?] at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:846) ~[hadoop-common-3.1.1.7.1.7.1000-141.jar:?] at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1566) ~[hadoop-common-3.1.1.7.1.7.1000-141.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1508) ~[hadoop-common-3.1.1.7.1.7.1000-141.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1405) ~[hadoop-common-3.1.1.7.1.7.1000-141.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:233) ~[hadoop-common-3.1.1.7.1.7.1000-141.jar:?] ... 27 more Caused by: java.io.EOFException at java.io.DataInputStream.readInt(DataInputStream.java:392) ~[?:1.8.0_322] at org.apache.hadoop.ipc.Client$IpcStreams.readResponse(Client.java:1880) ~[hadoop-common-3.1.1.7.1.7.1000-141.jar:?] at org.apache.hadoop.ipc.Client$Connection.receiveRpcResponse(Client.java:1191) ~[hadoop-common-3.1.1.7.1.7.1000-141.jar:?] at org.apache.hadoop.ipc.Client$Connection.run(Client.java:1087) ~[hadoop-common-3.1.1.7.1.7.1000-141.jar:?]
1:25:52.379 AM	INFO	TezClient	[main]: App did not succeed. Diagnostics: Application application_1650968366405_0009 failed 2 times due to AM Container for appattempt_1650968366405_0009_000002 exited with exitCode: 1 Failing this attempt.Diagnostics: [2022-04-27 01:25:51.885]Exception from container-launch. Container id: container_e02_1650968366405_0009_02_000001 Exit code: 1 [2022-04-27 01:25:51.890]Container exited with a non-zero exit code 1. Error file: prelaunch.err. Last 4096 bytes of prelaunch.err : Last 4096 bytes of stderr : [2022-04-27 01:25:51.892]Container exited with a non-zero exit code 1. Error file: prelaunch.err. Last 4096 bytes of prelaunch.err : Last 4096 bytes of stderr : For more detailed output, check the application tracking page: https://b7-40.lab.archivas.com:8090/cluster/app/application_1650968366405_0009 Then click on links to logs of each attempt. . Failing the application.
1:25:52.380 AM	ERROR	HiveServer2	[main]: Error starting Tez sessions: org.apache.hive.service.ServiceException: Unable to setup tez session pool at org.apache.hive.service.server.HiveServer2.initAndStartTezSessionPoolManager(HiveServer2.java:825) ~[hive-service-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141] at org.apache.hive.service.server.HiveServer2.startOrReconnectTezSessions(HiveServer2.java:795) ~[hive-service-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141] at org.apache.hive.service.server.HiveServer2.start(HiveServer2.java:718) [hive-service-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141] at org.apache.hive.service.server.HiveServer2.startHiveServer2(HiveServer2.java:1059) [hive-service-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141] at org.apache.hive.service.server.HiveServer2.access$1400(HiveServer2.java:138) [hive-service-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141] at org.apache.hive.service.server.HiveServer2$StartOptionExecutor.execute(HiveServer2.java:1333) [hive-service-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141] at org.apache.hive.service.server.HiveServer2.main(HiveServer2.java:1177) [hive-service-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_322] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_322] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_322] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_322] at org.apache.hadoop.util.RunJar.run(RunJar.java:318) [hadoop-common-3.1.1.7.1.7.1000-141.jar:?] at org.apache.hadoop.util.RunJar.main(RunJar.java:232) [hadoop-common-3.1.1.7.1.7.1000-141.jar:?] Caused by: org.apache.tez.dag.api.SessionNotRunning: TezSession has already shutdown. Application application_1650968366405_0009 failed 2 times due to AM Container for appattempt_1650968366405_0009_000002 exited with exitCode: 1 Failing this attempt.Diagnostics: [2022-04-27 01:25:51.885]Exception from container-launch. Container id: container_e02_1650968366405_0009_02_000001 Exit code: 1 [2022-04-27 01:25:51.890]Container exited with a non-zero exit code 1. Error file: prelaunch.err. Last 4096 bytes of prelaunch.err : Last 4096 bytes of stderr : [2022-04-27 01:25:51.892]Container exited with a non-zero exit code 1. Error file: prelaunch.err. Last 4096 bytes of prelaunch.err : Last 4096 bytes of stderr : For more detailed output, check the application tracking page: https://b7-40.lab.archivas.com:8090/cluster/app/application_1650968366405_0009 Then click on links to logs of each attempt. . Failing the application. at org.apache.tez.client.TezClient.waitTillReady(TezClient.java:979) ~[tez-api-0.9.1.7.1.7.1000-141.jar:0.9.1.7.1.7.1000-141] at org.apache.tez.client.TezClient.waitTillReady(TezClient.java:948) ~[tez-api-0.9.1.7.1.7.1000-141.jar:0.9.1.7.1.7.1000-141] at org.apache.hadoop.hive.ql.exec.tez.TezSessionState.startSessionAndContainers(TezSessionState.java:567) ~[hive-exec-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141] at org.apache.hadoop.hive.ql.exec.tez.TezSessionState.openInternal(TezSessionState.java:385) ~[hive-exec-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141] at org.apache.hadoop.hive.ql.exec.tez.TezSessionState.open(TezSessionState.java:314) ~[hive-exec-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141] at org.apache.hadoop.hive.ql.exec.tez.TezSessionPoolSession.open(TezSessionPoolSession.java:118) ~[hive-exec-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141] at org.apache.hadoop.hive.ql.exec.tez.TezSessionPool.startInitialSession(TezSessionPool.java:359) ~[hive-exec-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141] at org.apache.hadoop.hive.ql.exec.tez.TezSessionPool.startUnderInitLock(TezSessionPool.java:171) ~[hive-exec-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141] at org.apache.hadoop.hive.ql.exec.tez.TezSessionPool.start(TezSessionPool.java:123) ~[hive-exec-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141] at org.apache.hadoop.hive.ql.exec.tez.TezSessionPoolManager.startPool(TezSessionPoolManager.java:115) ~[hive-exec-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141] at org.apache.hive.service.server.HiveServer2.initAndStartTezSessionPoolManager(HiveServer2.java:822) ~[hive-service-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141] ... 12 more
1:25:52.382 AM	INFO	HiveServer2	`[main]: Shutting down HiveServer2`
1:25:52.382 AM	INFO	ThriftCLIService	`[main]: Thrift server has stopped`
1:25:52.382 AM	INFO	AbstractService	`[main]: Service:ThriftBinaryCLIService is stopped.`
1:25:52.382 AM	INFO	AbstractService	`[main]: Service:OperationManager is stopped.`
1:25:52.382 AM	INFO	AbstractService	`[main]: Service:SessionManager is stopped.`
1:25:52.384 AM	INFO	AbstractService	`[main]: Service:CLIService is stopped.`
1:25:52.385 AM	INFO	HiveMetaStoreClient	`[main]: Closed a connection to metastore, current connections: 16`
1:25:52.385 AM	INFO	AbstractService	`[main]: Service:HiveServer2 is stopped.`
1:25:53.066 AM	INFO	ScheduledQueryExecutionService	`[Scheduled Query Poller]: Thread finished; renaming back to: Scheduled Query Thread 0`
1:25:53.386 AM	WARN	ScheduledQueryExecutionService	`[Scheduled Query Progress Reporter]: interrupt discarded`
1:25:53.386 AM	INFO	ScheduledQueryExecutionService	`[Scheduled Query Progress Reporter]: Thread finished; renaming back to: Scheduled Query Thread 1`
1:25:53.388 AM	INFO	AbstractConnector	`[main]: Stopped ServerConnector@7df4709e{SSL, (ssl, http/1.1)}{0.0.0.0:10002}`
1:25:53.389 AM	INFO	session	`[main]: node0 Stopped scavenging`
1:25:53.390 AM	INFO	ContextHandler	`[main]: Stopped o.e.j.s.ServletContextHandler@7ab2ba47{static,/static,jar:file:/opt/cloudera/parcels/CDH-7.1.7-1.cdh7.1.7.p1000.24102687/jars/hive-service-3.1.3000.7.1.7.1000-141.jar!/hive-webapps/static,STOPPED}`
1:25:53.392 AM	INFO	ContextHandler	`[main]: Stopped o.e.j.w.WebAppContext@559fd5ec{hiveserver2,/,null,STOPPED}{jar:file:/opt/cloudera/parcels/CDH-7.1.7-1.cdh7.1.7.p1000.24102687/jars/hive-service-3.1.3000.7.1.7.1000-141.jar!/hive-webapps/hiveserver2}`
1:25:53.394 AM	INFO	HiveServer2	`[main]: Web UI has stopped`
1:25:53.401 AM	WARN	ZooKeeperHiveHelper	`[main-EventThread]: This server instance with path null is now de-registered from ZooKeeper.`
1:25:53.401 AM	WARN	HiveServer2	`[main-EventThread]: This instance of HiveServer2 has been removed from the list of server instances available for dynamic service discovery. The last client session has ended - will shutdown now.`
1:25:53.401 AM	INFO	CuratorFrameworkImpl	`[Curator-Framework-0]: backgroundOperationsLoop exiting`
1:25:53.507 AM	INFO	ZooKeeper	`[main]: Session: 0x10023db368405e2 closed`
1:25:53.507 AM	INFO	ZooKeeperHiveHelper	`[main]: Server instance removed from ZooKeeper.`
1:25:53.507 AM	INFO	HiveServer2	`[main]: Stopping/Disconnecting tez sessions.`
1:25:53.507 AM	INFO	HiveServer2	`[main]: Stopped tez session pool manager.`
1:25:53.508 AM	INFO	HiveServer2	`[main-EventThread]: Shutting down HiveServer2`

HiveServer2	[main]: Error starting HiveServer2 on attempt 2, will retry in 60000ms org.apache.hive.service.ServiceException: org.apache.hive.service.ServiceException: Unable to setup tez session pool at org.apache.hive.service.server.HiveServer2.start(HiveServer2.java:721) ~[hive-service-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141] at org.apache.hive.service.server.HiveServer2.startHiveServer2(HiveServer2.java:1059) [hive-service-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141] at org.apache.hive.service.server.HiveServer2.access$1400(HiveServer2.java:138) [hive-service-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141] at org.apache.hive.service.server.HiveServer2$StartOptionExecutor.execute(HiveServer2.java:1333) [hive-service-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141] at org.apache.hive.service.server.HiveServer2.main(HiveServer2.java:1177) [hive-service-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_322] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_322] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_322] at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_322] at org.apache.hadoop.util.RunJar.run(RunJar.java:318) [hadoop-common-3.1.1.7.1.7.1000-141.jar:?] at org.apache.hadoop.util.RunJar.main(RunJar.java:232) [hadoop-common-3.1.1.7.1.7.1000-141.jar:?] Caused by: org.apache.hive.service.ServiceException: Unable to setup tez session pool at org.apache.hive.service.server.HiveServer2.initAndStartTezSessionPoolManager(HiveServer2.java:825) ~[hive-service-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141] at org.apache.hive.service.server.HiveServer2.startOrReconnectTezSessions(HiveServer2.java:795) ~[hive-service-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141] at org.apache.hive.service.server.HiveServer2.start(HiveServer2.java:718) ~[hive-service-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141] ... 10 more Caused by: org.apache.tez.dag.api.SessionNotRunning: TezSession has already shutdown. Application application_1650968366405_0009 failed 2 times due to AM Container for appattempt_1650968366405_0009_000002 exited with exitCode: 1 Failing this attempt.Diagnostics: [2022-04-27 01:25:51.885]Exception from container-launch. Container id: container_e02_1650968366405_0009_02_000001 Exit code: 1 [2022-04-27 01:25:51.890]Container exited with a non-zero exit code 1. Error file: prelaunch.err. Last 4096 bytes of prelaunch.err : Last 4096 bytes of stderr : [2022-04-27 01:25:51.892]Container exited with a non-zero exit code 1. Error file: prelaunch.err. Last 4096 bytes of prelaunch.err : Last 4096 bytes of stderr : For more detailed output, check the application tracking page: https://b7-40.lab.archivas.com:8090/cluster/app/application_1650968366405_0009 Then click on links to logs of each attempt. . Failing the application. at org.apache.tez.client.TezClient.waitTillReady(TezClient.java:979) ~[tez-api-0.9.1.7.1.7.1000-141.jar:0.9.1.7.1.7.1000-141] at org.apache.tez.client.TezClient.waitTillReady(TezClient.java:948) ~[tez-api-0.9.1.7.1.7.1000-141.jar:0.9.1.7.1.7.1000-141] at org.apache.hadoop.hive.ql.exec.tez.TezSessionState.startSessionAndContainers(TezSessionState.java:567) ~[hive-exec-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141] at org.apache.hadoop.hive.ql.exec.tez.TezSessionState.openInternal(TezSessionState.java:385) ~[hive-exec-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141] at org.apache.hadoop.hive.ql.exec.tez.TezSessionState.open(TezSessionState.java:314) ~[hive-exec-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141] at org.apache.hadoop.hive.ql.exec.tez.TezSessionPoolSession.open(TezSessionPoolSession.java:118) ~[hive-exec-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141] at org.apache.hadoop.hive.ql.exec.tez.TezSessionPool.startInitialSession(TezSessionPool.java:359) ~[hive-exec-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141] at org.apache.hadoop.hive.ql.exec.tez.TezSessionPool.startUnderInitLock(TezSessionPool.java:171) ~[hive-exec-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141] at org.apache.hadoop.hive.ql.exec.tez.TezSessionPool.start(TezSessionPool.java:123) ~[hive-exec-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141] at org.apache.hadoop.hive.ql.exec.tez.TezSessionPoolManager.startPool(TezSessionPoolManager.java:115) ~[hive-exec-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141] at org.apache.hive.service.server.HiveServer2.initAndStartTezSessionPoolManager(HiveServer2.java:822) ~[hive-service-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141] at org.apache.hive.service.server.HiveServer2.startOrReconnectTezSessions(HiveServer2.java:795) ~[hive-service-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141] at org.apache.hive.service.server.HiveServer2.start(HiveServer2.java:718) ~[hive-service-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141] ... 10 more
1:25:53.508 AM	ERROR	HiveServer2	[main-EventThread]: Error stopping schq java.lang.IllegalStateException: The current ScheduledQueryExecutionService INSTANCE is invalid at org.apache.hadoop.hive.ql.scheduled.ScheduledQueryExecutionService.close(ScheduledQueryExecutionService.java:312) ~[hive-exec-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141] at org.apache.hive.service.server.HiveServer2.stop(HiveServer2.java:892) [hive-service-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141] at org.apache.hive.service.server.HiveServer2$DeRegisterWatcher.process(HiveServer2.java:617) [hive-service-3.1.3000.7.1.7.1000-141.jar:3.1.3000.7.1.7.1000-141] at org.apache.curator.framework.imps.NamespaceWatcher.process(NamespaceWatcher.java:77) [curator-framework-4.3.0.7.1.7.1000-141.jar:4.3.0.7.1.7.1000-141] at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:532) [zookeeper-3.5.5.7.1.7.1000-141.jar:3.5.5.7.1.7.1000-141] at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:507) [zookeeper-3.5.5.7.1.7.1000-141.jar:3.5.5.7.1.7.1000-141]

Sign up

Please use your Cloudera account.

Login to the community

Please use your Cloudera account.