28806
DISCUSSIONS
102200
MEMBERS
3161
ARTICLES
Created 10-26-2015 01:37 PM
I am getting a connection error in which I am suspecting network latency to be the primary contributor
I set in cloudera manager hbase.rpc.timeout to 10 minutes but error still shows 60 sec
2015-10-26 15:25:26,303 WARN org.apache.hadoop.hbase.thrift.ThriftServerRunner$HBaseHandler: Failed after attempts=36, exceptions: Mon Oct 26 15:25:26 CDT 2015, null, java.net.SocketTimeoutException: callTimeout=60000, callDuration=98589: row '1month|EUR|rhub8VRN55s94qWKDv6jmDy1pUykJzF3wq|EUR|rLEsXccBGNR3UPuPu2hUXPjziKC3qKSBun|20140101000000' on table 'prod_agg_exchanges' at region=prod_agg_exchanges,1minute|JPY|rMAz5ZnK73nyNUL4foAvaxdreczCkG3vA6|USD|rMwjYedjc7qqtKYVLiAccJSmCwih4LnE2q|20150306162900,1437375197929.fe9d00dc05ac8156f8117dedb6acb594., hostname=hadoop3-private.wdc01.infra.ripple.com,60020,1445891075197, seqNum=1453001 org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=36, exceptions: Mon Oct 26 15:25:26 CDT 2015, null, java.net.SocketTimeoutException: callTimeout=60000, callDuration=98589: row '1month|EUR|rhub8VRN55s94qWKDv6jmDy1pUykJzF3wq|EUR|rLEsXccBGNR3UPuPu2hUXPjziKC3qKSBun|20140101000000' on table 'prod_agg_exchanges' at region=prod_agg_exchanges,1minute|JPY|rMAz5ZnK73nyNUL4foAvaxdreczCkG3vA6|USD|rMwjYedjc7qqtKYVLiAccJSmCwih4LnE2q|20150306162900,1437375197929.fe9d00dc05ac8156f8117dedb6acb594., hostname=hadoop3-private.wdc01.infra.ripple.com,60020,1445891075197, seqNum=1453001 at org.apache.hadoop.hbase.client.RpcRetryingCallerWithReadReplicas.throwEnrichedException(RpcRetryingCallerWithReadReplicas.java:270) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:219) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:57) at org.apache.hadoop.hbase.client.RpcRetryingCaller.callWithoutRetries(RpcRetryingCaller.java:200) at org.apache.hadoop.hbase.client.ClientScanner.call(ClientScanner.java:293) at org.apache.hadoop.hbase.client.ClientScanner.nextScanner(ClientScanner.java:268) at org.apache.hadoop.hbase.client.ClientScanner.initializeScannerInConstruction(ClientScanner.java:140) at org.apache.hadoop.hbase.client.ClientScanner.<init>(ClientScanner.java:135) at org.apache.hadoop.hbase.client.HTable.getScanner(HTable.java:888) at org.apache.hadoop.hbase.thrift.ThriftServerRunner$HBaseHandler.scannerOpenWithScan(ThriftServerRunner.java:1446) at sun.reflect.GeneratedMethodAccessor1.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:606) at org.apache.hadoop.hbase.thrift.HbaseHandlerMetricsProxy.invoke(HbaseHandlerMetricsProxy.java:67) at com.sun.proxy.$Proxy14.scannerOpenWithScan(Unknown Source) at org.apache.hadoop.hbase.thrift.generated.Hbase$Processor$scannerOpenWithScan.getResult(Hbase.java:4609) at org.apache.hadoop.hbase.thrift.generated.Hbase$Processor$scannerOpenWithScan.getResult(Hbase.java:4593) at org.apache.thrift.ProcessFunction.process(ProcessFunction.java:39) at org.apache.thrift.TBaseProcessor.process(TBaseProcessor.java:39) at org.apache.hadoop.hbase.thrift.TBoundedThreadPoolServer$ClientConnnection.run(TBoundedThreadPoolServer.java:289) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:745) Caused by: java.net.SocketTimeoutException: callTimeout=60000, callDuration=98589: row '1month|EUR|rhub8VRN55s94qWKDv6jmDy1pUykJzF3wq|EUR|rLEsXccBGNR3UPuPu2hUXPjziKC3qKSBun|20140101000000' on table 'prod_agg_exchanges' at region=prod_agg_exchanges,1minute|JPY|rMAz5ZnK73nyNUL4foAvaxdreczCkG3vA6|USD|rMwjYedjc7qqtKYVLiAccJSmCwih4LnE2q|20150306162900,1437375197929.fe9d00dc05ac8156f8117dedb6acb594., hostname=hadoop3-private.wdc01.infra.ripple.com,60020,1445891075197, seqNum=1453001 at org.apache.hadoop.hbase.client.RpcRetryingCaller.callWithRetries(RpcRetryingCaller.java:159) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:64) ... 3 more Caused by: org.apache.hadoop.hbase.NotServingRegionException: org.apache.hadoop.hbase.NotServingRegionException: Region prod_agg_exchanges,1minute|JPY|rMAz5ZnK73nyNUL4foAvaxdreczCkG3vA6|USD|rMwjYedjc7qqtKYVLiAccJSmCwih4LnE2q|20150306162900,1437375197929.fe9d00dc05ac8156f8117dedb6acb594. is not online on hadoop3-private.wdc01.infra.ripple.com,60020,1445891117541 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2794) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:892) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:2108) at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:31443) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2035) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107) at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130) at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107) at java.lang.Thread.run(Thread.java:745) at sun.reflect.GeneratedConstructorAccessor8.newInstance(Unknown Source) at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) at java.lang.reflect.Constructor.newInstance(Constructor.java:526) at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:106) at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:95) at org.apache.hadoop.hbase.protobuf.ProtobufUtil.getRemoteException(ProtobufUtil.java:326) at org.apache.hadoop.hbase.client.ScannerCallable.openScanner(ScannerCallable.java:357) at org.apache.hadoop.hbase.client.ScannerCallable.call(ScannerCallable.java:193) at org.apache.hadoop.hbase.client.ScannerCallable.call(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RpcRetryingCaller.callWithoutRetries(RpcRetryingCaller.java:200) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:332) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:306) at org.apache.hadoop.hbase.client.RpcRetryingCaller.callWithRetries(RpcRetryingCaller.java:126) ... 4 more Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException(org.apache.hadoop.hbase.NotServingRegionException): org.apache.hadoop.hbase.NotServingRegionException: Region prod_agg_exchanges,1minute|JPY|rMAz5ZnK73nyNUL4foAvaxdreczCkG3vA6|USD|rMwjYedjc7qqtKYVLiAccJSmCwih4LnE2q|20150306162900,1437375197929.fe9d00dc05ac8156f8117dedb6acb594. is not online on hadoop3-private.wdc01.infra.ripple.com,60020,1445891117541 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:2794) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:892) at org.apache.hadoop.hbase.regionserver.RSRpcServices.scan(RSRpcServices.java:2108) at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:31443) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:2035) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:107) at org.apache.hadoop.hbase.ipc.RpcExecutor.consumerLoop(RpcExecutor.java:130) at org.apache.hadoop.hbase.ipc.RpcExecutor$1.run(RpcExecutor.java:107) at java.lang.Thread.run(Thread.java:745) at org.apache.hadoop.hbase.ipc.RpcClientImpl.call(RpcClientImpl.java:1210) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:216) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:300) at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$BlockingStub.scan(ClientProtos.java:31889) at org.apache.hadoop.hbase.client.ScannerCallable.openScanner(ScannerCallable.java:349) ... 10 more 2015-10-26 15:25:26,325 WARN org.apache.hadoop.hbase.thrift.ThriftServerRunner$HBaseHandler: Failed after attempts=36, exceptions: Mon Oct 26 15:25:26 CDT 2015, null, java.net.SocketTimeoutException: callTimeout=60000, callDuration=98725: row '1minute|EUR|rhub8VRN55s94qWKDv6jmDy1pUykJzF3wq|EUR|rLEsXccBGNR3UPuPu2hUXPjziKC3qKSBun|20151026170000' on table 'prod_agg_exchanges' at region=prod_agg_exchanges,1minute|CNY|razqQKzJRdB4UxFPWf5NEpEG3WMkmwgcXA|STR|rKiCet8SdvWxPXnAgYarFUXMh1zCPz432Y|20141016074700,1438564543660.186625dc63689c992d7551527a172cca., hostname=hadoop2-private.wdc01.infra.ripple.com,60020,1445891082985, seqNum=433654 org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=36, exceptions: Mon Oct 26 15:25:26 CDT 2015, null, java.net.SocketTimeoutException: callTimeout=60000, callDuration=98725: row '1minute|EUR|rhub8VRN55s94qWKDv6jmDy1pUykJzF3wq|EUR|rLEsXccBGNR3UPuPu2hUXPjziKC3qKSBun|20151026170000' on table 'prod_agg_exchanges' at region=prod_agg_exchanges,1minute|CNY|razqQKzJRdB4UxFPWf5NEpEG3WMkmwgcXA|STR|rKiCet8SdvWxPXnAgYarFUXMh1zCPz432Y|20141016074700,1438564543660.186625dc63689c992d7551527a172cca., hostname=hadoop2-private.wdc01.infra.ripple.com,60020,1445891082985, seqNum=433654 at org.apache.hadoop.hbase.client.RpcRetryingCallerWithReadReplicas.throwEnrichedException(RpcRetryingCallerWithReadReplicas.java:270) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:219) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:57) at org.apache.hadoop.hbase.client.RpcRetryingCaller.callWithoutRetries(RpcRetryingCaller.java:200) at org.apache.hadoop.hbase.client.ClientScanner.call(ClientScanner.java:293) at org.apache.hadoop.hbase.client.ClientScanner.nextScanner(ClientScanner.java:268) at org.apache.hadoop.hbase.client.ClientScanner.initializeScannerInConstruction(ClientScanner.java:140) at org.apache.hadoop.hbase.client.ClientScanner.<init>(ClientScanner.java:135) at org.apache.hadoop.hbase.client.HTable.getScanner(HTable.java:888) at org.apache.hadoop.hbase.thrift.ThriftServerRunner$HBaseHandler.scannerOpenWithScan(ThriftServerRunner.java:1446) at sun.reflect.GeneratedMethodAccessor1.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:606) at org.apache.hadoop.hbase.thrift.HbaseHandlerMetricsProxy.invoke(HbaseHandlerMetricsProxy.java:67) at com.sun.proxy.$Proxy14.scannerOpenWithScan(Unknown Source) at org.apache.hadoop.hbase.thrift.generated.Hbase$Processor$scannerOpenWithScan.getResult(Hbase.java:4609) at org.apache.hadoop.hbase.thrift.generated.Hbase$Processor$scannerOpenWithScan.getResult(Hbase.java:4593) at org.apache.thrift.ProcessFunction.process(ProcessFunction.java:39) at org.apache.thrift.TBaseProcessor.process(TBaseProcessor.java:39) at org.apache.hadoop.hbase.thrift.TBoundedThreadPoolServer$ClientConnnection.run(TBoundedThreadPoolServer.java:289) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:745) Caused by: java.net.SocketTimeoutException: callTimeout=60000, callDuration=98725: row '1minute|EUR|rhub8VRN55s94qWKDv6jmDy1pUykJzF3wq|EUR|rLEsXccBGNR3UPuPu2hUXPjziKC3qKSBun|20151026170000' on table 'prod_agg_exchanges' at region=prod_agg_exchanges,1minute|CNY|razqQKzJRdB4UxFPWf5NEpEG3WMkmwgcXA|STR|rKiCet8SdvWxPXnAgYarFUXMh1zCPz432Y|20141016074700,1438564543660.186625dc63689c992d7551527a172cca., hostname=hadoop2-private.wdc01.infra.ripple.com,60020,1445891082985, seqNum=433654 at org.apache.hadoop.hbase.client.RpcRetryingCaller.callWithRetries(RpcRetryingCaller.java:159) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:64) ... 3 more Caused by: org.apache.hadoop.hbase.exceptions.ConnectionClosingException: Call to hadoop2-private.wdc01.infra.ripple.com/10.125.122.222:60020 failed on local exception: org.apache.hadoop.hbase.exceptions.ConnectionClosingException: Connection to hadoop2-private.wdc01.infra.ripple.com/10.125.122.222:60020 is closing. Call id=838, waitTime=0 at org.apache.hadoop.hbase.ipc.RpcClientImpl.wrapException(RpcClientImpl.java:1243) at org.apache.hadoop.hbase.ipc.RpcClientImpl.call(RpcClientImpl.java:1214) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:216) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:300) at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$BlockingStub.scan(ClientProtos.java:31889) at org.apache.hadoop.hbase.client.ScannerCallable.openScanner(ScannerCallable.java:349) at org.apache.hadoop.hbase.client.ScannerCallable.call(ScannerCallable.java:193) at org.apache.hadoop.hbase.client.ScannerCallable.call(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RpcRetryingCaller.callWithoutRetries(RpcRetryingCaller.java:200) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:332) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:306) at org.apache.hadoop.hbase.client.RpcRetryingCaller.callWithRetries(RpcRetryingCaller.java:126) ... 4 more Caused by: org.apache.hadoop.hbase.exceptions.ConnectionClosingException: Connection to hadoop2-private.wdc01.infra.ripple.com/10.125.122.222:60020 is closing. Call id=838, waitTime=0 at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.cleanupCalls(RpcClientImpl.java:1033) at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.close(RpcClientImpl.java:840) at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.run(RpcClientImpl.java:568) 2015-10-26 15:25:42,844 WARN org.apache.hadoop.hbase.thrift.ThriftServerRunner$HBaseHandler: Failed after attempts=36, exceptions: Mon Oct 26 15:25:42 CDT 2015, null, java.net.SocketTimeoutException: callTimeout=60000, callDuration=80726: row '000016648981' on table 'prod_lu_ledgers_by_index' at region=prod_lu_ledgers_by_index,000014434619,1438568606685.486dedaca4863d62a31cbfd6a8e9de57., hostname=hadoop4-private.wdc01.infra.ripple.com,60020,1445891059883, seqNum=3336890 org.apache.hadoop.hbase.client.RetriesExhaustedException: Failed after attempts=36, exceptions: Mon Oct 26 15:25:42 CDT 2015, null, java.net.SocketTimeoutException: callTimeout=60000, callDuration=80726: row '000016648981' on table 'prod_lu_ledgers_by_index' at region=prod_lu_ledgers_by_index,000014434619,1438568606685.486dedaca4863d62a31cbfd6a8e9de57., hostname=hadoop4-private.wdc01.infra.ripple.com,60020,1445891059883, seqNum=3336890 at org.apache.hadoop.hbase.client.RpcRetryingCallerWithReadReplicas.throwEnrichedException(RpcRetryingCallerWithReadReplicas.java:270) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:219) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas.call(ScannerCallableWithReplicas.java:57) at org.apache.hadoop.hbase.client.RpcRetryingCaller.callWithoutRetries(RpcRetryingCaller.java:200) at org.apache.hadoop.hbase.client.ReversedClientScanner.nextScanner(ReversedClientScanner.java:121) at org.apache.hadoop.hbase.client.ClientScanner.initializeScannerInConstruction(ClientScanner.java:140) at org.apache.hadoop.hbase.client.ClientScanner.<init>(ClientScanner.java:135) at org.apache.hadoop.hbase.client.ReversedClientScanner.<init>(ReversedClientScanner.java:60) at org.apache.hadoop.hbase.client.HTable.getScanner(HTable.java:877) at org.apache.hadoop.hbase.thrift.ThriftServerRunner$HBaseHandler.scannerOpenWithScan(ThriftServerRunner.java:1446) at sun.reflect.GeneratedMethodAccessor1.invoke(Unknown Source) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:606) at org.apache.hadoop.hbase.thrift.HbaseHandlerMetricsProxy.invoke(HbaseHandlerMetricsProxy.java:67) at com.sun.proxy.$Proxy14.scannerOpenWithScan(Unknown Source) at org.apache.hadoop.hbase.thrift.generated.Hbase$Processor$scannerOpenWithScan.getResult(Hbase.java:4609) at org.apache.hadoop.hbase.thrift.generated.Hbase$Processor$scannerOpenWithScan.getResult(Hbase.java:4593) at org.apache.thrift.ProcessFunction.process(ProcessFunction.java:39) at org.apache.thrift.TBaseProcessor.process(TBaseProcessor.java:39) at org.apache.hadoop.hbase.thrift.TBoundedThreadPoolServer$ClientConnnection.run(TBoundedThreadPoolServer.java:289) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:745) Caused by: java.net.SocketTimeoutException: callTimeout=60000, callDuration=80726: row '000016648981' on table 'prod_lu_ledgers_by_index' at region=prod_lu_ledgers_by_index,000014434619,1438568606685.486dedaca4863d62a31cbfd6a8e9de57., hostname=hadoop4-private.wdc01.infra.ripple.com,60020,1445891059883, seqNum=3336890 at org.apache.hadoop.hbase.client.RpcRetryingCaller.callWithRetries(RpcRetryingCaller.java:159) at org.apache.hadoop.hbase.client.ResultBoundedCompletionService$QueueingFuture.run(ResultBoundedCompletionService.java:64) ... 3 more Caused by: org.apache.hadoop.hbase.exceptions.ConnectionClosingException: Call to hadoop4-private.wdc01.infra.ripple.com/10.125.122.203:60020 failed on local exception: org.apache.hadoop.hbase.exceptions.ConnectionClosingException: Connection to hadoop4-private.wdc01.infra.ripple.com/10.125.122.203:60020 is closing. Call id=865, waitTime=0 at org.apache.hadoop.hbase.ipc.RpcClientImpl.wrapException(RpcClientImpl.java:1243) at org.apache.hadoop.hbase.ipc.RpcClientImpl.call(RpcClientImpl.java:1214) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:216) at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:300) at org.apache.hadoop.hbase.protobuf.generated.ClientProtos$ClientService$BlockingStub.scan(ClientProtos.java:31889) at org.apache.hadoop.hbase.client.ScannerCallable.openScanner(ScannerCallable.java:349) at org.apache.hadoop.hbase.client.ScannerCallable.call(ScannerCallable.java:193) at org.apache.hadoop.hbase.client.ScannerCallable.call(ScannerCallable.java:62) at org.apache.hadoop.hbase.client.RpcRetryingCaller.callWithoutRetries(RpcRetryingCaller.java:200) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:332) at org.apache.hadoop.hbase.client.ScannerCallableWithReplicas$RetryingRPC.call(ScannerCallableWithReplicas.java:306) at org.apache.hadoop.hbase.client.RpcRetryingCaller.callWithRetries(RpcRetryingCaller.java:126) ... 4 more Caused by: org.apache.hadoop.hbase.exceptions.ConnectionClosingException: Connection to hadoop4-private.wdc01.infra.ripple.com/10.125.122.203:60020 is closing. Call id=865, waitTime=0 at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.cleanupCalls(RpcClientImpl.java:1033) at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.close(RpcClientImpl.java:840) at org.apache.hadoop.hbase.ipc.RpcClientImpl$Connection.run(RpcClientImpl.java:568)