Support Questions
Find answers, ask questions, and share your expertise
Announcements
Alert: Welcome to the Unified Cloudera Community. Former HCC members be sure to read and learn how to activate your account here.

error when i start Cloudera Manager

error when i start Cloudera Manager

Contributor

Hi

I have Cloudera Manager 5.11. When i start the server, i have this:

2017-07-27 16:21:57,934 WARN 2006132096@scm-web-2:com.cloudera.server.web.cmf.StatusProvider: Failed to get status from SERVICE_MONITORING
java.util.concurrent.ExecutionException: org.apache.avro.AvroRemoteException: java.net.SocketTimeoutException: Read timed out
        at java.util.concurrent.FutureTask.report(FutureTask.java:122)
        at java.util.concurrent.FutureTask.get(FutureTask.java:188)
        at com.cloudera.server.web.cmf.StatusProvider$SubjectStatusCustomFuture.safeGetResponse(StatusProvider.java:645)
        at com.cloudera.server.web.cmf.StatusProvider.getStatus(StatusProvider.java:1080)
        at com.cloudera.api.dao.impl.ServiceManagerDaoImpl.getServicesByType(ServiceManagerDaoImpl.java:595)
        at com.cloudera.api.dao.impl.ServiceManagerDaoImpl.getMgmtService(ServiceManagerDaoImpl.java:622)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:606)
        at com.cloudera.api.dao.impl.ManagerDaoBase.runInNewTransaction(ManagerDaoBase.java:193)
        at com.cloudera.api.dao.impl.ManagerDaoBase.access$100(ManagerDaoBase.java:78)
        at com.cloudera.api.dao.impl.ManagerDaoBase$TransactionCallable.call(ManagerDaoBase.java:229)
        at com.cloudera.server.common.RetryWrapper.executeWithRetry(RetryWrapper.java:32)
        at com.cloudera.server.common.RetryUtils.executeWithRetry(RetryUtils.java:124)
        at com.cloudera.api.dao.impl.ManagerDaoBase.runInNewTransactionWithRetry(ManagerDaoBase.java:165)
        at com.cloudera.api.dao.impl.ManagerDaoBase.invoke(ManagerDaoBase.java:262)
        at com.sun.proxy.$Proxy121.getMgmtService(Unknown Source)
        at com.cloudera.api.v1.impl.MgmtServiceResourceImpl.readService(MgmtServiceResourceImpl.java:42)
        at com.cloudera.api.v1.impl.MgmtServiceResourceImpl$RolesResourceWrapper.<init>(MgmtServiceResourceImpl.java:206)
        at com.cloudera.api.v1.impl.MgmtServiceResourceImpl.getRolesResource(MgmtServiceResourceImpl.java:157)
        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
        at java.lang.reflect.Method.invoke(Method.java:606)
        at org.apache.cxf.service.invoker.AbstractInvoker.performInvocation(AbstractInvoker.java:180)
        at org.apache.cxf.service.invoker.AbstractInvoker.invoke(AbstractInvoker.java:96)
        at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:192)
        at com.cloudera.api.ApiInvoker.invoke(ApiInvoker.java:116)
        at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:256)
        at com.cloudera.api.ApiInvoker.invoke(ApiInvoker.java:116)
        at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:256)
        at com.cloudera.api.ApiInvoker.invoke(ApiInvoker.java:116)
        at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:256)
        at com.cloudera.api.ApiInvoker.invoke(ApiInvoker.java:116)
        at org.apache.cxf.jaxrs.JAXRSInvoker.invoke(JAXRSInvoker.java:100)
        at org.apache.cxf.interceptor.ServiceInvokerInterceptor$1.run(ServiceInvokerInterceptor.java:57)
        at org.apache.cxf.interceptor.ServiceInvokerInterceptor.handleMessage(ServiceInvokerInterceptor.java:93)
        at org.apache.cxf.phase.PhaseInterceptorChain.doIntercept(PhaseInterceptorChain.java:272)
        at org.apache.cxf.transport.ChainInitiationObserver.onMessage(ChainInitiationObserver.java:121)
        at org.apache.cxf.transport.http.AbstractHTTPDestination.invoke(AbstractHTTPDestination.java:239)
        at org.apache.cxf.transport.servlet.ServletController.invokeDestination(ServletController.java:248)
        at org.apache.cxf.transport.servlet.ServletController.invoke(ServletController.java:222)
        at org.apache.cxf.transport.servlet.ServletController.invoke(ServletController.java:153)
        at org.apache.cxf.transport.servlet.CXFNonSpringServlet.invoke(CXFNonSpringServlet.java:167)
        at org.apache.cxf.transport.servlet.AbstractHTTPServlet.handleRequest(AbstractHTTPServlet.java:286)
        at org.apache.cxf.transport.servlet.AbstractHTTPServlet.doGet(AbstractHTTPServlet.java:211)
        at javax.servlet.http.HttpServlet.service(HttpServlet.java:575)
        at org.apache.cxf.transport.servlet.AbstractHTTPServlet.service(AbstractHTTPServlet.java:262)
        at org.mortbay.jetty.servlet.ServletHolder.handle(ServletHolder.java:511)
        at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1221)
        at com.cloudera.enterprise.JavaMelodyFacade$MonitoringFilter.doFilter(JavaMelodyFacade.java:109)
        at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1212)
        at org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:311)
        at org.springframework.security.web.access.intercept.FilterSecurityInterceptor.invoke(FilterSecurityInterceptor.java:116)
        at org.springframework.security.web.access.intercept.FilterSecurityInterceptor.doFilter(FilterSecurityInterceptor.java:83)
        at org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:323)
        at org.springframework.security.web.access.ExceptionTranslationFilter.doFilter(ExceptionTranslationFilter.java:113)
        at org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:323)
        at org.springframework.security.web.session.SessionManagementFilter.doFilter(SessionManagementFilter.java:101)
        at org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:323)
        at org.springframework.security.web.authentication.AnonymousAuthenticationFilter.doFilter(AnonymousAuthenticationFilter.java:113)
        at org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:323)
        at org.springframework.security.web.servletapi.SecurityContextHolderAwareRequestFilter.doFilter(SecurityContextHolderAwareRequestFilter.java:54)
        at org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:323)
        at org.springframework.security.web.savedrequest.RequestCacheAwareFilter.doFilter(RequestCacheAwareFilter.java:45)
        at org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:323)
        at org.springframework.security.web.authentication.www.BasicAuthenticationFilter.doFilter(BasicAuthenticationFilter.java:201)
        at org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:323)
        at com.cloudera.api.ApiBasicAuthFilter.doFilter(ApiBasicAuthFilter.java:44)
        at org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:323)
        at org.springframework.security.web.context.SecurityContextPersistenceFilter.doFilter(SecurityContextPersistenceFilter.java:87)
        at org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:323)
        at org.springframework.security.web.FilterChainProxy.doFilter(FilterChainProxy.java:173)
        at org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:237)
        at org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:167)
        at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1212)
        at org.springframework.web.filter.CharacterEncodingFilter.doFilterInternal(CharacterEncodingFilter.java:88)
        at org.springframework.web.filter.OncePerRequestFilter.doFilter(OncePerRequestFilter.java:76)
        at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1212)
        at org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:399)
        at org.mortbay.jetty.security.SecurityHandler.handle(SecurityHandler.java:216)
        at org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:182)
        at org.mortbay.jetty.security.SecurityHandler.handle(SecurityHandler.java:216)
        at org.mortbay.jetty.security.SecurityHandler.handle(SecurityHandler.java:216)
        at org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:767)
        at org.mortbay.jetty.webapp.WebAppContext.handle(WebAppContext.java:450)
        at org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152)
        at org.mortbay.jetty.handler.StatisticsHandler.handle(StatisticsHandler.java:53)
        at org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152)
        at org.mortbay.jetty.Server.handle(Server.java:326)
        at org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:542)
        at org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConnection.java:928)
        at org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:549)
        at org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:212)
        at org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:404)
        at org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:410)
        at org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582)
Caused by: org.apache.avro.AvroRemoteException: java.net.SocketTimeoutException: Read timed out
        at org.apache.avro.ipc.specific.SpecificRequestor.invoke(SpecificRequestor.java:88)
        at com.sun.proxy.$Proxy111.getSubjectStatus(Unknown Source)
        at com.cloudera.cmf.protocol.firehose.nozzle.TimeoutNozzleIPC.getSubjectStatus(TimeoutNozzleIPC.java:419)
        at com.cloudera.server.web.cmf.StatusProvider$SubjectStatusCustomFuture$1.call(StatusProvider.java:734)
        at com.cloudera.server.web.cmf.StatusProvider$SubjectStatusCustomFuture$1.call(StatusProvider.java:730)
        at java.util.concurrent.FutureTask.run(FutureTask.java:262)
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
        at java.lang.Thread.run(Thread.java:745)
Caused by: java.net.SocketTimeoutException: Read timed out
        at java.net.SocketInputStream.socketRead0(Native Method)
        at java.net.SocketInputStream.read(SocketInputStream.java:152)
        at java.net.SocketInputStream.read(SocketInputStream.java:122)
        at java.io.BufferedInputStream.fill(BufferedInputStream.java:235)
        at java.io.BufferedInputStream.read1(BufferedInputStream.java:275)
        at java.io.BufferedInputStream.read(BufferedInputStream.java:334)
        at sun.net.www.http.HttpClient.parseHTTPHeader(HttpClient.java:687)
        at sun.net.www.http.HttpClient.parseHTTP(HttpClient.java:633)
        at sun.net.www.protocol.http.HttpURLConnection.getInputStream(HttpURLConnection.java:1323)
        at org.apache.avro.ipc.HttpTransceiver.readBuffers(HttpTransceiver.java:54)
        at org.apache.avro.ipc.Transceiver.transceive(Transceiver.java:59)
        at org.apache.avro.ipc.Transceiver.transceive(Transceiver.java:72)
        at org.apache.avro.ipc.Requestor.request(Requestor.java:147)
        at org.apache.avro.ipc.Requestor.request(Requestor.java:101)
        at org.apache.avro.ipc.specific.SpecificRequestor.invoke(SpecificRequestor.java:72)
        ... 8 more

can you explain it to me?

how can i resolve it?

2 REPLIES 2

Re: error when i start Cloudera Manager

Champion
Does the CM server fail to start? Is this from the CM server logs? Can you get to the UI?

The error is complaining that it can't reach the Service Monitor which is a secondary service of CM. The CM server should still start. If you can get to the UI, check the Service Monitor logs for more information on why it is failing.

Re: error when i start Cloudera Manager

Contributor

thanks for your help and disponibility

yes the server start

yes it's from the server log

i have acces to the Web UI

this what i have in service monitor log

Failed to collect information on NameNode hdfs-NAMENODE-fa173cfa0f5889213dd73172954cf1a5
java.io.IOException: Failed to connect to: http://ha01vmlx.toto:50070/jmx?qry=Hadoop:*
	at com.cloudera.cmf.cdhclient.common.security.SecureUrlUtil.readSecureUrlWithTimeouts(SecureUrlUtil.java:234)
	at com.cloudera.cmf.cdhclient.common.security.SecureUrlUtil.readSecureUrlWithTimeouts(SecureUrlUtil.java:124)
	at com.cloudera.cmf.cdhclient.util.SecurityUtil.readSecureUrlWithTimeouts(SecurityUtil.java:61)
	at com.cloudera.enterprise.UrlUtil.readUrlWithTimeouts(UrlUtil.java:52)
	at com.cloudera.enterprise.HttpBasedJMXJsonInfoRetriever.doQuery(HttpBasedJMXJsonInfoRetriever.java:298)
	at com.cloudera.enterprise.HttpBasedJMXJsonInfoRetriever.getMBeansByDomain(HttpBasedJMXJsonInfoRetriever.java:173)
	at com.cloudera.cmon.firehose.polling.hdfs.HdfsServiceStateFetcher.getNameNodeInfo(HdfsServiceStateFetcher.java:117)
	at com.cloudera.cmon.firehose.polling.hdfs.HdfsServiceStateFetcher.doWorkWithClientConfig(HdfsServiceStateFetcher.java:88)
	at com.cloudera.cmon.firehose.polling.hdfs.HdfsServiceStateFetcher.doWorkWithClientConfig(HdfsServiceStateFetcher.java:41)
	at com.cloudera.cmon.firehose.polling.AbstractCdhWorkUsingClientConfigs.doWork(AbstractCdhWorkUsingClientConfigs.java:45)
	at com.cloudera.cmon.firehose.polling.CdhTask$InstrumentedWork.doWork(CdhTask.java:230)
	at com.cloudera.cmf.cdhclient.CdhExecutor$SecurityWrapper$1.run(CdhExecutor.java:189)
	at java.security.AccessController.doPrivileged(Native Method)
	at javax.security.auth.Subject.doAs(Subject.java:415)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1693)
	at com.cloudera.cmf.cdh5client.security.UserGroupInformationImpl.doAs(UserGroupInformationImpl.java:41)
	at com.cloudera.cmf.cdhclient.CdhExecutor$SecurityWrapper.doWork(CdhExecutor.java:186)
	at com.cloudera.cmf.cdhclient.CdhExecutor$1.call(CdhExecutor.java:125)
	at java.util.concurrent.FutureTask.run(FutureTask.java:262)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
	at java.lang.Thread.run(Thread.java:745)