Support Questions

Find answers, ask questions, and share your expertise
Announcements
Celebrating as our community reaches 100,000 members! Thank you!

Tez UI not coming up, failing with spnego auth

avatar
Explorer

Adapter operation failed » 500: Failed to fetch results by the proxy from url: http://hostname:8188/ws/v1/timeline/TEZ_DAG_ID?limit=11&_=1490004805683. Internal Error.. SPNego authentication failed, can not get hadoop.auth cookie Details:

{ "message": "Failed to fetch results by the proxy from url: http://hostname:8188/ws/v1/timeline/TEZ_DAG_ID?limit=11&_=1490004805683. Internal Error.. SPNego authentication failed, can not get hadoop.auth cookie", "status": 500, "trace": "java.io.IOException: SPNego authentication failed, can not get hadoop.auth cookie\n\tat org.apache.ambari.server.controller.internal.AppCookieManager.getAppCookie(AppCookieManager.java:123)\n\tat org.apache.ambari.server.controller.internal.URLStreamProvider.processURL(URLStreamProvider.java:228)\n\tat org.apache.ambari.server.view.ViewURLStreamProvider.getHttpURLConnection(ViewURLStreamProvider.java:239)\n\tat org.apache.ambari.server.view.ViewURLStreamProvider.getConnection(ViewURLStreamProvider.java:146)\n\tat org.apache.ambari.server.view.ViewURLStreamProvider.getConnectionAs(ViewURLStreamProvider.java:163)\n\tat org.apache.ambari.server.view.ViewURLStreamProvider.getConnectionAsCurrent(ViewURLStreamProvider.java:180)\n\tat org.apache.ambari.view.tez.utils.ProxyHelper.getResponse(ProxyHelper.java:61)\n\tat org.apache.ambari.view.tez.rest.BaseProxyResource.getData(BaseProxyResource.java:64)\n\tat sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\n\tat sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)\n\tat sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\n\tat java.lang.reflect.Method.invoke(Method.java:606)\n\tat com.sun.jersey.spi.container.JavaMethodInvokerFactory$1.invoke(JavaMethodInvokerFactory.java:60)\n\tat com.sun.jersey.server.impl.model.method.dispatch.AbstractResourceMethodDispatchProvider$ResponseOutInvoker._dispatch(AbstractResourceMethodDispatchProvider.java:205)\n\tat com.sun.jersey.server.impl.model.method.dispatch.ResourceJavaMethodDispatcher.dispatch(ResourceJavaMethodDispatcher.java:75)\n\tat com.sun.jersey.server.impl.uri.rules.HttpMethodRule.accept(HttpMethodRule.java:302)\n\tat com.sun.jersey.server.impl.uri.rules.RightHandPathRule.accept(RightHandPathRule.java:147)\n\tat com.sun.jersey.server.impl.uri.rules.SubLocatorRule.accept(SubLocatorRule.java:137)\n\tat com.sun.jersey.server.impl.uri.rules.RightHandPathRule.accept(RightHandPathRule.java:147)\n\tat com.sun.jersey.server.impl.uri.rules.SubLocatorRule.accept(SubLocatorRule.java:137)\n\tat com.sun.jersey.server.impl.uri.rules.RightHandPathRule.accept(RightHandPathRule.java:147)\n\tat com.sun.jersey.server.impl.uri.rules.SubLocatorRule.accept(SubLocatorRule.java:137)\n\tat com.sun.jersey.server.impl.uri.rules.RightHandPathRule.accept(RightHandPathRule.java:147)\n\tat com.sun.jersey.server.impl.uri.rules.SubLocatorRule.accept(SubLocatorRule.java:137)\n\tat com.sun.jersey.server.impl.uri.rules.RightHandPathRule.accept(RightHandPathRule.java:147)\n\tat com.sun.jersey.server.impl.uri.rules.ResourceClassRule.accept(ResourceClassRule.java:108)\n\tat com.sun.jersey.server.impl.uri.rules.RightHandPathRule.accept(RightHandPathRule.java:147)\n\tat com.sun.jersey.server.impl.uri.rules.RootResourceClassesRule.accept(RootResourceClassesRule.java:84)\n\tat com.sun.jersey.server.impl.application.WebApplicationImpl._handleRequest(WebApplicationImpl.java:1542)\n\tat com.sun.jersey.server.impl.application.WebApplicationImpl._handleRequest(WebApplicationImpl.java:1473)\n\tat com.sun.jersey.server.impl.application.WebApplicationImpl.handleRequest(WebApplicationImpl.java:1419)\n\tat com.sun.jersey.server.impl.application.WebApplicationImpl.handleRequest(WebApplicationImpl.java:1409)\n\tat com.sun.jersey.spi.container.servlet.WebComponent.service(WebComponent.java:409)\n\tat com.sun.jersey.spi.container.servlet.ServletContainer.service(ServletContainer.java:558)\n\tat com.sun.jersey.spi.container.servlet.ServletContainer.service(ServletContainer.java:733)\n\tat javax.servlet.http.HttpServlet.service(HttpServlet.java:790)\n\tat org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:684)\n\tat org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1507)\n\tat org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:330)\n\tat org.springframework.security.web.access.intercept.FilterSecurityInterceptor.invoke(FilterSecurityInterceptor.java:118)\n\tat org.springframework.security.web.access.intercept.FilterSecurityInterceptor.doFilter(FilterSecurityInterceptor.java:84)\n\tat org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:342)\n\tat org.apache.ambari.server.security.authorization.AmbariAuthorizationFilter.doFilter(AmbariAuthorizationFilter.java:257)\n\tat org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:342)\n\tat org.springframework.security.web.access.ExceptionTranslationFilter.doFilter(ExceptionTranslationFilter.java:113)\n\tat org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:342)\n\tat org.springframework.security.web.session.SessionManagementFilter.doFilter(SessionManagementFilter.java:103)\n\tat org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:342)\n\tat org.springframework.security.web.authentication.AnonymousAuthenticationFilter.doFilter(AnonymousAuthenticationFilter.java:113)\n\tat org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:342)\n\tat org.springframework.security.web.servletapi.SecurityContextHolderAwareRequestFilter.doFilter(SecurityContextHolderAwareRequestFilter.java:54)\n\tat org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:342)\n\tat org.springframework.security.web.savedrequest.RequestCacheAwareFilter.doFilter(RequestCacheAwareFilter.java:45)\n\tat org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:342)\n\tat org.apache.ambari.server.security.authorization.jwt.JwtAuthenticationFilter.doFilter(JwtAuthenticationFilter.java:96)\n\tat org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:342)\n\tat org.springframework.security.web.authentication.www.BasicAuthenticationFilter.doFilter(BasicAuthenticationFilter.java:150)\n\tat org.apache.ambari.server.security.authentication.AmbariAuthenticationFilter.doFilter(AmbariAuthenticationFilter.java:88)\n\tat org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:342)\n\tat org.apache.ambari.server.security.authorization.AmbariUserAuthorizationFilter.doFilter(AmbariUserAuthorizationFilter.java:91)\n\tat org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:342)\n\tat org.springframework.security.web.context.SecurityContextPersistenceFilter.doFilter(SecurityContextPersistenceFilter.java:87)\n\tat org.springframework.security.web.FilterChainProxy$VirtualFilterChain.doFilter(FilterChainProxy.java:342)\n\tat org.springframework.security.web.FilterChainProxy.doFilterInternal(FilterChainProxy.java:192)\n\tat org.springframework.security.web.FilterChainProxy.doFilter(FilterChainProxy.java:160)\n\tat org.springframework.web.filter.DelegatingFilterProxy.invokeDelegate(DelegatingFilterProxy.java:237)\n\tat org.springframework.web.filter.DelegatingFilterProxy.doFilter(DelegatingFilterProxy.java:167)\n\tat org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1478)\n\tat org.apache.ambari.server.api.MethodOverrideFilter.doFilter(MethodOverrideFilter.java:72)\n\tat org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1478)\n\tat org.apache.ambari.server.api.AmbariPersistFilter.doFilter(AmbariPersistFilter.java:47)\n\tat org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1478)\n\tat org.apache.ambari.server.view.AmbariViewsMDCLoggingFilter.doFilter(AmbariViewsMDCLoggingFilter.java:54)\n\tat org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1478)\n\tat org.apache.ambari.server.view.ViewThrottleFilter.doFilter(ViewThrottleFilter.java:161)\n\tat org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1478)\n\tat org.apache.ambari.server.security.AbstractSecurityHeaderFilter.doFilter(AbstractSecurityHeaderFilter.java:109)\n\tat org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1478)\n\tat org.apache.ambari.server.security.AbstractSecurityHeaderFilter.doFilter(AbstractSecurityHeaderFilter.java:109)\n\tat org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1478)\n\tat org.eclipse.jetty.servlets.UserAgentFilter.doFilter(UserAgentFilter.java:82)\n\tat org.eclipse.jetty.servlets.GzipFilter.doFilter(GzipFilter.java:294)\n\tat org.eclipse.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1478)\n\tat org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:499)\n\tat org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:137)\n\tat org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:557)\n\tat org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:231)\n\tat org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1086)\n\tat org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:427)\n\tat org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:193)\n\tat org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1020)\n\tat org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:135)\n\tat org.apache.ambari.server.controller.AmbariHandlerList.processHandlers(AmbariHandlerList.java:212)\n\tat org.apache.ambari.server.controller.AmbariHandlerList.processHandlers(AmbariHandlerList.java:201)\n\tat org.apache.ambari.server.controller.AmbariHandlerList.handle(AmbariHandlerList.java:150)\n\tat org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:116)\n\tat org.eclipse.jetty.server.Server.handle(Server.java:370)\n\tat org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:494)\n\tat org.eclipse.jetty.server.AbstractHttpConnection.headerComplete(AbstractHttpConnection.java:973)\n\tat org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.headerComplete(AbstractHttpConnection.java:1035)\n\tat org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:641)\n\tat org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:231)\n\tat org.eclipse.jetty.server.AsyncHttpConnection.handle(AsyncHttpConnection.java:82)\n\tat org.eclipse.jetty.io.nio.SelectChannelEndPoint.handle(SelectChannelEndPoint.java:696)\n\tat org.eclipse.jetty.io.nio.SelectChannelEndPoint$1.run(SelectChannelEndPoint.java:53)\n\tat org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608)\n\tat org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543)\n\tat java.lang.Thread.run(Thread.java:745)\n" }

Request:

{ "adapterName": "dag", "url": "http://hostname2:8080/api/v1/views/TEZ/versions/0.7.0.2.5.3.0-136/instances/TEZ_CLUSTER_INSTANCE/resources/atsproxy/ws/v1/timeline/TEZ_DAG_ID", "responseHeaders": { "User": "admin", "Server": "Jetty(8.1.19.v20160209)", "X-Frame-Options": "SAMEORIGIN", "Content-Length": "11137", "X-XSS-Protection": "1; mode=block", "Content-Type": "application/json" }, "queryParams": { "limit": 11 }, "namespace": "dags" }

Stack:

Error: Adapter operation failed at ember$data$lib$adapters$errors$AdapterError.EmberError (http://hostname28080/views/TEZ/0.7.0.2.5.3.0-136/TEZ_CLUSTER_INSTANCE/assets/vendor.js:24586:21) at ember$data$lib$adapters$errors$AdapterError (http://hostname2:8080/views/TEZ/0.7.0.2.5.3.0-136/TEZ_CLUSTER_INSTANCE/assets/vendor.js:80222:50) at Class.handleResponse (http://hostname2:8080/views/TEZ/0.7.0.2.5.3.0-136/TEZ_CLUSTER_INSTANCE/assets/vendor.js:81517:16) at Class.hash.error (http://hostname2:8080/views/TEZ/0.7.0.2.5.3.0-136/TEZ_CLUSTER_INSTANCE/assets/vendor.js:81597:33) at fire (http://hostname2:8080/views/TEZ/0.7.0.2.5.3.0-136/TEZ_CLUSTER_INSTANCE/assets/vendor.js:3320:30) at Object.fireWith [as rejectWith] (http://hostname2:8080/views/TEZ/0.7.0.2.5.3.0-136/TEZ_CLUSTER_INSTANCE/assets/vendor.js:3432:7) at done (http://hostname2:8080/views/TEZ/0.7.0.2.5.3.0-136/TEZ_CLUSTER_INSTANCE/assets/vendor.js:8487:14) at XMLHttpRequest.<anonymous> (http://hostname2:8080/views/TEZ/0.7.0.2.5.3.0-136/TEZ_CLUSTER_INSTANCE/assets/vendor.js:8826:9)

1 ACCEPTED SOLUTION
3 REPLIES 3

avatar

avatar
Explorer

Thanks Mugdha. All the steps were followed initially except ambari-server setup-security. After this was run, the UI came up.

avatar
Rising Star

Hi @Mugdha,

I am facing same kind of exception when tried to integrate Knox with AD on Kerberized cluster and followed ambari-server setup-security document which is suggested by you but still same exception remains.

Log:

cat /usr/hdp/current/knox-server/logs/gateway.log

ERROR hadoop.gateway (AppCookieManager.java:getAppCookie(126)) - Failed Knox->Hadoop SPNegotiation authentication for URL: http://hostname1:50070/webhdfs/v1/?op=GETHOMEDIRECTORY&doAs=username
WARN hadoop.gateway (DefaultDispatch.java:executeOutboundRequest(138)) - Connection exception dispatching request: http://hostname1:50070/webhdfs/v1/?op=GETHOMEDIRECTORY&doAs=username java.io.IOException: SPNego authn failed, can not get hadoop.auth cookie
java.io.IOException: SPNego authn failed, can not get hadoop.auth cookie

cat /usr/hdp/current/knox-server/conf/topologies/sample5.xml

<topology>
<gateway><provider>
<role>authentication</role>
<name>ShiroProvider</name>
<enabled>true</enabled>
<param name="main.ldapRealm" value="org.apache.hadoop.gateway.shirorealm.KnoxLdapRealm"/>
<param name="main.ldapContextFactory" value="org.apache.hadoop.gateway.shirorealm.KnoxLdapContextFactory"/>
<param name="main.ldapRealm.contextFactory" value="$ldapContextFactory"/>
<param name="main.ldapRealm.contextFactory.url" value="ldaps://abcd123:636"/>
<param name="main.ldapRealm.contextFactory.systemUsername" value="testuser"/>
<param name="main.ldapRealm.contextFactory.systemPassword" value="testpassword"/>

<param name="main.ldapRealm.searchBase" value="DC=org,DC=apache,DC=com"/>
<param name="main.ldapRealm.userSearchAttributeName" value="sAMAccountName"/>
<param name="main.ldapRealm.userObjectClass" value="person"/>

<param name="main.ldapRealm.authorizationEnabled" value="true"/>
<param name="main.ldapRealm.groupSearchBase" value="OU=Service Accounts,OU=Applications,DC=org,DC=apache,DC=com"/>
<param name="main.ldapRealm.groupObjectClass" value="group"/>
<param name="main.ldapRealm.groupIdAttribute" value="sAMAccountName"/>
<param name="main.ldapRealm.memberAttribute" value="member"/>

<param name="main.cacheManager" value="org.apache.shiro.cache.ehcache.EhCacheManager"/>
<param name="main.securityManager.cacheManager" value="$cacheManager"/>
<param name="main.ldapRealm.authenticationCachingEnabled" value="true"/>

<param name="urls./**" value="authcBasic"/>
</provider>

<provider>
<role>authorization</role>
<name>AclsAuthz</name>
<enabled>true</enabled>
</provider>

<provider>
<role>identity-assertion</role>
<name>Default</name>
<enabled>true</enabled>
</provider>

</gateway>

<service>
<role>NAMENODE</role>
<url>hdfs://hostname1:8020</url>
</service>

<service>
<role>JOBTRACKER</role>
<url>rpc://hostname2:8050</url>
</service>

<service>
<role>WEBHDFS</role>
<url>http://hostname1:50070/webhdfs</url>
</service>

<service>
<role>WEBHCAT</role>
<url>http://hostname1:50111/templeton</url>
</service>

<service>
<role>OOZIE</role>
<url>http://hostname3:11000/oozie</url>
</service>

<service>
<role>WEBHBASE</role>
<url>http://hostname2:8080</url>
</service>

<service>
<role>HIVE</role>
<url>http://hostname1:10001/cliservice</url>
</service>

<service>
<role>RESOURCEMANAGER</role>
<url>http://hostname2:8088/ws</url>
</service>
<service>
<role>KNOX</role>
<url>hostname1</url>
</service>

</topology>

url1:

curl -u username:password -ik 'https://knoxhost:8443/gateway/sample5/api/v1/version'

HTTP/1.1 200 OK
Set-Cookie: JSESSIONID=123;Path=/gateway/sample5;Secure;HttpOnly
Expires: Thu, 01 Jan 1970 00:00:00 GMT
Content-Length: 169
Content-Type: application/xml
Server: Jetty(8.1.14.v20131031)

<?xml version="1.0" encoding="UTF-8"?>
<ServerVersion>
<version>0.6.0.2.4.3.0-227</version>
<hash>12322</hash>
</ServerVersion>

url2:

curl -u username:password -ik

'https://knoxhost:8443/gateway/sample5/webhdfs/v1?op=GETHOMEDIRECTORY'

<html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=ISO-8859-1"/>
<title>Error 500 Server Error</title>
</head>
<body><h2>HTTP ERROR 500</h2>
<p>Problem accessing /gateway/sample5/webhdfs/v1. Reason:
<pre> Server Error</pre></p><hr /><i><small>Powered by Jetty://</small></i><br/>