Member since 
    
	
		
		
		12-30-2015
	
	
	
	
	
	
	
	
	
	
	
	
	
	
			
      
                164
            
            
                Posts
            
        
                29
            
            
                Kudos Received
            
        
                10
            
            
                Solutions
            
        My Accepted Solutions
| Title | Views | Posted | 
|---|---|---|
| 28725 | 01-07-2019 06:17 AM | |
| 1475 | 12-27-2018 07:28 AM | |
| 4434 | 11-26-2018 10:12 AM | |
| 1926 | 11-16-2018 12:15 PM | |
| 4162 | 10-22-2018 09:31 AM | 
			
    
	
		
		
		10-09-2019
	
		
		09:28 AM
	
	
	
	
	
	
	
	
	
	
	
	
	
	
		
	
				
		
			
					
				
		
	
		
					
							 PFA the below error logs :  19/10/09 16:09:32 DEBUG ServletHandler: chain=org.apache.hadoop.security.authentication.server.AuthenticationFilter-418c020b->org.apache.spark.ui.JettyUtils$$anon$3-75e710b@986efce7==org.apache.spark.ui.JettyUtils$$anon$3,jsp=null,order=-1,inst=true  19/10/09 16:09:32 DEBUG ServletHandler: call filter org.apache.hadoop.security.authentication.server.AuthenticationFilter-418c020b  19/10/09 16:09:32 DEBUG AuthenticationFilter: Got token null from httpRequest http://ip-10-0-10.184. ************:18081/  19/10/09 16:09:32 DEBUG AuthenticationFilter: Request [http://ip-10-0-10-184.*****:18081/] triggering authentication. handler: class org.apache.hadoop.security.authentication.server.KerberosAuthenticationHandler  19/10/09 16:09:32 DEBUG AuthenticationFilter: Authentication exception: java.lang.IllegalArgumentException  org.apache.hadoop.security.authentication.client.AuthenticationException: java.lang.IllegalArgumentException  at org.apache.hadoop.security.authentication.server.KerberosAuthenticationHandler.authenticate(KerberosAuthenticationHandler.java:306)  at org.apache.hadoop.security.authentication.server.AuthenticationFilter.doFilter(AuthenticationFilter.java:536)  at org.spark_project.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1759)  at org.spark_project.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:582)  at org.spark_project.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1180)  at org.spark_project.jetty.servlet.ServletHandler.doScope(ServletHandler.java:512)  at org.spark_project.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:1112)  at org.spark_project.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:141)  at org.spark_project.jetty.server.handler.gzip.GzipHandler.handle(GzipHandler.java:493)  at org.spark_project.jetty.server.handler.ContextHandlerCollection.handle(ContextHandlerCollection.java:213)  at org.spark_project.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:134)  at org.spark_project.jetty.server.Server.handle(Server.java:539)  at org.spark_project.jetty.server.HttpChannel.handle(HttpChannel.java:333)  at org.spark_project.jetty.server.HttpConnection.onFillable(HttpConnection.java:251)  at org.spark_project.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:283)  at org.spark_project.jetty.io.FillInterest.fillable(FillInterest.java:108)  at org.spark_project.jetty.io.SelectChannelEndPoint$2.run(SelectChannelEndPoint.java:93)  at org.spark_project.jetty.util.thread.strategy.ExecuteProduceConsume.executeProduceConsume(ExecuteProduceConsume.java:303)  at org.spark_project.jetty.util.thread.strategy.ExecuteProduceConsume.produceConsume(ExecuteProduceConsume.java:148)  at org.spark_project.jetty.util.thread.strategy.ExecuteProduceConsume.run(ExecuteProduceConsume.java:136)  at org.spark_project.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:671)  at org.spark_project.jetty.util.thread.QueuedThreadPool$2.run(QueuedThreadPool.java:589)  at java.lang.Thread.run(Thread.java:748)  Caused by: java.lang.IllegalArgumentException  at java.nio.Buffer.limit(Buffer.java:275)  at org.apache.hadoop.security.authentication.util.KerberosUtil$DER.<init>(KerberosUtil.java:365)  at org.apache.hadoop.security.authentication.util.KerberosUtil$DER.<init>(KerberosUtil.java:358)  at org.apache.hadoop.security.authentication.util.KerberosUtil.getTokenServerName(KerberosUtil.java:291)  at org.apache.hadoop.security.authentication.server.KerberosAuthenticationHandler.authenticate(KerberosAuthenticationHandler.java:285)  ... 22 more  19/10/09 16:09:32 DEBUG GzipHttpOutputInterceptor: org.spark_project.jetty.server.handler.gzip.GzipHttpOutputInterceptor@17d4d832 exclude by status 403  19/10/09 16:09:32 DEBUG HttpChannel: sendResponse info=null content=HeapByteBuffer@26ea8849[p=0,l=365,c=32768,r=365]={<<<<html>\n<head>\n<me.../body>\n</html>\n>>>\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00...\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00} complete=true committing=true callback=Blocker@137652aa{null}  19/10/09 16:09:32 DEBUG HttpChannel: COMMIT for / on HttpChannelOverHttp@4d71d816{r=2,c=true,a=DISPATCHED,uri=//ip-10-0-10-184.******:18081/}  403 java.lang.IllegalArgumentException HTTP/1.1  Date: Wed, 09 Oct 2019 16:09:32 GMT  Set-Cookie: hadoop.auth=; HttpOnly  Cache-Control: must-revalidate,no-cache,no-store  Content-Type: text/html;charset=iso-8859-1    
						
					
					... View more
				
			
			
			
			
			
			
			
			
			
		
			
    
	
		
		
		10-08-2019
	
		
		11:07 PM
	
	
	
	
	
	
	
	
	
	
	
	
	
	
		
	
				
		
			
					
				
		
	
		
					
							 Hi @Shelton ,     have tried with the below code but still facing the same issue ?     {% if security_enabled %}  export SPARK_HISTORY_OPTS='-Dspark.ui.filters=org.apache.hadoop.security.authentication.server.AuthenticationFilter -Dspark.org.apache.hadoop.security.authentication.server.AuthenticationFilter.params="type=kerberos,kerberos.principal={{spnego_principal}},kerberos.keytab={{spnego_keytab}}"'    {% endif %}        do we need enable spengo authenitcation in browser ?       
						
					
					... View more
				
			
			
			
			
			
			
			
			
			
		
			
    
	
		
		
		07-08-2019
	
		
		09:49 AM
	
	
	
	
	
	
	
	
	
	
	
	
	
	
		
	
				
		
			
					
				
		
	
		
					
							 Dear community members,  we were unable to connect spark-history server ui even we have enabled spengo authentication in Advanced spark-env config as following:      export SPARK_HISTORY_OPTS='-Dspark.ui.filters=org.apache.hadoop.security.authentication.server.AuthenticationFilter -Dspark.org.apache.hadoop.security.authentication.server.AuthenticationFilter.params="type=kerberos,kerberos.principal=HTTP/ip-0-0-0-0@HDPCLUSTER.LOCAL,kerberos.keytab=/etc/security/keytabs/spnego.service.keytab"'  Any help much appreciated to resolve the above issue ?  Thanks in advance         
						
					
					... View more
				
			
			
			
			
			
			
			
			
			
		
		
			
				
						
							Labels:
						
						
		
			
	
					
			
		
	
	
	
	
				
		
	
	
- Labels:
- 
						
							
		
			Apache Spark
			
    
	
		
		
		05-07-2019
	
		
		11:38 AM
	
	
	
	
	
	
	
	
	
	
	
	
	
	
		
	
				
		
			
					
				
		
	
		
					
							 please check below article:  https://antnix07.blogspot.com/2017/05/normal-0-false-false-false-en-us-x-none.html  
						
					
					... View more
				
			
			
			
			
			
			
			
			
			
		
			
    
	
		
		
		02-19-2019
	
		
		07:36 AM
	
	
	
	
	
	
	
	
	
	
	
	
	
	
		
	
				
		
			
					
				
		
	
		
					
							 Hi @Dan Hops,  It seems you are working with single node cluster and configured LLAP in top of hive with yarn 100% utilization  Please configure the LLAP to use 25% of your Yarn capacity  
						
					
					... View more
				
			
			
			
			
			
			
			
			
			
		
			
    
	
		
		
		01-11-2019
	
		
		10:35 AM
	
	
	
	
	
	
	
	
	
	
	
	
	
	
		
	
				
		
			
					
				
		
	
		
					
							 Try the following, lets assume your hive.tez.container.size=2048.  
set hive.tez.java.opts=-Xmx1640m (0.8 times hive.tez.container.size)   set tez.runtime.io.sort.mb=820 (0.4 times hive.tez.container.size)   set tez.runtime.unordered.output.buffer.size-mb=205 (0.1 times hive.tez.container.size) 
						
					
					... View more
				
			
			
			
			
			
			
			
			
			
		
			
    
	
		
		
		01-07-2019
	
		
		07:29 AM
	
	
	
	
	
	
	
	
	
	
	
	
	
	
		
	
				
		
			
					
				
		
	
		
					
							 Hi @Rajesh Sampath,  please follow the below article :  https://github.com/hortonworks-spark/spark-llap/tree/master 
						
					
					... View more
				
			
			
			
			
			
			
			
			
			
		
			
    
	
		
		
		01-07-2019
	
		
		06:17 AM
	
	
	
	
	
	
	
	
	
	
	
	
	
	
		
	
				
		
			
					
	
		1 Kudo
		
	
				
		
	
		
					
							 Hi Vinay,  use the below code to connect hive and list the databases :  spark-shell --conf spark.sql.hive.hiveserver2.jdbc.url="jdbc:hive2://hiveserverip:10000/" spark.datasource.hive.warehouse.load.staging.dir="/tmp" spark.hadoop.hive.zookeeper.quorum="zookeeperquoremip:2181" --jars /usr/hdp/current/hive_warehouse_connector/hive-warehouse-connector-assembly-1.0.0.3.0.0.0-1634.jar  val hive = com.hortonworks.spark.sql.hive.llap.HiveWarehouseBuilder.session(spark).build()  hive.showDatabases().show(100, false)  Reference article  https://github.com/hortonworks-spark/spark-llap/tree/master 
						
					
					... View more
				
			
			
			
			
			
			
			
			
			
		
			
    
	
		
		
		01-07-2019
	
		
		06:16 AM
	
	
	
	
	
	
	
	
	
	
	
	
	
	
		
	
				
		
			
					
				
		
	
		
					
							 Hi Vinay,  use the below code to connect hive and list the databases :  spark-shell --conf spark.sql.hive.hiveserver2.jdbc.url="jdbc:hive2://hiveserverip:10000/" spark.datasource.hive.warehouse.load.staging.dir="/tmp" spark.hadoop.hive.zookeeper.quorum="zookeeperquoremip:2181" --jars /usr/hdp/current/hive_warehouse_connector/hive-warehouse-connector-assembly-1.0.0.3.0.0.0-1634.jar  val hive = com.hortonworks.spark.sql.hive.llap.HiveWarehouseBuilder.session(spark).build()  hive.showDatabases().show(100, false)  Reference article   https://github.com/hortonworks-spark/spark-llap/tree/master 
						
					
					... View more
				
			
			
			
			
			
			
			
			
			
		
			
    
	
		
		
		01-02-2019
	
		
		12:21 PM
	
	
	
	
	
	
	
	
	
	
	
	
	
	
		
	
				
		
			
					
				
		
	
		
					
							 Hi VInay,  can you post spark code which is using to display the databases ? 
						
					
					... View more
				
			
			
			
			
			
			
			
			
			
		 
         
					
				













