Member since 
    
	
		
		
		04-28-2017
	
	
	
	
	
	
	
	
	
	
	
	
	
	
			
      
                3
            
            
                Posts
            
        
                0
            
            
                Kudos Received
            
        
                0
            
            
                Solutions
            
        
			
    
	
		
		
		05-04-2017
	
		
		03:23 AM
	
	
	
	
	
	
	
	
	
	
	
	
	
	
		
	
				
		
			
					
				
		
	
		
					
							 @slachtermanWe have created an instance profile for the node, and not added credentials in core-site.xml. hadoop fs -ls s3a:// works and even selecting few rows from the external table (whose data is in s3) works, but I try to do aggregation function like :  select max(updated_at) from s3_table;  This query fails with the below mentioned error. Could you please help.  Caused by: java.lang.RuntimeException: java.io.IOException: java.io.IOException: Cannot find password option fs.s3a.access.key
  at org.apache.hadoop.mapred.split.TezGroupedSplitsInputFormat$TezGroupedSplitsRecordReader.initNextRecordReader(TezGroupedSplitsInputFormat.java:206)
  at org.apache.hadoop.mapred.split.TezGroupedSplitsInputFormat$TezGroupedSplitsRecordReader.(TezGroupedSplitsInputFormat.java:145)
  at org.apache.hadoop.mapred.split.TezGroupedSplitsInputFormat.getRecordReader(TezGroupedSplitsInputFormat.java:111)
  at org.apache.tez.mapreduce.lib.MRReaderMapred.setupOldRecordReader(MRReaderMapred.java:157)
  at org.apache.tez.mapreduce.lib.MRReaderMapred.setSplit(MRReaderMapred.java:83)
  at org.apache.tez.mapreduce.input.MRInput.initFromEventInternal(MRInput.java:694)
  at org.apache.tez.mapreduce.input.MRInput.initFromEvent(MRInput.java:653)
  at org.apache.tez.mapreduce.input.MRInputLegacy.checkAndAwaitRecordReaderInitialization(MRInputLegacy.java:145)
  at org.apache.tez.mapreduce.input.MRInputLegacy.init(MRInputLegacy.java:109)
  at org.apache.hadoop.hive.ql.exec.tez.MapRecordProcessor.getMRInput(MapRecordProcessor.java:525)
  at org.apache.hadoop.hive.ql.exec.tez.MapRecordProcessor.init(MapRecordProcessor.java:171)
  at org.apache.hadoop.hive.ql.exec.tez.TezProcessor.initializeAndRunProcessor(TezProcessor.java:184)
  ... 15 more
Caused by: java.io.IOException: java.io.IOException: Cannot find password option fs.s3a.access.key
  at org.apache.hadoop.hive.io.HiveIOExceptionHandlerChain.handleRecordReaderCreationException(HiveIOExceptionHandlerChain.java:97)
  at org.apache.hadoop.hive.io.HiveIOExceptionHandlerUtil.handleRecordReaderCreationException(HiveIOExceptionHandlerUtil.java:57)
  at org.apache.hadoop.hive.ql.io.HiveInputFormat.getRecordReader(HiveInputFormat.java:382)
  at org.apache.hadoop.mapred.split.TezGroupedSplitsInputFormat$TezGroupedSplitsRecordReader.initNextRecordReader(TezGroupedSplitsInputFormat.java:203)
  ... 26 more
Caused by: java.io.IOException: Cannot find password option fs.s3a.access.key
  at org.apache.hadoop.fs.s3a.S3AUtils.lookupPassword(S3AUtils.java:489)
  at org.apache.hadoop.fs.s3a.S3AUtils.getPassword(S3AUtils.java:468)
  at org.apache.hadoop.fs.s3a.S3AUtils.getAWSAccessKeys(S3AUtils.java:451)
  at org.apache.hadoop.fs.s3a.S3AUtils.createAWSCredentialProviderSet(S3AUtils.java:341)
  at org.apache.hadoop.fs.s3a.S3ClientFactory$DefaultS3ClientFactory.createS3Client(S3ClientFactory.java:73)
  at org.apache.hadoop.fs.s3a.S3AFileSystem.initialize(S3AFileSystem.java:185)
  at org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:2795)
  at org.apache.hadoop.fs.FileSystem.access$200(FileSystem.java:99)
  at org.apache.hadoop.fs.FileSystem$Cache.getInternal(FileSystem.java:2829)
  at org.apache.hadoop.fs.FileSystem$Cache.get(FileSystem.java:2811)
  at org.apache.hadoop.fs.FileSystem.get(FileSystem.java:390)
  at org.apache.hadoop.fs.Path.getFileSystem(Path.java:295)
  at org.apache.parquet.hadoop.ParquetFileReader.readFooter(ParquetFileReader.java:385)
  at org.apache.parquet.hadoop.ParquetFileReader.readFooter(ParquetFileReader.java:372)
  at org.apache.hadoop.hive.ql.io.parquet.read.ParquetRecordReaderWrapper.getSplit(ParquetRecordReaderWrapper.java:244)
  at org.apache.hadoop.hive.ql.io.parquet.read.ParquetRecordReaderWrapper.(ParquetRecordReaderWrapper.java:94)
  at org.apache.hadoop.hive.ql.io.parquet.read.ParquetRecordReaderWrapper.(ParquetRecordReaderWrapper.java:80)
  at org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat.getRecordReader(MapredParquetInputFormat.java:72)
  at org.apache.hadoop.hive.ql.io.HiveInputFormat.getRecordReader(HiveInputFormat.java:380)
  ... 27 more
Caused by: java.io.IOException: Configuration problem with provider path.
  at org.apache.hadoop.conf.Configuration.getPasswordFromCredentialProviders(Configuration.java:1999)
  at org.apache.hadoop.conf.Configuration.getPassword(Configuration.java:1959)
  at org.apache.hadoop.fs.s3a.S3AUtils.lookupPassword(S3AUtils.java:484)
  ... 45 more
Caused by: java.io.IOException: No CredentialProviderFactory for jceks://file/usr/hdp/current/hive-server2-hive2/conf/conf.server/hive-site.jceks in hadoop.security.credential.provider.path
  at org.apache.hadoop.security.alias.CredentialProviderFactory.getProviders(CredentialProviderFactory.java:66)
  at org.apache.hadoop.conf.Configuration.getPasswordFromCredentialProviders(Configuration.java:1979) 
						
					
					... View more