Support Questions
Find answers, ask questions, and share your expertise
Announcements
Alert: Welcome to the Unified Cloudera Community. Former HCC members be sure to read and learn how to activate your account here.

Hive select from spark-shell

Highlighted

Hive select from spark-shell

Explorer

Hi guys I tried to run a select on a table hive from spark-shell this my code : 

 

import org.apache.spark.sql.hive.HiveContext                                                                                                                     
val sqlContext = new HiveContext(sc)   
val df = sqlContext.sql("select count(*) from timeserie")
df.head  

 

this is error :

 

org.apache.spark.sql.catalyst.errors.package$TreeNodeException: execute, tree:                                                                                          
Exchange SinglePartition                                                                                                                                                
+- *(1) HashAggregate(keys=[], functions=[partial_count(1)], output=[count#13L])                                                                                        
   +- HiveTableScan HiveTableRelation `default`.`timeserie`, org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe, [data#0, temperature#1, hum#2]                         
                                                                                                                                                                        
  at org.apache.spark.sql.catalyst.errors.package$.attachTree(package.scala:56)                                                                                         
  at org.apache.spark.sql.execution.exchange.ShuffleExchangeExec.doExecute(ShuffleExchangeExec.scala:119)                                                               
  at org.apache.spark.sql.execution.SparkPlan$$anonfun$execute$1.apply(SparkPlan.scala:131)                                                                             
  at org.apache.spark.sql.execution.SparkPlan$$anonfun$execute$1.apply(SparkPlan.scala:127)                                                                             
  at org.apache.spark.sql.execution.SparkPlan$$anonfun$executeQuery$1.apply(SparkPlan.scala:155)                                                                        
  at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)                                                                                     
  at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:152)                                                                                         
  at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:127)                                                                                              
  at org.apache.spark.sql.execution.InputAdapter.inputRDDs(WholeStageCodegenExec.scala:371)                                                                             
  at org.apache.spark.sql.execution.aggregate.HashAggregateExec.inputRDDs(HashAggregateExec.scala:150)                                                                  
  at org.apache.spark.sql.execution.WholeStageCodegenExec.doExecute(WholeStageCodegenExec.scala:605)                                                                    
  at org.apache.spark.sql.execution.SparkPlan$$anonfun$execute$1.apply(SparkPlan.scala:131)                                                                             
  at org.apache.spark.sql.execution.SparkPlan$$anonfun$execute$1.apply(SparkPlan.scala:127)                                                                             
  at org.apache.spark.sql.execution.SparkPlan$$anonfun$executeQuery$1.apply(SparkPlan.scala:155)                                                                        
  at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)                                                                                     
  at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:152)                                                                                         
  at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:127)                                                                                              
  at org.apache.spark.sql.execution.SparkPlan.getByteArrayRdd(SparkPlan.scala:247)                                                                                      
  at org.apache.spark.sql.execution.SparkPlan.executeCollect(SparkPlan.scala:294)                                                                                       
  at org.apache.spark.sql.Dataset$$anonfun$count$1.apply(Dataset.scala:2775)                                                                                            
  at org.apache.spark.sql.Dataset$$anonfun$count$1.apply(Dataset.scala:2774)                                                                                            
  at org.apache.spark.sql.Dataset$$anonfun$52.apply(Dataset.scala:3259)                                                                                                 
  at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:77)                                                                             
  at org.apache.spark.sql.Dataset.withAction(Dataset.scala:3258)                                                                                                        
  at org.apache.spark.sql.Dataset.count(Dataset.scala:2774)                                                                                                             
  ... 49 elided                                                                                                                                                         
Caused by: java.io.IOException: Not a file: hdfs://sandbox-hdp.hortonworks.com:8020/warehouse/tablespace/managed/hive/timeserie/delta_0000001_0000001_0000              
  at org.apache.hadoop.mapred.FileInputFormat.getSplits(FileInputFormat.java:337)                                                                                       
  at org.apache.spark.rdd.HadoopRDD.getPartitions(HadoopRDD.scala:200)                                                                                                  
  at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:253)                                                                                                
  at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:251)                                                                                                
  at scala.Option.getOrElse(Option.scala:121)                                                                                                                           
  at org.apache.spark.rdd.RDD.partitions(RDD.scala:251)                                                                                                                 
  at org.apache.spark.rdd.MapPartitionsRDD.getPartitions(MapPartitionsRDD.scala:35)                                                                                     
  at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:253)                                                                                                
  at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:251)                                                                                                
  at scala.Option.getOrElse(Option.scala:121)                                                                                                                           
  at org.apache.spark.rdd.RDD.partitions(RDD.scala:251)                                                                                                                 
  at org.apache.spark.rdd.MapPartitionsRDD.getPartitions(MapPartitionsRDD.scala:35)                                                                                     
  at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:253)                                                                                                
  at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:251)                                                                                                
  at scala.Option.getOrElse(Option.scala:121)                                                                                                                           
  at org.apache.spark.rdd.RDD.partitions(RDD.scala:251)                                                                                                                 
  at org.apache.spark.rdd.MapPartitionsRDD.getPartitions(MapPartitionsRDD.scala:35)                                                                                     
  at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:253)                                                                                                
  at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:251)                                                                                                
  at scala.Option.getOrElse(Option.scala:121)                                                                                                                           
  at org.apache.spark.rdd.RDD.partitions(RDD.scala:251)                                                                                                                 
  at org.apache.spark.rdd.MapPartitionsRDD.getPartitions(MapPartitionsRDD.scala:35)                                                                                     
  at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:253)                                                                                                
  at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:251)                                                                                                
  at scala.Option.getOrElse(Option.scala:121)                                                                                                                           
  at org.apache.spark.rdd.RDD.partitions(RDD.scala:251)                                                                                                                 
  at org.apache.spark.rdd.MapPartitionsRDD.getPartitions(MapPartitionsRDD.scala:35)                                                                                     
  at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:253)                                                                                                
  at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:251)                                                                                                
  at scala.Option.getOrElse(Option.scala:121)                                                                                                                           
  at org.apache.spark.rdd.RDD.partitions(RDD.scala:251)                                                                                                                 
  at org.apache.spark.ShuffleDependency.<init>(Dependency.scala:91)                                                                                                     
  at org.apache.spark.sql.execution.exchange.ShuffleExchangeExec$.prepareShuffleDependency(ShuffleExchangeExec.scala:318)                                               
  at org.apache.spark.sql.execution.exchange.ShuffleExchangeExec.prepareShuffleDependency(ShuffleExchangeExec.scala:91)                                                 
  at org.apache.spark.sql.execution.exchange.ShuffleExchangeExec$$anonfun$doExecute$1.apply(ShuffleExchangeExec.scala:128)                                              
  at org.apache.spark.sql.execution.exchange.ShuffleExchangeExec$$anonfun$doExecute$1.apply(ShuffleExchangeExec.scala:119)                                              
  at org.apache.spark.sql.catalyst.errors.package$.attachTree(package.scala:52)                                                                                         
  ... 73 more  

 

Is there anyone who can help me...

Don't have an account?
Coming from Hortonworks? Activate your account here