A solution to import your data as parquet file and be able to treat the TIMESTAMP and DATE format which come from RDBMS such as IBM DB2 or MySQL is to import using the sqoop import --as-parquet command and map each field using --map-column-java which are TIMESTAMP and DATE to a String Java type. After that, you should be able to interrogate the Hive database though a SparkSession by changing the configuration of the actual Spark Session and set spark.sql.hive.convertMetastoreParquet to false. SparkSQL will use the Hive SerDe for reading parquet tables instead of the built in support. spark.sql.hive.convertMetastoreParquet false import org.apache.spark.sql.SparkSession
val sparkSession = SparkSession.builder()
.appName("test interrogate Hive parquet file using Spark")
.config("spark.sql.parquet.compression.codec", "snappy")
.config("spark.sql.warehouse.dir","/apps/hive/warehouse")
.config("hive.metastore.uris","thrift://sdsl-hdp-01.mycluster:9083")
.config("spark.sql.hive.convertMetastoreParquet", false)
.enableHiveSupport()
.getOrCreate()
import spark.implicits._
import spark.sql
val df = sql("SELECT CAST(COL1 AS TIMESTAMP), COL2, COL3, CAST(COL4 AS TIMESTAMP), COL5 FROM db.mytable")
df.printSchema
root
|-- COL1: timestamp (nullable = true)
|-- COL2: string (nullable = true)
|-- COL3: string (nullable = true)
|-- COL4: timestamp (nullable = true)
|-- COL5: integer (nullable = true)
df.show(5, false)
+--------------------------+--------+--------+--------------------------+------+
|COL1 |COL2 |COL3 |COL4 |COL5|
+--------------------------+--------+--------+--------------------------+------+
|2003-01-01 00:00:00.100001| |00001 |2003-01-01 00:00:00.10361 |1 |
|2003-01-01 00:00:00.100002| |00002 |2003-01-01 00:00:00.100002|2 |
|2003-01-01 00:00:00.100003| |00003 |2003-01-01 00:00:00.100003|3 |
|2003-01-01 00:00:00.100004| |00004 |2003-01-01 00:00:00.100004|4 |
|2003-01-01 00:00:00.100005| |00005 |2003-01-01 00:00:00.100005|5 |
+--------------------------+--------+--------+--------------------------+------+
only showing top 5 row
... View more
