spark.sql("set pyspark.hadoop.hive.exec.dynamic.partition=true") spark.sql("set pyspark.hadoop.hive.exec.dynamic.partition.mode=nonstrict") spark.sql("set hive.exec.dynamic.partition=true") spark.sql("set hive.exec.dynamic.partition.mode=nonstrict") spark.sql("set hive.merge.tezfiles=true") spark.sql("SET hive.merge.sparkfiles = true") spark.sql("set hive.merge.smallfiles.avgsize=128000000") spark.sql("set hive.merge.size.per.task=128000000") ... View more