Spark 覆盖写Hive分区表,只覆盖部分对应分区

要求Spark版本2.3以上，亲测2.2无效

配置 config("spark.sql.sources.partitionOverwriteMode","dynamic") 注意 1、saveAsTable方法无效，会全表覆盖写，需要用insertInto，详情见代码 2、insertInto需要主要DataFrame列的顺序要和Hive表里的顺序一致，不然会数据错误！

package com.dkl.blog.spark.hive

import org.apache.spark.sql.SparkSession

/**

• Created by dongkelun on 2020/1/16 15:25
• 博客：Spark 覆盖写Hive分区表,只覆盖部分对应分区
• 要求Spark版本2.3以上 */ object SparkHivePartitionOverwrite { def main(args: Array[String]): Unit = { val spark = SparkSession .builder() .appName("SparkHivePartitionOverwrite") .master("local") .config("spark.sql.parquet.writeLegacyFormat", true) .config("spark.sql.sources.partitionOverwriteMode","dynamic") .enableHiveSupport() .getOrCreate() import spark.sql val data = Array(("001", "张三", 21, "2018"), ("002", "李四", 18, "2017")) val df = spark.createDataFrame(data).toDF("id", "name", "age", "year") //创建临时表 df.createOrReplaceTempView("temp_table") val tableName="test_partition" //切换hive的数据库 sql("use test") // 1、创建分区表，并写入数据 df.write.mode("overwrite").partitionBy("year").saveAsTable(tableName) spark.table(tableName).show() val data1 = Array(("011", "Sam", 21, "2018")) val df1 = spark.createDataFrame(data1).toDF("id", "name", "age", "year") // df1.write.mode("overwrite").partitionBy("year").saveAsTable(tableName) //不成功，全表覆盖 // df1.write.mode("overwrite").format("Hive").partitionBy("year").saveAsTable(tableName) //不成功，全表覆盖 df1.write.mode("overwrite").insertInto(tableName) spark.table(tableName).show() spark.stop }

} 结果 +---+----+---+----+ | id|name|age|year| +---+----+---+----+ |002| 李四| 18|2017| |001| 张三| 21|2018| +---+----+---+----+

+---+----+---+----+ | id|name|age|year| +---+----+---+----+ |011| Sam| 21|2018| +---+----+---+----+