使用Spark和Java8从数据帧中获取多个列的非重复值计数

import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;
import org.apache.spark.sql.SparkSession;
import java.util.Arrays;
import java.util.List;
import java.util.stream.Collectors;

SparkSession spark = SparkSession.builder()
        .appName("ColumnDistinctCount")
        .master("local")
        .getOrCreate();

Dataset<Row> dataframe = spark.read().format("csv")
        .option("header", "true")
        .load("path/to/your/data.csv");

List<String> columns = Arrays.asList("column1", "column2", "column3");
List<Long> distinctCounts = columns.stream()
        .map(column -> dataframe.select(column).distinct().count())
        .collect(Collectors.toList());

for (int i = 0; i < columns.size(); i++) {
    System.out.println("Distinct count of " + columns.get(i) + ": " + distinctCounts.get(i));
}