Apache Spark (Java)中列的自定义处理

import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;
import org.apache.spark.sql.SparkSession;
import org.apache.spark.sql.functions;

public class CustomColumnProcessing {
    public static void main(String[] args) {
        // 创建SparkSession
        SparkSession spark = SparkSession.builder()
                .appName("CustomColumnProcessing")
                .master("local")
                .getOrCreate();

        // 读取数据集
        Dataset<Row> dataset = spark.read().format("csv")
                .option("header", "true")
                .load("path/to/input.csv");

        // 对某一列进行自定义处理
        Dataset<Row> processedDataset = dataset.withColumn("customColumn", functions.expr("length(columnName)"));

        // 显示处理后的数据集
        processedDataset.show();

        // 关闭SparkSession
        spark.close();
    }
}