问HowTo在中扁平简单Json文件并转换为Parquet

EN

using System;
using System.Collections.Generic;
using Microsoft.Spark.Sql;
using Microsoft.Spark.Sql.Types;
using System.Diagnostics;


var df = spark.Read().Json("{Your source file path here}");

//get the schema of the data frame
var dfSchema = df.Schema() ;
// traverse the schema of the dataframe
foreach(var parentSchemaField in dfSchema.Fields) {
    
    if (parentSchemaField.DataType is StructType) {
        // get a new dataframe that just contains the child data from the parent
        var childFrame = df.Select($"{parentSchemaField.Name}.*") ;
        // traverse the schema of the child dataframe
        foreach(var childSchemaField in childFrame.Schema().Fields) {   
            //make a new column in the parent dataframe for each parents child property
            df = df.WithColumn($"{parentSchemaField.Name}.{childSchemaField.Name}",Col($"{parentSchemaField.Name}.{childSchemaField.Name}")) ;
        }
        // drop the parent column from the data frame its no longer needed
        df = df.Drop(parentSchemaField.Name) ;
    }
}
df.Write().Parquet("{Your sink file path here}") ;