从Scala Spark到PySpark的熵计算转换

import org.apache.spark.ml.feature.{VectorAssembler, StringIndexer}
import org.apache.spark.ml.linalg.Vectors
import org.apache.spark.sql.SparkSession
import org.apache.spark.ml.stat.Entropy

val spark = SparkSession.builder()
  .appName("Entropy Calculation")
  .getOrCreate()

val data = spark.read.format("csv")
  .option("header", "true")
  .option("inferSchema", "true")
  .load("path/to/dataset.csv")

val assembler = new VectorAssembler()
  .setInputCols(Array("feature1", "feature2", ...))
  .setOutputCol("features")

val assembledData = assembler.transform(data)

val indexer = new StringIndexer()
  .setInputCol("label")
  .setOutputCol("indexedLabel")

val indexedData = indexer.fit(assembledData).transform(assembledData)

val entropy = Entropy.calculateEntropy(indexedData, "indexedLabel")

from pyspark.ml.feature import VectorAssembler, StringIndexer
from pyspark.ml.linalg import Vectors
from pyspark.sql import SparkSession
from pyspark.ml.stat import Entropy

spark = SparkSession.builder \
    .appName("Entropy Calculation") \
    .getOrCreate()

data = spark.read.format("csv") \
    .option("header", "true") \
    .option("inferSchema", "true") \
    .load("path/to/dataset.csv")

assembler = VectorAssembler() \
    .setInputCols(["feature1", "feature2", ...]) \
    .setOutputCol("features")

assembledData = assembler.transform(data)

indexer = StringIndexer() \
    .setInputCol("label") \
    .setOutputCol("indexedLabel")

indexedData = indexer.fit(assembledData).transform(assembledData)

entropy = Entropy.calculateEntropy(indexedData, "indexedLabel")