在Scala中使用SparkML训练/测试数据

import org.apache.spark.ml.Pipeline
import org.apache.spark.ml.classification.LogisticRegression
import org.apache.spark.ml.evaluation.BinaryClassificationEvaluator
import org.apache.spark.ml.feature.{HashingTF, Tokenizer}
import org.apache.spark.sql.{SparkSession, DataFrame}

object SparkMLExample {
  def main(args: Array[String]): Unit = {
    val spark = SparkSession.builder()
      .appName("Spark ML Example")
      .master("local[*]")
      .getOrCreate()

    // 加载数据集
    val data = spark.read.format("libsvm").load("data/sample_libsvm_data.txt")

    // 数据预处理
    val tokenizer = new Tokenizer().setInputCol("text").setOutputCol("words")
    val hashingTF = new HashingTF().setInputCol(tokenizer.getOutputCol).setOutputCol("features")

    // 模型训练
    val lr = new LogisticRegression().setMaxIter(10).setRegParam(0.001)

    // 构建管道
    val pipeline = new Pipeline().setStages(Array(tokenizer, hashingTF, lr))

    // 划分训练集和测试集
    val Array(trainingData, testData) = data.randomSplit(Array(0.8, 0.2))

    // 训练模型
    val model = pipeline.fit(trainingData)

    // 进行预测
    val predictions = model.transform(testData)

    // 评估模型
    val evaluator = new BinaryClassificationEvaluator()
    val accuracy = evaluator.evaluate(predictions)
    println(s"Test Accuracy = $accuracy")

    spark.stop()
  }
}