使用Pyspark进行交叉验证

from pyspark.ml import Pipeline
from pyspark.ml.evaluation import BinaryClassificationEvaluator
from pyspark.ml.tuning import ParamGridBuilder, CrossValidator

from pyspark.ml.classification import RandomForestClassifier

# 定义随机森林分类器
rf = RandomForestClassifier()

# 定义参数网格
paramGrid = ParamGridBuilder() \
    .addGrid(rf.maxDepth, [5, 10, 20]) \
    .addGrid(rf.numTrees, [10, 20, 30]) \
    .build()

# 定义二分类评估器
evaluator = BinaryClassificationEvaluator()

# 定义交叉验证器
crossval = CrossValidator(estimator=rf,
                          estimatorParamMaps=paramGrid,
                          evaluator=evaluator,
                          numFolds=3)

# 训练模型
cvModel = crossval.fit(train_data)

# 选择最佳模型
bestModel = cvModel.bestModel