模型评估与选择：交叉验证、偏差方差、过拟合与调参实战

from sklearn.model_selection import train_test_split

X_temp, X_test, y_temp, y_test = train_test_split(
    X, y, test_size=0.2, random_state=42)
X_train, X_val, y_train, y_val = train_test_split(
    X_temp, y_temp, test_size=0.25, random_state=42)   # 0.25*0.8=20%

print(X_train.shape, X_val.shape, X_test.shape)

from sklearn.model_selection import cross_val_score
from sklearn.ensemble import RandomForestClassifier

rf = RandomForestClassifier(n_estimators=100, random_state=42)
scores = cross_val_score(rf, X, y, cv=5, scoring='roc_auc')

print("每折 AUC:", scores.round(3))
print("平均 AUC:", scores.mean().round(3))
print("标准差:", scores.std().round(3))

from sklearn.model_selection import StratifiedKFold

cv = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)
scores = cross_val_score(rf, X, y, cv=cv, scoring='roc_auc')

import numpy as np
from sklearn.preprocessing import PolynomialFeatures
from sklearn.linear_model import LinearRegression
from sklearn.pipeline import make_pipeline

np.random.seed(42)
x = np.linspace(-3, 3, 100)
y_true = 0.5*x**2 + x
y = y_true + np.random.normal(0, 2, 100)

for degree in [1, 2, 15]:
    pipe = make_pipeline(PolynomialFeatures(degree), LinearRegression())
    pipe.fit(x.reshape(-1,1), y)
    # degree=1 欠拟合，degree=2 合适，degree=15 过拟合

from sklearn.model_selection import learning_curve

train_sizes, train_scores, val_scores = learning_curve(
    rf, X, y, cv=5, train_sizes=[0.2, 0.4, 0.6, 0.8, 1.0],
    scoring='roc_auc')

train_mean = train_scores.mean(axis=1)
val_mean = val_scores.mean(axis=1)

import matplotlib.pyplot as plt
plt.plot(train_sizes, train_mean, 'o-', label='训练分')
plt.plot(train_sizes, val_mean, 'o-', label='验证分')
plt.xlabel('训练样本量')
plt.ylabel('AUC')
plt.legend()
plt.show()

# 决策树限制深度 = 减复杂度
from sklearn.tree import DecisionTreeClassifier

dt_deep = DecisionTreeClassifier(max_depth=None)   # 易过拟合
dt_lim  = DecisionTreeClassifier(max_depth=5)      # 控制复杂度

# 集成降低方差
from sklearn.ensemble import RandomForestClassifier
rf = RandomForestClassifier(n_estimators=200, max_depth=8, random_state=42)

from sklearn.model_selection import GridSearchCV

param_grid = {
    'n_estimators': [50, 100, 200],
    'max_depth': [5, 10, None],
    'min_samples_split': [2, 5, 10],
}

search = GridSearchCV(
    RandomForestClassifier(random_state=42),
    param_grid, cv=5, scoring='roc_auc', n_jobs=-1)
search.fit(X, y)

print("最优参数:", search.best_params_)
print("最优交叉验证分:", search.best_score_.round(3))

from sklearn.model_selection import RandomizedSearchCV
from scipy.stats import randint, uniform

param_dist = {
    'n_estimators': randint(50, 300),
    'max_depth': randint(3, 15),
    'min_samples_split': randint(2, 10),
}

search = RandomizedSearchCV(
    RandomForestClassifier(random_state=42),
    param_dist, n_iter=50, cv=5, scoring='roc_auc', random_state=42, n_jobs=-1)
search.fit(X, y)
print("最优:", search.best_params_)

models = {
    'LogisticRegression': LogisticRegression(max_iter=1000),
    'DecisionTree': DecisionTreeClassifier(max_depth=8, random_state=42),
    'RandomForest': RandomForestClassifier(n_estimators=100, random_state=42),
    'GradientBoosting': GradientBoostingClassifier(random_state=42),
}

for name, model in models.items():
    scores = cross_val_score(model, X, y, cv=5, scoring='roc_auc')
    print(f"{name:20s} AUC={scores.mean():.3f} ±{scores.std():.3f}")

# 选定的模型只在最终测试集上评估一次
best = RandomForestClassifier(n_estimators=200, max_depth=10, random_state=42)
best.fit(X_train, y_train)
final_auc = roc_auc_score(y_test, best.predict_proba(X_test)[:, 1])
print("最终测试集 AUC:", round(final_auc, 3))