特征工程实战：从原始数据到有效特征的完整流程

import pandas as pd

df = pd.read_csv('sales.csv')
print(df.isnull().sum())          # 看缺失分布

# 方案矩阵：
# 少量缺失数值列 → 均值/中位数填充
df['age'] = df['age'].fillna(df['age'].median())

# 类别列 → 众数或"未知"占位
df['city'] = df['city'].fillna('未知')

# 关键字段缺失 → 删行
df = df.dropna(subset=['order_id'])

missing_ratio = df.isnull().mean()
drop_cols = missing_ratio[missing_ratio > 0.6].index
df = df.drop(columns=drop_cols)
print("弃列:", list(drop_cols))

Q1 = df['amount'].quantile(0.25)
Q3 = df['amount'].quantile(0.75)
IQR = Q3 - Q1
low  = Q1 - 1.5 * IQR
high = Q3 + 1.5 * IQR

outliers = df[(df['amount'] < low) | (df['amount'] > high)]
print("异常值数量:", len(outliers))

print("重复行:", df.duplicated().sum())
df = df.drop_duplicates()

# 组合特征（比值往往比原始值更有信息）
df['客单价']   = df['消费额'] / df['消费次数']
df['月均消费'] = df['消费额'] / (df['注册月数'] + 1)

# 分箱：连续值转区间
df['年龄段'] = pd.cut(df['age'], bins=[0, 18, 30, 45, 60, 100],
                     labels=['<18','18-30','30-45','45-60','60+'])

df['下单日期'] = pd.to_datetime(df['下单日期'])

df['年']   = df['下单日期'].dt.year
df['月']   = df['下单日期'].dt.month
df['星期'] = df['下单日期'].dt.dayofweek        # 0=周一
df['小时'] = df['下单日期'].dt.hour
df['是否周末'] = (df['星期'] >= 5).astype(int)

# 每个用户的统计特征
agg = df.groupby('user_id').agg(
    order_count = ('order_id', 'count'),
    total_amount = ('amount', 'sum'),
    avg_amount = ('amount', 'mean'),
).reset_index()

df_user = df.merge(agg, on='user_id')

# 显式交互（线性模型尤其需要）
df['消费额_x_频次'] = df['消费额'] * df['消费次数']

from sklearn.preprocessing import OneHotEncoder

df = pd.get_dummies(df, columns=['城市'], prefix='城市')
# 城市 → 城市_北京, 城市_上海, 城市_深圳 ...

# 简单的目标编码（训练集）
city_target = df.groupby('城市')['是否复购'].mean()

# ⚠️ 必须加平滑，防止小样本类别的均值过拟合
smooth = 10
global_mean = df['是否复购'].mean()
df['城市_编码'] = df['城市'].map(
    lambda c: (city_target[c]*df['城市'].eq(c).sum() + global_mean*smooth)
              / (df['城市'].eq(c).sum() + smooth)
)

# 有序类别直接映射
df['等级'] = df['等级'].map({'普通':0, '银卡':1, '金卡':2, '铂金':3})

# 方差极低（几乎不变）的特征无信息
from sklearn.feature_selection import VarianceThreshold

selector = VarianceThreshold(threshold=0.01)
X_reduced = selector.fit_transform(X)

# 与目标高度相关 → 保留；两特征间高度相关 → 去冗余
corr = pd.DataFrame(X).corrwith(y).abs().sort_values(ascending=False)

from sklearn.ensemble import RandomForestClassifier

rf = RandomForestClassifier(n_estimators=100, random_state=42)
rf.fit(X, y)

importance = pd.Series(rf.feature_importances_, index=X.columns)
keep = importance[importance > 0.02].index.sort_values()
print("保留特征:", list(keep))

from sklearn.feature_selection import RFE
from sklearn.linear_model import LogisticRegression

rfe = RFE(LogisticRegression(max_iter=1000), n_features_to_select=20)
rfe.fit(X, y)
print("选中的特征:", X.columns[rfe.support_])

from sklearn.feature_extraction.text import TfidfVectorizer

tfidf = TfidfVectorizer(max_features=1000, stop_words='english')
X_text = tfidf.fit_transform(df['评论'])

# 用预训练 CNN 的倒数第二层做特征（迁移学习）
from torchvision.models import resnet18
import torchvision.transforms as T

model = resnet18(pretrained=True)
model.fc = torch.nn.Identity()      # 去掉分类头，输出特征向量

# ❌ 泄漏1：用全量数据归一化/填充
scaler = StandardScaler()
X_scaled = scaler.fit_transform(X)      # fit 用到了测试集统计量

# 正确：fit 只用训练集
scaler = StandardScaler()
X_train_s = scaler.fit_transform(X_train)
X_test_s = scaler.transform(X_test)

# ❌ 泄漏2：目标编码用全量目标均值
city_target = df.groupby('城市')['是否复购'].mean()   # 用到 y 了

# 正确：在训练集内算，测试集用训练集映射
city_target = df_train.groupby('城市')['是否复购'].mean()

# ❌ 泄漏3：时序问题用了未来信息
# 训练某天的预测，特征里却包含当天之后的统计量

import numpy as np
import pandas as pd
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import roc_auc_score

# 1. 加载
df = pd.read_csv('users.csv')

# 2. 清洗
df['age'] = df['age'].fillna(df['age'].median())
df = df.dropna(subset=['user_id'])
df = df.drop_duplicates()

# 3. 构造
df['注册月数'] = (pd.Timestamp.now() - pd.to_datetime(df['注册日期'])).dt.days / 30
df['客单价'] = df['消费额'] / (df['消费次数'] + 1)
df['是否周末下单'] = (pd.to_datetime(df['最近下单日期']).dt.dayofweek >= 5).astype(int)

# 4. 编码
df = pd.get_dummies(df, columns=['城市'], prefix='城市')

# 5. 选择特征
features = ['age','消费额','消费次数','客单价','注册月数',
            '是否周末下单'] + [c for c in df if c.startswith('城市_')]
X = df[features]
y = df['是否复购']

# 6. 切分 + 标准化（防泄漏）
X_train, X_test, y_train, y_test = train_test_split(
    X, y, test_size=0.2, random_state=42)
scaler = StandardScaler()
X_train_s = scaler.fit_transform(X_train)
X_test_s = scaler.transform(X_test)

# 7. 训练 + 评估
rf = RandomForestClassifier(n_estimators=100, random_state=42)
rf.fit(X_train_s, y_train)
print("AUC:", roc_auc_score(y_test, rf.predict_proba(X_test_s)[:,1]).round(4))