模型评估与基准：从分类指标到 LLM-as-Judge

from sklearn.metrics import (
    precision_score, recall_score, f1_score,
    roc_auc_score, roc_curve, confusion_matrix
)

y_true = [1, 0, 1, 1, 0, 1, 0, 0, 1, 0]
y_pred = [1, 0, 0, 1, 0, 1, 0, 1, 1, 0]
y_prob = [0.9, 0.2, 0.3, 0.8, 0.1, 0.85, 0.15, 0.6, 0.7, 0.05]

print("Precision:", precision_score(y_true, y_pred))   # 5/6
print("Recall:", recall_score(y_true, y_pred))         # 4/5
print("F1:", f1_score(y_true, y_pred))                 # 0.727
print("AUC:", roc_auc_score(y_true, y_prob))           # 0.960

import numpy as np
from sklearn.calibration import calibration_curve

def expected_calibration_error(y_true, y_prob, n_bins=10):
    bins = np.linspace(0, 1, n_bins + 1)
    ece = 0.0
    for i in range(n_bins):
        mask = (y_prob > bins[i]) & (y_prob <= bins[i + 1])
        if mask.sum() == 0:
            continue
        conf = y_prob[mask].mean()          # 桶内平均预测概率
        acc = y_true[mask].mean()           # 桶内实际正例率
        ece += mask.sum() / len(y_true) * abs(acc - conf)
    return ece

# 校准后的模型应使 ECE 趋近 0
ece = expected_calibration_error(y_true, y_prob)
print(f"ECE = {ece:.3f}")   # 越接近 0 校准越好

from nltk.translate.bleu_score import sentence_bleu
from rouge_score import rouge_scorer

reference = ["大模型评估需要结合自动指标与人工判断"]
generated = "大模型评估要结合自动指标和人工判断"

bleu = sentence_bleu([reference], generated, weights=(0.25, 0.25, 0.25, 0.25))
print(f"BLEU = {bleu:.3f}")

scorer = rouge_scorer.RougeScorer(["rouge1", "rouge2", "rougeL"], lang="zh")
scores = scorer.score(reference, generated)
print({k: v.fmeasure for k, v in scores.items()})

JUDGE_PROMPT = """你是评估员。比较模型回答与参考答案，按以下维度打分（1-5）：
- 正确性（事实准确，无幻觉）
- 完整性（覆盖所有要点）
- 相关性（紧扣问题）
输出 JSON：{"correctness": n, "completeness": n, "relevance": n, "comment": "..."}"""

def llm_judge(question, answer, reference=None):
    response = judge_llm.invoke([
        {"role": "system", "content": JUDGE_PROMPT},
        {"role": "user", "content": f"问题：{question}\n回答：{answer}\n参考答案：{reference or '无'}"}
    ])
    return parse_json(response.content)

{
  "test_suite": {
    "normal_cases": 120,     // 正常业务问题
    "edge_cases": 40,        // 边界（空输入、超长、非常规表达）
    "adversarial": 30,       // 对抗样本（错别字、歧义、诱导）
    "domain_shift": 10       // 新领域样本
  },
  "labeling": {
    "standard": "人工标注标准答案",
    "difficulty": ["easy", "medium", "hard"],
    "count_per_sample": 3
  }
}

import random

def ab_assign(user_id: str, exp_ratio: float = 0.2) -> str:
    """简单哈希分桶：保证同一用户始终进同一组。"""
    return "experiment" if (hash(user_id) % 100) < exp_ratio * 100 else "control"

# 影子部署：新模型结果仅记录
for request in traffic:
    online_result = prod_model(request)        # 返回用户
    shadow_result = candidate_model(request)  # 仅记录，用于离线分析
    log_to_store(request, online_result, shadow_result)

# benchmark_report.yaml 示例
model:
  name: "ChatQwen-7B-sft-v3"
  base: "Qwen2.5-7B-Instruct"
  checkpoint: "run_20260920_epoch2"
evaluation:
  dataset: "golden_set_v2.1"
  metric: ["accuracy", "faithfulness", "harmlessness"]
  judge_model: "gpt-4o"        # LLM-as-Judge 的评审模型
  temperature: 0.0             # 评估用确定性采样
  seed: 42
results:
  accuracy: 0.882
  faithfulness: 0.91
  harmlessness_pass_rate: 0.99
date: "2026-09-25"

import numpy as np

def psi(expected: np.ndarray, actual: np.ndarray, n_bins: int = 10) -> float:
    """Population Stability Index：衡量两个分布的分桶差异。"""
    eps = 1e-6
    bins = np.linspace(0, 1, n_bins + 1)
    e_hist, _ = np.histogram(expected, bins=bins, density=True)
    a_hist, _ = np.histogram(actual, bins=bins, density=True)
    e_hist += eps; a_hist += eps
    e_pct = e_hist / e_hist.sum()
    a_pct = a_hist / a_hist.sum()
    return float(np.sum((a_pct - e_pct) * np.log(a_pct / e_pct)))

# 行业经验：PSI < 0.1 稳定；0.1-0.25 需关注；>0.25 需处理
score = psi(expected_dist, production_dist)
print(f"PSI = {score:.3f}")