《互联网产品的 AI 架构·多能力编排与任务闭环》7.1 端到端评测集

// file: src/orch/eval.ts
// tier: L2
// verify: node --test test/orch/eval.test.ts

import { AiError } from "../core/errors.ts";

/** 端到端评测的五类样本，按用户可感知的任务结果划分，而不是按节点返回码划分。 */
export type Outcome = "success" | "partial" | "failure" | "timeout" | "unauthorized";

/** 顺序固定，统计表与门禁都按这个顺序输出，保证报告可比。 */
export const OUTCOMES: readonly Outcome[] = [
  "success",
  "partial",
  "failure",
  "timeout",
  "unauthorized",
];

export interface EvalExpectation {
  outcome: Outcome;
  /** 任务级判定：用户任务是否完成，而不是某个节点是否返回 200。 */
  completed: boolean;
  /** 可验收产出占比的下限；成功样本为 1，部分成功样本通常取 0.6。 */
  minDeliveredRatio: number;
  /** 失败路径样本必须携带首个失败节点，否则评测集无法支撑归因。 */
  requiresFailureNode: boolean;
}

export interface EvalObservation {
  outcome: Outcome;
  completed: boolean;
  deliveredRatio: number;
  firstFailureNode?: string;
}

export interface EvalSample {
  id: string;
  scenario: string;
  expected: EvalExpectation;
  observed: EvalObservation;
  costMicros: number;
  latencyMs: number;
}

/** 评测单位固定为 task：一条样本就是一个用户任务，不是一个节点调用。 */
export interface EvalSet {
  id: string;
  version: string;
  unit: "task";
  samples: readonly EvalSample[];
}

export type VerdictCode =
  | "pass"
  | "outcome-mismatch"
  | "task-completion"
  | "partial-delivery"
  | "missing-failure-node";

export interface SampleVerdict {
  id: string;
  passed: boolean;
  code: VerdictCode;
}

export interface EvalPolicy {
  /** 每一类样本至少要有多少条，缺一类就不算覆盖失败路径。 */
  minPerOutcome: number;
  /** 非成功样本占全集的最低比例。 */
  minFailurePathShare: number;
  /** 成功样本占全集的上限，防止评测集退化成 happy path 集合。 */
  maxSuccessShare: number;
}

export const DEFAULT_EVAL_POLICY: EvalPolicy = {
  minPerOutcome: 3,
  minFailurePathShare: 0.4,
  maxSuccessShare: 0.6,
};

export interface OutcomeStat {
  outcome: Outcome;
  samples: number;
  share: number;
  passed: number;
  passRate: number;
}

export interface EvalSetReport {
  total: number;
  byOutcome: OutcomeStat[];
  passed: number;
  passRate: number;
  /** 任务完成率的分母是全部样本，包含失败、超时与越权样本。 */
  taskCompletionRate: number;
  successShare: number;
  failurePathShare: number;
  gaps: string[];
}

function fail(message: string): never {
  throw new AiError("invalid-request", `eval: ${message}`);
}

function ratio(value: number, field: string): number {
  if (!Number.isFinite(value) || value < 0 || value > 1) fail(`${field} 必须落在 [0, 1]`);
  return value;
}

function metric(value: number, field: string): number {
  if (!Number.isInteger(value) || value < 0) fail(`${field} 必须是 >= 0 的整数`);
  return value;
}

/**
 * 单条样本的判定严格按“任务是否完成”执行：
 * 先比对结果类别，再比对任务完成标志，最后检查部分交付与失败节点。
 */
export function judgeSample(sample: EvalSample): SampleVerdict {
  ratio(sample.observed.deliveredRatio, `${sample.id}.deliveredRatio`);
  ratio(sample.expected.minDeliveredRatio, `${sample.id}.minDeliveredRatio`);
  metric(sample.costMicros, `${sample.id}.costMicros`);
  metric(sample.latencyMs, `${sample.id}.latencyMs`);
  if (sample.observed.outcome !== sample.expected.outcome) {
    return { id: sample.id, passed: false, code: "outcome-mismatch" };
  }
  if (sample.observed.completed !== sample.expected.completed) {
    return { id: sample.id, passed: false, code: "task-completion" };
  }
  if (sample.observed.deliveredRatio + 1e-9 < sample.expected.minDeliveredRatio) {
    return { id: sample.id, passed: false, code: "partial-delivery" };
  }
  if (sample.expected.requiresFailureNode && !sample.observed.firstFailureNode) {
    return { id: sample.id, passed: false, code: "missing-failure-node" };
  }
  return { id: sample.id, passed: true, code: "pass" };
}

/** 统计五类样本的数量、通过率与覆盖缺口；缺类、成功样本过多都会写进 gaps。 */
export function summarizeEvalSet(
  set: EvalSet,
  policy: EvalPolicy = DEFAULT_EVAL_POLICY,
): EvalSetReport {
  if (set.unit !== "task") fail(`评测单位必须是 task，当前 ${String(set.unit)}`);
  if (set.id.trim() === "") fail("评测集 id 不能为空");
  if (set.samples.length === 0) fail("评测集不能为空");
  if (!Number.isInteger(policy.minPerOutcome) || policy.minPerOutcome < 1) {
    fail("minPerOutcome 必须是 >= 1 的整数");
  }
  ratio(policy.minFailurePathShare, "minFailurePathShare");
  ratio(policy.maxSuccessShare, "maxSuccessShare");
  if (policy.minFailurePathShare + policy.maxSuccessShare > 1) {
    fail("失败路径下限与成功样本上限互相矛盾");
  }

  const seen = new Set<string>();
  const counts = new Map<Outcome, number>();
  const passes = new Map<Outcome, number>();
  let passed = 0;
  let completed = 0;

  for (const sample of set.samples) {
    if (sample.id.trim() === "") fail("样本 id 不能为空");
    if (seen.has(sample.id)) fail(`重复的样本 id ${sample.id}`);
    seen.add(sample.id);
    const verdict = judgeSample(sample);
    if (verdict.passed) passed += 1;
    if (sample.observed.completed) completed += 1;
    const outcome = sample.observed.outcome;
    counts.set(outcome, (counts.get(outcome) ?? 0) + 1);
    if (verdict.passed) passes.set(outcome, (passes.get(outcome) ?? 0) + 1);
  }

  const total = set.samples.length;
  const byOutcome = OUTCOMES.map((outcome): OutcomeStat => {
    const samples = counts.get(outcome) ?? 0;
    const ok = passes.get(outcome) ?? 0;
    return {
      outcome,
      samples,
      share: Number((samples / total).toFixed(4)),
      passed: ok,
      passRate: samples === 0 ? 0 : Number((ok / samples).toFixed(4)),
    };
  });

  const successShare = Number(((counts.get("success") ?? 0) / total).toFixed(4));
  const failurePathShare = Number((1 - successShare).toFixed(4));
  const gaps: string[] = [];
  for (const stat of byOutcome) {
    if (stat.samples < policy.minPerOutcome) {
      gaps.push(`${stat.outcome} 样本不足：${stat.samples}/${policy.minPerOutcome}`);
    }
  }
  if (failurePathShare < policy.minFailurePathShare) {
    gaps.push(`失败路径占比 ${failurePathShare}，低于下限 ${policy.minFailurePathShare}`);
  }
  if (successShare > policy.maxSuccessShare) {
    gaps.push(`成功样本占比 ${successShare}，高于上限 ${policy.maxSuccessShare}`);
  }

  return {
    total,
    byOutcome,
    passed,
    passRate: Number((passed / total).toFixed(4)),
    taskCompletionRate: Number((completed / total).toFixed(4)),
    successShare,
    failurePathShare,
    gaps,
  };
}

/** 发布门禁：有缺口就拒绝把这次评测结果当作“可上线”的证据。 */
export function assertEvalSet(
  set: EvalSet,
  policy: EvalPolicy = DEFAULT_EVAL_POLICY,
): EvalSetReport {
  const report = summarizeEvalSet(set, policy);
  if (report.gaps.length > 0) {
    fail(`评测集未通过门禁：${report.gaps.join("；")}`);
  }
  return report;
}