《互联网产品的 AI 架构·平台化与治理》5.1 评测集的治理：版本、来源与污染

// file: src/gov/eval-platform.ts
// tier: L2
// verify: node --test test/gov/eval-platform.test.ts

import { createHash } from "node:crypto";

/** 评测集是平台资产，不是某个团队的本地夹具；版本、来源与污染都必须可复核。 */
export type EvalSource = "human" | "production" | "synthetic";
export interface EvalCase {
  id: string;
  capability: string;
  input: string;
  expected: string;
  source: EvalSource;
  tags: readonly string[];
  /** 非合成样本追溯到采集批次；合成样本还必须追溯到生成器与整数种子。 */
  provenance: { batch: string; collectedAt: string; generator?: string; seed?: number };
}
export interface EvalSet { id: string; version: string; cases: readonly EvalCase[]; updatedAt: string }
export interface EvalPolicy {
  minHumanBp: number;
  maxSyntheticBp: number;
  minCases: number;
  maxCapabilityBp: number;
}

/** 万分比口径，避免浮点在门禁与报表之间产生不同的舍入。 */
export const DEFAULT_EVAL_POLICY: EvalPolicy = {
  minHumanBp: 3_000, maxSyntheticBp: 4_000, minCases: 40, maxCapabilityBp: 6_000,
};
export interface SourceMix {
  total: number;
  human: number;
  production: number;
  synthetic: number;
  humanBp: number;
  productionBp: number;
  syntheticBp: number;
}
export function sourceMix(cases: readonly EvalCase[]): SourceMix {
  const total = cases.length;
  const count = (source: EvalSource): number => cases.filter((item) => item.source === source).length;
  const bp = (value: number): number => (total === 0 ? 0 : Math.round((value * 10_000) / total));
  const human = count("human");
  const production = count("production");
  const synthetic = count("synthetic");
  return {
    total, human, production, synthetic,
    humanBp: bp(human), productionBp: bp(production), syntheticBp: bp(synthetic),
  };
}
export type EvalIssueCode =
  | "invalid-id" | "invalid-version" | "no-cases" | "below-min-cases"
  | "duplicate-case" | "invalid-source" | "missing-provenance"
  | "synthetic-without-seed" | "human-share-low" | "synthetic-share-high"
  | "capability-concentration";
export interface EvalIssue { path: string; code: EvalIssueCode; detail: string }
const EVAL_ID = /^eval-\d+$/;
const SEMVER = /^\d+\.\d+\.\d+$/;
const DATE = /^\d{4}-\d{2}-\d{2}$/;
export function validateEvalSet(set: EvalSet, policy: EvalPolicy = DEFAULT_EVAL_POLICY): EvalIssue[] {
  const issues: EvalIssue[] = [];
  if (!EVAL_ID.test(set.id)) {
    issues.push({ path: set.id, code: "invalid-id", detail: "评测集 id 必须是 eval-<整数>" });
  }
  if (!SEMVER.test(set.version)) {
    issues.push({ path: `${set.id}@${set.version}`, code: "invalid-version", detail: "版本必须是三段语义化版本" });
  }
  if (set.cases.length === 0) {
    issues.push({ path: set.id, code: "no-cases", detail: "评测集不能为空" });
    return issues;
  }
  if (set.cases.length < policy.minCases) {
    issues.push({ path: set.id, code: "below-min-cases", detail: `样本数 ${set.cases.length} 低于下限 ${policy.minCases}` });
  }
  const seen = new Set<string>();
  for (const item of set.cases) {
    if (seen.has(item.id)) issues.push({ path: item.id, code: "duplicate-case", detail: "同一评测集内 case id 重复" });
    seen.add(item.id);
    if (item.source !== "human" && item.source !== "production" && item.source !== "synthetic") {
      issues.push({ path: item.id, code: "invalid-source", detail: `未知来源 ${String(item.source)}` });
    }
    if (item.provenance.batch.trim() === "" || !DATE.test(item.provenance.collectedAt)) {
      issues.push({ path: item.id, code: "missing-provenance", detail: "缺少采集批次或合法采集日期" });
    }
    if (item.source === "synthetic" && (item.provenance.generator === undefined || item.provenance.seed === undefined)) {
      issues.push({ path: item.id, code: "synthetic-without-seed", detail: "合成样本必须记录生成器与种子" });
    }
  }
  const mix = sourceMix(set.cases);
  if (mix.humanBp < policy.minHumanBp) {
    issues.push({ path: set.id, code: "human-share-low", detail: `人工标注占比 ${mix.humanBp}bp 低于 ${policy.minHumanBp}bp` });
  }
  if (mix.syntheticBp > policy.maxSyntheticBp) {
    issues.push({ path: set.id, code: "synthetic-share-high", detail: `合成样本占比 ${mix.syntheticBp}bp 高于 ${policy.maxSyntheticBp}bp` });
  }
  const byCapability = new Map<string, number>();
  for (const item of set.cases) {
    byCapability.set(item.capability, (byCapability.get(item.capability) ?? 0) + 1);
  }
  for (const [capability, count] of byCapability) {
    const shareBp = Math.round((count * 10_000) / set.cases.length);
    if (shareBp > policy.maxCapabilityBp) {
      issues.push({
        path: `${set.id}/${capability}`, code: "capability-concentration",
        detail: `${capability} 占 ${shareBp}bp，超过单能力上限 ${policy.maxCapabilityBp}bp`,
      });
    }
  }
  return issues;
}

/** 归一化只抹掉大小写、空白与标点，不改变词序；这样近似重复不会被误判为完全相同。 */
export function normalizeText(text: string): string {
  return text.toLowerCase().replace(/[\s\p{P}]+/gu, "");
}
export function textHash(text: string): string {
  return createHash("sha256").update(normalizeText(text)).digest("hex").slice(0, 16);
}
function editDistance(left: string, right: string): number {
  const previous = Array.from({ length: right.length + 1 }, (_, index) => index);
  for (let i = 1; i <= left.length; i += 1) {
    const current = [i];
    for (let j = 1; j <= right.length; j += 1) {
      const cost = left[i - 1] === right[j - 1] ? 0 : 1;
      current[j] = Math.min(previous[j] + 1, current[j - 1] + 1, previous[j - 1] + cost);
    }
    for (let j = 0; j <= right.length; j += 1) previous[j] = current[j];
  }
  return previous[right.length];
}

/** 归一化编辑距离比 n-gram 集合更稳：插入一个词不会让后半段的相似度全部归零。 */
export function similarity(left: string, right: string): number {
  const a = normalizeText(left);
  const b = normalizeText(right);
  if (a.length === 0 && b.length === 0) return 1;
  return 1 - editDistance(a, b) / Math.max(a.length, b.length);
}
export type ContaminationKind =
  | "duplicate-in-set" | "exact-training" | "near-training"
  | "exact-tuning" | "near-tuning";
export interface ContaminationFinding { caseId: string; kind: ContaminationKind; reference: string; score: number }
export interface ContaminationCorpus { training: readonly string[]; tuning: readonly string[] }
export interface ContaminationReport {
  total: number;
  findings: ContaminationFinding[];
  contaminated: string[];
  clean: boolean;
}

/**
 * 污染检查按「精确命中优先、近似命中其次」的顺序报告；
 * 同一 case 对同一语料只保留最强的一条结论。
 */
export function checkContamination(
  candidate: readonly EvalCase[],
  corpus: ContaminationCorpus,
  threshold = 0.8,
): ContaminationReport {
  const findings: ContaminationFinding[] = [];
  const seen = new Map<string, string>();
  for (const item of candidate) {
    const hash = textHash(item.input);
    const duplicate = seen.get(hash);
    if (duplicate !== undefined) {
      findings.push({ caseId: item.id, kind: "duplicate-in-set", reference: duplicate, score: 1 });
    } else {
      seen.set(hash, item.id);
    }
    for (const [kind, source] of [["training", corpus.training], ["tuning", corpus.tuning]] as const) {
      if (source.some((text) => textHash(text) === hash)) {
        findings.push({ caseId: item.id, kind: kind === "training" ? "exact-training" : "exact-tuning", reference: kind, score: 1 });
        continue;
      }
      const near = nearest(item.input, source, threshold);
      if (near !== null) {
        findings.push({
          caseId: item.id, kind: kind === "training" ? "near-training" : "near-tuning",
          reference: near.reference, score: near.score,
        });
      }
    }
  }
  const contaminated = [...new Set(findings.map((item) => item.caseId))].sort();
  return { total: candidate.length, findings, contaminated, clean: contaminated.length === 0 };
}
function nearest(text: string, corpus: readonly string[], threshold: number): { reference: string; score: number } | null {
  let best: { reference: string; score: number } | null = null;
  for (const [index, item] of corpus.entries()) {
    const score = similarity(text, item);
    if (score >= threshold && (best === null || score > best.score)) {
      best = { reference: `corpus-${index}`, score: Math.round(score * 1_000) / 1_000 };
    }
  }
  return best;
}
export type EvalBump = "major" | "minor" | "patch";
export interface EvalChangeReport {
  added: string[];
  removed: string[];
  relabeled: string[];
  rewritten: string[];
  requiredBump: EvalBump;
}

/** 删除、改标注、改输入都会让历史分数失去可比性，必须发主版本。 */
export function classifyEvalChange(before: EvalSet, after: EvalSet): EvalChangeReport {
  const oldCases = new Map(before.cases.map((item) => [item.id, item]));
  const newCases = new Map(after.cases.map((item) => [item.id, item]));
  const added = [...newCases.keys()].filter((id) => !oldCases.has(id)).sort();
  const removed = [...oldCases.keys()].filter((id) => !newCases.has(id)).sort();
  const relabeled: string[] = [];
  const rewritten: string[] = [];
  for (const [id, oldCase] of oldCases) {
    const newCase = newCases.get(id);
    if (newCase === undefined) continue;
    if (newCase.expected !== oldCase.expected) relabeled.push(id);
    else if (normalizeText(newCase.input) !== normalizeText(oldCase.input)) rewritten.push(id);
  }
  const requiredBump: EvalBump =
    removed.length + relabeled.length + rewritten.length > 0 ? "major" : added.length > 0 ? "minor" : "patch";
  return { added, removed, relabeled: relabeled.sort(), rewritten: rewritten.sort(), requiredBump };
}
const BUMP_RANK: Record<EvalBump, number> = { patch: 0, minor: 1, major: 2 };
export function checkEvalBump(report: EvalChangeReport, declared: EvalBump): { ok: boolean; reason: string } {
  const ok = BUMP_RANK[declared] >= BUMP_RANK[report.requiredBump];
  return {
    ok,
    reason: ok
      ? `声明的 ${declared} 覆盖 ${report.requiredBump} 要求`
      : `删除 ${report.removed.length}、改标注 ${report.relabeled.length}、改输入 ${report.rewritten.length}，要求 ${report.requiredBump}`,
  };
}
export interface Cadence { reviewDays: number; refreshBp: number }
export const DEFAULT_CADENCE: Cadence = { reviewDays: 90, refreshBp: 2_000 };
function addDays(date: string, days: number): string {
  return new Date(Date.parse(`${date}T00:00:00Z`) + days * 86_400_000).toISOString().slice(0, 10);
}
export interface CadenceStatus {
  setId: string;
  dueAt: string;
  overdueDays: number;
  due: boolean;
  refreshTarget: number;
}
export function cadenceStatus(set: EvalSet, today: string, cadence: Cadence = DEFAULT_CADENCE): CadenceStatus {
  const dueAt = addDays(set.updatedAt, cadence.reviewDays);
  const days = (Date.parse(`${today}T00:00:00Z`) - Date.parse(`${dueAt}T00:00:00Z`)) / 86_400_000;
  return {
    setId: set.id,
    dueAt,
    overdueDays: Math.max(0, Math.round(days)),
    due: days > 0,
    refreshTarget: Math.ceil((set.cases.length * cadence.refreshBp) / 10_000),
  };
}