《互联网产品的 AI 架构·模型接入与能力边界》12.3 质量信号的线上采集

// file: src/observe/quality.ts
// tier: L1
// verify: node --test test/observe/quality.test.ts

import { AiError } from "../core/errors.ts";
import type { CapabilityId } from "../core/types.ts";

/** 指标定义本身也要版本化，否则线上与离线只是两个同名的数字。 */
export const QUALITY_METRIC_VERSION = 1;

export type FeedbackKind =
  | "accepted_unchanged"
  | "accepted_edited"
  | "rejected"
  | "ignored";

export interface SuggestionEvent {
  suggestionId: string;
  capability: CapabilityId;
  modelId: string;
  surfacedAt: number;
  /** null 表示用户没有做出任何反馈，不能当作拒绝。 */
  feedback: FeedbackKind | null;
  /** 编辑幅度 0 到 1，只有 accepted_edited 参与轻改判定。 */
  editDistanceRatio?: number;
  retried: boolean;
  handoff: boolean;
  complaint: boolean;
}

export type AddStatus = "accepted" | "duplicate" | "conflict";

const FEEDBACK_KINDS = new Set<string>([
  "accepted_unchanged",
  "accepted_edited",
  "rejected",
  "ignored",
]);

function fail(message: string): never {
  throw new AiError("invalid-request", `quality: ${message}`);
}

function validateEvent(event: SuggestionEvent): void {
  for (const field of ["suggestionId", "capability", "modelId"] as const) {
    if (typeof event[field] !== "string" || event[field].trim() === "") fail(`${field} is required`);
  }
  if (!Number.isInteger(event.surfacedAt) || event.surfacedAt < 0) {
    fail("surfacedAt must be a non-negative integer");
  }
  if (event.feedback !== null && !FEEDBACK_KINDS.has(event.feedback)) {
    fail("feedback is not a known kind");
  }
  if (event.editDistanceRatio !== undefined) {
    const ratio = event.editDistanceRatio;
    if (typeof ratio !== "number" || !Number.isFinite(ratio) || ratio < 0 || ratio > 1) {
      fail("editDistanceRatio must be between 0 and 1");
    }
  }
  for (const field of ["retried", "handoff", "complaint"] as const) {
    if (typeof event[field] !== "boolean") fail(`${field} must be a boolean`);
  }
}

const SAME_FIELDS: Array<keyof SuggestionEvent> = [
  "capability", "modelId", "surfacedAt", "feedback", "editDistanceRatio", "retried", "handoff", "complaint",
];

function sameEvent(left: SuggestionEvent, right: SuggestionEvent): boolean {
  return SAME_FIELDS.every((field) => left[field] === right[field]);
}

export interface QualityMetrics {
  metricVersion: number;
  surfaced: number;
  withFeedback: number;
  noFeedbackRate: number;
  /** 点击口径：原样采纳 + 改编采纳，容易虚高。 */
  clickThroughRate: number;
  /** 真正口径：只有原样采纳才计入。 */
  adoptionRate: number;
  /** 改写幅度不超过阈值的那部分采纳。 */
  lightEditRate: number;
  /** 原样采纳 + 轻改，作为采纳率的上界参考。 */
  adjustedAdoptionRate: number;
  retryRate: number;
  handoffRate: number;
  complaintsPerThousand: number;
}

function rate(numerator: number, denominator: number): number {
  return denominator === 0 ? 0 : numerator / denominator;
}

/** 分母是“曝光过的建议数”，不是“有反馈的建议数”，否则沉默用户会被抹掉。 */
export function computeQuality(
  events: SuggestionEvent[],
  options: { lightEditThreshold?: number } = {},
): QualityMetrics {
  const threshold = options.lightEditThreshold ?? 0.2;
  const surfaced = events.length;
  let withFeedback = 0;
  let unchanged = 0;
  let edited = 0;
  let light = 0;
  let retried = 0;
  let handoff = 0;
  let complaints = 0;

  for (const event of events) {
    if (event.feedback !== null) withFeedback += 1;
    if (event.feedback === "accepted_unchanged") unchanged += 1;
    if (event.feedback === "accepted_edited") {
      edited += 1;
      if (event.editDistanceRatio !== undefined && event.editDistanceRatio <= threshold) light += 1;
    }
    if (event.retried) retried += 1;
    if (event.handoff) handoff += 1;
    if (event.complaint) complaints += 1;
  }

  return {
    metricVersion: QUALITY_METRIC_VERSION,
    surfaced,
    withFeedback,
    noFeedbackRate: rate(surfaced - withFeedback, surfaced),
    clickThroughRate: rate(unchanged + edited, surfaced),
    adoptionRate: rate(unchanged, surfaced),
    lightEditRate: rate(light, surfaced),
    adjustedAdoptionRate: rate(unchanged + light, surfaced),
    retryRate: rate(retried, surfaced),
    handoffRate: rate(handoff, surfaced),
    complaintsPerThousand: rate(complaints, surfaced) * 1000,
  };
}

/**
 * 采集器按 suggestionId 去重。同一建议被上报两次是常态，
 * 但两次内容不同说明上游状态错乱，应暴露为冲突而不是静默覆盖。
 */
export class QualityCollector {
  private readonly events: Map<string, SuggestionEvent>;

  constructor() {
    this.events = new Map();
  }

  add(event: SuggestionEvent): { status: AddStatus; duplicateOf?: string } {
    validateEvent(event);
    const seen = this.events.get(event.suggestionId);
    if (seen === undefined) {
      this.events.set(event.suggestionId, event);
      return { status: "accepted" };
    }
    return sameEvent(seen, event)
      ? { status: "duplicate", duplicateOf: seen.suggestionId }
      : { status: "conflict", duplicateOf: seen.suggestionId };
  }

  get size(): number {
    return this.events.size;
  }

  metrics(filter: { capability?: string; modelId?: string; fromMs?: number; toMs?: number } = {}, options: { lightEditThreshold?: number } = {}): QualityMetrics {
    const selected: SuggestionEvent[] = [];
    for (const event of this.events.values()) {
      if (filter.capability !== undefined && event.capability !== filter.capability) continue;
      if (filter.modelId !== undefined && event.modelId !== filter.modelId) continue;
      if (filter.fromMs !== undefined && event.surfacedAt < filter.fromMs) continue;
      if (filter.toMs !== undefined && event.surfacedAt >= filter.toMs) continue;
      selected.push(event);
    }
    return computeQuality(selected, options);
  }
}

export interface OfflineBaseline {
  metricVersion: number;
  capability: string;
  modelId: string;
  window: string;
  sampleSize: number;
  adoptionRate: number;
  retryRate: number;
  handoffRate: number;
}

export interface AlignmentOptions {
  /** 低于该样本量时线上数字不足以与离线比较，默认 200。 */
  minSample: number;
  adoptionTolerance: number;
  retryTolerance: number;
  handoffTolerance: number;
}

export const DEFAULT_ALIGNMENT: AlignmentOptions = {
  minSample: 200,
  adoptionTolerance: 0.05,
  retryTolerance: 0.03,
  handoffTolerance: 0.02,
};

export type DivergenceMetric = "adoption" | "retry" | "handoff";

export interface Divergence {
  metric: DivergenceMetric;
  online: number;
  offline: number;
  /** 线上减离线。 */
  delta: number;
  direction: "online-better" | "online-worse";
}

export interface AlignmentResult {
  comparable: boolean;
  reason: "metric-version-mismatch" | "insufficient-sample" | null;
  divergences: Divergence[];
}

function divergence(
  metric: DivergenceMetric,
  online: number,
  offline: number,
  tolerance: number,
  higherIsBetter: boolean,
): Divergence | null {
  const delta = online - offline;
  if (Math.abs(delta) <= tolerance) return null;
  const better = higherIsBetter ? delta > 0 : delta < 0;
  return { metric, online, offline, delta, direction: better ? "online-better" : "online-worse" };
}

/**
 * 线上与离线的对应关系有三个前提：指标版本一致、样本量足够、
 * 比较的是同一个能力与模型。任何前提不满足都返回不可比。
 */
export function compareWithOffline(
  online: QualityMetrics,
  offline: OfflineBaseline,
  options: AlignmentOptions = DEFAULT_ALIGNMENT,
): AlignmentResult {
  if (online.metricVersion !== offline.metricVersion) {
    return { comparable: false, reason: "metric-version-mismatch", divergences: [] };
  }
  if (online.surfaced < options.minSample) {
    return { comparable: false, reason: "insufficient-sample", divergences: [] };
  }

  const checks = [
    divergence("adoption", online.adoptionRate, offline.adoptionRate, options.adoptionTolerance, true),
    divergence("retry", online.retryRate, offline.retryRate, options.retryTolerance, false),
    divergence("handoff", online.handoffRate, offline.handoffRate, options.handoffTolerance, false),
  ];
  return {
    comparable: true,
    reason: null,
    divergences: checks.filter((item): item is Divergence => item !== null),
  };
}

node --test test/observe/quality.test.ts

ℹ tests 12
ℹ pass 12
ℹ fail 0
ℹ duration_ms 162.122208