《互联网产品的 AI 架构·平台化与治理》6.2 质量事故的分级与响应

// file: src/gov/quality-incidents.ts
// tier: L2
// verify: node --test test/gov/quality-ops.test.ts

/** P0 最严重，P3 最轻；分级只看事实，不看谁在群里先说话。 */
export type IncidentSeverity = "P0" | "P1" | "P2" | "P3";

export interface IncidentFacts {
  affectedTenants: number;
  totalTenants: number;
  failedDeliveries: number;
  totalDeliveries: number;
  privacyExposure: boolean;
  dataLoss: boolean;
  financialImpact: boolean;
  safetyViolation: boolean;
}

export interface ResponsePolicy {
  severity: IncidentSeverity;
  acknowledgeMinutes: number;
  mitigateMinutes: number;
  updateEveryMinutes: number;
  postmortemHours: number;
  commander: string;
  notify: readonly string[];
  stopBleeding: readonly string[];
}

export const RESPONSE_POLICIES: Readonly<Record<IncidentSeverity, ResponsePolicy>> = {
  P0: {
    severity: "P0",
    acknowledgeMinutes: 5,
    mitigateMinutes: 30,
    updateEveryMinutes: 15,
    postmortemHours: 24,
    commander: "事件指挥官（平台负责人授权）",
    notify: ["值班", "平台负责人", "安全", "合规", "受影响租户负责人", "业务高管"],
    stopBleeding: [
      "关闭受影响能力的入口或路由",
      "切回上一个稳定能力版本",
      "冻结相关模型与提示变更",
      "保留请求、回放包与审计日志",
    ],
  },
  P1: {
    severity: "P1",
    acknowledgeMinutes: 15,
    mitigateMinutes: 60,
    updateEveryMinutes: 30,
    postmortemHours: 72,
    commander: "平台值班负责人",
    notify: ["值班", "平台负责人", "能力提供方", "受影响租户负责人"],
    stopBleeding: [
      "把受影响租户切到降级档或人工队列",
      "回滚最近一次能力版本或路由变更",
      "保留失败请求与抽检样本",
    ],
  },
  P2: {
    severity: "P2",
    acknowledgeMinutes: 60,
    mitigateMinutes: 240,
    updateEveryMinutes: 120,
    postmortemHours: 168,
    commander: "能力提供方值班",
    notify: ["值班", "能力提供方", "受影响租户负责人"],
    stopBleeding: ["限制问题能力的流量比例", "提高转人工阈值", "记录受影响请求范围"],
  },
  P3: {
    severity: "P3",
    acknowledgeMinutes: 240,
    mitigateMinutes: 1440,
    updateEveryMinutes: 1440,
    postmortemHours: 336,
    commander: "能力提供方负责人",
    notify: ["能力提供方", "质量运营"],
    stopBleeding: ["登记样本并纳入下次抽检", "按常规迭代修复，不打断当前发布"],
  },
};

export interface IncidentGrade {
  severity: IncidentSeverity;
  failureRate: number;
  affectedRate: number;
  reasons: readonly string[];
  policy: ResponsePolicy;
}

/** 危害类事实优先于比例：只要越权、丢数据或动钱，就不能按 P3 处理。 */
export function gradeIncident(facts: IncidentFacts): IncidentGrade {
  if (
    !Number.isInteger(facts.totalTenants) ||
    facts.totalTenants <= 0 ||
    !Number.isInteger(facts.affectedTenants) ||
    facts.affectedTenants < 0 ||
    facts.affectedTenants > facts.totalTenants
  ) {
    throw new Error("quality-incidents: invalid tenant counts");
  }
  if (
    !Number.isInteger(facts.totalDeliveries) ||
    facts.totalDeliveries <= 0 ||
    !Number.isInteger(facts.failedDeliveries) ||
    facts.failedDeliveries < 0 ||
    facts.failedDeliveries > facts.totalDeliveries
  ) {
    throw new Error("quality-incidents: invalid delivery counts");
  }

  const failureRate = facts.failedDeliveries / facts.totalDeliveries;
  const affectedRate = facts.affectedTenants / facts.totalTenants;
  const reasons: string[] = [];
  if (facts.privacyExposure) reasons.push("隐私或数据越权暴露");
  if (facts.dataLoss) reasons.push("数据丢失");
  if (facts.financialImpact) reasons.push("资金或订单动作错误");
  if (facts.safetyViolation) reasons.push("安全违规内容到达用户");

  let severity: IncidentSeverity;
  if (facts.privacyExposure || facts.dataLoss || facts.financialImpact) {
    severity = "P0";
  } else if ((facts.safetyViolation && affectedRate >= 0.5) || (failureRate >= 0.5 && facts.affectedTenants >= 3)) {
    severity = "P0";
    if (facts.safetyViolation) reasons.push("安全违规覆盖半数以上租户");
    else reasons.push("失败率不低于 50% 且影响至少 3 个租户");
  } else if (facts.safetyViolation || failureRate >= 0.2 || affectedRate >= 0.5) {
    severity = "P1";
    if (failureRate >= 0.2) reasons.push("交付失败率不低于 20%");
    if (affectedRate >= 0.5) reasons.push("受影响租户不少于一半");
  } else if (failureRate >= 0.05 || facts.affectedTenants >= 2) {
    severity = "P2";
    if (failureRate >= 0.05) reasons.push("交付失败率不低于 5%");
    if (facts.affectedTenants >= 2) reasons.push("影响多个租户");
  } else {
    severity = "P3";
    reasons.push("影响限于单一租户且失败率低于 5%");
  }

  return { severity, failureRate, affectedRate, reasons, policy: RESPONSE_POLICIES[severity] };
}

export interface IncidentTimeline {
  detectedAt: number;
  acknowledgeBy: number;
  mitigateBy: number;
  firstUpdateBy: number;
  postmortemBy: number;
}

const MINUTE_MS = 60_000;
const HOUR_MS = 60 * MINUTE_MS;

/** 时间线从发现时刻起算，不从「有人开始在群里讨论」起算。 */
export function responseDeadlines(detectedAt: number, policy: ResponsePolicy): IncidentTimeline {
  if (!Number.isFinite(detectedAt)) throw new Error("quality-incidents: detectedAt must be finite");
  return {
    detectedAt,
    acknowledgeBy: detectedAt + policy.acknowledgeMinutes * MINUTE_MS,
    mitigateBy: detectedAt + policy.mitigateMinutes * MINUTE_MS,
    firstUpdateBy: detectedAt + policy.updateEveryMinutes * MINUTE_MS,
    postmortemBy: detectedAt + policy.postmortemHours * HOUR_MS,
  };
}

const SEVERITY_ORDER: readonly IncidentSeverity[] = ["P0", "P1", "P2", "P3"];

/** 超时未确认自动升一级；P0 已是最高的，仍返回 P0 表示继续升级通知范围。 */
export function escalationTarget(
  grade: IncidentGrade,
  elapsedMinutes: number,
  acknowledged: boolean,
): IncidentSeverity | null {
  if (acknowledged || elapsedMinutes <= grade.policy.acknowledgeMinutes) return null;
  const index = SEVERITY_ORDER.indexOf(grade.severity);
  return SEVERITY_ORDER[Math.max(0, index - 1)];
}