《互联网产品的 AI 架构·多能力编排与任务闭环》10.3 编排事故的止血与复盘

// file: src/orch/observability-incident.ts
// tier: L2
// verify: node --test test/orch/observability.test.ts

import { AiError } from "../core/errors.ts";

export type IncidentStage =
  | "detected"
  | "intake-stopped"
  | "node-degraded"
  | "version-rolled-back"
  | "data-preserved"
  | "stabilized";

export type IncidentAction =
  | "stop-intake"
  | "degrade-node"
  | "rollback-version"
  | "preserve-data"
  | "stabilize";

/** 止血顺序是硬约束：先停新增，再降级，再回滚，最后固化数据。 */
export const CONTAINMENT_ORDER: readonly IncidentAction[] = [
  "stop-intake",
  "degrade-node",
  "rollback-version",
  "preserve-data",
  "stabilize",
];

export const STAGE_AFTER: Record<IncidentAction, IncidentStage> = {
  "stop-intake": "intake-stopped",
  "degrade-node": "node-degraded",
  "rollback-version": "version-rolled-back",
  "preserve-data": "data-preserved",
  "stabilize": "stabilized",
};

export interface IncidentEvidence {
  nodeId?: string;
  hasFallback?: boolean;
  currentVersion?: string;
  targetVersion?: string;
  snapshotId?: string;
  preservedRows?: number;
  verifiedRuns?: number;
  observedErrorRate?: number;
  sloTarget?: number;
}

export interface IncidentLogEntry {
  action: IncidentAction;
  stage: IncidentStage;
  atOffsetMs: number;
  detail: string;
}

export interface IncidentState {
  stage: IncidentStage;
  log: IncidentLogEntry[];
}

export interface Transition {
  ok: boolean;
  state: IncidentState;
  code: "applied" | "already-done" | "out-of-order" | "missing-evidence" | "invalid-evidence";
  message: string;
}

export const INITIAL_INCIDENT: IncidentState = { stage: "detected", log: [] };

function fail(message: string): never {
  throw new AiError("invalid-request", `incident: ${message}`);
}

function text(value: unknown): value is string {
  return typeof value === "string" && value.trim() !== "";
}

function ratio(value: unknown, field: string): number {
  if (typeof value !== "number" || !Number.isFinite(value) || value < 0 || value > 1) {
    fail(`${field} 必须落在 [0, 1]`);
  }
  return value;
}

function integer(value: unknown, field: string, min = 0): number {
  if (typeof value !== "number" || !Number.isInteger(value) || value < min) {
    fail(`${field} 必须是 >= ${min} 的整数`);
  }
  return value;
}

function refuse(state: IncidentState, code: Transition["code"], message: string): Transition {
  return { ok: false, state, code, message };
}

function evidenceCheck(action: IncidentAction, evidence: IncidentEvidence): string | null {
  if (action === "degrade-node") {
    if (!text(evidence.nodeId)) return "缺少要降级的节点";
    if (evidence.hasFallback !== true) return "节点没有可用的降级产物";
  }
  if (action === "rollback-version") {
    if (!text(evidence.currentVersion)) return "缺少当前版本";
    if (!text(evidence.targetVersion)) return "缺少回滚目标版本";
    if (evidence.currentVersion === evidence.targetVersion) return "回滚目标与当前版本相同";
  }
  if (action === "preserve-data") {
    if (!text(evidence.snapshotId)) return "缺少数据快照标识";
    if (!Number.isInteger(evidence.preservedRows) || (evidence.preservedRows ?? -1) < 0) {
      return "缺少已固化行数";
    }
  }
  if (action === "stabilize") {
    if (!Number.isInteger(evidence.verifiedRuns) || (evidence.verifiedRuns ?? 0) < 100) {
      return "至少需要 100 次验证运行";
    }
    const errorRate = ratio(evidence.observedErrorRate, "observedErrorRate");
    const target = ratio(evidence.sloTarget, "sloTarget");
    if (target === 1) return "SLO 目标为 100% 时无法判定恢复";
    if (errorRate > 1 - target) {
      return `错误率 ${errorRate} 仍高于预算 ${Number((1 - target).toFixed(4))}`;
    }
  }
  return null;
}

/** 按固定顺序推进；重复动作幂等，跳步或证据不足一律拒绝。 */
export function applyIncidentAction(
  state: IncidentState,
  action: IncidentAction,
  evidence: IncidentEvidence = {},
  atOffsetMs = 0,
): Transition {
  if (!Number.isFinite(atOffsetMs) || atOffsetMs < 0) fail("atOffsetMs 必须是非负数");
  const index = CONTAINMENT_ORDER.indexOf(action);
  if (index < 0) return refuse(state, "out-of-order", `未知动作 ${String(action)}`);
  if (state.log.some((entry) => entry.action === action)) {
    return { ok: true, state, code: "already-done", message: `${action} 已执行，保持原状态` };
  }
  const expectedStage = index === 0 ? "detected" : STAGE_AFTER[CONTAINMENT_ORDER[index - 1]];
  if (state.stage !== expectedStage) {
    return refuse(
      state,
      "out-of-order",
      `${action} 需要先处于 ${expectedStage}，当前是 ${state.stage}`,
    );
  }
  const missing = evidenceCheck(action, evidence);
  if (missing !== null) {
    const code = missing.startsWith("缺少") ? "missing-evidence" : "invalid-evidence";
    return refuse(state, code, missing);
  }
  const next: IncidentState = {
    stage: STAGE_AFTER[action],
    log: [
      ...state.log,
      {
        action,
        stage: STAGE_AFTER[action],
        atOffsetMs: Math.round(atOffsetMs),
        detail: action === "degrade-node" ? String(evidence.nodeId) : action,
      },
    ],
  };
  return { ok: true, state: next, code: "applied", message: `进入 ${next.stage}` };
}

/** 错误预算消耗倍数：成功率 98.12%、目标 99% 时，1% 的预算已经烧掉 1.88 倍。 */
export function errorBudgetBurn(successRate: number, sloTarget: number): number {
  const success = ratio(successRate, "successRate");
  const target = ratio(sloTarget, "sloTarget");
  if (target === 1) fail("sloTarget 不能是 100%");
  return Number(((1 - success) / (1 - target)).toFixed(4));
}

export interface ReviewField {
  key: string;
  label: string;
  required: boolean;
}

export const REVIEW_FIELDS: readonly ReviewField[] = [
  { key: "detectedAt", label: "发现时间", required: true },
  { key: "mitigatedAt", label: "止血完成时间", required: true },
  { key: "affectedRuns", label: "影响面（工作流/运行数）", required: true },
  { key: "budgetBurn", label: "错误预算消耗倍数", required: true },
  { key: "triggerNode", label: "触发节点", required: true },
  { key: "triggerVersion", label: "触发版本（模型/提示/编排）", required: true },
  { key: "amplifier", label: "放大因素", required: true },
  { key: "rootCause", label: "根因", required: true },
  { key: "guardGap", label: "防线缺口", required: true },
  { key: "action", label: "行动项", required: true },
  { key: "owner", label: "负责人", required: true },
  { key: "dueAt", label: "截止时间", required: true },
];

export interface ReviewRow extends ReviewField {
  value: string;
  filled: boolean;
}

export interface ReviewTable {
  rows: ReviewRow[];
  missing: string[];
  complete: boolean;
}

/** 复盘表先判定“能不能结案”，再谈文风：必填项为空就不允许关闭。 */
export function buildReviewTable(input: Record<string, string>): ReviewTable {
  const rows = REVIEW_FIELDS.map((field) => {
    const value = (input[field.key] ?? "").trim();
    return { ...field, value, filled: value !== "" };
  });
  const missing = rows.filter((row) => row.required && !row.filled).map((row) => row.key);
  return { rows, missing, complete: missing.length === 0 };
}