《互联网产品的 AI 架构·多能力编排与任务闭环》10.2 节点级 SLO 与整体 SLO

// file: src/orch/observability-slo.ts
// tier: L2
// verify: node --test test/orch/observability.test.ts

import { AiError } from "../core/errors.ts";

export type SloNodeKind = "capability" | "human" | "action";
export type JoinMode = "all" | "any";

/** 单个节点的月度 SLO。可用率与延迟都带自己的目标，整体目标另算。 */
export interface NodeSlo {
  id: string;
  kind: SloNodeKind;
  /** 月度可用率，0..1；人工节点表示按时完成率。 */
  availability: number;
  /** 节点自身 P95 延迟，毫秒。 */
  p95LatencyMs: number;
  /** 单次执行成本，整数微元。 */
  costMicros: number;
  targetAvailability: number;
  targetP95LatencyMs: number;
}

export type SloPlan =
  | { kind: "node"; node: NodeSlo }
  | { kind: "serial"; children: readonly SloPlan[]; handoffLatencyMs: number }
  | { kind: "parallel"; join: JoinMode; children: readonly SloPlan[] };

export interface SloTarget {
  availability: number;
  p95LatencyMs: number;
  /** 人工节点的按时完成率目标；不传则不参与整体判定。 */
  humanOnTimeRate?: number;
}

export interface NodeVerdict {
  id: string;
  kind: SloNodeKind;
  availability: number;
  p95LatencyMs: number;
  availabilityPass: boolean;
  latencyPass: boolean;
  pass: boolean;
}

export interface HumanRollup {
  count: number;
  onTimeRate: number;
  p95LatencyMs: number;
  nodes: string[];
}

export interface SloComposition {
  machineAvailability: number;
  machineP95LatencyMs: number;
  /** 含人工等待的墙钟 P95；不能与机器 P95 混在同一个 SLO 里。 */
  wallClockP95LatencyMs: number;
  costMicros: number;
  human: HumanRollup | null;
  nodes: NodeVerdict[];
  /** 机器节点中可用率最低者；并行结构下整体不会高于它。 */
  weakestNodeId: string | null;
  overallPass: boolean;
  /** 未达标的整体维度：availability / p95 / human。 */
  misses: string[];
  /** 自身未达标、却被整体指标掩盖的节点。 */
  masked: string[];
}

function fail(message: string): never {
  throw new AiError("invalid-request", `slo: ${message}`);
}

function ratio(value: number, field: string): number {
  if (!Number.isFinite(value) || value < 0 || value > 1) fail(`${field} 必须落在 [0, 1]`);
  return value;
}

function finite(value: number, field: string, min = 0): number {
  if (!Number.isFinite(value) || value < min) fail(`${field} 必须是 >= ${min} 的有限数`);
  return value;
}

function validateNode(node: NodeSlo): void {
  if (node.id.trim() === "") fail("node.id 不能为空");
  ratio(node.availability, `${node.id}.availability`);
  ratio(node.targetAvailability, `${node.id}.targetAvailability`);
  finite(node.p95LatencyMs, `${node.id}.p95LatencyMs`);
  finite(node.targetP95LatencyMs, `${node.id}.targetP95LatencyMs`);
  if (!Number.isInteger(node.costMicros) || node.costMicros < 0) {
    fail(`${node.id}.costMicros 必须是非负整数`);
  }
}

interface Folded {
  machineAvailability: number;
  machineP95LatencyMs: number;
  machineWallMs: number;
  costMicros: number;
  humans: NodeSlo[];
}

function combine(plan: SloPlan, verdicts: NodeVerdict[]): Folded {
  if (plan.kind === "node") {
    const node = plan.node;
    validateNode(node);
    verdicts.push({
      id: node.id,
      kind: node.kind,
      availability: node.availability,
      p95LatencyMs: node.p95LatencyMs,
      availabilityPass: node.availability >= node.targetAvailability,
      latencyPass: node.p95LatencyMs <= node.targetP95LatencyMs,
      pass: node.availability >= node.targetAvailability && node.p95LatencyMs <= node.targetP95LatencyMs,
    });
    if (node.kind === "human") {
      // 机器可用率不乘人工节点，但墙钟时间必须把等待加回去。
      return {
        machineAvailability: 1,
        machineP95LatencyMs: 0,
        machineWallMs: node.p95LatencyMs,
        costMicros: node.costMicros,
        humans: [node],
      };
    }
    return {
      machineAvailability: node.availability,
      machineP95LatencyMs: node.p95LatencyMs,
      machineWallMs: node.p95LatencyMs,
      costMicros: node.costMicros,
      humans: [],
    };
  }
  if (plan.children.length === 0) fail(`${plan.kind} 至少需要一个子计划`);

  if (plan.kind === "serial") {
    const folded = plan.children.map((child) => combine(child, verdicts));
    const handoff = finite(plan.handoffLatencyMs, "handoffLatencyMs");
    const boundaries = plan.children.length - 1;
    return {
      machineAvailability: folded.reduce((value, item) => value * item.machineAvailability, 1),
      machineP95LatencyMs: folded.reduce((value, item) => value + item.machineP95LatencyMs, 0) + handoff * boundaries,
      machineWallMs: folded.reduce((value, item) => value + item.machineWallMs, 0) + handoff * boundaries,
      costMicros: folded.reduce((value, item) => value + item.costMicros, 0),
      humans: folded.flatMap((item) => item.humans),
    };
  }

  const folded = plan.children.map((child) => combine(child, verdicts));
  const all = plan.join === "all";
  return {
    machineAvailability: all
      ? folded.reduce((value, item) => value * item.machineAvailability, 1)
      : 1 - folded.reduce((value, item) => value * (1 - item.machineAvailability), 1),
    machineP95LatencyMs: all
      ? Math.max(...folded.map((item) => item.machineP95LatencyMs))
      : Math.min(...folded.map((item) => item.machineP95LatencyMs)),
    machineWallMs: all
      ? Math.max(...folded.map((item) => item.machineWallMs))
      : Math.min(...folded.map((item) => item.machineWallMs)),
    costMicros: folded.reduce((value, item) => value + item.costMicros, 0),
    humans: folded.flatMap((item) => item.humans),
  };
}

/**
 * 串行可用率相乘、延迟相加；并行 all 汇合乘积、P95 取最大；
 * 人工节点不进入机器可用率，单独汇总按时率与等待时间。
 */
export function composeSlo(plan: SloPlan, target: SloTarget): SloComposition {
  ratio(target.availability, "target.availability");
  finite(target.p95LatencyMs, "target.p95LatencyMs");
  if (target.humanOnTimeRate !== undefined) ratio(target.humanOnTimeRate, "target.humanOnTimeRate");

  const verdicts: NodeVerdict[] = [];
  const folded = combine(plan, verdicts);
  const machineAvailability = Number(folded.machineAvailability.toFixed(9));
  const machineP95LatencyMs = Math.round(folded.machineP95LatencyMs);
  const humans = folded.humans;
  const human: HumanRollup | null = humans.length === 0
    ? null
    : {
      count: humans.length,
      onTimeRate: Math.min(...humans.map((node) => node.availability)),
      p95LatencyMs: Math.max(...humans.map((node) => node.p95LatencyMs)),
      nodes: humans.map((node) => node.id),
    };

  const misses: string[] = [];
  if (machineAvailability < target.availability) misses.push("availability");
  if (machineP95LatencyMs > target.p95LatencyMs) misses.push("p95");
  if (human && target.humanOnTimeRate !== undefined && human.onTimeRate < target.humanOnTimeRate) {
    misses.push("human");
  }
  const machineNodes = verdicts.filter((node) => node.kind !== "human");
  const weakest = machineNodes.reduce<NodeVerdict | null>(
    (low, node) => (low === null || node.availability < low.availability ? node : low),
    null,
  );
  const overallPass = misses.length === 0;
  return {
    machineAvailability,
    machineP95LatencyMs,
    wallClockP95LatencyMs: Math.round(folded.machineWallMs),
    costMicros: folded.costMicros,
    human,
    nodes: verdicts,
    weakestNodeId: weakest?.id ?? null,
    overallPass,
    misses,
    masked: overallPass ? verdicts.filter((node) => !node.pass).map((node) => node.id) : [],
  };
}