《互联网产品的客户端架构·规模化与架构演进》7.3 从告警到定位的闭环

// file: src/observability/alert-loop.ts
// tier: L1
// verify: node --test test/observability/alert-loop.test.ts

export type Severity = "warning" | "critical";
export type Comparison = ">" | ">=" | "<" | "<=";
export type Aggregate = "avg" | "max" | "min";
export type AlertStatus = "firing" | "acked" | "resolved";

export type Sample = { at: number; value: number };

export type AlertRule = {
  id: string;
  metric: string;
  comparison: Comparison;
  threshold: number;
  windowMs: number;
  aggregate: Aggregate;
  severity: Severity;
  owner: string;
  runbookId: string;
};

export type Alert = {
  id: string;
  ruleId: string;
  fingerprint: string;
  severity: Severity;
  owner: string;
  runbookId: string;
  labels: Record<string, string>;
  firedAt: number;
  status: AlertStatus;
  ackedAt?: number;
  resolvedAt?: number;
  unclosed?: boolean;
};

export type AlertGroup = {
  fingerprint: string;
  ruleId: string;
  severity: Severity;
  owner: string;
  count: number;
  firstAt: number;
  lastAt: number;
  status: AlertStatus;
  ids: string[];
};

export type Route = { match: Record<string, string>; owner: string };
export type Runbook = { id: string; title: string; url: string; steps: string[] };
export type LoopReport = {
  total: number;
  open: number;
  resolved: number;
  unclosed: number;
  missingRunbook: number;
  avgMttrMs: number | null;
};
export type RuleEvaluation = { firing: boolean; observed: number | null; windowSize: number };
export function aggregateValues(values: number[], mode: Aggregate): number | null {
  if (values.length === 0) {
    return null;
  }
  if (mode === "max") {
    return Math.max(...values);
  }
  if (mode === "min") {
    return Math.min(...values);
  }
  return values.reduce((sum, value) => sum + value, 0) / values.length;
}

export function compare(observed: number, comparison: Comparison, threshold: number): boolean {
  if (comparison === ">") {
    return observed > threshold;
  }
  if (comparison === ">=") {
    return observed >= threshold;
  }
  if (comparison === "<") {
    return observed < threshold;
  }
  return observed <= threshold;
}

export function evaluateRule(rule: AlertRule, samples: Sample[], now: number): RuleEvaluation {
  const from = now - rule.windowMs;
  const windowValues = samples.filter((s) => s.at >= from && s.at <= now).map((s) => s.value);
  const observed = aggregateValues(windowValues, rule.aggregate);
  if (observed === null) {
    return { firing: false, observed: null, windowSize: 0 };
  }
  return { firing: compare(observed, rule.comparison, rule.threshold), observed, windowSize: windowValues.length };
}

export function fingerprint(ruleId: string, labels: Record<string, string>): string {
  const pairs = Object.keys(labels).sort().map((key) => `${key}=${labels[key]}`);
  return `${ruleId}|${pairs.join(",")}`;
}

export function createAlert(
  rule: AlertRule,
  labels: Record<string, string>,
  firedAt: number,
  id?: string,
): Alert {
  const key = fingerprint(rule.id, labels);
  return {
    id: id ?? `${key}@${firedAt}`,
    ruleId: rule.id,
    fingerprint: key,
    severity: rule.severity,
    owner: rule.owner,
    runbookId: rule.runbookId,
    labels: { ...labels },
    firedAt,
    status: "firing",
  };
}

function mergeStatus(current: AlertStatus, incoming: AlertStatus): AlertStatus {
  if (current === "firing" || incoming === "firing") {
    return "firing";
  }
  if (current === "acked" || incoming === "acked") {
    return "acked";
  }
  return "resolved";
}

export function dedupe(alerts: Alert[], windowMs = 5 * 60 * 1000): AlertGroup[] {
  const sorted = [...alerts].sort((a, b) => a.firedAt - b.firedAt || a.id.localeCompare(b.id));
  const groups: AlertGroup[] = [];
  for (const alert of sorted) {
    const windowStart = alert.firedAt - windowMs;
    let target: AlertGroup | undefined;
    for (let i = groups.length - 1; i >= 0; i -= 1) {
      const group = groups[i];
      if (group.fingerprint === alert.fingerprint && group.firstAt >= windowStart) {
        target = group;
        break;
      }
      if (group.firstAt < windowStart) {
        break;
      }
    }
    if (!target) {
      groups.push({
        fingerprint: alert.fingerprint,
        ruleId: alert.ruleId,
        severity: alert.severity,
        owner: alert.owner,
        count: 1,
        firstAt: alert.firedAt,
        lastAt: alert.firedAt,
        status: alert.status,
        ids: [alert.id],
      });
      continue;
    }
    target.count += 1;
    target.lastAt = alert.firedAt;
    target.status = mergeStatus(target.status, alert.status);
    target.ids.push(alert.id);
    if (alert.severity === "critical") {
      target.severity = "critical";
    }
  }
  return groups.sort((a, b) => a.firstAt - b.firstAt || a.fingerprint.localeCompare(b.fingerprint));
}

export function routeOwner(labels: Record<string, string>, routes: Route[], fallback: string): string {
  let bestOwner = fallback;
  let bestScore = -1;
  for (const route of routes) {
    const keys = Object.keys(route.match);
    const matched = keys.every((key) => labels[key] === route.match[key]);
    if (matched && keys.length > bestScore) {
      bestScore = keys.length;
      bestOwner = route.owner;
    }
  }
  return bestOwner;
}

export function runbookFor(rule: AlertRule, runbooks: Runbook[]): Runbook | undefined {
  return runbooks.find((runbook) => runbook.id === rule.runbookId);
}

export function acknowledge(alert: Alert, at: number): Alert {
  if (alert.status !== "firing") {
    throw new Error(`告警 ${alert.id} 当前状态为 ${alert.status}，不能确认`);
  }
  return { ...alert, status: "acked", ackedAt: at };
}

export function resolve(alert: Alert, at: number): Alert {
  if (alert.status === "resolved") {
    throw new Error(`告警 ${alert.id} 已经关闭`);
  }
  return { ...alert, status: "resolved", resolvedAt: at };
}

export function unclosedAlerts(alerts: Alert[], now: number, slaMs: number): Alert[] {
  return alerts
    .filter((alert) => alert.status !== "resolved" && now - alert.firedAt >= slaMs)
    .map((alert) => ({ ...alert, unclosed: true }));
}

export function loopReport(alerts: Alert[], now: number, slaMs: number): LoopReport {
  const resolved = alerts.filter((alert) => alert.status === "resolved" && alert.resolvedAt !== undefined);
  const unclosed = unclosedAlerts(alerts, now, slaMs).length;
  const durations = resolved.map((alert) => (alert.resolvedAt as number) - alert.firedAt);
  return {
    total: alerts.length,
    open: alerts.filter((alert) => alert.status !== "resolved").length,
    resolved: resolved.length,
    unclosed,
    missingRunbook: alerts.filter((alert) => !alert.runbookId).length,
    avgMttrMs: durations.length === 0 ? null : durations.reduce((sum, value) => sum + value, 0) / durations.length,
  };
}

✔ 平均值聚合 (0.559709ms)
✔ 最大值与最小值聚合 (0.062042ms)
✔ 空样本返回 null (0.047917ms)
✔ 窗口外的样本不参与计算 (0.100833ms)
✔ 阈值比较支持四个方向 (0.061292ms)
✔ 聚合值超过阈值时触发 (0.068625ms)
✔ 没有样本时不触发 (0.385959ms)
✔ 指纹与标签顺序无关 (0.077791ms)
✔ 不同规则产生不同指纹 (0.078625ms)
✔ createAlert 继承规则并复制标签 (0.1235ms)
✔ 同一指纹在窗口内合并为一条 (0.143417ms)
✔ 超出窗口后另起一组 (0.093583ms)
✔ 聚合组状态优先保留 firing (0.06975ms)
✔ 聚合组严重度升级为 critical (0.053167ms)
✔ 路由选择最具体的匹配 (0.081333ms)
✔ 没有匹配时回落到默认 owner (0.04225ms)
✔ 同具体度按声明顺序取第一个 (0.035ms)
✔ 规则可以映射到 runbook (0.057833ms)
✔ 未知 runbook 返回 undefined (0.037042ms)
✔ 确认告警进入 acked 并记录时间 (0.054709ms)
✔ 重复确认被拒绝 (0.187875ms)
✔ 关闭告警并计算 MTTR (0.093459ms)
✔ 重复关闭被拒绝 (0.046542ms)
✔ 超过 SLA 未关闭的告警被标记 (0.0485ms)
✔ 闭环报告汇总未闭环与缺失 runbook (0.041584ms)
✔ 没有已关闭告警时 MTTR 为 null (0.030209ms)
ℹ tests 26
ℹ suites 0
ℹ pass 26
ℹ fail 0
ℹ cancelled 0
ℹ skipped 0
ℹ todo 0
ℹ duration_ms 122.132208