《后端架构设计·高性能与高可用》10.2 告警：从症状到行动的闭环

// file: internal/platform/alert/alert.go
// tier: L1
// verify: go test ./internal/platform/alert/
package alert

import (
	"errors"
	"fmt"
	"math"
	"sort"
	"strings"
	"time"
)

type Rule struct {
	ID        string
	Signal    string
	Above     bool
	Threshold float64
	For       time.Duration
	Severity  string
	Runbook   string
	Inhibits  []string
}

type Notification struct {
	RuleID       string
	Signal       string
	Severity     string
	Runbook      string
	Kind         string
	Value        float64
	At           time.Time
	SuppressedBy string
}

type state struct {
	breach int
	good   int
	firing bool
}

type Engine struct {
	Rules []Rule
	Tick  time.Duration
	state map[string]*state
}

// New 在加载期拒绝“没有 runbook 的告警”：告警如果不能导向动作，就只是噪音。
func New(rules []Rule, tick time.Duration) (*Engine, error) {
	if tick <= 0 {
		return nil, errors.New("tick must be positive")
	}
	seen := map[string]bool{}
	for _, r := range rules {
		switch {
		case r.ID == "" || r.Signal == "":
			return nil, errors.New("rule id and signal are required")
		case seen[r.ID]:
			return nil, fmt.Errorf("duplicate rule id %q", r.ID)
		case !strings.HasPrefix(r.Runbook, "RB-"):
			return nil, fmt.Errorf("rule %s must bind a RB-* runbook", r.ID)
		case r.Severity != "page" && r.Severity != "ticket":
			return nil, fmt.Errorf("rule %s has unknown severity %q", r.ID, r.Severity)
		case r.For <= 0:
			return nil, fmt.Errorf("rule %s must have a positive for duration", r.ID)
		}
		seen[r.ID] = true
	}
	for _, r := range rules {
		for _, id := range r.Inhibits {
			if !seen[id] {
				return nil, fmt.Errorf("rule %s inhibits unknown rule %s", r.ID, id)
			}
		}
	}
	return &Engine{Rules: rules, Tick: tick, state: map[string]*state{}}, nil
}

func (e *Engine) ticks(r Rule) int {
	n := int(math.Ceil(float64(r.For) / float64(e.Tick)))
	if n < 1 {
		n = 1
	}
	return n
}

// Observe 每 Tick 调用一次。只有连续越线达到 For 才触发；恢复需要连续两个
// 正常窗口，避免边界抖动造成“告警-恢复”刷屏。
func (e *Engine) Observe(at time.Time, values map[string]float64) []Notification {
	var out []Notification
	firing := map[string]bool{}
	for _, r := range e.Rules {
		v, ok := values[r.Signal]
		if !ok {
			continue // 指标缺失不等于健康，保持原状态，等待显式恢复或缺失告警。
		}
		st := e.state[r.ID]
		if st == nil {
			st = &state{}
			e.state[r.ID] = st
		}
		breach := v >= r.Threshold
		if !r.Above {
			breach = v <= r.Threshold
		}
		if breach {
			st.breach++
			st.good = 0
		} else {
			st.breach = 0
			st.good++
			if st.firing && st.good >= 2 {
				st.firing = false
				out = append(out, Notification{RuleID: r.ID, Signal: r.Signal, Severity: r.Severity,
					Runbook: r.Runbook, Kind: "resolved", Value: v, At: at})
			}
		}
		if breach && !st.firing && st.breach >= e.ticks(r) {
			st.firing = true
			out = append(out, Notification{RuleID: r.ID, Signal: r.Signal, Severity: r.Severity,
				Runbook: r.Runbook, Kind: "firing", Value: v, At: at})
		}
		if st.firing {
			firing[r.ID] = true
		}
	}
	// 抑制只影响“是否送达”，不改变规则自身的 firing 状态，
	// 这样下游恢复后上游可以自然重新评估，而不是丢掉状态。
	for i := range out {
		if out[i].Kind != "firing" {
			continue
		}
		for _, r := range e.Rules {
			if firing[r.ID] && contains(r.Inhibits, out[i].RuleID) {
				out[i].SuppressedBy = r.ID
				break
			}
		}
	}
	return out
}

func contains(list []string, want string) bool {
	for _, s := range list {
		if s == want {
			return true
		}
	}
	return false
}

// Histogram 是 P99 的来源。分桶必须有界且有序，跨实例不能直接平均 P99。
type Histogram struct {
	bounds []float64
	counts []uint64
	total  uint64
}

func NewHistogram() *Histogram {
	return &Histogram{
		bounds: []float64{5, 10, 25, 50, 100, 250, 500, 1000, 2500, 5000},
		counts: make([]uint64, 11),
	}
}

func (h *Histogram) Observe(ms float64) {
	i := sort.SearchFloat64s(h.bounds, ms)
	h.counts[i]++
	h.total++
}

func (h *Histogram) Quantile(q float64) float64 {
	if h.total == 0 {
		return 0
	}
	target := uint64(math.Ceil(q * float64(h.total)))
	var seen uint64
	for i, c := range h.counts {
		seen += c
		if seen >= target {
			if i < len(h.bounds) {
				return h.bounds[i]
			}
			return h.bounds[len(h.bounds)-1]
		}
	}
	return h.bounds[len(h.bounds)-1]
}

type Window struct {
	Requests, ServerErrors           uint64
	ReserveAttempts, ReserveFailures uint64
	Latency                          *Histogram
	OldestOutbox                     time.Duration
}

func (w *Window) Snapshots() map[string]float64 {
	s := map[string]float64{
		"outbox.backlog.age_seconds": w.OldestOutbox.Seconds(),
		"order.http.latency.p99_ms":  w.Latency.Quantile(0.99),
	}
	if w.Requests > 0 {
		s["order.http.5xx.ratio"] = float64(w.ServerErrors) / float64(w.Requests)
	}
	if w.ReserveAttempts > 0 {
		s["inventory.reserve.failure_ratio"] = float64(w.ReserveFailures) / float64(w.ReserveAttempts)
	}
	return s
}

$ go test -v ./... -count=1
=== RUN   TestSpikeAndSustained
--- PASS: TestSpikeAndSustained (0.00s)
=== RUN   TestInhibitionAndRouting
    alert_test.go:86: p99_ms=1000 5xx=0.12 outbox_s=300 reserve=0.15 rules=4 suppressed=1
--- PASS: TestInhibitionAndRouting (0.00s)
=== RUN   TestRuleMustBindRunbook
--- PASS: TestRuleMustBindRunbook (0.00s)
PASS
ok  	archlab/internal/platform/alert	0.448s