《游戏服务端架构·平台化与架构治理》5.3 告警设计与值班

// file: internal/alertoncall/router.go
// tier: L1
// verify: go test ./internal/alertoncall/
package alertoncall

import (
	"fmt"
	"time"
)

// Window is a maintenance interval: start inclusive, end exclusive.
type Window struct {
	Start time.Time
	End   time.Time
}

func (w Window) Contains(t time.Time) bool {
	return !t.Before(w.Start) && t.Before(w.End)
}

func (w Window) Validate() error {
	if !w.Start.Before(w.End) {
		return fmt.Errorf("alertoncall: window start must precede end")
	}
	return nil
}

type Policy struct {
	DedupWindow  time.Duration
	MuteWindows  []Window
	InhibitLower bool
}

type Router struct {
	policy Policy
	seen   map[string]time.Time
	counts map[string]int
	active map[string]Alert
}

func New(p Policy) (*Router, error) {
	if p.DedupWindow < 0 {
		return nil, fmt.Errorf("alertoncall: dedup window must be >= 0")
	}
	for _, w := range p.MuteWindows {
		if err := w.Validate(); err != nil {
			return nil, err
		}
	}
	return &Router{
		policy: p,
		seen:   make(map[string]time.Time),
		counts: make(map[string]int),
		active: make(map[string]Alert),
	}, nil
}

// Decision explains why an alert was or was not delivered. Count is how many
// occurrences were folded into the current notification.
type Decision struct {
	Deliver bool
	Reason  string
	Count   int
}

func (r *Router) Ingest(a Alert) (Decision, error) {
	if err := a.Validate(); err != nil {
		return Decision{}, err
	}
	for _, w := range r.policy.MuteWindows {
		if w.Contains(a.StartedAt) {
			return Decision{Reason: "maintenance-window", Count: 1}, nil
		}
	}
	if prev, ok := r.seen[a.Key]; ok {
		delta := a.StartedAt.Sub(prev)
		if delta >= 0 && delta <= r.policy.DedupWindow {
			r.counts[a.Key]++
			return Decision{Reason: "deduplicated", Count: r.counts[a.Key]}, nil
		}
	}
	if r.policy.InhibitLower && a.Symptom {
		if sev, ok := r.highest(a.Service); ok && sev.rank() < a.Severity.rank() {
			return Decision{Reason: "inhibited-by-" + string(sev), Count: 1}, nil
		}
	}
	r.seen[a.Key] = a.StartedAt
	r.counts[a.Key] = 1
	r.active[a.Key] = a
	return Decision{Deliver: true, Reason: "deliver", Count: 1}, nil
}

func (r *Router) highest(service string) (Severity, bool) {
	var best Severity
	found := false
	for _, a := range r.active {
		if a.Service != service {
			continue
		}
		if !found || a.Severity.rank() < best.rank() {
			best, found = a.Severity, true
		}
	}
	return best, found
}

// Resolve removes a firing alert. Dedup history is kept so a flapping alert
// does not page again just because it recovered for one evaluation cycle.
func (r *Router) Resolve(key string) {
	delete(r.active, key)
}

// file: internal/alertoncall/escalation.go
// tier: L1
// verify: go test ./internal/alertoncall/
package alertoncall

import (
	"fmt"
	"time"
)

type Step struct {
	After  time.Duration
	Target string
}

type Plan struct {
	Steps []Step
}

func (p Plan) Validate() error {
	if len(p.Steps) == 0 {
		return fmt.Errorf("alertoncall: plan has no steps")
	}
	for i, s := range p.Steps {
		if s.Target == "" {
			return fmt.Errorf("alertoncall: step %d needs a target", i)
		}
		if i == 0 && s.After != 0 {
			return fmt.Errorf("alertoncall: first step must start at zero")
		}
		if i > 0 && s.After <= p.Steps[i-1].After {
			return fmt.Errorf("alertoncall: step %d must be later than step %d", i, i-1)
		}
	}
	return nil
}

// At returns the step responsible after elapsed time. The last matching step
// wins, so the plan can move from team to lead to incident commander.
func (p Plan) At(elapsed time.Duration) (Step, bool) {
	var out Step
	found := false
	for _, s := range p.Steps {
		if elapsed >= s.After {
			out, found = s, true
		}
	}
	return out, found
}

// Next returns the next step and how long the caller must wait from now. ok is
// false when the plan is exhausted.
func (p Plan) Next(elapsed time.Duration) (Step, time.Duration, bool) {
	for _, s := range p.Steps {
		if s.After > elapsed {
			return s, s.After - elapsed, true
		}
	}
	return Step{}, 0, false
}

// Breach reports whether an alert was acknowledged later than its severity
// allows. A zero ack means it is still unacknowledged.
func Breach(a Alert, ackedAt, now time.Time) (bool, error) {
	limit, err := a.Severity.Response()
	if err != nil {
		return false, err
	}
	if ackedAt.IsZero() {
		return now.Sub(a.StartedAt) > limit, nil
	}
	return ackedAt.Sub(a.StartedAt) > limit, nil
}