《后端架构设计·演进与治理》13.2 一次带 SLO 门禁与混沌演练的发布

// file: internal/evolve/releasegate.go
// tier: L1
// verify: go test ./internal/evolve/ -count=1 -run 'TestGate|TestChaos|TestDecide'
package evolve

import (
	"fmt"
	"sort"
	"time"
)

// SLIWindow is a count-based availability sample. Total is the number of
// eligible requests, Good the number that met the SLI.
type SLIWindow struct {
	Name  string
	Total int
	Good  int
}

// Availability returns Good/Total. An empty window reports 1; it carries no
// evidence and must be rejected by MinSamples, not silently accepted.
func (w SLIWindow) Availability() float64 {
	if w.Total == 0 {
		return 1
	}
	return float64(w.Good) / float64(w.Total)
}

// BurnRate is the observed error rate divided by the error budget (1-target).
// A burn of 1 means the window consumed exactly its budget.
func BurnRate(w SLIWindow, target float64) float64 {
	if target <= 0 || target >= 1 {
		return 0
	}
	return (1 - w.Availability()) / (1 - target)
}

// FreezeWindow blocks releases in a calendar interval, for example a campaign
// or a finance close.
type FreezeWindow struct {
	Start  time.Time
	End    time.Time
	Reason string
}

// GateConfig is the release policy. Fast and slow windows catch different
// failures: a spike and a slow leak.
type GateConfig struct {
	Target             float64
	FastBurn           float64
	SlowBurn           float64
	MinSamples         int
	MinBudgetRemaining float64
	Freeze             []FreezeWindow
}

// GateResult carries the decision and every number that produced it, so a
// rejection can be explained without re-querying the metrics backend.
type GateResult struct {
	Allow     bool
	FastBurn  float64
	SlowBurn  float64
	Remaining float64
	Reasons   []string
}

// EvaluateGate applies all rules and reports every violation. The order of
// reasons is sorted to keep output stable for tests and logs.
func EvaluateGate(cfg GateConfig, fast, slow SLIWindow, remaining float64, now time.Time) GateResult {
	res := GateResult{
		FastBurn:  BurnRate(fast, cfg.Target),
		SlowBurn:  BurnRate(slow, cfg.Target),
		Remaining: remaining,
	}
	if cfg.Target <= 0 || cfg.Target >= 1 {
		res.Reasons = append(res.Reasons, fmt.Sprintf("invalid SLO target %.4f", cfg.Target))
	}
	if cfg.FastBurn <= 0 || cfg.SlowBurn <= 0 {
		res.Reasons = append(res.Reasons, "burn thresholds must be positive")
	}
	if fast.Total < cfg.MinSamples || slow.Total < cfg.MinSamples {
		res.Reasons = append(res.Reasons, fmt.Sprintf("insufficient samples: fast=%d slow=%d min=%d", fast.Total, slow.Total, cfg.MinSamples))
	}
	if res.FastBurn > cfg.FastBurn {
		res.Reasons = append(res.Reasons, fmt.Sprintf("fast burn %.2f exceeds %.2f", res.FastBurn, cfg.FastBurn))
	}
	if res.SlowBurn > cfg.SlowBurn {
		res.Reasons = append(res.Reasons, fmt.Sprintf("slow burn %.2f exceeds %.2f", res.SlowBurn, cfg.SlowBurn))
	}
	if remaining < cfg.MinBudgetRemaining {
		res.Reasons = append(res.Reasons, fmt.Sprintf("budget remaining %.2f below %.2f", remaining, cfg.MinBudgetRemaining))
	}
	for _, f := range cfg.Freeze {
		if !now.Before(f.Start) && now.Before(f.End) {
			res.Reasons = append(res.Reasons, fmt.Sprintf("freeze window: %s", f.Reason))
			break
		}
	}
	sort.Strings(res.Reasons)
	res.Allow = len(res.Reasons) == 0
	return res
}

// ChaosStep is one fault and the steady-state assertion that must hold while
// the fault is active. AbortOnFail stops the whole plan on violation.
type ChaosStep struct {
	Name        string
	Action      func() error
	Assert      func() error
	AbortOnFail bool
}

// ChaosPlan is a bounded experiment: hypothesis first, blast radius second,
// actions last.
type ChaosPlan struct {
	Name        string
	Hypothesis  string
	BlastRadius string
	Steps       []ChaosStep
}

// ChaosReport is the audit record of one run.
type ChaosReport struct {
	Name     string
	Executed int
	Passed   bool
	Aborted  bool
	Failures []string
}

// RunChaos refuses a plan that has no hypothesis or no declared blast radius;
// an unbounded experiment is not an experiment, it is an incident.
func RunChaos(p ChaosPlan) ChaosReport {
	rep := ChaosReport{Name: p.Name}
	if p.Hypothesis == "" {
		rep.Failures = append(rep.Failures, "missing steady-state hypothesis")
	}
	if p.BlastRadius == "" {
		rep.Failures = append(rep.Failures, "missing blast radius")
	}
	if len(p.Steps) == 0 {
		rep.Failures = append(rep.Failures, "no steps")
	}
	if len(rep.Failures) > 0 {
		return rep
	}
	for _, step := range p.Steps {
		rep.Executed++
		var err error
		if step.Action != nil {
			err = step.Action()
		}
		if err == nil && step.Assert != nil {
			err = step.Assert()
		}
		if err != nil {
			rep.Failures = append(rep.Failures, fmt.Sprintf("%s: %v", step.Name, err))
			if step.AbortOnFail {
				rep.Aborted = true
				break
			}
		}
	}
	rep.Passed = len(rep.Failures) == 0
	return rep
}

// ReleaseDecision combines both gates. Chaos is not run when the SLO gate
// already rejects the release: injecting faults cannot fix a bad burn rate.
type ReleaseDecision struct {
	Allow   bool
	Gate    GateResult
	Chaos   ChaosReport
	Reasons []string
}

// DecideRelease evaluates the SLO gate, then the chaos plan, and allows the
// release only when both pass.
func DecideRelease(cfg GateConfig, fast, slow SLIWindow, remaining float64, now time.Time, plan ChaosPlan) ReleaseDecision {
	d := ReleaseDecision{Gate: EvaluateGate(cfg, fast, slow, remaining, now)}
	if !d.Gate.Allow {
		d.Reasons = append(d.Reasons, "slo gate rejected release")
		sort.Strings(d.Reasons)
		return d
	}
	d.Chaos = RunChaos(plan)
	if !d.Chaos.Passed {
		d.Reasons = append(d.Reasons, "chaos experiment did not pass")
	}
	sort.Strings(d.Reasons)
	d.Allow = len(d.Reasons) == 0
	return d
}

// file: internal/evolve/releasegate_test.go
// tier: L1
// verify: go test ./internal/evolve/ -count=1 -run 'TestGate|TestChaos|TestDecide'
var releaseAt = time.Date(2026, 11, 2, 18, 10, 0, 0, time.UTC)

func gateConfig() GateConfig {
	return GateConfig{
		Target:             0.999,
		FastBurn:           2,
		SlowBurn:           1,
		MinSamples:         1000,
		MinBudgetRemaining: 0.10,
	}
}

func TestGateRejectsFastBurn(t *testing.T) {
	fast := SLIWindow{Name: "fast-1h", Total: 10000, Good: 9900}
	slow := SLIWindow{Name: "slow-6h", Total: 100000, Good: 99950}
	got := EvaluateGate(gateConfig(), fast, slow, 0.75, releaseAt)
	if got.Allow {
		t.Fatal("allowed a 10x fast burn")
	}
	if !contains(got.Reasons, "fast burn 10.00 exceeds 2.00") {
		t.Fatalf("reasons = %v", got.Reasons)
	}
}

func TestChaosAbortsOnSteadyStateViolation(t *testing.T) {
	ranThird := false
	plan := ChaosPlan{
		Name:        "payment-latency",
		Hypothesis:  "order success stays above 99.9% while one payment replica is slow",
		BlastRadius: "payment-service/1 of 4 replicas",
		Steps: []ChaosStep{
			{Name: "baseline", AbortOnFail: true},
			{Name: "inject-3000ms", Assert: func() error { return errors.New("success rate 99.1%") }, AbortOnFail: true},
			{Name: "inject-5000ms", Action: func() error { ranThird = true; return nil }},
		},
	}
	rep := RunChaos(plan)
	if rep.Passed || !rep.Aborted {
		t.Fatalf("report = %+v, want failed and aborted", rep)
	}
	if rep.Executed != 2 || ranThird {
		t.Fatalf("executed = %d, third ran = %v", rep.Executed, ranThird)
	}
	if !strings.Contains(rep.Failures[0], "success rate 99.1%") {
		t.Fatalf("failures = %v", rep.Failures)
	}
}

func TestDecideReleaseSkipsChaosWhenGateFails(t *testing.T) {
	ran := false
	plan := ChaosPlan{
		Name:        "should-not-run",
		Hypothesis:  "h",
		BlastRadius: "b",
		Steps:       []ChaosStep{{Name: "step", Action: func() error { ran = true; return nil }}},
	}
	fast := SLIWindow{Name: "fast-1h", Total: 10, Good: 10}
	d := DecideRelease(gateConfig(), fast, fast, 0.75, releaseAt, plan)
	if d.Allow || ran {
		t.Fatalf("allow = %v, chaos ran = %v", d.Allow, ran)
	}
	if !contains(d.Reasons, "slo gate rejected release") {
		t.Fatalf("reasons = %v", d.Reasons)
	}
}

$ go test ./internal/evolve/ -count=1 -v -run 'TestGate|TestChaos|TestDecide'
=== RUN   TestGateAllowsHealthyWindows
--- PASS: TestGateAllowsHealthyWindows (0.00s)
=== RUN   TestGateRejectsFastBurn
--- PASS: TestGateRejectsFastBurn (0.00s)
=== RUN   TestGateRejectsInsufficientSamples
--- PASS: TestGateRejectsInsufficientSamples (0.00s)
=== RUN   TestGateRejectsFreezeWindow
--- PASS: TestGateRejectsFreezeWindow (0.00s)
=== RUN   TestGateRejectsExhaustedBudget
--- PASS: TestGateRejectsExhaustedBudget (0.00s)
=== RUN   TestChaosAbortsOnSteadyStateViolation
--- PASS: TestChaosAbortsOnSteadyStateViolation (0.00s)
=== RUN   TestChaosRejectsUnboundedPlan
--- PASS: TestChaosRejectsUnboundedPlan (0.00s)
=== RUN   TestChaosPassesBoundedPlan
--- PASS: TestChaosPassesBoundedPlan (0.00s)
=== RUN   TestDecideReleaseSkipsChaosWhenGateFails
--- PASS: TestDecideReleaseSkipsChaosWhenGateFails (0.00s)
=== RUN   TestDecideReleaseRequiresBothGates
--- PASS: TestDecideReleaseRequiresBothGates (0.00s)
PASS
ok  	archlab/internal/evolve	0.241s