《互联网产品的服务端架构·产品、流量与基础能力》11.3 一次线上问题定位复盘

// file: internal/obs/incident.go
// tier: L1
// verify: go test ./internal/obs/

package obs

import "fmt"

// Phase 是线上问题处理时间线上的四个阶段。
type Phase string

const (
	PhaseDetect   Phase = "detect"
	PhaseAck      Phase = "ack"
	PhaseMitigate Phase = "mitigate"
	PhaseResolve  Phase = "resolve"
)

// Event 是时间线上的一次状态变化，At 是 Unix 秒。
type Event struct {
	At    int64
	Phase Phase
	Note  string
}

// Report 把一次问题的处理过程拆成三段耗时，单位与 Event.At 一致。
type Report struct {
	DetectToAck       int64
	AckToMitigate     int64
	MitigateToResolve int64
}

// Analyze 从事件序列还原处理时间线。
// 四个阶段必须各出现一次且顺序正确，时间必须非递减；
// 任一条件不满足都返回错误和空的 Report，不返回半截结果。
func Analyze(events []Event) (Report, error) {
	order := []Phase{PhaseDetect, PhaseAck, PhaseMitigate, PhaseResolve}
	position := make(map[Phase]int, len(order))

	var prev int64
	for i, ev := range events {
		if i > 0 && ev.At < prev {
			return Report{}, fmt.Errorf("obs: event %d goes back in time: %d < %d", i, ev.At, prev)
		}
		prev = ev.At

		if !validPhase(ev.Phase) {
			return Report{}, fmt.Errorf("obs: event %d has unknown phase %q", i, ev.Phase)
		}
		if _, ok := position[ev.Phase]; ok {
			return Report{}, fmt.Errorf("obs: phase %q appears more than once", ev.Phase)
		}
		position[ev.Phase] = i
	}

	for _, p := range order {
		if _, ok := position[p]; !ok {
			return Report{}, fmt.Errorf("obs: phase %q missing", p)
		}
	}
	for i := 1; i < len(order); i++ {
		if position[order[i]] < position[order[i-1]] {
			return Report{}, fmt.Errorf("obs: phase %q appears before %q", order[i], order[i-1])
		}
	}

	detect := events[position[PhaseDetect]].At
	ack := events[position[PhaseAck]].At
	mitigate := events[position[PhaseMitigate]].At
	resolve := events[position[PhaseResolve]].At

	return Report{
		DetectToAck:       ack - detect,
		AckToMitigate:     mitigate - ack,
		MitigateToResolve: resolve - mitigate,
	}, nil
}

func validPhase(p Phase) bool {
	switch p {
	case PhaseDetect, PhaseAck, PhaseMitigate, PhaseResolve:
		return true
	default:
		return false
	}
}

// file: internal/obs/incident_test.go
// tier: L1
// verify: go test ./internal/obs/

package obs

import (
	"reflect"
	"testing"
)

func TestAnalyzeBuildsTimeline(t *testing.T) {
	events := []Event{
		{At: 1000, Phase: PhaseDetect, Note: "告警触发"},
		{At: 1120, Phase: PhaseAck, Note: "值班确认"},
		{At: 1900, Phase: PhaseMitigate, Note: "切流量"},
		{At: 2800, Phase: PhaseResolve, Note: "索引重建完成"},
	}

	got, err := Analyze(events)
	if err != nil {
		t.Fatal(err)
	}
	want := Report{DetectToAck: 120, AckToMitigate: 780, MitigateToResolve: 900}
	if !reflect.DeepEqual(got, want) {
		t.Fatalf("report = %+v, want %+v", got, want)
	}
}

func TestAnalyzeAllowsEqualTimestamps(t *testing.T) {
	events := []Event{
		{At: 10, Phase: PhaseDetect},
		{At: 10, Phase: PhaseAck},
		{At: 20, Phase: PhaseMitigate},
		{At: 20, Phase: PhaseResolve},
	}
	got, err := Analyze(events)
	if err != nil {
		t.Fatal(err)
	}
	want := Report{DetectToAck: 0, AckToMitigate: 10, MitigateToResolve: 0}
	if !reflect.DeepEqual(got, want) {
		t.Fatalf("report = %+v, want %+v", got, want)
	}
}

func TestAnalyzeRejectsIncompleteOrDisorderedTimeline(t *testing.T) {
	cases := []struct {
		name   string
		events []Event
	}{
		{
			name: "missing resolve",
			events: []Event{
				{At: 1, Phase: PhaseDetect},
				{At: 2, Phase: PhaseAck},
				{At: 3, Phase: PhaseMitigate},
			},
		},
		{
			name: "wrong order",
			events: []Event{
				{At: 1, Phase: PhaseDetect},
				{At: 2, Phase: PhaseMitigate},
				{At: 3, Phase: PhaseAck},
				{At: 4, Phase: PhaseResolve},
			},
		},
		{
			name: "duplicate phase",
			events: []Event{
				{At: 1, Phase: PhaseDetect},
				{At: 2, Phase: PhaseAck},
				{At: 3, Phase: PhaseAck},
				{At: 4, Phase: PhaseMitigate},
				{At: 5, Phase: PhaseResolve},
			},
		},
		{
			name: "time goes back",
			events: []Event{
				{At: 5, Phase: PhaseDetect},
				{At: 4, Phase: PhaseAck},
				{At: 6, Phase: PhaseMitigate},
				{At: 7, Phase: PhaseResolve},
			},
		},
		{
			name: "unknown phase",
			events: []Event{
				{At: 1, Phase: PhaseDetect},
				{At: 2, Phase: Phase("diagnose")},
				{At: 3, Phase: PhaseResolve},
			},
		},
	}

	for _, tc := range cases {
		t.Run(tc.name, func(t *testing.T) {
			got, err := Analyze(tc.events)
			if err == nil {
				t.Fatal("want error")
			}
			if got != (Report{}) {
				t.Fatalf("report = %+v, want zero report on error", got)
			}
		})
	}
}

cd .local/checks/webapp-lab
go test ./internal/obs/ -run 'TestAnalyze' -count=1 -v

=== RUN   TestAnalyzeBuildsTimeline
--- PASS: TestAnalyzeBuildsTimeline (0.00s)
=== RUN   TestAnalyzeAllowsEqualTimestamps
--- PASS: TestAnalyzeAllowsEqualTimestamps (0.00s)
=== RUN   TestAnalyzeRejectsIncompleteOrDisorderedTimeline
=== RUN   TestAnalyzeRejectsIncompleteOrDisorderedTimeline/missing_resolve
=== RUN   TestAnalyzeRejectsIncompleteOrDisorderedTimeline/wrong_order
=== RUN   TestAnalyzeRejectsIncompleteOrDisorderedTimeline/duplicate_phase
=== RUN   TestAnalyzeRejectsIncompleteOrDisorderedTimeline/time_goes_back
=== RUN   TestAnalyzeRejectsIncompleteOrDisorderedTimeline/unknown_phase
--- PASS: TestAnalyzeRejectsIncompleteOrDisorderedTimeline (0.00s)
    --- PASS: TestAnalyzeRejectsIncompleteOrDisorderedTimeline/missing_resolve (0.00s)
    --- PASS: TestAnalyzeRejectsIncompleteOrDisorderedTimeline/wrong_order (0.00s)
    --- PASS: TestAnalyzeRejectsIncompleteOrDisorderedTimeline/duplicate_phase (0.00s)
    --- PASS: TestAnalyzeRejectsIncompleteOrDisorderedTimeline/time_goes_back (0.00s)
    --- PASS: TestAnalyzeRejectsIncompleteOrDisorderedTimeline/unknown_phase (0.00s)
PASS
ok  	webapp/internal/obs	0.219s