《游戏服务端架构·平台化与架构治理》6.3 事故分级与响应

// file: internal/incidentseverity/severity.go
// tier: L1
// verify: go test ./internal/incidentseverity/

package incidentseverity

import (
	"errors"
	"fmt"
	"time"
)

var (
	ErrInvalidInput = errors.New("incidentseverity: invalid input")
	ErrAlreadyAtTop = errors.New("incidentseverity: incident is already S1")
)

type Severity string

const (
	S1 Severity = "S1"
	S2 Severity = "S2"
	S3 Severity = "S3"
	S4 Severity = "S4"
)

func (s Severity) valid() bool { return s.level() != 0 }

func (s Severity) level() int {
	switch s {
	case S1:
		return 1
	case S2:
		return 2
	case S3:
		return 3
	case S4:
		return 4
	default:
		return 0
	}
}

func fromLevel(n int) Severity {
	switch n {
	case 1:
		return S1
	case 2:
		return S2
	case 3:
		return S3
	default:
		return S4
	}
}

func (s Severity) worse() Severity  { return fromLevel(s.level() - 1) }
func (s Severity) better() Severity { return fromLevel(s.level() + 1) }

type PublicRisk string

const (
	RiskLow    PublicRisk = "low"
	RiskMedium PublicRisk = "medium"
	RiskHigh   PublicRisk = "high"
)

type Input struct {
	AffectedPlayers int
	OnlinePlayers   int
	Duration        time.Duration
	PaymentAffected bool
	Workaround      bool // true means players can still play or bypass the failure
	PublicRisk      PublicRisk
	DataLoss        bool
}

type Role string

const (
	IncidentCommander Role = "incident_commander"
	Communications    Role = "communications"
	Operations        Role = "operations"
	ExecutiveSponsor  Role = "executive_sponsor"
)

type Plan struct {
	Severity    Severity
	Reasons     []string
	Roles       []Role
	Notify      []string
	UpdateEvery time.Duration
	WarRoom     bool
}

func planFor(sev Severity) Plan {
	switch sev {
	case S1:
		return Plan{
			Severity:    S1,
			Roles:       []Role{IncidentCommander, Communications, Operations, ExecutiveSponsor},
			Notify:      []string{"pager", "leadership", "customer_support", "community"},
			UpdateEvery: 15 * time.Minute,
			WarRoom:     true,
		}
	case S2:
		return Plan{
			Severity:    S2,
			Roles:       []Role{IncidentCommander, Communications, Operations},
			Notify:      []string{"pager", "leadership", "customer_support"},
			UpdateEvery: 30 * time.Minute,
			WarRoom:     true,
		}
	case S3:
		return Plan{
			Severity:    S3,
			Roles:       []Role{IncidentCommander, Operations},
			Notify:      []string{"oncall", "customer_support"},
			UpdateEvery: 2 * time.Hour,
		}
	default:
		return Plan{
			Severity:    S4,
			Roles:       []Role{Operations},
			Notify:      []string{"oncall"},
			UpdateEvery: 24 * time.Hour,
		}
	}
}

func atLeast(current, floor Severity) Severity {
	if current.level() > floor.level() {
		return floor
	}
	return current
}

func worsen(sev Severity, levels int) Severity {
	n := sev.level() - levels
	if n < 1 {
		n = 1
	}
	return fromLevel(n)
}

// Classify turns five observable facts into a severity and a response plan.
// Hard floors run before duration: data loss or confirmed payment damage cannot
// be talked down to S3 just because the affected ratio is small.
func Classify(in Input) (Plan, error) {
	if in.OnlinePlayers <= 0 {
		return Plan{}, fmt.Errorf("%w: online players must be positive", ErrInvalidInput)
	}
	if in.AffectedPlayers < 0 || in.AffectedPlayers > in.OnlinePlayers {
		return Plan{}, fmt.Errorf("%w: affected players out of range", ErrInvalidInput)
	}
	if in.Duration < 0 {
		return Plan{}, fmt.Errorf("%w: duration cannot be negative", ErrInvalidInput)
	}
	if in.PublicRisk != "" && in.PublicRisk != RiskLow && in.PublicRisk != RiskMedium && in.PublicRisk != RiskHigh {
		return Plan{}, fmt.Errorf("%w: unknown public risk %q", ErrInvalidInput, in.PublicRisk)
	}

	ratio := float64(in.AffectedPlayers) / float64(in.OnlinePlayers)
	sev := S4
	reasons := make([]string, 0, 5)
	switch {
	case ratio >= 0.5:
		sev = S1
		reasons = append(reasons, "half or more of online players affected")
	case ratio >= 0.1:
		sev = S2
		reasons = append(reasons, "at least 10% of online players affected")
	case ratio >= 0.01:
		sev = S3
		reasons = append(reasons, "at least 1% of online players affected")
	default:
		reasons = append(reasons, "limited player impact")
	}

	if in.DataLoss {
		sev = atLeast(sev, S1)
		reasons = append(reasons, "data loss")
	}
	if in.PaymentAffected && ratio >= 0.1 {
		sev = atLeast(sev, S1)
		reasons = append(reasons, "payment affected for a material share of players")
	}
	if in.PublicRisk == RiskHigh {
		sev = atLeast(sev, S2)
		reasons = append(reasons, "high public attention")
		if in.PaymentAffected {
			sev = atLeast(sev, S1)
			reasons = append(reasons, "payment plus public attention")
		}
	}
	if !in.Workaround && ratio >= 0.1 {
		sev = atLeast(sev, S2)
		reasons = append(reasons, "no player workaround")
	}
	if in.AffectedPlayers > 0 {
		switch {
		case in.Duration >= 6*time.Hour:
			sev = worsen(sev, 2)
			reasons = append(reasons, "unresolved for six hours or more")
		case in.Duration >= 2*time.Hour:
			sev = worsen(sev, 1)
			reasons = append(reasons, "unresolved for two hours or more")
		}
	}

	p := planFor(sev)
	p.Reasons = reasons
	return p, nil
}