《游戏服务端架构·规模化、全球化与成本》8.2 主备、双活与故障切换

// file: internal/failoverplan/plan.go
// tier: L1
// verify: go test ./internal/failoverplan/
package failoverplan

import (
	"errors"
	"fmt"
	"time"
)

// State 是切换状态机的对外状态，必须能被值班同学一眼读懂。
type State string

const (
	Healthy   State = "healthy"
	Suspect   State = "suspect"
	Fencing   State = "fencing"
	Promoting State = "promoting"
	Active    State = "active"
	Standby   State = "standby"
)

// EventKind 是驱动状态机的外部事实，不是“操作按钮”。
type EventKind string

const (
	HeartbeatMiss EventKind = "heartbeat_miss"
	HeartbeatOK   EventKind = "heartbeat_ok"
	VoteGranted   EventKind = "vote_granted"
	VoteDenied    EventKind = "vote_denied"
	FenceDone     EventKind = "fence_done"
	FenceFailed   EventKind = "fence_failed"
	PromoteDone   EventKind = "promote_done"
	PromoteFailed EventKind = "promote_failed"
	Demote        EventKind = "demote"
)

// Config 是切换的判定参数；默认值让零值配置也能工作。
type Config struct {
	MissThreshold  int
	Quorum         int
	FenceTimeout   time.Duration
	PromoteTimeout time.Duration
}

func (c Config) withDefaults() Config {
	if c.MissThreshold <= 0 {
		c.MissThreshold = 3
	}
	if c.Quorum <= 0 {
		c.Quorum = 1
	}
	if c.FenceTimeout <= 0 {
		c.FenceTimeout = 10 * time.Second
	}
	if c.PromoteTimeout <= 0 {
		c.PromoteTimeout = 60 * time.Second
	}
	return c
}

// Event 带时间戳，超时判定不依赖真实时钟，测试可以完全复现。
type Event struct {
	Kind EventKind
	At   time.Duration
}

var (
	ErrTransition = errors.New("invalid transition")
	ErrTimeout    = errors.New("phase timeout")
)

// Machine 是单方向的切换状态机：健康 → 怀疑 → fencing → 提升。
// 回切由运维显式 Demote，不自动发生。
type Machine struct {
	cfg      Config
	state    State
	misses   int
	votes    int
	epoch    int64
	deadline time.Duration
}

func New(cfg Config) *Machine {
	return &Machine{cfg: cfg.withDefaults(), state: Healthy}
}

func (m *Machine) State() State { return m.state }

// Epoch 是 fencing token 的当前值；每次成功 fencing 后递增。
func (m *Machine) Epoch() int64 { return m.epoch }

// Apply 消费一个外部事实并推进状态机。非法迁移返回 ErrTransition，
// 由调用方决定是告警还是丢弃，状态机本身不做隐式纠正。
func (m *Machine) Apply(ev Event) (State, error) {
	switch ev.Kind {
	case HeartbeatMiss:
		switch m.state {
		case Healthy:
			m.misses++
			if m.misses >= m.cfg.MissThreshold {
				m.state = Suspect
				m.votes = 0
			}
		case Suspect:
			// 已经怀疑，继续计数没有额外语义，保持原状。
		default:
			return m.state, fmt.Errorf("%w: heartbeat_miss in %s", ErrTransition, m.state)
		}
	case HeartbeatOK:
		if m.state != Healthy && m.state != Suspect {
			return m.state, fmt.Errorf("%w: heartbeat_ok in %s", ErrTransition, m.state)
		}
		m.state = Healthy
		m.misses = 0
		m.votes = 0
	case VoteGranted:
		if m.state != Suspect {
			return m.state, fmt.Errorf("%w: vote_granted in %s", ErrTransition, m.state)
		}
		m.votes++
		if m.votes >= m.cfg.Quorum {
			m.state = Fencing
			m.deadline = ev.At + m.cfg.FenceTimeout
		}
	case VoteDenied:
		if m.state != Suspect {
			return m.state, fmt.Errorf("%w: vote_denied in %s", ErrTransition, m.state)
		}
		m.votes = 0
	case FenceDone:
		if m.state != Fencing {
			return m.state, fmt.Errorf("%w: fence_done in %s", ErrTransition, m.state)
		}
		m.epoch++
		m.state = Promoting
		m.deadline = ev.At + m.cfg.PromoteTimeout
	case FenceFailed:
		if m.state != Fencing {
			return m.state, fmt.Errorf("%w: fence_failed in %s", ErrTransition, m.state)
		}
		m.state = Suspect
		m.votes = 0
	case PromoteDone:
		if m.state != Promoting {
			return m.state, fmt.Errorf("%w: promote_done in %s", ErrTransition, m.state)
		}
		m.state = Active
		m.deadline = 0
	case PromoteFailed:
		if m.state != Promoting {
			return m.state, fmt.Errorf("%w: promote_failed in %s", ErrTransition, m.state)
		}
		m.state = Standby
		m.deadline = 0
	case Demote:
		if m.state != Active {
			return m.state, fmt.Errorf("%w: demote in %s", ErrTransition, m.state)
		}
		m.state = Standby
	default:
		return m.state, fmt.Errorf("%w: unknown event %q", ErrTransition, ev.Kind)
	}
	return m.state, nil
}

// Tick 检查当前阶段是否超时；超时后放弃本次切换，回到 Suspect 重新取证。
// 注意：Active 状态没有超时。
func (m *Machine) Tick(now time.Duration) (State, error) {
	if m.deadline == 0 || now < m.deadline {
		return m.state, nil
	}
	switch m.state {
	case Fencing, Promoting:
		m.state = Suspect
		m.votes = 0
		m.deadline = 0
		return m.state, ErrTimeout
	default:
		return m.state, nil
	}
}

cd .local/checks/gsarch-lab
gofmt -l internal/failoverplan
go vet ./internal/failoverplan/
go test -count=1 ./internal/failoverplan/