《MMO 服务器开发》6.2 心跳与超时检测

长连接最大的敌人不是掉线,而是你以为它还活着。本节给 aegis 写一个真实的 WebSocket 服务端:一条读超时、一个看门狗、一路可选的服务端 ping,然后用真起端口、真连的 L2 测试验证「不发心跳的客户端会被主动关闭并清理会话」,最后报告本机实测的超时精度与 1000 个空连接的心跳开销。

6.2 心跳与超时检测

上一节把帧格式写通了,但一条「能收发帧」的连接离「可用」还差一步:它不知道对端什么时候已经死了。玩家在地铁里切进隧道、手机锁屏、NAT 映射到期、进程被 kill -9,这些场景下 TCP 都不会立刻通知服务端,连接会以「半开」状态一直挂着。

这一节解决的就是这件事。我们要的是一个可观测、可复现的策略:多久算超时、谁来判、判定之后怎么清理,以及这套机制在 1000 条连接上要花多少代价。

沉默的连接:TCP 不会告诉你对端已经走了

很多人第一反应是开 SO_KEEPALIVE。但 TCP keepalive 的默认探测间隔是 2 小时,就算调到系统允许的最小值,粒度也远达不到 MMO 的要求——玩家掉线几十秒内就该被判定离线,否则队友看到的是一具「站在原地不动」的僵尸。

所以游戏服务端几乎都做应用层心跳:客户端周期性地发一个空包或 ping 帧,服务端只要「最近收到过任何字节」就认为连接活着。反向也一样,服务端可以主动 ping,让 NAT 设备上的映射不过期。注意心跳是双向的:只做单向探测,在对称 NAT 后面依然会被静默丢弃。

服务端实现:读超时、看门狗与主动 ping

aegis 的接入层做法很朴素:每条连接一个读循环 + 一个看门狗。读循环每收到一帧就更新一次「最后活跃时间」,看门狗按固定节拍检查这个时间戳,超过 ReadTimeout 就发一个 close 帧(状态码 1001 “going away”)并回收会话。可选的服务端 pinger 则按 PingInterval 周期性发 ping。

下面这段是 L2,go test 会真起一个 TCP 端口并真连:

// file: internal/ws/server.go
// tier: L2
// verify: go test ./internal/ws/ -run 'TestHeartbeatTimeoutClosesIdleClient|TestServerPingKeepsIdleClientAlive|TestServerEcho' -v

package ws

import (
	"bufio"
	"net"
	"sync"
	"sync/atomic"
	"time"
)

// Close codes. 1001 "going away" is what a gateway sends when it drops a peer
// whose heartbeat went silent.
const (
	CloseGoingAway    uint16 = 1001
	CloseProtocolErr  uint16 = 1002
	ClosePolicyViolat uint16 = 1003
)

// Server is a minimal WebSocket echo/heartbeat gateway. Per connection there is
// one reader goroutine, one watchdog enforcing the read deadline, and one
// optional pinger keeping NAT bindings alive.
type Server struct {
	// ReadTimeout is the longest silence tolerated before the peer is dropped.
	ReadTimeout time.Duration
	// PingInterval, when non-zero, makes the server send periodic pings.
	PingInterval time.Duration
	// MaxPayload caps a single reassembled message; 0 means unlimited.
	MaxPayload uint64
	// OnMessage handles data frames. When nil the server echoes them back.
	OnMessage func(s *Session, opcode byte, payload []byte)
	// RequireClientMask enforces RFC 6455 §5.1 on the read path.
	RequireClientMask bool

	mu       sync.Mutex
	sessions map[*Session]struct{}
	nextID   uint64
}

// NewServer returns a Server with sane defaults.
func NewServer() *Server {
	return &Server{
		MaxPayload:        1 << 20,
		RequireClientMask: true,
		sessions:          make(map[*Session]struct{}),
	}
}

// Session is one accepted WebSocket connection.
type Session struct {
	ID  uint64
	seq uint64 // reconnection sequence handed to the client

	conn   net.Conn
	last   atomic.Int64 // UnixNano of the last inbound frame
	closed atomic.Bool
	wmu    sync.Mutex
}

// LastActive reports the time of the most recent inbound frame.
func (s *Session) LastActive() time.Time { return time.Unix(0, s.last.Load()) }

func (s *Session) touch() { s.last.Store(time.Now().UnixNano()) }

// WriteMessage sends one unfragmented frame. Server-to-client frames are never
// masked, as the RFC requires.
func (s *Session) WriteMessage(opcode byte, payload []byte) error {
	s.wmu.Lock()
	defer s.wmu.Unlock()
	return WriteFrame(s.conn, true, opcode, payload, false)
}

// Close sends a close frame best-effort and then closes the socket. Idempotent,
// so the watchdog and the reader can race without double-closing.
func (s *Session) Close(code uint16, reason string) error {
	if s.closed.Swap(true) {
		return nil
	}
	buf := make([]byte, 2+len(reason))
	buf[0] = byte(code >> 8)
	buf[1] = byte(code)
	copy(buf[2:], reason)
	s.wmu.Lock()
	_ = WriteFrame(s.conn, true, OpClose, buf, false)
	s.wmu.Unlock()
	return s.conn.Close()
}

// SessionCount returns the number of live sessions.
func (s *Server) SessionCount() int {
	s.mu.Lock()
	defer s.mu.Unlock()
	return len(s.sessions)
}

// PingAll sends one ping to every live session. The table is snapshotted under
// the lock and the writes happen outside it, so one slow socket cannot stall
// the whole sweep.
func (s *Server) PingAll() int {
	s.mu.Lock()
	batch := make([]*Session, 0, len(s.sessions))
	for sess := range s.sessions {
		batch = append(batch, sess)
	}
	s.mu.Unlock()

	sent := 0
	for _, sess := range batch {
		if err := sess.WriteMessage(OpPing, nil); err == nil {
			sent++
		}
	}
	return sent
}

func (s *Server) add(sess *Session) {
	s.mu.Lock()
	s.nextID++
	sess.ID = s.nextID
	s.sessions[sess] = struct{}{}
	s.mu.Unlock()
}

func (s *Server) remove(sess *Session) {
	s.mu.Lock()
	delete(s.sessions, sess)
	s.mu.Unlock()
}

// ServeListener accepts connections until the listener is closed.
func (s *Server) ServeListener(ln net.Listener) error {
	for {
		c, err := ln.Accept()
		if err != nil {
			return err
		}
		go func() { _ = s.Serve(c) }()
	}
}

// Serve runs the full lifecycle for one accepted TCP connection: handshake,
// frame loop, keepalive, and timeout enforcement.
func (s *Server) Serve(conn net.Conn) error {
	br := bufio.NewReader(conn)
	if _, err := Handshake(br, conn); err != nil {
		_ = conn.Close()
		return err
	}

	sess := &Session{conn: conn}
	sess.touch()
	s.add(sess)

	defer func() {
		sess.closed.Store(true)
		_ = conn.Close()
		s.remove(sess)
	}()

	done := make(chan struct{})
	go func() {
		defer close(done)
		for {
			op, payload, err := ReadMessageMasked(br, s.MaxPayload, s.RequireClientMask)
			if err != nil {
				return
			}
			sess.touch()
			switch op {
			case OpPing:
				_ = sess.WriteMessage(OpPong, payload)
			case OpPong:
				// Liveness only; touch() already recorded it.
			case OpClose:
				_ = sess.WriteMessage(OpClose, payload)
				return
			default:
				if s.OnMessage != nil {
					s.OnMessage(sess, op, payload)
				} else {
					_ = sess.WriteMessage(op, payload)
				}
			}
		}
	}()

	if s.PingInterval > 0 {
		go func() {
			t := time.NewTicker(s.PingInterval)
			defer t.Stop()
			for {
				select {
				case <-done:
					return
				case <-t.C:
					_ = sess.WriteMessage(OpPing, nil)
				}
			}
		}()
	}

	deadline := s.ReadTimeout
	if deadline <= 0 {
		deadline = 60 * time.Second
	}
	tick := deadline / 8
	if tick < time.Millisecond {
		tick = time.Millisecond
	}
	t := time.NewTicker(tick)
	defer t.Stop()
	for {
		select {
		case <-done:
			return nil
		case <-t.C:
			if time.Since(sess.LastActive()) > deadline {
				_ = sess.Close(CloseGoingAway, "heartbeat timeout")
				return nil
			}
		}
	}
}

有两个细节值得单独说。第一,Close 用 atomic.Bool 做了幂等,因为看门狗和读循环都可能触发关闭,重复关同一个 socket 会让日志里出现毫无意义的 use of closed network connection。第二,看门狗的检查节拍是 ReadTimeout / 8,不是每毫秒轮询——1000 条连接各跑一个 1ms 的 ticker,光调度开销就足够吃掉一个核。

真跑:L2 端到端验证

测试不 mock 任何东西:startServer 用 net.Listen("tcp", "127.0.0.1:0") 起真端口,dialWS 真发 HTTP 升级请求并校验 Sec-WebSocket-Accept。核心断言是下面这个:客户端握手成功后什么都不发,服务端必须在超时后主动发 close 帧、关闭 socket、并把会话从表里摘掉。

// snippet: internal/ws/server_test.go(节选)

// TestHeartbeatTimeoutClosesIdleClient is the core L2 assertion: a client that
// never sends anything is dropped and its session is reclaimed.
func TestHeartbeatTimeoutClosesIdleClient(t *testing.T) {
	const timeout = 300 * time.Millisecond
	s := NewServer()
	s.ReadTimeout = timeout
	s.PingInterval = 0 // no server pings: pure silence must trip the watchdog
	addr := startServer(t, s)

	c, br := dialWS(t, addr)
	defer c.Close()

	if got := s.SessionCount(); got != 1 {
		t.Fatalf("session count after handshake = %d, want 1", got)
	}

	start := time.Now()
	op, _, err := ReadMessage(br, 0) // blocks until the server's close frame
	elapsed := time.Since(start)
	if err != nil {
		t.Fatalf("expected a close frame, got error: %v", err)
	}
	if op != OpClose {
		t.Fatalf("op = %#x, want close", op)
	}
	if elapsed < timeout {
		t.Fatalf("closed after %v, before the %v deadline", elapsed, timeout)
	}
	if elapsed > timeout+500*time.Millisecond {
		t.Fatalf("closed after %v, far past the %v deadline", elapsed, timeout)
	}

	deadline := time.Now().Add(2 * time.Second)
	for s.SessionCount() != 0 && time.Now().Before(deadline) {
		time.Sleep(5 * time.Millisecond)
	}
	if got := s.SessionCount(); got != 0 {
		t.Fatalf("session leaked: count = %d", got)
	}
	t.Logf("timeout precision: configured %v, observed %v, delta %v",
		timeout, elapsed.Round(time.Millisecond), (elapsed - timeout).Round(time.Millisecond))
}

实测结论

以下是本机(Go 1.26,macOS arm64)实测数字,同机相对量级,绝对数值不可移植:

指标实测值说明
超时精度配置 300ms,实测 304–305ms(偏差 4–5ms)看门狗节拍 37.5ms,偏差小于一拍
1000 空连接内存约 6.5 KB/连接runtime.MemStats.HeapAlloc 增量 / 1000
1000 空连接 goroutine2.00 个/连接读循环 + Serve 各一个
心跳 sweep 成本约 65–67 µs/ping1000 连接 × 100 轮,共 10 万次写
本机 socket write 底噪约 80 µs/次单连接校准,说明上一条基本就是 syscall 底噪

两个反直觉的结论。其一,超时精度不是问题:300ms 的判定被实测在 305ms 内完成,误差远小于一帧 RTT 的量级,没必要为了「更准」把节拍调密。其二,心跳本身的协议开销可以忽略:每 ping 约 66µs 里几乎全是写 socket 的 syscall 成本,2 字节的帧头 vs 12 字节的帧头在这个量级上毫无区别——真正要防的是「1000 个 ticker 的调度开销」,所以看门狗必须共享节拍或走时间轮,这一点留到第 9 章展开。

最后提醒一个 macOS 的坑:kern.ipc.somaxconn 默认只有 128,千连接压测如果一口气 dial 会被内核直接 RST。上面的测试因此每连一条 time.Sleep(time.Millisecond) 错峰,1000 条大约 5 秒才建完。

阅读导航:上一节:6.1 WebSocket 握手、掩码与分片 · 下一节:6.3 断线重连与重连票据 。

继续阅读

探索更多技术文章

浏览归档,发现更多关于系统设计、工具链和工程实践的内容。

全部文章 返回首页

「mmo」更多文章

  1. 《MMO 服务器开发》工程化与运营 · 目录
  2. 《MMO 服务器开发》9.3 上线检查清单
  3. 《MMO 服务器开发》9.2 性能预算达标