Go 字符串处理与内存优化实战：Interning、零拷贝与构建器模式

type StringHeader struct {
    Data uintptr // 指向底层字节数组的指针
    Len  int     // 字节长度
}

package main

import (
	"fmt"
	"unsafe"
)

func main() {
	// 证明 string header 结构
	s := "hello, 世界"

	type StringHeader struct {
		Data uintptr
		Len  int
	}

	sh := (*StringHeader)(unsafe.Pointer(&s))
	fmt.Printf("Data 指针: %p\n", unsafe.Pointer(sh.Data))
	fmt.Printf("长度: %d\n", sh.Len)
	fmt.Printf("len(s) 返回: %d\n", len(s))

	// 字符串数值是 13 字节（hello, + 空格 + 世 3 字节 + 界 3 字节）
	// 但 rune 数只有 9 个
	fmt.Printf("字符数: %d\n", len([]rune(s)))
}

s := "hello"
// s[0] = 'H'  // 编译错误：cannot assign to s[0]

type SliceHeader struct {
    Data uintptr
    Len  int
    Cap  int  // string 没有 Cap 字段！
}

s := "hello"
b := []byte(s) // 拷贝了 5 个字节！

// s 和 b 指向不同内存
fmt.Printf("s[0]=%02x -> %p\n", s[0], unsafe.Pointer(&s[0]))
fmt.Printf("b[0]=%02x -> %p\n", b[0],unsafe.Pointer(&b[0]))

package main

import (
	"bytes"
	"fmt"
	"strings"
	"testing"
)

const TestCount = 10000

// 方式一：+ 拼接
func concatPlus(n int) string {
	var s string
	for i := 0; i < n; i++ {
		s += "hello"
	}
	return s
}

// 方式二：strings.Builder
func concatBuilder(n int) string {
	var b strings.Builder
	for i := 0; i < n; i++ {
		b.WriteString("hello")
	}
	return b.String()
}

// 方式三：bytes.Buffer
func concatBuffer(n int) string {
	var b bytes.Buffer
	for i := 0; i < n; i++ {
		b.WriteString("hello")
	}
	return b.String()
}

// 方式四：fmt.Sprintf
func concatSprintf(n int) string {
	var s string
	for i := 0; i < n; i++ {
		s = fmt.Sprintf("%shello", s)
	}
	return s
}

// 方式五：预分配 Builder
func concatBuilderGrow(n int) string {
	var b strings.Builder
	b.Grow(n * 5)
	for i := 0; i < n; i++ {
		b.WriteString("hello")
	}
	return b.String()
}

func BenchmarkConcatPlus(b *testing.B) {
	for i := 0; i < b.N; i++ {
		concatPlus(TestCount)
	}
}

func BenchmarkConcatBuilder(b *testing.B) {
	for i := 0; i < b.N; i++ {
		concatBuilder(TestCount)
	}
}

func BenchmarkConcatBuffer(b *testing.B) {
	for i := 0; i < b.N; i++ {
		concatBuffer(TestCount)
	}
}

func BenchmarkConcatSprintf(b *testing.B) {
	for i := 0; i < b.N; i++ {
		concatSprintf(TestCount)
	}
}

func BenchmarkConcatBuilderGrow(b *testing.B) {
	for i := 0; i < b.N; i++ {
		concatBuilderGrow(TestCount)
	}
}

// go test -bench=BenchmarkConcat -benchmem

var b strings.Builder
b.Grow(100)
b.WriteString("Hello, ")
b.WriteRune('世')
b.WriteRune('界')
b.WriteString("!")
fmt.Println(b.String()) // Hello, 世界!

func formatBatchResponse(items []Result) string {
	var b strings.Builder

	// 技巧 1: 如果知道大致长度，预分配
	if len(items) > 0 {
		// 估算：每个项目大约需要 50 字节
		b.Grow(len(items) * 50)
	}

	b.WriteByte('{')
	for i, item := range items {
		if i > 0 {
			b.WriteByte(',')
		}
		// 使用 strconv 而不是 fmt.Sprintf，减少分配
		b.WriteString(`"`)
		b.WriteString(item.Name)
		b.WriteString(`":`)
		b.WriteString(strconv.Itoa(item.Value))
	}
	b.WriteByte('}')

	return b.String()
}

var builderPool = sync.Pool{
	New: func() interface{} {
		return &strings.Builder{}
	},
}

func processLogEntries(entries []LogEntry) []string {
	results := make([]string, len(entries))
	for i, entry := range entries {
		b := builderPool.Get().(*strings.Builder)
		b.Reset()
		b.Grow(256)

		// 构建日志消息
		b.WriteString(entry.Level)
		b.WriteString(" | ")
		b.WriteString(entry.Time.Format(time.RFC3339))
		b.WriteString(" | ")
		b.WriteString(entry.Message)

		results[i] = b.String()
		builderPool.Put(b)
	}
	return results
}

b := []byte(s)  // 拷贝 len(s) 字节
s := string(b)  // 拷贝 len(b) 字节

package main

import (
	"reflect"
	"unsafe"
)

// BytesToString 零拷贝：[]byte -> string
func BytesToString(b []byte) string {
	return unsafe.String(unsafe.SliceData(b), len(b))
}

// StringToBytes 零拷贝：string -> []byte
func StringToBytes(s string) []byte {
	return unsafe.Slice(unsafe.StringData(s), len(s))
}

// 零拷贝方式写入日志（底层只读）
func (l *Logger) WriteString(data string) {
	// 不拷贝，直接将 string 看作 []byte 写入输出
	l.writer.Write(StringToBytes(data))
}

package main

import (
	"fmt"
	"sync"
)

// StringPool 手动 interning 池
type StringPool struct {
	mu   sync.RWMutex
	pool map[string]string
}

func NewStringPool() *StringPool {
	return &StringPool{
		pool: make(map[string]string, 1024),
	}
}

func (p *StringPool) Intern(s string) string {
	// 1. 先读锁查找
	p.mu.RLock()
	if interned, ok := p.pool[s]; ok {
		p.mu.RUnlock()
		return interned
	}
	p.mu.RUnlock()

	// 2. 未找到，加写锁并重新检查（防止并发竞争）
	p.mu.Lock()
	defer p.mu.Unlock()

	if interned, ok := p.pool[s]; ok {
		return interned
	}

	// 3. 存入池并返回
	p.pool[s] = s
	return s
}

package main

import (
	"sync"
)

// SyncMapStringPool 使用 sync.Map 的 interning 池
type SyncMapStringPool struct {
	pool sync.Map
}

func NewSyncMapStringPool() *SyncMapStringPool {
	return &SyncMapStringPool{}
}

func (p *SyncMapStringPool) Intern(s string) string {
	if loaded, ok := p.pool.Load(s); ok {
		return loaded.(string)
	}

	// 只有第一次出现时才触发 CompareAndSwap
	actual, loaded := p.pool.LoadOrStore(s, s)
	if loaded {
		return actual.(string)
	}
	return s
}

package main

import (
	"container/list"
	"sync"
)

type LRUStringPool struct {
	mu       sync.Mutex
	capacity int
	pool     map[string]*list.Element // 字符串 -> list element
	lru      *list.List               // 维护访问顺序
}

type lruEntry struct {
	key   string
	value string
}

func NewLRUStringPool(capacity int) *LRUStringPool {
	return &LRUStringPool{
		capacity: capacity,
		pool:     make(map[string]*list.Element, capacity),
		lru:      list.New(),
	}
}

func (p *LRUStringPool) Intern(s string) string {
	p.mu.Lock()
	defer p.mu.Unlock()

	if elem, ok := p.pool[s]; ok {
		// 移动到队尾（最近使用）
		p.lru.MoveToBack(elem)
		return elem.Value.(*lruEntry).value
	}

	// 新增：如果超出容量，淘汰最旧的
	if p.lru.Len() >= p.capacity {
		oldest := p.lru.Front()
		oldestEntry := oldest.Value.(*lruEntry)
		delete(p.pool, oldestEntry.key)
		p.lru.Remove(oldest)
	}

	entry := &lruEntry{key: s, value: s}
	elem := p.lru.PushBack(entry)
	p.pool[s] = elem
	return s
}

func internHTTPMethod(method string) string {
	switch method {
	case "GET":
		return "GET"
	case "POST":
		return "POST"
	case "PUT":
		return "PUT"
	case "DELETE":
		return "DELETE"
	case "HEAD":
		return "HEAD"
	case "OPTIONS":
		return "OPTIONS"
	case "PATCH":
		return "PATCH"
	default:
		return method
	}
}

package main

import (
	"sync"
)

var bytePool = sync.Pool{
	New: func() interface{} {
		b := make([]byte, 0, 1024)
		return &b
	},
}

func getBuffer() []byte {
	b := bytePool.Get().(*[]byte)
	return (*b)[:0] // 重置长度但保留容量
}

func putBuffer(b []byte) {
	if cap(b) <= 4*1024 { // 只回收小 buffer，避免池膨胀
		bytePool.Put(&b)
	}
}

package main

import (
	"sync"
)

// 分级 buffer 池，对应典型大小
type tieredBufferPool struct {
	pools [5]*sync.Pool // 128, 512, 2048, 8192, 32768
}

func newTieredPool() *tieredBufferPool {
	var p tieredBufferPool
	for i, size := range []int{128, 512, 2048, 8192, 32768} {
		size := size
		p.pools[i] = &sync.Pool{
			New: func() interface{} {
				b := make([]byte, size)
				return &b
			},
		}
	}
	return &p
}

func (p *tieredBufferPool) getBuffer(required int) []byte {
	tierIdx := 0
	for _, size := range []int{128, 512, 2048, 8192, 32768} {
		if required <= size {
			b := p.pools[tierIdx].Get().(*[]byte)
			return (*b)[:required]
		}
		tierIdx++
	}
	// 超出所有 tier，直接分配
	return make([]byte, required)
}

func (p *tieredBufferPool) putBuffer(b []byte) {
	cap_ := cap(b)
	tierIdx := 0
	for _, size := range []int{128, 512, 2048, 8192, 32768} {
		if cap_ == size {
			p.pools[tierIdx].Put(&b)
			return
		}
		tierIdx++
	}
}

package main

import (
	"regexp"
	"strings"
	"testing"
)

var reColon = regexp.MustCompile(`:`)
var testString = "Content-Type: application/json; charset=utf-8"

func BenchmarkStringsIndex(b *testing.B) {
	for i := 0; i < b.N; i++ {
		_ = strings.Index(testString, ":")
	}
}

func BenchmarkRegexpFindIndex(b *testing.B) {
	for i := 0; i < b.N; i++ {
		_ = reColon.FindStringIndex(testString)
	}
}

// 使用 strings.FieldsFunc
parts := strings.FieldsFunc(line, func(r rune) bool {
	return r == ',' || r == '\t' || r == '|'
})

// Go 会在内部自动选择最优实现
idx := strings.IndexByte(haystack, '\n')

// 旧方式：通用但稍慢
parts := strings.SplitN(s, ":", 2)
key, value := parts[0], parts[1]

// 新方式：专为首个分隔符优化
key, value, ok := strings.Cut(s, ":")

package main

import (
	"fmt"
	"unicode/utf8"
)

func main() {
	s := "Hello, 世界！"

	// 方式一：range 遍历（推荐）
	// range 按 UTF-8 解码，每次返回一个 rune
	fmt.Println("=== range 遍历 ===")
	for i, r := range s {
		fmt.Printf("index=%d, rune=%c (U+%04X)\n", i, r, r)
	}

	// 方式二：按字节遍历（错误方式）
	fmt.Println("\n=== 按字节遍历（不对中文生效） ===")
	for i := 0; i < len(s); i++ {
		fmt.Printf("byte[%d] = %02x\n", i, s[i])
	}

	// 方式三：utf8.DecodeRuneInString
	fmt.Println("\n=== utf8.DecodeRuneInString ===")
	for i := 0; i < len(s); {
		r, size := utf8.DecodeRuneInString(s[i:])
		fmt.Printf("index=%d, size=%d, rune=%c\n", i, size, r)
		i += size
	}

	// 计算 rune 数量
	fmt.Println("\nrune 计数:", utf8.RuneCountInString(s))
}

// 是否有效 UTF-8
if !utf8.ValidString(input) {
	// 处理非法字符
	input = strings.ToValidUTF8(input, "�") // 替换为替换字符
}

// ❌ 错误：按字节截断，中文可能被劈半
func truncateBytes(s string, n int) string {
	if len(s) <= n {
		return s
	}
	return s[:n] // 可能断在中文字符中间！
}

// ✅ 正确：按 rune 截断
func truncateRune(s string, n int) string {
	runes := []rune(s)
	if len(runes) <= n {
		return s
	}
	return string(runes[:n])
}

// ✅ 更优：不额外分配 []rune，用 utf8.DecodeRune 遍历
func truncateRuneFast(s string, n int) string {
	if n <= 0 {
		return ""
	}
	count := 0
	for i := 0; i < len(s); {
		_, size := utf8.DecodeRuneInString(s[i:])
		count++
		if count > n {
			return s[:i]
		}
		i += size
	}
	return s
}

// ❌ 高分配版
func (l *Logger) Infof(format string, args ...interface{}) {
	msg := fmt.Sprintf(format, args...) // 分配 + 反射开销
	l.output.println(msg)
}

// ✅ 低分配版：固定字段 + 预分配 Builder
var logLevelStr = map[Level]string{
	Debug: "DEBUG",
	Info:  "INFO",
	Warn:  "WARN",
	Error: "ERROR",
}

func (l *Logger) Info(msg string) {
	var b strings.Builder
	b.Grow(128 + len(msg))

	b.WriteString("2024-01-01T12:00:00Z") // 实际用纳秒时间戳
	b.WriteByte(' ')
	b.WriteString(logLevelStr[Info])
	b.WriteByte(' ')
	b.WriteString(l.prefix)
	b.WriteString(msg)
	b.WriteByte('\n')

	l.output.Write(StringToBytes(b.String()))
}

package main

import (
	"strings"
)

// 常见的 header 值常量（interning）
var (
	HeaderContentType     = "Content-Type"
	HeaderContentLength   = "Content-Length"
	HeaderAuthorization   = "Authorization"
	HeaderAcceptEncoding  = "Accept-Encoding"
)

// SimpleHeaderMap 只存储关心的 headers，使用 interned key
type SimpleHeaderMap struct {
	data map[string]string
}

func (h *SimpleHeaderMap) Get(key string) string {
	if h.data == nil {
		return ""
	}
	return h.data[key]
}

// ParseHeadersLine 解析单行 "Header: value"
func ParseHeadersLine(line string, pool *StringPool) (k, v string, ok bool) {
	idx := strings.IndexByte(line, ':')
	if idx <= 0 {
		return "", "", false
	}
	key := strings.TrimSpace(line[:idx])
	value := strings.TrimSpace(line[idx+1:])
	return pool.Intern(key), pool.Intern(value), true
}

package main

import (
	"encoding/json"
	"strconv"
)

// internKeyPool 共享 JSON field name 的 interning 池
var internKeyPool = NewStringPool()

// CustomMap 使用 interned key 的 map
type CustomMap map[string]interface{}

func (m *CustomMap) UnmarshalJSON(data []byte) error {
	tmp := make(map[string]interface{})
	if err := json.Unmarshal(data, &tmp); err != nil {
		return err
	}

	result := make(CustomMap, len(tmp))
	for k, v := range tmp {
		internedKey := internKeyPool.Intern(k)
		result[internedKey] = v
	}
	*m = result
	return nil
}

package main

import (
	"net/http"
	_ "net/http/pprof"
	"runtime"
)

func main() {
	// 在后台暴露 pprof 接口
	go func() {
		http.ListenAndServe("localhost:6060", nil)
	}()

	// 业务代码...
}

import (
	"os"
	"runtime/pprof"
)

func dumpHeap(filename string) {
	f, _ := os.Create(filename)
	defer f.Close()
	pprof.WriteHeapProfile(f)
}

# 生成堆分配的火焰图（SVG 格式）
go tool pprof -svg http://localhost:6060/debug/pprof/heap > heap.svg

# 查看分配次数最多的调用栈（按 alloc_objects 排序）
go tool pprof -alloc_objects http://localhost:6060/debug/pprof/heap

# 交互式命令
(pprof) top 20          # 查看前 20 个消耗最高的函数
(pprof) list concat     # 查看特定函数的分配详情
(pprof) peek strings    # 查看包含 "strings" 的调用链

# 优化前
$ go test -bench=. -benchmem -count=5 > old.txt

# 优化后
$ go test -bench=. -benchmem -count=5 > new.txt

# 对比
$ go install golang.org/x/perf/cmd/benchstat@latest
$ benchstat old.txt new.txt