日志处理：构建可观测的应用

package main

import (
	"log"
	"os"
)

func main() {
	// 基本使用
	log.Println("这是一条普通日志")
	log.Printf("用户 %s 登录了系统", "张三")
	
	// 设置日志前缀
	log.SetPrefix("[MyApp] ")
	log.Println("带前缀的日志")
	
	// 设置日志标志
	log.SetFlags(log.Ldate | log.Ltime | log.Lshortfile)
	log.Println("带日期、时间和文件名的日志")
	
	// 输出到文件
	file, err := os.OpenFile("app.log", os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666)
	if err != nil {
		log.Fatal("打开日志文件失败:", err)
	}
	defer file.Close()
	
	logger := log.New(file, "[FILE] ", log.LstdFlags)
	logger.Println("这条日志会写入文件")
	
	// Fatal 和 Panic
	// log.Fatal("程序会退出")  // 调用 os.Exit(1)
	// log.Panic("程序会 panic") // 调用 panic()
}

const (
	Ldate         = 1 << iota     // 日期：2009/01/23
	Ltime                         // 时间：01:23:23
	Lmicroseconds                 // 微秒：01:23:23.123123
	Llongfile                     // 完整文件名和行号：/a/b/c/d.go:23
	Lshortfile                    // 短文件名和行号：d.go:23
	LUTC                          // 使用 UTC 时间
	Lmsgprefix                    // 将前缀移到消息前面
	LstdFlags     = Ldate | Ltime // 标准标志
)

package main

import (
	"os"
	
	"github.com/sirupsen/logrus"
)

func main() {
	// 基本使用
	logrus.Info("这是一条 Info 日志")
	logrus.Warn("这是一条 Warning 日志")
	logrus.Error("这是一条 Error 日志")
	
	// 带字段的结构化日志
	logrus.WithFields(logrus.Fields{
		"user_id": 123,
		"username": "张三",
		"action":   "login",
	}).Info("用户登录")
	
	// 设置日志级别
	logrus.SetLevel(logrus.DebugLevel)
	logrus.Debug("这条 Debug 日志会显示")
	
	// 设置输出格式
	logrus.SetFormatter(&logrus.JSONFormatter{})
	logrus.Info("JSON 格式的日志")
	
	// 输出到文件
	file, _ := os.OpenFile("app.log", os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666)
	logrus.SetOutput(file)
	logrus.Info("这条日志会写入文件")
}

package main

import (
	"go.uber.org/zap"
)

func main() {
	// 创建 logger
	logger, _ := zap.NewProduction()
	defer logger.Sync()
	
	// 基本使用
	logger.Info("这是一条 Info 日志")
	logger.Warn("这是一条 Warning 日志")
	logger.Error("这是一条 Error 日志")
	
	// 带字段
	logger.Info("用户登录",
		zap.Int("user_id", 123),
		zap.String("username", "张三"),
		zap.String("action", "login"),
	)
	
	// 开发环境 logger（更易读）
	devLogger, _ := zap.NewDevelopment()
	devLogger.Info("开发环境日志")
	
	// 自定义配置
	config := zap.Config{
		Level:       zap.NewAtomicLevelAt(zap.DebugLevel),
		Development: true,
		Encoding:    "console",
		OutputPaths: []string{"stdout"},
	}
	customLogger, _ := config.Build()
	customLogger.Debug("自定义配置的日志")
}

package main

import (
	"os"
	
	"github.com/rs/zerolog"
	"github.com/rs/zerolog/log"
)

func main() {
	// 基本使用
	log.Info().Msg("这是一条 Info 日志")
	log.Warn().Msg("这是一条 Warning 日志")
	log.Error().Msg("这是一条 Error 日志")
	
	// 带字段
	log.Info().
		Int("user_id", 123).
		Str("username", "张三").
		Str("action", "login").
		Msg("用户登录")
	
	// 设置全局级别
	zerolog.SetGlobalLevel(zerolog.DebugLevel)
	log.Debug().Msg("这条 Debug 日志会显示")
	
	// 人类友好的输出
	log.Logger = log.Output(zerolog.ConsoleWriter{Out: os.Stdout})
	log.Info().Msg("易读的日志格式")
}

package main

import (
	"github.com/sirupsen/logrus"
)

type Logger struct {
	*logrus.Entry
}

func NewLogger(component string) *Logger {
	return &Logger{
		Entry: logrus.WithField("component", component),
	}
}

func (l *Logger) WithRequestID(requestID string) *Logger {
	return &Logger{
		Entry: l.WithField("request_id", requestID),
	}
}

func (l *Logger) WithUserID(userID int) *Logger {
	return &Logger{
		Entry: l.WithField("user_id", userID),
	}
}

func main() {
	// 创建带组件标识的 logger
	logger := NewLogger("UserService")
	
	// 在请求处理中添加上下文
	requestLogger := logger.WithRequestID("req-123").WithUserID(456)
	
	requestLogger.Info("开始处理请求")
	// 输出: {"component":"UserService","request_id":"req-123","user_id":456,"level":"info","msg":"开始处理请求"}
	
	// 处理业务逻辑
	requestLogger.WithField("action", "query").Info("查询用户信息")
	
	requestLogger.Info("请求处理完成")
}

package main

import (
	"os"
	"time"

	"go.uber.org/zap"
	"go.uber.org/zap/zapcore"
)

func NewZapLogger() *zap.Logger {
	encoderConfig := zapcore.EncoderConfig{
		TimeKey:        "ts",
		LevelKey:       "level",
		NameKey:        "logger",
		CallerKey:      "caller",
		MessageKey:     "msg",
		StacktraceKey:  "stacktrace",
		LineEnding:     zapcore.DefaultLineEnding,
		EncodeLevel:    zapcore.LowercaseLevelEncoder,
		EncodeTime:     zapcore.ISO8601TimeEncoder,
		EncodeDuration: zapcore.SecondsDurationEncoder,
		EncodeCaller:   zapcore.ShortCallerEncoder,
	}

	level := zap.NewAtomicLevelAt(zap.InfoLevel)
	core := zapcore.NewCore(
		zapcore.NewJSONEncoder(encoderConfig),
		zapcore.NewMultiWriteSyncer(zapcore.AddSync(os.Stdout)),
		level,
	)

	logger := zap.New(core, zap.AddCaller(), zap.AddStacktrace(zap.ErrorLevel))
	return logger
}

func main() {
	logger := NewZapLogger()
	defer logger.Sync()

	logger.Info("服务启动",
		zap.String("service", "order-api"),
		zap.String("version", "v1.2.3"),
		zap.Time("start_time", time.Now()),
	)

	logger.Error("数据库连接异常",
		zap.String("dsn", "***masked***"),
		zap.Int("retry", 3),
	)
}

package main

import (
	"log/slog"
	"os"
	"time"
)

func main() {
	// 创建 JSON Handler，带自定义时间格式
	opts := &slog.HandlerOptions{
		Level: slog.LevelDebug,
		ReplaceAttr: func(groups []string, a slog.Attr) slog.Attr {
			if a.Key == slog.TimeKey {
				return slog.Attr{
					Key:   slog.TimeKey,
					Value: slog.StringValue(a.Value.Time().Format(time.RFC3339)),
				}
			}
			return a
		},
	}
	handler := slog.NewJSONHandler(os.Stdout, opts)
	logger := slog.New(handler)

	// 替换默认 logger
	slog.SetDefault(logger)

	// 使用
	slog.Info("用户下单", slog.Int("user_id", 42), slog.String("order_id", "ORD-20250813"))
	slog.Debug("请求详情", slog.Any("headers", map[string]string{"Accept": "application/json"}))
}

package main

import (
	"net/http"
	"time"
	
	"github.com/sirupsen/logrus"
)

type responseWriter struct {
	http.ResponseWriter
	statusCode int
}

func (rw *responseWriter) WriteHeader(code int) {
	rw.statusCode = code
	rw.ResponseWriter.WriteHeader(code)
}

func loggingMiddleware(logger *logrus.Logger) func(http.Handler) http.Handler {
	return func(next http.Handler) http.Handler {
		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
			start := time.Now()
			
			// 创建带请求信息的 logger
			reqLogger := logger.WithFields(logrus.Fields{
				"method":     r.Method,
				"path":       r.URL.Path,
				"remote_addr": r.RemoteAddr,
				"user_agent": r.UserAgent(),
			})
			
			reqLogger.Info("请求开始")
			
			// 包装 ResponseWriter 以捕获状态码
			wrapped := &responseWriter{w, http.StatusOK}
			
			// 调用下一个处理器
			next.ServeHTTP(wrapped, r)
			
			// 记录请求完成
			duration := time.Since(start)
			reqLogger.WithFields(logrus.Fields{
				"status":   wrapped.statusCode,
				"duration": duration.String(),
			}).Info("请求完成")
		})
	}
}

func main() {
	logger := logrus.New()
	logger.SetFormatter(&logrus.JSONFormatter{})
	
	mux := http.NewServeMux()
	mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		w.Write([]byte("Hello, World!"))
	})
	
	handler := loggingMiddleware(logger)(mux)
	
	http.ListenAndServe(":8080", handler)
}

package main

import (
	"github.com/sirupsen/logrus"
	"gopkg.in/natefinish/lumberjack.v2"
)

func main() {
	logger := logrus.New()
	
	// 配置日志轮转
	logger.SetOutput(&lumberjack.Logger{
		Filename:   "app.log",
		MaxSize:    100,  // MB
		MaxBackups: 30,   // 保留旧文件数量
		MaxAge:     7,    // 保留天数
		Compress:   true, // 压缩旧文件
	})
	
	logger.SetFormatter(&logrus.JSONFormatter{})
	
	// 使用 logger
	for i := 0; i < 1000; i++ {
		logger.WithField("iteration", i).Info("循环日志")
	}
}

package main

import (
	"context"
	"fmt"
	"log/slog"
	"os"

	"go.opentelemetry.io/otel/trace"
)

func TraceLogger(handler slog.Handler) *slog.Logger {
	return slog.New(&traceHandler{Handler: handler})
}

type traceHandler struct {
	slog.Handler
}

func (h *traceHandler) Handle(ctx context.Context, r slog.Record) error {
	span := trace.SpanFromContext(ctx)
	if span.SpanContext().IsValid() {
		r.AddAttrs(
			slog.String("trace_id", span.SpanContext().TraceID().String()),
			slog.String("span_id", span.SpanContext().SpanID().String()),
		)
	}
	return h.Handler.Handle(ctx, r)
}

func main() {
	logger := TraceLogger(slog.NewJSONHandler(os.Stdout, nil))
	ctx := context.Background()
	// 实际使用时应注入真实的 tracing context
	logger.InfoContext(ctx, "处理支付请求", slog.String("order_id", "ORD-999"))
	fmt.Println("trace_id 已注入日志")
}

// Debug：调试信息，开发环境使用
logger.Debug("数据库连接池状态", zap.Int("active", 5), zap.Int("idle", 10))

// Info：正常操作信息
logger.Info("用户登录成功", zap.Int("user_id", 123))

// Warn：警告信息，可能的问题
logger.Warn("数据库查询慢", zap.Duration("duration", 2*time.Second))

// Error：错误信息，但不影响系统运行
logger.Error("发送邮件失败", zap.Error(err), zap.String("email", "user@example.com"))

// Fatal：严重错误，程序会退出
logger.Fatal("数据库连接失败", zap.Error(err))

// 不要记录敏感信息
// logger.Info("用户登录", zap.String("password", password))

// 只记录必要信息
logger.Info("用户登录", zap.Int("user_id", userID), zap.String("ip", clientIP))

type contextKey string

const loggerKey contextKey = "logger"

func WithLogger(ctx context.Context, logger *logrus.Entry) context.Context {
	return context.WithValue(ctx, loggerKey, logger)
}

func LoggerFromContext(ctx context.Context) *logrus.Entry {
	logger, ok := ctx.Value(loggerKey).(*logrus.Entry)
	if !ok {
		return logrus.NewEntry(logrus.StandardLogger())
	}
	return logger
}

func handleRequest(w http.ResponseWriter, r *http.Request) {
	// 创建带请求 ID 的 logger
	requestID := r.Header.Get("X-Request-ID")
	logger := logrus.WithField("request_id", requestID)
	
	// 将 logger 放入 context
	ctx := WithLogger(r.Context(), logger)
	
	// 在后续处理中使用
	processRequest(ctx)
}

func processRequest(ctx context.Context) {
	logger := LoggerFromContext(ctx)
	logger.Info("处理请求")
}

// 即使日志级别不够，也会执行字符串拼接
// logger.Debug("用户信息: " + user.String())

// 使用 WithFields 延迟求值
logger.WithField("user", user).Debug("用户信息")

// 使用 zap 的懒加载
logger.Debug("用户信息", zap.Stringer("user", user))

package main

import (
	"context"
	"net/http"
	"os"
	"time"
	
	"github.com/google/uuid"
	"github.com/sirupsen/logrus"
	"gopkg.in/natefinish/lumberjack.v2"
)

type Logger struct {
	*logrus.Logger
}

func NewLogger() *Logger {
	logger := logrus.New()
	
	// 设置输出
	if os.Getenv("ENV") == "production" {
		logger.SetOutput(&lumberjack.Logger{
			Filename:   "/var/log/myapp/app.log",
			MaxSize:    100,
			MaxBackups: 30,
			MaxAge:     7,
			Compress:   true,
		})
		logger.SetFormatter(&logrus.JSONFormatter{})
	} else {
		logger.SetFormatter(&logrus.TextFormatter{
			FullTimestamp: true,
		})
	}
	
	// 设置级别
	level, err := logrus.ParseLevel(os.Getenv("LOG_LEVEL"))
	if err != nil {
		level = logrus.InfoLevel
	}
	logger.SetLevel(level)
	
	return &Logger{logger}
}

func (l *Logger) WithContext(ctx context.Context) *logrus.Entry {
	entry := l.Logger.WithContext(ctx)
	
	if requestID, ok := ctx.Value("request_id").(string); ok {
		entry = entry.WithField("request_id", requestID)
	}
	
	if userID, ok := ctx.Value("user_id").(int); ok {
		entry = entry.WithField("user_id", userID)
	}
	
	return entry
}

type contextKey string

const loggerKey contextKey = "logger"

func LoggingMiddleware(logger *Logger) func(http.Handler) http.Handler {
	return func(next http.Handler) http.Handler {
		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
			// 生成请求 ID
			requestID := r.Header.Get("X-Request-ID")
			if requestID == "" {
				requestID = uuid.New().String()
			}
			
			// 创建请求上下文
			ctx := context.WithValue(r.Context(), "request_id", requestID)
			
			// 创建请求 logger
			reqLogger := logger.WithContext(ctx).WithFields(logrus.Fields{
				"method":      r.Method,
				"path":        r.URL.Path,
				"remote_addr": r.RemoteAddr,
			})
			
			// 将 logger 放入 context
			ctx = context.WithValue(ctx, loggerKey, reqLogger)
			
			start := time.Now()
			reqLogger.Info("请求开始")
			
			// 添加请求 ID 到响应头
			w.Header().Set("X-Request-ID", requestID)
			
			// 调用下一个处理器
			next.ServeHTTP(w, r.WithContext(ctx))
			
			duration := time.Since(start)
			reqLogger.WithField("duration", duration.String()).Info("请求完成")
		})
	}
}

var logger = NewLogger()

func handler(w http.ResponseWriter, r *http.Request) {
	// 从 context 获取 logger
	reqLogger := r.Context().Value(loggerKey).(*logrus.Entry)
	
	reqLogger.Info("处理业务逻辑")
	
	// 模拟处理
	time.Sleep(100 * time.Millisecond)
	
	reqLogger.Info("业务逻辑完成")
	
	w.Write([]byte("Hello, World!"))
}

func main() {
	mux := http.NewServeMux()
	mux.HandleFunc("/", handler)
	
	handler := LoggingMiddleware(logger)(mux)
	
	logger.Info("服务器启动在 :8080")
	http.ListenAndServe(":8080", handler)
}

{"time":"2024-01-15T10:23:45+08:00","level":"INFO","msg":"请求完成","method":"GET","path":"/api/users","status":200,"duration_ms":45}

type AsyncWriter struct {
    ch     chan []byte
    writer io.Writer
    stop   chan struct{}
}

func NewAsyncWriter(w io.Writer, bufferSize int) *AsyncWriter {
    aw := &AsyncWriter{
        ch:     make(chan []byte, bufferSize),
        writer: w,
        stop:   make(chan struct{}),
    }
    go aw.processLoop()
    return aw
}

func (aw *AsyncWriter) processLoop() {
    ticker := time.NewTicker(100 * time.Millisecond)
    defer ticker.Stop()
    var batch [][]byte
    for {
        select {
        case data := <-aw.ch:
            batch = append(batch, data)
            if len(batch) >= 100 {
                aw.flush(batch)
                batch = batch[:0]
            }
        case <-ticker.C:
            if len(batch) > 0 {
                aw.flush(batch)
                batch = batch[:0]
            }
        case <-aw.stop:
            aw.flush(batch)
            return
        }
    }
}

func (aw *AsyncWriter) flush(batch [][]byte) {
    for _, data := range batch {
        aw.writer.Write(data)
    }
}

// 概率采样：1% 的日志被记录
func ProbabilisticSample(next slog.Handler) slog.Handler {
    return &sampleHandler{next: next, rate: 0.01}
}

type sampleHandler struct {
    next slog.Handler
    rate float64
}

func (h *sampleHandler) Enabled(ctx context.Context, level slog.Level) bool {
    return h.next.Enabled(ctx, level)
}

func (h *sampleHandler) Handle(ctx context.Context, r slog.Record) error {
    if rand.Float64() < h.rate {
        return h.next.Handle(ctx, r)
    }
    return nil
}

package main

import (
    "go.uber.org/zap"
    "go.uber.org/zap/zapcore"
    "gopkg.in/natefinch/lumberjack.v2"
)

func NewZapLogger() *zap.Logger {
    // 文件轮转 + 控制台双输出
    fileWriter := zapcore.AddSync(&lumberjack.Logger{
        Filename:   "/var/log/app/app.log",
        MaxSize:    100, // MB
        MaxBackups: 30,
        MaxAge:     7,
        Compress:   true,
    })

    consoleWriter := zapcore.AddSync(os.Stdout)

    encoderConfig := zapcore.EncoderConfig{
        TimeKey:        "ts",
        LevelKey:       "level",
        NameKey:        "logger",
        CallerKey:      "caller",
        MessageKey:     "msg",
        StacktraceKey:  "stacktrace",
        EncodeLevel:    zapcore.LowercaseLevelEncoder,
        EncodeTime:     zapcore.ISO8601TimeEncoder,
        EncodeDuration: zapcore.SecondsDurationEncoder,
    }

    fileEncoder := zapcore.NewJSONEncoder(encoderConfig)
    consoleEncoder := zapcore.NewConsoleEncoder(encoderConfig)

    core := zapcore.NewTee(
        zapcore.NewCore(fileEncoder, fileWriter, zapcore.InfoLevel),
        zapcore.NewCore(consoleEncoder, consoleWriter, zapcore.DebugLevel),
    )

    return zap.New(core, zap.AddCaller(), zap.AddStacktrace(zapcore.ErrorLevel))
}

// 使用
logger.Info("用户登录",
    zap.String("user_id", "12345"),
    zap.String("ip", "192.168.1.1"),
    zap.Duration("latency", 45*time.Millisecond),
)

package main

import (
    "log/slog"
    "os"
    "time"
)

func NewSlogLogger() *slog.Logger {
    handler := slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{
        Level:       slog.LevelDebug,
        AddSource:   true,
        ReplaceAttr: func(groups []string, a slog.Attr) slog.Attr {
            if a.Key == slog.TimeKey {
                return slog.Attr{Key: a.Key, Value: slog.StringValue(a.Value.Time().Format(time.RFC3339Nano))}
            }
            return a
        },
    })
    return slog.New(handler)
}

// 使用
logger := NewSlogLogger()
logger.Info("请求处理",
    slog.String("method", "GET"),
    slog.String("path", "/api/users"),
    slog.Int("status", 200),
    slog.Duration("duration", 45*time.Millisecond),
    slog.Group("meta",
        slog.String("version", "1.0.0"),
        slog.String("env", "production"),
    ),
)

type traceHandler struct {
    handler slog.Handler
}

func (h *traceHandler) Handle(ctx context.Context, r slog.Record) error {
    if traceID, ok := ctx.Value("trace_id").(string); ok {
        r.AddAttrs(slog.String("trace_id", traceID))
    }
    return h.handler.Handle(ctx, r)
}

func (h *traceHandler) Enabled(ctx context.Context, level slog.Level) bool {
    return h.handler.Enabled(ctx, level)
}

func (h *traceHandler) WithAttrs(attrs []slog.Attr) slog.Handler {
    return &traceHandler{handler: h.handler.WithAttrs(attrs)}
}

func (h *traceHandler) WithGroup(name string) slog.Handler {
    return &traceHandler{handler: h.handler.WithGroup(name)}
}

func WithTraceID(ctx context.Context, traceID string) context.Context {
    return context.WithValue(ctx, "trace_id", traceID)
}

// HTTP 中间件中提取并传递 trace_id
func TraceMiddleware(next http.Handler) http.Handler {
    return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
        traceID := r.Header.Get("X-Trace-ID")
        if traceID == "" {
            traceID = generateTraceID()
        }
        ctx := WithTraceID(r.Context(), traceID)
        w.Header().Set("X-Trace-ID", traceID)
        next.ServeHTTP(w, r.WithContext(ctx))
    })
}

// 已使用 lumberjack 实现按大小轮转
// ELK 收集：使用 Filebeat 监控日志目录

// filebeat.yml 示例配置：
// filebeat.inputs:
// - type: log
//   enabled: true
//   paths:
//     - /var/log/app/*.log
//   json.keys_under_root: true
//   json.add_error_key: true
// output.logstash:
//   hosts: ["logstash:5044"]

func BenchmarkLogStdlib(b *testing.B) {
    log.SetOutput(io.Discard)
    for i := 0; i < b.N; i++ {
        log.Printf("user %d logged in", i)
    }
}

func BenchmarkZapStructured(b *testing.B) {
    logger := zap.New(zapcore.NewCore(
        zapcore.NewJSONEncoder(zap.NewProductionEncoderConfig()),
        zapcore.AddSync(io.Discard), zap.InfoLevel,
    ))
    for i := 0; i < b.N; i++ {
        logger.Info("user logged in", zap.Int("user_id", i))
    }
}