「产品矩阵平台」DevOps 与可观测性

trace_id -> gateway log -> service span -> db query -> event publish -> worker log

# .github/workflows/platform.yml
name: Platform CI/CD

on:
  push:
    branches: [main, develop]
  pull_request:
    branches: [main]

env:
  GO_VERSION: "1.22"
  REGISTRY: ghcr.io
  IMAGE_NAME: ${{ github.repository }}

jobs:
  lint-and-test:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-go@v5
        with:
          go-version: ${{ env.GO_VERSION }}
      - name: Lint
        uses: golangci/golangci-lint-action@v6
        with:
          version: latest
          args: --timeout=5m
      - name: Test
        run: go test -race -coverprofile=coverage.out ./...
      - name: Coverage
        run: go tool cover -func=coverage.out

  security-scan:
    needs: lint-and-test
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: Trivy Scan
        uses: aquasecurity/trivy-action@master
        with:
          scan-type: 'fs'
          severity: 'HIGH,CRITICAL'
      - name: Secret Scan
        uses: trufflesecurity/trufflehog@main
        with:
          path: ./
          base: main
          head: HEAD

  build-and-push:
    needs: [lint-and-test, security-scan]
    runs-on: ubuntu-latest
    if: github.ref == 'refs/heads/main'
    steps:
      - uses: actions/checkout@v4
      - uses: docker/setup-buildx-action@v3
      - uses: docker/login-action@v3
        with:
          registry: ${{ env.REGISTRY }}
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}
      - name: Build and Push
        uses: docker/build-push-action@v5
        with:
          context: .
          push: true
          tags: |
            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ github.sha }}
            ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest
          cache-from: type=gha
          cache-to: type=gha,mode=max
          platforms: linux/amd64,linux/arm64

  deploy-staging:
    needs: build-and-push
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: Deploy to Staging
        run: |
          echo "${{ secrets.KUBECONFIG_STAGING }}" | base64 -d > kubeconfig
          kubectl --kubeconfig=kubeconfig set image deployment/platform-app             app=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ github.sha }}
          kubectl --kubeconfig=kubeconfig rollout status deployment/platform-app --timeout=5m
          kubectl --kubeconfig=kubeconfig delete pod -l app=platform-app --grace-period=30

# Build stage
FROM golang:1.22-alpine AS builder
WORKDIR /app
COPY go.mod go.sum ./
RUN go mod download
COPY . .

# 启用 CGO=0 以获得静态链接二进制文件
# -ldflags 去掉符号表和调试信息，减小体积
RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build     -ldflags="-s -w -X main.Version=$(git describe --tags) -X main.BuildTime=$(date -u +%Y-%m-%dT%H:%M:%SZ)"     -o platform     ./cmd/platform

# Runtime stage（最小化攻击面）
FROM gcr.io/distroless/static:nonroot
WORKDIR /
COPY --from=builder /app/platform /platform
COPY --from=builder /app/config /config

USER nonroot:nonroot

EXPOSE 8080
HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3     CMD ["/platform", "healthcheck"] || exit 1

ENTRYPOINT ["/platform"]

# k8s/deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: platform-app
  namespace: default
  labels:
    app: platform-app
    version: {{ .Values.image.tag }}
spec:
  replicas: 3
  strategy:
    type: RollingUpdate
    rollingUpdate:
      maxSurge: 1
      maxUnavailable: 0
  selector:
    matchLabels:
      app: platform-app
  template:
    metadata:
      labels:
        app: platform-app
      annotations:
        prometheus.io/scrape: "true"
        prometheus.io/port: "8080"
        prometheus.io/path: "/metrics"
    spec:
      affinity:
        podAntiAffinity:
          preferredDuringSchedulingIgnoredDuringExecution:
            - weight: 100
              podAffinityTerm:
                labelSelector:
                  matchExpressions:
                    - key: app
                      operator: In
                      values:
                        - platform-app
                topologyKey: kubernetes.io/hostname
      containers:
        - name: app
          image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
          ports:
            - containerPort: 8080
              name: http
          env:
            - name: GO_ENV
              value: "production"
            - name: DATABASE_URL
              valueFrom:
                secretKeyRef:
                  name: platform-secrets
                  key: database_url
            - name: REDIS_URL
              valueFrom:
                secretKeyRef:
                  name: platform-secrets
                  key: redis_url
          resources:
            requests:
              memory: "256Mi"
              cpu: "250m"
            limits:
              memory: "512Mi"
              cpu: "500m"
          livenessProbe:
            httpGet:
              path: /healthz
              port: 8080
            initialDelaySeconds: 5
            periodSeconds: 10
          readinessProbe:
            httpGet:
              path: /readyz
              port: 8080
            initialDelaySeconds: 3
            periodSeconds: 5
          volumeMounts:
            - name: config
              mountPath: /config
      volumes:
        - name: config
          configMap:
            name: platform-config
---
# Horizontal Pod Autoscaler
apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
  name: platform-app-hpa
spec:
  scaleTargetRef:
    apiVersion: apps/v1
    kind: Deployment
    name: platform-app
  minReplicas: 3
  maxReplicas: 20
  metrics:
    - type: Resource
      resource:
        name: cpu
        target:
          type: Utilization
          averageUtilization: 70
    - type: Pods
      pods:
        metric:
          name: http_requests_per_second
        target:
          type: AverageValue
          averageValue: "100"

---
# Service + Ingress
apiVersion: v1
kind: Service
metadata:
  name: platform-app
spec:
  selector:
    app: platform-app
  ports:
    - port: 80
      targetPort: 8080
---
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: platform-ingress
  annotations:
    cert-manager.io/cluster-issuer: "letsencrypt"
    nginx.ingress.kubernetes.io/rate-limit: "100"
spec:
  tls:
    - hosts:
        - api.platform.com
      secretName: platform-tls
  rules:
    - host: api.platform.com
      http:
        paths:
          - path: /
            pathType: Prefix
            backend:
              service:
                name: platform-app
                port:
                  number: 80

package logger

import (
    "context"
    "encoding/json"
    "net/http"
    "os"
    "time"
    "github.com/google/uuid"
)

type StructuredLog struct {
    Timestamp time.Time              `json:"ts"`
    Level     string                 `json:"level"`
    TraceID   string                 `json:"trace_id"`
    TenantID  string                 `json:"tenant_id,omitempty"`
    AppID     string                 `json:"app_id,omitempty"`
    UserID    string                 `json:"user_id,omitempty"`
    Method    string                 `json:"method,omitempty"`
    Path      string                 `json:"path,omitempty"`
    Status    int                    `json:"status,omitempty"`
    Duration  int64                  `json:"duration_ms,omitempty"`
    Error     string                 `json:"error,omitempty"`
    Message   string                 `json:"msg"`
    Extra     map[string]interface{} `json:"extra,omitempty"`
}

func Log(level string, msg string, fields ...map[string]interface{}) {
    entry := StructuredLog{
        Timestamp: time.Now().UTC(),
        Level:     level,
        Message:   msg,
    }
    if len(fields) > 0 {
        entry.Extra = fields[0]
    }
    data, _ := json.Marshal(entry)
    os.Stdout.Write(data)
    os.Stdout.Write([]byte("\n"))
}

// RequestLogger 中间件：记录每个 HTTP 请求
func RequestLogger(next http.Handler) http.Handler {
    return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
        start := time.Now()
        traceID := r.Header.Get("X-Request-ID")
        if traceID == "" {
            traceID = uuid.New().String()
        }

        ctx := context.WithValue(r.Context(), "trace_id", traceID)
        ctx = context.WithValue(ctx, "tenant_id", r.Header.Get("X-Tenant-ID"))
        ctx = context.WithValue(ctx, "app_id", r.Header.Get("X-App-ID"))

        ww := &responseWriter{ResponseWriter: w, statusCode: 200}
        next.ServeHTTP(ww, r.WithContext(ctx))

        entry := StructuredLog{
            Timestamp: time.Now().UTC(),
            Level:     "INFO",
            TraceID:   traceID,
            TenantID:  r.Header.Get("X-Tenant-ID"),
            AppID:     r.Header.Get("X-App-ID"),
            Method:    r.Method,
            Path:      r.URL.Path,
            Status:    ww.statusCode,
            Duration:  time.Since(start).Milliseconds(),
            Message:   "http_request",
        }
        if ww.statusCode >= 500 {
            entry.Level = "ERROR"
        } else if ww.statusCode >= 400 {
            entry.Level = "WARN"
        }
        data, _ := json.Marshal(entry)
        os.Stdout.Write(data)
        os.Stdout.Write([]byte("\n"))
    })
}

type responseWriter struct {
    http.ResponseWriter
    statusCode int
}

func (w *responseWriter) WriteHeader(code int) {
    w.statusCode = code
    w.ResponseWriter.WriteHeader(code)
}

package metrics

import (
    "context"
    "time"
    "github.com/prometheus/client_golang/prometheus"
    "github.com/prometheus/client_golang/prometheus/promauto"
    "github.com/prometheus/client_golang/prometheus/promhttp"
)

var (
    httpRequestsTotal = promauto.NewCounterVec(prometheus.CounterOpts{
        Name: "platform_http_requests_total",
        Help: "Total HTTP requests",
    }, []string{"method", "path", "status", "tenant_id"})

    httpRequestDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{
        Name:    "platform_http_request_duration_seconds",
        Help:    "HTTP request latency",
        Buckets: prometheus.DefBuckets,
    }, []string{"method", "path"})

    activeConnections = promauto.NewGauge(prometheus.GaugeOpts{
        Name: "platform_active_connections",
        Help: "Current active connections",
    })

    dbQueryDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{
        Name:    "platform_db_query_duration_seconds",
        Help:    "Database query latency",
        Buckets: []float64{.001, .005, .01, .025, .05, .1, .25, .5, 1},
    }, []string{"operation", "table"})

    businessEvents = promauto.NewCounterVec(prometheus.CounterOpts{
        Name: "platform_business_events_total",
        Help: "Business events count",
    }, []string{"event_type", "tenant_id"})
)

// InstrumentHandler 包装 HTTP Handler，自动采集 RED 指标
func InstrumentHandler(next http.HandlerFunc, route string) http.HandlerFunc {
    return func(w http.ResponseWriter, r *http.Request) {
        start := time.Now()
        activeConnections.Inc()
        defer activeConnections.Dec()

        tenantID := r.Header.Get("X-Tenant-ID")
        ww := &responseWriter{ResponseWriter: w, statusCode: 200}
        next(ww, r)

        duration := time.Since(start).Seconds()
        httpRequestsTotal.WithLabelValues(r.Method, route, fmt.Sprintf("%d", ww.statusCode), tenantID).Inc()
        httpRequestDuration.WithLabelValues(r.Method, route).Observe(duration)
    }
}

// InstrumentDB 包装数据库操作
func InstrumentDB(ctx context.Context, operation, table string, fn func() error) error {
    start := time.Now()
    err := fn()
    dbQueryDuration.WithLabelValues(operation, table).Observe(time.Since(start).Seconds())
    return err
}

// RecordBusinessEvent 记录业务事件
func RecordBusinessEvent(eventType, tenantID string) {
    businessEvents.WithLabelValues(eventType, tenantID).Inc()
}

// MetricsHandler 暴露 /metrics 端点
func MetricsHandler() http.Handler {
    return promhttp.Handler()
}