可观测性成本治理：采样降噪、数据生命周期与存储成本优化实战

# ❌ 反模式：请求耗时按 user_id 分桶 → 百万序列
http_request_duration_ms{user_id="12345"} 0.1
http_request_duration_ms{user_id="67890"} 0.2
# 100 万用户 = 100 万条时间序列 = 存储与查询爆炸

// ❌ 不推荐：把 user_id 写入指标标签
histogram.WithLabelValues(userID, endpoint).Observe(dur)

// ✅ 推荐：指标只保留低基数维度
// 用户维度的耗时分析交给 tracing（span attribute 携带 user_id）
histogram.WithLabelValues(endpoint).Observe(dur)

# prometheus rules：原始数据保留 15 天，预聚合保留 1 年
groups:
- name: service_aggregation
  interval: 1m
  rules:
  - record: job:http_requests:rate5m
    expr: sum(rate(http_requests_total[5m])) by (job)
  - record: job:http_request_duration:p95_5m
    expr: histogram_quantile(0.95, sum(rate(http_request_duration_seconds_bucket[5m])) by (le, job))

# vector / fluent-bit 处理管道示例
transforms:
  filter_low_value:
    type: filter
    inputs: [logs]
    condition:
      type: or
      conditions:
        - type: level_selector
          level: warn
          operator: gte
        - type: message_selector   # 业务白名单，如订单完成事件
          match: "^order_(created|paid|shipped):"

# 丢弃高体积低价值的字段
remap:
  drop_fields:
    - "user_agent"      # 高基数且可重建
    - "request_headers"
    - "debug_stack"
  convert:
    - from: "duration_ns"
      to: "duration_ms"
      type: float

transforms:
  sample_healthcheck:
    type: sampler
    inputs: [logs]
    rate: 0.01          # 只保留 1%
    key_field: "log_key" # 按 log_key 保持一致性（同 key 同采样决策）

// OpenTelemetry 按比例采样
tp := sdktrace.NewTracerProvider(
	sdktrace.WithSampler(
		sdktrace.ParentBased(
			sdktrace.TraceIDRatioBased(0.1), // 10% 采样率
		),
	),
)

# OpenTelemetry Collector tail sampling 配置
processors:
  tail_sampling:
    decision_wait: 10s          # 缓冲窗口，等待整条 trace 到达
    policies:
      - name: errors
        type: status_code
        status_code: { status_codes: [ERROR] }
      - name: slow
        type: latency
        latency: { threshold_ms: 500 }
      - name: random-sample
        type: probabilistic
        probabilistic: { sampling_percentage: 1 }

-- ClickHouse 表级 TTL：热→温→冷→删
CREATE TABLE app_logs (
  ts          DateTime64(3),
  level       LowCardinality(String),
  trace_id    String,
  service     LowCardinality(String),
  message     String
)
ENGINE = MergeTree
PARTITION BY toYYYYMM(ts)
ORDER BY (service, ts)
TTL ts + INTERVAL 7 DAY  TO VOLUME 'hot',
    ts + INTERVAL 90 DAY TO VOLUME 'warm',
    ts + INTERVAL 365 DAY DELETE;

chunk_store_config:
  chunk_target_size: 1536000      # 增大块，提升压缩比
chunk_encoding: gzip               # gzip 压缩比优于 snappy
compactor:
  compaction_interval: 10m
  retention_enabled: true
  retention_period: 168h           # 7 天热数据

# Prometheus 丢弃高基数 label（当心：会影响查询，需谨慎）
scrape_configs:
  - job_name: app
    metric_relabel_configs:
      - source_labels: [user_id]
        regex: ".*"
        action: drop

# Prometheus Alertmanager
route:
  group_by: ['alertname', 'cluster']   # 按告警名+集群聚合
  group_wait: 30s
  group_interval: 5m
  repeat_interval: 4h
  receiver: ops-critical
  routes:
  - match:
      severity: critical
    receiver: oncall-page
  - match:
      severity: warning
    receiver: oncall-slack

inhibit_rules:
- source_match: { severity: critical }
  target_match: { severity: warning }
  equal: ['cluster']                    # 同一集群 critical 抑制 warning

// OpenTelemetry resource 标签
res := resource.NewWithAttributes(
	semconv.SchemaURL,
	attribute.String("service.name", "payment-service"),
	attribute.String("team", "payments"),
	attribute.String("env", "prod"),
)