OpenTelemetry 完全指南：标准化可观测性框架

// Go: Context 自动传播
ctx, span := tracer.Start(ctx, "process-payment")
defer span.End()

// HTTP 请求自动注入追踪头
req, _ := http.NewRequestWithContext(ctx, "POST", url, body)
// 自动注入: traceparent: 00-xxx-xxx-01

// Baggage = 随 Trace 传播的键值对（会出现在所有下游 Span 中）
import "go.opentelemetry.io/otel/baggage"

// 设置 Baggage
member, _ := baggage.NewMember("user.id", "user_123")
bag, _ := baggage.New(member)
ctx = baggage.ContextWithBaggage(ctx, bag)

// 下游 Span 自动读取
userID := baggage.FromContext(ctx).Member("user.id").Value()

// otel/setup.go
package otel

import (
    "context"
    "time"

    "go.opentelemetry.io/otel"
    "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc"
    "go.opentelemetry.io/otel/sdk/resource"
    sdktrace "go.opentelemetry.io/otel/sdk/trace"
    semconv "go.opentelemetry.io/otel/semconv/v1.24.0"
)

func InitTracer(endpoint string) (*sdktrace.TracerProvider, error) {
    ctx := context.Background()

    // 创建 OTLP gRPC Exporter
    exporter, err := otlptracegrpc.New(ctx,
        otlptracegrpc.WithEndpoint(endpoint),
        otlptracegrpc.WithInsecure(),
    )
    if err != nil {
        return nil, err
    }

    // 资源信息（服务元数据）
    res, err := resource.New(ctx,
        resource.WithAttributes(
            semconv.ServiceName("payment-service"),
            semconv.ServiceVersion("1.2.3"),
            semconv.DeploymentEnvironment("production"),
            semconv.HostName("web-01"),
        ),
    )
    if err != nil {
        return nil, err
    }

    // TracerProvider
    tp := sdktrace.NewTracerProvider(
        sdktrace.WithBatcher(exporter,
            sdktrace.WithBatchTimeout(5*time.Second),
        ),
        sdktrace.WithResource(res),
        sdktrace.WithSampler(sdktrace.ParentBased(
            sdktrace.TraceIDRatioBased(0.1), // 10% 采样
        )),
    )

    otel.SetTracerProvider(tp)
    return tp, nil
}

// main.go
package main

import (
    "context"
    "fmt"
    "net/http"
    "time"

    "go.opentelemetry.io/otel"
    "go.opentelemetry.io/otel/attribute"
    "go.opentelemetry.io/otel/codes"
    "go.opentelemetry.io/otel/trace"
)

var tracer = otel.Tracer("payment-service")

func main() {
    tp, _ := otel.InitTracer("otel-collector:4317")
    defer tp.Shutdown(context.Background())

    http.HandleFunc("/api/payment", handlePayment)
    http.ListenAndServe(":8080", nil)
}

func handlePayment(w http.ResponseWriter, r *http.Request) {
    ctx, span := tracer.Start(r.Context(), "handle-payment",
        trace.WithSpanKind(trace.SpanKindServer),
        trace.WithAttributes(
            attribute.String("http.method", r.Method),
            attribute.String("http.route", "/api/payment"),
            attribute.String("http.target", r.URL.Path),
        ),
    )
    defer span.End()

    // 解析参数
    userID := r.URL.Query().Get("user_id")
    span.SetAttributes(attribute.String("user.id", userID))

    // 验证
    if err := validateUser(ctx, userID); err != nil {
        span.SetStatus(codes.Error, err.Error())
        span.RecordError(err)
        http.Error(w, err.Error(), 400)
        return
    }

    // 处理支付
    if err := processPayment(ctx, userID); err != nil {
        span.SetStatus(codes.Error, "payment failed")
        span.RecordError(err)
        http.Error(w, "Payment failed", 500)
        return
    }

    span.SetAttributes(attribute.Bool("payment.success", true))
    w.Write([]byte("Payment successful"))
}

func processPayment(ctx context.Context, userID string) error {
    ctx, span := tracer.Start(ctx, "process-payment",
        trace.WithSpanKind(trace.SpanKindInternal),
    )
    defer span.End()

    start := time.Now()

    // 调用银行接口
    if err := callBankAPI(ctx); err != nil {
        span.SetAttributes(
            attribute.Bool("bank.api.success", false),
            attribute.Float64("bank.api.duration_ms", float64(time.Since(start).Milliseconds())),
        )
        return err
    }

    span.SetAttributes(
        attribute.Bool("bank.api.success", true),
        attribute.Float64("bank.api.duration_ms", float64(time.Since(start).Milliseconds())),
    )
    return nil
}

// gRPC 服务端自动埋点
import (
    "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc"
    "google.golang.org/grpc"
)

server := grpc.NewServer(
    grpc.UnaryInterceptor(otelgrpc.UnaryServerInterceptor()),
    grpc.StreamInterceptor(otelgrpc.StreamServerInterceptor()),
)

// gRPC 客户端自动传播
conn, _ := grpc.Dial(addr,
    grpc.WithUnaryInterceptor(otelgrpc.UnaryClientInterceptor()),
)

# otel-collector-config.yaml
receivers:
  otlp:
    protocols:
      grpc:
        endpoint: 0.0.0.0:4317
      http:
        endpoint: 0.0.0.0:4318

  prometheus:
    config:
      scrape_configs:
        - job_name: 'otel-collector'
          scrape_interval: 10s
          static_configs:
            - targets: ['localhost:8888']

processors:
  batch:
    timeout: 1s
    send_batch_size: 1024

  memory_limiter:
    limit_mib: 512
    spike_limit_mib: 128

  resource:
    attributes:
      - key: environment
        value: production
        action: upsert

  tail_sampling:
    decision_wait: 10s
    num_traces: 100
    expected_new_traces_per_sec: 1000
    policies:
      - name: errors
        type: status_code
        status_code: { status_codes: [ERROR] }
      - name: slow_requests
        type: latency
        latency: { threshold_ms: 1000 }

exporters:
  otlp/jaeger:
    endpoint: jaeger:4317
    tls:
      insecure: true

  prometheusremotewrite:
    endpoint: http://prometheus:9090/api/v1/write

  loki:
    endpoint: http://loki:3100/loki/api/v1/push

  otlp/tempo:
    endpoint: tempo:4317
    tls:
      insecure: true

service:
  pipelines:
    traces:
      receivers: [otlp, jaeger, zipkin]
      processors: [memory_limiter, tail_sampling, batch]
      exporters: [otlp/jaeger, otlp/tempo]

    metrics:
      receivers: [otlp, prometheus]
      processors: [memory_limiter, resource, batch]
      exporters: [prometheusremotewrite]

    logs:
      receivers: [otlp]
      processors: [memory_limiter, resource, batch]
      exporters: [loki]

// AlwaysOn — 全部采样（开发环境）
sdktrace.AlwaysSample()

// AlwaysOff — 不采样（仅埋点测试）
sdktrace.NeverSample()

// TraceIDRatioBased — 按比例采样
sdktrace.TraceIDRatioBased(0.01) // 1%

// ParentBased — 跟随父 Span 采样决策（推荐）
sdktrace.ParentBased(
    sdktrace.TraceIDRatioBased(0.1),
    // 可自定义父 Span 状态的行为
)

// 按 Span 属性采样（Collector Tail Sampling）
// 保留所有错误和慢请求，其他随机 1%

// Spring Boot Auto Instrumentation（零代码）
// java -javaagent:opentelemetry-javaagent.jar \
//      -Dotel.service.name=order-service \
//      -Dotel.traces.exporter=otlp \
//      -jar app.jar

// 手动埋点
import io.opentelemetry.api.trace.Span;
import io.opentelemetry.api.trace.Tracer;

Span span = tracer.spanBuilder("process-order").startSpan();
try (Scope scope = span.makeCurrent()) {
    span.setAttribute("order.id", orderId);
    processOrder();
} catch (Exception e) {
    span.recordException(e);
    span.setStatus(StatusCode.ERROR);
    throw e;
} finally {
    span.end();
}

// Auto Instrumentation
const { NodeSDK } = require('@opentelemetry/sdk-node');
const { OTLPTraceExporter } = require('@opentelemetry/exporter-trace-otlp-grpc');

const sdk = new NodeSDK({
  traceExporter: new OTLPTraceExporter({ url: 'http://otel-collector:4317' }),
  serviceName: 'api-service'
});
sdk.start();

// Express 自动拦截
const { registerInstrumentations } = require('@opentelemetry/instrumentation');
const { HttpInstrumentation } = require('@opentelemetry/instrumentation-http');
const { ExpressInstrumentation } = require('@opentelemetry/instrumentation-express');

registerInstrumentations({
  instrumentations: [new HttpInstrumentation(), new ExpressInstrumentation()]
});

# Auto Instrumentation
# opentelemetry-instrument --traces_exporter otlp \
#     --service_name payment-service \
#     python app.py

# 手动埋点
from opentelemetry import trace
from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import BatchSpanProcessor

trace.set_tracer_provider(TracerProvider())
tracer = trace.get_tracer(__name__)

span_processor = BatchSpanProcessor(OTLPSpanExporter(endpoint="otel-collector:4317"))
trace.get_tracer_provider().add_span_processor(span_processor)

with tracer.start_as_current_span("process-payment") as span:
    span.set_attribute("user.id", user_id)
    process_payment()