Kubernetes 安全加固

# kube-apiserver 启动参数
--anonymous-auth=false                    # 禁止匿名访问
--basic-auth-file=""                      # 禁用基本认证
--token-auth-file=""                      # 禁用静态 Token 文件
--oidc-issuer-url=https://auth.example.com  # 使用 OIDC
--oidc-client-id=kubernetes
--oidc-username-claim=email
--oidc-groups-claim=groups
--authorization-mode=RBAC,Node             # RBAC + Node 授权
--enable-admission-plugins=NodeRestriction,PodSecurityPolicy,...
--audit-log-path=/var/log/audit.log
--audit-policy-file=/etc/kubernetes/audit-policy.yaml
--tls-cert-file=/etc/kubernetes/pki/apiserver.crt
--tls-private-key-file=/etc/kubernetes/pki/apiserver.key

# audit-policy.yaml
apiVersion: audit.k8s.io/v1
kind: Policy
rules:
  # 记录所有 Secret 变更
  - level: RequestResponse
    resources:
      - group: ""
        resources: ["secrets", "configmaps"]
    verbs: ["create", "update", "patch", "delete"]

  # 记录所有认证失败
  - level: Metadata
    omitStages:
      - RequestReceived
    userGroups: ["system:authenticated"]

  # 记录所有权限拒绝
  - level: Metadata
    omitStages:
      - RequestReceived
    verbs: ["create", "update", "patch", "delete"]

  # 默认：不记录读操作
  - level: None
    verbs: ["get", "list", "watch"]

# 开发人员权限（仅特定 namespace）
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
  namespace: development
  name: developer
rules:
- apiGroups: [""]
  resources: ["pods", "services", "configmaps"]
  verbs: ["get", "list", "watch", "create", "update", "patch"]
- apiGroups: ["apps"]
  resources: ["deployments", "replicasets"]
  verbs: ["get", "list", "watch", "create", "update", "patch"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
  name: developer-binding
  namespace: development
subjects:
- kind: Group
  name: "dev-team@example.com"
  apiGroup: rbac.authorization.k8s.io
roleRef:
  kind: Role
  name: developer
  apiGroup: rbac.authorization.k8s.io

# 只读集群权限（运维监控）
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: cluster-reader
rules:
- apiGroups: ["*"]
  resources: ["*"]
  verbs: ["get", "list", "watch"]
---
# 禁止的权限模式
# ❌ cluster-admin 给所有人
# ❌ secrets 读权限给不需要的人
# ❌ 允许 exec 进入容器（除非必要）

apiVersion: v1
kind: ServiceAccount
metadata:
  name: app-sa
  namespace: production
automountServiceAccountToken: false  # ❗ 不需要 API 访问时不挂载 Token
---
# 需要 API 访问时，给最小权限
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
  name: minimal-api
  namespace: production
rules:
- apiGroups: [""]
  resources: ["configmaps"]
  verbs: ["get", "list"]
  resourceNames: ["app-config"]  # 仅限特定 ConfigMap

# 默认拒绝所有入站和出站流量（除 DNS）
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
  name: default-deny-all
  namespace: production
spec:
  podSelector: {}
  policyTypes:
    - Ingress
    - Egress
---
# 允许 DNS 解析
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
  name: allow-dns
  namespace: production
spec:
  podSelector: {}
  policyTypes:
    - Egress
  egress:
    - to:
      - namespaceSelector:
          matchLabels:
            kubernetes.io/metadata.name: kube-system
      ports:
        - protocol: UDP
          port: 53

# frontend 只允许访问 backend 的 8080 端口
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
  name: frontend-policy
  namespace: production
spec:
  podSelector:
    matchLabels:
      app: frontend
  policyTypes:
    - Egress
  egress:
    - to:
        - podSelector:
            matchLabels:
              app: backend
      ports:
        - protocol: TCP
          port: 8080
    - to:  # 允许访问 kube-dns
        - namespaceSelector:
            matchLabels:
              name: kube-system
      ports:
        - protocol: UDP
          port: 53

# 命名空间级策略
apiVersion: v1
kind: Namespace
metadata:
  name: restricted-ns
  labels:
    pod-security.kubernetes.io/enforce: restricted
    pod-security.kubernetes.io/enforce-version: latest
    pod-security.kubernetes.io/audit: restricted
    pod-security.kubernetes.io/warn: restricted
---
# 受限 Pod 示例
apiVersion: v1
kind: Pod
metadata:
  name: secure-app
  namespace: restricted-ns
spec:
  securityContext:
    runAsNonRoot: true
    seccompProfile:
      type: RuntimeDefault
  containers:
    - name: app
      image: myapp:latest
      securityContext:
        allowPrivilegeEscalation: false
        readOnlyRootFilesystem: true
        capabilities:
          drop:
            - ALL
        runAsUser: 1000
        runAsGroup: 1000

# 禁止镜像使用 latest 标签
apiVersion: templates.gatekeeper.sh/v1beta1
kind: ConstraintTemplate
metadata:
  name: k8srequiredtags
spec:
  crd:
    spec:
      names:
        kind: K8sRequiredTags
  targets:
    - target: admission.k8s.gatekeeper.sh
      rego: |
        package k8srequiredtags
        violation[{"msg": msg}] {
          image := input.review.object.spec.containers[_].image
          endswith(image, ":latest")
          msg := sprintf("Container image must not use latest tag: %v", [image])
        }
---
apiVersion: constraints.gatekeeper.sh/v1beta1
kind: K8sRequiredTags
metadata:
  name: require-image-tags
spec:
  match:
    kinds:
      - apiGroups: [""]
        kinds: ["Pod"]

apiVersion: kyverno.io/v1
kind: ClusterPolicy
metadata:
  name: require-ro-rootfs
spec:
  validationFailureAction: enforce
  rules:
    - name: check-read-only-root-fs
      match:
        resources:
          kinds:
            - Pod
      validate:
        message: "Pod 必须设置 readOnlyRootFilesystem: true"
        pattern:
          spec:
            containers:
              - securityContext:
                  readOnlyRootFilesystem: true

# 创建加密配置文件
apiVersion: apiserver.config.k8s.io/v1
kind: EncryptionConfiguration
resources:
  - resources:
      - secrets
      - configmaps
    providers:
      - aescbc:
          keys:
            - name: key1
              secret: <base64-encoded-key>
      - identity: {}  # 允许未加密读取

# kube-apiserver 添加参数
--encryption-provider-config=/etc/kubernetes/encryption-config.yaml

# External Secrets Operator + HashiCorp Vault
apiVersion: external-secrets.io/v1beta1
kind: ExternalSecret
metadata:
  name: db-credentials
  namespace: production
spec:
  refreshInterval: 1h
  secretStoreRef:
    kind: ClusterSecretStore
    name: vault-backend
  target:
    name: db-credentials
  data:
    - secretKey: username
      remoteRef:
        key: secret/data/db
        property: username
    - secretKey: password
      remoteRef:
        key: secret/data/db
        property: password

# /var/lib/kubelet/config.yaml
authentication:
  anonymous:
    enabled: false  # 禁用匿名认证
  webhook:
    enabled: true
  x509:
    clientCAFile: /etc/kubernetes/pki/ca.crt
authorization:
  mode: Webhook
readOnlyPort: 0  # 禁用只读端口（10255）
protectKernelDefaults: true  # 保护内核参数

# CIS Kubernetes Benchmark 检查（kube-bench）
docker run --pid=host --network=host \
  -v /etc:/etc:ro -v /var:/var:ro \
  aquasec/kube-bench:latest run --targets node

# 关键检查项：
# - 1.2.1: API Server 匿名认证
# - 4.2.1: Kubelet 匿名认证
# - 4.2.6: Kubelet 保护内核默认值

# falco-k8s-rules.yaml
- rule: Unauthorized K8s API Access
  desc: Detect access to K8s API from unexpected pods
  condition: >
    k8s_audit and ka.target.resource=pods
    and not (ka.auth.decision=allow and authorized_sa)
  output: Unauthorized K8s API access
  priority: CRITICAL

- rule: Privilege Escalation via Setuid
  desc: Detect setuid binary execution
  condition: >
    spawned_process and container and (setuid or setgid)
  output: Privilege escalation attempt
  priority: WARNING

# 命名空间级 PSA 标签配置
apiVersion: v1
kind: Namespace
metadata:
  name: web-frontend
  labels:
    # 强制执行 Restricted 策略
    pod-security.kubernetes.io/enforce: restricted
    pod-security.kubernetes.io/enforce-version: latest
    # 审计和告警（记录但不阻止）
    pod-security.kubernetes.io/audit: restricted
    pod-security.kubernetes.io/warn: restricted
    # 豁免系统命名空间的审计噪音
    pod-security.kubernetes.io/audit-version: latest
    pod-security.kubernetes.io/warn-version: latest
---
# 严格的 Restricted Pod 示例
apiVersion: v1
kind: Pod
metadata:
  name: secure-app
  namespace: web-frontend
spec:
  securityContext:
    runAsNonRoot: true
    seccompProfile:
      type: RuntimeDefault
    runAsUser: 65534  # nobody
    runAsGroup: 65534
    fsGroup: 65534
  automountServiceAccountToken: false
  containers:
    - name: app
      image: myapp:v1.2.3
      resources:
        limits:
          cpu: "500m"
          memory: "256Mi"
        requests:
          cpu: "100m"
          memory: "128Mi"
      securityContext:
        allowPrivilegeEscalation: false
        readOnlyRootFilesystem: true
        capabilities:
          drop:
            - ALL
        runAsUser: 65534
        runAsGroup: 65534
        seccompProfile:
          type: RuntimeDefault
      volumeMounts:
        - name: tmp
          mountPath: /tmp
        - name: cache
          mountPath: /var/cache
  volumes:
    - name: tmp
      emptyDir: {}
    - name: cache
      emptyDir: {}

# GitHub Actions 镜像扫描示例
jobs:
  build-and-scan:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      
      - name: Build image
        run: docker build -t myapp:${{ github.sha }} .
      
      - name: Trivy vulnerability scan
        uses: aquasecurity/trivy-action@master
        with:
          image-ref: myapp:${{ github.sha }}
          format: 'sarif'
          output: 'trivy-results.sarif'
          severity: 'CRITICAL,HIGH'
          exit-code: '1'  # 发现严重漏洞自动失败
      
      - name: Upload scan results
        uses: github/codeql-action/upload-sarif@v2
        with:
          sarif_file: 'trivy-results.sarif'

# 在 Admission Controller 中阻止漏洞镜像部署
apiVersion: templates.gatekeeper.sh/v1beta1
kind: ConstraintTemplate
metadata:
  name: k8sallowedimagerepos
spec:
  crd:
    spec:
      names:
        kind: K8sAllowedRepos
  targets:
    - target: admission.k8s.gatekeeper.sh
      rego: |
        package k8sallowedrepos
        violation[{"msg": msg}] {
          container := input.review.object.spec.containers[_]
          satisfied := [good | repo = input.parameters.repos[_] ; good = startswith(container.image, repo)]
          not any(satisfied)
          msg := sprintf("容器 %v 使用的镜像 %v 不在白名单仓库中", [container.name, container.image])
        }

# 多阶段构建 + Distroless 基础镜像
FROM golang:1.21-alpine AS builder
WORKDIR /app
COPY go.mod go.sum ./
RUN go mod download
COPY . .
RUN CGO_ENABLED=0 GOOS=linux go build -ldflags="-w -s" -o server ./cmd/server

# 使用 distroless 运行时镜像
FROM gcr.io/distroless/static:nonroot
COPY --from=builder /app/server /server
USER nonroot:nonroot
EXPOSE 8080
ENTRYPOINT ["/server"]

# Cilium L7 可视化监测（替代单纯 L4 NetworkPolicy）
apiVersion: cilium.io/v2
kind: CiliumNetworkPolicy
metadata:
  name: l7-allow-http
spec:
  selector:
    matchLabels:
      app: frontend
  ingress:
    - fromEndpoints:
        - matchLabels:
            app: backend
      toPorts:
        - ports:
            - port: "80"
              protocol: TCP
          rules:
            http:
              - method: GET
                path: "/api/v1/*"
              - method: POST
                path: "/api/v1/orders"

# 生成加密密钥
ENCRYPTION_KEY=$(head -c 32 /dev/urandom | base64)

# 创建加密配置文件
cat > /etc/kubernetes/encryption-config.yaml <<EOF
apiVersion: apiserver.config.k8s.io/v1
kind: EncryptionConfiguration
resources:
  - resources:
      - secrets
      - configmaps
    providers:
      - aescbc:
          keys:
            - name: key1
              secret: ${ENCRYPTION_KEY}
      - identity: {}  # 允许未加密数据读取（逐步迁移用）
EOF

# kube-apiserver 添加启动参数
--encryption-provider-config=/etc/kubernetes/encryption-config.yaml

# 验证 Secret 是否已加密
ETCDCTL_API=3 etcdctl get /registry/secrets/default/my-secret \
  --cert=/etc/kubernetes/pki/etcd/server.crt \
  --key=/etc/kubernetes/pki/etcd/server.key \
  --cacert=/etc/kubernetes/pki/etcd/ca.crt | hexdump -C | head

# 输出应显示乱码（k8s:enc:aescbc:v1:key1:... 前辍），而非明文 JSON

# 密钥轮转流程
# 1. 在 EncryptionConfiguration 中添加新密钥
# 2. 重启 kube-apiserver
# 3. 触发所有 Secret 重新加密: kubectl get secrets --all-namespaces -o json | kubectl replace -f -
# 4. 移除旧密钥，再次重启

# External Secrets Operator 从 Vault 自动同步
apiVersion: external-secrets.io/v1beta1
kind: ExternalSecret
metadata:
  name: db-credentials
  namespace: production
spec:
  refreshInterval: 1h
  secretStoreRef:
    kind: ClusterSecretStore
    name: vault-backend
  target:
    name: db-credentials
    creationPolicy: Owner
    deletionPolicy: Retain
    template:
      type: Opaque
      data:
        # 注入连接字符串而非单独字段
        DATABASE_URL: "postgresql://{{ .username }}:{{ .password }}@db.production.svc:5432/app"
  data:
    - secretKey: username
      remoteRef:
        key: secret/data/production/db
        property: username
    - secretKey: password
      remoteRef:
        key: secret/data/production/db
        property: password

# audit-policy.yaml
apiVersion: audit.k8s.io/v1
kind: Policy
rules:
  # 1. 记录所有 Secret/ConfigMap 变更（RequestResponse 级别）
  - level: RequestResponse
    resources:
      - group: ""
        resources: ["secrets", "configmaps"]
    verbs: ["create", "update", "patch", "delete"]
    omitStages:
      - RequestReceived
  
  # 2. 记录所有认证失败
  - level: Metadata
    omitStages:
      - RequestReceived
    userGroups: ["system:authenticated"]
    verbs: ["create", "update", "patch", "delete"]
  
  # 3. 记录所有权限拒绝
  - level: Metadata
    omitStages:
      - RequestReceived
    verbs: ["create", "update", "patch", "delete"]
  
  # 4. 记录 Pod/Deployment 创建（生产环境敏感操作）
  - level: Request
    resources:
      - group: ""
        resources: ["pods", "pods/status"]
      - group: "apps"
        resources: ["deployments", "statefulsets", "daemonsets"]
    namespaces: ["production", "staging"]
    verbs: ["create", "update", "delete"]
  
  # 5. 记录 RBAC 变更
  - level: RequestResponse
    resources:
      - group: "rbac.authorization.k8s.io"
        resources: ["roles", "rolebindings", "clusterroles", "clusterrolebindings"]
    verbs: ["create", "update", "patch", "delete"]
  
  # 6. 默认：不记录读操作（减少日志量）
  - level: None
    verbs: ["get", "list", "watch"]

# 审计日志分析示例
# 查询生产环境敏感资源变更
grep '"requestURI":"/api/v1/namespaces/production/pods"' /var/log/audit.log | \
  jq -c '{user: .user.username, verb: .verb, resource: .objectRef.resource, time: .requestReceivedTimestamp}'

# 查询 RBAC 变更（潜在提权行为）
grep '"resource":"clusterroles"' /var/log/audit.log | \
  jq -c '{user: .user.username, verb: .verb, name: .objectRef.name, rules: .requestObject.rules}'

# falco-rules-custom.yaml
# 自定义规则：检测容器中异常网络连接
- rule: Outbound Connection from Sensitive Container
  desc: 检测敏感容器（数据库、密钥管理）的出网连接
  condition: >
    spawned_process and container and outbound
    and (container.image.repository contains "postgres"
         or container.image.repository contains "vault"
         or container.image.repository contains "mysql")
  output: >
    敏感容器异常出网: connection=%fd.name
    container=%container.name image=%container.image
    proc=%proc.name cmdline=%proc.cmdline
    remote=%fd.ip
  priority: CRITICAL

# 自定义规则：检测容器中写入 /etc 或 /bin（持久化攻击）
- rule: Write Below Etc/Bin
  desc: 检测容器中向/etc、/bin等系统目录写入文件
  condition: >
    write_etc_common or write_below_bin_dir
    and container
    and not proc.name in ("yum", "apt", "dpkg", "rpm", "apk")
  output: >
    系统目录写入: file=%fd.name
    proc=%proc.name cmdline=%proc.cmdline
    container=%container.name
  priority: WARNING

# 使用 Tracee 监控 K8s 运行时事件（基于 eBPF，性能更高）
docker run --rm --privileged \
  -v /proc:/proc -v /tmp/tracee:/tmp/tracee \
  aquasec/tracee:latest \
  --trace event=execve,connect,openat \
  --trace container=new \
  --output json

# Tracee vs Falco 对比
# Tracee: eBPF 原语，CPU 开销 < 1%，启动即监控，无需规则编译
# Falco: 规则引擎成熟，社区规则丰富，与 K8s 审计日志集成完善

# 生产建议：Falco + Tracee 组合
# - Falco: K8s 审计 + 已知攻击特征检测
# - Tracee: 全系统行为基线 + 未知攻击模式发现