「可观测性与运维自动化」

# 一次 plan 就能暴露全部差异
terraform plan -detailed-exitcode
# 退出码 0 = 无变更，1 = 报错，2 = 有变更

# 从 plan 中提取本次变更的完整清单
terraform show -json plan.tfplan | \
  jq -c '[.resource_changes[]
          | select(.change.actions != ["no-op"])
          | {address, actions: .change.actions}]'

# 留存审计证据：plan 哈希 + 变更清单 + 审批记录
sha256sum plan.tfplan > audit/plan-$(date +%Y%m%d-%H%M%S).sha256
git rev-parse HEAD > audit/commit.txt

# .github/workflows/apply.yml
jobs:
  plan:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - run: terraform init && terraform plan -out=plan.tfplan
      - uses: actions/upload-artifact@v4
        with:
          name: tfplan
          path: plan.tfplan

  apply:
    needs: plan
    runs-on: ubuntu-latest
    environment: production   # 需人工审批
    steps:
      - uses: actions/download-artifact@v4
        with: { name: tfplan }
      - run: terraform init && terraform apply plan.tfplan

# 把 plan 摘要格式化成 Slack 消息
terraform show -json plan.tfplan | jq -r '
  "计划变更: \(.resource_changes | map(select(.change.actions != ["no-op"])) | length) 个资源\n" +
  (.resource_changes
   | map(select(.change.actions != ["no-op"]))
   | map("• \(.address) [\(.change.actions | join(","))]")
   | join("\n"))' > summary.txt

curl -s -X POST -H 'Content-type: application/json' \
  --data "$(jq -n --rawfile s summary.txt '{
    text: ("Terraform 计划摘要\n" + $s)
  }')" \
  "$SLACK_WEBHOOK_URL"

# 通用 Webhook 通知（适用于 Teams / 钉钉 / 自建服务）
- name: Notify
  if: always()
  run: |
    curl -s -X POST "$WEBHOOK_URL" \
      -H "Content-Type: application/json" \
      -d "{\"status\":\"${{ job.status }}\",\"commit\":\"$GITHUB_SHA\"}"

# .github/workflows/drift.yml
name: drift-detection

on:
  schedule:
    - cron: "0 6 * * *"   # 每天 UTC 06:00
  workflow_dispatch:

jobs:
  drift:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        stack: [network, data, app]
    steps:
      - uses: actions/checkout@v4
      - uses: hashicorp/setup-terraform@v3
      - run: terraform init
        working-directory: stacks/${{ matrix.stack }}
      - name: Detect drift
        id: plan
        continue-on-error: true
        run: terraform plan -detailed-exitcode -no-color
        working-directory: stacks/${{ matrix.stack }}
      - name: Report
        if: steps.plan.outputs.exitcode == '2'
        run: |
          echo "检测到漂移: ${{ matrix.stack }}"
          # 推送到告警频道

# 本地或定时任务中手工执行
terraform plan -detailed-exitcode -refresh-only
echo "exit=$?"

# 只看漂移的资源
terraform show -json drift.tfplan | \
  jq '[.resource_changes[] | select(.change.actions != ["no-op"]) | .address]'

# 自动化修复脚本：仅处理白名单内的漂移资源
DRIFT=$(terraform show -json drift.tfplan | \
  jq -r '.resource_changes[]
         | select(.change.actions != ["no-op"])
         | .address')

ALLOWED='aws_cloudwatch_log_group|aws_s3_bucket_lifecycle|aws_ssm_parameter'

if echo "$DRIFT" | grep -vqE "^($ALLOWED)"; then
  echo "存在白名单外的漂移，跳过自动修复，转人工处理"
  exit 1
fi

terraform apply -auto-approve -refresh-only

# 用 prevent_destroy 从代码层面挡住最危险的操作
resource "aws_db_instance" "main" {
  lifecycle {
    prevent_destroy = true
  }
}

运行手册：Terraform apply 失败

[现象]
apply 中断，报错 ResourceInUse / DependencyViolation

[处理步骤]
1. 确认是否有并发 apply：检查锁状态
2. 查看具体失败资源：terraform show plan.tfplan | grep 资源名
3. 判断是否部分成功：terraform state list 对比期望清单
4. 若部分成功：修正配置后重新 apply（Terraform 幂等）
5. 若状态损坏：从备份 state push 恢复
6. 记录事件：时间、影响范围、根因、后续动作

[升级路径]
15 分钟未恢复 → 通知平台负责人
涉及生产数据 → 立即拉群并冻结相关流水线

# 手册中常用的诊断命令
terraform state list
terraform state show aws_instance.web
terraform providers
terraform version

# 从历史 apply 日志中统计成功率
grep -c "Apply complete" apply-*.log
grep -c "Error:" apply-*.log

# 统计当前漂移资源数
terraform show -json drift.tfplan | \
  jq '[.resource_changes[] | select(.change.actions != ["no-op"])] | length'

# 把关键指标推送到 CloudWatch，纳入统一看板
resource "aws_cloudwatch_metric_alarm" "drift_count" {
  alarm_name          = "terraform-drift-count"
  comparison_operator = "GreaterThanThreshold"
  evaluation_periods  = 1
  metric_name         = "DriftResourceCount"
  namespace           = "TerraformOps"
  period              = 86400
  statistic           = "Maximum"
  threshold           = 5
  alarm_description   = "漂移资源数超过 5 个，需要人工收敛"
}