《移动游戏产品的架构·网络、资源与热更新》10.3 热更失败与回滚兜底

// file: src/hotfix/failure-drill.ts
// tier: L1
// verify: node --test test/hotfix/failure-drill.test.ts

export type FailureStage = "fetch" | "verify" | "apply" | "activate";
export type RecoveryAction =
  | "resume"
  | "retry"
  | "rollback"
  | "disable-feature"
  | "store-release";

export interface HotfixAttempt {
  readonly id: string;
  readonly stage: FailureStage;
  readonly bytes: number;
  readonly retryable: boolean;
  readonly partialDownload: boolean;
  readonly rollbackAvailable: boolean;
  readonly featureFlagAvailable: boolean;
  readonly minutesIntoWindow: number;
}

export interface DrillPolicy {
  readonly name: string;
  readonly rollbackBudgetMinutes: number;
  readonly batchBudgetBytes: number;
}

export interface RecoveryPlan {
  readonly id: string;
  readonly stage: FailureStage;
  readonly action: RecoveryAction;
  readonly remainingMinutes: number;
  readonly terminal: boolean;
  readonly reason: string;
}

export interface DrillReport {
  readonly policy: string;
  readonly plans: readonly RecoveryPlan[];
  readonly counts: Readonly<Record<RecoveryAction, number>>;
  readonly oversize: boolean;
  readonly maxRemainingMinutes: number;
}

export const MB = 1024 * 1024;

export const TIDE_ARK_DRILL: DrillPolicy = {
  name: "潮汐方舟",
  rollbackBudgetMinutes: 15,
  batchBudgetBytes: 45 * MB,
};

export const TIDE_MATCH3_DRILL: DrillPolicy = {
  name: "潮汐消消",
  rollbackBudgetMinutes: 30,
  batchBudgetBytes: 12 * MB,
};

const STAGES: readonly FailureStage[] = ["fetch", "verify", "apply", "activate"];

function validateAttempt(attempt: HotfixAttempt): void {
  if (attempt.id.trim() === "") throw new Error("attempt id is required");
  if (!STAGES.includes(attempt.stage)) {
    throw new Error(`unknown stage: ${String(attempt.stage)}`);
  }
  if (!Number.isInteger(attempt.bytes) || attempt.bytes < 0) {
    throw new Error(`invalid bytes for ${attempt.id}: ${attempt.bytes}`);
  }
  if (
    !Number.isInteger(attempt.minutesIntoWindow) ||
    attempt.minutesIntoWindow < 0
  ) {
    throw new Error(
      `invalid minutes for ${attempt.id}: ${attempt.minutesIntoWindow}`,
    );
  }
}

export function planRecovery(
  attempt: HotfixAttempt,
  policy: DrillPolicy,
): RecoveryPlan {
  validateAttempt(attempt);
  if (
    !Number.isInteger(policy.rollbackBudgetMinutes) ||
    policy.rollbackBudgetMinutes < 0
  ) {
    throw new Error(`invalid rollback budget: ${policy.rollbackBudgetMinutes}`);
  }

  const remainingMinutes =
    policy.rollbackBudgetMinutes - attempt.minutesIntoWindow;

  if (remainingMinutes < 0) {
    return {
      id: attempt.id,
      stage: attempt.stage,
      action: "store-release",
      remainingMinutes: 0,
      terminal: true,
      reason: `已用 ${attempt.minutesIntoWindow} 分钟，超出 ${policy.rollbackBudgetMinutes} 分钟回滚窗口`,
    };
  }

  if (attempt.stage === "fetch" && attempt.retryable) {
    const action: RecoveryAction = attempt.partialDownload ? "resume" : "retry";
    return {
      id: attempt.id,
      stage: attempt.stage,
      action,
      remainingMinutes,
      terminal: false,
      reason: attempt.partialDownload
        ? "断点信息完整，从已下载偏移继续"
        : "拉取阶段失败，可重新发起下载",
    };
  }

  if (attempt.rollbackAvailable) {
    return {
      id: attempt.id,
      stage: attempt.stage,
      action: "rollback",
      remainingMinutes,
      terminal: false,
      reason: "恢复上一份已校验版本",
    };
  }

  if (attempt.featureFlagAvailable) {
    return {
      id: attempt.id,
      stage: attempt.stage,
      action: "disable-feature",
      remainingMinutes,
      terminal: false,
      reason: "无法回滚，先关闭功能开关隔离影响",
    };
  }

  return {
    id: attempt.id,
    stage: attempt.stage,
    action: "store-release",
    remainingMinutes,
    terminal: true,
    reason: "无法回滚且没有功能开关，只能发商店新版本覆盖",
  };
}

export function runDrill(
  attempts: readonly HotfixAttempt[],
  policy: DrillPolicy,
): DrillReport {
  const seen = new Set<string>();
  const plans: RecoveryPlan[] = [];
  for (const attempt of attempts) {
    if (seen.has(attempt.id)) throw new Error(`duplicate attempt id: ${attempt.id}`);
    seen.add(attempt.id);
    plans.push(planRecovery(attempt, policy));
  }

  const counts: Record<RecoveryAction, number> = {
    resume: 0,
    retry: 0,
    rollback: 0,
    "disable-feature": 0,
    "store-release": 0,
  };
  let oversize = false;
  let maxRemainingMinutes = 0;

  for (let index = 0; index < attempts.length; index += 1) {
    counts[plans[index].action] += 1;
    if (attempts[index].bytes > policy.batchBudgetBytes) oversize = true;
    if (plans[index].remainingMinutes > maxRemainingMinutes) {
      maxRemainingMinutes = plans[index].remainingMinutes;
    }
  }

  return {
    policy: policy.name,
    plans,
    counts,
    oversize,
    maxRemainingMinutes,
  };
}

export function formatDrillReport(report: DrillReport): string {
  const lines = ["id\tstage\taction\tremaining\tterminal\treason"];
  for (const plan of report.plans) {
    lines.push(
      `${plan.id}\t${plan.stage}\t${plan.action}\t${plan.remainingMinutes}min\t${plan.terminal ? "yes" : "no"}\t${plan.reason}`,
    );
  }
  lines.push(
    `summary\tresume=${report.counts.resume}\tretry=${report.counts.retry}\trollback=${report.counts.rollback}\tdisable-feature=${report.counts["disable-feature"]}\tstore-release=${report.counts["store-release"]}\toversize=${report.oversize ? "yes" : "no"}`,
  );
  return lines.join("\n");
}

id	stage	action	remaining	terminal	reason
partial	fetch	resume	13min	no	断点信息完整，从已下载偏移继续
corrupt	verify	rollback	13min	no	恢复上一份已校验版本
blocked	apply	disable-feature	13min	no	无法回滚，先关闭功能开关隔离影响
late	activate	store-release	0min	yes	已用 16 分钟，超出 15 分钟回滚窗口
summary	resume=1	retry=0	rollback=1	disable-feature=1	store-release=1	oversize=no