《互联网产品的 AI 架构·平台化与治理》8.2 留存、删除与审计

// file: src/gov/retention.ts
// tier: L2
// verify: node --test test/gov/compliance.test.ts

import { createHash } from "node:crypto";
import { EGRESS_POLICY } from "./compliance.ts";
import type { DataClass } from "./compliance.ts";

export type StoreKind = "primary" | "cache" | "log" | "search-index" | "eval-set" | "backup";

export const HOUR_MS = 3_600_000;
export const DAY_MS = 24 * HOUR_MS;

/** deferAllowed 表示不可变存储可登记延期删除，但必须给出确定的到期时间。 */
export interface RetentionRule { maxDays: number; deletable: boolean; slaHours: number; deferAllowed: boolean }

export const RETENTION_RULES: Record<StoreKind, RetentionRule> = {
  primary: { maxDays: 30, deletable: true, slaHours: 24, deferAllowed: false },
  cache: { maxDays: 7, deletable: true, slaHours: 24, deferAllowed: false },
  log: { maxDays: 30, deletable: true, slaHours: 72, deferAllowed: false },
  "search-index": { maxDays: 30, deletable: true, slaHours: 72, deferAllowed: false },
  "eval-set": { maxDays: 180, deletable: true, slaHours: 168, deferAllowed: false },
  backup: { maxDays: 90, deletable: false, slaHours: 720, deferAllowed: true },
};

export const STORE_KINDS = Object.keys(RETENTION_RULES) as StoreKind[];

export interface StoreRetention { store: StoreKind; retentionDays: number }

export interface AssetRetention { assetId: string; dataClass: DataClass; createdAt: number; stores: readonly StoreRetention[] }

export type RetentionIssueCode = "unknown-store" | "duplicate-store" | "retention-too-long" | "retention-exceeds-class";

export interface RetentionIssue { assetId: string; store: StoreKind | ""; code: RetentionIssueCode; detail: string }

/** 每条留存声明同时受存储上限与数据级别上限约束，取两者更严的一条。 */
export function checkRetention(assets: readonly AssetRetention[]): RetentionIssue[] {
  const issues: RetentionIssue[] = [];
  for (const asset of assets) {
    const seen = new Set<StoreKind>();
    const classCap = EGRESS_POLICY[asset.dataClass].maxRetentionDays;
    for (const item of asset.stores) {
      const rule = RETENTION_RULES[item.store];
      if (rule === undefined) {
        issues.push({ assetId: asset.assetId, store: item.store, code: "unknown-store", detail: `${item.store} is not a registered store` });
        continue;
      }
      if (seen.has(item.store)) issues.push({ assetId: asset.assetId, store: item.store, code: "duplicate-store", detail: `${item.store} appears twice` });
      seen.add(item.store);
      if (item.retentionDays > rule.maxDays) issues.push({ assetId: asset.assetId, store: item.store, code: "retention-too-long", detail: `${item.retentionDays}d exceeds store cap ${rule.maxDays}d` });
      if (item.retentionDays > classCap) issues.push({ assetId: asset.assetId, store: item.store, code: "retention-exceeds-class", detail: `${item.retentionDays}d exceeds ${asset.dataClass} cap ${classCap}d` });
    }
  }
  return issues;
}

export interface PurgeDue { assetId: string; store: StoreKind; expiresAt: number }

export function purgeDue(assets: readonly AssetRetention[], now: number): PurgeDue[] {
  const due: PurgeDue[] = [];
  for (const asset of assets) {
    for (const item of asset.stores) {
      if (RETENTION_RULES[item.store] === undefined) continue;
      const expiresAt = asset.createdAt + item.retentionDays * DAY_MS;
      if (expiresAt <= now) due.push({ assetId: asset.assetId, store: item.store, expiresAt });
    }
  }
  return due;
}

export type ReceiptStatus = "deleted" | "pending" | "failed" | "deferred";

export interface DeletionReceipt { assetId: string; store: StoreKind; status: ReceiptStatus; completedAt?: number; expiresAt?: number }

export interface DeletionTarget { assetId: string; stores: readonly StoreKind[] }

export interface DeletionRequest { requestId: string; subjectRef: string; issuedAt: number; targets: readonly DeletionTarget[] }

export type DeletionIssueCode =
  | "missing-receipt" | "pending" | "failed" | "deleted-without-time" | "sla-missed"
  | "deferred-not-allowed" | "deferred-without-expiry" | "backup-expiry-too-late";

export interface DeletionIssue { assetId: string; store: StoreKind | ""; code: DeletionIssueCode; detail: string }

export interface DeletionReport {
  requestId: string; complete: boolean; withinSla: boolean; required: number;
  deleted: number; deferred: number; outstanding: number;
  issues: DeletionIssue[]; proofId: string; storesCovered: StoreKind[];
}

function relevant(request: DeletionRequest, receipts: readonly DeletionReceipt[]): DeletionReceipt[] {
  return receipts.filter((receipt) => request.targets.some((target) => target.assetId === receipt.assetId && target.stores.includes(receipt.store)));
}

/** 删除证明对请求与被覆盖的存储回执做规范化哈希，同一输入必须得到同一证明。 */
export function deletionProof(request: DeletionRequest, receipts: readonly DeletionReceipt[]): string {
  const lines = relevant(request, receipts)
    .map((receipt) => `${receipt.assetId}/${receipt.store}/${receipt.status}/${receipt.completedAt ?? 0}/${receipt.expiresAt ?? 0}`)
    .sort();
  const canonical = [request.requestId, request.subjectRef, String(request.issuedAt), ...lines].join("|");
  return `sha256:${createHash("sha256").update(canonical).digest("hex")}`;
}

/** 删除必须覆盖主库、缓存、日志、检索索引、评测集与备份；主库删除不等于传播完成。 */
export function checkDeletion(request: DeletionRequest, receipts: readonly DeletionReceipt[], now: number): DeletionReport {
  const issues: DeletionIssue[] = [];
  const covered = new Set<StoreKind>();
  let required = 0, deleted = 0, deferred = 0, outstanding = 0, withinSla = true;
  for (const target of request.targets) {
    for (const store of target.stores) {
      required += 1;
      covered.add(store);
      const rule = RETENTION_RULES[store];
      if (rule === undefined) {
        outstanding += 1;
        issues.push({ assetId: target.assetId, store, code: "missing-receipt", detail: `${store} is not a registered store` });
        continue;
      }
      const deadline = request.issuedAt + rule.slaHours * HOUR_MS;
      const receipt = receipts.find((item) => item.assetId === target.assetId && item.store === store);
      if (receipt === undefined) {
        outstanding += 1;
        issues.push({ assetId: target.assetId, store, code: "missing-receipt", detail: `no deletion receipt for ${store}` });
        if (now > deadline) {
          withinSla = false;
          issues.push({ assetId: target.assetId, store, code: "sla-missed", detail: `nothing recorded by deadline ${deadline}` });
        }
        continue;
      }
      if (receipt.status === "deleted") {
        if (receipt.completedAt === undefined) {
          outstanding += 1;
          issues.push({ assetId: target.assetId, store, code: "deleted-without-time", detail: "deleted without a completion timestamp" });
          continue;
        }
        deleted += 1;
        if (receipt.completedAt > deadline) {
          withinSla = false;
          issues.push({ assetId: target.assetId, store, code: "sla-missed", detail: `completed ${receipt.completedAt} after deadline ${deadline}` });
        }
        continue;
      }
      if (receipt.status === "pending") {
        outstanding += 1;
        issues.push({ assetId: target.assetId, store, code: "pending", detail: "deletion pending" });
        if (now > deadline) {
          withinSla = false;
          issues.push({ assetId: target.assetId, store, code: "sla-missed", detail: `still pending after deadline ${deadline}` });
        }
        continue;
      }
      if (receipt.status === "failed") {
        outstanding += 1;
        withinSla = false;
        issues.push({ assetId: target.assetId, store, code: "failed", detail: "deletion failed" });
        continue;
      }
      if (!rule.deferAllowed) {
        outstanding += 1;
        issues.push({ assetId: target.assetId, store, code: "deferred-not-allowed", detail: `${store} cannot defer deletion` });
        continue;
      }
      if (receipt.expiresAt === undefined) {
        outstanding += 1;
        issues.push({ assetId: target.assetId, store, code: "deferred-without-expiry", detail: "deferred backup has no expiry" });
        continue;
      }
      const latest = request.issuedAt + rule.maxDays * DAY_MS;
      if (receipt.expiresAt > latest) {
        outstanding += 1;
        issues.push({ assetId: target.assetId, store, code: "backup-expiry-too-late", detail: `expiry ${receipt.expiresAt} beyond ${latest}` });
        continue;
      }
      deferred += 1;
    }
  }
  return {
    requestId: request.requestId, complete: outstanding === 0, withinSla, required, deleted, deferred, outstanding,
    issues, proofId: deletionProof(request, receipts), storesCovered: STORE_KINDS.filter((store) => covered.has(store)),
  };
}

/** 审计主体只保存不可逆的引用，不保存原始账号、手机号或邮箱。 */
export function pseudonymizeSubject(rawId: string): string {
  return `subj_${createHash("sha256").update(rawId).digest("hex").slice(0, 16)}`;
}

export interface AuditEntry {
  requestId: string; subjectRef: string; issuedAt: number; recordedAt: number;
  required: number; deleted: number; deferred: number; outstanding: number;
  complete: boolean; proofId: string; storesCovered: readonly StoreKind[];
}

export function buildAuditEntry(request: DeletionRequest, report: DeletionReport, recordedAt: number): AuditEntry {
  return {
    requestId: request.requestId, subjectRef: request.subjectRef, issuedAt: request.issuedAt, recordedAt,
    required: report.required, deleted: report.deleted, deferred: report.deferred,
    outstanding: report.outstanding, complete: report.complete, proofId: report.proofId, storesCovered: report.storesCovered,
  };
}

export type AuditIssueCode =
  | "raw-subject-id" | "missing-proof" | "invalid-proof" | "recorded-before-issued"
  | "incomplete-without-outstanding" | "missing-store-coverage";

export interface AuditIssue { requestId: string; code: AuditIssueCode; detail: string }

export function validateAuditEntry(entry: AuditEntry, rawSubjectId: string): AuditIssue[] {
  const issues: AuditIssue[] = [];
  const push = (code: AuditIssueCode, detail: string): void => { issues.push({ requestId: entry.requestId, code, detail }); };
  if (entry.subjectRef === rawSubjectId || entry.subjectRef.includes(rawSubjectId)) push("raw-subject-id", "subjectRef contains the raw subject identifier");
  if (entry.proofId.trim() === "") push("missing-proof", "deletion proof is required");
  else if (!/^sha256:[0-9a-f]{64}$/.test(entry.proofId)) push("invalid-proof", "proof must be a sha256 digest");
  if (entry.recordedAt < entry.issuedAt) push("recorded-before-issued", "recorded before request");
  if (!entry.complete && entry.outstanding === 0) push("incomplete-without-outstanding", "incomplete entry must list outstanding items");
  if (entry.storesCovered.length === 0) push("missing-store-coverage", "entry must name the stores it covered");
  return issues;
}