《互联网产品的 AI 架构·推理性能与成本工程》4.3 生成长度的非线性成本

// file: src/perf/output-length.ts
// tier: L1
// verify: node --test test/perf/output-length.test.ts

import { AiError } from "../core/errors.ts";

export interface LengthSample {
  capability: string;
  tokens: number;
}

export interface LengthSummary {
  samples: number;
  totalTokens: number;
  meanTokens: number;
  p50Tokens: number;
  p90Tokens: number;
  p95Tokens: number;
  p99Tokens: number;
  maxTokens: number;
  /** 最长的 tailRatio 那部分请求的数量。 */
  tailCount: number;
  /** 长尾请求贡献的 token 占总 token 的比例。 */
  tailTokenShare: number;
}

function fail(message: string): never {
  throw new AiError("invalid-request", `output-length: ${message}`);
}

function requireInteger(value: number, field: string, min: number): number {
  if (!Number.isInteger(value) || value < min) fail(`${field} must be an integer >= ${min}`);
  return value;
}

function requireRatio(value: number, field: string): number {
  if (typeof value !== "number" || !Number.isFinite(value) || value <= 0 || value > 1) {
    fail(`${field} must be in (0, 1]`);
  }
  return value;
}

/** 最近秩法：第 ceil(ratio * n) 个值，结果可以逐条样本复算。 */
export function percentileNearestRank(values: readonly number[], ratio: number): number {
  requireRatio(ratio, "ratio");
  if (values.length === 0) fail("percentile of empty sample");
  const sorted = [...values].sort((left, right) => left - right);
  const rank = Math.ceil(ratio * sorted.length) - 1;
  return sorted[Math.max(0, Math.min(sorted.length - 1, rank))];
}

/** 目标不是平均值，而是分位数与长尾占比：P50 看典型，P99 看预算尾部。 */
export function summarizeLengths(lengths: readonly number[], tailRatio = 0.05): LengthSummary {
  if (lengths.length === 0) fail("cannot summarize an empty sample");
  requireRatio(tailRatio, "tailRatio");
  for (const value of lengths) requireInteger(value, "tokens", 0);

  const sorted = [...lengths].sort((left, right) => left - right);
  const total = sorted.reduce((sum, value) => sum + value, 0);
  const tailCount = Math.max(1, Math.ceil(sorted.length * tailRatio));
  const tailTotal = sorted.slice(sorted.length - tailCount).reduce((sum, value) => sum + value, 0);
  return {
    samples: sorted.length,
    totalTokens: total,
    meanTokens: total / sorted.length,
    p50Tokens: percentileNearestRank(sorted, 0.5),
    p90Tokens: percentileNearestRank(sorted, 0.9),
    p95Tokens: percentileNearestRank(sorted, 0.95),
    p99Tokens: percentileNearestRank(sorted, 0.99),
    maxTokens: sorted[sorted.length - 1],
    tailCount,
    tailTokenShare: total === 0 ? 0 : tailTotal / total,
  };
}

export interface CapabilityLengthReport {
  capability: string;
  summary: LengthSummary;
}

/** 按能力分组，顺序稳定，避免同一份样本在不同机器上得到不同表格。 */
export function summarizeByCapability(
  samples: readonly LengthSample[],
  tailRatio = 0.05,
): CapabilityLengthReport[] {
  const groups = new Map<string, number[]>();
  for (const sample of samples) {
    if (sample.capability.trim() === "") fail("capability is required");
    requireInteger(sample.tokens, "tokens", 0);
    const bucket = groups.get(sample.capability) ?? [];
    bucket.push(sample.tokens);
    groups.set(sample.capability, bucket);
  }
  return [...groups.entries()]
    .sort(([left], [right]) => left.localeCompare(right))
    .map(([capability, lengths]) => ({ capability, summary: summarizeLengths(lengths, tailRatio) }));
}

export interface LengthTarget {
  capability: string;
  p90TargetTokens: number;
  p99CeilingTokens: number;
}

export interface LengthTargetResult {
  capability: string;
  p90Met: boolean;
  p99Met: boolean;
  detail: string;
}

/** 按能力设目标，而不是全站一个平均数；不同能力的长度分布本来就不该相同。 */
export function evaluateLengthTargets(
  reports: readonly CapabilityLengthReport[],
  targets: readonly LengthTarget[],
): LengthTargetResult[] {
  const byCapability = new Map(targets.map((target) => [target.capability, target]));
  return reports.map((report) => {
    const target = byCapability.get(report.capability);
    if (target === undefined) {
      return { capability: report.capability, p90Met: false, p99Met: false, detail: "no target defined" };
    }
    requireInteger(target.p90TargetTokens, "p90TargetTokens", 1);
    requireInteger(target.p99CeilingTokens, "p99CeilingTokens", 1);
    return {
      capability: report.capability,
      p90Met: report.summary.p90Tokens <= target.p90TargetTokens,
      p99Met: report.summary.p99Tokens <= target.p99CeilingTokens,
      detail: `P90 ${report.summary.p90Tokens}/${target.p90TargetTokens}, P99 ${report.summary.p99Tokens}/${target.p99CeilingTokens}`,
    };
  });
}

export interface LengthCostPolicy {
  ttftMs: number;
  decodeTokensPerSecond: number;
  microsPerToken: number;
  /** 超过该长度后，截断与修复概率开始上升。 */
  truncationThresholdTokens: number;
  failurePer100Tokens: number;
  maxFailureProbability: number;
  /** 失败后重生成的总成本倍数，含输入与重建输出。 */
  retryCostMultiplier: number;
}

export interface LengthCostPoint {
  tokens: number;
  latencyMs: number;
  baseMicros: number;
  failureProbability: number;
  expectedMicros: number;
  nonlinearPremiumMicros: number;
}

function requireNumber(value: number, field: string, min: number): number {
  if (typeof value !== "number" || !Number.isFinite(value) || value < min) {
    fail(`${field} must be a finite number >= ${min}`);
  }
  return value;
}

/** 延迟 = 首 token 固定开销 + 线性解码；成本在截断阈值后叠加重生成成本。 */
export function lengthCost(tokens: number, policy: LengthCostPolicy): LengthCostPoint {
  requireInteger(tokens, "tokens", 0);
  requireNumber(policy.ttftMs, "ttftMs", 0);
  requireNumber(policy.decodeTokensPerSecond, "decodeTokensPerSecond", 1);
  const micros = requireInteger(policy.microsPerToken, "microsPerToken", 0);
  const threshold = requireInteger(policy.truncationThresholdTokens, "truncationThresholdTokens", 0);
  const failurePer100 = requireNumber(policy.failurePer100Tokens, "failurePer100Tokens", 0);
  const maxFailure = requireRatio(policy.maxFailureProbability, "maxFailureProbability");
  const retryMultiplier = requireNumber(policy.retryCostMultiplier, "retryCostMultiplier", 1);

  const latencyMs = Math.ceil(policy.ttftMs) + Math.ceil((tokens * 1_000) / policy.decodeTokensPerSecond);
  const baseMicros = Math.ceil(tokens * micros);
  const over = Math.max(0, tokens - threshold);
  const failureProbability = Math.min(maxFailure, (over / 100) * failurePer100);
  const expectedMicros = Math.ceil(baseMicros * (1 + failureProbability * (retryMultiplier - 1)));
  return {
    tokens, latencyMs, baseMicros, failureProbability, expectedMicros,
    nonlinearPremiumMicros: expectedMicros - baseMicros,
  };
}

export function costCurve(lengths: readonly number[], policy: LengthCostPolicy): LengthCostPoint[] {
  return lengths.map((tokens) => lengthCost(tokens, policy));
}

export interface LengthCapPlan {
  capTokens: number;
  affected: number;
  affectedShare: number;
  savedTokens: number;
  savedMicros: number;
  savedShare: number;
}

/** 试算“把输出截到 cap”的 token 收益；质量损失必须另用评测集验证。 */
export function planLengthCap(
  lengths: readonly number[],
  capTokens: number,
  microsPerToken: number,
): LengthCapPlan {
  if (lengths.length === 0) fail("cannot plan a cap on an empty sample");
  requireInteger(capTokens, "capTokens", 1);
  requireInteger(microsPerToken, "microsPerToken", 0);
  let affected = 0;
  let savedTokens = 0;
  let totalTokens = 0;
  for (const value of lengths) {
    requireInteger(value, "tokens", 0);
    totalTokens += value;
    if (value > capTokens) {
      affected += 1;
      savedTokens += value - capTokens;
    }
  }
  return {
    capTokens, affected,
    affectedShare: affected / lengths.length,
    savedTokens,
    savedMicros: savedTokens * microsPerToken,
    savedShare: totalTokens === 0 ? 0 : savedTokens / totalTokens,
  };
}

node --test test/perf/output-length.test.ts

ℹ tests 9
ℹ pass 9
ℹ fail 0
ℹ duration_ms 152.29525