站内已有一篇广告变现设计清单 ，那篇是清单式短文，把「该不该加广告、加在哪、频次多少」列成一张检查表；本章给的是可跑的实验与显著性模型——同样一句「新频次更赚钱」，怎么用分流、样本量和显著性把它验成一个数字。区别在写法：那篇回答「广告该怎么设计」，本章回答「一个改动到底该不该上线，让数据说了算」。
前面几章把变现、增长、留存各自算了一遍，但它们都是模型：模型告诉你「新频次理论上更好」，不告诉你「这次改动在真实用户身上是否真的更好」。要把一个改动从「我觉得」变成「数据支持」，唯一的办法是做实验：把用户随机分成两组，一组用旧方案、一组用新方案，比两组的指标差异。这一章三节分别解决三件事：怎么分（8.1）、怎么判（8.2）、怎么决策（8.3）。分流最直觉的做法是「用户 ID 对 2 取模」，但它有两个致命问题：一是不均衡（ID 分布有偏时两臂人数会差出一大截），二是不可复现（每次请求重新随机，同一用户会在两臂间漂移）。正确做法是确定性哈希：把用户 ID 哈希成桶号，同一个用户永远落同一个桶、同一个臂。
D 层：把实验定义落成配置
下面这份是 D 层（静态层）产物，全整数。它定义实验 ID、哈希桶数、各臂权重‰，以及一个容易被忽略的参数 logDropPermille——某臂的日志丢失率；SRM 检查的期望取「分析样本总数 × 权重‰」，偏离超过 srmTolerancePermille 即判失衡。
{
"version": 1,
"game": "merge-island",
"seed": 20261012,
"note": "A/B 实验分流：用 FNV-1a 32 位哈希把用户 ID 确定性映射到 [0, hashModulo) 的一个桶，再按各臂 weightPermille 把桶区间依次切给各臂。同一用户永远落同一臂（可复现、无会话漂移）。分配本身应当均衡，但真正的 SRM 常来自「日志丢失」：logDropPermille 给某臂一个丢失率（用第二条独立哈希 logHashSalt 决定谁被丢），于是被分析到的样本比例失衡。SRM 用「分析样本总数 × 权重‰」作期望，偏离超过 srmTolerancePermille 即判失衡。真实丢失率需真实业务环境，命令行不可复现（X）。全整数，禁浮点。",
"experimentId": "exp-interstitial-on-loss",
"unit": "userIdHash",
"hashModulo": 10000,
"logHashSalt": ":log",
"populationUsers": 200000,
"srmTolerancePermille": 10,
"logDropPermille": { "control": 0, "treat": 30 },
"arms": [
{ "id": "control", "name": "对照(原频次)", "weightPermille": 500 },
{ "id": "treat", "name": "实验(降频次)", "weightPermille": 500 }
]
}
logDropPermille 是这一节的关键。分配本身（哈希）永远是均衡的，但被分析到的样本未必均衡：如果实验臂因为埋点 bug、崩溃、或客户端版本差异丢了一部分日志，进入分析的两臂人数就会失衡——这就是 SRM（Sample Ratio Mismatch，样本比例失衡）。SRM 不是「实验做错了」，而是**「实验数据不可信」的红灯**：一旦 SRM 报警，任何显著性结论都作废。
S 层：确定性哈希分流 + SRM 检查
下面这段是 S 层仿真脚本，读 config/l08_experiment.json，把 20 万个合成用户按哈希分到两臂，再按丢失率丢掉一部分日志，最后做 SRM 检查，产出 JSON + CSV + SVG。
#!/usr/bin/env python3
# file: sim/l08_split.py
# tier: S
# verify: /opt/homebrew/bin/python3 sim/l08_split.py --config config/ --out out/
"""A/B 分流仿真：FNV-1a 32 位哈希把用户 ID 映射到桶，再按各臂权重‰切桶区间。
同一用户永远落同一臂；再用带盐哈希模拟日志丢失，做 SRM（样本比例失衡）检查。
产出 JSON + CSV + SVG。真实丢失率需真实业务环境，命令行不可复现（X）。全整数。"""
from __future__ import annotations
import argparse, csv, json, os
import xml.etree.ElementTree as ET
FNV_OFFSET, FNV_PRIME, MASK32 = 0x811C9DC5, 0x01000193, 0xFFFFFFFF
def load(d: str) -> dict:
with open(os.path.join(d, "l08_experiment.json"), encoding="utf-8") as f:
return json.load(f)
def fnv1a32(s: str) -> int:
"""FNV-1a 32 位哈希（ASCII 输入逐字符即逐字节）。"""
h = FNV_OFFSET
for ch in s:
h = ((h ^ ord(ch)) * FNV_PRIME) & MASK32
return h
def bucket_of(uid: str, modulo: int) -> int:
return fnv1a32(uid) % modulo
def assign_arm(cfg: dict, uid: str) -> str:
"""按累计权重把桶区间分给各臂；同一用户永远落同一臂。"""
b, cum = bucket_of(uid, cfg["hashModulo"]), 0
for arm in cfg["arms"]:
cum += arm["weightPermille"]
if b < cum * cfg["hashModulo"] // 1000:
return arm["id"]
return cfg["arms"][-1]["id"]
def is_logged(cfg: dict, uid: str, arm: str) -> bool:
"""日志是否落库：带盐第二哈希决定，按该臂丢失率‰丢弃。"""
drop = cfg["logDropPermille"].get(arm, 0)
return True if drop <= 0 else fnv1a32(uid + cfg["logHashSalt"]) % 1000 >= drop
def tally(cfg: dict, population: int) -> tuple:
assigned = {a["id"]: 0 for a in cfg["arms"]}
logged = {a["id"]: 0 for a in cfg["arms"]}
for i in range(population):
uid = "u%d" % i
arm = assign_arm(cfg, uid)
assigned[arm] += 1
if is_logged(cfg, uid, arm):
logged[arm] += 1
return assigned, logged
def srm_rows(cfg: dict, logged: dict) -> tuple:
"""期望 = 分析样本总数 × 权重‰；偏差‰ 与卡方（×1000，整数）。"""
total = sum(logged.values())
rows, chi2, max_dev = [], 0, 0
for arm in cfg["arms"]:
exp = total * arm["weightPermille"] // 1000
obs = logged[arm["id"]]
dev = abs(obs - exp) * 1000 // exp if exp else 0
chi2 += (obs - exp) ** 2 * 1000 // exp if exp else 0
max_dev = max(max_dev, dev)
rows.append({"id": arm["id"], "name": arm["name"], "expected": exp,
"observed": obs, "deviationPermille": dev})
return rows, max_dev, chi2
def draw_svg(path: str, rows: list, tol: int) -> int:
W, PAD, ROW = 720, 190, 46
H = 70 + ROW * len(rows)
peak = max((r["observed"] for r in rows), default=1) or 1
svg = ET.Element("svg", {"xmlns": "http://www.w3.org/2000/svg", "width": str(W),
"height": str(H), "viewBox": f"0 0 {W} {H}"})
ET.SubElement(svg, "rect", {"width": str(W), "height": str(H), "fill": "#ffffff"})
ET.SubElement(svg, "text", {"x": "16", "y": "28", "font-size": "15", "fill": "#111827"}).text = \
f"分析样本数 vs 期望（SRM 容差 ±{tol}‰，红线 = 期望）"
for i, r in enumerate(rows):
y = 54 + i * ROW
w_obs = (W - PAD - 60) * r["observed"] // peak
w_exp = (W - PAD - 60) * r["expected"] // peak
ET.SubElement(svg, "text", {"x": "16", "y": str(y + 20), "font-size": "12",
"fill": "#374151"}).text = r["name"]
ET.SubElement(svg, "rect", {"x": str(PAD), "y": str(y + 6), "width": str(w_obs),
"height": "22", "rx": "4", "fill": "#2563eb"})
ET.SubElement(svg, "line", {"x1": str(PAD + w_exp), "y1": str(y + 2),
"x2": str(PAD + w_exp), "y2": str(y + 32),
"stroke": "#dc2626", "stroke-width": "2"})
ET.SubElement(svg, "text", {"x": str(PAD + w_obs + 8), "y": str(y + 22), "font-size": "12",
"fill": "#6b7280"}).text = f'实测 {r["observed"]} · 偏差 {r["deviationPermille"]}‰'
os.makedirs(os.path.dirname(path), exist_ok=True)
ET.ElementTree(svg).write(path, encoding="utf-8", xml_declaration=True)
return os.path.getsize(path)
def main() -> int:
ap = argparse.ArgumentParser()
ap.add_argument("--config", default="config/")
ap.add_argument("--out", default="out/")
a = ap.parse_args()
cfg = load(a.config)
pop = cfg["populationUsers"]
assigned, logged = tally(cfg, pop)
rows, max_dev, chi2 = srm_rows(cfg, logged)
for r in rows:
r["assigned"] = assigned[r["id"]]
ok = max_dev <= cfg["srmTolerancePermille"]
os.makedirs(a.out, exist_ok=True)
svg = draw_svg(os.path.join(a.out, "l08_split.svg"), rows, cfg["srmTolerancePermille"])
with open(os.path.join(a.out, "l08_split.csv"), "w", encoding="utf-8", newline="") as f:
w = csv.writer(f)
w.writerow(["armId", "name", "assigned", "expected", "observed", "deviationPermille"])
for r in rows:
w.writerow([r["id"], r["name"], r["assigned"], r["expected"], r["observed"], r["deviationPermille"]])
rep = {"version": cfg["version"], "game": cfg["game"], "seed": cfg["seed"],
"experimentId": cfg["experimentId"], "hashModulo": cfg["hashModulo"], "populationUsers": pop,
"srmTolerancePermille": cfg["srmTolerancePermille"], "logDropPermille": cfg["logDropPermille"],
"arms": rows, "assigned": assigned, "logged": logged, "maxDeviationPermille": max_dev,
"chi2Milli": chi2, "srmPass": ok,
"deterministic": assign_arm(cfg, "u12345") == assign_arm(cfg, "u12345"),
"csv": os.path.join(a.out, "l08_split.csv"),
"svg": os.path.join(a.out, "l08_split.svg"), "svgBytes": svg}
with open(os.path.join(a.out, "l08_split.json"), "w", encoding="utf-8") as f:
json.dump(rep, f, ensure_ascii=False, indent=2)
print(json.dumps({"assigned": assigned, "logged": logged, "maxDeviationPermille": max_dev,
"chi2Milli": chi2, "srmPass": ok, "deterministic": rep["deterministic"]},
ensure_ascii=False))
return 0
if __name__ == "__main__":
raise SystemExit(main())
脚本只有四步：fnv1a32 把 ID 哈希成整数；bucket_of 取模得到桶号；assign_arm 按累计权重把桶分给各臂；srm_rows 用「分析样本总数 × 权重‰」作期望、算各臂偏差‰。在本机跑它，输出如下（同机相对量级，绝对数值不可移植）：
{"assigned": {"control": 100071, "treat": 99929}, "logged": {"control": 100071, "treat": 96892}, "maxDeviationPermille": 16, "chi2Milli": 51308, "srmPass": false, "deterministic": true}
读这三个数字：分配是 100071 : 99929，几乎完美均衡（偏差不足 1‰）；但日志落库后变成 100071 : 96892——treat 臂丢了约 3% 的日志，SRM 偏差冲到 16‰，超过容差 10‰，SRM 未通过。这正是 SRM 的价值：它先于任何指标结论，告诉你「这组数据的两臂不可比」。
P 层：同一份 config 的分流内核
仿真跑通了，但分流逻辑最终要跑在服务端（或客户端 SDK）里。下面这段是 P 层内核 core/src/l08/split.ts，与 Python 脚本同一份 config、同一套整数运算，因此逐位一致。注意 Math.imul(h, FNV_PRIME) >>> 0——JS 的 * 在超过 2^53 时会丢精度，必须用 Math.imul 做 32 位整数乘法、>>> 0 转无符号。
// file: core/src/l08/split.ts
// tier: P
// verify: node --test 'core/test/l08_*.test.ts'
// 《合成小岛》A/B 分流内核：FNV-1a 32 位哈希把用户 ID 确定性映射到桶，
// 再按各臂权重‰把桶区间分给各臂；另用带盐的第二条哈希模拟日志丢失，做 SRM 检查。
// 与 sim/l08_split.py 同一份 config、同一套整数运算，逐位一致。全整数、无 Math.random()。
export interface Arm { id: string; name: string; weightPermille: number }
export interface SplitConfig {
arms: Arm[]; hashModulo: number; logHashSalt: string; logDropPermille: Record<string, number>;
}
export interface ArmTally {
id: string; name: string; assigned: number; expected: number; observed: number; deviationPermille: number;
}
const FNV_OFFSET = 0x811c9dc5;
const FNV_PRIME = 0x01000193;
/** FNV-1a 32 位哈希（ASCII 输入逐字符即逐字节）。 */
export function fnv1a32(s: string): number {
let h = FNV_OFFSET;
for (let i = 0; i < s.length; i += 1) {
h ^= s.charCodeAt(i);
h = Math.imul(h, FNV_PRIME) >>> 0;
}
return h >>> 0;
}
export const bucketOf = (uid: string, modulo: number): number => fnv1a32(uid) % modulo;
/** 按累计权重把桶区间分给各臂；同一用户永远落同一臂。 */
export function assignArm(cfg: SplitConfig, uid: string): string {
const b = bucketOf(uid, cfg.hashModulo);
let cum = 0;
for (const arm of cfg.arms) {
cum += arm.weightPermille;
if (b < Math.floor(cum * cfg.hashModulo / 1000)) return arm.id;
}
return cfg.arms[cfg.arms.length - 1].id;
}
/** 日志是否落库：用带盐的第二条哈希决定，按该臂丢失率‰丢弃。 */
export function isLogged(cfg: SplitConfig, uid: string, arm: string): boolean {
const drop = cfg.logDropPermille[arm] ?? 0;
if (drop <= 0) return true;
return fnv1a32(uid + cfg.logHashSalt) % 1000 >= drop;
}
/** 统计 population 个合成用户（u0..u{N-1}）的分配数与日志落库数。 */
export function tallyArms(cfg: SplitConfig, population: number): { assigned: Record<string, number>; logged: Record<string, number> } {
const assigned: Record<string, number> = {};
const logged: Record<string, number> = {};
for (const arm of cfg.arms) { assigned[arm.id] = 0; logged[arm.id] = 0; }
for (let i = 0; i < population; i += 1) {
const uid = "u" + i;
const arm = assignArm(cfg, uid);
assigned[arm] += 1;
if (isLogged(cfg, uid, arm)) logged[arm] += 1;
}
return { assigned, logged };
}
/** SRM 检查：期望 = 分析样本总数 × 权重‰，返回各臂偏差‰。 */
export function srmTally(cfg: SplitConfig, assigned: Record<string, number>, logged: Record<string, number>): ArmTally[] {
const total = cfg.arms.reduce((s, a) => s + logged[a.id], 0);
return cfg.arms.map((arm) => {
const expected = Math.floor(total * arm.weightPermille / 1000);
const observed = logged[arm.id];
return {
id: arm.id, name: arm.name, assigned: assigned[arm.id], expected, observed,
deviationPermille: expected ? Math.floor(Math.abs(observed - expected) * 1000 / expected) : 0,
};
});
}
P 层：把分流钉进单元测试
// file: core/test/l08_split.test.ts
// tier: P
// verify: node --test 'core/test/l08_*.test.ts'
// 断言：哈希确定、同一用户永远落同一臂；分配均衡但日志丢失造成 SRM；
// 关键计数与 sim/l08_split.py 逐位一致。
import { test } from "node:test";
import assert from "node:assert/strict";
import { readFileSync } from "node:fs";
import { dirname, join } from "node:path";
import { fileURLToPath } from "node:url";
import { fnv1a32, bucketOf, assignArm, isLogged, tallyArms, srmTally } from "../src/l08/split.ts";
const cfg = JSON.parse(readFileSync(join(dirname(fileURLToPath(import.meta.url)), "..", "..", "config", "l08_experiment.json"), "utf-8"));
test("fnv1a32：确定且与 Python 逐位一致", () => {
assert.equal(fnv1a32("u0"), 54699616);
assert.equal(fnv1a32("u2"), 88254854);
assert.equal(fnv1a32("u12345"), 192589931);
assert.equal(fnv1a32("u0"), fnv1a32("u0"));
});
test("assignArm：同一用户永远落同一臂，且落在 [0, hashModulo)", () => {
assert.equal(bucketOf("u0", cfg.hashModulo), 9616);
assert.equal(assignArm(cfg, "u0"), "treat");
assert.equal(assignArm(cfg, "u2"), "control");
assert.equal(assignArm(cfg, "u12345"), "treat");
assert.equal(assignArm(cfg, "u12345"), assignArm(cfg, "u12345"));
for (let i = 0; i < 500; i += 1) {
const b = bucketOf("u" + i, cfg.hashModulo);
assert.ok(b >= 0 && b < cfg.hashModulo);
}
});
test("分配本身均衡，日志丢失让 treat 臂样本比例失衡（SRM）", () => {
const { assigned, logged } = tallyArms(cfg, cfg.populationUsers);
assert.deepEqual(assigned, { control: 100071, treat: 99929 });
assert.deepEqual(logged, { control: 100071, treat: 96892 });
assert.equal(assigned.control - assigned.treat, 142); // 分配只差 142
assert.equal(logged.control - logged.treat, 3179); // 日志落库后差 3179
});
test("isLogged：带盐哈希按丢失率丢弃，control 丢失率为 0 时不丢", () => {
assert.equal(isLogged(cfg, "u0", "control"), true);
assert.equal(isLogged(cfg, "u0", "treat"), fnv1a32("u0:log") % 1000 >= 30);
});
test("srmTally：期望按分析样本总数算，偏差 16‰ 超容差 10‰ → SRM 未通过", () => {
const { assigned, logged } = tallyArms(cfg, cfg.populationUsers);
const rows = srmTally(cfg, assigned, logged);
const maxDev = Math.max(...rows.map((r) => r.deviationPermille));
assert.equal(maxDev, 16);
assert.ok(maxDev > cfg.srmTolerancePermille);
assert.equal(rows[0].assigned, 100071);
assert.equal(rows[1].observed, 96892);
});
本机跑 node --test 'core/test/l08_*.test.ts'，5 项全过；其中「分配本身均衡，日志丢失让 treat 臂样本比例失衡」一条同时断言了 assigned 与 logged 两组计数，与 Python 仿真逐位一致（100071 / 99929 / 96892 / 16‰）。
小结与下一步
这一节把「分流」做成了一个可复现、可断言的工程件：确定性哈希保证同臂、权重切桶保证比例、带盐哈希模拟丢失、SRM 检查保证数据可信。分流分得对，实验才有资格谈结论——但分好组只是第一步，怎么判断两组的差异是「真的」还是「碰巧」、需要多少样本才够、看到一半就想下结论会出什么错，是下一节的主题。
阅读导航：上一节：目录 · 下一节：显著性、样本量与常见陷阱 。
继续阅读
探索更多技术文章
浏览归档，发现更多关于系统设计、工具链和工程实践的内容。