端侧推理：移动端与浏览器部署

Qwen2.5-1.5B Q4 (约 1 GB) 在带宽 50 GB/s 的手机上：
  理论上限 ≈ 1000 MB / 50 GB/s = 20 ms/token → 50 token/s
  实测通常 15~30 token/s（受内核效率与调度影响）

# 交叉编译 Android（arm64-v8a）
cmake -B build-android \
  -DCMAKE_TOOLCHAIN_FILE=$NDK/build/cmake/android.toolchain.cmake \
  -DANDROID_ABI=arm64-v8a \
  -DANDROID_PLATFORM=android-28 \
  -DGGML_OPENMP=OFF \
  -DLLAMA_CURL=OFF
cmake --build build-android --config Release -j8

# iOS 用 XCFramework
./build-xcframework.sh

// Android JNI 调用示例
#include "llama.h"

extern "C" JNIEXPORT jstring JNICALL
Java_com_example_LlmBridge_generate(JNIEnv *env, jobject, jstring jprompt) {
    const char *prompt = env->GetStringUTFChars(jprompt, nullptr);

    llama_model_params mp = llama_model_default_params();
    mp.n_gpu_layers = 99;                 // 尽量卸载到 GPU
    llama_model *model = llama_load_model_from_file("/data/data/app/files/qwen.gguf", mp);

    llama_context_params cp = llama_context_default_params();
    cp.n_ctx = 2048;
    cp.n_batch = 256;
    cp.n_threads = 4;                     // 大核数量
    llama_context *ctx = llama_new_context_with_model(model, cp);

    // ... tokenize → decode 循环 → detokenize
    env->ReleaseStringUTFChars(jprompt, prompt);
    return env->NewStringUTF(result.c_str());
}

# MLX（macOS / iOS，Python 侧原型验证）
import mlx.core as mx
from mlx_lm import load, generate

model, tokenizer = load("mlx-community/Qwen2.5-1.5B-Instruct-4bit")
out = generate(model, tokenizer, prompt="用一句话解释量化", max_tokens=100)
print(out)

# 用 coremltools 把模型转成 Core ML（ANE 原生支持 INT8/FP16）
python -m coremltools.converters.llm \
  --model Qwen/Qwen2.5-1.5B-Instruct \
  --output-dir ./coreml-model \
  --compute-units ALL          # CPU + GPU + ANE

// Android NNAPI：让系统调度到最佳加速器（NPU/DSP/GPU）
Model model = Model.create(modelBuffer, new Model.Options.Builder()
        .setDevice(NNAPI_DEVICE)
        .setPriority(PRIORITY_HIGH)
        .build());

<script type="module">
import * as webllm from "https://esm.run/@mlc-ai/web-llm";

const engine = await webllm.CreateMLCEngine(
  "Qwen2.5-1.5B-Instruct-q4f16_1-MLC",
  {
    initProgressCallback: (p) => {
      console.log(`加载进度: ${(p.progress * 100).toFixed(1)}%`, p.text);
    },
  }
);

const chunks = await engine.chat.completions.create({
  messages: [{ role: "user", content: "用一句话介绍 WebGPU" }],
  stream: true,
  temperature: 0.7,
});

for await (const chunk of chunks) {
  process.stdout.write(chunk.choices[0]?.delta?.content ?? "");
}
</script>

// 用 Cache API 缓存权重，二次访问秒开
if ("caches" in window) {
  const cache = await caches.open("webllm-weights-v1");
  // WebLLM 内部已做，也可手动预取
}

import { Wllama } from "@wllama/wllama";

const wllama = new Wllama({
  "single-thread/wllama.wasm": "/wasm/single-thread/wllama.wasm",
  "multi-thread/wllama.wasm": "/wasm/multi-thread/wllama.wasm",
});

await wllama.loadModelFromUrl("/models/qwen2.5-0.5b-q4_k_m.gguf", {
  n_threads: navigator.hardwareConcurrency || 4,
});

const stream = await wllama.createCompletion("你好", {
  nPredict: 128,
  sampling: { temp: 0.7, top_p: 0.9 },
});

Cross-Origin-Opener-Policy: same-origin
Cross-Origin-Embedder-Policy: require-corp

async function detectBackend() {
  if (navigator.gpu) {
    const adapter = await navigator.gpu.requestAdapter();
    if (adapter) {
      const hasF16 = adapter.features.has("shader-f16");
      return hasF16 ? "webgpu-f16" : "webgpu-f32";
    }
  }
  if (crossOriginIsolated) return "wasm-multithread";
  return "wasm-singlethread";
}

const backend = await detectBackend();
// 按后端选择不同的模型档位
const MODEL_BY_BACKEND = {
  "webgpu-f16": "Qwen2.5-1.5B-Instruct-q4f16_1-MLC",
  "webgpu-f32": "Qwen2.5-0.5B-Instruct-q4f32_1-MLC",
  "wasm-multithread": "qwen2.5-0.5b-q4_k_m.gguf",
  "wasm-singlethread": "qwen2.5-0.5b-q4_0.gguf",
};

Qwen2.5-1.5B Q4: 权重 ~1.0 GB
KV Cache (2048 ctx, GQA): 约 0.1 GB
运行时 + 词表 + 中间张量: 约 0.3 GB
──────────────────────────────
合计 ≈ 1.4 GB  →  8GB 手机的 17%，安全

典型曲线（旗舰手机，1.5B Q4）：
  0~60s:   25 tok/s（满血）
  60~180s: 18 tok/s（温升）
  180s+:   10~12 tok/s（降频）

async def hybrid_generate(query: str) -> str:
    # 端侧做轻量判断（分类模型，几毫秒）
    complexity = await on_device_classify(query)

    if complexity == "simple" and not needs_fresh_knowledge(query):
        return await on_device_generate(query)      # 本地出结果
    return await cloud_generate(query)              # 上云

def redact(text: str, entities: list[tuple[str, str]]) -> tuple[str, dict]:
    mapping = {}
    for i, (span, label) in enumerate(entities):
        placeholder = f"<{label}_{i}>"
        mapping[placeholder] = span
        text = text.replace(span, placeholder)
    return text, mapping

// llama.cpp：保存/加载会话状态，复用系统提示的 KV
llama_state_seq_save(ctx, "session.bin", 0, tokens, n_tokens);
// 下次启动
llama_state_seq_load(ctx, "session.bin", 0, tokens, n_tokens);

llama-cli -m qwen2.5-3b-q4.gguf \
  -md qwen2.5-0.5b-q4.gguf \
  --draft-max 8 --draft-min 4 \
  -p "解释一下什么是端侧推理"

const metrics = {
  loadMs: 0,          // 模型加载
  ttftMs: 0,          // 首 token
  tps: 0,             // 生成速度
  peakMemoryMB: 0,    // 峰值内存（performance.measureUserAgentSpecificMemory）
  thermalThrottled: false,
};