LLM 推理部署与优化：vLLM PagedAttention、量化技术与生产级服务搭建

# 安装
pip install vllm

# 启动 OpenAI 兼容 API 服务
python -m vllm.entrypoints.openai.api_server \
    --model Qwen/Qwen2.5-7B-Instruct \
    --tensor-parallel-size 1 \
    --max-model-len 8192 \
    --dtype bfloat16 \
    --gpu-memory-utilization 0.9

# 客户端调用（与 OpenAI API 100% 兼容）
curl http://localhost:8000/v1/chat/completions \
  -H "Content-Type: application/json" \
  -d '{
    "model": "Qwen/Qwen2.5-7B-Instruct",
    "messages": [{"role": "user", "content": "Hello!"}],
    "max_tokens": 100
  }'

from vllm import LLM, SamplingParams

# 加载模型
llm = LLM(
    model="Qwen/Qwen2.5-7B-Instruct",
    tensor_parallel_size=2,          # 双卡张量并行
    dtype="bfloat16",
    max_model_len=8192,
    gpu_memory_utilization=0.9,
)

# 采样参数
sampling_params = SamplingParams(
    temperature=0.7,
    top_p=0.9,
    max_tokens=512,
)

# 批量推理
prompts = [
    "Explain quantum computing in simple terms.",
    "Write a Python function to calculate factorial.",
    "What are the benefits of microservices?",
]

outputs = llm.generate(prompts, sampling_params)
for output in outputs:
    print(f"Prompt: {output.prompt}\nGenerated: {output.outputs[0].text}\n")

from vllm import LLM, SamplingParams
from vllm.lora.request import LoRARequest

# LoRA 适配器热加载（生产多租户场景）
llm = LLM(
    model="meta-llama/Llama-2-7b-hf",
    enable_lora=True,
    max_loras=4,               # 同时加载的 LoRA 数量
    max_lora_rank=64,
)

sampling = SamplingParams(temperature=0.8)

# 使用不同 LoRA 适配器
output = llm.generate(
    "What is machine learning?",
    sampling,
    lora_request=LoRARequest("math-lora", 1, "/path/to/math-lora"),
)

# 量化加载
quant_llm = LLM(
    model="TheBloke/Llama-2-7B-AWQ",
    quantization="awq",
    dtype="auto",
)

# Docker 启动
docker run --gpus all -p 8080:80 \
  -v $PWD/data:/data \
  ghcr.io/huggingface/text-generation-inference:2.0 \
  --model-id Qwen/Qwen2.5-7B-Instruct \
  --quantize awq

# 调用
curl http://localhost:8080/generate \
  -X POST -H "Content-Type: application/json" \
  -d '{"inputs":"What is AI?","parameters":{"max_new_tokens":100}}'

# 启动 server
./server -m model-Q4_K_M.gguf \
  -c 4096 -n 512 --host 0.0.0.0 --port 8080 \
  --parallel 4                    # 并发槽数

# 批处理推理
python -c "
from llama_cpp import Llama
llm = Llama('model-Q4_K_M.gguf', n_ctx=4096, n_threads=8)
out = llm('Q: What is RAG?\nA:', max_tokens=200)
print(out['choices'][0]['text'])
"

from awq import AutoAWQForCausalLM
from transformers import AutoTokenizer

model_path = "Qwen/Qwen2.5-7B-Instruct"
quant_path = "qwen-7b-awq"
quant_config = {"zero_point": True, "q_group_size": 128, "w_bit": 4}

# 加载并量化
model = AutoAWQForCausalLM.from_pretrained(model_path)
tokenizer = AutoTokenizer.from_pretrained(model_path)
model.quantize(tokenizer, quant_config=quant_config)

# 保存
model.save_quantized(quant_path)
tokenizer.save_pretrained(quant_path)

# vLLM 加载 AWQ
from vllm import LLM
llm = LLM(model=quant_path, quantization="awq")

from auto_gptq import AutoGPTQForCausalLM, BaseQuantizeConfig

quantize_config = BaseQuantizeConfig(
    bits=4,
    group_size=128,
    desc_act=False,  # True 质量更好但速度更慢
)

model = AutoGPTQForCausalLM.from_pretrained(
    "Qwen/Qwen2.5-7B-Instruct",
    quantize_config,
)
model.quantize(examples, batch_size=1)
model.save_quantized("qwen-7b-gptq")

from transformers import AutoModelForCausalLM

# 仅支持 Hopper 架构 GPU
model = AutoModelForCausalLM.from_pretrained(
    "meta-llama/Llama-3.1-8B",
    torch_dtype=torch.float8_e4m3fn,
    device_map="auto",
)

from vllm import LLM

llm = LLM(
    model="Qwen/Qwen2.5-7B-Instruct",
    max_num_seqs=256,          # 最大并发序列数
    max_num_batched_tokens=4096,  # 每批最大 token 数
    scheduling_policy="fcfs",  # fcfs / priority
)

# vLLM 默认启用 FlashAttention
llm = LLM(
    model="Qwen/Qwen2.5-7B-Instruct",
    max_model_len=32768,       # 32K 上下文
    rope_scaling={"type": "dynamic", "factor": 2.0},  # 位置编码扩展
)

# 手动控制 attention 后端
import os
os.environ["VLLM_ATTENTION_BACKEND"] = "FLASH_ATTN"  # FLASH_ATTN / XFORMERS / FLASHINFER

# 2 GPU 张量并行
python -m vllm.entrypoints.openai.api_server \
    --model Qwen/Qwen2.5-72B-Instruct \
    --tensor-parallel-size 2

# 4 GPU：2 路张量并行 × 2 路流水线并行
python -m vllm.entrypoints.openai.api_server \
    --model Qwen/Qwen2.5-72B-Instruct \
    --tensor-parallel-size 2 \
    --pipeline-parallel-size 2

def estimate_inference_memory(
    params_b: float,      # 参数量（B）
    dtype_bytes: float,   # 数据类型字节数（fp16=2, int8=1, int4=0.5）
    batch: int = 1,
    seq_len: int = 2048,
    layers: int = 32,
    hidden_dim: int = 4096,
    kv_cache_ratio: float = 0.8,  # KV cache 占比
) -> dict:
    """估算推理显存需求。"""
    # 模型权重
    model_mem = params_b * 1e9 * dtype_bytes / (1024**3)

    # KV Cache
    kv_cache = 2 * layers * batch * seq_len * hidden_dim * 2 / (1024**3)

    # 激活值（粗略估计）
    activation = batch * seq_len * hidden_dim * 4 / (1024**3)

    # 开销系数
    overhead = 1.2
    total = (model_mem + kv_cache + activation) * overhead

    return {
        "model_weights_gb": round(model_mem, 2),
        "kv_cache_gb": round(kv_cache, 2),
        "activation_gb": round(activation, 2),
        "total_gb": round(total, 2),
    }

# Llama-2-7B FP16 @ 4K 上下文
print(estimate_inference_memory(7, 2, seq_len=4096))
# {'model_weights_gb': 13.02, 'kv_cache_gb': 2.0, 'activation_gb': 0.12, 'total_gb': 18.17}

from fastapi import FastAPI, HTTPException
from fastapi.responses import StreamingResponse
from pydantic import BaseModel, Field
from vllm import LLM, SamplingParams, AsyncLLMEngine
from vllm.sampling_params import SamplingParams
from vllm.utils import random_uuid
import asyncio
import json

app = FastAPI(title="LLM Inference Service")

class ChatRequest(BaseModel):
    messages: list[dict]
    model: str = "default"
    temperature: float = Field(default=0.7, ge=0.0, le=2.0)
    top_p: float = Field(default=0.9, ge=0.0, le=1.0)
    max_tokens: int = Field(default=512, ge=1, le=8192)
    stream: bool = False
    stop: list[str] = []

class ChatResponse(BaseModel):
    content: str
    usage: dict
    model: str

class InferenceEngine:
    """vLLM 推理引擎封装。"""
    def __init__(self, model_path: str):
        self.engine = AsyncLLMEngine.from_engine_args(
            model_path,
            tensor_parallel_size=1,
            dtype="bfloat16",
            max_model_len=8192,
        )

    async def generate(
        self,
        prompt: str,
        sampling_params: SamplingParams,
        request_id: str = None,
    ):
        request_id = request_id or random_uuid()
        generator = self.engine.generate(prompt, sampling_params, request_id)

        final_output = None
        async for output in generator:
            final_output = output

        return final_output

    async def stream_generate(
        self,
        prompt: str,
        sampling_params: SamplingParams,
        request_id: str = None,
    ):
        request_id = request_id or random_uuid()
        generator = self.engine.generate(prompt, sampling_params, request_id)

        async for output in generator:
            yield output.outputs[0].text

# 使用更简单的同步方式（vLLM 的 LLM 类）
engine = LLM(
    model="Qwen/Qwen2.5-7B-Instruct",
    tensor_parallel_size=1,
    dtype="bfloat16",
    max_model_len=8192,
    gpu_memory_utilization=0.9,
)

async def format_messages(messages: list[dict]) -> str:
    """将消息列表格式化为模型输入提示。"""
    # 简单 format（实际使用 tokenizer.apply_chat_template）
    formatted = ""
    for m in messages:
        if m["role"] == "system":
            formatted += f"System: {m['content']}\n"
        elif m["role"] == "user":
            formatted += f"User: {m['content']}\n"
        elif m["role"] == "assistant":
            formatted += f"Assistant: {m['content']}\n"
    formatted += "Assistant: "
    return formatted

@app.post("/v1/chat/completions", response_model=ChatResponse)
async def chat_completions(req: ChatRequest):
    prompt = await format_messages(req.messages)

    sampling = SamplingParams(
        temperature=req.temperature,
        top_p=req.top_p,
        max_tokens=req.max_tokens,
        stop=req.stop,
    )

    if req.stream:
        async def stream_generator():
            outputs = engine.generate([prompt], sampling)
            for output in outputs[0].outputs:
                data = json.dumps({"text": output.text})
                yield f"data: {data}\n\n"
            yield "data: [DONE]\n\n"

        return StreamingResponse(stream_generator(), media_type="text/event-stream")

    outputs = engine.generate([prompt], sampling)
    generated = outputs[0].outputs[0].text

    return ChatResponse(
        content=generated,
        usage={
            "prompt_tokens": len(outputs[0].prompt_token_ids),
            "completion_tokens": len(outputs[0].outputs[0].token_ids),
            "total_tokens": len(outputs[0].prompt_token_ids) + len(outputs[0].outputs[0].token_ids),
        },
        model=req.model,
    )

@app.get("/health")
async def health():
    return {"status": "healthy", "model": "Qwen/Qwen2.5-7B-Instruct"}

@app.get("/metrics")
async def metrics():
    """Prometheus 风格指标。"""
    return {
        "gpu_utilization": 0.85,
        "requests_total": 12345,
        "avg_latency_ms": 120,
    }

# docker-compose.yml
version: '3.8'

services:
  vllm:
    image: vllm/vllm-openai:latest
    runtime: nvidia
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              count: all
              capabilities: [gpu]
    volumes:
      - ./models:/models
      - ./hf_cache:/root/.cache/huggingface
    environment:
      - HF_HOME=/root/.cache/huggingface
      - CUDA_VISIBLE_DEVICES=0,1
    command: >
      --model /models/Qwen2.5-7B-Instruct
      --tensor-parallel-size 2
      --max-model-len 8192
      --dtype bfloat16
      --gpu-memory-utilization 0.9
      --max-num-seqs 256
      --enable-prefix-caching
      --port 8000
    ports:
      - "8000:8000"
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:8000/health"]
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 60s

  api:
    build: ./api
    ports:
      - "8080:8080"
    environment:
      - VLLM_BASE_URL=http://vllm:8000
      - API_KEY=${API_KEY}
    depends_on:
      vllm:
        condition: service_healthy
    restart: unless-stopped

  redis:
    image: redis:7-alpine
    volumes:
      - redis_data:/data
    restart: unless-stopped

  nginx:
    image: nginx:alpine
    ports:
      - "80:80"
      - "443:443"
    volumes:
      - ./nginx.conf:/etc/nginx/nginx.conf:ro
      - ./ssl:/etc/nginx/ssl:ro
    depends_on:
      - api
    restart: unless-stopped

volumes:
  redis_data:

upstream llm_backend {
    least_conn;
    server api1:8080;
    server api2:8080;
}

server {
    listen 80;
    server_name llm-api.example.com;

    location /v1/ {
        proxy_pass http://llm_backend;
        proxy_set_header Host $host;
        proxy_set_header X-Real-IP $remote_addr;

        # SSE 支持
        proxy_buffering off;
        proxy_cache off;
        proxy_read_timeout 300s;
    }

    location /metrics {
        allow 10.0.0.0/8;
        deny all;
        proxy_pass http://llm_backend;
    }
}