大模型本地部署与私有托管：从 Ollama 到 vLLM 生产级推理集群

# macOS / Linux
curl -fsSL https://ollama.com/install.sh | sh

# 验证安装
ollama --version

# 拉取并运行模型（首次自动下载）
ollama run llama3.2
ollama run qwen2.5
ollama run deepseek-r1:14b

# 查看本地模型列表
ollama list

# 查看模型信息
ollama show llama3.2

# Modelfile
FROM llama3.2

# 系统提示词
SYSTEM """你是一个专业的 Python 编程助手。回答应简洁、准确，并提供可运行的代码示例。"""

# 超参数配置
PARAMETER temperature 0.3
PARAMETER top_p 0.9
PARAMETER num_ctx 8192

# 添加知识库文件（RAG 前置）
ADD ./python_style_guide.md .

# 构建自定义模型
ollama create python-assistant -f Modelfile

# 运行自定义模型
ollama run python-assistant

# 原生 Ollama API
curl http://localhost:11434/api/generate -d '{
  "model": "llama3.2",
  "prompt": "解释 Python 的装饰器",
  "stream": false
}'

# OpenAI 兼容模式（强烈推荐，可直接替换现有代码）
curl http://localhost:11434/v1/chat/completions \
  -H "Content-Type: application/json" \
  -d '{
    "model": "llama3.2",
    "messages": [{"role": "user", "content": "Hello!"}],
    "temperature": 0.7
  }'

# Python 客户端集成（OpenAI SDK 直接替换 base_url）
from openai import OpenAI

client = OpenAI(
    base_url="http://localhost:11434/v1",
    api_key="ollama",  # 任意字符串，Ollama 不校验
)

response = client.chat.completions.create(
    model="llama3.2",
    messages=[{"role": "user", "content": "什么是量子计算？"}],
    temperature=0.7,
)
print(response.choices[0].message.content)

# 启动时设置并发槽位
OLLAMA_NUM_PARALLEL=4 OLLAMA_MAX_LOADED_MODELS=2 ollama serve

# 推荐环境变量（写入 ~/.zshrc）
export OLLAMA_HOST=0.0.0.0:11434        # 允许局域网访问
export OLLAMA_NUM_PARALLEL=4             # 并发请求数
export OLLAMA_MAX_LOADED_MODELS=2        # 内存中同时驻留的模型数
export OLLAMA_KEEP_ALIVE=30m             # 模型保留时间
export OLLAMA_FLASH_ATTENTION=1          # 启用 Flash Attention（加速）

# 克隆并编译（带 CUDA 支持）
git clone https://github.com/ggerganov/llama.cpp.git
cd llama.cpp
cmake -B build -DGGML_CUDA=ON
cmake --build build --config Release -j$(nproc)

# 下载并运行 GGUF 模型
./build/bin/llama-cli \
  -m models/Qwen2.5-7B-Instruct-Q4_K_M.gguf \
  -c 8192 -n 512 \
  -p "User: 解释什么是 RAG\nAssistant:"

# 启动 server 模式（OpenAI 兼容 API）
./build/bin/llama-server \
  -m models/Qwen2.5-7B-Instruct-Q4_K_M.gguf \
  -c 8192 --host 0.0.0.0 --port 8080 \
  --threads 8 --parallel 4

# 安装（CPU 版）
pip install llama-cpp-python

# 安装（CUDA 版）
CMAKE_ARGS="-DGGML_CUDA=ON" pip install llama-cpp-python --force-reinstall --no-cache-dir

from llama_cpp import Llama

# 加载模型
llm = Llama(
    model_path="models/Qwen2.5-7B-Instruct-Q4_K_M.gguf",
    n_ctx=8192,           # 上下文长度
    n_threads=8,          # CPU 线程数（建议 = 物理核心数）
    n_batch=512,          # 批处理大小
    verbose=False,
)

# 推理
output = llm(
    "User: 什么是向量数据库？\nAssistant:",
    max_tokens=256,
    temperature=0.7,
    stop=["User:", "\n\n"],
)
print(output["choices"][0]["text"])

# Chat 模式（自动处理对话模板）
response = llm.create_chat_completion(
    messages=[
        {"role": "system", "content": "你是一个有帮助的助手。"},
        {"role": "user", "content": "推荐三本 Python 进阶书籍。"},
    ],
    max_tokens=512,
    temperature=0.7,
)

# 下载 llamafile（约 4-8GB 单文件）
wget https://huggingface.co/Mozilla/Qwen2.5-7B-Instruct-llamafile/resolve/main/qwen2.5-7b-instruct.Q4_K_M.llamafile
chmod +x qwen2.5-7b-instruct.Q4_K_M.llamafile

# 直接运行（内置 HTTP server）
./qwen2.5-7b-instruct.Q4_K_M.llamafile --server --host 0.0.0.0 --port 8080

# 安装
pip install vllm

# 单卡启动 OpenAI 兼容服务
python -m vllm.entrypoints.openai.api_server \
  --model Qwen/Qwen2.5-7B-Instruct \
  --dtype bfloat16 \
  --max-model-len 8192 \
  --gpu-memory-utilization 0.9 \
  --enable-prefix-caching \
  --port 8000

# 多卡张量并行（72B 模型）
python -m vllm.entrypoints.openai.api_server \
  --model Qwen/Qwen2.5-72B-Instruct \
  --tensor-parallel-size 4 \
  --dtype bfloat16 \
  --max-model-len 32768 \
  --port 8000

# AWQ 4-bit 量化（显存减半，速度提升）
python -m vllm.entrypoints.openai.api_server \
  --model Qwen/Qwen2.5-7B-Instruct-AWQ \
  --quantization awq \
  --dtype auto \
  --max-model-len 8192 \
  --port 8000

# GPTQ 量化
python -m vllm.entrypoints.openai.api_server \
  --model TheBloke/Llama-2-7B-GPTQ \
  --quantization gptq \
  --port 8000

# docker-compose.yml
version: '3.8'

services:
  vllm:
    image: vllm/vllm-openai:latest
    runtime: nvidia
    shm_size: '16gb'
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              count: all
              capabilities: [gpu]
    volumes:
      - ./models:/models:ro
      - hf_cache:/root/.cache/huggingface
    environment:
      - HF_HOME=/root/.cache/huggingface
      - CUDA_VISIBLE_DEVICES=0,1
      - VLLM_WORKER_MULTIPROC_METHOD=spawn
    command: >
      --model /models/Qwen2.5-7B-Instruct
      --tensor-parallel-size 2
      --max-model-len 8192
      --dtype bfloat16
      --gpu-memory-utilization 0.92
      --max-num-seqs 256
      --enable-prefix-caching
      --port 8000
    ports:
      - "8000:8000"
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:8000/health"]
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 120s
    restart: unless-stopped

  nginx:
    image: nginx:alpine
    ports:
      - "80:80"
    volumes:
      - ./nginx.conf:/etc/nginx/nginx.conf:ro
    depends_on:
      - vllm
    restart: unless-stopped

volumes:
  hf_cache:

# nginx.conf
upstream vllm_backend {
    least_conn;
    server vllm:8000;
}

server {
    listen 80;
    client_max_body_size 10M;

    location /v1/ {
        proxy_pass http://vllm_backend;
        proxy_set_header Host $host;
        proxy_set_header X-Real-IP $remote_addr;
        proxy_buffering off;
        proxy_read_timeout 300s;
    }

    location /health {
        proxy_pass http://vllm_backend/health;
    }
}

# Docker 启动（自动从 Hub 下载）
docker run --gpus all -p 8080:80 \
  -v $PWD/data:/data \
  ghcr.io/huggingface/text-generation-inference:2.4 \
  --model-id Qwen/Qwen2.5-7B-Instruct \
  --quantize awq \
  --max-input-length 8192 \
  --max-total-tokens 16384

# 调用
 curl http://localhost:8080/v1/chat/completions \
  -X POST -H "Content-Type: application/json" \
  -d '{
    "model": "tgi",
    "messages": [{"role": "user", "content": "Hello!"}],
    "max_tokens": 100
  }'

# 下载转换脚本
git clone https://github.com/ggerganov/llama.cpp
cd llama.cpp

# 从 HuggingFace 模型转 GGUF（FP16）
python convert_hf_to_gguf.py ../Qwen2.5-7B-Instruct \
  --outfile qwen2.5-7b-f16.gguf \
  --outtype f16

# 量化（推荐 Q4_K_M：平衡速度与质量）
./llama-quantize qwen2.5-7b-f16.gguf qwen2.5-7b-Q4_K_M.gguf Q4_K_M

# 更高质量选项：Q5_K_M
./llama-quantize qwen2.5-7b-f16.gguf qwen2.5-7b-Q5_K_M.gguf Q5_K_M

# 近乎无损：Q8_0
./llama-quantize qwen2.5-7b-f16.gguf qwen2.5-7b-Q8_0.gguf Q8_0

from awq import AutoAWQForCausalLM
from transformers import AutoTokenizer

model_path = "Qwen/Qwen2.5-7B-Instruct"
quant_path = "qwen2.5-7b-awq"

quant_config = {
    "zero_point": True,
    "q_group_size": 128,
    "w_bit": 4,
    "version": "GEMM",
}

model = AutoAWQForCausalLM.from_pretrained(model_path)
tokenizer = AutoTokenizer.from_pretrained(model_path)
model.quantize(tokenizer, quant_config=quant_config)

model.save_quantized(quant_path)
tokenizer.save_pretrained(quant_path)

# docker-compose.localai.yml
version: '3.8'
services:
  localai:
    image: localai/localai:latest-aio-gpu-nvidia-cuda-12
    runtime: nvidia
    ports:
      - "8080:8080"
    volumes:
      - ./models:/build/models:ro
    environment:
      - MODELS_PATH=/build/models
      - THREADS=8
      - CONTEXT_SIZE=8192

def estimate_vram(
    params_b: float,
    bytes_per_param: float,
    context_length: int = 4096,
    batch_size: int = 1,
    kv_cache_dtype: float = 2,  # fp16 = 2 bytes
) -> dict:
    """估算推理所需显存（GB）。

    Args:
        params_b: 参数量（十亿），如 7、13、70
        bytes_per_param: 每个参数字节数
            - FP16: 2
            - INT8: 1
            - INT4 / Q4: 0.5
        context_length: 最大上下文长度
        batch_size: 批大小
    """
    # 模型权重
    model_weights = params_b * 1e9 * bytes_per_param / (1024**3)

    # KV Cache（简化公式：Llama 架构近似）
    # 注意：不同架构系数不同，此公式为经验估计
    layers = params_b * 4.5  # 每 1B 参数约 4.5 层（近似）
    hidden_dim = 4096 if params_b <= 8 else 5120 if params_b <= 13 else 8192
    kv_cache = 2 * layers * batch_size * context_length * hidden_dim * kv_cache_dtype / (1024**3)

    # 激活值与系统开销（通常占权重的 10-20%）
    activation_overhead = model_weights * 0.15

    # 总计 + 10% 安全裕量
    total = (model_weights + kv_cache + activation_overhead) * 1.1

    return {
        "model_weights_gb": round(model_weights, 1),
        "kv_cache_gb": round(kv_cache, 1),
        "activation_gb": round(activation_overhead, 1),
        "total_estimate_gb": round(total, 1),
    }

# 示例：Qwen2.5-72B AWQ 4-bit @ 8K上下文，batch=4
print(estimate_vram(72, 0.5, context_length=8192, batch_size=4))
# {'model_weights_gb': 33.6, 'kv_cache_gb': 124.4, 'activation_gb': 5.0, 'total_estimate_gb': 179.4}
# => 需要约 180GB 显存，即 3×A100 80GB

# LangChain
from langchain_openai import ChatOpenAI

llm = ChatOpenAI(
    model="llama3.2",
    base_url="http://localhost:11434/v1",
    api_key="not-needed",
    temperature=0.7,
)

# LlamaIndex
from llama_index.llms.openai import OpenAI as LlamaOpenAI

llm = LlamaOpenAI(
    model="qwen2.5",
    api_base="http://localhost:8000/v1",
    api_key="dummy",
)

# CrewAI
import os
os.environ["OPENAI_API_BASE"] = "http://localhost:11434/v1"
os.environ["OPENAI_API_KEY"] = "dummy"
os.environ["OPENAI_MODEL_NAME"] = "llama3.2"

# 统一网关客户端（支持 fallback）
import random
from openai import OpenAI

class LLMRouter:
    def __init__(self, backends: list[dict]):
        self.backends = backends
        self.clients = {
            b["name"]: OpenAI(base_url=b["url"], api_key=b.get("key", "dummy"))
            for b in backends
        }

    def chat(self, messages, **kwargs):
        # 按权重选择后端
        backend = random.choices(
            self.backends,
            weights=[b.get("weight", 1) for b in self.backends],
        )[0]

        client = self.clients[backend["name"]]
        model = backend["model"]

        try:
            return client.chat.completions.create(
                model=model, messages=messages, **kwargs
            )
        except Exception:
            # Fallback 到下一个后端
            for b in self.backends:
                if b["name"] != backend["name"]:
                    return self.clients[b["name"]].chat.completions.create(
                        model=b["model"], messages=messages, **kwargs
                    )
            raise

router = LLMRouter([
    {"name": "vllm", "url": "http://vllm:8000/v1", "model": "Qwen2.5-72B", "weight": 3},
    {"name": "ollama", "url": "http://ollama:11434/v1", "model": "qwen2.5", "weight": 1},
])

# docker-compose.security.yml
services:
  vllm:
    # 不直接暴露端口，仅通过 sidecar 访问
    expose:
      - "8000"
    networks:
      - backend

  auth-proxy:
    image: oauth2-proxy/oauth2-proxy:latest
    ports:
      - "8080:8080"
    environment:
      - OAUTH2_PROXY_UPSTREAMS=http://vllm:8000
      - OAUTH2_PROXY_PROVIDER=oidc
      - OAUTH2_PROXY_CLIENT_ID=${OIDC_CLIENT_ID}
      - OAUTH2_PROXY_CLIENT_SECRET=${OIDC_CLIENT_SECRET}
    networks:
      - backend
      - frontend

networks:
  backend:
    internal: true  # 无外部路由
  frontend:

# FastAPI 中间件示例
from fastapi import FastAPI, Request, HTTPException
from fastapi.middleware.cors import CORSMiddleware
import time
from collections import defaultdict

app = FastAPI()

# 简单 Token Bucket（生产请用 Redis + lua）
rate_limits = defaultdict(lambda: {"tokens": 10, "last": time.time()})

@app.middleware("http")
async def rate_limit(request: Request, call_next):
    client = request.client.host
    now = time.time()
    limit = rate_limits[client]

    # 补充令牌
    limit["tokens"] = min(10, limit["tokens"] + (now - limit["last"]) * 2)
    limit["last"] = now

    if limit["tokens"] < 1:
        raise HTTPException(429, "Too many requests")

    limit["tokens"] -= 1
    return await call_next(request)

# CORS 严格限制
app.add_middleware(
    CORSMiddleware,
    allow_origins=["https://internal.company.com"],
    allow_methods=["POST"],
    allow_headers=["Authorization", "Content-Type"],
)