多模态图像理解：从 GPT-4o Vision 到开源视觉大模型的生产实践

from openai import OpenAI
import base64

client = OpenAI()

def encode_image(image_path):
    with open(image_path, "rb") as f:
        return base64.b64encode(f.read()).decode("utf-8")

image_b64 = encode_image("./cat.jpg")

response = client.chat.completions.create(
    model="gpt-4o",
    messages=[
        {
            "role": "user",
            "content": [
                {"type": "text", "text": "这张图里有什么动物？它在做什么？"},
                {
                    "type": "image_url",
                    "image_url": {
                        "url": f"data:image/jpeg;base64,{image_b64}",
                        "detail": "auto",  # auto / low / high
                    },
                },
            ],
        }
    ],
    max_tokens=300,
)
print(response.choices[0].message.content)

def extract_text_from_image(image_path):
    """从图片中提取结构化文本"""
    image_b64 = encode_image(image_path)
    response = client.chat.completions.create(
        model="gpt-4o",
        messages=[
            {
                "role": "user",
                "content": [
                    {
                        "type": "text",
                        "text": """请从这张图片中提取所有文本内容。
如果是表格，请输出 Markdown 格式。
如果是发票/票据，请提取：日期、金额、商户名称、商品列表。"""
                    },
                    {
                        "type": "image_url",
                        "image_url": {"url": f"data:image/jpeg;base64,{image_b64}"},
                    },
                ],
            }
        ],
    )
    return response.choices[0].message.content

def analyze_chart(image_path):
    """分析数据图表"""
    image_b64 = encode_image(image_path)
    response = client.chat.completions.create(
        model="gpt-4o",
        messages=[
            {
                "role": "user",
                "content": [
                    {
                        "type": "text",
                        "text": """分析这张图表：
1. 图表类型和数据维度
2. 关键趋势和异常点（附具体数值）
3. 可能的数据洞察
4. 建议的可视化改进"""
                    },
                    {
                        "type": "image_url",
                        "image_url": {"url": f"data:image/jpeg;base64,{image_b64}"},
                    },
                ],
            }
        ],
    )
    return response.choices[0].message.content

from playwright.sync_api import sync_playwright

def ui_agent(task: str, screenshot_func):
    """通过视觉理解完成 UI 任务"""
    max_steps = 10
    for step in range(max_steps):
        screenshot = screenshot_func()
        b64 = base64.b64encode(screenshot).decode()

        response = client.chat.completions.create(
            model="gpt-4o",
            messages=[
                {
                    "role": "user",
                    "content": [
                        {
                            "type": "text",
                            "text": f"""当前网页截图。目标：{task}
请按以下格式回复：
分析：描述当前页面状态
行动：click(x,y) / type(x,y,"text") / scroll(up/down) / done
坐标基于 1920x1080 分辨率。"""
                        },
                        {
                            "type": "image_url",
                            "image_url": {"url": f"data:image/png;base64,{b64}"},
                        },
                    ],
                }
            ],
        )

        action = response.choices[0].message.content
        print(f"Step {step}: {action}")

        if "done" in action.lower():
            break
        # 执行 action（需接入 Playwright）
        execute_action(action)

# 使用 Playwright 截图
with sync_playwright() as p:
    browser = p.chromium.launch()
    page = browser.new_page()
    page.goto("https://example.com")

    def screenshot():
        return page.screenshot()

    ui_agent("帮我登录并查看订单列表", screenshot)

# 安装 Ollama
brew install ollama  # macOS
curl -fsSL https://ollama.com/install.sh | sh  # Linux

# 拉取 Llava 模型
ollama pull llava:34b

# 运行
ollama run llava:34b

import ollama

response = ollama.chat(
    model="llava:34b",
    messages=[{
        "role": "user",
        "content": "描述这张图片",
        "images": ["./photo.jpg"]
    }]
)
print(response["message"]["content"])

from transformers import Qwen2VLForConditionalGeneration, AutoProcessor
from PIL import Image
import torch

model = Qwen2VLForConditionalGeneration.from_pretrained(
    "Qwen/Qwen2-VL-7B-Instruct",
    torch_dtype=torch.float16,
    device_map="auto",
)
processor = AutoProcessor.from_pretrained("Qwen/Qwen2-VL-7B-Instruct")

image = Image.open("./chart.png")
conversation = [
    {
        "role": "user",
        "content": [
            {"type": "image"},
            {"type": "text", "text": "分析这张图表的关键趋势"},
        ],
    }
]

text_prompt = processor.apply_chat_template(conversation, add_generation_prompt=True)
inputs = processor(text=[text_prompt], images=[image], return_tensors="pt").to(model.device)

output_ids = model.generate(**inputs, max_new_tokens=512)
response = processor.batch_decode(output_ids, skip_special_tokens=True)[0]
print(response)

def smart_resize(image_path, max_pixels=512 * 768):
    """智能调整图像大小，控制 token 成本"""
    from PIL import Image
    img = Image.open(image_path)
    current_pixels = img.width * img.height

    if current_pixels > max_pixels:
        ratio = (max_pixels / current_pixels) ** 0.5
        new_size = (int(img.width * ratio), int(img.height * ratio))
        img = img.resize(new_size, Image.LANCZOS)

    return img

def create_image_grid(image_paths, grid_size=(2, 2)):
    """将多张图片拼接为一张，减少 API 调用次数"""
    from PIL import Image

    images = [Image.open(p) for p in image_paths]
    w, h = images[0].size
    grid_w, grid_h = grid_size

    grid = Image.new("RGB", (w * grid_w, h * grid_h))
    for idx, img in enumerate(images):
        x = (idx % grid_w) * w
        y = (idx // grid_w) * h
        grid.paste(img.resize((w, h)), (x, y))

    return grid