多模态视频分析：从关键帧抽取到时序理解的 LLM 视频理解实战

import google.generativeai as genai

genai.configure(api_key="YOUR_KEY")
model = genai.GenerativeModel("gemini-1.5-pro")

video = genai.upload_file("/path/to/video.mp4")
response = model.generate_content([
    video,
    "用中文总结这个视频的主要内容，列出 3 个关键要点。"
])
print(response.text)

import cv2
import numpy as np
from scenedetect import detect, ContentDetector

def extract_keyframes(video_path, method="scene", max_frames=32):
    """智能关键帧提取
    method: "uniform" 均匀采样 | "scene" 场景切换 | "motion" 运动变化
    """
    cap = cv2.VideoCapture(video_path)
    total_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
    fps = cap.get(cv2.CAP_PROP_FPS)

    if method == "uniform":
        # 均匀采样
        indices = np.linspace(0, total_frames - 1, max_frames, dtype=int)
        frames = [read_frame(cap, i) for i in indices]

    elif method == "scene":
        # 基于内容变化的场景检测
        scene_list = detect(video_path, ContentDetector(threshold=27))
        key_indices = [scene[0].get_frames() for scene in scene_list]
        # 如果场景太少，补充均匀采样
        if len(key_indices) < max_frames:
            extra = np.linspace(0, total_frames - 1, max_frames - len(key_indices), dtype=int)
            key_indices.extend(extra)
        frames = [read_frame(cap, i) for i in key_indices[:max_frames]]

    cap.release()
    return frames

def read_frame(cap, index):
    cap.set(cv2.CAP_PROP_POS_FRAMES, index)
    ret, frame = cap.read()
    if ret:
        return cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
    return None

from openai import OpenAI
import base64
from PIL import Image
import io

client = OpenAI()

def frames_to_grid(frames, grid_size=(4, 4)):
    """将帧序列拼合成一张大图减少 API 调用"""
    rows, cols = grid_size
    frame_size = frames[0].shape[:2]

    grid = Image.new("RGB", (frame_size[1] * cols, frame_size[0] * rows))
    for i, frame in enumerate(frames[:rows * cols]):
        img = Image.fromarray(frame)
        x = (i % cols) * frame_size[1]
        y = (i // cols) * frame_size[0]
        grid.paste(img, (x, y))

    return grid

def summarize_video(video_path, max_frames=16):
    frames = extract_keyframes(video_path, method="scene", max_frames=max_frames)
    grid = frames_to_grid(frames, grid_size=(4, 4))

    # 转为 base64
    buffer = io.BytesIO()
    grid.save(buffer, format="PNG")
    b64 = base64.b64encode(buffer.getvalue()).decode()

    response = client.chat.completions.create(
        model="gpt-4o",
        messages=[
            {
                "role": "user",
                "content": [
                    {
                        "type": "text",
                        "text": """这是一段视频的关键帧序列（按时间顺序排列）。
请生成一段 200 字的中文摘要，包含：
1. 视频主题
2. 主要人物/物体
3. 关键情节发展
4. 情感基调"""
                    },
                    {
                        "type": "image_url",
                        "image_url": {"url": f"data:image/png;base64,{b64}"}
                    },
                ],
            }
        ],
        max_tokens=500,
    )
    return response.choices[0].message.content

def detect_anomaly(video_path):
    """检测视频中的异常行为"""
    frames = extract_keyframes(video_path, method="motion", max_frames=8)

    actions = []
    for frame in frames:
        b64 = frame_to_base64(frame)
        response = client.chat.completions.create(
            model="gpt-4o",
            messages=[{
                "role": "user",
                "content": [
                    {
                        "type": "text",
                        "text": "描述图中人物的行为（正常/异常）。如果异常，说明原因。"
                    },
                    {"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{b64}"}}
                ]
            }]
        )
        actions.append(response.choices[0].message.content)

    # 时序一致性检查
    anomaly_count = sum(1 for a in actions if "异常" in a)
    return {
        "anomaly_detected": anomaly_count >= 2,
        "details": actions,
    }

def analyze_lesson_video(video_path):
    """分析教学视频的结构与质量"""
    frames = extract_keyframes(video_path, max_frames=12)
    grid = frames_to_grid(frames, grid_size=(3, 4))
    b64 = image_to_base64(grid)

    response = client.chat.completions.create(
        model="gpt-4o",
        messages=[{
            "role": "user",
            "content": [
                {
                    "type": "text",
                    "text": """这是一段教学视频的关键帧。请分析：
1. 教学结构（开场 → 内容 → 总结）
2. 使用的教学媒介（PPT/板书/实物演示）
3. 师生互动频率评估
4. 改进建议"""
                },
                {"type": "image_url", "image_url": {"url": f"data:image/png;base64,{b64}"}}
            ]
        }]
    )
    return response.choices[0].message.content

import google.generativeai as genai
import time

genai.configure(api_key="YOUR_API_KEY")

# 上传视频
video_file = genai.upload_file(path="lecture.mp4")
print(f"上传完成: {video_file.uri}")

# 等待处理
while video_file.state.name == "PROCESSING":
    time.sleep(10)
    video_file = genai.get_file(video_file.name)

# 生成分析
model = genai.GenerativeModel("gemini-1.5-pro")
response = model.generate_content([
    video_file,
    """你是一个教育视频分析专家。请：
1. 按时间戳列出主要教学节点
2. 提取板书/PPT 中的核心知识点
3. 评估讲解清晰度（1-10分）
4. 给出剪辑建议（哪些部分可以加速/删减）"""
])

print(response.text)

# 安装
git clone https://github.com/DAMO-NLP-SG/Video-LLaMA.git
cd Video-LLaMA
pip install -r requirements.txt

# 下载权重
python download_weights.py

# 运行推理
python inference.py \
  --cfg-path eval_configs/video_llama_eval_withaudio.yaml \
  --video-path ./sample.mp4 \
  --question "描述这段视频的内容"