多模态生成式 AI：从 DALL-E 3 到视频生成与图生图的生产实战

from openai import OpenAI
client = OpenAI()

response = client.images.generate(
    model="dall-e-3",
    prompt="一只橙色猫咪在温暖的午后阳光下打盹，背景是日式庭院，柔和的水彩画风",
    size="1024x1024",  # 1024x1024, 1024x1792, 1792x1024
    quality="standard",  # standard | hd
    n=1,
)

image_url = response.data[0].url
print(image_url)

# 优质 Prompt 结构
"""
[主体], [动作/状态], [环境/背景], [艺术风格], [光照/氛围], [视角/构图], [质量修饰词]

示例:
"一只银灰色的波斯猫[主体]在窗台上优雅地伸展[动作],
窗外是雨后的城市天际线[背景],
赛璐珞动画风格[风格],
柔和的侧光[光照],
中景平视[视角],
8K 超清细节[质量]"
"""

# 负面提示（Negative Prompt）— SD/Flux 专用
negative_prompt = """
blur, low quality, distorted face, extra fingers, mutated hands,
poorly drawn, bad anatomy, watermark, signature, text, cropped
"""

# 方法 1：使用 Ollama
ollama pull llava  # Ollama 也支持部分视觉生成

# 方法 2：ComfyUI（推荐）
git clone https://github.com/comfyanonymous/ComfyUI.git
cd ComfyUI
pip install -r requirements.txt
python main.py

# 方法 3：Stable Diffusion WebUI AUTOMATIC1111
git clone https://github.com/AUTOMATIC1111/stable-diffusion-webui.git
cd stable-diffusion-webui
./webui.sh

# 使用 diffusers 库程序化调用
from diffusers import StableDiffusionXLPipeline
import torch

pipe = StableDiffusionXLPipeline.from_pretrained(
    "stabilityai/stable-diffusion-xl-base-1.0",
    torch_dtype=torch.float16,
    variant="fp16",
).to("cuda")

image = pipe(
    prompt="一只穿着太空服的柯基犬在火星表面奔跑，皮克斯动画风格",
    negative_prompt="模糊，低质量，变形",
    num_inference_steps=30,
    guidance_scale=7.5,
).images[0]

image.save("corgi_mars.png")

import requests

API_KEY = "YOUR_RUNWAY_KEY"

response = requests.post(
    "https://api.runwayml.com/v1/generations",
    headers={"Authorization": f"Bearer {API_KEY}"},
    json={
        "task_type": "text_to_video",
        "model": "gen3",
        "prompt": "Aerial drone shot of a serene Japanese garden in autumn, with koi pond and red maple leaves gently falling",
        "duration": 10,  # seconds
        "ratio": "16:9",
    },
)

task_id = response.json()["id"]

# 轮询结果
import time
while True:
    status = requests.get(
        f"https://api.runwayml.com/v1/generations/{task_id}",
        headers={"Authorization": f"Bearer {API_KEY}"},
    ).json()

    if status["status"] == "succeeded":
        print(f"视频 URL: {status['url']}")
        break
    elif status["status"] == "failed":
        print("生成失败")
        break
    time.sleep(5)

# 差的 Prompt
"一只狗在公园玩"

# 好的 Prompt
"一只金毛犬在阳光明媚的公园草地上奔跑，
镜头从左向右缓慢平移，
背景中可以看到儿童游乐设施和其他遛狗的人，
画面从远景逐渐推近到狗狗的特写，
4K 电影级画质，温暖的午后光线"

from diffusers import StableDiffusionControlNetPipeline, ControlNetModel
from PIL import Image
import torch
import cv2
import numpy as np

# 加载 ControlNet (Canny 边缘检测)
controlnet = ControlNetModel.from_pretrained(
    "lllyasviel/sd-controlnet-canny",
    torch_dtype=torch.float16,
).to("cuda")

pipe = StableDiffusionControlNetPipeline.from_pretrained(
    "runwayml/stable-diffusion-v1-5",
    controlnet=controlnet,
    torch_dtype=torch.float16,
).to("cuda")

# 提取 Canny 边缘
image = Image.open("pose_reference.jpg")
image_np = np.array(image)
low_threshold = 100
high_threshold = 200
canny = cv2.Canny(image_np, low_threshold, high_threshold)
canny_image = Image.fromarray(canny)

# 基于边缘 + Prompt 生成
generated = pipe(
    prompt="一个赛博朋克风格的少女，霓虹灯光",
    image=canny_image,
    num_inference_steps=20,
).images[0]

from diffusers import StableDiffusionPipeline
import torch

# IP-Adapter 允许用参考图控制生成风格/人物
pipe = StableDiffusionPipeline.from_pretrained(
    "runwayml/stable-diffusion-v1-5",
    torch_dtype=torch.float16,
).to("cuda")

pipe.load_ip_adapter(
    "h94/IP-Adapter",
    subfolder="models",
    weight_name="ip-adapter_sd15.bin",
)

reference_image = Image.open("reference_style.jpg")

generated = pipe(
    prompt="一只猫在森林中",
    ip_adapter_image=reference_image,
    num_inference_steps=25,
).images[0]

import asyncio
from dataclasses import dataclass
from typing import List

@dataclass
class GenerationTask:
    prompt: str
    style_reference: str = None
    output_path: str = None

class BatchGenerator:
    def __init__(self, model="dall-e-3"):
        self.model = model
        self.client = OpenAI()

    async def generate_single(self, task: GenerationTask):
        response = self.client.images.generate(
            model=self.model,
            prompt=task.prompt,
            size="1024x1024",
            n=1,
        )

        # 下载并保存
        import aiohttp
        async with aiohttp.ClientSession() as session:
            async with session.get(response.data[0].url) as resp:
                image_data = await resp.read()

        output = task.output_path or f"output/{hash(task.prompt)}.png"
        with open(output, "wb") as f:
            f.write(image_data)

        return {"task": task.prompt, "output": output}

    async def generate_batch(self, tasks: List[GenerationTask], concurrency=3):
        semaphore = asyncio.Semaphore(concurrency)

        async def bounded_generate(task):
            async with semaphore:
                return await self.generate_single(task)

        results = await asyncio.gather(*[bounded_generate(t) for t in tasks])
        return results

# 使用
tasks = [
    GenerationTask("一只柴犬在樱花树下", output_path="shiba_sakura.png"),
    GenerationTask("未来城市的夜景，赛博朋克风格", output_path="cyberpunk.png"),
    GenerationTask("手绘风格的热带海滩", output_path="beach_sketch.png"),
]

generator = BatchGenerator()
results = asyncio.run(generator.generate_batch(tasks))