AI漫剧制作：从零搭建自动化漫画视频生成管线

User Prompt → 剧本脚本(JSON) → 分镜图序列 → 动态视频片段 → 配音轨 → 合成MP4

SYSTEM_PROMPT = """
你是一个漫剧编剧，请根据用户故事生成分镜脚本。
输出格式为JSON数组，每个分镜包含：
- scene_id: 场景编号
- description: 场景描述（用于绘图）
- dialogue: 角色台词（含情感标签）
- duration: 预计时长（秒）
- camera: 镜头运动（pan/zoom/static）
- bgm_style: 背景音乐风格
- characters: 场景中角色列表（含服饰、表情）
"""

def generate_script(user_prompt: str) -> list[dict]:
    response = llm.chat(
        messages=[
            {"role": "system", "content": SYSTEM_PROMPT},
            {"role": "user", "content": user_prompt}
        ],
        response_format={"type": "json_object"}  # 强制JSON
    )
    return json.loads(response.content)["scenes"]

from diffusers import StableDiffusionPipeline
from peft import LoraConfig, get_peft_model

# 1. 准备角色数据集（统一裁剪512x512）
# 2. 配置LoRA
lora_config = LoraConfig(
    r=32,  # rank
    lora_alpha=64,
    target_modules=["to_q", "to_v", "to_k", "to_out.0"],
    lora_dropout=0.1,
)

# 3. 训练（用diffusers + accelerate）
# 实际代码略，核心是逐张生成角色图并计算CLIP相似度作为Reward

def generate_scene_image(prompt: str, character_lora_path: str) -> Image:
    pipe = StableDiffusionPipeline.from_pretrained(
        "runwayml/stable-diffusion-v1-5"
    )
    pipe.load_lora_weights(character_lora_path)
    
    # 负面提示词
    negative_prompt = "bad anatomy, distorted, blurry, low quality, watermark"
    
    image = pipe(
        prompt=prompt,
        negative_prompt=negative_prompt,
        num_inference_steps=30,
        guidance_scale=7.5,
        height=512,
        width=768,  # 16:9宽高比
    ).images[0]
    return image

import cv2
import numpy as np

def apply_camera_motion(image: np.ndarray, motion: str, duration: int, fps: int = 30) -> list[np.ndarray]:
    """生成运动帧序列"""
    h, w = image.shape[:2]
    frames = []
    total_frames = duration * fps
    
    if motion == "pan_left":
        for i in range(total_frames):
            offset = int((i / total_frames) * w * 0.3)
            M = np.float32([[1, 0, -offset], [0, 1, 0]])
            frame = cv2.warpAffine(image, M, (w, h))
            frames.append(frame)
    elif motion == "zoom_in":
        for i in range(total_frames):
            scale = 1 + 0.15 * (i / total_frames)
            M = cv2.getRotationMatrix2D((w/2, h/2), 0, scale)
            frame = cv2.warpAffine(image, M, (w, h))
            frames.append(frame)
    # ... 其他运动
    
    return frames

from diffusers import StableVideoDiffusionPipeline

def generate_video_from_image(image: Image, prompt: str, duration: int = 3):
    pipe = StableVideoDiffusionPipeline.from_pretrained(
        "stabilityai/stable-video-diffusion-img2vid"
    )
    pipe.to("cuda")
    
    # SVD默认生成25帧，调整fps控制时长
    frames = pipe(
        image=image,
        decode_chunk_size=8,
        generator=torch.manual_seed(42),
    ).frames[0]
    
    return frames  # list of PIL Images

# 用Depth ControlNet约束结构
from diffusers import ControlNetModel
controlnet = ControlNetModel.from_pretrained("lllyasviel/control_v11f1p_sd15_depth")
# 结合AnimateDiff做运动生成

import edge_tts

async def generate_voice(text: str, voice: str = "zh-CN-XiaoxiaoNeural"):
    """生成语音"""
    communicate = edge_tts.Communicate(text, voice)
    audio_path = f"audio_{hash(text)}.mp3"
    await communicate.save(audio_path)
    return audio_path

# 支持情感调节：开心、悲伤、愤怒等
# voice参数可选不同音色

# 1. 准备角色音频样本（3-10分钟干净人声）
# 2. 训练GPT-SoVITS模型（约2小时）
# 3. 推理时加载模型
from GPT_SoVITS.inference import TTS

tts = TTS(
    model_path="character_voice.pth",
    config_path="config.yaml"
)
audio = tts.generate("台词文本", emotion="happy")

def estimate_duration(text: str) -> float:
    """估算朗读时长"""
    char_count = len(text)
    base_duration = char_count / 4.0  # 4字/秒
    return max(base_duration, 1.5)  # 最短1.5秒

import subprocess
import json
from moviepy.editor import *

def compose_video(scenes: list[dict], scene_videos: list, audio_tracks: list):
    """合成最终视频"""
    clips = []
    
    for i, (scene, video_frames, audio_path) in enumerate(zip(scenes, scene_videos, audio_tracks)):
        duration = scene["duration"]
        
        # 生成视频片段
        if isinstance(video_frames, list):
            # 帧序列 → VideoClip
            clip = ImageSequenceClip(video_frames, fps=24)
        else:
            clip = VideoFileClip(video_frames)
        
        # 设置时长
        clip = clip.subclip(0, duration)
        
        # 加载音频
        audio_clip = AudioFileClip(audio_path)
        if audio_clip.duration > duration:
            audio_clip = audio_clip.subclip(0, duration)
        else:
            # 音频短则静音补齐
            silence = CompositeAudioClip([audio_clip])
            clip = clip.set_audio(silence)
        
        clip = clip.set_audio(audio_clip)
        clips.append(clip)
    
    # 拼接所有分镜
    final_video = concatenate_videoclips(clips)
    final_video.write_videofile("output.mp4", fps=24, codec="libx264", audio_codec="aac")

from pydub import AudioSegment

def add_bgm(video_path: str, bgm_path: str, volume_reduction: int = 20):
    video_audio = AudioSegment.from_file(video_path)
    bgm = AudioSegment.from_file(bgm_path)
    
    # 循环BGM到视频长度
    bgm = bgm * (len(video_audio) // len(bgm) + 1)
    bgm = bgm[:len(video_audio)]
    
    # 降低BGM音量（-20dB）
    bgm = bgm - volume_reduction
    
    # 混合
    mixed = video_audio.overlay(bgm)
    mixed.export("output_with_bgm.mp4", format="mp4")

class AIMangaPipeline:
    def __init__(self):
        self.llm = init_llm()
        self.sd_pipe = init_sd_pipe()
        self.tts = init_tts()
    
    def run(self, user_prompt: str) -> str:
        # Step 1: 生成分镜脚本
        script = self.generate_script(user_prompt)
        
        # Step 2: 逐分镜生成
        scene_data = []
        for scene in script:
            # 2.1 生成图像
            image = self.generate_image(scene)
            # 2.2 生成动态视频
            video_frames = self.animate_scene(image, scene)
            # 2.3 生成配音
            audio = self.generate_audio(scene["dialogue"])
            scene_data.append((scene, video_frames, audio))
        
        # Step 3: 合成输出
        output_path = self.compose_video(scene_data)
        return output_path

import librosa

def align_subtitles(audio_path: str, text: str, duration: float):
    """根据音频能量做词级对齐（简化版）"""
    y, sr = librosa.load(audio_path)
    energy = librosa.feature.rms(y=y)
    # 根据能量峰值切分时间点
    # 返回SRT格式字幕