虚拟偶像技术深度解析：语音与动作合成的工程化实践

# 目录结构
dataset/
├─ wav/
│  ├─ 0001.wav
│  └─ ...
├─ train.txt  # 格式：0001|你好，我是虚拟偶像 Alice
└─ val.txt

# vits_finetune.py
import torch, yaml, argparse
from vits.train import train

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("-c", "--config", default="configs/alice.json")
    args = parser.parse_args()
    with open(args.config) as f:
        cfg = yaml.safe_load(f)
    train(cfg,  # 加载原始 config
          base_ckpt="pretrained_vits.pth",  # 官方 checkpoint
          data_path="dataset")

# inference_emotion.py
from vits.commons import intersperse
from vits.models import SynthesizerTrn
import torch

model = SynthesizerTrn(*cfg["model_params"])
model.load_state_dict(torch.load("alice_emotion.pth"))
model.eval()

text = "今天天气真不错！"
text_ids = intersperse(text_to_sequence(text, cleaner), 0)
x = torch.LongTensor(text_ids).unsqueeze(0)
sid = torch.LongTensor([0])          # 角色 ID
emo = torch.FloatTensor([0,1,0,0])   # 快乐 one-hot

with torch.no_grad():
    audio = model.infer(x, sid=sid, emo=emo, noise_scale=0.667)[0]
torchaudio.save("alice_happy.wav", audio.unsqueeze(0), 24000)

# 用 librosa 提取节拍
import librosa, numpy as np
y, sr = librosa.load("song.wav", sr=None)
tempo, beats = librosa.beat.beat_track(y=y, sr=sr)
np.save("beats.npy", librosa.frames_to_time(beats, sr=sr))

# beat2dance.py
import torch.nn as nn

class Beat2Dance(nn.Module):
    def __init__(self, n_joints=52):
        super().__init__()
        self.backbone = nn.Sequential(
            nn.Conv1d(1, 64, kernel_size=9, padding=4),
            nn.ReLU(),
            nn.Conv1d(64, 128, 9, padding=4),
            nn.AdaptiveAvgPool1d(1))
        self.fc = nn.Linear(128, n_joints*3)  # 每关节 XYZ 旋转角

    def forward(self, beat):  # beat: [B, T] 0/1
        feat = self.backbone(beat.unsqueeze(1)).squeeze(-1)
        rot = self.fc(feat).view(-1, 52, 3)
        return rot

# real_time_dance.py
import sounddevice as sd, numpy as np
window = np.zeros(int(sr*4*60/120))  # 4 拍
def callback(indata, frames, time, status):
    global window
    window = np.roll(window, -frames)
    window[-frames:] = np.mean(indata, axis=1)
    beat_env = librosa.onset.onset_strength(y=window, sr=sr)
    beats,_ = librosa.beat.beat_track(onset_envelope=beat_env)
    if len(beats)>1:
        rot = model(torch.from_numpy(beat_env).float().unsqueeze(0))
        send_to_unity(rot)  # 通过 UDP 发旋转角
stream = sd.InputStream(callback=callback, blocksize=512)
stream.start()

# mel2viseme.py
from scipy.signal import resample
mel = librosa.feature.melspectrogram(y=audio, sr=24000, n_mels=80)
mel_rs = resample(mel, num=int(len(audio)/24000*60))  # 60 fps
viseme = mel2lip(torch.from_numpy(mel_rs.T).unsqueeze(0))

rot_emo = torch.cat([rot, emo.unsqueeze(1).repeat(1,52,1)], dim=-1)  # [B,52,7]

using UnityEngine;
using System.Net.Sockets;
using MessagePack;

[MessagePackObject]
public struct RotPacket {
    [Key(0)] public float[] rot;  // length=52*3
}

public class UDPServer : MonoBehaviour {
    UdpClient client;
    void Start() {
        client = new UdpClient(9999);
        client.BeginReceive(OnReceive, null);
    }
    void OnReceive(System.IAsyncResult ar) {
        byte[] data = client.EndReceive(ar, ref RemoteEndPoint);
        var pkt = MessagePackSerializer.Deserialize<RotPacket>(data);
        SetJoints(pkt.rot);
        client.BeginReceive(OnReceive, null);
    }
}

// GrabCam.cs
IEnumerator StartPipe() {
    var tex = new Texture2D(Screen.width, Screen.height, TextureFormat.RGB24, false);
    var ffmpeg = new System.Diagnostics.Process();
    ffmpeg.StartInfo.FileName = "ffmpeg";
    ffmpeg.StartInfo.Arguments =
        "-f rawvideo -pix_fmt rgb24 -s 1920x1080 -r 30 -i - -preset ultrafast -tune zerolatency -f flv rtmp://bilibili/live/YOUR_KEY";
    ffmpeg.StartInfo.RedirectStandardInput = true;
    ffmpeg.Start();
    var sw = ffmpeg.StandardInput.BaseStream;
    while (true) {
        yield return new WaitForEndOfFrame();
        tex.ReadPixels(new Rect(0,0,1920,1080), 0, 0);
        tex.Apply();
        var bytes = tex.GetRawTextureData();
        sw.Write(bytes, 0, bytes.Length);
    }
}

# 1. 启动推理服务
python real_time_dance.py &

# 2. 启动 Unity（已打包可执行）
./VirtualIdol.x86_64 &

# 3. 一键推流
./GrabCam &