使用nemo训练语音合成模型

export DL_SITE=https://mirrors.tuna.tsinghua.edu.cn/anaconda/miniconda
wget -c $DL_SITE/Miniconda3-py38_4.10.3-Linux-x86_64.sh
bash Miniconda3-py38_4.10.3-Linux-x86_64.sh
source ~/.bashrc

apt-get update && apt-get install -y libsndfile1 ffmpeg
pip install Cython 
pip install --user pytest-runner 
pip install rosa numpy==1.19.4 
pip install torchmetrics==0.6.0 
pip install nemo_toolkit[all]==1.4.0 
pip install ASR-metrics

import pytorch_lightning as pl
from nemo.collections.common.callbacks import LogEpochTimeCallback
from nemo.collections.tts.models import Tacotron2Model
from nemo.core.config import hydra_runner
from nemo.utils.exp_manager import exp_manager
# hydra_runner is a thin NeMo wrapper around Hydra
# It looks for a config named tacotron2.yaml inside the conf folder
# Hydra parses the yaml and returns it as a Omegaconf DictConfig
@hydra_runner(config_path="conf", config_name="tacotron2")
def main(cfg):
    # Define the Lightning trainer
    trainer = pl.Trainer(**cfg.trainer)
    # exp_manager is a NeMo construct that helps with logging and checkpointing
    exp_manager(trainer, cfg.get("exp_manager", None))
    # Define the Tacotron 2 model, this will construct the model as well as
    # define the training and validation dataloaders
    model = Tacotron2Model(cfg=cfg.model, trainer=trainer)
    # Let's add a few more callbacks
    lr_logger = pl.callbacks.LearningRateMonitor()
    epoch_time_logger = LogEpochTimeCallback()
    trainer.callbacks.extend([lr_logger, epoch_time_logger])
    # Call lightning trainer's fit() to train the model
    trainer.fit(model)
if __name__ == '__main__':
    main()  # noqa pylint: disable=no-value-for-parameter

name: Tacotron2
sample_rate: 22050
# <PAD>, <BOS>, <EOS> will be added by the tacotron2.py script
labels: [' ', '!', '"', "'", '(', ')', ',', '-', '.', ':', ';', '?', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H',
        'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', '[', ']',
        'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't',
        'u', 'v', 'w', 'x', 'y', 'z']
n_fft: 1024
n_mels: 80
fmax: 8000
n_stride: 256
pad_value: -11.52
train_dataset: ???
validation_datasets: ???

model:
  labels: ${labels}
  train_ds:
    dataset:
      _target_: "nemo.collections.asr.data.audio_to_text.AudioToCharDataset"
      manifest_filepath: ${train_dataset}
      max_duration: null
      min_duration: 0.1
      trim: false
      int_values: false
      normalize: true
      sample_rate: ${sample_rate}
      # bos_id: 66
      # eos_id: 67
      # pad_id: 68  These parameters are added automatically in Tacotron2
    dataloader_params:
      drop_last: false
      shuffle: true
      batch_size: 48
      num_workers: 4


  validation_ds:
    dataset:
      _target_: "nemo.collections.asr.data.audio_to_text.AudioToCharDataset"
      manifest_filepath: ${validation_datasets}
      max_duration: null
      min_duration: 0.1
      int_values: false
      normalize: true
      sample_rate: ${sample_rate}
      trim: false
      # bos_id: 66
      # eos_id: 67
      # pad_id: 68  These parameters are added automatically in Tacotron2
    dataloader_params:
      drop_last: false
      shuffle: false
      batch_size: 48
      num_workers: 8

  preprocessor:
    _target_: nemo.collections.asr.parts.preprocessing.features.FilterbankFeatures
    dither: 0.0
    nfilt: ${n_mels}
    frame_splicing: 1
    highfreq: ${fmax}
    log: true
    log_zero_guard_type: clamp
    log_zero_guard_value: 1e-05
    lowfreq: 0
    mag_power: 1.0
    n_fft: ${n_fft}
    n_window_size: 1024
    n_window_stride: ${n_stride}
    normalize: null
    pad_to: 16
    pad_value: ${pad_value}
    preemph: null
    sample_rate: ${sample_rate}
    window: hann

  encoder:
    _target_: nemo.collections.tts.modules.tacotron2.Encoder
    encoder_kernel_size: 5
    encoder_n_convolutions: 3
    encoder_embedding_dim: 512

  decoder:
    _target_: nemo.collections.tts.modules.tacotron2.Decoder
    decoder_rnn_dim: 1024
    encoder_embedding_dim: ${model.encoder.encoder_embedding_dim}
    gate_threshold: 0.3
    max_decoder_steps: 5000
    n_frames_per_step: 1  # currently only 1 is supported
    n_mel_channels: ${n_mels}
    p_attention_dropout: 0.1
    p_decoder_dropout: 0.1
    prenet_dim: 256
    prenet_p_dropout: 0.5
    # Attention parameters
    attention_dim: 128
    attention_rnn_dim: 1024
    # AttentionLocation Layer parameters
    attention_location_kernel_size: 31
    attention_location_n_filters: 32
    early_stopping: true

  postnet:
    _target_: nemo.collections.tts.modules.tacotron2.Postnet
    n_mel_channels: ${n_mels}
    p_dropout: 0.5
    postnet_embedding_dim: 512
    postnet_kernel_size: 5
    postnet_n_convolutions: 5

  optim:
    name: adam
    lr: 1e-3
    weight_decay: 1e-6

    # scheduler setup
    sched:
      name: CosineAnnealing
      min_lr: 1e-5


trainer:
  gpus: 1 # number of gpus
  max_epochs: ???
  num_nodes: 1
  accelerator: ddp
  accumulate_grad_batches: 1
  checkpoint_callback: False  # Provided by exp_manager
  logger: False  # Provided by exp_manager
  gradient_clip_val: 1.0
  flush_logs_every_n_steps: 1000
  log_every_n_steps: 200
  check_val_every_n_epoch: 25

exp_manager:
  exp_dir: null
  name: ${name}
  create_tensorboard_logger: True
  create_checkpoint_callback: True

{"audio_filepath":"语音文件位置", "duration":语音时长, "text"："语音表示的文本内容"}

import nemo
import nemo.collections.tts as nemo_tts
from nemo.collections.tts.models import Tacotron2Model
from matplotlib.pyplot import imshow
from matplotlib import pyplot as plt
#导入所需要模块
! HYDRA_FULL_ERROR=1 \
python tacotron2.py train_dataset=训练集的json文件路径 \
validation_datasets=测试集的json文件路径 \
trainer.max_epochs=4000 \
trainer.accelerator=null \
trainer.check_val_every_n_epoch=1

model = Tacotron2Model.restore_from("模型的路径")
vocoder = nemo_tts.models.MelGanModel.restore_from("tts_melgan.nemo")
#生成语音对应频谱图
text = "需要发声的语音内容"
tokens = model.parse(text)
spectrogram = model.generate_spectrogram(tokens = tokens)
%matplotlib inline
imshow(spectrogram.cpu().detach().numpy()[0,...], origin="lower")
plt.show()

#通过声码器播放合成的语音内容
audio = vocoder.convert_spectrogram_to_audio(spec=spectrogram)
import IPython
IPython.display.Audio(audio.to('cpu').detach().numpy(), rate=22050)