【Python】tacotron2を使って音声合成をローカル環境で試した時の備忘録

import os
# transcript.txtの変換
in_path = 'archive/transcript.txt'
out_path = 'filelists/transcript.txt'
output = []
nums = ['一', '二', '三', '四', '五', '六', '七', '八', '九', '十', '百', '章おわり。']
with open(in_path) as f:
    lines = f.readlines()
    for line in lines:
        strs = line.split('|')

        flag = False
        for num in nums:
            if strs[1].startswith(num):
                print(strs[0])
                flag = True
                break
        if flag:
            continue
        if len(strs[2]) < 10:
            continue

        strs[2] = strs[2].replace('&#12289;',',')
        strs[2] = strs[2].replace('&#12290;','.')
        strs[2] = strs[2].replace('&#8213;&#8213;','')
        strs[2] = strs[2].replace('&#65311;','')
        strs[2] = strs[2].replace('&#65281;','')
        strs[2] = strs[2].replace(' ','')
        output.append(strs[0]+'|'+strs[2]+'\n')

with open(out_path, 'w') as f:
    f.writelines(output)

linuxの場合
head -n 6500 filelists/transcript.txt > filelists/transcript_train.txt
tail -n 85 filelists/transcript.txt > filelists/transcript_val.txt
Windowsの場合
Get-Content "filelists/transcript.txt" | Select-Object -first 6500 | Out-File "filelists/transcript_train.txt"
Get-Content "filelists/transcript.txt" | Select-Object -last 85 | Out-File "filelists/transcript_val.txt"

# 事前実行
# $ pip install librosa==0.8.0
# $ pip install pysoundfile==0.9.0.post1

import os
import librosa
import soundfile as sf

# サンプリングビットの変換
in_path = 'ダウンロード/archive/meian/'
out_path = 'ダウンロード/filelists/meian/'#事前にmeianというフォルダを作っておく
filenames = os.listdir(in_path)
for filename in filenames:
   print(out_path+filename)
   y, sr = librosa.core.load(in_path+filename, sr=22050, mono=True) # 22050Hz、モノラルで読み込み
   sf.write(out_path+filename, y, sr, subtype="PCM_16") #16bitで書き込み

#実行後の完成したmeianフォルダをtacotron2フォルダの配下に移動しておきます

pip install tensorflow==1.15.5

Successfully installed
absl-py-1.3.0
astor-0.8.1
gast-0.2.2
google-pasta-0.2.0
grpcio-1.48.2
h5py-2.10.0
keras-applications-1.0.8
keras-preprocessing-1.1.2
numpy-1.18.5
opt-einsum-3.3.0
protobuf-3.19.6
six-1.16.0
tensorboard-1.15.0
tensorflow-1.15.5
tensorflow-estimator-1.15.1
termcolor-1.1.0
wrapt-1.14.1

pip install torch==1.6.0+cu101 torchvision==0.7.0+cu101 -f https://download.pytorch.org/whl/torch_stable.html

Successfully installed 
future-0.18.2
pillow-8.4.0
torch-1.6.0+cu101
torchvision-0.7.0+cu101

# NVIDIA/tacotron2のインストール
!git clone https://github.com/NVIDIA/tacotron2.git
%cd tacotron2
!git submodule init; git submodule update

matplotlib==2.1.0
#tensorflow==1.15.2 15系の最新バージョンでいいのでは、どっちでもいいかも
tensorflow==1.15.5
#numpy==1.13.3
numpy==1.18
inflect==0.2.5
#librosa==0.6.0
librosa==0.5.1
#scipy==1.0.0
scipy==1.4.0
Unidecode==1.0.22
#pillow Python3.7以上じゃないと最新盤は動かないので下げる
pillow==8.4.0

# Apexのインストール
!git clone https://github.com/NVIDIA/apex
%cd apex
!pip install -v --disable-pip-version-check --no-cache-dir --global-option="--cpp_ext" --global-option="--cuda_ext" ./

pip install Unidecode==1.0.22

Successfully installed
PyYAML-6.0
atomicwrites-1.4.1
attrs-22.2.0
colorama-0.4.5
cxxfilt-0.3.0
importlib-metadata-4.8.3
importlib-resources-5.4.0
iniconfig-1.1.1
pluggy-1.0.0
py-1.11.0
pytest-7.0.1
tomli-1.2.3
tqdm-4.64.1
typing-extensions-4.1.1
zipp-3.6.0

tacotron2:.
│   .gitmodules
│   audio_processing.py
│   data_utils.py
│   demo.wav
│   distributed.py
│   Dockerfile
│   hparams.py
│   inference.ipynb
│   layers.py
│   LICENSE
│   logger.py
│   loss_function.py
│   loss_scaler.py
│   model.py
│   multiproc.py
│   my_inference.ipynb
│   plotting_utils.py
│   README.md
│   requirements.txt
│   stft.py
│   tacotron2_statedict.pt
│   tensorboard.png
│   train.py
│   utils.py
│   waveglow_256channels_universal_v5.pt
│
├───filelists
│       ljs_audio_text_test_filelist.txt
│       ljs_audio_text_train_filelist.txt
│       ljs_audio_text_val_filelist.txt
│       transcript.txt
│       transcript_train.txt
│       transcript_val.txt
│
├───meian
│       meian_0000.wav
│       meian_0001.wav
│       meian_0002.wav
│       meian_0003.wav
│       meian_0004.wav
~~~~~~~~
│       meian_6838.wav
│       meian_6839.wav
│       meian_6840.wav
├───text
│   │   cleaners.py
│   │   cmudict.py
│   │   LICENSE
│   │   numbers.py
│   │   symbols.py
│   │   __init__.py
│   │
│   └───__pycache__
│           cleaners.cpython-36.pyc
│           cleaners.cpython-37.pyc
│           cmudict.cpython-36.pyc
│           cmudict.cpython-37.pyc
│           numbers.cpython-36.pyc
│           numbers.cpython-37.pyc
│           symbols.cpython-36.pyc
│           symbols.cpython-37.pyc
│           __init__.cpython-36.pyc
│           __init__.cpython-37.pyc
│
├───waveglow
│   │   .gitmodules
│   │   config.json
│   │   convert_model.py
│   │   denoiser.py
│   │   distributed.py
│   │   glow.py
│   │   glow_old.py
│   │   inference.py
│   │   LICENSE
│   │   mel2samp.py
│   │   README.md
│   │   requirements.txt
│   │   train.py
│   │   waveglow_logo.png
│   │
│   ├───tacotron2
│   └───__pycache__
│           denoiser.cpython-37.pyc
│           glow.cpython-37.pyc
│
└───__pycache__
        audio_processing.cpython-36.pyc
        audio_processing.cpython-37.pyc
        data_utils.cpython-36.pyc
        data_utils.cpython-37.pyc
        distributed.cpython-36.pyc
        distributed.cpython-37.pyc
        hparams.cpython-36.pyc
        hparams.cpython-37.pyc
        layers.cpython-36.pyc
        layers.cpython-37.pyc
        logger.cpython-36.pyc
        logger.cpython-37.pyc
        loss_function.cpython-36.pyc
        loss_function.cpython-37.pyc
        model.cpython-36.pyc
        model.cpython-37.pyc
        plotting_utils.cpython-36.pyc
        plotting_utils.cpython-37.pyc
        stft.cpython-36.pyc
        stft.cpython-37.pyc
        train.cpython-37.pyc
        utils.cpython-36.pyc
        utils.cpython-37.pyc

python train.py --output_directory=outdir --log_directory=logdir -c tacotron2_statedict.pt --warm_start

WARNING:tensorflow:
The TensorFlow contrib module will not be included in TensorFlow 2.0.
For more information, please see:
 * https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md
 * https://github.com/tensorflow/addons
 * https://github.com/tensorflow/io (for I/O related ops)
If you depend on functionality not listed there, please file an issue.

FP16 Run: False
Dynamic Loss Scaling: True
Distributed Run: False
cuDNN Enabled: True
cuDNN Benchmark: False
Warm starting model from checkpoint 'tacotron2_statedict.pt'
Epoch: 0
Train loss 0 4.530630 Grad Norm 106.327721 4.36s/it
Validation loss 0: 74.075213  
Saving model and optimizer state at iteration 0 to outdir/checkpoint_0
Train loss 1 1.618356 Grad Norm 14.370405 3.57s/it
Train loss 2 3.936485 Grad Norm 68.261269 3.69s/it
Train loss 3 2.413089 Grad Norm 33.520912 3.78s/it
Train loss 4 1.628413 Grad Norm 13.512403 3.56s/it
    :

(py36) C:\Users\○○>pip list
Package              Version
-------------------- -----------
absl-py              0.15.0
apex                 0.1
astor                0.8.1
atomicwrites         1.4.1
attrs                22.2.0
audioread            3.0.0
cachetools           4.2.4
certifi              2021.5.30
charset-normalizer   2.0.12
colorama             0.4.5
cxxfilt              0.3.0
cycler               0.11.0
dataclasses          0.8
decorator            5.1.1
future               0.18.2
gast                 0.2.2
google-auth          1.35.0
google-auth-oauthlib 0.4.6
google-pasta         0.2.0
grpcio               1.48.2
h5py                 2.10.0
idna                 3.4
importlib-metadata   4.8.3
importlib-resources  5.4.0
inflect              0.2.5
iniconfig            1.1.1
joblib               1.1.1
Keras-Applications   1.0.8
Keras-Preprocessing  1.1.2
librosa              0.6.2
llvmlite             0.31.0
Markdown             3.3.4
matplotlib           2.1.0
mkl-fft              1.3.0
mkl-random           1.1.1
mkl-service          2.3.0
numba                0.48.0
numpy                1.18.0
oauthlib             3.2.2
opt-einsum           3.3.0
packaging            21.3
Pillow               8.4.0
pip                  21.2.2
pluggy               1.0.0
protobuf             3.19.6
py                   1.11.0
pyasn1               0.4.8
pyasn1-modules       0.2.8
pyparsing            3.0.9
pyreadline           2.1
pytest               7.0.1
python-dateutil      2.8.2
pytz                 2022.7
PyYAML               6.0
requests             2.27.1
requests-oauthlib    1.3.1
resampy              0.3.1
rsa                  4.9
scikit-learn         0.24.2
scipy                1.5.3
setuptools           58.0.4
six                  1.16.0
tensorboard          1.15.0
tensorflow           1.15.0
tensorflow-estimator 1.15.1
termcolor            1.1.0
threadpoolctl        3.1.0
tomli                1.2.3
torch                1.6.0+cu101
torchvision          0.7.0+cu101
tqdm                 4.64.1
typing_extensions    4.1.1
Unidecode            1.0.22
urllib3              1.26.13
webencodings         0.5.1
Werkzeug             0.16.1
wheel                0.37.1
wincertstore         0.2
wrapt                1.14.1
zipp                 3.6.0

conda install -c conda-forge tensorboard==1.15.0

Traceback (most recent call last):
  File "train.py", line 13, in <module>
    from model import Tacotron2
  File "C:\Users\sena\Documents\Python Scripts\onsei\tacotron2\model.py", line 6, in <module>
    from layers import ConvNorm, LinearNorm
  File "C:\Users\sena\Documents\Python Scripts\onsei\tacotron2\layers.py", line 2, in <module>
    from librosa.filters import mel as librosa_mel_fn
  File "C:\Users\sena\anaconda3\envs\py36\lib\site-packages\librosa\__init__.py", line 12, in <module>
    from . import core
  File "C:\Users\sena\anaconda3\envs\py36\lib\site-packages\librosa\core\__init__.py", line 110, in <module>
    from .audio import *  # pylint: disable=wildcard-import
  File "C:\Users\sena\anaconda3\envs\py36\lib\site-packages\librosa\core\audio.py", line 12, in <module>
    import resampy
  File "C:\Users\sena\anaconda3\envs\py36\lib\site-packages\resampy\__init__.py", line 7, in <module>
    from .core import *
  File "C:\Users\sena\anaconda3\envs\py36\lib\site-packages\resampy\core.py", line 11, in <module>
    from .interpn import resample_f_s, resample_f_p
  File "C:\Users\sena\anaconda3\envs\py36\lib\site-packages\resampy\interpn.py", line 75, in <module>
    nopython=True,
TypeError: guvectorize() missing 1 required positional argument: 'signature'

Warning! ***HDF5 library version mismatched error***
The HDF5 header files used to compile this application do not match
the version used by the HDF5 library to which this application is linked.
Data corruption or segmentation faults may occur if the application continues.
This can happen when an application was compiled by one version of HDF5 but
linked with a different version of static or shared HDF5 library.
You should recompile the application or check your shared library related
settings such as 'LD_LIBRARY_PATH'.
You can, at your own risk, disable this warning by setting the environment
variable 'HDF5_DISABLE_VERSION_CHECK' to a value of '1'.
Setting it to 2 or higher will suppress the warning messages totally.
Headers are 1.10.6, library is 1.10.5

conda install -c conda-forge hdf5=1.10.5

pip install typing-extensions

future feature annotations is not defined

pip install joblib==0.17.0

pip install numba==0.49.1

pip install resampy==0.3.1

pip install -U scipy

batch_size=20,#デフォルトで64 https://github.com/a8568730/Tacotron-2/issues/8

pip install protobuf==3.20.*

import matplotlib
%matplotlib inline
import matplotlib.pylab as plt

import IPython.display as ipd

import sys
sys.path.append('waveglow/')
import numpy as np
import torch

from hparams import create_hparams
from model import Tacotron2
from layers import TacotronSTFT, STFT
from audio_processing import griffin_lim
from train import load_model
from text import text_to_sequence
from denoiser import Denoiser

def plot_data(data, figsize=(16, 4)):
    fig, axes = plt.subplots(1, len(data), figsize=figsize)
    for i in range(len(data)):
        axes[i].imshow(data[i], aspect='auto', origin='bottom', 
                       interpolation='none')

hparams = create_hparams()
hparams.sampling_rate = 22050

checkpoint_path = "tacotron2_statedict.pt"
model = load_model(hparams)
model.load_state_dict(torch.load(checkpoint_path)['state_dict'])
_ = model.cuda().eval().half()

waveglow_path = 'waveglow_256channels_universal_v5.pt'
waveglow = torch.load(waveglow_path)['model']
waveglow.cuda().eval().half()
for k in waveglow.convinv:
    k.float()
denoiser = Denoiser(waveglow)

text = "konnichiwa"
sequence = np.array(text_to_sequence(text, ['basic_cleaners']))[None, :]
sequence = torch.autograd.Variable(
    torch.from_numpy(sequence)).cuda().long()

mel_outputs, mel_outputs_postnet, _, alignments = model.inference(sequence)
with torch.no_grad():
    audio = waveglow.infer(mel_outputs_postnet, sigma=0.666)
ipd.Audio(audio[0].data.cpu().numpy(), rate=hparams.sampling_rate)