DiffSinger_colab_notebook.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "collapsed_sections": [
        "MP5rRkbTpnG8",
        "Wv0gfI5feBSc",
        "eexZl_OCDmQ3",
        "0J3b18EKdzMC",
        "FY40fGHEg9_i",
        "4sbU1aH5kGFE"
      ],
      "gpuType": "T4",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/MLo7Ghinsan/DiffSinger_colab_notebook_MLo7/blob/main/DiffSinger_colab_notebook.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "MP5rRkbTpnG8"
      },
      "source": [
        "# _**[DiffSinger](https://github.com/openvpi/DiffSinger)**_\n",
        "_Singing Voice Synthesis via Shallow Diffusion Mechanism (SVS & TTS)_\n",
        "\n",
        "\\\n",
        "____\n",
        "\n",
        "Note:\n",
        "- This notebook will get update semi-frequently based from the feedback or response from users\n",
        "\n",
        "```We refer \"variance\" as \"parameters\" to avoid the confusion```\n",
        "\n",
        "\\\n",
        "____\n",
        "\\\n",
        "#### **This notebook is an edited copy of Kei's Diffsinger [colab notebook](https://colab.research.google.com/drive/1kUg9dz8PPH92NfnLZwgq0_9B9an39t1J?usp=sharing)**\n",
        "####**This notebook is maintained by MLo7**\n",
        "\\\n",
        "___"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# **Setup**"
      ],
      "metadata": {
        "id": "Wv0gfI5feBSc"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "pK8aicf8A2sj",
        "cellView": "form"
      },
      "outputs": [],
      "source": [
        "#@title # Mount Google Drive and Setup\n",
        "\n",
        "\n",
        "from IPython.display import clear_output\n",
        "from IPython.display import Audio, display, HTML\n",
        "import os\n",
        "from google.colab import drive\n",
        "drive.mount(\"/content/drive\")\n",
        "\n",
        "if not os.path.exists(\"/content/pretrain_models\"):\n",
        "    os.makedirs(\"/content/pretrain_models\")\n",
        "\n",
        "%cd /content\n",
        "!wget https://github.com/MLo7Ghinsan/DiffSinger_colab_notebook_MLo7/releases/download/OU_files/jpn_dict.txt\n",
        "\n",
        "!rm -rf /content/sample_data\n",
        "!apt-get install aria2\n",
        "clear_output()\n",
        "\n",
        "!git clone https://github.com/UtaUtaUtau/nnsvs-db-converter\n",
        "!git clone https://github.com/openvpi/DiffSinger.git\n",
        "!git clone https://github.com/openvpi/MakeDiffSinger #for ds training\n",
        "!git clone https://github.com/MLo7Ghinsan/ghin_shenanigans #for ds segmenting <3 ig and for setup complete wav\n",
        "!git clone https://github.com/openvpi/SOME #yass some midi estimation\n",
        "\n",
        "clear_output()\n",
        "!pip install torch torchvision torchaudio\n",
        "clear_output()\n",
        "!pip install -r /content/DiffSinger/requirements.txt\n",
        "!pip install -r /content/SOME/requirements.txt\n",
        "!pip install mido einops\n",
        "clear_output()\n",
        "!aria2c https://github.com/openvpi/vocoders/releases/download/nsf-hifigan-44.1k-hop512-128bin-2024.02/nsf_hifigan_44.1k_hop512_128bin_2024.02.zip\n",
        "!aria2c https://github.com/openvpi/DiffSinger/releases/download/v2.1.0/rmvpe.zip\n",
        "!aria2c https://github.com/openvpi/SOME/releases/download/v1.0.0-baseline/0119_continuous128_5spk.zip\n",
        "!wget https://github.com/yxlllc/vocal-remover/releases/download/hnsep_240512/hnsep_240512.zip #using wget cus aria2c takes a lot longer for some reason\n",
        "!unzip -q /content/hnsep_240512.zip -d /content/DiffSinger/checkpoints\n",
        "!unzip -q 0119_continuous128_5spk.zip -d /content/DiffSinger/checkpoints/SOME\n",
        "!unzip -q nsf_hifigan_44.1k_hop512_128bin_2024.02.zip -d /content/DiffSinger/checkpoints\n",
        "!unzip -q /content/rmvpe.zip -d /content/DiffSinger/checkpoints\n",
        "!unzip -q /content/rmvpe.zip -d /content/MakeDiffSinger/variance-temp-solution/assets\n",
        "!rm /content/nsf_hifigan_44.1k_hop512_128bin_2024.02.zip\n",
        "!rm /content/rmvpe.zip\n",
        "!rm /content/0119_continuous128_5spk.zip\n",
        "\n",
        "# pretrain stuff\n",
        "!aria2c -d /content/pretrain_models -o acoustic_pretrain.ckpt https://github.com/haru0l/diffsinger_models/releases/download/acoustic/model_ckpt_steps_49000.ckpt\n",
        "!aria2c -d /content/pretrain_models -o variance_pretrain.ckpt https://github.com/haru0l/diffsinger_models/releases/download/variance/model_ckpt_steps_51000.ckpt\n",
        "\n",
        "clear_output()\n",
        "!pip install --upgrade tensorboard\n",
        "clear_output()\n",
        "!pip install protobuf\n",
        "clear_output()\n",
        "!pip install onnxruntime\n",
        "clear_output()\n",
        "!pip install pydub\n",
        "clear_output()\n",
        "#shit tons of clear output cus i dont wanna see anything <3\n",
        "\n",
        "print(\"setup complete!\")\n",
        "print(\"|\")\n",
        "print(\"|\")\n",
        "print(\"|\")\n",
        "\n",
        "chika_dance = '<img src=\"https://raw.githack.com/MLo7Ghinsan/ghin_shenanigans/main/image_and_gif/chika_dance.gif\"/>'\n",
        "display(HTML(chika_dance))\n",
        "\n",
        "with open(\"/content/ghin_shenanigans/audio/setup_complete.wav\", \"rb\") as f:\n",
        "    setup_complete_sound = f.read()\n",
        "Audio(data=setup_complete_sound, autoplay=True)"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# **Preprocess data for training**"
      ],
      "metadata": {
        "id": "eexZl_OCDmQ3"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#@title #Extract Data\n",
        "#@markdown ___\n",
        "%cd /content\n",
        "#@markdown this cell will create a folder name [raw_data] in the root folder of colab (/content) and extract your data into it\n",
        "\n",
        "data_type = \"lab + wav (NNSVS format)\" # @param [\"lab + wav (NNSVS format)\", \"csv + wav (DiffSinger format)\", \"ds (DiffSinger format)\"]\n",
        "\n",
        "#@markdown <font size=\"-1.5\"> The path to your data zip file\n",
        "\n",
        "data_zip_path = \"\" #@param {type:\"string\"}\n",
        "\n",
        "#@markdown ___\n",
        "\n",
        "#@markdown nnsvs-db-converter settings (lab + wav ONLY)\n",
        "\n",
        "#@markdown <font size=\"-1.5\"> _These values can exceed the amount that's in your data to maximize the segment length or to keep the data as is_\n",
        "\n",
        "#@markdown <font size=\"-1.5\"> This option is necessary for variance's pitch training\n",
        "estimate_midi_option = \"False\" # @param [\"False\", \"True | parselmouth\", \"True | harvest\", \"True | SOME\"]\n",
        "if estimate_midi_option == \"True | parselmouth\":\n",
        "    estimate_midi = True\n",
        "    midi_pitch_ext = \"parselmouth\"\n",
        "elif estimate_midi_option == \"True | harvest\":\n",
        "    estimate_midi = True\n",
        "    midi_pitch_ext = \"harvest\"\n",
        "else:\n",
        "    estimate_midi = False\n",
        "    midi_pitch_ext = None\n",
        "#@markdown <font size=\"-1.5\"> Determine how long it will segment your data to based on silence phoneme placement (seconds)\n",
        "segment_length = 15 #@param {type:\"slider\", min:5, max:35, step:1}\n",
        "\n",
        "#@markdown <font size=\"-1.5\"> Determine how many silence phoneme is allowed in the middle of each segment\n",
        "max_silence_phoneme_amount = 2 #@param {type:\"slider\", min:0, max:50, step:1}\n",
        "\n",
        "# leaving -S at 60 so max silence can be 60 seconds that exceeds the segment legnth cap idk why///\n",
        "# making the segment length cap at 35 secs because any longer than that would make training goes really slow\n",
        "\n",
        "# my ass dont remember why i made two... i think one is unnecessary extra but mehhh\n",
        "all_shits = \"/content/raw_data\"\n",
        "all_shits_not_wav_n_lab = \"/content/raw_data/diffsinger_db\"\n",
        "\n",
        "import os\n",
        "import csv\n",
        "import json\n",
        "import shutil\n",
        "from pydub import AudioSegment\n",
        "\n",
        "if os.path.exists(\"/content/raw_data\"):\n",
        "    shutil.rmtree(\"/content/raw_data\")\n",
        "\n",
        "if not os.path.exists(all_shits_not_wav_n_lab):\n",
        "  os.makedirs(all_shits_not_wav_n_lab)\n",
        "\n",
        "# using 'if not' bc i edited the wrong section which im also too lazy to fix it <3\n",
        "if not data_type == \"lab + wav (NNSVS format)\":\n",
        "    #changed to 7zip to support more compression types\n",
        "    !7z x \"$data_zip_path\" -o{all_shits_not_wav_n_lab}\n",
        "    for root, dirs, files in os.walk(all_shits):\n",
        "        for filename in files:\n",
        "            if filename.endswith(\".lab\"):\n",
        "                file_path = os.path.join(root, filename)\n",
        "                with open(file_path, \"r\") as file:\n",
        "                    file_data = file.read()\n",
        "                file_data = file_data.replace(\"SP\", \"pau\")\n",
        "                file_data = file_data.replace(\"br\", \"AP\")\n",
        "                with open(file_path, \"w\") as file:\n",
        "                    file.write(file_data)\n",
        "\n",
        "else:\n",
        "    !7z x \"$data_zip_path\" -o{all_shits_not_wav_n_lab}\n",
        "\n",
        "\n",
        "# for funny auto dict generator lmao\n",
        "out = \"/content/DiffSinger/dictionaries/custom_dict.txt\"\n",
        "\n",
        "phonemes = set()\n",
        "\n",
        "def is_excluded(phoneme):\n",
        "    return phoneme in [\"pau\", \"AP\", \"SP\", \"sil\"]\n",
        "\n",
        "if data_type == \"lab + wav (NNSVS format)\":\n",
        "    phoneme_folder_path = all_shits\n",
        "    for root, dirs, files in os.walk(phoneme_folder_path):\n",
        "        for file in files:\n",
        "            if file.endswith(\".lab\"):\n",
        "                fpath = os.path.join(root, file)\n",
        "                with open(fpath, \"r\") as lab_file:\n",
        "                    for line in lab_file:\n",
        "                        line = line.strip()\n",
        "                        if line:\n",
        "                            phoneme = line.split()[2]\n",
        "                            if not is_excluded(phoneme):\n",
        "                                phonemes.add(phoneme)\n",
        "elif data_type == \"csv + wav (DiffSinger format)\":\n",
        "    phoneme_folder_path = all_shits_not_wav_n_lab\n",
        "    for root, dirs, files in os.walk(phoneme_folder_path):\n",
        "        for file in files:\n",
        "            if file.endswith(\".csv\"):\n",
        "                fpath = os.path.join(root, file)\n",
        "                with open(fpath, \"r\", newline=\"\") as csv_file:\n",
        "                    csv_reader = csv.DictReader(csv_file)\n",
        "                    for row in csv_reader:\n",
        "                        if \"ph_seq\" in row:\n",
        "                            ph_seq = row[\"ph_seq\"].strip()\n",
        "                            for phoneme in ph_seq.split():\n",
        "                                if not is_excluded(phoneme):\n",
        "                                    phonemes.add(phoneme)\n",
        "\n",
        "else:\n",
        "    phoneme_folder_path = all_shits_not_wav_n_lab\n",
        "    for root, dirs, files in os.walk(phoneme_folder_path):\n",
        "        for file in files:\n",
        "            if file.endswith(\".ds\"):\n",
        "                fpath = os.path.join(root, file)\n",
        "                with open(fpath, \"r\") as json_file:\n",
        "                    data = json.load(json_file)\n",
        "                    for entry in data:\n",
        "                        if \"ph_seq\" in entry:\n",
        "                            ph_seq = entry[\"ph_seq\"].strip()\n",
        "                            phoneme_list = ph_seq.split()\n",
        "                            for phoneme in phoneme_list:\n",
        "                                if not is_excluded(phoneme):\n",
        "                                    phonemes.add(phoneme)\n",
        "\n",
        "with open(out, \"w\") as f:\n",
        "    for phoneme in sorted(phonemes):\n",
        "        f.write(phoneme + \"\t\" + phoneme + \"\\n\")\n",
        "\n",
        "# for vowels and consonants.txt.... well adding luquid type for uta's script\n",
        "dict_path = out\n",
        "vowel_types = {\"a\", \"i\", \"u\", \"e\", \"o\", \"N\", \"M\", \"NG\"}\n",
        "liquid_types = {\"y\", \"w\", \"l\", \"r\"} # r for english labels, it should be fine with jp too\n",
        "vowel_data = []\n",
        "consonant_data = []\n",
        "liquid_data = []\n",
        "\n",
        "with open(dict_path, \"r\") as f:\n",
        "    for line in f:\n",
        "        phoneme, _ = line.strip().split(\"\\t\")\n",
        "        if phoneme[0] in vowel_types:\n",
        "            vowel_data.append(phoneme)\n",
        "        elif phoneme[0] in liquid_types:\n",
        "            liquid_data.append(phoneme)\n",
        "        else:\n",
        "            consonant_data.append(phoneme)\n",
        "\n",
        "vowel_data.sort()\n",
        "liquid_data.sort()\n",
        "consonant_data.sort()\n",
        "directory = os.path.dirname(dict_path)\n",
        "\n",
        "# make txt for language json file\n",
        "vowel_txt_path = os.path.join(directory, \"vowels.txt\")\n",
        "with open(vowel_txt_path, \"w\") as f:\n",
        "    f.write(\" \".join(vowel_data))\n",
        "liquid_txt_path = os.path.join(directory, \"liquids.txt\")\n",
        "with open(liquid_txt_path, \"w\") as f:\n",
        "    f.write(\" \".join(liquid_data))\n",
        "consonant_txt_path = os.path.join(directory, \"consonants.txt\")\n",
        "with open(consonant_txt_path, \"w\") as f:\n",
        "    f.write(\" \".join(consonant_data))\n",
        "\n",
        "\n",
        "# here's a funny json append\n",
        "with open(vowel_txt_path, \"r\") as f:\n",
        "    vowel_data = f.read().split()\n",
        "with open(liquid_txt_path, \"r\") as f:\n",
        "    liquid_data = f.read().split()\n",
        "with open(consonant_txt_path, \"r\") as f:\n",
        "    consonant_data = f.read().split()\n",
        "liquid_list = {liquid: True for liquid in liquid_data} #temp fix, might need more research about the push in timing'''\n",
        "phones4json = {\"vowels\": vowel_data, \"liquids\": liquid_list}\n",
        "with open(\"/content/nnsvs-db-converter/lang.sample.json\", \"w\") as rawr:\n",
        "    json.dump(phones4json, rawr, indent=4)\n",
        "\n",
        "\n",
        "if data_type == \"lab + wav (NNSVS format)\":\n",
        "    db_converter_script = \"/content/nnsvs-db-converter/db_converter.py\"\n",
        "    for raw_folder_name in os.listdir(all_shits_not_wav_n_lab):\n",
        "        raw_folder_path = os.path.join(all_shits_not_wav_n_lab, raw_folder_name)\n",
        "        if os.path.isdir(raw_folder_path):\n",
        "            if estimate_midi:\n",
        "                !python {db_converter_script} -s {max_silence_phoneme_amount} -l {segment_length} -m -c -L \"/content/nnsvs-db-converter/lang.sample.json\" {raw_folder_path}\n",
        "            else:\n",
        "                !python {db_converter_script} -s {max_silence_phoneme_amount} -l {segment_length} -L \"/content/nnsvs-db-converter/lang.sample.json\" {raw_folder_path}\n",
        "            !rm -rf {raw_folder_path}/*.wav {raw_folder_path}/*.lab\n",
        "            !mv {raw_folder_path}/diffsinger_db/* {raw_folder_path} 2> /dev/null\n",
        "            !rm -rf {raw_folder_path}/diffsinger_db\n",
        "            if estimate_midi_option == \"True | SOME\":\n",
        "                !python /content/SOME/batch_infer.py --model \"/content/DiffSinger/checkpoints/SOME/0119_continuous256_5spk/model_ckpt_steps_100000_simplified.ckpt\" --dataset {raw_folder_path} --overwrite\n",
        "\n",
        "elif data_type == \"ds (DiffSinger format)\":\n",
        "    ds_segment_script = \"/content/ghin_shenanigans/scripts/ds_segmentor.py\"\n",
        "    ds2csv_script = \"/content/MakeDiffSinger/variance-temp-solution/convert_ds.py\"\n",
        "    for raw_folder_name in os.listdir(all_shits_not_wav_n_lab):\n",
        "        raw_folder_path = os.path.join(all_shits_not_wav_n_lab, raw_folder_name)\n",
        "        if os.path.isdir(raw_folder_path):\n",
        "            ds_exp_path = os.path.join(raw_folder_path, \"ds\")\n",
        "            csv_exp_path = os.path.join(raw_folder_path, \"transcriptions.csv\")\n",
        "            !python {ds_segment_script} {raw_folder_path} --export_path {ds_exp_path}\n",
        "            !rm -rf {raw_folder_path}/*.ds #clean it cus why not\n",
        "            !python {ds2csv_script} ds2csv {ds_exp_path} {csv_exp_path}\n",
        "else:\n",
        "    pass\n",
        "\n",
        "# make it replace the first SP to AP cus it seems like people always forgot about it\n",
        "for root, _, files in os.walk(all_shits_not_wav_n_lab):\n",
        "    for file in files:\n",
        "        if file.endswith(\".csv\"):\n",
        "            file_path = os.path.join(root, file)\n",
        "            with open(file_path, \"r\", newline=\"\") as input_file:\n",
        "                csv_reader = csv.reader(input_file)\n",
        "                data = [row for row in csv_reader]\n",
        "                header = data[0]\n",
        "                if \"ph_seq\" in header:\n",
        "                    ph_seq_index = header.index(\"ph_seq\")\n",
        "                    if len(data) > 1 and len(data[1]) > ph_seq_index:\n",
        "                        data[1][ph_seq_index] = data[1][ph_seq_index].replace(\"SP\", \"AP\", 1)\n",
        "            with open(file_path, \"w\", newline=\"\") as output_file:\n",
        "                csv_writer = csv.writer(output_file)\n",
        "                csv_writer.writerows(data)\n",
        "\n",
        "print(\"extraction complete!\")\n",
        "print(\"|\")\n",
        "print(\"|\")\n",
        "print(\"|\")\n",
        "print(\"I'm also nice enough to convert your data and also write your dict.txt lmao. You are welcome :)\")"
      ],
      "metadata": {
        "cellView": "form",
        "id": "JsP1TGg2F1g3"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#@title #Edit Config\n",
        "#@markdown ___\n",
        "\n",
        "import re\n",
        "import os\n",
        "import yaml\n",
        "import random #for the random test files lmaoz\n",
        "\n",
        "%cd /content\n",
        "clear_output()\n",
        "#@markdown The model type user is training\n",
        "model_type = \"acoustic\" # @param [\"acoustic\", \"variance\"]\n",
        "config_cap = model_type.upper()\n",
        "diffusion_type = \"reflow\" # @param [\"ddpm\", \"reflow\"]\n",
        "diff_accelerator = \"ddim\" # @param [\"ddim\", \"pndm\", \"dpm-solver\", \"unipc\"]\n",
        "loss_type = \"l2\" # @param [\"l1\", \"l2\"]\n",
        "\n",
        "spk_name = [folder_name for folder_name in os.listdir(all_shits_not_wav_n_lab) if os.path.isdir(os.path.join(all_shits_not_wav_n_lab, folder_name))]\n",
        "# i used spk_name for something else cus i forgor now imma just copy and paste it\n",
        "spk_names = [folder_name for folder_name in os.listdir(all_shits_not_wav_n_lab) if os.path.isdir(os.path.join(all_shits_not_wav_n_lab, folder_name))]\n",
        "num_spk = len(spk_name)\n",
        "raw_dir = []\n",
        "for folder_name in spk_name:\n",
        "    folder_path = os.path.join(all_shits_not_wav_n_lab, folder_name)\n",
        "    raw_dir.append(folder_path)\n",
        "if num_spk == 1:\n",
        "    singer_type = \"SINGLE-SPEAKER\"\n",
        "    use_spk_id = False\n",
        "    all_wav_files = []\n",
        "    for root, dirs, files in os.walk(\"/content/raw_data/diffsinger_db\"):\n",
        "        for file in files:\n",
        "            if file.endswith((\".wav\", \".ds\")):\n",
        "                full_path = os.path.join(root, file)\n",
        "                all_wav_files.append(full_path)\n",
        "    random.shuffle(all_wav_files)\n",
        "    random_ass_wavs = all_wav_files[:3]\n",
        "    random_ass_test_files = [os.path.splitext(os.path.basename(file))[0] for file in random_ass_wavs]\n",
        "\n",
        "else:\n",
        "    singer_type = \"MULTI-SPEAKER\"\n",
        "    use_spk_id = True\n",
        "    folder_to_id = {folder_name: i for i, folder_name in enumerate(spk_name)}\n",
        "    random_ass_test_files = []\n",
        "    for folder_path in raw_dir:\n",
        "        if data_type == \"ds (DiffSinger format)\":\n",
        "            audio_files = [f[:-4] for f in os.listdir(folder_path) if f.endswith(\".ds\")]\n",
        "        else:\n",
        "            audio_files = [f[:-4] for f in os.listdir(folder_path + \"/wavs\") if f.endswith(\".wav\")]\n",
        "        folder_name = os.path.basename(folder_path)\n",
        "        folder_id = folder_to_id.get(folder_name, -1)\n",
        "        prefixed_audio_files = [f\"{folder_id}:{audio_file}\" for audio_file in audio_files]\n",
        "        random_ass_test_files.extend(prefixed_audio_files[:3])\n",
        "spk_id = []\n",
        "for i, spk_name in enumerate(spk_name):\n",
        "    spk_id_format = f\"{i}:{spk_name}\"\n",
        "    spk_id.append(spk_id_format)\n",
        "\n",
        "#@markdown Shallow Diffusion training\n",
        "use_shallow_diffusion = \"true | gt_val\" # @param [\"false\", \"true | aux_val\", \"true | gt_val\"]\n",
        "if use_shallow_diffusion == \"false\":\n",
        "    shallow = False\n",
        "    gt_shallow = False\n",
        "elif use_shallow_diffusion == \"true | aux_val\":\n",
        "    shallow = True\n",
        "    gt_shallow = False\n",
        "else:\n",
        "    shallow = True\n",
        "    gt_shallow = True\n",
        "\n",
        "#@markdown Half precision, or mixed precision can result in improved performance, achieving speedups on training (from [doc](https://lightning.ai/docs/pytorch/stable/common/trainer.html#precision))\n",
        "# the reason why i dont add 64 is because colab is already dreadfully slow at 32 so yes im leaving it out\n",
        "precision = \"16-mixed\" # @param [\"32-true\", \"bf16-mixed\", \"16-mixed\", \"bf16\", \"16\"]\n",
        "\n",
        "#@markdown User model save path\n",
        "save_dir = \"\" #@param {type:\"string\"}\n",
        "\n",
        "binary_save_dir = save_dir + \"/binary\"\n",
        "\n",
        "conf_dir = save_dir\n",
        "\n",
        "#@markdown ....................................................................................................................................................................................................................................................................................................................................................................................................................................\n",
        "\n",
        "#@markdown Option to use base model for finetuning\n",
        "\n",
        "enable_finetuning = False # @param {type:\"boolean\"}\n",
        "\n",
        "\n",
        "#@markdown Path to custom base model, leave blank to use [default](https://github.com/haru0l/diffsinger_models) models\n",
        "#wtf haru i just looked at your readme\"\"\"\"\"\n",
        "\n",
        "base_model_path = \"\" # @param {type:\"string\"}\n",
        "\n",
        "if enable_finetuning:\n",
        "    pretrain = True\n",
        "    if base_model_path:\n",
        "        pretrain_ckpt = base_model_path\n",
        "    else:\n",
        "        pretrain_ckpt = f\"/content/pretrain_models/{model_type}_pretrain.ckpt\"\n",
        "    finetune_strict_shapes = False\n",
        "    finetune_ckpt_path = pretrain_ckpt\n",
        "else:\n",
        "    pretrain = False\n",
        "    finetune_strict_shapes = True #default value\n",
        "    finetune_ckpt_path = None #default value\n",
        "\n",
        "#@markdown ....................................................................................................................................................................................................................................................................................................................................................................................................................................\n",
        "\n",
        "#@markdown Model embeds check; Tension, Energy, Breathiness, Voicing | for both <font color = \"yellow\">acoustic and variance\n",
        "\n",
        "#@markdown <font size=\"-1.5\"> we limited the pair up choice to prevent the quality and usage issue, if user wish to enable option(s) outside of these choices then please keep in mind that most of these embeds do not work well together except for [energy + breathiness]\n",
        "\n",
        "selected_param = \"tension\" # @param [\"energy + breathiness\", \"tension\", \"voicing\", \"none\"]\n",
        "if selected_param ==  \"energy + breathiness\":\n",
        "    tension_training = False\n",
        "    energy_training = True\n",
        "    breathiness_training = True\n",
        "    voicing_training = True\n",
        "elif selected_param == \"tension\":\n",
        "    tension_training = True\n",
        "    energy_training = False\n",
        "    breathiness_training = False\n",
        "    voicing_training = False\n",
        "elif selected_param == \"voicing\":\n",
        "    tension_training = False\n",
        "    energy_training = False\n",
        "    breathiness_training = False\n",
        "    voicing_training = True\n",
        "else:\n",
        "    tension_training = False\n",
        "    energy_training = False\n",
        "    breathiness_training = False\n",
        "    voicing_training = False\n",
        "\n",
        "parameter_extraction_method = \"vr\" # @param [\"vr\", \"world\"]\n",
        "\n",
        "### forcing data aug to be true by default cus i dont think anyone would disable it and its good to be on by default\n",
        "data_aug = True #param {type:\"boolean\"}\n",
        "\n",
        "#@markdown ....................................................................................................................................................................................................................................................................................................................................................................................................................................\n",
        "\n",
        "#@markdown Model training check | for <font color = \"yellow\">variance </font>only\n",
        "\n",
        "\n",
        "\n",
        "#@markdown <font size=\"-1.5\"> due to skill issues, if user wish to train with glide embed, please enable it manually in the config\n",
        "pitch_training = \"False\" # @param [\"False\", \"True | Standard\", \"True | MelodyEncoder\"]\n",
        "if pitch_training == \"False\":\n",
        "    pitch_training = False\n",
        "    use_melody_encoder = False\n",
        "    use_glide_embed = False\n",
        "elif pitch_training == \"True | Standard\":\n",
        "    pitch_training = True\n",
        "    use_melody_encoder = False\n",
        "    use_glide_embed = False\n",
        "else:\n",
        "    pitch_training = True\n",
        "    use_melody_encoder = True\n",
        "    use_glide_embed = False\n",
        "\n",
        "duration_training = True #@param {type: \"boolean\"}\n",
        "\n",
        "#@markdown ....................................................................................................................................................................................................................................................................................................................................................................................................................................\n",
        "\n",
        "#@markdown Pitch extractor algorithm\n",
        "\n",
        "f0_ext = \"parselmouth\" # @param [\"parselmouth\", \"rmvpe\", \"harvest\"]\n",
        "if f0_ext == \"rmvpe\":\n",
        "    pe_ckpt_pth = \"checkpoints/rmvpe/model.pt\"\n",
        "else:\n",
        "    pe_ckpt_pth = None\n",
        "\n",
        "#@markdown ....................................................................................................................................................................................................................................................................................................................................................................................................................................\n",
        "\n",
        "#@markdown Proceeding sections are the parameters that will greatly affect the model's final quality and size. Read about them [here](https://github.com/openvpi/DiffSinger/blob/main/docs/ConfigurationSchemas.md)\n",
        "\n",
        "#@markdown So if you don't know what they do then please <font color = \"red\">leave these options at default </font>, otherwise it could affect your model badly\n",
        "\n",
        "#@markdown <font size=\"-2.5\">anyone is welcome to experiment though\n",
        "\n",
        "#@markdown <font size=\"-1.5\">model_hidden_size: hidden layers for FS2 and token param embeds\n",
        "\n",
        "#@markdown <font size=\"-1.5\">model_residual_layers | model_residual_channels: the model's main layers and channels\n",
        "\n",
        "#@markdown ....................................................................................................................................................................................................................................................................................................................................................................................................................................\n",
        "#@markdown Model's network/layer size for <font color = \"yellow\">acoustic\n",
        "\n",
        "#@markdown <font size=\"-1.5\"> The quality of samplig_algorithm is in order, range from euler being the LEAST accurate to rk5 being the MOST accurate.... Though euler works fine on most cases\n",
        "sampling_algorithm = \"euler\" # @param [\"euler\", \"rk2\", \"rk4\", \"rk5\"]\n",
        "\n",
        "acoustic_hidden_size = 256 # @param {type:\"slider\", min:2, max:1024, step:2}\n",
        "\n",
        "acoustic_num_layers = 6 # @param {type:\"slider\", min:2, max:42, step:2}\n",
        "acoustic_num_channels = 1024 # @param {type:\"slider\", min:2, max:2048, step:2}\n",
        "\n",
        "#@markdown Model's network/layer size for <font color = \"yellow\">variance\n",
        "variance_hidden_size = 256 # @param {type:\"slider\", min:2, max:1024, step:2}\n",
        "duration_hidden_size = 512 # @param {type:\"slider\", min:2, max:1024, step:2}\n",
        "melody_encoder_hidden_size = 128 # @param {type:\"slider\", min:2, max:1024, step:2}\n",
        "\n",
        "pitch_num_layers = 6 # @param {type:\"slider\", min:2, max:100, step:2}\n",
        "pitch_num_channels = 512 # @param {type:\"slider\", min:2, max:1024, step:2}\n",
        "variance_num_layers = 6 # @param {type:\"slider\", min:2, max:100, step:2}\n",
        "variance_num_channels = 384 # @param {type:\"slider\", min:2, max:1024, step:2}\n",
        "\n",
        "\n",
        "\n",
        "with open(\"/content/DiffSinger/configs/base.yaml\", \"r\") as config:\n",
        "    mother = yaml.safe_load(config)\n",
        "mother[\"pl_trainer_precision\"] = precision\n",
        "with open(\"/content/DiffSinger/configs/base.yaml\", \"w\") as config:\n",
        "    yaml.dump(mother, config)\n",
        "\n",
        "if  data_type == \"ds (DiffSinger format)\":\n",
        "    prefer_ds = True\n",
        "else:\n",
        "    prefer_ds = False\n",
        "\n",
        "if model_type == \"acoustic\":\n",
        "    with open(\"/content/DiffSinger/configs/acoustic.yaml\", \"r\") as config:\n",
        "        bitch_ass_config = yaml.safe_load(config)\n",
        "    bitch_ass_config[\"speakers\"] = spk_names\n",
        "    bitch_ass_config[\"test_prefixes\"] = random_ass_test_files\n",
        "    bitch_ass_config[\"raw_data_dir\"] = raw_dir\n",
        "    bitch_ass_config[\"num_spk\"] = num_spk\n",
        "    bitch_ass_config[\"use_spk_id\"] = use_spk_id\n",
        "    #bitch_ass_config[\"spk_ids\"] = spk_id\n",
        "    bitch_ass_config[\"diffusion_type\"] = diffusion_type\n",
        "    bitch_ass_config[\"diff_accelerator\"] = diff_accelerator\n",
        "    bitch_ass_config[\"main_loss_type\"] = loss_type\n",
        "    bitch_ass_config[\"binary_data_dir\"] = binary_save_dir\n",
        "    bitch_ass_config[\"dictionary\"] = \"dictionaries/custom_dict.txt\"\n",
        "    bitch_ass_config[\"augmentation_args\"][\"random_pitch_shifting\"][\"enabled\"] = data_aug\n",
        "    bitch_ass_config[\"augmentation_args\"][\"random_time_stretching\"][\"enabled\"] = data_aug\n",
        "    bitch_ass_config[\"use_key_shift_embed\"] = data_aug\n",
        "    bitch_ass_config[\"use_speed_embed\"] = data_aug\n",
        "    bitch_ass_config[\"pe\"] = f0_ext\n",
        "    bitch_ass_config[\"use_energy_embed\"] = energy_training\n",
        "    bitch_ass_config[\"use_breathiness_embed\"] = breathiness_training\n",
        "    bitch_ass_config[\"use_tension_embed\"] = tension_training\n",
        "    bitch_ass_config[\"use_voicing_embed\"] = voicing_training\n",
        "\n",
        "    bitch_ass_config[\"pe_ckpt\"] = pe_ckpt_pth\n",
        "    #shallow diff stuff\n",
        "    bitch_ass_config[\"use_shallow_diffusion\"] = shallow\n",
        "    bitch_ass_config[\"shallow_diffusion_args\"][\"val_gt_start\"] = gt_shallow\n",
        "    #finetue stuff\n",
        "    bitch_ass_config[\"finetune_enabled\"] = enable_finetuning\n",
        "    bitch_ass_config[\"finetune_ckpt_path\"] = finetune_ckpt_path\n",
        "    bitch_ass_config[\"finetune_strict_shapes\"] = finetune_strict_shapes\n",
        "    #vr\n",
        "    bitch_ass_config[\"hnsep\"] = parameter_extraction_method\n",
        "    #layers\n",
        "    bitch_ass_config[\"sampling_algorithm\"] = sampling_algorithm\n",
        "    bitch_ass_config[\"hidden_size\"] = acoustic_hidden_size\n",
        "    bitch_ass_config[\"backbone_type\"] = \"lynxnet\"\n",
        "    bitch_ass_config[\"backbone_args\"][\"num_layers\"] = acoustic_num_layers\n",
        "    bitch_ass_config[\"backbone_args\"][\"num_channels\"] = acoustic_num_channels\n",
        "\n",
        "    with open(\"/content/DiffSinger/configs/acoustic.yaml\", \"w\") as config:\n",
        "        yaml.dump(bitch_ass_config, config)\n",
        "else:\n",
        "    with open(\"/content/DiffSinger/configs/variance.yaml\", \"r\") as config:\n",
        "        bitch_ass_config = yaml.safe_load(config)\n",
        "    bitch_ass_config[\"speakers\"] = spk_names\n",
        "    bitch_ass_config[\"test_prefixes\"] = random_ass_test_files\n",
        "    bitch_ass_config[\"raw_data_dir\"] = raw_dir\n",
        "    bitch_ass_config[\"num_spk\"] = num_spk\n",
        "    bitch_ass_config[\"use_spk_id\"] = use_spk_id\n",
        "    bitch_ass_config[\"main_loss_type\"] = loss_type\n",
        "    bitch_ass_config[\"diffusion_type\"] = diffusion_type\n",
        "    bitch_ass_config[\"diff_accelerator\"] = diff_accelerator\n",
        "    bitch_ass_config[\"binary_data_dir\"] = binary_save_dir\n",
        "    bitch_ass_config[\"dictionary\"] = \"dictionaries/custom_dict.txt\"\n",
        "    bitch_ass_config[\"pe\"] = f0_ext # i think variance uses it for pitch ref as ground-truth for pitch training soooo\n",
        "    bitch_ass_config[\"pe_ckpt\"] = pe_ckpt_pth #same goes to this one\n",
        "\n",
        "    bitch_ass_config[\"predict_energy\"] = energy_training\n",
        "    bitch_ass_config[\"predict_breathiness\"] = breathiness_training\n",
        "    bitch_ass_config[\"predict_tension\"] = tension_training\n",
        "    bitch_ass_config[\"predict_pitch\"] = pitch_training\n",
        "    bitch_ass_config[\"predict_voicing\"] = voicing_training\n",
        "\n",
        "    bitch_ass_config[\"use_melody_encoder\"] = use_melody_encoder\n",
        "    bitch_ass_config[\"use_glide_embed\"] = use_glide_embed\n",
        "    bitch_ass_config[\"predict_dur\"] = duration_training\n",
        "    bitch_ass_config[\"binarization_args\"][\"prefer_ds\"] = prefer_ds\n",
        "    #finetune stuff\n",
        "    bitch_ass_config[\"finetune_enabled\"] = enable_finetuning\n",
        "    bitch_ass_config[\"finetune_ckpt_path\"] = finetune_ckpt_path\n",
        "    bitch_ass_config[\"finetune_strict_shapes\"] = finetune_strict_shapes\n",
        "    #vr\n",
        "    bitch_ass_config[\"hnsep\"] = parameter_extraction_method\n",
        "    bitch_ass_config[\"hnsep_ckpt\"] = \"checkpoints/vr/model.pt\"\n",
        "    #layers\n",
        "    bitch_ass_config[\"hidden_size\"] = variance_hidden_size\n",
        "    bitch_ass_config[\"dur_prediction_args\"][\"hidden_size\"] = duration_hidden_size\n",
        "    bitch_ass_config[\"melody_encoder_args\"][\"hidden_size\"] = melody_encoder_hidden_size\n",
        "    bitch_ass_config[\"variances_prediction_args\"][\"backbone_type\"] = \"lynxnet\"\n",
        "    bitch_ass_config[\"variances_prediction_args\"][\"backbone_args\"][\"num_layers\"] = variance_num_layers\n",
        "    bitch_ass_config[\"variances_prediction_args\"][\"backbone_args\"][\"num_channels\"] = variance_num_channels\n",
        "    bitch_ass_config[\"pitch_prediction_args\"][\"backbone_type\"] = \"lynxnet\"\n",
        "    bitch_ass_config[\"pitch_prediction_args\"][\"backbone_args\"][\"num_layers\"] = pitch_num_layers\n",
        "    bitch_ass_config[\"pitch_prediction_args\"][\"backbone_args\"][\"num_channels\"] = pitch_num_channels\n",
        "\n",
        "    with open(\"/content/DiffSinger/configs/variance.yaml\", \"w\") as config:\n",
        "        yaml.dump(bitch_ass_config, config)\n",
        "\n",
        "os.makedirs(save_dir, exist_ok=True)\n",
        "\n",
        "\n",
        "with open(\"/content/DiffSinger/utils/hparams.py\", \"r\") as f:\n",
        "    hparams_py_read = f.read()\n",
        "hparams_py_read = re.sub(r\"args_work_dir\\s*=\\s*.*\", f\"args_work_dir = '{save_dir}'\", hparams_py_read)\n",
        "with open(\"/content/DiffSinger/utils/hparams.py\", \"w\") as f:\n",
        "    f.write(hparams_py_read)\n",
        "\n",
        "with open(\"/content/DiffSinger/utils/training_utils.py\", \"r\") as f:\n",
        "    training_utils_stuff = f.read()\n",
        "training_utils_stuff = re.sub(\"relative_path\\s*=\\s*.*\", \"relative_path = filepath.relative_to(Path('/content').resolve())\", training_utils_stuff)\n",
        "with open(\"/content/DiffSinger/utils/training_utils.py\", \"w\") as f:\n",
        "    f.write(training_utils_stuff)\n",
        "\n",
        "print(\"config updated! see below for config's information\")\n",
        "print(\"|\")\n",
        "print(\"|\")\n",
        "print(\"|\")\n",
        "print(f\"+++---{config_cap} {singer_type} TRAINING---+++\")\n",
        "print(\"|\")\n",
        "print(\"|\")\n",
        "print(\"|\")\n",
        "print(\"+++---user's settings---+++\")\n",
        "print(\"\\n\")\n",
        "print(f\"speaker name: {spk_names}\")\n",
        "print(\"\\n\")\n",
        "print(f\"data augmentation: {data_aug}\")\n",
        "print(\"\\n\")\n",
        "print(f\"pitch extractor: {f0_ext}\")\n",
        "print(\"\\n\")\n",
        "print(f\"binary data save directory: {binary_save_dir}\")\n",
        "print(\"\\n\")\n",
        "print(f\"your model will be saved to: {save_dir}\")\n",
        "print(\"\\n\")\n",
        "print(\"==========================================================================================\")\n",
        "print(\"\\n\")\n",
        "print(\"+++---other auto-defined settings---+++\")\n",
        "print(\"\\n\")\n",
        "print(f\"test files (auto selected): {random_ass_test_files}\")\n",
        "print(\"\\n\")\n",
        "print(\"dictionary (auto generated): custom_dict.txt\")\n",
        "print(\"\\n\")\n",
        "print(\"==========================================================================================\")\n",
        "print(\"\\n\")\n",
        "print(\"if you don't like or disagree with any of these options,\")\n",
        "print(f\"you can go and edit the config at [/content/DiffSinger/configs/{model_type}.yaml]\")\n"
      ],
      "metadata": {
        "cellView": "form",
        "id": "nI3dzDv_Mr9Y"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#@markdown # Preprocess data\n",
        "import os\n",
        "#we dont need that old f0 limit change anymore <3\n",
        "training_config = f\"/content/DiffSinger/configs/{model_type}.yaml\"\n",
        "%cd /content/DiffSinger\n",
        "os.environ['PYTHONPATH']='.'\n",
        "!CUDA_VISIBLE_DEVICES=0 python /content/DiffSinger/scripts/binarize.py --config {training_config} --reset"
      ],
      "metadata": {
        "cellView": "form",
        "id": "76NvDR1cXlDM"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# **Training**"
      ],
      "metadata": {
        "id": "0J3b18EKdzMC"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#@markdown #Train your model\n",
        "%cd /content/DiffSinger\n",
        "import re\n",
        "import os\n",
        "import yaml\n",
        "#@markdown ___\n",
        "\n",
        "#@markdown <font size=\"-1.5\"> Step interval of when your model will be validate and save\n",
        "save_interval = 2000 #@param {type:\"slider\", min:100, max:10000, step:100}\n",
        "\n",
        "#@markdown <font size=\"-1.5\"> batch size setting, too low can cause bottleneck, too high can cause oom\n",
        "batch_size = 9 # @param {type:\"slider\", min:1, max:100, step:1}\n",
        "\n",
        "#@markdown <font size=\"-1.5\"> step interval of when your model will stop training automatically\n",
        "max_updates = 160000 # @param {type:\"slider\", min:100, max:2000000, step:100}\n",
        "\n",
        "#@markdown ___\n",
        "\n",
        "#@markdown ###**Only edit this section if you want to resume training**\n",
        "resume_training = False #@param {type:\"boolean\"}\n",
        "\n",
        "#@markdown <font size=\"-1.5\"> select this option if you locally binarized your data | this option will only append your binary data path in your config | \"binary\" folder must be in the same directory as config.yaml\n",
        "local_data = False #@param {type:\"boolean\"}\n",
        "\n",
        "#@markdown <font size=\"-1.5\"> path to the config you got from training\n",
        "re_config_path = \"\" #@param {type:\"string\"}\n",
        "model_dir = os.path.dirname(re_config_path)\n",
        "save_dir = model_dir\n",
        "if resume_training:\n",
        "    with open(\"/content/DiffSinger/utils/hparams.py\", \"r\") as f:\n",
        "        hparams_py_read = f.read()\n",
        "    hparams_py_read = re.sub(r\"args_work_dir\\s*=\\s*.*\", f\"args_work_dir = '{save_dir}'\", hparams_py_read)\n",
        "    with open(\"/content/DiffSinger/utils/hparams.py\", \"w\") as f:\n",
        "        f.write(hparams_py_read)\n",
        "    with open(\"/content/DiffSinger/utils/training_utils.py\", \"r\") as f:\n",
        "        training_utils_stuff = f.read()\n",
        "    training_utils_stuff = re.sub(\"relative_path\\s*=\\s*.*\", \"relative_path = filepath.relative_to(Path('/content').resolve())\", training_utils_stuff)\n",
        "    with open(\"/content/DiffSinger/utils/training_utils.py\", \"w\") as f:\n",
        "        f.write(training_utils_stuff)\n",
        "\n",
        "    config_path = re_config_path\n",
        "    log_dir = save_dir\n",
        "\n",
        "    !cp {model_dir}/dictionary.txt /content/DiffSinger/dictionaries/custom_dict.txt\n",
        "\n",
        "else:\n",
        "    config_path = training_config\n",
        "    log_dir = conf_dir\n",
        "\n",
        "with open(config_path, \"r\") as config:\n",
        "    ehe = yaml.safe_load(config)\n",
        "config_dir = os.path.dirname(config_path)\n",
        "yuh = os.path.join(config_dir, \"binary\")\n",
        "\n",
        "ehe[\"val_check_interval\"] = save_interval\n",
        "ehe[\"max_batch_size\"] = batch_size\n",
        "ehe[\"max_updates\"] = max_updates\n",
        "if local_data:\n",
        "    ehe[\"binary_data_dir\"] = yuh\n",
        "with open(config_path, \"w\") as config:\n",
        "    yaml.dump(ehe, config)\n",
        "\n",
        "logs = log_dir\n",
        "%reload_ext tensorboard\n",
        "%tensorboard --logdir {logs}/lightning_logs\n",
        "\n",
        "!python /content/DiffSinger/scripts/train.py --config {config_path} --exp_name ${save_dir} --reset"
      ],
      "metadata": {
        "cellView": "form",
        "id": "Lu5w72UWgccC"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# **Convert model to ONNX format**"
      ],
      "metadata": {
        "id": "FY40fGHEg9_i"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#@markdown # Drop Speakers from Model (Optional)\n",
        "#@markdown ___\n",
        "#@markdown <font size=\"-1.5\"> Use this to drop speakers from your model for distribution. You will need to do it for both acoustic and variance models.\n",
        "\n",
        "drop_model_path = '' #@param {type: \"string\"}\n",
        "#@markdown <font size=\"-1.5\"> Type the ID of speakers you'd like to KEEP separated by commas. Ex: \"0,3,4\" <br>\n",
        "#@markdown <font size=\"-1.5\"> Note: You can find the ID of speakers in the model by opening the ```spk_map.json``` file in the model folder.<br>\n",
        "#@markdown <font size=\"-1.5\"> If you see ```{\"natural\": 0, \"power\": 1, \"silly\": 2}``` but only want to keep \"natural\" and \"power\", type ```0,1``` below.\n",
        "retain_speakers = '' #@param {type: \"string\"}\n",
        "#@markdown <font size=\"-1.5\"> If you don't know what this means, don't change it.\n",
        "fill_embed = 'zeros' #@param ['zeros', 'random', 'mean', 'cyclic']\n",
        "\n",
        "drop_out_path = drop_model_path[:-5] + '_spk-dropped.ckpt'\n",
        "\n",
        "!python /content/DiffSinger/scripts/drop_spk.py {drop_model_path} {drop_out_path} --retain {retain_speakers} --fill {fill_embed}\n",
        "\n",
        "\n"
      ],
      "metadata": {
        "id": "21ILzW4OEnh4",
        "cellView": "form"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#@markdown # Export ONNX\n",
        "#@markdown ___\n",
        "%cd /content\n",
        "from IPython.display import clear_output\n",
        "clear_output()\n",
        "import os\n",
        "import zipfile\n",
        "import shutil\n",
        "\n",
        "#move this here so users doesnt have to wait for this to install for training\n",
        "print(\"installing dependencies for onnx conversion...\")\n",
        "#I HATE colab\n",
        "!pip install condacolab\n",
        "import condacolab\n",
        "condacolab.install()\n",
        "env_name = \"ONNX_exporter\"\n",
        "env_check = f\"conda env list | grep '^{env_name} '\"\n",
        "\n",
        "if os.system(env_check) != 0:\n",
        "    !conda create -n ONNX_exporter python=3.10 -y --quiet\n",
        "    !conda run -n ONNX_exporter pip install -r /content/DiffSinger/requirements-onnx.txt -q -q -q 2>/dev/null\n",
        "else:\n",
        "    print(f\"Environment '{env_name}' already exists. Skipping installation.\")\n",
        "\n",
        "# to counter IF the user is to re-run this cell <3\n",
        "if os.path.exists(\"/content/OU_compatible_files\"):\n",
        "    shutil.rmtree(\"/content/OU_compatible_files\")\n",
        "    os.remove(\"/content/jpn_dict.txt\")\n",
        "else:\n",
        "    pass\n",
        "\n",
        "#@markdown <font size=\"-1.5\"> select this if you don't want to see the onnx converter's output\n",
        "no_output = True # @param {type:\"boolean\"}\n",
        "\n",
        "#@markdown <font size=\"-1.5\"> path to your **ACOUSTIC CHECKPOINT** (leave blank if you don't have any): automatically use latest checkpoint that is in the same folder\n",
        "acoustic_checkpoint_path = \"\" #@param{type:\"string'}\n",
        "acoustic_folder_name = os.path.basename(os.path.dirname(acoustic_checkpoint_path)) + \"_acoustic\"\n",
        "acoustic_folder_path = os.path.dirname(acoustic_checkpoint_path)\n",
        "\n",
        "#@markdown <font size=\"-1.5\"> path to your **VARIANCE CHECKPOINT** (leave blank if you don't have any): automatically use latest checkpoint that is in the same folder\n",
        "variance_checkpoint_path = \"\" #@param{type:\"string'}\n",
        "variance_folder_name = os.path.basename(os.path.dirname(variance_checkpoint_path)) + \"_variance\"\n",
        "variance_folder_path = os.path.dirname(variance_checkpoint_path)\n",
        "\n",
        "#@markdown <font size=\"-1.5\"> path to where you want to save your ONNX files (it will create a folder named \"onnx\" in this path)\n",
        "exp_folder = \"\" #@param{type:\"string\"}\n",
        "\n",
        "acoustic_onnx_exp = exp_folder + \"/onnx/acoustic\"\n",
        "variance_onnx_exp = exp_folder + \"/onnx/variance\"\n",
        "\n",
        "if not acoustic_checkpoint_path:\n",
        "    print(\"\\n\")\n",
        "    print(\"acoustic ckeckpoint path not specified, not exporting acoustic ONNX...\")\n",
        "else:\n",
        "    print(\"\\n\")\n",
        "    print(\"converting acoustic to onnx...\")\n",
        "    #cp stuff cus apparently exporter doesnt work without it\n",
        "    !cp {acoustic_folder_path}/config.yaml -r /content/DiffSinger/checkpoints/{acoustic_folder_name}\n",
        "    search_text = \"        args_work_dir = os.path.join(\"\n",
        "    replacement = f\"        args_work_dir = '{acoustic_folder_path}'\"\n",
        "    with open(\"/content/DiffSinger/utils/hparams.py\", \"r\") as file:\n",
        "        lines = file.readlines()\n",
        "    for i, line in enumerate(lines):\n",
        "        if search_text in line:\n",
        "            lines[i] = replacement + \"\\n\"\n",
        "            break\n",
        "    with open(\"/content/DiffSinger/utils/hparams.py\", \"w\") as file:\n",
        "            file.writelines(lines)\n",
        "    #incase if anyone wanna change it lmao\n",
        "    search_text_alt = \"        args_work_dir = '\"\n",
        "    replacement_alt = f\"        args_work_dir = '{acoustic_folder_path}'\"\n",
        "    with open(\"/content/DiffSinger/utils/hparams.py\", \"r\") as file:\n",
        "        lines = file.readlines()\n",
        "    for i, line in enumerate(lines):\n",
        "        if search_text_alt in line:\n",
        "            lines[i] = replacement_alt + \"\\n\"\n",
        "            break\n",
        "    with open(\"/content/DiffSinger/utils/hparams.py\", \"w\") as file:\n",
        "            file.writelines(lines)\n",
        "\n",
        "    if no_output:\n",
        "        !conda run -n ONNX_exporter python /content/DiffSinger/scripts/export.py acoustic --exp {acoustic_folder_name} --out {exp_folder}/onnx/acoustic >/dev/null 2>&1\n",
        "    else:\n",
        "        !conda run -n ONNX_exporter python /content/DiffSinger/scripts/export.py acoustic --exp {acoustic_folder_name} --out {exp_folder}/onnx/acoustic\n",
        "\n",
        "\n",
        "if not variance_checkpoint_path:\n",
        "    print(\"\\n\")\n",
        "    print(\"variance ckeckpoint path not specified, not exporting variance ONNX...\")\n",
        "else:\n",
        "    print(\"\\n\")\n",
        "    print(\"converting variance to onnx...\")\n",
        "    #cp stuff cus apparently exporter doesnt work without it\n",
        "    !cp {variance_folder_path}/config.yaml -r /content/DiffSinger/checkpoints/{variance_folder_name}\n",
        "    search_text = \"        args_work_dir = os.path.join(\"\n",
        "    replacement = f\"        args_work_dir = '{variance_folder_path}'\"\n",
        "    with open(\"/content/DiffSinger/utils/hparams.py\", \"r\") as file:\n",
        "        lines = file.readlines()\n",
        "    for i, line in enumerate(lines):\n",
        "        if search_text in line:\n",
        "            lines[i] = replacement + \"\\n\"\n",
        "            break\n",
        "    with open(\"/content/DiffSinger/utils/hparams.py\", \"w\") as file:\n",
        "            file.writelines(lines)\n",
        "    #incase if anyone wanna change it lmao\n",
        "    search_text_alt = \"        args_work_dir = '\"\n",
        "    replacement_alt = f\"        args_work_dir = '{variance_folder_path}'\"\n",
        "    with open(\"/content/DiffSinger/utils/hparams.py\", \"r\") as file:\n",
        "        lines = file.readlines()\n",
        "    for i, line in enumerate(lines):\n",
        "        if search_text_alt in line:\n",
        "            lines[i] = replacement_alt + \"\\n\"\n",
        "            break\n",
        "    with open(\"/content/DiffSinger/utils/hparams.py\", \"w\") as file:\n",
        "            file.writelines(lines)\n",
        "    if no_output:\n",
        "        !conda run -n ONNX_exporter python /content/DiffSinger/scripts/export.py variance --exp {variance_folder_name} --out {exp_folder}/onnx/variance >/dev/null 2>&1\n",
        "    else:\n",
        "        !conda run -n ONNX_exporter python /content/DiffSinger/scripts/export.py variance --exp {variance_folder_name} --out {exp_folder}/onnx/variance\n",
        "\n",
        "\n",
        "if not variance_checkpoint_path:\n",
        "    folder_paths = [acoustic_onnx_exp]\n",
        "elif not acoustic_checkpoint_path:\n",
        "    folder_paths = [variance_onnx_exp]\n",
        "else:\n",
        "    folder_paths = [acoustic_onnx_exp, variance_onnx_exp]\n",
        "\n",
        "patterns = {\"acoustic.onnx\": \"acoustic.onnx\", \"dur.onnx\": \"dur.onnx\", \"linguistic.onnx\": \"linguistic.onnx\", \"pitch.onnx\": \"pitch.onnx\", \"variance.onnx\": \"variance.onnx\", \"phonemes.txt\": \"phonemes.txt\"}\n",
        "\n",
        "for folder_path in folder_paths:\n",
        "    for filename in os.listdir(folder_path):\n",
        "        for pattern, new_name in patterns.items():\n",
        "            if pattern in filename:\n",
        "                old_path = os.path.join(folder_path, filename)\n",
        "                new_path = os.path.join(folder_path, new_name)\n",
        "                if os.path.exists(old_path):\n",
        "                    os.rename(old_path, new_path)\n",
        "for folder_path in folder_paths:\n",
        "    for filename in os.listdir(folder_path):\n",
        "        if \"acoustic_acoustic.\" in filename:\n",
        "            new_filename = filename.replace(\"acoustic_acoustic.\", \"acoustic_\")\n",
        "        elif \"variance_variance.\" in filename:\n",
        "            new_filename = filename.replace(\"variance_variance.\", \"variance_\")\n",
        "        else:\n",
        "            new_filename = filename\n",
        "        old_path = os.path.join(folder_path, filename)\n",
        "        new_path = os.path.join(folder_path, new_filename)\n",
        "        os.rename(old_path, new_path)\n",
        "print(\"\\n\")\n",
        "print(\"ONNX export complete! Please refer to https://github.com/xunmengshe/OpenUtau/wiki/Voicebank-Development to make your model OU compatible\")\n",
        "print(\"\\n\")\n",
        "print(\"Or use the 'Build OpenUtau VB' cell to have things set up for you\")\n"
      ],
      "metadata": {
        "id": "x33iZhZchEMW",
        "cellView": "form"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# **Miscellaneous**"
      ],
      "metadata": {
        "id": "4sbU1aH5kGFE"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#@title #Raw data conversion\n",
        "#@markdown ___\n",
        "%cd /content\n",
        "#@markdown This cell will export .lab and .ds files along with your data\n",
        "\n",
        "data_type = \"lab + wav (NNSVS format)\" # @param [\"lab + wav (NNSVS format)\"]\n",
        "\n",
        "#@markdown <font size=\"-1.5\"> The path to your data zip file\n",
        "\n",
        "data_zip_path = \"\" #@param {type:\"string\"}\n",
        "\n",
        "#@markdown <font size=\"-1.5\"> The path you will be saving the data to\n",
        "\n",
        "data_save_path = \"\" #@param {type:\"string\"}\n",
        "\n",
        "#@markdown ___\n",
        "\n",
        "export_ds = True\n",
        "\n",
        "#@markdown <font size=\"-1.5\"> _These values can exceed the amount that's in your data to maximize the segment length or to keep the data as is_\n",
        "\n",
        "#@markdown <font size=\"-1.5\"> Determine how long it will segment your data to based on silence phoneme placement (seconds)\n",
        "segment_length = 15 #@param {type:\"slider\", min:5, max:35, step:1}\n",
        "\n",
        "#@markdown <font size=\"-1.5\"> Determine how many silence phoneme is allowed in the middle of each segment\n",
        "max_silence_phoneme_amount = 2 #@param {type:\"slider\", min:0, max:50, step:1}\n",
        "\n",
        "# leaving -S at 60 so max silence can be 60 seconds that exceeds the segment legnth cap idk why///\n",
        "# making the segment length cap at 35 secs because any longer than that would make training goes really slow\n",
        "\n",
        "# my ass dont remember why i made two... i think one is unnecessary extra but mehhh\n",
        "all_shits = \"/content/raw_data\"\n",
        "all_shits_not_wav_n_lab = \"/content/raw_data/diffsinger_db\"\n",
        "\n",
        "import os\n",
        "import csv\n",
        "import json\n",
        "import shutil\n",
        "from pydub import AudioSegment\n",
        "\n",
        "if os.path.exists(\"/content/raw_data\"):\n",
        "    shutil.rmtree(\"/content/raw_data\")\n",
        "\n",
        "if not os.path.exists(all_shits_not_wav_n_lab):\n",
        "  os.makedirs(all_shits_not_wav_n_lab)\n",
        "\n",
        "# using 'if not' bc i edited the wrong section which im also too lazy to fix it <3\n",
        "if not data_type == \"lab + wav (NNSVS format)\":\n",
        "    #changed to 7zip to support more compression types\n",
        "    !7z x \"$data_zip_path\" -o{all_shits_not_wav_n_lab}\n",
        "    for root, dirs, files in os.walk(all_shits):\n",
        "        for filename in files:\n",
        "            if filename.endswith(\".lab\"):\n",
        "                file_path = os.path.join(root, filename)\n",
        "                with open(file_path, \"r\") as file:\n",
        "                    file_data = file.read()\n",
        "                file_data = file_data.replace(\"SP\", \"pau\")\n",
        "                file_data = file_data.replace(\"br\", \"AP\")\n",
        "                with open(file_path, \"w\") as file:\n",
        "                    file.write(file_data)\n",
        "\n",
        "else:\n",
        "    !7z x \"$data_zip_path\" -o{all_shits_not_wav_n_lab}\n",
        "\n",
        "\n",
        "# for funny auto dict generator lmao\n",
        "out = \"/content/raw_data/custom_dict.txt\"\n",
        "\n",
        "phonemes = set()\n",
        "\n",
        "def is_excluded(phoneme):\n",
        "    return phoneme in [\"pau\", \"AP\", \"SP\"]\n",
        "\n",
        "if data_type == \"lab + wav (NNSVS format)\":\n",
        "    phoneme_folder_path = all_shits\n",
        "    for root, dirs, files in os.walk(phoneme_folder_path):\n",
        "        for file in files:\n",
        "            if file.endswith(\".lab\"):\n",
        "                fpath = os.path.join(root, file)\n",
        "                with open(fpath, \"r\") as lab_file:\n",
        "                    for line in lab_file:\n",
        "                        line = line.strip()\n",
        "                        if line:\n",
        "                            phoneme = line.split()[2]\n",
        "                            if not is_excluded(phoneme):\n",
        "                                phonemes.add(phoneme)\n",
        "\n",
        "with open(out, \"w\") as f:\n",
        "    for phoneme in sorted(phonemes):\n",
        "        f.write(phoneme + \"\t\" + phoneme + \"\\n\")\n",
        "\n",
        "# for vowels and consonants.txt.... well adding liquid type for uta's script\n",
        "dict_path = out\n",
        "vowel_types = {\"a\", \"i\", \"u\", \"e\", \"o\", \"N\", \"M\", \"NG\"}\n",
        "liquid_types = {\"y\", \"w\", \"l\", \"r\"} # r for english labels, it should be fine with jp too\n",
        "vowel_data = []\n",
        "consonant_data = []\n",
        "liquid_data = []\n",
        "\n",
        "with open(dict_path, \"r\") as f:\n",
        "    for line in f:\n",
        "        phoneme, _ = line.strip().split(\"\\t\")\n",
        "        if phoneme[0] in vowel_types:\n",
        "            vowel_data.append(phoneme)\n",
        "        elif phoneme[0] in liquid_types:\n",
        "            liquid_data.append(phoneme)\n",
        "        else:\n",
        "            consonant_data.append(phoneme)\n",
        "\n",
        "vowel_data.sort()\n",
        "liquid_data.sort()\n",
        "consonant_data.sort()\n",
        "directory = os.path.dirname(dict_path)\n",
        "\n",
        "# make txt for language json file\n",
        "vowel_txt_path = os.path.join(directory, \"vowels.txt\")\n",
        "with open(vowel_txt_path, \"w\") as f:\n",
        "    f.write(\" \".join(vowel_data))\n",
        "liquid_txt_path = os.path.join(directory, \"liquids.txt\")\n",
        "with open(liquid_txt_path, \"w\") as f:\n",
        "    f.write(\" \".join(liquid_data))\n",
        "consonant_txt_path = os.path.join(directory, \"consonants.txt\")\n",
        "with open(consonant_txt_path, \"w\") as f:\n",
        "    f.write(\" \".join(consonant_data))\n",
        "\n",
        "\n",
        "# here's a funny json append\n",
        "with open(vowel_txt_path, \"r\") as f:\n",
        "    vowel_data = f.read().split()\n",
        "with open(liquid_txt_path, \"r\") as f:\n",
        "    liquid_data = f.read().split()\n",
        "with open(consonant_txt_path, \"r\") as f:\n",
        "    consonant_data = f.read().split()\n",
        "phones4json = {\"vowels\": vowel_data, \"liquids\": liquid_data}\n",
        "with open(\"/content/nnsvs-db-converter/lang.sample.json\", \"w\") as rawr:\n",
        "    json.dump(phones4json, rawr, indent=4)\n",
        "\n",
        "\n",
        "if data_type == \"lab + wav (NNSVS format)\":\n",
        "    db_converter_script = \"/content/nnsvs-db-converter/db_converter.py\"\n",
        "    for raw_folder_name in os.listdir(all_shits_not_wav_n_lab):\n",
        "        raw_folder_path = os.path.join(all_shits_not_wav_n_lab, raw_folder_name)\n",
        "        if os.path.isdir(raw_folder_path):\n",
        "            !python {db_converter_script} -s {max_silence_phoneme_amount} -S 60 -l {segment_length} ${export_lab} -mD -c -L \"/content/nnsvs-db-converter/lang.sample.json\" -w htk --folder {raw_folder_path}\n",
        "\n",
        "if data_type == \"lab + wav (NNSVS format)\":\n",
        "    for raw_folder_name in os.listdir(all_shits_not_wav_n_lab):\n",
        "        raw_folder_path = os.path.join(all_shits_not_wav_n_lab, raw_folder_name)\n",
        "        !rm -rf {raw_folder_path}/*.wav {raw_folder_path}/*.lab\n",
        "        !mv {raw_folder_path}/diffsinger_db/* {raw_folder_path} 2> /dev/null\n",
        "        !rm -rf {raw_folder_path}/diffsinger_db\n",
        "        #!cp {raw_folder_path}/wavs/*.wav {raw_folder_path}\n",
        "\n",
        "# make it replace the first SP to AP cus it seems like people always forgot about it\n",
        "for root, _, files in os.walk(all_shits_not_wav_n_lab):\n",
        "    for file in files:\n",
        "        if file.endswith(\".csv\"):\n",
        "            file_path = os.path.join(root, file)\n",
        "            with open(file_path, \"r\", newline=\"\") as input_file:\n",
        "                csv_reader = csv.reader(input_file)\n",
        "                data = [row for row in csv_reader]\n",
        "                header = data[0]\n",
        "                if \"ph_seq\" in header:\n",
        "                    ph_seq_index = header.index(\"ph_seq\")\n",
        "                    if len(data) > 1 and len(data[1]) > ph_seq_index:\n",
        "                        data[1][ph_seq_index] = data[1][ph_seq_index].replace(\"SP\", \"AP\", 1)\n",
        "            with open(file_path, \"w\", newline=\"\") as output_file:\n",
        "                csv_writer = csv.writer(output_file)\n",
        "                csv_writer.writerows(data)\n",
        "\n",
        "print(\"extraction complete!\")\n",
        "print(\"\\n\")\n",
        "print(\"zipping up files...\")\n",
        "!zip -q -9 -r {data_save_path}/data.zip /content/raw_data/*"
      ],
      "metadata": {
        "cellView": "form",
        "id": "AI7EQ2jQkGEq"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#@markdown # Build OpenUtau VB\n",
        "#@markdown ___\n",
        "#i need to clean this up it seems\n",
        "#plan: add a build ou section here by inserting onnx paths (or just the folder containing the folders to the onnx files) to build ou\n",
        "# ill have a config read function too so i dont have to add checkmark of if people train with embeds or shallow diff or not <3\n",
        "# yes im lazy rawr x3\n",
        "%cd /content\n",
        "import os\n",
        "import shutil\n",
        "import yaml\n",
        "from IPython.display import clear_output\n",
        "\n",
        "constr_folder = \"/content/OU_voicebank\"\n",
        "if not os.path.exists(constr_folder):\n",
        "    os.makedirs(constr_folder)\n",
        "else:\n",
        "    shutil.rmtree(constr_folder)\n",
        "    os.makedirs(constr_folder)\n",
        "\n",
        "clear_output()\n",
        "\n",
        "#@markdown <font size=\"-1.5\"> path to your **ACOUSTIC ONNX FOLDER**\n",
        "acoustic_onnx_folder = \"\" #@param{type:\"string'}\n",
        "#@markdown <font size=\"-1.5\"> path to the config.yaml of acoustic model\n",
        "acoustic_config = \"\" #@param{type:\"string'}\n",
        "\n",
        "#@markdown <font size=\"-1.5\"> path to your **VARIANCE ONNX FOLDER**\n",
        "variance_onnx_folder = \"\" #@param{type:\"string'}\n",
        "#@markdown <font size=\"-1.5\"> path to the config.yaml of variance model\n",
        "variance_config = \"\" #@param{type:\"string'}\n",
        "\n",
        "#@markdown <font size=\"-1.5\"> path to your word to phoneme dict (leave blank to use default Japanese dict)\n",
        "dictionary_path = \"\" #@param{type:\"string\"}\n",
        "\n",
        "#@markdown <font size=\"-1.5\"> path to the folder you want to save the zip file to\n",
        "save_path = \"\" #@param{type:\"string\"}\n",
        "\n",
        "#@markdown ___\n",
        "\n",
        "#@markdown ## Character Configuration | character.txt and character.yaml\n",
        "\n",
        "#@markdown <font size=\"-1.5\"> your character display name| **required**\n",
        "name = \"\" #@param{type:\"string\"}\n",
        "\n",
        "print(\"copying files...\")\n",
        "main_stuff = f\"{constr_folder}/{name}\"\n",
        "if not os.path.exists(main_stuff):\n",
        "    os.makedirs(main_stuff)\n",
        "if not os.path.exists(f\"{main_stuff}/dsmain\"):\n",
        "    os.makedirs(f\"{main_stuff}/dsmain/embeds/acoustic\")\n",
        "    os.makedirs(f\"{main_stuff}/dsmain/embeds/variance\")\n",
        "!cp {acoustic_onnx_folder}/acoustic.onnx {main_stuff}/dsmain\n",
        "!cp {acoustic_onnx_folder}/phonemes.txt {main_stuff}/dsmain\n",
        "!cp {acoustic_onnx_folder}/*.emb {main_stuff}/dsmain/embeds/acoustic >/dev/null 2>&1\n",
        "!cp {variance_onnx_folder}/*.emb {main_stuff}/dsmain/embeds/variance >/dev/null 2>&1\n",
        "\n",
        "if variance_onnx_folder:\n",
        "    !cp {variance_onnx_folder}/linguistic.onnx {main_stuff}/dsmain\n",
        "else:\n",
        "    pass\n",
        "\n",
        "print(\"\\n\")\n",
        "print(\"writing character.txt...\")\n",
        "with open(f\"{main_stuff}/character.txt\", \"w\") as file:\n",
        "    file.write(f\"name={name}\\n\")\n",
        "    file.write(\"image=\\n\")\n",
        "    file.write(\"author=\\n\")\n",
        "    file.write(\"voice=\\n\")\n",
        "    file.write(\"web=\\n\")\n",
        "\n",
        "print(\"\\n\")\n",
        "print(\"writing character.yaml...\")\n",
        "with open(f\"{main_stuff}/character.yaml\", \"w\") as file:\n",
        "    file.write(\"text_file_encoding: utf-8\\n\")\n",
        "    file.write(\"portrait:\\n\")\n",
        "    file.write(\"portrait_opacity: 0.45\\n\")\n",
        "    file.write(\"default_phonemizer: OpenUtau.Core.DiffSinger.DiffSingerPhonemizer\\n\")\n",
        "    file.write(\"singer_type: diffsinger\\n\")\n",
        "acoustic_emb_files = os.listdir(acoustic_onnx_folder)\n",
        "acoustic_embeds = []\n",
        "acoustic_color_suffix = []\n",
        "for file in acoustic_emb_files:\n",
        "    if file.endswith(\".emb\"):\n",
        "        acoustic_emb = os.path.splitext(file)[0]\n",
        "        acoustic_embeds.append(\"dsmain/embeds/acoustic/\" + acoustic_emb)\n",
        "        acoustic_color_suffix.append(acoustic_emb)\n",
        "subbanks = []\n",
        "for i, (acoustic_embed_color, acoustic_embed_suffix) in enumerate(zip(acoustic_color_suffix, acoustic_embeds), start=1):\n",
        "    color = f\"{i:02}: {acoustic_embed_color}\"\n",
        "    suffix = f\"{acoustic_embed_suffix}\"\n",
        "    subbanks.append({\"color\": color, \"suffix\": suffix})\n",
        "if subbanks:\n",
        "    with open(f\"{main_stuff}/character.yaml\", \"r\") as config:\n",
        "        i_wanna_die_slash_j = yaml.safe_load(config)\n",
        "    i_wanna_die_slash_j[\"subbanks\"] = subbanks\n",
        "    with open(f\"{main_stuff}/character.yaml\", \"w\") as config:\n",
        "        yaml.dump(i_wanna_die_slash_j, config)\n",
        "\n",
        "print(\"\\n\")\n",
        "print(\"writing dsconfig.yaml for acoustic...\")\n",
        "with open(f\"{main_stuff}/dsconfig.yaml\", \"w\") as file:\n",
        "    file.write(\"phonemes: dsmain/phonemes.txt\\n\")\n",
        "    file.write(\"acoustic: dsmain/acoustic.onnx\\n\")\n",
        "    file.write(\"vocoder: nsf_hifigan\\n\")\n",
        "    file.write(\"singer_type: diffsinger\\n\")\n",
        "with open(acoustic_config, \"r\") as config:\n",
        "    mfking_config = yaml.safe_load(config)\n",
        "use_energy_embed = mfking_config.get(\"use_energy_embed\")\n",
        "use_breathiness_embed = mfking_config.get(\"use_breathiness_embed\")\n",
        "use_shallow_diffusion = mfking_config.get(\"use_shallow_diffusion\")\n",
        "max_depth = mfking_config.get(\"T_start\")\n",
        "speakers = mfking_config.get(\"speakers\") #looking back here, why is this even here lmao cus i used acoustic_embeds instead of speakers\n",
        "augmentation_arg = mfking_config.get(\"augmentation_args\")\n",
        "pitch_aug = mfking_config.get(\"use_key_shift_embed\")\n",
        "time_aug = mfking_config.get(\"use_speed_embed\")\n",
        "voicing = mfking_config.get(\"use_voicing_embed\")\n",
        "tension = mfking_config.get(\"use_tension_embed\")\n",
        "sample_rate = mfking_config.get(\"audio_sample_rate\")\n",
        "hop_size = mfking_config.get(\"hop_size\")\n",
        "win_size = mfking_config.get(\"win_size\")\n",
        "fft_size = mfking_config.get(\"fft_size\")\n",
        "num_mel_bins = mfking_config.get(\"audio_num_mel_bins\")\n",
        "mel_fmin = mfking_config.get(\"fmin\")\n",
        "mel_fmax = mfking_config.get(\"fmax\")\n",
        "mel_base = mfking_config.get(\"mel_base\")\n",
        "\n",
        "with open(f\"{main_stuff}/dsconfig.yaml\", \"r\") as config:\n",
        "    why_are_there_so_many_i_could_prob_make_it_one = yaml.safe_load(config)\n",
        "why_are_there_so_many_i_could_prob_make_it_one[\"use_energy_embed\"] = use_energy_embed\n",
        "why_are_there_so_many_i_could_prob_make_it_one[\"use_breathiness_embed\"] = use_breathiness_embed\n",
        "why_are_there_so_many_i_could_prob_make_it_one[\"use_variable_depth\"] = use_shallow_diffusion\n",
        "why_are_there_so_many_i_could_prob_make_it_one[\"max_depth\"] = max_depth\n",
        "why_are_there_so_many_i_could_prob_make_it_one[\"augmentation_args\"] = augmentation_arg\n",
        "why_are_there_so_many_i_could_prob_make_it_one[\"use_key_shift_embed\"] = pitch_aug\n",
        "why_are_there_so_many_i_could_prob_make_it_one[\"use_speed_embed\"] = time_aug\n",
        "why_are_there_so_many_i_could_prob_make_it_one[\"use_voicing_embed\"] = voicing\n",
        "why_are_there_so_many_i_could_prob_make_it_one[\"use_tension_embed\"] = tension\n",
        "why_are_there_so_many_i_could_prob_make_it_one[\"use_continuous_acceleration\"] = True\n",
        "why_are_there_so_many_i_could_prob_make_it_one[\"sample_rate\"] = sample_rate\n",
        "why_are_there_so_many_i_could_prob_make_it_one[\"hop_size\"] = hop_size\n",
        "why_are_there_so_many_i_could_prob_make_it_one[\"win_size\"] = win_size\n",
        "why_are_there_so_many_i_could_prob_make_it_one[\"fft_size\"] = fft_size\n",
        "why_are_there_so_many_i_could_prob_make_it_one[\"num_mel_bins\"] = num_mel_bins\n",
        "why_are_there_so_many_i_could_prob_make_it_one[\"fmin\"] = mel_fmin\n",
        "why_are_there_so_many_i_could_prob_make_it_one[\"fmax\"] = mel_fmax\n",
        "why_are_there_so_many_i_could_prob_make_it_one[\"mel_base\"] = mel_base\n",
        "why_are_there_so_many_i_could_prob_make_it_one[\"mel_scale\"] = \"slaney\"\n",
        "\n",
        "\n",
        "if subbanks:\n",
        "    why_are_there_so_many_i_could_prob_make_it_one[\"speakers\"] = acoustic_embeds\n",
        "with open(f\"{main_stuff}/dsconfig.yaml\", \"w\") as config:\n",
        "    yaml.dump(why_are_there_so_many_i_could_prob_make_it_one, config)\n",
        "\n",
        "\n",
        "variance_emb_files = os.listdir(variance_onnx_folder)\n",
        "variance_embeds = []\n",
        "for file in variance_emb_files:\n",
        "    if file.endswith(\".emb\"):\n",
        "        variance_emb = os.path.splitext(file)[0]\n",
        "        variance_embeds.append(\"../dsmain/embeds/variance/\" + variance_emb)\n",
        "\n",
        "print(\"\\n\")\n",
        "print(\"writing dsdict.yaml...\")\n",
        "if not dictionary_path:\n",
        "    dict_path = \"/content/jpn_dict.txt\"\n",
        "else:\n",
        "    dict_path = dictionary_path\n",
        "\n",
        "# for symbols list\n",
        "phoneme_dict_path = f\"{acoustic_onnx_folder}/dictionary.txt\"\n",
        "\n",
        "dsdict = \"dsdict.yaml\"\n",
        "\n",
        "def parse_phonemes(phonemes_str):\n",
        "    return phonemes_str.split()\n",
        "\n",
        "entries = []\n",
        "vowel_types = {\"a\", \"i\", \"u\", \"e\", \"o\", \"N\", \"M\", \"NG\", \"cl\", \"vf\"}\n",
        "vowel_data = []\n",
        "stop_data = []\n",
        "\n",
        "# Process the specified dictionary\n",
        "with open(dict_path, \"r\") as f:\n",
        "    for line in f:\n",
        "        word, phonemes_str = line.strip().split(\"\\t\")\n",
        "        phonemes = parse_phonemes(phonemes_str)\n",
        "        if len(phonemes) == 1:\n",
        "            entries.append({\"grapheme\": word, \"phonemes\": phonemes})\n",
        "        else:\n",
        "            entries.append({\"grapheme\": word, \"phonemes\": phonemes})\n",
        "\n",
        "with open(phoneme_dict_path, \"r\") as f:\n",
        "    for line in f:\n",
        "        phoneme, _ = line.strip().split(\"\\t\")\n",
        "        phoneme_type = \"vowel\" if phoneme[0] in vowel_types else \"stop\"\n",
        "        entry = {\"symbol\": phoneme, \"type\": phoneme_type}\n",
        "        if phoneme_type == \"vowel\":\n",
        "            vowel_data.append(entry)\n",
        "        else:\n",
        "            stop_data.append(entry)\n",
        "\n",
        "vowel_data.sort(key=lambda x: x[\"symbol\"])\n",
        "stop_data.sort(key=lambda x: x[\"symbol\"])\n",
        "\n",
        "dsdict_path = os.path.join(constr_folder, dsdict)\n",
        "with open(dsdict_path, \"w\") as f:\n",
        "    f.write(\"entries:\\n\")\n",
        "    for entry in entries:\n",
        "        f.write(f\"- grapheme: {entry['grapheme']}\\n\")\n",
        "        f.write(\"  phonemes:\\n\")\n",
        "        for phoneme in entry[\"phonemes\"]:\n",
        "            f.write(f\"  - {phoneme}\\n\")\n",
        "\n",
        "    f.write(\"\\nsymbols:\\n\")\n",
        "    for entry in vowel_data + stop_data:\n",
        "        f.write(f\"- symbol: {entry['symbol']}\\n\")\n",
        "        f.write(f\"  type: {entry['type']}\\n\")\n",
        "\n",
        "with open(variance_config, \"r\") as config:\n",
        "    mfking_config = yaml.safe_load(config)\n",
        "sample_rate = mfking_config.get(\"audio_sample_rate\")\n",
        "hop_size = mfking_config.get(\"hop_size\")\n",
        "predict_dur = mfking_config.get(\"predict_dur\")\n",
        "predict_pitch = mfking_config.get(\"predict_pitch\")\n",
        "use_melody_encoder = mfking_config.get(\"use_melody_encoder\")\n",
        "predict_voicing = mfking_config.get(\"predict_voicing\")\n",
        "predict_tension = mfking_config.get(\"predict_tension\")\n",
        "predict_energy = mfking_config.get(\"predict_energy\")\n",
        "predict_breathiness = mfking_config.get(\"predict_breathiness\")\n",
        "\n",
        "dur_onnx_path = variance_onnx_folder + \"/dur.onnx\"\n",
        "if os.path.exists(dur_onnx_path):\n",
        "    print(\"\\n\")\n",
        "    print(\"making dsdur directory and necessary files...\")\n",
        "    os.makedirs(f\"{main_stuff}/dsdur\")\n",
        "    !cp {dur_onnx_path} {main_stuff}/dsdur\n",
        "    !cp {dsdict_path} {main_stuff}/dsdur\n",
        "    with open(f\"{main_stuff}/dsdur/dsconfig.yaml\", \"w\") as file:\n",
        "        file.write(\"phonemes: ../dsmain/phonemes.txt\\n\")\n",
        "        file.write(\"linguistic: ../dsmain/linguistic.onnx\\n\")\n",
        "        file.write(\"dur: dur.onnx\\n\")\n",
        "    with open(f\"{main_stuff}/dsdur/dsconfig.yaml\", \"r\") as config:\n",
        "        dsdur_config = yaml.safe_load(config)\n",
        "    dsdur_config[\"use_continuous_acceleration\"] = True\n",
        "    dsdur_config[\"sample_rate\"] = sample_rate\n",
        "    dsdur_config[\"hop_size\"] = hop_size\n",
        "    dsdur_config[\"predict_dur\"] = predict_dur\n",
        "    if subbanks:\n",
        "        dsdur_config[\"speakers\"] = variance_embeds\n",
        "    with open(f\"{main_stuff}/dsdur/dsconfig.yaml\", \"w\") as config:\n",
        "        yaml.dump(dsdur_config, config)\n",
        "else:\n",
        "    print(\"\\n\")\n",
        "    print(\"dur.onnx not found, skipping on making dsdur folder...\")\n",
        "\n",
        "pitch_onnx_path = variance_onnx_folder + \"/pitch.onnx\"\n",
        "if os.path.exists(pitch_onnx_path):\n",
        "    print(\"\\n\")\n",
        "    print(\"making dspitch directory and necessary files...\")\n",
        "    os.makedirs(f\"{main_stuff}/dspitch\")\n",
        "    !cp {pitch_onnx_path} {main_stuff}/dspitch\n",
        "    !cp {dsdict_path} {main_stuff}/dspitch\n",
        "    with open(f\"{main_stuff}/dspitch/dsconfig.yaml\", \"w\") as file:\n",
        "        file.write(\"phonemes: ../dsmain/phonemes.txt\\n\")\n",
        "        file.write(\"linguistic: ../dsmain/linguistic.onnx\\n\")\n",
        "        file.write(\"pitch: pitch.onnx\\n\")\n",
        "        file.write(\"use_expr: true\\n\")\n",
        "    with open(f\"{main_stuff}/dspitch/dsconfig.yaml\", \"r\") as config:\n",
        "        dspitch_config = yaml.safe_load(config)\n",
        "    dspitch_config[\"use_continuous_acceleration\"] = True\n",
        "    dspitch_config[\"sample_rate\"] = sample_rate\n",
        "    dspitch_config[\"hop_size\"] = hop_size\n",
        "    dspitch_config[\"predict_dur\"] = predict_pitch\n",
        "    if subbanks:\n",
        "        dspitch_config[\"speakers\"] = variance_embeds\n",
        "    dspitch_config[\"use_note_rest\"] = use_melody_encoder\n",
        "    with open(f\"{main_stuff}/dspitch/dsconfig.yaml\", \"w\") as config:\n",
        "        yaml.dump(dspitch_config, config)\n",
        "else:\n",
        "    print(\"\\n\")\n",
        "    print(\"pitch.onnx not found, skipping on making dspitch folder...\")\n",
        "\n",
        "variance_onnx_path = variance_onnx_folder + \"/variance.onnx\"\n",
        "if os.path.exists(variance_onnx_path):\n",
        "    print(\"\\n\")\n",
        "    print(\"making dsvariance directory and necessary files...\")\n",
        "    os.makedirs(f\"{main_stuff}/dsvariance\")\n",
        "    !cp {variance_onnx_path} {main_stuff}/dsvariance\n",
        "    !cp {dsdict_path} {main_stuff}/dsvariance\n",
        "    with open(f\"{main_stuff}/dsvariance/dsconfig.yaml\", \"w\") as file:\n",
        "        file.write(\"phonemes: ../dsmain/phonemes.txt\\n\")\n",
        "        file.write(\"linguistic: ../dsmain/linguistic.onnx\\n\")\n",
        "        file.write(\"variance: variance.onnx\\n\")\n",
        "    with open(f\"{main_stuff}/dsvariance/dsconfig.yaml\", \"r\") as config:\n",
        "        dsvariance_config = yaml.safe_load(config)\n",
        "    dsvariance_config[\"use_continuous_acceleration\"] = True\n",
        "    dsvariance_config[\"sample_rate\"] = sample_rate\n",
        "    dsvariance_config[\"hop_size\"] = hop_size\n",
        "    dsvariance_config[\"predict_dur\"] = True #this one will always be true cus if there's no variance model, it shouldnt make this folder in the first place\n",
        "    dsvariance_config[\"predict_voicing\"] = predict_voicing\n",
        "    dsvariance_config[\"predict_tension\"] = predict_tension\n",
        "    dsvariance_config[\"predict_energy\"] = predict_energy\n",
        "    dsvariance_config[\"predict_breathiness\"] = predict_breathiness\n",
        "    if subbanks:\n",
        "        dsvariance_config[\"speakers\"] = variance_embeds\n",
        "    with open(f\"{main_stuff}/dsvariance/dsconfig.yaml\", \"w\") as config:\n",
        "        yaml.dump(dsvariance_config, config)\n",
        "else:\n",
        "    print(\"\\n\")\n",
        "    print(\"variance.onnx not found, skipping on making dsvariance folder...\")\n",
        "\n",
        "!rm -rf {dsdict_path}\n",
        "#im too lazy to write codes so ill just do this, itll only remove those folders if they're empty anyway\n",
        "!rm -d {main_stuff}/dsmain/embeds/* >/dev/null 2>&1\n",
        "!rm -d {main_stuff}/dsmain/embeds >/dev/null 2>&1\n",
        "\n",
        "print(\"\\n\")\n",
        "print(\"zipping up files...\")\n",
        "!zip -q -9 -r {save_path}/{name}.zip {main_stuff}/*\n",
        "\n",
        "print(\"\\n\")\n",
        "print(\"done!\")\n",
        "\n",
        "print(\"\\n\")\n",
        "print(\"You can download your model zip and use it in OpenUtau! If anything needed to be edit in the config then please do so\")"
      ],
      "metadata": {
        "cellView": "form",
        "id": "A70Sc3Hbmxh0"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}