tools/convert_to_onnx.py

#!/usr/bin/env python3
"""
Convert IndexTTS-2 PyTorch models to ONNX format for Rust inference!

This script converts the three main models:
1. GPT model (gpt.pth) - Autoregressive text-to-semantic generation
2. S2Mel model (s2mel.pth) - Semantic-to-mel spectrogram conversion
3. BigVGAN - Mel-to-waveform vocoder (already available as ONNX from NVIDIA)

Usage:
    python tools/convert_to_onnx.py

Output:
    models/gpt.onnx
    models/s2mel.onnx
    models/bigvgan.onnx (if needed, otherwise use NVIDIA's)

Why ONNX?
    - Cross-platform: Works on Windows, Linux, macOS, M1/M2 Macs
    - Fast: ONNX Runtime is highly optimized
    - Rust-native: ort crate provides excellent ONNX Runtime bindings
    - No Python: Production inference without Python dependency hell!

Author: Aye & Hue @ 8b.is
"""

import os
import sys

# Setup paths
script_dir = os.path.dirname(os.path.abspath(__file__))
project_root = os.path.dirname(script_dir)
os.chdir(project_root)

# Set HF cache
os.environ['HF_HUB_CACHE'] = './checkpoints/hf_cache'

print("=" * 70)
print("  IndexTTS-2 PyTorch to ONNX Converter")
print("  For Rust inference with ort crate!")
print("=" * 70)
print()

# Check for models
if not os.path.exists("checkpoints/gpt.pth"):
    print("ERROR: Models not found!")
    print("Run: python tools/download_files.py -s huggingface")
    sys.exit(1)

import torch
import torch.onnx
import numpy as np
from pathlib import Path

# Add reference code to path
sys.path.insert(0, "indextts - REMOVING - REF ONLY")

# Create output directory
output_dir = Path("models")
output_dir.mkdir(exist_ok=True)

print(f"PyTorch version: {torch.__version__}")
print(f"Output directory: {output_dir}")
print()


def export_speaker_encoder():
    """
    Export the CAM++ speaker encoder to ONNX.

    This model extracts speaker embeddings from reference audio.
    Input: mel spectrogram [batch, n_mels, time]
    Output: speaker embedding [batch, 192]
    """
    print("\n" + "=" * 50)
    print("Exporting Speaker Encoder (CAM++)")
    print("=" * 50)

    try:
        from omegaconf import OmegaConf
        from indextts.s2mel.modules.campplus.DTDNN import CAMPPlus

        # Load config
        cfg = OmegaConf.load("checkpoints/config.yaml")

        # Create model
        model = CAMPPlus(feat_dim=80, embedding_size=192)

        # Load weights
        weights_path = "./checkpoints/hf_cache/models--funasr--campplus/snapshots/fb71fe990cbf6031ae6987a2d76fe64f94377b7e/campplus_cn_common.bin"
        if os.path.exists(weights_path):
            state_dict = torch.load(weights_path, map_location='cpu')
            model.load_state_dict(state_dict)
            print(f"Loaded weights from: {weights_path}")

        model.eval()

        # CAMPPlus expects [batch, time, n_mels] NOT [batch, n_mels, time]!
        # This is the key insight - the model processes time-series of mel features
        dummy_input = torch.randn(1, 100, 80)  # [batch, time, features]

        # Verify forward pass works before export
        with torch.no_grad():
            test_output = model(dummy_input)
            print(f"Forward pass works! Output shape: {test_output.shape}")

        # Export to ONNX
        output_path = output_dir / "speaker_encoder.onnx"
        torch.onnx.export(
            model,
            dummy_input,
            str(output_path),
            input_names=['mel_spectrogram'],
            output_names=['speaker_embedding'],
            dynamic_axes={
                'mel_spectrogram': {0: 'batch', 1: 'time'},  # time is dim 1!
                'speaker_embedding': {0: 'batch'}
            },
            opset_version=18,  # Use 18+ for latest features
            do_constant_folding=True,
        )

        # Verify the export
        import onnx
        onnx_model = onnx.load(str(output_path))
        onnx.checker.check_model(onnx_model)

        print(f"✓ Exported: {output_path}")
        print(f"  Input: mel_spectrogram [batch, time, 80]")  # Corrected!
        print(f"  Output: speaker_embedding [batch, 192]")
        print(f"✓ ONNX model verified!")
        return True

    except Exception as e:
        print(f"✗ Failed to export speaker encoder: {e}")
        import traceback
        traceback.print_exc()
        return False


def export_gpt_model():
    """
    Export the GPT autoregressive model to ONNX.

    This is the most complex model - generates semantic tokens from text.
    We may need to export it in parts due to KV caching.

    Input: text_tokens [batch, seq_len], speaker_embedding [batch, 192]
    Output: semantic_codes [batch, code_len]
    """
    print("\n" + "=" * 50)
    print("Exporting GPT Model (Autoregressive)")
    print("=" * 50)

    try:
        from omegaconf import OmegaConf

        # Load the full model config
        cfg = OmegaConf.load("checkpoints/config.yaml")

        # This is tricky - GPT models with KV caching are hard to export
        # We might need to:
        # 1. Export just the forward pass without caching
        # 2. Or export separate encoder/decoder parts

        print("GPT model export is complex due to:")
        print("  - Autoregressive generation with KV caching")
        print("  - Dynamic sequence lengths")
        print("  - Multiple internal components")
        print()
        print("Options:")
        print("  A) Export without KV cache (slower but simpler)")
        print("  B) Export encoder + single-step decoder (efficient)")
        print("  C) Use torch.compile + ONNX tracing")
        print()

        # For now, let's try the simpler approach
        from infer_v2 import IndexTTS2

        # Load model
        tts = IndexTTS2(
            cfg_path="checkpoints/config.yaml",
            model_dir="checkpoints",
            use_fp16=False,
            device="cpu"
        )

        # Get the GPT component
        gpt = tts.gpt
        gpt.eval()

        print(f"GPT model loaded: {type(gpt)}")
        print(f"Parameters: {sum(p.numel() for p in gpt.parameters()):,}")

        # The GPT model architecture:
        # - Text encoder (embeddings + transformer)
        # - Speaker conditioning
        # - Autoregressive decoder

        # Let's export the text encoder first
        output_path = output_dir / "gpt_encoder.onnx"

        # Create dummy inputs
        text_tokens = torch.randint(0, 30000, (1, 32), dtype=torch.int64)

        # This will likely fail due to complex control flow
        # but let's try!
        print(f"Attempting GPT export (may require modifications)...")

        # For now, just report what we learned
        print()
        print("Note: Full GPT export requires modifying the model code")
        print("to remove dynamic control flow. Creating a wrapper...")

        return False

    except Exception as e:
        print(f"✗ Failed to export GPT: {e}")
        import traceback
        traceback.print_exc()
        return False


def export_s2mel_model():
    """
    Export the Semantic-to-Mel model (flow matching).

    This converts semantic codes to mel spectrograms.
    Input: semantic_codes [batch, code_len], speaker_embedding [batch, 192]
    Output: mel_spectrogram [batch, 80, mel_len]
    """
    print("\n" + "=" * 50)
    print("Exporting S2Mel Model (Flow Matching)")
    print("=" * 50)

    try:
        from omegaconf import OmegaConf

        cfg = OmegaConf.load("checkpoints/config.yaml")

        print("S2Mel model (Diffusion/Flow Matching) is also complex:")
        print("  - Multiple denoising steps (iterative)")
        print("  - CFM (Conditional Flow Matching) requires ODE solving")
        print()
        print("Export strategy:")
        print("  1. Export the single denoising step")
        print("  2. Run iteration loop in Rust")
        print()

        return False

    except Exception as e:
        print(f"✗ Failed to export S2Mel: {e}")
        import traceback
        traceback.print_exc()
        return False


def export_bigvgan():
    """
    Export BigVGAN vocoder to ONNX.

    Good news: NVIDIA provides pre-trained BigVGAN models!
    Even better: They're designed for easy ONNX export.

    Input: mel_spectrogram [batch, 80, mel_len]
    Output: waveform [batch, 1, wave_len]
    """
    print("\n" + "=" * 50)
    print("Exporting BigVGAN Vocoder")
    print("=" * 50)

    try:
        # BigVGAN from NVIDIA is easier to export
        # Let's check if we already have it

        print("BigVGAN options:")
        print("  1. Use NVIDIA's pre-exported ONNX (recommended)")
        print("     https://github.com/NVIDIA/BigVGAN")
        print()
        print("  2. Export from PyTorch weights (we'll do this)")
        print()

        # Try to load BigVGAN
        try:
            from bigvgan import bigvgan
            model = bigvgan.BigVGAN.from_pretrained(
                'nvidia/bigvgan_v2_22khz_80band_256x',
                use_cuda_kernel=False
            )
            model.eval()
            model.remove_weight_norm()  # Important for ONNX!

            print(f"BigVGAN loaded from HuggingFace")
            print(f"Parameters: {sum(p.numel() for p in model.parameters()):,}")

            # Create dummy input
            dummy_mel = torch.randn(1, 80, 100)

            # Export
            output_path = output_dir / "bigvgan.onnx"
            torch.onnx.export(
                model,
                dummy_mel,
                str(output_path),
                input_names=['mel_spectrogram'],
                output_names=['waveform'],
                dynamic_axes={
                    'mel_spectrogram': {0: 'batch', 2: 'mel_length'},
                    'waveform': {0: 'batch', 2: 'wave_length'}
                },
                opset_version=18,  # Use 18+ for latest features
                do_constant_folding=True,
            )

            print(f"✓ Exported: {output_path}")
            print(f"  Input: mel_spectrogram [batch, 80, mel_len]")
            print(f"  Output: waveform [batch, 1, wave_len]")

            # Verify the export
            import onnx
            onnx_model = onnx.load(str(output_path))
            onnx.checker.check_model(onnx_model)
            print(f"✓ ONNX model verified!")

            return True

        except ImportError:
            print("bigvgan package not installed, installing...")
            os.system("pip install bigvgan")
            print("Please re-run the script.")
            return False

    except Exception as e:
        print(f"✗ Failed to export BigVGAN: {e}")
        import traceback
        traceback.print_exc()
        return False


def main():
    print("\nStarting ONNX conversion...\n")

    results = {}

    # Export each component
    results['speaker_encoder'] = export_speaker_encoder()
    results['gpt'] = export_gpt_model()
    results['s2mel'] = export_s2mel_model()
    results['bigvgan'] = export_bigvgan()

    # Summary
    print("\n" + "=" * 70)
    print("  CONVERSION SUMMARY")
    print("=" * 70)

    for name, success in results.items():
        status = "✓ SUCCESS" if success else "✗ NEEDS WORK"
        print(f"  {name:20} {status}")

    print()

    if all(results.values()):
        print("All models converted! Ready for Rust inference.")
    else:
        print("Some models need manual intervention.")
        print()
        print("For complex models (GPT, S2Mel), consider:")
        print("  1. Modifying the Python code to remove dynamic control flow")
        print("  2. Using torch.jit.trace with concrete inputs")
        print("  3. Exporting subcomponents separately")
        print("  4. Using ONNX Runtime's transformer optimizations")

    print()
    print("Output directory:", output_dir.absolute())


if __name__ == "__main__":
    main()