mirror of
https://github.com/OpenBMB/VoxCPM.git
synced 2026-09-20 02:43:33 +08:00
Merge pull request #327 from DEVAIEXP/feat-add-seed
feat: Add Seed Support for Reproducible Voice Generation (v1 & v2) Resolved conflict in src/voxcpm/cli.py: kept both --seed (PR #327) and timestamp alignment args (PR #332).
This commit is contained in:
@@ -108,6 +108,7 @@ wav = model.generate(
|
||||
text="VoxCPM2 is the current recommended release for realistic multilingual speech synthesis.",
|
||||
cfg_value=2.0,
|
||||
inference_timesteps=10,
|
||||
seed=42,
|
||||
)
|
||||
sf.write("demo.wav", wav, model.tts_model.sample_rate)
|
||||
print("saved: demo.wav")
|
||||
@@ -131,6 +132,7 @@ wav = model.generate(
|
||||
text="VoxCPM2 is the current recommended release for realistic multilingual speech synthesis.",
|
||||
cfg_value=2.0,
|
||||
inference_timesteps=10,
|
||||
seed=42,
|
||||
)
|
||||
sf.write("demo.wav", wav, model.tts_model.sample_rate)
|
||||
```
|
||||
@@ -144,6 +146,7 @@ wav = model.generate(
|
||||
text="(A young woman, gentle and sweet voice)Hello, welcome to VoxCPM2!",
|
||||
cfg_value=2.0,
|
||||
inference_timesteps=10,
|
||||
seed=42,
|
||||
)
|
||||
sf.write("voice_design.wav", wav, model.tts_model.sample_rate)
|
||||
```
|
||||
@@ -164,6 +167,7 @@ wav = model.generate(
|
||||
reference_wav_path="path/to/voice.wav",
|
||||
cfg_value=2.0,
|
||||
inference_timesteps=10,
|
||||
seed=42,
|
||||
)
|
||||
sf.write("controllable_clone.wav", wav, model.tts_model.sample_rate)
|
||||
```
|
||||
@@ -210,6 +214,7 @@ voxcpm design \
|
||||
voxcpm design \
|
||||
--text "VoxCPM2 brings studio-quality multilingual speech synthesis." \
|
||||
--control "Young female voice, warm and gentle, slightly smiling" \
|
||||
--seed 42 \
|
||||
--output out.wav
|
||||
|
||||
# Voice cloning (reference audio)
|
||||
|
||||
@@ -10,7 +10,7 @@
|
||||
<a href="https://voxcpm.readthedocs.io/zh-cn/latest/"><img src="https://img.shields.io/badge/Docs-ReadTheDocs-8CA1AF" alt="Documentation"></a>
|
||||
<a href="https://huggingface.co/openbmb/VoxCPM2"><img src="https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-VoxCPM2-yellow" alt="Hugging Face"></a>
|
||||
<a href="https://modelscope.cn/models/OpenBMB/VoxCPM2"><img src="https://img.shields.io/badge/ModelScope-VoxCPM2-purple" alt="ModelScope"></a>
|
||||
<a href="https://openbmb.github.io/voxcpm2-demopage/"><img src="https://img.shields.io/badge/DemoPage-Audio Samples-red"></a>
|
||||
<a href="https://openbmb.github.io/voxcpm2-demopage/"><img src="https://img.shields.io/badge/DemoPage-Audio Samples-red" alt="DemoPage"></a>
|
||||
|
||||
</p>
|
||||
|
||||
@@ -110,6 +110,7 @@ wav = model.generate(
|
||||
text="VoxCPM2 是目前推荐使用的多语言语音合成版本。",
|
||||
cfg_value=2.0,
|
||||
inference_timesteps=10,
|
||||
seed=42,
|
||||
)
|
||||
sf.write("demo.wav", wav, model.tts_model.sample_rate)
|
||||
print("已保存: demo.wav")
|
||||
@@ -133,6 +134,7 @@ wav = model.generate(
|
||||
text="VoxCPM2 是目前推荐使用的多语言语音合成版本。",
|
||||
cfg_value=2.0,
|
||||
inference_timesteps=10,
|
||||
seed=42,
|
||||
)
|
||||
sf.write("demo.wav", wav, model.tts_model.sample_rate)
|
||||
```
|
||||
@@ -146,6 +148,7 @@ wav = model.generate(
|
||||
text="(年轻女性,声音温柔甜美)你好,欢迎使用VoxCPM2!",
|
||||
cfg_value=2.0,
|
||||
inference_timesteps=10,
|
||||
seed=42,
|
||||
)
|
||||
sf.write("voice_design.wav", wav, model.tts_model.sample_rate)
|
||||
```
|
||||
@@ -166,6 +169,7 @@ wav = model.generate(
|
||||
reference_wav_path="path/to/voice.wav",
|
||||
cfg_value=2.0,
|
||||
inference_timesteps=10,
|
||||
seed=42,
|
||||
)
|
||||
sf.write("controllable_clone.wav", wav, model.tts_model.sample_rate)
|
||||
```
|
||||
@@ -212,6 +216,7 @@ voxcpm design \
|
||||
voxcpm design \
|
||||
--text "VoxCPM2带来全新语音合成体验。" \
|
||||
--control "年轻女声,温暖温柔,略带微笑" \
|
||||
--seed 42 \
|
||||
--output out.wav
|
||||
|
||||
# 声音克隆(参考音频)
|
||||
|
||||
81
app.py
81
app.py
@@ -2,6 +2,7 @@ import os
|
||||
import re
|
||||
import sys
|
||||
import logging
|
||||
import random
|
||||
import numpy as np
|
||||
import gradio as gr
|
||||
from typing import Optional, Tuple
|
||||
@@ -44,8 +45,8 @@ _EXAMPLES_FOOTER_EN = (
|
||||
"**Example 1 — Gentle & Melancholic Girl** \n"
|
||||
'`Control Instruction`: *"A young girl with a soft, sweet voice. '
|
||||
'Speaks slowly with a melancholic, slightly tsundere tone."* \n'
|
||||
'`Target Text`: *"I never asked you to stay… It\'s not like I care or anything. '
|
||||
'But… why does it still hurt so much now that you\'re gone?"* \n\n'
|
||||
"`Target Text`: *\"I never asked you to stay… It's not like I care or anything. "
|
||||
"But… why does it still hurt so much now that you're gone?\"* \n\n"
|
||||
"**Example 2 — Laid-Back Surfer Dude** \n"
|
||||
'`Control Instruction`: *"Relaxed young male voice, slightly nasal, '
|
||||
'lazy drawl, very casual and chill."* \n'
|
||||
@@ -115,6 +116,10 @@ _I18N_TRANSLATIONS = {
|
||||
"cfg_info": "Higher → closer to the prompt / reference; lower → more creative variation",
|
||||
"dit_steps_label": "LocDiT flow-matching steps",
|
||||
"dit_steps_info": "LocDiT flow-matching steps — more steps → maybe better audio quality, but slower",
|
||||
"seed_label": "Seed",
|
||||
"seed_info": "Seed used for reproducible generation. Updated with the actual successful seed after generation.",
|
||||
"random_seed_label": "Random Seed",
|
||||
"random_seed_info": "Generate a new seed before each inference run.",
|
||||
"usage_instructions": _USAGE_INSTRUCTIONS_EN,
|
||||
"examples_footer": _EXAMPLES_FOOTER_EN,
|
||||
},
|
||||
@@ -155,8 +160,7 @@ for _d in _I18N_TRANSLATIONS.values():
|
||||
I18N = gr.I18n(**_I18N_TRANSLATIONS)
|
||||
|
||||
DEFAULT_TARGET_TEXT = (
|
||||
"VoxCPM2 is a creative multilingual TTS model from ModelBest, "
|
||||
"designed to generate highly realistic speech."
|
||||
"VoxCPM2 is a creative multilingual TTS model from ModelBest, " "designed to generate highly realistic speech."
|
||||
)
|
||||
|
||||
_CUSTOM_CSS = """
|
||||
@@ -219,6 +223,7 @@ _APP_THEME = gr.themes.Soft(
|
||||
|
||||
# ---------- Model ----------
|
||||
|
||||
|
||||
class VoxCPMDemo:
|
||||
def __init__(self, model_id: str = "openbmb/VoxCPM2", device: str = "auto") -> None:
|
||||
self.device = resolve_runtime_device(device, "cuda")
|
||||
@@ -247,9 +252,7 @@ class VoxCPMDemo:
|
||||
def get_or_load_asr_model(self) -> AutoModel:
|
||||
if self.asr_model is not None:
|
||||
return self.asr_model
|
||||
logger.info(
|
||||
f"Loading ASR model: {self.asr_model_id} on device: {self.asr_device}"
|
||||
)
|
||||
logger.info(f"Loading ASR model: {self.asr_model_id} on device: {self.asr_device}")
|
||||
self.asr_model = AutoModel(
|
||||
model=self.asr_model_id,
|
||||
disable_update=True,
|
||||
@@ -279,6 +282,7 @@ class VoxCPMDemo:
|
||||
do_normalize: bool,
|
||||
denoise: bool,
|
||||
inference_timesteps: int = 10,
|
||||
seed: Optional[int] = None,
|
||||
) -> dict:
|
||||
generate_kwargs = dict(
|
||||
text=final_text,
|
||||
@@ -287,6 +291,7 @@ class VoxCPMDemo:
|
||||
inference_timesteps=inference_timesteps,
|
||||
normalize=do_normalize,
|
||||
denoise=denoise,
|
||||
seed=seed,
|
||||
)
|
||||
if prompt_text_clean and audio_path:
|
||||
generate_kwargs["prompt_wav_path"] = audio_path
|
||||
@@ -303,7 +308,8 @@ class VoxCPMDemo:
|
||||
do_normalize: bool = True,
|
||||
denoise: bool = True,
|
||||
inference_timesteps: int = 10,
|
||||
) -> Tuple[int, np.ndarray]:
|
||||
seed: Optional[int] = None,
|
||||
) -> Tuple[int, np.ndarray, Optional[int]]:
|
||||
current_model = self.get_or_load_voxcpm()
|
||||
|
||||
text = (text_input or "").strip()
|
||||
@@ -335,16 +341,32 @@ class VoxCPMDemo:
|
||||
do_normalize=do_normalize,
|
||||
denoise=denoise,
|
||||
inference_timesteps=inference_timesteps,
|
||||
seed=seed,
|
||||
)
|
||||
wav = current_model.generate(**generate_kwargs)
|
||||
return (current_model.tts_model.sample_rate, wav)
|
||||
last_successful_seed = getattr(current_model.tts_model, "last_successful_seed", seed)
|
||||
return (current_model.tts_model.sample_rate, wav, last_successful_seed)
|
||||
|
||||
|
||||
# ---------- UI ----------
|
||||
|
||||
|
||||
def create_demo_interface(demo: VoxCPMDemo):
|
||||
gr.set_static_paths(paths=[Path.cwd().absolute() / "assets"])
|
||||
|
||||
def _coerce_seed(seed_value) -> Optional[int]:
|
||||
if seed_value is None or seed_value == "":
|
||||
return None
|
||||
return int(seed_value)
|
||||
|
||||
def _prepare_seed(use_random_seed: bool, seed_value):
|
||||
if use_random_seed:
|
||||
return random.randint(0, 2**32 - 1)
|
||||
return _coerce_seed(seed_value)
|
||||
|
||||
def _on_random_seed_toggle(checked):
|
||||
return gr.update(interactive=not checked)
|
||||
|
||||
def _generate(
|
||||
text: str,
|
||||
control_instruction: str,
|
||||
@@ -355,10 +377,12 @@ def create_demo_interface(demo: VoxCPMDemo):
|
||||
do_normalize: bool,
|
||||
denoise: bool,
|
||||
dit_steps: int,
|
||||
seed_value,
|
||||
):
|
||||
actual_prompt_text = prompt_text_value.strip() if use_prompt_text else ""
|
||||
actual_control = "" if use_prompt_text else control_instruction
|
||||
sr, wav_np = demo.generate_tts_audio(
|
||||
seed = _coerce_seed(seed_value)
|
||||
sr, wav_np, last_successful_seed = demo.generate_tts_audio(
|
||||
text_input=text,
|
||||
control_instruction=actual_control,
|
||||
reference_wav_path_input=ref_wav,
|
||||
@@ -367,8 +391,9 @@ def create_demo_interface(demo: VoxCPMDemo):
|
||||
do_normalize=do_normalize,
|
||||
denoise=denoise,
|
||||
inference_timesteps=int(dit_steps),
|
||||
seed=seed,
|
||||
)
|
||||
return (sr, wav_np)
|
||||
return (sr, wav_np), last_successful_seed
|
||||
|
||||
def _on_toggle_instant(checked):
|
||||
"""Instant UI toggle — no ASR, no blocking."""
|
||||
@@ -465,6 +490,20 @@ def create_demo_interface(demo: VoxCPMDemo):
|
||||
label=I18N("dit_steps_label"),
|
||||
info=I18N("dit_steps_info"),
|
||||
)
|
||||
with gr.Row():
|
||||
seed_value = gr.Number(
|
||||
value=random.randint(0, 2**32 - 1),
|
||||
precision=0,
|
||||
label=I18N("seed_label"),
|
||||
info=I18N("seed_info"),
|
||||
interactive=False,
|
||||
)
|
||||
random_seed = gr.Checkbox(
|
||||
value=True,
|
||||
label=I18N("random_seed_label"),
|
||||
elem_classes=["switch-toggle"],
|
||||
info=I18N("random_seed_info"),
|
||||
)
|
||||
|
||||
run_btn = gr.Button(I18N("generate_btn"), variant="primary", size="lg")
|
||||
|
||||
@@ -482,7 +521,18 @@ def create_demo_interface(demo: VoxCPMDemo):
|
||||
outputs=[prompt_text],
|
||||
)
|
||||
|
||||
random_seed.change(
|
||||
fn=_on_random_seed_toggle,
|
||||
inputs=[random_seed],
|
||||
outputs=[seed_value],
|
||||
)
|
||||
|
||||
run_btn.click(
|
||||
fn=_prepare_seed,
|
||||
inputs=[random_seed, seed_value],
|
||||
outputs=[seed_value],
|
||||
show_progress=False,
|
||||
).then(
|
||||
fn=_generate,
|
||||
inputs=[
|
||||
text,
|
||||
@@ -494,14 +544,16 @@ def create_demo_interface(demo: VoxCPMDemo):
|
||||
DoNormalizeText,
|
||||
DoDenoisePromptAudio,
|
||||
dit_steps,
|
||||
seed_value,
|
||||
],
|
||||
outputs=[audio_output],
|
||||
outputs=[audio_output, seed_value],
|
||||
show_progress=True,
|
||||
api_name="generate",
|
||||
)
|
||||
|
||||
return interface
|
||||
|
||||
|
||||
def run_demo(
|
||||
server_name: str = "0.0.0.0",
|
||||
server_port: int = 8808,
|
||||
@@ -523,9 +575,12 @@ def run_demo(
|
||||
|
||||
if __name__ == "__main__":
|
||||
import argparse
|
||||
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument(
|
||||
"--model-id", type=str, default="openbmb/VoxCPM2",
|
||||
"--model-id",
|
||||
type=str,
|
||||
default="openbmb/VoxCPM2",
|
||||
help="Local path or HuggingFace repo ID (default: openbmb/VoxCPM2)",
|
||||
)
|
||||
parser.add_argument("--port", type=int, default=8808, help="Server port")
|
||||
|
||||
32
app_old.py
32
app_old.py
@@ -6,6 +6,7 @@ import gradio as gr
|
||||
from typing import Optional, Tuple
|
||||
from funasr import AutoModel
|
||||
from pathlib import Path
|
||||
|
||||
os.environ["TOKENIZERS_PARALLELISM"] = "false"
|
||||
if os.environ.get("HF_REPO_ID", "").strip() == "":
|
||||
os.environ["HF_REPO_ID"] = "openbmb/VoxCPM1.5"
|
||||
@@ -23,7 +24,7 @@ class VoxCPMDemo:
|
||||
self.asr_model: Optional[AutoModel] = AutoModel(
|
||||
model=self.asr_model_id,
|
||||
disable_update=True,
|
||||
log_level='DEBUG',
|
||||
log_level="DEBUG",
|
||||
device="cuda:0" if self.device == "cuda" else "cpu",
|
||||
)
|
||||
|
||||
@@ -48,6 +49,7 @@ class VoxCPMDemo:
|
||||
if not os.path.isdir(target_dir):
|
||||
try:
|
||||
from huggingface_hub import snapshot_download # type: ignore
|
||||
|
||||
os.makedirs(target_dir, exist_ok=True)
|
||||
print(f"Downloading model from HF repo '{repo_id}' to '{target_dir}' ...", file=sys.stderr)
|
||||
snapshot_download(repo_id=repo_id, local_dir=target_dir, local_dir_use_symlinks=False)
|
||||
@@ -72,7 +74,7 @@ class VoxCPMDemo:
|
||||
if prompt_wav is None:
|
||||
return ""
|
||||
res = self.asr_model.generate(input=prompt_wav, language="auto", use_itn=True)
|
||||
text = res[0]["text"].split('|>')[-1]
|
||||
text = res[0]["text"].split("|>")[-1]
|
||||
return text
|
||||
|
||||
def generate_tts_audio(
|
||||
@@ -153,7 +155,9 @@ def create_demo_interface(demo: VoxCPMDemo):
|
||||
|
||||
with gr.Blocks() as interface:
|
||||
# Header logo
|
||||
gr.HTML('<div class="logo-container"><img src="/gradio_api/file=assets/voxcpm_logo.png" alt="VoxCPM Logo"></div>')
|
||||
gr.HTML(
|
||||
'<div class="logo-container"><img src="/gradio_api/file=assets/voxcpm_logo.png" alt="VoxCPM Logo"></div>'
|
||||
)
|
||||
|
||||
# Quick Start
|
||||
with gr.Accordion("📋 Quick Start Guide |快速入门", open=False, elem_id="acc_quick"):
|
||||
@@ -201,7 +205,7 @@ def create_demo_interface(demo: VoxCPMDemo):
|
||||
with gr.Row():
|
||||
with gr.Column():
|
||||
prompt_wav = gr.Audio(
|
||||
sources=["upload", 'microphone'],
|
||||
sources=["upload", "microphone"],
|
||||
type="filepath",
|
||||
label="Prompt Speech (Optional, or let VoxCPM improvise)",
|
||||
value="./examples/example.wav",
|
||||
@@ -210,13 +214,13 @@ def create_demo_interface(demo: VoxCPMDemo):
|
||||
value=False,
|
||||
label="Prompt Speech Enhancement",
|
||||
elem_id="chk_denoise",
|
||||
info="We use ZipEnhancer model to denoise the prompt audio."
|
||||
info="We use ZipEnhancer model to denoise the prompt audio.",
|
||||
)
|
||||
with gr.Row():
|
||||
prompt_text = gr.Textbox(
|
||||
value="Just by listening a few minutes a day, you'll be able to eliminate negative thoughts by conditioning your mind to be more positive.",
|
||||
label="Prompt Text",
|
||||
placeholder="Please enter the prompt text. Automatic recognition is supported, and you can correct the results yourself..."
|
||||
placeholder="Please enter the prompt text. Automatic recognition is supported, and you can correct the results yourself...",
|
||||
)
|
||||
run_btn = gr.Button("Generate Speech", variant="primary")
|
||||
|
||||
@@ -227,7 +231,7 @@ def create_demo_interface(demo: VoxCPMDemo):
|
||||
value=2.0,
|
||||
step=0.1,
|
||||
label="CFG Value (Guidance Scale)",
|
||||
info="Higher values increase adherence to prompt, lower values allow more creativity"
|
||||
info="Higher values increase adherence to prompt, lower values allow more creativity",
|
||||
)
|
||||
inference_timesteps = gr.Slider(
|
||||
minimum=4,
|
||||
@@ -235,7 +239,7 @@ def create_demo_interface(demo: VoxCPMDemo):
|
||||
value=10,
|
||||
step=1,
|
||||
label="Inference Timesteps",
|
||||
info="Number of inference timesteps for generation (higher values may improve quality but slower)"
|
||||
info="Number of inference timesteps for generation (higher values may improve quality but slower)",
|
||||
)
|
||||
with gr.Row():
|
||||
text = gr.Textbox(
|
||||
@@ -247,14 +251,22 @@ def create_demo_interface(demo: VoxCPMDemo):
|
||||
value=False,
|
||||
label="Text Normalization",
|
||||
elem_id="chk_normalize",
|
||||
info="We use wetext library to normalize the input text."
|
||||
info="We use wetext library to normalize the input text.",
|
||||
)
|
||||
audio_output = gr.Audio(label="Output Audio")
|
||||
|
||||
# Wiring
|
||||
run_btn.click(
|
||||
fn=demo.generate_tts_audio,
|
||||
inputs=[text, prompt_wav, prompt_text, cfg_value, inference_timesteps, DoNormalizeText, DoDenoisePromptAudio],
|
||||
inputs=[
|
||||
text,
|
||||
prompt_wav,
|
||||
prompt_text,
|
||||
cfg_value,
|
||||
inference_timesteps,
|
||||
DoNormalizeText,
|
||||
DoDenoisePromptAudio,
|
||||
],
|
||||
outputs=[audio_output],
|
||||
show_progress=True,
|
||||
api_name="generate",
|
||||
|
||||
@@ -308,7 +308,8 @@ def run_inference(text, prompt_wav, prompt_text, lora_selection, cfg_scale, step
|
||||
if new_r is not None and current_r is not None and new_r != current_r:
|
||||
print(f"LoRA rank mismatch (model r={current_r}, checkpoint r={new_r}), reloading...", file=sys.stderr)
|
||||
reload_base = (
|
||||
new_base_model if new_base_model and os.path.exists(new_base_model)
|
||||
new_base_model
|
||||
if new_base_model and os.path.exists(new_base_model)
|
||||
else (pretrained_path if pretrained_path and pretrained_path.strip() else default_pretrained_path)
|
||||
)
|
||||
try:
|
||||
@@ -982,9 +983,7 @@ with gr.Blocks(title="VoxCPM LoRA WebUI", theme=gr.themes.Soft(), css=custom_css
|
||||
|
||||
gr.Markdown("#### 分发选项 (Distribution)")
|
||||
with gr.Row():
|
||||
hf_model_id = gr.Textbox(
|
||||
label="HuggingFace Model ID (e.g., openbmb/VoxCPM2)", value=""
|
||||
)
|
||||
hf_model_id = gr.Textbox(label="HuggingFace Model ID (e.g., openbmb/VoxCPM2)", value="")
|
||||
distribute = gr.Checkbox(label="分发模式 (distribute)", value=False)
|
||||
|
||||
with gr.Column(scale=2, elem_classes="form-section"):
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
Loads src/voxcpm/model/utils.py directly to avoid the heavy voxcpm package
|
||||
init. Run with: `python scripts/test_pick_runtime_dtype.py`.
|
||||
"""
|
||||
|
||||
import importlib.util
|
||||
import os
|
||||
import pathlib
|
||||
|
||||
@@ -19,6 +19,7 @@ With voice cloning:
|
||||
--text "Hello, this is voice cloning result." \
|
||||
--prompt_audio path/to/ref.wav \
|
||||
--prompt_text "Reference audio transcript" \
|
||||
--seed 42 \
|
||||
--output ft_clone.wav
|
||||
"""
|
||||
|
||||
@@ -86,6 +87,12 @@ def parse_args():
|
||||
action="store_true",
|
||||
help="Enable text normalization",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--seed",
|
||||
type=int,
|
||||
default=None,
|
||||
help="Random seed for generation (default: None)",
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
@@ -109,6 +116,9 @@ def main():
|
||||
print(f"[FT Inference] Using reference audio: {prompt_wav_path}", file=sys.stderr)
|
||||
print(f"[FT Inference] Reference text: {prompt_text}", file=sys.stderr)
|
||||
|
||||
if args.seed is not None:
|
||||
print(f"[FT Inference] Using seed: {args.seed}", file=sys.stderr)
|
||||
|
||||
audio_np = model.generate(
|
||||
text=args.text,
|
||||
prompt_wav_path=prompt_wav_path,
|
||||
@@ -118,6 +128,7 @@ def main():
|
||||
max_len=args.max_len,
|
||||
normalize=args.normalize,
|
||||
denoise=False,
|
||||
seed=args.seed,
|
||||
)
|
||||
|
||||
# Save audio
|
||||
|
||||
@@ -16,6 +16,7 @@ With voice cloning:
|
||||
--text "This is voice cloning result." \
|
||||
--prompt_audio path/to/ref.wav \
|
||||
--prompt_text "Reference audio transcript" \
|
||||
--seed 42 \
|
||||
--output lora_clone.wav
|
||||
|
||||
Note: The script reads base_model path and lora_config from lora_config.json
|
||||
@@ -94,6 +95,12 @@ def parse_args():
|
||||
action="store_true",
|
||||
help="Enable text normalization",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--seed",
|
||||
type=int,
|
||||
default=None,
|
||||
help="Random seed for generation (default: None)",
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
@@ -130,6 +137,8 @@ def main():
|
||||
print(
|
||||
f" LoRA config: r={lora_cfg.r}, alpha={lora_cfg.alpha}" if lora_cfg else " LoRA config: None", file=sys.stderr
|
||||
)
|
||||
if args.seed is not None:
|
||||
print(f" Seed: {args.seed}", file=sys.stderr)
|
||||
|
||||
# 3. Load model with LoRA (no denoiser)
|
||||
print(f"\n[1/2] Loading model with LoRA: {pretrained_path}", file=sys.stderr)
|
||||
@@ -161,6 +170,7 @@ def main():
|
||||
max_len=args.max_len,
|
||||
normalize=args.normalize,
|
||||
denoise=False,
|
||||
seed=args.seed,
|
||||
)
|
||||
lora_output = out_path.with_stem(out_path.stem + "_with_lora")
|
||||
sf.write(str(lora_output), audio_np, model.tts_model.sample_rate)
|
||||
@@ -181,6 +191,7 @@ def main():
|
||||
max_len=args.max_len,
|
||||
normalize=args.normalize,
|
||||
denoise=False,
|
||||
seed=args.seed,
|
||||
)
|
||||
disabled_output = out_path.with_stem(out_path.stem + "_lora_disabled")
|
||||
sf.write(str(disabled_output), audio_np, model.tts_model.sample_rate)
|
||||
@@ -201,6 +212,7 @@ def main():
|
||||
max_len=args.max_len,
|
||||
normalize=args.normalize,
|
||||
denoise=False,
|
||||
seed=args.seed,
|
||||
)
|
||||
reenabled_output = out_path.with_stem(out_path.stem + "_lora_reenabled")
|
||||
sf.write(str(reenabled_output), audio_np, model.tts_model.sample_rate)
|
||||
@@ -221,6 +233,7 @@ def main():
|
||||
max_len=args.max_len,
|
||||
normalize=args.normalize,
|
||||
denoise=False,
|
||||
seed=args.seed,
|
||||
)
|
||||
reset_output = out_path.with_stem(out_path.stem + "_lora_reset")
|
||||
sf.write(str(reset_output), audio_np, model.tts_model.sample_rate)
|
||||
@@ -242,6 +255,7 @@ def main():
|
||||
max_len=args.max_len,
|
||||
normalize=args.normalize,
|
||||
denoise=False,
|
||||
seed=args.seed,
|
||||
)
|
||||
reload_output = out_path.with_stem(out_path.stem + "_lora_reloaded")
|
||||
sf.write(str(reload_output), audio_np, model.tts_model.sample_rate)
|
||||
|
||||
@@ -599,7 +599,9 @@ def generate_sample_audio(
|
||||
)
|
||||
with torch.no_grad():
|
||||
with autocast_ctx:
|
||||
generated = unwrapped_model.generate(target_text=text, inference_timesteps=10, cfg_value=2.0)
|
||||
generated = unwrapped_model.generate(
|
||||
target_text=text, inference_timesteps=10, cfg_value=2.0, seed=42
|
||||
)
|
||||
|
||||
# Restore training setup
|
||||
# unwrapped_model.to(torch.float32)
|
||||
|
||||
@@ -248,6 +248,7 @@ def _run_single(args, parser, *, text: str, output: str, prompt_text: str | None
|
||||
inference_timesteps=args.inference_timesteps,
|
||||
normalize=args.normalize,
|
||||
denoise=args.denoise and (args.prompt_audio is not None or args.reference_audio is not None),
|
||||
seed=args.seed,
|
||||
)
|
||||
|
||||
import soundfile as sf
|
||||
@@ -332,6 +333,7 @@ def cmd_batch(args, parser):
|
||||
inference_timesteps=args.inference_timesteps,
|
||||
normalize=args.normalize,
|
||||
denoise=args.denoise and (prompt_audio_path is not None or reference_audio_path is not None),
|
||||
seed=args.seed,
|
||||
)
|
||||
|
||||
output_file = output_dir / f"output_{i:03d}.wav"
|
||||
@@ -413,6 +415,12 @@ def _add_common_generation_args(parser):
|
||||
help="Inference steps (int, recommended 4–30, default: 10)",
|
||||
)
|
||||
parser.add_argument("--normalize", action="store_true", help="Enable text normalization")
|
||||
parser.add_argument(
|
||||
"--seed",
|
||||
type=int,
|
||||
default=None,
|
||||
help="Random seed for generation (default: None)",
|
||||
)
|
||||
|
||||
|
||||
def _add_prompt_reference_args(parser):
|
||||
@@ -587,6 +595,12 @@ Examples:
|
||||
help="Inference steps (int, recommended 4–30, default: 10)",
|
||||
)
|
||||
batch_parser.add_argument("--normalize", action="store_true", help="Enable text normalization")
|
||||
batch_parser.add_argument(
|
||||
"--seed",
|
||||
type=int,
|
||||
default=None,
|
||||
help="Random seed for generation (default: None)",
|
||||
)
|
||||
_add_model_args(batch_parser)
|
||||
_add_lora_args(batch_parser)
|
||||
_add_timestamp_args(batch_parser, include_output=False)
|
||||
|
||||
@@ -196,6 +196,7 @@ class VoxCPM:
|
||||
retry_badcase_max_times: int = 3,
|
||||
retry_badcase_ratio_threshold: float = 6.0,
|
||||
streaming: bool = False,
|
||||
seed: Optional[int] = None,
|
||||
) -> Generator[np.ndarray, None, None]:
|
||||
"""Synthesize speech for the given text and return a single waveform.
|
||||
|
||||
@@ -218,6 +219,7 @@ class VoxCPM:
|
||||
retry_badcase_max_times: Maximum number of times to retry badcase.
|
||||
retry_badcase_ratio_threshold: Threshold for audio-to-text ratio.
|
||||
streaming: Whether to return a generator of audio chunks.
|
||||
seed: Optional random seed for reproducibility.
|
||||
Returns:
|
||||
Generator of numpy.ndarray: 1D waveform array (float32) on CPU.
|
||||
Yields audio chunks for each generation step if ``streaming=True``,
|
||||
@@ -294,6 +296,7 @@ class VoxCPM:
|
||||
retry_badcase_max_times=retry_badcase_max_times,
|
||||
retry_badcase_ratio_threshold=retry_badcase_ratio_threshold,
|
||||
streaming=streaming,
|
||||
seed=seed,
|
||||
)
|
||||
|
||||
if streaming:
|
||||
|
||||
@@ -5,9 +5,12 @@ from transformers import PreTrainedTokenizer
|
||||
|
||||
_LOW_PRECISION_DTYPES = {"bfloat16", "bf16", "float16", "fp16"}
|
||||
_VALID_DTYPE_OVERRIDES = {
|
||||
"bfloat16", "bf16",
|
||||
"float16", "fp16",
|
||||
"float32", "fp32",
|
||||
"bfloat16",
|
||||
"bf16",
|
||||
"float16",
|
||||
"fp16",
|
||||
"float32",
|
||||
"fp32",
|
||||
}
|
||||
|
||||
|
||||
@@ -21,6 +24,19 @@ def next_and_close(gen):
|
||||
gen.close()
|
||||
|
||||
|
||||
def materialize_generation_seed(seed: Optional[int]) -> int:
|
||||
"""Return a concrete seed for a generation request."""
|
||||
if seed is not None:
|
||||
return int(seed)
|
||||
return int(torch.seed() & 0xFFFFFFFF)
|
||||
|
||||
|
||||
def apply_generation_seed(seed: int) -> None:
|
||||
torch.manual_seed(seed)
|
||||
if torch.cuda.is_available():
|
||||
torch.cuda.manual_seed_all(seed)
|
||||
|
||||
|
||||
def mask_multichar_chinese_tokens(tokenizer: PreTrainedTokenizer):
|
||||
"""Create a tokenizer wrapper that converts multi-character Chinese tokens to single characters.
|
||||
|
||||
@@ -160,10 +176,7 @@ def pick_runtime_dtype(device: str, configured_dtype: str) -> str:
|
||||
override = os.environ.get("VOXCPM_MPS_DTYPE", "").strip().lower()
|
||||
if override:
|
||||
if override not in _VALID_DTYPE_OVERRIDES:
|
||||
raise ValueError(
|
||||
f"VOXCPM_MPS_DTYPE='{override}' is not one of "
|
||||
f"{sorted(_VALID_DTYPE_OVERRIDES)}"
|
||||
)
|
||||
raise ValueError(f"VOXCPM_MPS_DTYPE='{override}' is not one of " f"{sorted(_VALID_DTYPE_OVERRIDES)}")
|
||||
return override
|
||||
|
||||
if (configured_dtype or "").lower() in _LOW_PRECISION_DTYPES:
|
||||
@@ -211,15 +224,13 @@ def resolve_runtime_device(device: Optional[str], configured_device: str = "cuda
|
||||
if explicit.startswith("cuda"):
|
||||
if not torch.cuda.is_available():
|
||||
raise ValueError(
|
||||
f"Requested device '{device}', but CUDA is not available. "
|
||||
"Use device='auto' for automatic fallback."
|
||||
f"Requested device '{device}', but CUDA is not available. " "Use device='auto' for automatic fallback."
|
||||
)
|
||||
return explicit
|
||||
if explicit == "mps":
|
||||
if not _has_mps():
|
||||
raise ValueError(
|
||||
"Requested device 'mps', but MPS is not available. "
|
||||
"Use device='auto' for automatic fallback."
|
||||
"Requested device 'mps', but MPS is not available. " "Use device='auto' for automatic fallback."
|
||||
)
|
||||
return "mps"
|
||||
if explicit == "cpu":
|
||||
|
||||
@@ -45,7 +45,9 @@ from ..modules.locdit import CfmConfig, UnifiedCFM, VoxCPMLocDiT
|
||||
from ..modules.locenc import VoxCPMLocEnc
|
||||
from ..modules.minicpm4 import MiniCPM4Config, MiniCPMModel
|
||||
from .utils import (
|
||||
apply_generation_seed,
|
||||
get_dtype,
|
||||
materialize_generation_seed,
|
||||
mask_multichar_chinese_tokens,
|
||||
next_and_close,
|
||||
pick_runtime_dtype,
|
||||
@@ -140,6 +142,7 @@ class VoxCPMModel(nn.Module):
|
||||
self.text_tokenizer = mask_multichar_chinese_tokens(tokenizer)
|
||||
self.audio_start_token = 101
|
||||
self.audio_end_token = 102
|
||||
self.last_successful_seed = None
|
||||
|
||||
# Residual Acoustic LM
|
||||
residual_lm_config = config.lm_config.model_copy(deep=True)
|
||||
@@ -367,6 +370,7 @@ class VoxCPMModel(nn.Module):
|
||||
retry_badcase_max_times: int = 3,
|
||||
retry_badcase_ratio_threshold: float = 6.0, # setting acceptable ratio of audio length to text length (for badcase detection)
|
||||
streaming: bool = False,
|
||||
seed: Optional[int] = None,
|
||||
) -> Generator[torch.Tensor, None, None]:
|
||||
if retry_badcase and streaming:
|
||||
warnings.warn("Retry on bad cases is not supported in streaming mode, setting retry_badcase=False.")
|
||||
@@ -452,7 +456,12 @@ class VoxCPMModel(nn.Module):
|
||||
target_text_length = len(self.text_tokenizer(target_text))
|
||||
|
||||
retry_badcase_times = 0
|
||||
current_seed = materialize_generation_seed(seed)
|
||||
last_attempt_seed = current_seed
|
||||
while retry_badcase_times < retry_badcase_max_times:
|
||||
last_attempt_seed = current_seed
|
||||
apply_generation_seed(last_attempt_seed)
|
||||
|
||||
inference_result = self._inference(
|
||||
text_token,
|
||||
text_mask,
|
||||
@@ -471,6 +480,7 @@ class VoxCPMModel(nn.Module):
|
||||
for latent_pred, _ in inference_result:
|
||||
decode_audio = self.audio_vae.decode(latent_pred.to(torch.float32))
|
||||
decode_audio = decode_audio[..., -patch_len:].squeeze(1).cpu()
|
||||
self.last_successful_seed = last_attempt_seed
|
||||
yield decode_audio
|
||||
break
|
||||
else:
|
||||
@@ -482,6 +492,7 @@ class VoxCPMModel(nn.Module):
|
||||
file=sys.stderr,
|
||||
)
|
||||
retry_badcase_times += 1
|
||||
current_seed += 1
|
||||
continue
|
||||
else:
|
||||
break
|
||||
@@ -489,6 +500,7 @@ class VoxCPMModel(nn.Module):
|
||||
break
|
||||
|
||||
if not streaming:
|
||||
self.last_successful_seed = last_attempt_seed
|
||||
decode_audio = self.audio_vae.decode(latent_pred.to(torch.float32)).squeeze(1).cpu()
|
||||
yield decode_audio
|
||||
|
||||
@@ -603,6 +615,7 @@ class VoxCPMModel(nn.Module):
|
||||
retry_badcase_ratio_threshold: float = 6.0,
|
||||
streaming: bool = False,
|
||||
streaming_prefix_len: int = 3,
|
||||
seed: Optional[int] = None,
|
||||
) -> Generator[Tuple[torch.Tensor, torch.Tensor, Union[torch.Tensor, List[torch.Tensor]]], None, None]:
|
||||
"""
|
||||
Generate audio using pre-built prompt cache.
|
||||
@@ -678,7 +691,12 @@ class VoxCPMModel(nn.Module):
|
||||
# run inference
|
||||
target_text_length = len(self.text_tokenizer(target_text))
|
||||
retry_badcase_times = 0
|
||||
current_seed = materialize_generation_seed(seed)
|
||||
last_attempt_seed = current_seed
|
||||
while retry_badcase_times < retry_badcase_max_times:
|
||||
last_attempt_seed = current_seed
|
||||
apply_generation_seed(last_attempt_seed)
|
||||
|
||||
inference_result = self._inference(
|
||||
text_token,
|
||||
text_mask,
|
||||
@@ -698,6 +716,7 @@ class VoxCPMModel(nn.Module):
|
||||
for latent_pred, pred_audio_feat in inference_result:
|
||||
decode_audio = self.audio_vae.decode(latent_pred.to(torch.float32))
|
||||
decode_audio = decode_audio[..., -patch_len:].squeeze(1).cpu()
|
||||
self.last_successful_seed = last_attempt_seed
|
||||
yield (decode_audio, target_text_token, pred_audio_feat)
|
||||
break
|
||||
else:
|
||||
@@ -709,12 +728,14 @@ class VoxCPMModel(nn.Module):
|
||||
file=sys.stderr,
|
||||
)
|
||||
retry_badcase_times += 1
|
||||
current_seed += 1
|
||||
continue
|
||||
else:
|
||||
break
|
||||
else:
|
||||
break
|
||||
if not streaming:
|
||||
self.last_successful_seed = last_attempt_seed
|
||||
decode_audio = self.audio_vae.decode(latent_pred.to(torch.float32))
|
||||
patch_len = self.patch_size * self.chunk_size
|
||||
if audio_mask.sum().item() > 0:
|
||||
|
||||
@@ -46,7 +46,9 @@ from ..modules.locdit import CfmConfig, UnifiedCFM, VoxCPMLocDiTV2
|
||||
from ..modules.locenc import VoxCPMLocEnc
|
||||
from ..modules.minicpm4 import MiniCPM4Config, MiniCPMModel
|
||||
from .utils import (
|
||||
apply_generation_seed,
|
||||
get_dtype,
|
||||
materialize_generation_seed,
|
||||
mask_multichar_chinese_tokens,
|
||||
next_and_close,
|
||||
pick_runtime_dtype,
|
||||
@@ -55,7 +57,9 @@ from .utils import (
|
||||
|
||||
|
||||
# A simple function to trim audio silence using VAD, not used default
|
||||
def _trim_audio_silence_vad(audio: torch.Tensor, sample_rate: int, max_silence_ms: float = 200.0, top_db: float = 35.0) -> torch.Tensor:
|
||||
def _trim_audio_silence_vad(
|
||||
audio: torch.Tensor, sample_rate: int, max_silence_ms: float = 200.0, top_db: float = 35.0
|
||||
) -> torch.Tensor:
|
||||
if audio.numel() == 0:
|
||||
return audio
|
||||
y = audio.squeeze(0).numpy()
|
||||
@@ -184,6 +188,7 @@ class VoxCPM2Model(nn.Module):
|
||||
self.audio_end_token = 102
|
||||
self.ref_audio_start_token = 103
|
||||
self.ref_audio_end_token = 104
|
||||
self.last_successful_seed = None
|
||||
|
||||
# Residual Acoustic LM
|
||||
residual_lm_config = config.lm_config.model_copy(deep=True)
|
||||
@@ -476,6 +481,7 @@ class VoxCPM2Model(nn.Module):
|
||||
trim_silence_vad: bool = False,
|
||||
streaming: bool = False,
|
||||
streaming_prefix_len: int = 4,
|
||||
seed: Optional[int] = None,
|
||||
) -> Generator[torch.Tensor, None, None]:
|
||||
if retry_badcase and streaming:
|
||||
warnings.warn("Retry on bad cases is not supported in streaming mode, setting retry_badcase=False.")
|
||||
@@ -633,7 +639,12 @@ class VoxCPM2Model(nn.Module):
|
||||
target_text_length = len(self.text_tokenizer(target_text))
|
||||
|
||||
retry_badcase_times = 0
|
||||
current_seed = materialize_generation_seed(seed)
|
||||
last_attempt_seed = current_seed
|
||||
while retry_badcase_times < retry_badcase_max_times:
|
||||
last_attempt_seed = current_seed
|
||||
apply_generation_seed(last_attempt_seed)
|
||||
|
||||
inference_result = self._inference(
|
||||
text_token,
|
||||
text_mask,
|
||||
@@ -651,6 +662,7 @@ class VoxCPM2Model(nn.Module):
|
||||
for latent_pred, _, _ctx in inference_result:
|
||||
decode_audio = vae_dec.decode_chunk(latent_pred.to(torch.float32))
|
||||
decode_audio = decode_audio.squeeze(1).cpu()
|
||||
self.last_successful_seed = last_attempt_seed
|
||||
yield decode_audio
|
||||
break
|
||||
else:
|
||||
@@ -662,6 +674,7 @@ class VoxCPM2Model(nn.Module):
|
||||
file=sys.stderr,
|
||||
)
|
||||
retry_badcase_times += 1
|
||||
current_seed += 1
|
||||
continue
|
||||
else:
|
||||
break
|
||||
@@ -669,6 +682,7 @@ class VoxCPM2Model(nn.Module):
|
||||
break
|
||||
|
||||
if not streaming:
|
||||
self.last_successful_seed = last_attempt_seed
|
||||
decode_audio = self.audio_vae.decode(latent_pred.to(torch.float32))
|
||||
decode_patch_len = self.patch_size * self._decode_chunk_size
|
||||
if context_len > 0:
|
||||
@@ -793,6 +807,7 @@ class VoxCPM2Model(nn.Module):
|
||||
retry_badcase_ratio_threshold: float = 6.0,
|
||||
streaming: bool = False,
|
||||
streaming_prefix_len: int = 4,
|
||||
seed: Optional[int] = None,
|
||||
) -> Generator[Tuple[torch.Tensor, torch.Tensor, Union[torch.Tensor, List[torch.Tensor]]], None, None]:
|
||||
"""
|
||||
Generate audio using pre-built prompt cache.
|
||||
@@ -920,7 +935,12 @@ class VoxCPM2Model(nn.Module):
|
||||
# run inference
|
||||
target_text_length = len(self.text_tokenizer(target_text))
|
||||
retry_badcase_times = 0
|
||||
current_seed = materialize_generation_seed(seed)
|
||||
last_attempt_seed = current_seed
|
||||
while retry_badcase_times < retry_badcase_max_times:
|
||||
last_attempt_seed = current_seed
|
||||
apply_generation_seed(last_attempt_seed)
|
||||
|
||||
inference_result = self._inference(
|
||||
text_token,
|
||||
text_mask,
|
||||
@@ -938,6 +958,7 @@ class VoxCPM2Model(nn.Module):
|
||||
for latent_pred, pred_audio_feat, _ctx in inference_result:
|
||||
decode_audio = vae_dec.decode_chunk(latent_pred.to(torch.float32))
|
||||
decode_audio = decode_audio.squeeze(1).cpu()
|
||||
self.last_successful_seed = last_attempt_seed
|
||||
yield (decode_audio, target_text_token, pred_audio_feat)
|
||||
break
|
||||
else:
|
||||
@@ -949,12 +970,14 @@ class VoxCPM2Model(nn.Module):
|
||||
file=sys.stderr,
|
||||
)
|
||||
retry_badcase_times += 1
|
||||
current_seed += 1
|
||||
continue
|
||||
else:
|
||||
break
|
||||
else:
|
||||
break
|
||||
if not streaming:
|
||||
self.last_successful_seed = last_attempt_seed
|
||||
decode_audio = self.audio_vae.decode(latent_pred.to(torch.float32))
|
||||
decode_patch_len = self.patch_size * self._decode_chunk_size
|
||||
if context_len > 0:
|
||||
|
||||
@@ -551,8 +551,7 @@ class StreamingVAEDecoder:
|
||||
if x.shape[-1] >= _p:
|
||||
states[_k] = x[:, :, -_p:].detach()
|
||||
else:
|
||||
prev = states.get(_k, torch.zeros(x.shape[0], x.shape[1], _p,
|
||||
device=x.device, dtype=x.dtype))
|
||||
prev = states.get(_k, torch.zeros(x.shape[0], x.shape[1], _p, device=x.device, dtype=x.dtype))
|
||||
states[_k] = torch.cat([prev, x], dim=-1)[:, :, -_p:].detach()
|
||||
return nn.Conv1d.forward(_m, x_pad)
|
||||
|
||||
|
||||
@@ -350,7 +350,8 @@ class AudioFeatureProcessingPacker:
|
||||
|
||||
# -- text token track --
|
||||
# [103, 0×R, 104, text_ids, 101, 0×A, 102]
|
||||
text_token_info = torch.cat([
|
||||
text_token_info = torch.cat(
|
||||
[
|
||||
_tok([self.audio_prompt_start_id]),
|
||||
torch.zeros(ref_len, dtype=torch.int32, device=device),
|
||||
_tok([self.audio_prompt_end_id]),
|
||||
@@ -358,37 +359,71 @@ class AudioFeatureProcessingPacker:
|
||||
_tok([self.audio_start_id]),
|
||||
torch.zeros(tgt_len, dtype=torch.int32, device=device),
|
||||
_tok([self.audio_end_id]),
|
||||
])
|
||||
]
|
||||
)
|
||||
|
||||
# -- audio feature track --
|
||||
zero_1 = torch.zeros((1,) + feat_shape, dtype=torch.float32, device=device)
|
||||
zero_txt = torch.zeros((txt_len,) + feat_shape, dtype=torch.float32, device=device)
|
||||
audio_feat_info = torch.cat([
|
||||
zero_1, ref_feats, zero_1, # 103, ref, 104
|
||||
audio_feat_info = torch.cat(
|
||||
[
|
||||
zero_1,
|
||||
ref_feats,
|
||||
zero_1, # 103, ref, 104
|
||||
zero_txt, # text
|
||||
zero_1, tgt_feats, zero_1, # 101, target, 102
|
||||
], dim=0)
|
||||
zero_1,
|
||||
tgt_feats,
|
||||
zero_1, # 101, target, 102
|
||||
],
|
||||
dim=0,
|
||||
)
|
||||
|
||||
# -- masks --
|
||||
text_mask = torch.cat([
|
||||
torch.ones(1), torch.zeros(ref_len), torch.ones(1),
|
||||
text_mask = (
|
||||
torch.cat(
|
||||
[
|
||||
torch.ones(1),
|
||||
torch.zeros(ref_len),
|
||||
torch.ones(1),
|
||||
torch.ones(txt_len),
|
||||
torch.ones(1), torch.zeros(tgt_len), torch.ones(1),
|
||||
]).to(torch.int32).to(device)
|
||||
torch.ones(1),
|
||||
torch.zeros(tgt_len),
|
||||
torch.ones(1),
|
||||
]
|
||||
)
|
||||
.to(torch.int32)
|
||||
.to(device)
|
||||
)
|
||||
|
||||
audio_mask = torch.cat([
|
||||
torch.zeros(1), torch.ones(ref_len), torch.zeros(1),
|
||||
audio_mask = (
|
||||
torch.cat(
|
||||
[
|
||||
torch.zeros(1),
|
||||
torch.ones(ref_len),
|
||||
torch.zeros(1),
|
||||
torch.zeros(txt_len),
|
||||
torch.zeros(1), torch.ones(tgt_len), torch.zeros(1),
|
||||
]).to(torch.int32).to(device)
|
||||
torch.zeros(1),
|
||||
torch.ones(tgt_len),
|
||||
torch.zeros(1),
|
||||
]
|
||||
)
|
||||
.to(torch.int32)
|
||||
.to(device)
|
||||
)
|
||||
|
||||
loss_mask = torch.cat([
|
||||
loss_mask = (
|
||||
torch.cat(
|
||||
[
|
||||
torch.zeros(1 + ref_len + 1), # ref part: no loss
|
||||
torch.zeros(txt_len), # text: no loss
|
||||
torch.zeros(1), # 101: no loss
|
||||
torch.ones(tgt_len), # target audio: LOSS
|
||||
torch.zeros(1), # 102: no loss
|
||||
]).to(torch.int32).to(device)
|
||||
]
|
||||
)
|
||||
.to(torch.int32)
|
||||
.to(device)
|
||||
)
|
||||
|
||||
total_len = 1 + ref_len + 1 + txt_len + 1 + tgt_len + 1
|
||||
labels = torch.zeros(total_len, dtype=torch.int32, device=device)
|
||||
|
||||
@@ -44,10 +44,7 @@ def _check_audio_file(audio_path: str, sample_rate: int) -> Optional[str]:
|
||||
if info.frames == 0:
|
||||
return f"Audio file is empty: {audio_path}"
|
||||
if info.samplerate != sample_rate:
|
||||
return (
|
||||
f"Sample rate mismatch in {audio_path}: "
|
||||
f"expected {sample_rate} Hz, got {info.samplerate} Hz"
|
||||
)
|
||||
return f"Sample rate mismatch in {audio_path}: " f"expected {sample_rate} Hz, got {info.samplerate} Hz"
|
||||
return None
|
||||
except ImportError:
|
||||
# soundfile not available; just check existence
|
||||
@@ -187,13 +184,9 @@ def validate_manifest(
|
||||
if duration is not None:
|
||||
result.audio_durations.append(duration)
|
||||
if duration < 0.3:
|
||||
result.warnings.append(
|
||||
f"Line {i + 1}: Very short audio ({duration:.2f}s)"
|
||||
)
|
||||
result.warnings.append(f"Line {i + 1}: Very short audio ({duration:.2f}s)")
|
||||
elif duration > 30.0:
|
||||
result.warnings.append(
|
||||
f"Line {i + 1}: Very long audio ({duration:.1f}s), may cause OOM"
|
||||
)
|
||||
result.warnings.append(f"Line {i + 1}: Very long audio ({duration:.1f}s), may cause OOM")
|
||||
else:
|
||||
result.errors.append(f"Line {i + 1}: Invalid audio path")
|
||||
has_error = True
|
||||
@@ -209,9 +202,7 @@ def validate_manifest(
|
||||
if os.path.isfile(ref_path):
|
||||
result.has_ref_audio += 1
|
||||
else:
|
||||
result.warnings.append(
|
||||
f"Line {i + 1}: ref_audio file not found: {ref_path}"
|
||||
)
|
||||
result.warnings.append(f"Line {i + 1}: ref_audio file not found: {ref_path}")
|
||||
|
||||
if not has_error:
|
||||
result.valid_samples += 1
|
||||
@@ -222,14 +213,10 @@ def validate_manifest(
|
||||
# Summarize truncated errors
|
||||
if missing_audio_count > 5:
|
||||
result.errors.append(
|
||||
f"... and {missing_audio_count - 5} more missing audio files "
|
||||
f"({missing_audio_count} total)"
|
||||
f"... and {missing_audio_count - 5} more missing audio files " f"({missing_audio_count} total)"
|
||||
)
|
||||
if empty_text_count > 5:
|
||||
result.warnings.append(
|
||||
f"... and {empty_text_count - 5} more empty text entries "
|
||||
f"({empty_text_count} total)"
|
||||
)
|
||||
result.warnings.append(f"... and {empty_text_count - 5} more empty text entries " f"({empty_text_count} total)")
|
||||
|
||||
return result
|
||||
|
||||
|
||||
@@ -676,3 +676,61 @@ def test_detect_model_architecture_uses_local_configs():
|
||||
|
||||
assert cli.detect_model_architecture(v1_args) == "voxcpm"
|
||||
assert cli.detect_model_architecture(v2_args) == "voxcpm2"
|
||||
|
||||
|
||||
def test_parser_accepts_seed():
|
||||
parser = cli._build_parser()
|
||||
# Default seed should be None
|
||||
args = parser.parse_args(["design", "--text", "hello", "--output", "out.wav"])
|
||||
assert args.seed is None
|
||||
|
||||
# Custom seed should be parsed as int
|
||||
args = parser.parse_args(["design", "--text", "hello", "--output", "out.wav", "--seed", "42"])
|
||||
assert args.seed == 42
|
||||
|
||||
|
||||
def test_design_subcommand_passes_seed(monkeypatch, tmp_path):
|
||||
dummy_model = DummyModel()
|
||||
monkeypatch.setattr(cli, "load_model", lambda args: dummy_model)
|
||||
patch_soundfile_write(monkeypatch)
|
||||
|
||||
run_main(
|
||||
monkeypatch,
|
||||
[
|
||||
"design",
|
||||
"--text",
|
||||
"hello",
|
||||
"--seed",
|
||||
"123",
|
||||
"--output",
|
||||
str(tmp_path / "out.wav"),
|
||||
],
|
||||
)
|
||||
|
||||
assert dummy_model.calls[0]["seed"] == 123
|
||||
|
||||
|
||||
def test_batch_subcommand_passes_seed(monkeypatch, tmp_path):
|
||||
dummy_model = DummyModel()
|
||||
input_file = tmp_path / "texts.txt"
|
||||
input_file.write_text("hello\nworld\n", encoding="utf-8")
|
||||
|
||||
monkeypatch.setattr(cli, "load_model", lambda args: dummy_model)
|
||||
patch_soundfile_write(monkeypatch)
|
||||
|
||||
run_main(
|
||||
monkeypatch,
|
||||
[
|
||||
"batch",
|
||||
"--input",
|
||||
str(input_file),
|
||||
"--output-dir",
|
||||
str(tmp_path / "outs"),
|
||||
"--seed",
|
||||
"999",
|
||||
],
|
||||
)
|
||||
|
||||
assert len(dummy_model.calls) == 2
|
||||
assert dummy_model.calls[0]["seed"] == 999
|
||||
assert dummy_model.calls[1]["seed"] == 999
|
||||
|
||||
@@ -47,3 +47,25 @@ def test_resolve_runtime_device_rejects_unavailable_explicit_cuda(monkeypatch):
|
||||
|
||||
with pytest.raises(ValueError, match="CUDA is not available"):
|
||||
utils.resolve_runtime_device("cuda:0", "cuda")
|
||||
|
||||
|
||||
def test_materialize_generation_seed_preserves_explicit_seed():
|
||||
assert utils.materialize_generation_seed(42) == 42
|
||||
|
||||
|
||||
def test_materialize_generation_seed_creates_concrete_seed_for_none(monkeypatch):
|
||||
monkeypatch.setattr(utils.torch, "seed", lambda: 0x123456789)
|
||||
|
||||
assert utils.materialize_generation_seed(None) == 0x23456789
|
||||
|
||||
|
||||
def test_apply_generation_seed_sets_cpu_and_cuda_rng(monkeypatch):
|
||||
calls = []
|
||||
|
||||
monkeypatch.setattr(utils.torch, "manual_seed", lambda seed: calls.append(("cpu", seed)))
|
||||
monkeypatch.setattr(utils.torch.cuda, "is_available", lambda: True)
|
||||
monkeypatch.setattr(utils.torch.cuda, "manual_seed_all", lambda seed: calls.append(("cuda", seed)))
|
||||
|
||||
utils.apply_generation_seed(123)
|
||||
|
||||
assert calls == [("cpu", 123), ("cuda", 123)]
|
||||
|
||||
@@ -207,6 +207,7 @@ class TestValidateManifest:
|
||||
|
||||
audio = tmp_dir / "audio_8k.wav"
|
||||
import numpy as np
|
||||
|
||||
samples = np.zeros(8000, dtype=np.float32)
|
||||
sf.write(str(audio), samples, 8000)
|
||||
|
||||
@@ -240,6 +241,7 @@ class TestValidateManifest:
|
||||
def test_cli_validate_exit_code(self, tmp_dir):
|
||||
"""validate subcommand must exit 1 on validation error (missing audio)."""
|
||||
import subprocess
|
||||
|
||||
manifest = tmp_dir / "bad.jsonl"
|
||||
_write_manifest(manifest, [{"text": "hi", "audio": "/nonexistent/x.wav"}])
|
||||
|
||||
|
||||
Reference in New Issue
Block a user