Merge pull request #327 from DEVAIEXP/feat-add-seed

feat: Add Seed Support for Reproducible Voice Generation (v1 & v2)

Resolved conflict in src/voxcpm/cli.py: kept both --seed (PR #327) and timestamp alignment args (PR #332).
This commit is contained in:
Labmem-Zhouyx
2026-06-29 13:19:31 +08:00
20 changed files with 377 additions and 98 deletions

View File

@@ -108,6 +108,7 @@ wav = model.generate(
text="VoxCPM2 is the current recommended release for realistic multilingual speech synthesis.",
cfg_value=2.0,
inference_timesteps=10,
seed=42,
)
sf.write("demo.wav", wav, model.tts_model.sample_rate)
print("saved: demo.wav")
@@ -131,6 +132,7 @@ wav = model.generate(
text="VoxCPM2 is the current recommended release for realistic multilingual speech synthesis.",
cfg_value=2.0,
inference_timesteps=10,
seed=42,
)
sf.write("demo.wav", wav, model.tts_model.sample_rate)
```
@@ -144,6 +146,7 @@ wav = model.generate(
text="(A young woman, gentle and sweet voice)Hello, welcome to VoxCPM2!",
cfg_value=2.0,
inference_timesteps=10,
seed=42,
)
sf.write("voice_design.wav", wav, model.tts_model.sample_rate)
```
@@ -164,6 +167,7 @@ wav = model.generate(
reference_wav_path="path/to/voice.wav",
cfg_value=2.0,
inference_timesteps=10,
seed=42,
)
sf.write("controllable_clone.wav", wav, model.tts_model.sample_rate)
```
@@ -210,6 +214,7 @@ voxcpm design \
voxcpm design \
--text "VoxCPM2 brings studio-quality multilingual speech synthesis." \
--control "Young female voice, warm and gentle, slightly smiling" \
--seed 42 \
--output out.wav
# Voice cloning (reference audio)

View File

@@ -10,7 +10,7 @@
<a href="https://voxcpm.readthedocs.io/zh-cn/latest/"><img src="https://img.shields.io/badge/Docs-ReadTheDocs-8CA1AF" alt="Documentation"></a>
<a href="https://huggingface.co/openbmb/VoxCPM2"><img src="https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-VoxCPM2-yellow" alt="Hugging Face"></a>
<a href="https://modelscope.cn/models/OpenBMB/VoxCPM2"><img src="https://img.shields.io/badge/ModelScope-VoxCPM2-purple" alt="ModelScope"></a>
<a href="https://openbmb.github.io/voxcpm2-demopage/"><img src="https://img.shields.io/badge/DemoPage-Audio Samples-red"></a>
<a href="https://openbmb.github.io/voxcpm2-demopage/"><img src="https://img.shields.io/badge/DemoPage-Audio Samples-red" alt="DemoPage"></a>
</p>
@@ -110,6 +110,7 @@ wav = model.generate(
text="VoxCPM2 是目前推荐使用的多语言语音合成版本。",
cfg_value=2.0,
inference_timesteps=10,
seed=42,
)
sf.write("demo.wav", wav, model.tts_model.sample_rate)
print("已保存: demo.wav")
@@ -133,6 +134,7 @@ wav = model.generate(
text="VoxCPM2 是目前推荐使用的多语言语音合成版本。",
cfg_value=2.0,
inference_timesteps=10,
seed=42,
)
sf.write("demo.wav", wav, model.tts_model.sample_rate)
```
@@ -146,6 +148,7 @@ wav = model.generate(
text="(年轻女性,声音温柔甜美)你好欢迎使用VoxCPM2",
cfg_value=2.0,
inference_timesteps=10,
seed=42,
)
sf.write("voice_design.wav", wav, model.tts_model.sample_rate)
```
@@ -166,6 +169,7 @@ wav = model.generate(
reference_wav_path="path/to/voice.wav",
cfg_value=2.0,
inference_timesteps=10,
seed=42,
)
sf.write("controllable_clone.wav", wav, model.tts_model.sample_rate)
```
@@ -212,6 +216,7 @@ voxcpm design \
voxcpm design \
--text "VoxCPM2带来全新语音合成体验。" \
--control "年轻女声,温暖温柔,略带微笑" \
--seed 42 \
--output out.wav
# 声音克隆(参考音频)

83
app.py
View File

@@ -2,6 +2,7 @@ import os
import re
import sys
import logging
import random
import numpy as np
import gradio as gr
from typing import Optional, Tuple
@@ -44,8 +45,8 @@ _EXAMPLES_FOOTER_EN = (
"**Example 1 — Gentle & Melancholic Girl** \n"
'`Control Instruction`: *"A young girl with a soft, sweet voice. '
'Speaks slowly with a melancholic, slightly tsundere tone."* \n'
'`Target Text`: *"I never asked you to stay… It\'s not like I care or anything. '
'But… why does it still hurt so much now that you\'re gone?"* \n\n'
"`Target Text`: *\"I never asked you to stay… It's not like I care or anything. "
"But… why does it still hurt so much now that you're gone?\"* \n\n"
"**Example 2 — Laid-Back Surfer Dude** \n"
'`Control Instruction`: *"Relaxed young male voice, slightly nasal, '
'lazy drawl, very casual and chill."* \n'
@@ -115,6 +116,10 @@ _I18N_TRANSLATIONS = {
"cfg_info": "Higher → closer to the prompt / reference; lower → more creative variation",
"dit_steps_label": "LocDiT flow-matching steps",
"dit_steps_info": "LocDiT flow-matching steps — more steps → maybe better audio quality, but slower",
"seed_label": "Seed",
"seed_info": "Seed used for reproducible generation. Updated with the actual successful seed after generation.",
"random_seed_label": "Random Seed",
"random_seed_info": "Generate a new seed before each inference run.",
"usage_instructions": _USAGE_INSTRUCTIONS_EN,
"examples_footer": _EXAMPLES_FOOTER_EN,
},
@@ -142,7 +147,7 @@ _I18N_TRANSLATIONS = {
"examples_footer": _EXAMPLES_FOOTER_ZH,
},
"zh-Hans": None, # alias, filled below
"zh": None, # alias, filled below
"zh": None, # alias, filled below
}
_I18N_TRANSLATIONS["zh-Hans"] = _I18N_TRANSLATIONS["zh-CN"]
_I18N_TRANSLATIONS["zh"] = _I18N_TRANSLATIONS["zh-CN"]
@@ -155,8 +160,7 @@ for _d in _I18N_TRANSLATIONS.values():
I18N = gr.I18n(**_I18N_TRANSLATIONS)
DEFAULT_TARGET_TEXT = (
"VoxCPM2 is a creative multilingual TTS model from ModelBest, "
"designed to generate highly realistic speech."
"VoxCPM2 is a creative multilingual TTS model from ModelBest, " "designed to generate highly realistic speech."
)
_CUSTOM_CSS = """
@@ -219,6 +223,7 @@ _APP_THEME = gr.themes.Soft(
# ---------- Model ----------
class VoxCPMDemo:
def __init__(self, model_id: str = "openbmb/VoxCPM2", device: str = "auto") -> None:
self.device = resolve_runtime_device(device, "cuda")
@@ -247,9 +252,7 @@ class VoxCPMDemo:
def get_or_load_asr_model(self) -> AutoModel:
if self.asr_model is not None:
return self.asr_model
logger.info(
f"Loading ASR model: {self.asr_model_id} on device: {self.asr_device}"
)
logger.info(f"Loading ASR model: {self.asr_model_id} on device: {self.asr_device}")
self.asr_model = AutoModel(
model=self.asr_model_id,
disable_update=True,
@@ -279,6 +282,7 @@ class VoxCPMDemo:
do_normalize: bool,
denoise: bool,
inference_timesteps: int = 10,
seed: Optional[int] = None,
) -> dict:
generate_kwargs = dict(
text=final_text,
@@ -287,6 +291,7 @@ class VoxCPMDemo:
inference_timesteps=inference_timesteps,
normalize=do_normalize,
denoise=denoise,
seed=seed,
)
if prompt_text_clean and audio_path:
generate_kwargs["prompt_wav_path"] = audio_path
@@ -303,7 +308,8 @@ class VoxCPMDemo:
do_normalize: bool = True,
denoise: bool = True,
inference_timesteps: int = 10,
) -> Tuple[int, np.ndarray]:
seed: Optional[int] = None,
) -> Tuple[int, np.ndarray, Optional[int]]:
current_model = self.get_or_load_voxcpm()
text = (text_input or "").strip()
@@ -335,16 +341,32 @@ class VoxCPMDemo:
do_normalize=do_normalize,
denoise=denoise,
inference_timesteps=inference_timesteps,
seed=seed,
)
wav = current_model.generate(**generate_kwargs)
return (current_model.tts_model.sample_rate, wav)
last_successful_seed = getattr(current_model.tts_model, "last_successful_seed", seed)
return (current_model.tts_model.sample_rate, wav, last_successful_seed)
# ---------- UI ----------
def create_demo_interface(demo: VoxCPMDemo):
gr.set_static_paths(paths=[Path.cwd().absolute() / "assets"])
def _coerce_seed(seed_value) -> Optional[int]:
if seed_value is None or seed_value == "":
return None
return int(seed_value)
def _prepare_seed(use_random_seed: bool, seed_value):
if use_random_seed:
return random.randint(0, 2**32 - 1)
return _coerce_seed(seed_value)
def _on_random_seed_toggle(checked):
return gr.update(interactive=not checked)
def _generate(
text: str,
control_instruction: str,
@@ -355,10 +377,12 @@ def create_demo_interface(demo: VoxCPMDemo):
do_normalize: bool,
denoise: bool,
dit_steps: int,
seed_value,
):
actual_prompt_text = prompt_text_value.strip() if use_prompt_text else ""
actual_control = "" if use_prompt_text else control_instruction
sr, wav_np = demo.generate_tts_audio(
seed = _coerce_seed(seed_value)
sr, wav_np, last_successful_seed = demo.generate_tts_audio(
text_input=text,
control_instruction=actual_control,
reference_wav_path_input=ref_wav,
@@ -367,8 +391,9 @@ def create_demo_interface(demo: VoxCPMDemo):
do_normalize=do_normalize,
denoise=denoise,
inference_timesteps=int(dit_steps),
seed=seed,
)
return (sr, wav_np)
return (sr, wav_np), last_successful_seed
def _on_toggle_instant(checked):
"""Instant UI toggle — no ASR, no blocking."""
@@ -465,6 +490,20 @@ def create_demo_interface(demo: VoxCPMDemo):
label=I18N("dit_steps_label"),
info=I18N("dit_steps_info"),
)
with gr.Row():
seed_value = gr.Number(
value=random.randint(0, 2**32 - 1),
precision=0,
label=I18N("seed_label"),
info=I18N("seed_info"),
interactive=False,
)
random_seed = gr.Checkbox(
value=True,
label=I18N("random_seed_label"),
elem_classes=["switch-toggle"],
info=I18N("random_seed_info"),
)
run_btn = gr.Button(I18N("generate_btn"), variant="primary", size="lg")
@@ -482,7 +521,18 @@ def create_demo_interface(demo: VoxCPMDemo):
outputs=[prompt_text],
)
random_seed.change(
fn=_on_random_seed_toggle,
inputs=[random_seed],
outputs=[seed_value],
)
run_btn.click(
fn=_prepare_seed,
inputs=[random_seed, seed_value],
outputs=[seed_value],
show_progress=False,
).then(
fn=_generate,
inputs=[
text,
@@ -494,14 +544,16 @@ def create_demo_interface(demo: VoxCPMDemo):
DoNormalizeText,
DoDenoisePromptAudio,
dit_steps,
seed_value,
],
outputs=[audio_output],
outputs=[audio_output, seed_value],
show_progress=True,
api_name="generate",
)
return interface
def run_demo(
server_name: str = "0.0.0.0",
server_port: int = 8808,
@@ -523,9 +575,12 @@ def run_demo(
if __name__ == "__main__":
import argparse
parser = argparse.ArgumentParser()
parser.add_argument(
"--model-id", type=str, default="openbmb/VoxCPM2",
"--model-id",
type=str,
default="openbmb/VoxCPM2",
help="Local path or HuggingFace repo ID (default: openbmb/VoxCPM2)",
)
parser.add_argument("--port", type=int, default=8808, help="Server port")

View File

@@ -6,6 +6,7 @@ import gradio as gr
from typing import Optional, Tuple
from funasr import AutoModel
from pathlib import Path
os.environ["TOKENIZERS_PARALLELISM"] = "false"
if os.environ.get("HF_REPO_ID", "").strip() == "":
os.environ["HF_REPO_ID"] = "openbmb/VoxCPM1.5"
@@ -23,7 +24,7 @@ class VoxCPMDemo:
self.asr_model: Optional[AutoModel] = AutoModel(
model=self.asr_model_id,
disable_update=True,
log_level='DEBUG',
log_level="DEBUG",
device="cuda:0" if self.device == "cuda" else "cpu",
)
@@ -48,6 +49,7 @@ class VoxCPMDemo:
if not os.path.isdir(target_dir):
try:
from huggingface_hub import snapshot_download # type: ignore
os.makedirs(target_dir, exist_ok=True)
print(f"Downloading model from HF repo '{repo_id}' to '{target_dir}' ...", file=sys.stderr)
snapshot_download(repo_id=repo_id, local_dir=target_dir, local_dir_use_symlinks=False)
@@ -72,7 +74,7 @@ class VoxCPMDemo:
if prompt_wav is None:
return ""
res = self.asr_model.generate(input=prompt_wav, language="auto", use_itn=True)
text = res[0]["text"].split('|>')[-1]
text = res[0]["text"].split("|>")[-1]
return text
def generate_tts_audio(
@@ -149,11 +151,13 @@ _CUSTOM_CSS = """
def create_demo_interface(demo: VoxCPMDemo):
"""Build the Gradio UI for VoxCPM demo."""
gr.set_static_paths(paths=[Path.cwd().absolute()/"assets"])
gr.set_static_paths(paths=[Path.cwd().absolute() / "assets"])
with gr.Blocks() as interface:
# Header logo
gr.HTML('<div class="logo-container"><img src="/gradio_api/file=assets/voxcpm_logo.png" alt="VoxCPM Logo"></div>')
gr.HTML(
'<div class="logo-container"><img src="/gradio_api/file=assets/voxcpm_logo.png" alt="VoxCPM Logo"></div>'
)
# Quick Start
with gr.Accordion("📋 Quick Start Guide |快速入门", open=False, elem_id="acc_quick"):
@@ -201,7 +205,7 @@ def create_demo_interface(demo: VoxCPMDemo):
with gr.Row():
with gr.Column():
prompt_wav = gr.Audio(
sources=["upload", 'microphone'],
sources=["upload", "microphone"],
type="filepath",
label="Prompt Speech (Optional, or let VoxCPM improvise)",
value="./examples/example.wav",
@@ -210,13 +214,13 @@ def create_demo_interface(demo: VoxCPMDemo):
value=False,
label="Prompt Speech Enhancement",
elem_id="chk_denoise",
info="We use ZipEnhancer model to denoise the prompt audio."
info="We use ZipEnhancer model to denoise the prompt audio.",
)
with gr.Row():
prompt_text = gr.Textbox(
value="Just by listening a few minutes a day, you'll be able to eliminate negative thoughts by conditioning your mind to be more positive.",
label="Prompt Text",
placeholder="Please enter the prompt text. Automatic recognition is supported, and you can correct the results yourself..."
placeholder="Please enter the prompt text. Automatic recognition is supported, and you can correct the results yourself...",
)
run_btn = gr.Button("Generate Speech", variant="primary")
@@ -227,7 +231,7 @@ def create_demo_interface(demo: VoxCPMDemo):
value=2.0,
step=0.1,
label="CFG Value (Guidance Scale)",
info="Higher values increase adherence to prompt, lower values allow more creativity"
info="Higher values increase adherence to prompt, lower values allow more creativity",
)
inference_timesteps = gr.Slider(
minimum=4,
@@ -235,7 +239,7 @@ def create_demo_interface(demo: VoxCPMDemo):
value=10,
step=1,
label="Inference Timesteps",
info="Number of inference timesteps for generation (higher values may improve quality but slower)"
info="Number of inference timesteps for generation (higher values may improve quality but slower)",
)
with gr.Row():
text = gr.Textbox(
@@ -247,14 +251,22 @@ def create_demo_interface(demo: VoxCPMDemo):
value=False,
label="Text Normalization",
elem_id="chk_normalize",
info="We use wetext library to normalize the input text."
info="We use wetext library to normalize the input text.",
)
audio_output = gr.Audio(label="Output Audio")
# Wiring
run_btn.click(
fn=demo.generate_tts_audio,
inputs=[text, prompt_wav, prompt_text, cfg_value, inference_timesteps, DoNormalizeText, DoDenoisePromptAudio],
inputs=[
text,
prompt_wav,
prompt_text,
cfg_value,
inference_timesteps,
DoNormalizeText,
DoDenoisePromptAudio,
],
outputs=[audio_output],
show_progress=True,
api_name="generate",
@@ -277,4 +289,4 @@ def run_demo(server_name: str = "localhost", server_port: int = 7860, show_error
if __name__ == "__main__":
run_demo()
run_demo()

View File

@@ -308,7 +308,8 @@ def run_inference(text, prompt_wav, prompt_text, lora_selection, cfg_scale, step
if new_r is not None and current_r is not None and new_r != current_r:
print(f"LoRA rank mismatch (model r={current_r}, checkpoint r={new_r}), reloading...", file=sys.stderr)
reload_base = (
new_base_model if new_base_model and os.path.exists(new_base_model)
new_base_model
if new_base_model and os.path.exists(new_base_model)
else (pretrained_path if pretrained_path and pretrained_path.strip() else default_pretrained_path)
)
try:
@@ -982,9 +983,7 @@ with gr.Blocks(title="VoxCPM LoRA WebUI", theme=gr.themes.Soft(), css=custom_css
gr.Markdown("#### 分发选项 (Distribution)")
with gr.Row():
hf_model_id = gr.Textbox(
label="HuggingFace Model ID (e.g., openbmb/VoxCPM2)", value=""
)
hf_model_id = gr.Textbox(label="HuggingFace Model ID (e.g., openbmb/VoxCPM2)", value="")
distribute = gr.Checkbox(label="分发模式 (distribute)", value=False)
with gr.Column(scale=2, elem_classes="form-section"):

View File

@@ -3,6 +3,7 @@
Loads src/voxcpm/model/utils.py directly to avoid the heavy voxcpm package
init. Run with: `python scripts/test_pick_runtime_dtype.py`.
"""
import importlib.util
import os
import pathlib

View File

@@ -19,6 +19,7 @@ With voice cloning:
--text "Hello, this is voice cloning result." \
--prompt_audio path/to/ref.wav \
--prompt_text "Reference audio transcript" \
--seed 42 \
--output ft_clone.wav
"""
@@ -86,6 +87,12 @@ def parse_args():
action="store_true",
help="Enable text normalization",
)
parser.add_argument(
"--seed",
type=int,
default=None,
help="Random seed for generation (default: None)",
)
return parser.parse_args()
@@ -109,6 +116,9 @@ def main():
print(f"[FT Inference] Using reference audio: {prompt_wav_path}", file=sys.stderr)
print(f"[FT Inference] Reference text: {prompt_text}", file=sys.stderr)
if args.seed is not None:
print(f"[FT Inference] Using seed: {args.seed}", file=sys.stderr)
audio_np = model.generate(
text=args.text,
prompt_wav_path=prompt_wav_path,
@@ -118,6 +128,7 @@ def main():
max_len=args.max_len,
normalize=args.normalize,
denoise=False,
seed=args.seed,
)
# Save audio

View File

@@ -16,6 +16,7 @@ With voice cloning:
--text "This is voice cloning result." \
--prompt_audio path/to/ref.wav \
--prompt_text "Reference audio transcript" \
--seed 42 \
--output lora_clone.wav
Note: The script reads base_model path and lora_config from lora_config.json
@@ -94,6 +95,12 @@ def parse_args():
action="store_true",
help="Enable text normalization",
)
parser.add_argument(
"--seed",
type=int,
default=None,
help="Random seed for generation (default: None)",
)
return parser.parse_args()
@@ -130,6 +137,8 @@ def main():
print(
f" LoRA config: r={lora_cfg.r}, alpha={lora_cfg.alpha}" if lora_cfg else " LoRA config: None", file=sys.stderr
)
if args.seed is not None:
print(f" Seed: {args.seed}", file=sys.stderr)
# 3. Load model with LoRA (no denoiser)
print(f"\n[1/2] Loading model with LoRA: {pretrained_path}", file=sys.stderr)
@@ -161,6 +170,7 @@ def main():
max_len=args.max_len,
normalize=args.normalize,
denoise=False,
seed=args.seed,
)
lora_output = out_path.with_stem(out_path.stem + "_with_lora")
sf.write(str(lora_output), audio_np, model.tts_model.sample_rate)
@@ -181,6 +191,7 @@ def main():
max_len=args.max_len,
normalize=args.normalize,
denoise=False,
seed=args.seed,
)
disabled_output = out_path.with_stem(out_path.stem + "_lora_disabled")
sf.write(str(disabled_output), audio_np, model.tts_model.sample_rate)
@@ -201,6 +212,7 @@ def main():
max_len=args.max_len,
normalize=args.normalize,
denoise=False,
seed=args.seed,
)
reenabled_output = out_path.with_stem(out_path.stem + "_lora_reenabled")
sf.write(str(reenabled_output), audio_np, model.tts_model.sample_rate)
@@ -221,6 +233,7 @@ def main():
max_len=args.max_len,
normalize=args.normalize,
denoise=False,
seed=args.seed,
)
reset_output = out_path.with_stem(out_path.stem + "_lora_reset")
sf.write(str(reset_output), audio_np, model.tts_model.sample_rate)
@@ -242,6 +255,7 @@ def main():
max_len=args.max_len,
normalize=args.normalize,
denoise=False,
seed=args.seed,
)
reload_output = out_path.with_stem(out_path.stem + "_lora_reloaded")
sf.write(str(reload_output), audio_np, model.tts_model.sample_rate)

View File

@@ -599,7 +599,9 @@ def generate_sample_audio(
)
with torch.no_grad():
with autocast_ctx:
generated = unwrapped_model.generate(target_text=text, inference_timesteps=10, cfg_value=2.0)
generated = unwrapped_model.generate(
target_text=text, inference_timesteps=10, cfg_value=2.0, seed=42
)
# Restore training setup
# unwrapped_model.to(torch.float32)

View File

@@ -248,6 +248,7 @@ def _run_single(args, parser, *, text: str, output: str, prompt_text: str | None
inference_timesteps=args.inference_timesteps,
normalize=args.normalize,
denoise=args.denoise and (args.prompt_audio is not None or args.reference_audio is not None),
seed=args.seed,
)
import soundfile as sf
@@ -332,6 +333,7 @@ def cmd_batch(args, parser):
inference_timesteps=args.inference_timesteps,
normalize=args.normalize,
denoise=args.denoise and (prompt_audio_path is not None or reference_audio_path is not None),
seed=args.seed,
)
output_file = output_dir / f"output_{i:03d}.wav"
@@ -413,6 +415,12 @@ def _add_common_generation_args(parser):
help="Inference steps (int, recommended 430, default: 10)",
)
parser.add_argument("--normalize", action="store_true", help="Enable text normalization")
parser.add_argument(
"--seed",
type=int,
default=None,
help="Random seed for generation (default: None)",
)
def _add_prompt_reference_args(parser):
@@ -587,6 +595,12 @@ Examples:
help="Inference steps (int, recommended 430, default: 10)",
)
batch_parser.add_argument("--normalize", action="store_true", help="Enable text normalization")
batch_parser.add_argument(
"--seed",
type=int,
default=None,
help="Random seed for generation (default: None)",
)
_add_model_args(batch_parser)
_add_lora_args(batch_parser)
_add_timestamp_args(batch_parser, include_output=False)

View File

@@ -196,6 +196,7 @@ class VoxCPM:
retry_badcase_max_times: int = 3,
retry_badcase_ratio_threshold: float = 6.0,
streaming: bool = False,
seed: Optional[int] = None,
) -> Generator[np.ndarray, None, None]:
"""Synthesize speech for the given text and return a single waveform.
@@ -218,6 +219,7 @@ class VoxCPM:
retry_badcase_max_times: Maximum number of times to retry badcase.
retry_badcase_ratio_threshold: Threshold for audio-to-text ratio.
streaming: Whether to return a generator of audio chunks.
seed: Optional random seed for reproducibility.
Returns:
Generator of numpy.ndarray: 1D waveform array (float32) on CPU.
Yields audio chunks for each generation step if ``streaming=True``,
@@ -294,6 +296,7 @@ class VoxCPM:
retry_badcase_max_times=retry_badcase_max_times,
retry_badcase_ratio_threshold=retry_badcase_ratio_threshold,
streaming=streaming,
seed=seed,
)
if streaming:

View File

@@ -5,9 +5,12 @@ from transformers import PreTrainedTokenizer
_LOW_PRECISION_DTYPES = {"bfloat16", "bf16", "float16", "fp16"}
_VALID_DTYPE_OVERRIDES = {
"bfloat16", "bf16",
"float16", "fp16",
"float32", "fp32",
"bfloat16",
"bf16",
"float16",
"fp16",
"float32",
"fp32",
}
@@ -21,6 +24,19 @@ def next_and_close(gen):
gen.close()
def materialize_generation_seed(seed: Optional[int]) -> int:
"""Return a concrete seed for a generation request."""
if seed is not None:
return int(seed)
return int(torch.seed() & 0xFFFFFFFF)
def apply_generation_seed(seed: int) -> None:
torch.manual_seed(seed)
if torch.cuda.is_available():
torch.cuda.manual_seed_all(seed)
def mask_multichar_chinese_tokens(tokenizer: PreTrainedTokenizer):
"""Create a tokenizer wrapper that converts multi-character Chinese tokens to single characters.
@@ -160,10 +176,7 @@ def pick_runtime_dtype(device: str, configured_dtype: str) -> str:
override = os.environ.get("VOXCPM_MPS_DTYPE", "").strip().lower()
if override:
if override not in _VALID_DTYPE_OVERRIDES:
raise ValueError(
f"VOXCPM_MPS_DTYPE='{override}' is not one of "
f"{sorted(_VALID_DTYPE_OVERRIDES)}"
)
raise ValueError(f"VOXCPM_MPS_DTYPE='{override}' is not one of " f"{sorted(_VALID_DTYPE_OVERRIDES)}")
return override
if (configured_dtype or "").lower() in _LOW_PRECISION_DTYPES:
@@ -211,15 +224,13 @@ def resolve_runtime_device(device: Optional[str], configured_device: str = "cuda
if explicit.startswith("cuda"):
if not torch.cuda.is_available():
raise ValueError(
f"Requested device '{device}', but CUDA is not available. "
"Use device='auto' for automatic fallback."
f"Requested device '{device}', but CUDA is not available. " "Use device='auto' for automatic fallback."
)
return explicit
if explicit == "mps":
if not _has_mps():
raise ValueError(
"Requested device 'mps', but MPS is not available. "
"Use device='auto' for automatic fallback."
"Requested device 'mps', but MPS is not available. " "Use device='auto' for automatic fallback."
)
return "mps"
if explicit == "cpu":

View File

@@ -45,7 +45,9 @@ from ..modules.locdit import CfmConfig, UnifiedCFM, VoxCPMLocDiT
from ..modules.locenc import VoxCPMLocEnc
from ..modules.minicpm4 import MiniCPM4Config, MiniCPMModel
from .utils import (
apply_generation_seed,
get_dtype,
materialize_generation_seed,
mask_multichar_chinese_tokens,
next_and_close,
pick_runtime_dtype,
@@ -140,6 +142,7 @@ class VoxCPMModel(nn.Module):
self.text_tokenizer = mask_multichar_chinese_tokens(tokenizer)
self.audio_start_token = 101
self.audio_end_token = 102
self.last_successful_seed = None
# Residual Acoustic LM
residual_lm_config = config.lm_config.model_copy(deep=True)
@@ -367,6 +370,7 @@ class VoxCPMModel(nn.Module):
retry_badcase_max_times: int = 3,
retry_badcase_ratio_threshold: float = 6.0, # setting acceptable ratio of audio length to text length (for badcase detection)
streaming: bool = False,
seed: Optional[int] = None,
) -> Generator[torch.Tensor, None, None]:
if retry_badcase and streaming:
warnings.warn("Retry on bad cases is not supported in streaming mode, setting retry_badcase=False.")
@@ -452,7 +456,12 @@ class VoxCPMModel(nn.Module):
target_text_length = len(self.text_tokenizer(target_text))
retry_badcase_times = 0
current_seed = materialize_generation_seed(seed)
last_attempt_seed = current_seed
while retry_badcase_times < retry_badcase_max_times:
last_attempt_seed = current_seed
apply_generation_seed(last_attempt_seed)
inference_result = self._inference(
text_token,
text_mask,
@@ -471,6 +480,7 @@ class VoxCPMModel(nn.Module):
for latent_pred, _ in inference_result:
decode_audio = self.audio_vae.decode(latent_pred.to(torch.float32))
decode_audio = decode_audio[..., -patch_len:].squeeze(1).cpu()
self.last_successful_seed = last_attempt_seed
yield decode_audio
break
else:
@@ -482,6 +492,7 @@ class VoxCPMModel(nn.Module):
file=sys.stderr,
)
retry_badcase_times += 1
current_seed += 1
continue
else:
break
@@ -489,6 +500,7 @@ class VoxCPMModel(nn.Module):
break
if not streaming:
self.last_successful_seed = last_attempt_seed
decode_audio = self.audio_vae.decode(latent_pred.to(torch.float32)).squeeze(1).cpu()
yield decode_audio
@@ -603,6 +615,7 @@ class VoxCPMModel(nn.Module):
retry_badcase_ratio_threshold: float = 6.0,
streaming: bool = False,
streaming_prefix_len: int = 3,
seed: Optional[int] = None,
) -> Generator[Tuple[torch.Tensor, torch.Tensor, Union[torch.Tensor, List[torch.Tensor]]], None, None]:
"""
Generate audio using pre-built prompt cache.
@@ -678,7 +691,12 @@ class VoxCPMModel(nn.Module):
# run inference
target_text_length = len(self.text_tokenizer(target_text))
retry_badcase_times = 0
current_seed = materialize_generation_seed(seed)
last_attempt_seed = current_seed
while retry_badcase_times < retry_badcase_max_times:
last_attempt_seed = current_seed
apply_generation_seed(last_attempt_seed)
inference_result = self._inference(
text_token,
text_mask,
@@ -698,6 +716,7 @@ class VoxCPMModel(nn.Module):
for latent_pred, pred_audio_feat in inference_result:
decode_audio = self.audio_vae.decode(latent_pred.to(torch.float32))
decode_audio = decode_audio[..., -patch_len:].squeeze(1).cpu()
self.last_successful_seed = last_attempt_seed
yield (decode_audio, target_text_token, pred_audio_feat)
break
else:
@@ -709,12 +728,14 @@ class VoxCPMModel(nn.Module):
file=sys.stderr,
)
retry_badcase_times += 1
current_seed += 1
continue
else:
break
else:
break
if not streaming:
self.last_successful_seed = last_attempt_seed
decode_audio = self.audio_vae.decode(latent_pred.to(torch.float32))
patch_len = self.patch_size * self.chunk_size
if audio_mask.sum().item() > 0:

View File

@@ -46,7 +46,9 @@ from ..modules.locdit import CfmConfig, UnifiedCFM, VoxCPMLocDiTV2
from ..modules.locenc import VoxCPMLocEnc
from ..modules.minicpm4 import MiniCPM4Config, MiniCPMModel
from .utils import (
apply_generation_seed,
get_dtype,
materialize_generation_seed,
mask_multichar_chinese_tokens,
next_and_close,
pick_runtime_dtype,
@@ -55,7 +57,9 @@ from .utils import (
# A simple function to trim audio silence using VAD, not used default
def _trim_audio_silence_vad(audio: torch.Tensor, sample_rate: int, max_silence_ms: float = 200.0, top_db: float = 35.0) -> torch.Tensor:
def _trim_audio_silence_vad(
audio: torch.Tensor, sample_rate: int, max_silence_ms: float = 200.0, top_db: float = 35.0
) -> torch.Tensor:
if audio.numel() == 0:
return audio
y = audio.squeeze(0).numpy()
@@ -184,6 +188,7 @@ class VoxCPM2Model(nn.Module):
self.audio_end_token = 102
self.ref_audio_start_token = 103
self.ref_audio_end_token = 104
self.last_successful_seed = None
# Residual Acoustic LM
residual_lm_config = config.lm_config.model_copy(deep=True)
@@ -476,6 +481,7 @@ class VoxCPM2Model(nn.Module):
trim_silence_vad: bool = False,
streaming: bool = False,
streaming_prefix_len: int = 4,
seed: Optional[int] = None,
) -> Generator[torch.Tensor, None, None]:
if retry_badcase and streaming:
warnings.warn("Retry on bad cases is not supported in streaming mode, setting retry_badcase=False.")
@@ -633,7 +639,12 @@ class VoxCPM2Model(nn.Module):
target_text_length = len(self.text_tokenizer(target_text))
retry_badcase_times = 0
current_seed = materialize_generation_seed(seed)
last_attempt_seed = current_seed
while retry_badcase_times < retry_badcase_max_times:
last_attempt_seed = current_seed
apply_generation_seed(last_attempt_seed)
inference_result = self._inference(
text_token,
text_mask,
@@ -651,6 +662,7 @@ class VoxCPM2Model(nn.Module):
for latent_pred, _, _ctx in inference_result:
decode_audio = vae_dec.decode_chunk(latent_pred.to(torch.float32))
decode_audio = decode_audio.squeeze(1).cpu()
self.last_successful_seed = last_attempt_seed
yield decode_audio
break
else:
@@ -662,6 +674,7 @@ class VoxCPM2Model(nn.Module):
file=sys.stderr,
)
retry_badcase_times += 1
current_seed += 1
continue
else:
break
@@ -669,10 +682,11 @@ class VoxCPM2Model(nn.Module):
break
if not streaming:
self.last_successful_seed = last_attempt_seed
decode_audio = self.audio_vae.decode(latent_pred.to(torch.float32))
decode_patch_len = self.patch_size * self._decode_chunk_size
if context_len > 0:
decode_audio = decode_audio[..., decode_patch_len * context_len:].squeeze(1).cpu()
decode_audio = decode_audio[..., decode_patch_len * context_len :].squeeze(1).cpu()
else:
decode_audio = decode_audio.squeeze(1).cpu()
yield decode_audio
@@ -793,6 +807,7 @@ class VoxCPM2Model(nn.Module):
retry_badcase_ratio_threshold: float = 6.0,
streaming: bool = False,
streaming_prefix_len: int = 4,
seed: Optional[int] = None,
) -> Generator[Tuple[torch.Tensor, torch.Tensor, Union[torch.Tensor, List[torch.Tensor]]], None, None]:
"""
Generate audio using pre-built prompt cache.
@@ -920,7 +935,12 @@ class VoxCPM2Model(nn.Module):
# run inference
target_text_length = len(self.text_tokenizer(target_text))
retry_badcase_times = 0
current_seed = materialize_generation_seed(seed)
last_attempt_seed = current_seed
while retry_badcase_times < retry_badcase_max_times:
last_attempt_seed = current_seed
apply_generation_seed(last_attempt_seed)
inference_result = self._inference(
text_token,
text_mask,
@@ -938,6 +958,7 @@ class VoxCPM2Model(nn.Module):
for latent_pred, pred_audio_feat, _ctx in inference_result:
decode_audio = vae_dec.decode_chunk(latent_pred.to(torch.float32))
decode_audio = decode_audio.squeeze(1).cpu()
self.last_successful_seed = last_attempt_seed
yield (decode_audio, target_text_token, pred_audio_feat)
break
else:
@@ -949,16 +970,18 @@ class VoxCPM2Model(nn.Module):
file=sys.stderr,
)
retry_badcase_times += 1
current_seed += 1
continue
else:
break
else:
break
if not streaming:
self.last_successful_seed = last_attempt_seed
decode_audio = self.audio_vae.decode(latent_pred.to(torch.float32))
decode_patch_len = self.patch_size * self._decode_chunk_size
if context_len > 0:
decode_audio = decode_audio[..., decode_patch_len * context_len:].squeeze(1).cpu()
decode_audio = decode_audio[..., decode_patch_len * context_len :].squeeze(1).cpu()
else:
decode_audio = decode_audio.squeeze(1).cpu()
yield (decode_audio, target_text_token, pred_audio_feat)

View File

@@ -551,8 +551,7 @@ class StreamingVAEDecoder:
if x.shape[-1] >= _p:
states[_k] = x[:, :, -_p:].detach()
else:
prev = states.get(_k, torch.zeros(x.shape[0], x.shape[1], _p,
device=x.device, dtype=x.dtype))
prev = states.get(_k, torch.zeros(x.shape[0], x.shape[1], _p, device=x.device, dtype=x.dtype))
states[_k] = torch.cat([prev, x], dim=-1)[:, :, -_p:].detach()
return nn.Conv1d.forward(_m, x_pad)

View File

@@ -350,45 +350,80 @@ class AudioFeatureProcessingPacker:
# -- text token track --
# [103, 0×R, 104, text_ids, 101, 0×A, 102]
text_token_info = torch.cat([
_tok([self.audio_prompt_start_id]),
torch.zeros(ref_len, dtype=torch.int32, device=device),
_tok([self.audio_prompt_end_id]),
text_token,
_tok([self.audio_start_id]),
torch.zeros(tgt_len, dtype=torch.int32, device=device),
_tok([self.audio_end_id]),
])
text_token_info = torch.cat(
[
_tok([self.audio_prompt_start_id]),
torch.zeros(ref_len, dtype=torch.int32, device=device),
_tok([self.audio_prompt_end_id]),
text_token,
_tok([self.audio_start_id]),
torch.zeros(tgt_len, dtype=torch.int32, device=device),
_tok([self.audio_end_id]),
]
)
# -- audio feature track --
zero_1 = torch.zeros((1,) + feat_shape, dtype=torch.float32, device=device)
zero_txt = torch.zeros((txt_len,) + feat_shape, dtype=torch.float32, device=device)
audio_feat_info = torch.cat([
zero_1, ref_feats, zero_1, # 103, ref, 104
zero_txt, # text
zero_1, tgt_feats, zero_1, # 101, target, 102
], dim=0)
audio_feat_info = torch.cat(
[
zero_1,
ref_feats,
zero_1, # 103, ref, 104
zero_txt, # text
zero_1,
tgt_feats,
zero_1, # 101, target, 102
],
dim=0,
)
# -- masks --
text_mask = torch.cat([
torch.ones(1), torch.zeros(ref_len), torch.ones(1),
torch.ones(txt_len),
torch.ones(1), torch.zeros(tgt_len), torch.ones(1),
]).to(torch.int32).to(device)
text_mask = (
torch.cat(
[
torch.ones(1),
torch.zeros(ref_len),
torch.ones(1),
torch.ones(txt_len),
torch.ones(1),
torch.zeros(tgt_len),
torch.ones(1),
]
)
.to(torch.int32)
.to(device)
)
audio_mask = torch.cat([
torch.zeros(1), torch.ones(ref_len), torch.zeros(1),
torch.zeros(txt_len),
torch.zeros(1), torch.ones(tgt_len), torch.zeros(1),
]).to(torch.int32).to(device)
audio_mask = (
torch.cat(
[
torch.zeros(1),
torch.ones(ref_len),
torch.zeros(1),
torch.zeros(txt_len),
torch.zeros(1),
torch.ones(tgt_len),
torch.zeros(1),
]
)
.to(torch.int32)
.to(device)
)
loss_mask = torch.cat([
torch.zeros(1 + ref_len + 1), # ref part: no loss
torch.zeros(txt_len), # text: no loss
torch.zeros(1), # 101: no loss
torch.ones(tgt_len), # target audio: LOSS
torch.zeros(1), # 102: no loss
]).to(torch.int32).to(device)
loss_mask = (
torch.cat(
[
torch.zeros(1 + ref_len + 1), # ref part: no loss
torch.zeros(txt_len), # text: no loss
torch.zeros(1), # 101: no loss
torch.ones(tgt_len), # target audio: LOSS
torch.zeros(1), # 102: no loss
]
)
.to(torch.int32)
.to(device)
)
total_len = 1 + ref_len + 1 + txt_len + 1 + tgt_len + 1
labels = torch.zeros(total_len, dtype=torch.int32, device=device)

View File

@@ -44,10 +44,7 @@ def _check_audio_file(audio_path: str, sample_rate: int) -> Optional[str]:
if info.frames == 0:
return f"Audio file is empty: {audio_path}"
if info.samplerate != sample_rate:
return (
f"Sample rate mismatch in {audio_path}: "
f"expected {sample_rate} Hz, got {info.samplerate} Hz"
)
return f"Sample rate mismatch in {audio_path}: " f"expected {sample_rate} Hz, got {info.samplerate} Hz"
return None
except ImportError:
# soundfile not available; just check existence
@@ -187,13 +184,9 @@ def validate_manifest(
if duration is not None:
result.audio_durations.append(duration)
if duration < 0.3:
result.warnings.append(
f"Line {i + 1}: Very short audio ({duration:.2f}s)"
)
result.warnings.append(f"Line {i + 1}: Very short audio ({duration:.2f}s)")
elif duration > 30.0:
result.warnings.append(
f"Line {i + 1}: Very long audio ({duration:.1f}s), may cause OOM"
)
result.warnings.append(f"Line {i + 1}: Very long audio ({duration:.1f}s), may cause OOM")
else:
result.errors.append(f"Line {i + 1}: Invalid audio path")
has_error = True
@@ -209,9 +202,7 @@ def validate_manifest(
if os.path.isfile(ref_path):
result.has_ref_audio += 1
else:
result.warnings.append(
f"Line {i + 1}: ref_audio file not found: {ref_path}"
)
result.warnings.append(f"Line {i + 1}: ref_audio file not found: {ref_path}")
if not has_error:
result.valid_samples += 1
@@ -222,14 +213,10 @@ def validate_manifest(
# Summarize truncated errors
if missing_audio_count > 5:
result.errors.append(
f"... and {missing_audio_count - 5} more missing audio files "
f"({missing_audio_count} total)"
f"... and {missing_audio_count - 5} more missing audio files " f"({missing_audio_count} total)"
)
if empty_text_count > 5:
result.warnings.append(
f"... and {empty_text_count - 5} more empty text entries "
f"({empty_text_count} total)"
)
result.warnings.append(f"... and {empty_text_count - 5} more empty text entries " f"({empty_text_count} total)")
return result

View File

@@ -676,3 +676,61 @@ def test_detect_model_architecture_uses_local_configs():
assert cli.detect_model_architecture(v1_args) == "voxcpm"
assert cli.detect_model_architecture(v2_args) == "voxcpm2"
def test_parser_accepts_seed():
parser = cli._build_parser()
# Default seed should be None
args = parser.parse_args(["design", "--text", "hello", "--output", "out.wav"])
assert args.seed is None
# Custom seed should be parsed as int
args = parser.parse_args(["design", "--text", "hello", "--output", "out.wav", "--seed", "42"])
assert args.seed == 42
def test_design_subcommand_passes_seed(monkeypatch, tmp_path):
dummy_model = DummyModel()
monkeypatch.setattr(cli, "load_model", lambda args: dummy_model)
patch_soundfile_write(monkeypatch)
run_main(
monkeypatch,
[
"design",
"--text",
"hello",
"--seed",
"123",
"--output",
str(tmp_path / "out.wav"),
],
)
assert dummy_model.calls[0]["seed"] == 123
def test_batch_subcommand_passes_seed(monkeypatch, tmp_path):
dummy_model = DummyModel()
input_file = tmp_path / "texts.txt"
input_file.write_text("hello\nworld\n", encoding="utf-8")
monkeypatch.setattr(cli, "load_model", lambda args: dummy_model)
patch_soundfile_write(monkeypatch)
run_main(
monkeypatch,
[
"batch",
"--input",
str(input_file),
"--output-dir",
str(tmp_path / "outs"),
"--seed",
"999",
],
)
assert len(dummy_model.calls) == 2
assert dummy_model.calls[0]["seed"] == 999
assert dummy_model.calls[1]["seed"] == 999

View File

@@ -47,3 +47,25 @@ def test_resolve_runtime_device_rejects_unavailable_explicit_cuda(monkeypatch):
with pytest.raises(ValueError, match="CUDA is not available"):
utils.resolve_runtime_device("cuda:0", "cuda")
def test_materialize_generation_seed_preserves_explicit_seed():
assert utils.materialize_generation_seed(42) == 42
def test_materialize_generation_seed_creates_concrete_seed_for_none(monkeypatch):
monkeypatch.setattr(utils.torch, "seed", lambda: 0x123456789)
assert utils.materialize_generation_seed(None) == 0x23456789
def test_apply_generation_seed_sets_cpu_and_cuda_rng(monkeypatch):
calls = []
monkeypatch.setattr(utils.torch, "manual_seed", lambda seed: calls.append(("cpu", seed)))
monkeypatch.setattr(utils.torch.cuda, "is_available", lambda: True)
monkeypatch.setattr(utils.torch.cuda, "manual_seed_all", lambda seed: calls.append(("cuda", seed)))
utils.apply_generation_seed(123)
assert calls == [("cpu", 123), ("cuda", 123)]

View File

@@ -207,6 +207,7 @@ class TestValidateManifest:
audio = tmp_dir / "audio_8k.wav"
import numpy as np
samples = np.zeros(8000, dtype=np.float32)
sf.write(str(audio), samples, 8000)
@@ -240,6 +241,7 @@ class TestValidateManifest:
def test_cli_validate_exit_code(self, tmp_dir):
"""validate subcommand must exit 1 on validation error (missing audio)."""
import subprocess
manifest = tmp_dir / "bad.jsonl"
_write_manifest(manifest, [{"text": "hi", "audio": "/nonexistent/x.wav"}])