mirror of
https://github.com/OpenBMB/VoxCPM.git
synced 2026-09-20 10:53:32 +08:00
Merge pull request #327 from DEVAIEXP/feat-add-seed
feat: Add Seed Support for Reproducible Voice Generation (v1 & v2) Resolved conflict in src/voxcpm/cli.py: kept both --seed (PR #327) and timestamp alignment args (PR #332).
This commit is contained in:
@@ -108,6 +108,7 @@ wav = model.generate(
|
|||||||
text="VoxCPM2 is the current recommended release for realistic multilingual speech synthesis.",
|
text="VoxCPM2 is the current recommended release for realistic multilingual speech synthesis.",
|
||||||
cfg_value=2.0,
|
cfg_value=2.0,
|
||||||
inference_timesteps=10,
|
inference_timesteps=10,
|
||||||
|
seed=42,
|
||||||
)
|
)
|
||||||
sf.write("demo.wav", wav, model.tts_model.sample_rate)
|
sf.write("demo.wav", wav, model.tts_model.sample_rate)
|
||||||
print("saved: demo.wav")
|
print("saved: demo.wav")
|
||||||
@@ -131,6 +132,7 @@ wav = model.generate(
|
|||||||
text="VoxCPM2 is the current recommended release for realistic multilingual speech synthesis.",
|
text="VoxCPM2 is the current recommended release for realistic multilingual speech synthesis.",
|
||||||
cfg_value=2.0,
|
cfg_value=2.0,
|
||||||
inference_timesteps=10,
|
inference_timesteps=10,
|
||||||
|
seed=42,
|
||||||
)
|
)
|
||||||
sf.write("demo.wav", wav, model.tts_model.sample_rate)
|
sf.write("demo.wav", wav, model.tts_model.sample_rate)
|
||||||
```
|
```
|
||||||
@@ -144,6 +146,7 @@ wav = model.generate(
|
|||||||
text="(A young woman, gentle and sweet voice)Hello, welcome to VoxCPM2!",
|
text="(A young woman, gentle and sweet voice)Hello, welcome to VoxCPM2!",
|
||||||
cfg_value=2.0,
|
cfg_value=2.0,
|
||||||
inference_timesteps=10,
|
inference_timesteps=10,
|
||||||
|
seed=42,
|
||||||
)
|
)
|
||||||
sf.write("voice_design.wav", wav, model.tts_model.sample_rate)
|
sf.write("voice_design.wav", wav, model.tts_model.sample_rate)
|
||||||
```
|
```
|
||||||
@@ -164,6 +167,7 @@ wav = model.generate(
|
|||||||
reference_wav_path="path/to/voice.wav",
|
reference_wav_path="path/to/voice.wav",
|
||||||
cfg_value=2.0,
|
cfg_value=2.0,
|
||||||
inference_timesteps=10,
|
inference_timesteps=10,
|
||||||
|
seed=42,
|
||||||
)
|
)
|
||||||
sf.write("controllable_clone.wav", wav, model.tts_model.sample_rate)
|
sf.write("controllable_clone.wav", wav, model.tts_model.sample_rate)
|
||||||
```
|
```
|
||||||
@@ -210,6 +214,7 @@ voxcpm design \
|
|||||||
voxcpm design \
|
voxcpm design \
|
||||||
--text "VoxCPM2 brings studio-quality multilingual speech synthesis." \
|
--text "VoxCPM2 brings studio-quality multilingual speech synthesis." \
|
||||||
--control "Young female voice, warm and gentle, slightly smiling" \
|
--control "Young female voice, warm and gentle, slightly smiling" \
|
||||||
|
--seed 42 \
|
||||||
--output out.wav
|
--output out.wav
|
||||||
|
|
||||||
# Voice cloning (reference audio)
|
# Voice cloning (reference audio)
|
||||||
|
|||||||
@@ -10,7 +10,7 @@
|
|||||||
<a href="https://voxcpm.readthedocs.io/zh-cn/latest/"><img src="https://img.shields.io/badge/Docs-ReadTheDocs-8CA1AF" alt="Documentation"></a>
|
<a href="https://voxcpm.readthedocs.io/zh-cn/latest/"><img src="https://img.shields.io/badge/Docs-ReadTheDocs-8CA1AF" alt="Documentation"></a>
|
||||||
<a href="https://huggingface.co/openbmb/VoxCPM2"><img src="https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-VoxCPM2-yellow" alt="Hugging Face"></a>
|
<a href="https://huggingface.co/openbmb/VoxCPM2"><img src="https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-VoxCPM2-yellow" alt="Hugging Face"></a>
|
||||||
<a href="https://modelscope.cn/models/OpenBMB/VoxCPM2"><img src="https://img.shields.io/badge/ModelScope-VoxCPM2-purple" alt="ModelScope"></a>
|
<a href="https://modelscope.cn/models/OpenBMB/VoxCPM2"><img src="https://img.shields.io/badge/ModelScope-VoxCPM2-purple" alt="ModelScope"></a>
|
||||||
<a href="https://openbmb.github.io/voxcpm2-demopage/"><img src="https://img.shields.io/badge/DemoPage-Audio Samples-red"></a>
|
<a href="https://openbmb.github.io/voxcpm2-demopage/"><img src="https://img.shields.io/badge/DemoPage-Audio Samples-red" alt="DemoPage"></a>
|
||||||
|
|
||||||
</p>
|
</p>
|
||||||
|
|
||||||
@@ -110,6 +110,7 @@ wav = model.generate(
|
|||||||
text="VoxCPM2 是目前推荐使用的多语言语音合成版本。",
|
text="VoxCPM2 是目前推荐使用的多语言语音合成版本。",
|
||||||
cfg_value=2.0,
|
cfg_value=2.0,
|
||||||
inference_timesteps=10,
|
inference_timesteps=10,
|
||||||
|
seed=42,
|
||||||
)
|
)
|
||||||
sf.write("demo.wav", wav, model.tts_model.sample_rate)
|
sf.write("demo.wav", wav, model.tts_model.sample_rate)
|
||||||
print("已保存: demo.wav")
|
print("已保存: demo.wav")
|
||||||
@@ -133,6 +134,7 @@ wav = model.generate(
|
|||||||
text="VoxCPM2 是目前推荐使用的多语言语音合成版本。",
|
text="VoxCPM2 是目前推荐使用的多语言语音合成版本。",
|
||||||
cfg_value=2.0,
|
cfg_value=2.0,
|
||||||
inference_timesteps=10,
|
inference_timesteps=10,
|
||||||
|
seed=42,
|
||||||
)
|
)
|
||||||
sf.write("demo.wav", wav, model.tts_model.sample_rate)
|
sf.write("demo.wav", wav, model.tts_model.sample_rate)
|
||||||
```
|
```
|
||||||
@@ -146,6 +148,7 @@ wav = model.generate(
|
|||||||
text="(年轻女性,声音温柔甜美)你好,欢迎使用VoxCPM2!",
|
text="(年轻女性,声音温柔甜美)你好,欢迎使用VoxCPM2!",
|
||||||
cfg_value=2.0,
|
cfg_value=2.0,
|
||||||
inference_timesteps=10,
|
inference_timesteps=10,
|
||||||
|
seed=42,
|
||||||
)
|
)
|
||||||
sf.write("voice_design.wav", wav, model.tts_model.sample_rate)
|
sf.write("voice_design.wav", wav, model.tts_model.sample_rate)
|
||||||
```
|
```
|
||||||
@@ -166,6 +169,7 @@ wav = model.generate(
|
|||||||
reference_wav_path="path/to/voice.wav",
|
reference_wav_path="path/to/voice.wav",
|
||||||
cfg_value=2.0,
|
cfg_value=2.0,
|
||||||
inference_timesteps=10,
|
inference_timesteps=10,
|
||||||
|
seed=42,
|
||||||
)
|
)
|
||||||
sf.write("controllable_clone.wav", wav, model.tts_model.sample_rate)
|
sf.write("controllable_clone.wav", wav, model.tts_model.sample_rate)
|
||||||
```
|
```
|
||||||
@@ -212,6 +216,7 @@ voxcpm design \
|
|||||||
voxcpm design \
|
voxcpm design \
|
||||||
--text "VoxCPM2带来全新语音合成体验。" \
|
--text "VoxCPM2带来全新语音合成体验。" \
|
||||||
--control "年轻女声,温暖温柔,略带微笑" \
|
--control "年轻女声,温暖温柔,略带微笑" \
|
||||||
|
--seed 42 \
|
||||||
--output out.wav
|
--output out.wav
|
||||||
|
|
||||||
# 声音克隆(参考音频)
|
# 声音克隆(参考音频)
|
||||||
|
|||||||
83
app.py
83
app.py
@@ -2,6 +2,7 @@ import os
|
|||||||
import re
|
import re
|
||||||
import sys
|
import sys
|
||||||
import logging
|
import logging
|
||||||
|
import random
|
||||||
import numpy as np
|
import numpy as np
|
||||||
import gradio as gr
|
import gradio as gr
|
||||||
from typing import Optional, Tuple
|
from typing import Optional, Tuple
|
||||||
@@ -44,8 +45,8 @@ _EXAMPLES_FOOTER_EN = (
|
|||||||
"**Example 1 — Gentle & Melancholic Girl** \n"
|
"**Example 1 — Gentle & Melancholic Girl** \n"
|
||||||
'`Control Instruction`: *"A young girl with a soft, sweet voice. '
|
'`Control Instruction`: *"A young girl with a soft, sweet voice. '
|
||||||
'Speaks slowly with a melancholic, slightly tsundere tone."* \n'
|
'Speaks slowly with a melancholic, slightly tsundere tone."* \n'
|
||||||
'`Target Text`: *"I never asked you to stay… It\'s not like I care or anything. '
|
"`Target Text`: *\"I never asked you to stay… It's not like I care or anything. "
|
||||||
'But… why does it still hurt so much now that you\'re gone?"* \n\n'
|
"But… why does it still hurt so much now that you're gone?\"* \n\n"
|
||||||
"**Example 2 — Laid-Back Surfer Dude** \n"
|
"**Example 2 — Laid-Back Surfer Dude** \n"
|
||||||
'`Control Instruction`: *"Relaxed young male voice, slightly nasal, '
|
'`Control Instruction`: *"Relaxed young male voice, slightly nasal, '
|
||||||
'lazy drawl, very casual and chill."* \n'
|
'lazy drawl, very casual and chill."* \n'
|
||||||
@@ -115,6 +116,10 @@ _I18N_TRANSLATIONS = {
|
|||||||
"cfg_info": "Higher → closer to the prompt / reference; lower → more creative variation",
|
"cfg_info": "Higher → closer to the prompt / reference; lower → more creative variation",
|
||||||
"dit_steps_label": "LocDiT flow-matching steps",
|
"dit_steps_label": "LocDiT flow-matching steps",
|
||||||
"dit_steps_info": "LocDiT flow-matching steps — more steps → maybe better audio quality, but slower",
|
"dit_steps_info": "LocDiT flow-matching steps — more steps → maybe better audio quality, but slower",
|
||||||
|
"seed_label": "Seed",
|
||||||
|
"seed_info": "Seed used for reproducible generation. Updated with the actual successful seed after generation.",
|
||||||
|
"random_seed_label": "Random Seed",
|
||||||
|
"random_seed_info": "Generate a new seed before each inference run.",
|
||||||
"usage_instructions": _USAGE_INSTRUCTIONS_EN,
|
"usage_instructions": _USAGE_INSTRUCTIONS_EN,
|
||||||
"examples_footer": _EXAMPLES_FOOTER_EN,
|
"examples_footer": _EXAMPLES_FOOTER_EN,
|
||||||
},
|
},
|
||||||
@@ -142,7 +147,7 @@ _I18N_TRANSLATIONS = {
|
|||||||
"examples_footer": _EXAMPLES_FOOTER_ZH,
|
"examples_footer": _EXAMPLES_FOOTER_ZH,
|
||||||
},
|
},
|
||||||
"zh-Hans": None, # alias, filled below
|
"zh-Hans": None, # alias, filled below
|
||||||
"zh": None, # alias, filled below
|
"zh": None, # alias, filled below
|
||||||
}
|
}
|
||||||
_I18N_TRANSLATIONS["zh-Hans"] = _I18N_TRANSLATIONS["zh-CN"]
|
_I18N_TRANSLATIONS["zh-Hans"] = _I18N_TRANSLATIONS["zh-CN"]
|
||||||
_I18N_TRANSLATIONS["zh"] = _I18N_TRANSLATIONS["zh-CN"]
|
_I18N_TRANSLATIONS["zh"] = _I18N_TRANSLATIONS["zh-CN"]
|
||||||
@@ -155,8 +160,7 @@ for _d in _I18N_TRANSLATIONS.values():
|
|||||||
I18N = gr.I18n(**_I18N_TRANSLATIONS)
|
I18N = gr.I18n(**_I18N_TRANSLATIONS)
|
||||||
|
|
||||||
DEFAULT_TARGET_TEXT = (
|
DEFAULT_TARGET_TEXT = (
|
||||||
"VoxCPM2 is a creative multilingual TTS model from ModelBest, "
|
"VoxCPM2 is a creative multilingual TTS model from ModelBest, " "designed to generate highly realistic speech."
|
||||||
"designed to generate highly realistic speech."
|
|
||||||
)
|
)
|
||||||
|
|
||||||
_CUSTOM_CSS = """
|
_CUSTOM_CSS = """
|
||||||
@@ -219,6 +223,7 @@ _APP_THEME = gr.themes.Soft(
|
|||||||
|
|
||||||
# ---------- Model ----------
|
# ---------- Model ----------
|
||||||
|
|
||||||
|
|
||||||
class VoxCPMDemo:
|
class VoxCPMDemo:
|
||||||
def __init__(self, model_id: str = "openbmb/VoxCPM2", device: str = "auto") -> None:
|
def __init__(self, model_id: str = "openbmb/VoxCPM2", device: str = "auto") -> None:
|
||||||
self.device = resolve_runtime_device(device, "cuda")
|
self.device = resolve_runtime_device(device, "cuda")
|
||||||
@@ -247,9 +252,7 @@ class VoxCPMDemo:
|
|||||||
def get_or_load_asr_model(self) -> AutoModel:
|
def get_or_load_asr_model(self) -> AutoModel:
|
||||||
if self.asr_model is not None:
|
if self.asr_model is not None:
|
||||||
return self.asr_model
|
return self.asr_model
|
||||||
logger.info(
|
logger.info(f"Loading ASR model: {self.asr_model_id} on device: {self.asr_device}")
|
||||||
f"Loading ASR model: {self.asr_model_id} on device: {self.asr_device}"
|
|
||||||
)
|
|
||||||
self.asr_model = AutoModel(
|
self.asr_model = AutoModel(
|
||||||
model=self.asr_model_id,
|
model=self.asr_model_id,
|
||||||
disable_update=True,
|
disable_update=True,
|
||||||
@@ -279,6 +282,7 @@ class VoxCPMDemo:
|
|||||||
do_normalize: bool,
|
do_normalize: bool,
|
||||||
denoise: bool,
|
denoise: bool,
|
||||||
inference_timesteps: int = 10,
|
inference_timesteps: int = 10,
|
||||||
|
seed: Optional[int] = None,
|
||||||
) -> dict:
|
) -> dict:
|
||||||
generate_kwargs = dict(
|
generate_kwargs = dict(
|
||||||
text=final_text,
|
text=final_text,
|
||||||
@@ -287,6 +291,7 @@ class VoxCPMDemo:
|
|||||||
inference_timesteps=inference_timesteps,
|
inference_timesteps=inference_timesteps,
|
||||||
normalize=do_normalize,
|
normalize=do_normalize,
|
||||||
denoise=denoise,
|
denoise=denoise,
|
||||||
|
seed=seed,
|
||||||
)
|
)
|
||||||
if prompt_text_clean and audio_path:
|
if prompt_text_clean and audio_path:
|
||||||
generate_kwargs["prompt_wav_path"] = audio_path
|
generate_kwargs["prompt_wav_path"] = audio_path
|
||||||
@@ -303,7 +308,8 @@ class VoxCPMDemo:
|
|||||||
do_normalize: bool = True,
|
do_normalize: bool = True,
|
||||||
denoise: bool = True,
|
denoise: bool = True,
|
||||||
inference_timesteps: int = 10,
|
inference_timesteps: int = 10,
|
||||||
) -> Tuple[int, np.ndarray]:
|
seed: Optional[int] = None,
|
||||||
|
) -> Tuple[int, np.ndarray, Optional[int]]:
|
||||||
current_model = self.get_or_load_voxcpm()
|
current_model = self.get_or_load_voxcpm()
|
||||||
|
|
||||||
text = (text_input or "").strip()
|
text = (text_input or "").strip()
|
||||||
@@ -335,16 +341,32 @@ class VoxCPMDemo:
|
|||||||
do_normalize=do_normalize,
|
do_normalize=do_normalize,
|
||||||
denoise=denoise,
|
denoise=denoise,
|
||||||
inference_timesteps=inference_timesteps,
|
inference_timesteps=inference_timesteps,
|
||||||
|
seed=seed,
|
||||||
)
|
)
|
||||||
wav = current_model.generate(**generate_kwargs)
|
wav = current_model.generate(**generate_kwargs)
|
||||||
return (current_model.tts_model.sample_rate, wav)
|
last_successful_seed = getattr(current_model.tts_model, "last_successful_seed", seed)
|
||||||
|
return (current_model.tts_model.sample_rate, wav, last_successful_seed)
|
||||||
|
|
||||||
|
|
||||||
# ---------- UI ----------
|
# ---------- UI ----------
|
||||||
|
|
||||||
|
|
||||||
def create_demo_interface(demo: VoxCPMDemo):
|
def create_demo_interface(demo: VoxCPMDemo):
|
||||||
gr.set_static_paths(paths=[Path.cwd().absolute() / "assets"])
|
gr.set_static_paths(paths=[Path.cwd().absolute() / "assets"])
|
||||||
|
|
||||||
|
def _coerce_seed(seed_value) -> Optional[int]:
|
||||||
|
if seed_value is None or seed_value == "":
|
||||||
|
return None
|
||||||
|
return int(seed_value)
|
||||||
|
|
||||||
|
def _prepare_seed(use_random_seed: bool, seed_value):
|
||||||
|
if use_random_seed:
|
||||||
|
return random.randint(0, 2**32 - 1)
|
||||||
|
return _coerce_seed(seed_value)
|
||||||
|
|
||||||
|
def _on_random_seed_toggle(checked):
|
||||||
|
return gr.update(interactive=not checked)
|
||||||
|
|
||||||
def _generate(
|
def _generate(
|
||||||
text: str,
|
text: str,
|
||||||
control_instruction: str,
|
control_instruction: str,
|
||||||
@@ -355,10 +377,12 @@ def create_demo_interface(demo: VoxCPMDemo):
|
|||||||
do_normalize: bool,
|
do_normalize: bool,
|
||||||
denoise: bool,
|
denoise: bool,
|
||||||
dit_steps: int,
|
dit_steps: int,
|
||||||
|
seed_value,
|
||||||
):
|
):
|
||||||
actual_prompt_text = prompt_text_value.strip() if use_prompt_text else ""
|
actual_prompt_text = prompt_text_value.strip() if use_prompt_text else ""
|
||||||
actual_control = "" if use_prompt_text else control_instruction
|
actual_control = "" if use_prompt_text else control_instruction
|
||||||
sr, wav_np = demo.generate_tts_audio(
|
seed = _coerce_seed(seed_value)
|
||||||
|
sr, wav_np, last_successful_seed = demo.generate_tts_audio(
|
||||||
text_input=text,
|
text_input=text,
|
||||||
control_instruction=actual_control,
|
control_instruction=actual_control,
|
||||||
reference_wav_path_input=ref_wav,
|
reference_wav_path_input=ref_wav,
|
||||||
@@ -367,8 +391,9 @@ def create_demo_interface(demo: VoxCPMDemo):
|
|||||||
do_normalize=do_normalize,
|
do_normalize=do_normalize,
|
||||||
denoise=denoise,
|
denoise=denoise,
|
||||||
inference_timesteps=int(dit_steps),
|
inference_timesteps=int(dit_steps),
|
||||||
|
seed=seed,
|
||||||
)
|
)
|
||||||
return (sr, wav_np)
|
return (sr, wav_np), last_successful_seed
|
||||||
|
|
||||||
def _on_toggle_instant(checked):
|
def _on_toggle_instant(checked):
|
||||||
"""Instant UI toggle — no ASR, no blocking."""
|
"""Instant UI toggle — no ASR, no blocking."""
|
||||||
@@ -465,6 +490,20 @@ def create_demo_interface(demo: VoxCPMDemo):
|
|||||||
label=I18N("dit_steps_label"),
|
label=I18N("dit_steps_label"),
|
||||||
info=I18N("dit_steps_info"),
|
info=I18N("dit_steps_info"),
|
||||||
)
|
)
|
||||||
|
with gr.Row():
|
||||||
|
seed_value = gr.Number(
|
||||||
|
value=random.randint(0, 2**32 - 1),
|
||||||
|
precision=0,
|
||||||
|
label=I18N("seed_label"),
|
||||||
|
info=I18N("seed_info"),
|
||||||
|
interactive=False,
|
||||||
|
)
|
||||||
|
random_seed = gr.Checkbox(
|
||||||
|
value=True,
|
||||||
|
label=I18N("random_seed_label"),
|
||||||
|
elem_classes=["switch-toggle"],
|
||||||
|
info=I18N("random_seed_info"),
|
||||||
|
)
|
||||||
|
|
||||||
run_btn = gr.Button(I18N("generate_btn"), variant="primary", size="lg")
|
run_btn = gr.Button(I18N("generate_btn"), variant="primary", size="lg")
|
||||||
|
|
||||||
@@ -482,7 +521,18 @@ def create_demo_interface(demo: VoxCPMDemo):
|
|||||||
outputs=[prompt_text],
|
outputs=[prompt_text],
|
||||||
)
|
)
|
||||||
|
|
||||||
|
random_seed.change(
|
||||||
|
fn=_on_random_seed_toggle,
|
||||||
|
inputs=[random_seed],
|
||||||
|
outputs=[seed_value],
|
||||||
|
)
|
||||||
|
|
||||||
run_btn.click(
|
run_btn.click(
|
||||||
|
fn=_prepare_seed,
|
||||||
|
inputs=[random_seed, seed_value],
|
||||||
|
outputs=[seed_value],
|
||||||
|
show_progress=False,
|
||||||
|
).then(
|
||||||
fn=_generate,
|
fn=_generate,
|
||||||
inputs=[
|
inputs=[
|
||||||
text,
|
text,
|
||||||
@@ -494,14 +544,16 @@ def create_demo_interface(demo: VoxCPMDemo):
|
|||||||
DoNormalizeText,
|
DoNormalizeText,
|
||||||
DoDenoisePromptAudio,
|
DoDenoisePromptAudio,
|
||||||
dit_steps,
|
dit_steps,
|
||||||
|
seed_value,
|
||||||
],
|
],
|
||||||
outputs=[audio_output],
|
outputs=[audio_output, seed_value],
|
||||||
show_progress=True,
|
show_progress=True,
|
||||||
api_name="generate",
|
api_name="generate",
|
||||||
)
|
)
|
||||||
|
|
||||||
return interface
|
return interface
|
||||||
|
|
||||||
|
|
||||||
def run_demo(
|
def run_demo(
|
||||||
server_name: str = "0.0.0.0",
|
server_name: str = "0.0.0.0",
|
||||||
server_port: int = 8808,
|
server_port: int = 8808,
|
||||||
@@ -523,9 +575,12 @@ def run_demo(
|
|||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
import argparse
|
import argparse
|
||||||
|
|
||||||
parser = argparse.ArgumentParser()
|
parser = argparse.ArgumentParser()
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
"--model-id", type=str, default="openbmb/VoxCPM2",
|
"--model-id",
|
||||||
|
type=str,
|
||||||
|
default="openbmb/VoxCPM2",
|
||||||
help="Local path or HuggingFace repo ID (default: openbmb/VoxCPM2)",
|
help="Local path or HuggingFace repo ID (default: openbmb/VoxCPM2)",
|
||||||
)
|
)
|
||||||
parser.add_argument("--port", type=int, default=8808, help="Server port")
|
parser.add_argument("--port", type=int, default=8808, help="Server port")
|
||||||
|
|||||||
36
app_old.py
36
app_old.py
@@ -6,6 +6,7 @@ import gradio as gr
|
|||||||
from typing import Optional, Tuple
|
from typing import Optional, Tuple
|
||||||
from funasr import AutoModel
|
from funasr import AutoModel
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
os.environ["TOKENIZERS_PARALLELISM"] = "false"
|
os.environ["TOKENIZERS_PARALLELISM"] = "false"
|
||||||
if os.environ.get("HF_REPO_ID", "").strip() == "":
|
if os.environ.get("HF_REPO_ID", "").strip() == "":
|
||||||
os.environ["HF_REPO_ID"] = "openbmb/VoxCPM1.5"
|
os.environ["HF_REPO_ID"] = "openbmb/VoxCPM1.5"
|
||||||
@@ -23,7 +24,7 @@ class VoxCPMDemo:
|
|||||||
self.asr_model: Optional[AutoModel] = AutoModel(
|
self.asr_model: Optional[AutoModel] = AutoModel(
|
||||||
model=self.asr_model_id,
|
model=self.asr_model_id,
|
||||||
disable_update=True,
|
disable_update=True,
|
||||||
log_level='DEBUG',
|
log_level="DEBUG",
|
||||||
device="cuda:0" if self.device == "cuda" else "cpu",
|
device="cuda:0" if self.device == "cuda" else "cpu",
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -48,6 +49,7 @@ class VoxCPMDemo:
|
|||||||
if not os.path.isdir(target_dir):
|
if not os.path.isdir(target_dir):
|
||||||
try:
|
try:
|
||||||
from huggingface_hub import snapshot_download # type: ignore
|
from huggingface_hub import snapshot_download # type: ignore
|
||||||
|
|
||||||
os.makedirs(target_dir, exist_ok=True)
|
os.makedirs(target_dir, exist_ok=True)
|
||||||
print(f"Downloading model from HF repo '{repo_id}' to '{target_dir}' ...", file=sys.stderr)
|
print(f"Downloading model from HF repo '{repo_id}' to '{target_dir}' ...", file=sys.stderr)
|
||||||
snapshot_download(repo_id=repo_id, local_dir=target_dir, local_dir_use_symlinks=False)
|
snapshot_download(repo_id=repo_id, local_dir=target_dir, local_dir_use_symlinks=False)
|
||||||
@@ -72,7 +74,7 @@ class VoxCPMDemo:
|
|||||||
if prompt_wav is None:
|
if prompt_wav is None:
|
||||||
return ""
|
return ""
|
||||||
res = self.asr_model.generate(input=prompt_wav, language="auto", use_itn=True)
|
res = self.asr_model.generate(input=prompt_wav, language="auto", use_itn=True)
|
||||||
text = res[0]["text"].split('|>')[-1]
|
text = res[0]["text"].split("|>")[-1]
|
||||||
return text
|
return text
|
||||||
|
|
||||||
def generate_tts_audio(
|
def generate_tts_audio(
|
||||||
@@ -149,11 +151,13 @@ _CUSTOM_CSS = """
|
|||||||
|
|
||||||
def create_demo_interface(demo: VoxCPMDemo):
|
def create_demo_interface(demo: VoxCPMDemo):
|
||||||
"""Build the Gradio UI for VoxCPM demo."""
|
"""Build the Gradio UI for VoxCPM demo."""
|
||||||
gr.set_static_paths(paths=[Path.cwd().absolute()/"assets"])
|
gr.set_static_paths(paths=[Path.cwd().absolute() / "assets"])
|
||||||
|
|
||||||
with gr.Blocks() as interface:
|
with gr.Blocks() as interface:
|
||||||
# Header logo
|
# Header logo
|
||||||
gr.HTML('<div class="logo-container"><img src="/gradio_api/file=assets/voxcpm_logo.png" alt="VoxCPM Logo"></div>')
|
gr.HTML(
|
||||||
|
'<div class="logo-container"><img src="/gradio_api/file=assets/voxcpm_logo.png" alt="VoxCPM Logo"></div>'
|
||||||
|
)
|
||||||
|
|
||||||
# Quick Start
|
# Quick Start
|
||||||
with gr.Accordion("📋 Quick Start Guide |快速入门", open=False, elem_id="acc_quick"):
|
with gr.Accordion("📋 Quick Start Guide |快速入门", open=False, elem_id="acc_quick"):
|
||||||
@@ -201,7 +205,7 @@ def create_demo_interface(demo: VoxCPMDemo):
|
|||||||
with gr.Row():
|
with gr.Row():
|
||||||
with gr.Column():
|
with gr.Column():
|
||||||
prompt_wav = gr.Audio(
|
prompt_wav = gr.Audio(
|
||||||
sources=["upload", 'microphone'],
|
sources=["upload", "microphone"],
|
||||||
type="filepath",
|
type="filepath",
|
||||||
label="Prompt Speech (Optional, or let VoxCPM improvise)",
|
label="Prompt Speech (Optional, or let VoxCPM improvise)",
|
||||||
value="./examples/example.wav",
|
value="./examples/example.wav",
|
||||||
@@ -210,13 +214,13 @@ def create_demo_interface(demo: VoxCPMDemo):
|
|||||||
value=False,
|
value=False,
|
||||||
label="Prompt Speech Enhancement",
|
label="Prompt Speech Enhancement",
|
||||||
elem_id="chk_denoise",
|
elem_id="chk_denoise",
|
||||||
info="We use ZipEnhancer model to denoise the prompt audio."
|
info="We use ZipEnhancer model to denoise the prompt audio.",
|
||||||
)
|
)
|
||||||
with gr.Row():
|
with gr.Row():
|
||||||
prompt_text = gr.Textbox(
|
prompt_text = gr.Textbox(
|
||||||
value="Just by listening a few minutes a day, you'll be able to eliminate negative thoughts by conditioning your mind to be more positive.",
|
value="Just by listening a few minutes a day, you'll be able to eliminate negative thoughts by conditioning your mind to be more positive.",
|
||||||
label="Prompt Text",
|
label="Prompt Text",
|
||||||
placeholder="Please enter the prompt text. Automatic recognition is supported, and you can correct the results yourself..."
|
placeholder="Please enter the prompt text. Automatic recognition is supported, and you can correct the results yourself...",
|
||||||
)
|
)
|
||||||
run_btn = gr.Button("Generate Speech", variant="primary")
|
run_btn = gr.Button("Generate Speech", variant="primary")
|
||||||
|
|
||||||
@@ -227,7 +231,7 @@ def create_demo_interface(demo: VoxCPMDemo):
|
|||||||
value=2.0,
|
value=2.0,
|
||||||
step=0.1,
|
step=0.1,
|
||||||
label="CFG Value (Guidance Scale)",
|
label="CFG Value (Guidance Scale)",
|
||||||
info="Higher values increase adherence to prompt, lower values allow more creativity"
|
info="Higher values increase adherence to prompt, lower values allow more creativity",
|
||||||
)
|
)
|
||||||
inference_timesteps = gr.Slider(
|
inference_timesteps = gr.Slider(
|
||||||
minimum=4,
|
minimum=4,
|
||||||
@@ -235,7 +239,7 @@ def create_demo_interface(demo: VoxCPMDemo):
|
|||||||
value=10,
|
value=10,
|
||||||
step=1,
|
step=1,
|
||||||
label="Inference Timesteps",
|
label="Inference Timesteps",
|
||||||
info="Number of inference timesteps for generation (higher values may improve quality but slower)"
|
info="Number of inference timesteps for generation (higher values may improve quality but slower)",
|
||||||
)
|
)
|
||||||
with gr.Row():
|
with gr.Row():
|
||||||
text = gr.Textbox(
|
text = gr.Textbox(
|
||||||
@@ -247,14 +251,22 @@ def create_demo_interface(demo: VoxCPMDemo):
|
|||||||
value=False,
|
value=False,
|
||||||
label="Text Normalization",
|
label="Text Normalization",
|
||||||
elem_id="chk_normalize",
|
elem_id="chk_normalize",
|
||||||
info="We use wetext library to normalize the input text."
|
info="We use wetext library to normalize the input text.",
|
||||||
)
|
)
|
||||||
audio_output = gr.Audio(label="Output Audio")
|
audio_output = gr.Audio(label="Output Audio")
|
||||||
|
|
||||||
# Wiring
|
# Wiring
|
||||||
run_btn.click(
|
run_btn.click(
|
||||||
fn=demo.generate_tts_audio,
|
fn=demo.generate_tts_audio,
|
||||||
inputs=[text, prompt_wav, prompt_text, cfg_value, inference_timesteps, DoNormalizeText, DoDenoisePromptAudio],
|
inputs=[
|
||||||
|
text,
|
||||||
|
prompt_wav,
|
||||||
|
prompt_text,
|
||||||
|
cfg_value,
|
||||||
|
inference_timesteps,
|
||||||
|
DoNormalizeText,
|
||||||
|
DoDenoisePromptAudio,
|
||||||
|
],
|
||||||
outputs=[audio_output],
|
outputs=[audio_output],
|
||||||
show_progress=True,
|
show_progress=True,
|
||||||
api_name="generate",
|
api_name="generate",
|
||||||
@@ -277,4 +289,4 @@ def run_demo(server_name: str = "localhost", server_port: int = 7860, show_error
|
|||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
run_demo()
|
run_demo()
|
||||||
|
|||||||
@@ -308,7 +308,8 @@ def run_inference(text, prompt_wav, prompt_text, lora_selection, cfg_scale, step
|
|||||||
if new_r is not None and current_r is not None and new_r != current_r:
|
if new_r is not None and current_r is not None and new_r != current_r:
|
||||||
print(f"LoRA rank mismatch (model r={current_r}, checkpoint r={new_r}), reloading...", file=sys.stderr)
|
print(f"LoRA rank mismatch (model r={current_r}, checkpoint r={new_r}), reloading...", file=sys.stderr)
|
||||||
reload_base = (
|
reload_base = (
|
||||||
new_base_model if new_base_model and os.path.exists(new_base_model)
|
new_base_model
|
||||||
|
if new_base_model and os.path.exists(new_base_model)
|
||||||
else (pretrained_path if pretrained_path and pretrained_path.strip() else default_pretrained_path)
|
else (pretrained_path if pretrained_path and pretrained_path.strip() else default_pretrained_path)
|
||||||
)
|
)
|
||||||
try:
|
try:
|
||||||
@@ -982,9 +983,7 @@ with gr.Blocks(title="VoxCPM LoRA WebUI", theme=gr.themes.Soft(), css=custom_css
|
|||||||
|
|
||||||
gr.Markdown("#### 分发选项 (Distribution)")
|
gr.Markdown("#### 分发选项 (Distribution)")
|
||||||
with gr.Row():
|
with gr.Row():
|
||||||
hf_model_id = gr.Textbox(
|
hf_model_id = gr.Textbox(label="HuggingFace Model ID (e.g., openbmb/VoxCPM2)", value="")
|
||||||
label="HuggingFace Model ID (e.g., openbmb/VoxCPM2)", value=""
|
|
||||||
)
|
|
||||||
distribute = gr.Checkbox(label="分发模式 (distribute)", value=False)
|
distribute = gr.Checkbox(label="分发模式 (distribute)", value=False)
|
||||||
|
|
||||||
with gr.Column(scale=2, elem_classes="form-section"):
|
with gr.Column(scale=2, elem_classes="form-section"):
|
||||||
|
|||||||
@@ -3,6 +3,7 @@
|
|||||||
Loads src/voxcpm/model/utils.py directly to avoid the heavy voxcpm package
|
Loads src/voxcpm/model/utils.py directly to avoid the heavy voxcpm package
|
||||||
init. Run with: `python scripts/test_pick_runtime_dtype.py`.
|
init. Run with: `python scripts/test_pick_runtime_dtype.py`.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import importlib.util
|
import importlib.util
|
||||||
import os
|
import os
|
||||||
import pathlib
|
import pathlib
|
||||||
|
|||||||
@@ -19,6 +19,7 @@ With voice cloning:
|
|||||||
--text "Hello, this is voice cloning result." \
|
--text "Hello, this is voice cloning result." \
|
||||||
--prompt_audio path/to/ref.wav \
|
--prompt_audio path/to/ref.wav \
|
||||||
--prompt_text "Reference audio transcript" \
|
--prompt_text "Reference audio transcript" \
|
||||||
|
--seed 42 \
|
||||||
--output ft_clone.wav
|
--output ft_clone.wav
|
||||||
"""
|
"""
|
||||||
|
|
||||||
@@ -86,6 +87,12 @@ def parse_args():
|
|||||||
action="store_true",
|
action="store_true",
|
||||||
help="Enable text normalization",
|
help="Enable text normalization",
|
||||||
)
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--seed",
|
||||||
|
type=int,
|
||||||
|
default=None,
|
||||||
|
help="Random seed for generation (default: None)",
|
||||||
|
)
|
||||||
return parser.parse_args()
|
return parser.parse_args()
|
||||||
|
|
||||||
|
|
||||||
@@ -109,6 +116,9 @@ def main():
|
|||||||
print(f"[FT Inference] Using reference audio: {prompt_wav_path}", file=sys.stderr)
|
print(f"[FT Inference] Using reference audio: {prompt_wav_path}", file=sys.stderr)
|
||||||
print(f"[FT Inference] Reference text: {prompt_text}", file=sys.stderr)
|
print(f"[FT Inference] Reference text: {prompt_text}", file=sys.stderr)
|
||||||
|
|
||||||
|
if args.seed is not None:
|
||||||
|
print(f"[FT Inference] Using seed: {args.seed}", file=sys.stderr)
|
||||||
|
|
||||||
audio_np = model.generate(
|
audio_np = model.generate(
|
||||||
text=args.text,
|
text=args.text,
|
||||||
prompt_wav_path=prompt_wav_path,
|
prompt_wav_path=prompt_wav_path,
|
||||||
@@ -118,6 +128,7 @@ def main():
|
|||||||
max_len=args.max_len,
|
max_len=args.max_len,
|
||||||
normalize=args.normalize,
|
normalize=args.normalize,
|
||||||
denoise=False,
|
denoise=False,
|
||||||
|
seed=args.seed,
|
||||||
)
|
)
|
||||||
|
|
||||||
# Save audio
|
# Save audio
|
||||||
|
|||||||
@@ -16,6 +16,7 @@ With voice cloning:
|
|||||||
--text "This is voice cloning result." \
|
--text "This is voice cloning result." \
|
||||||
--prompt_audio path/to/ref.wav \
|
--prompt_audio path/to/ref.wav \
|
||||||
--prompt_text "Reference audio transcript" \
|
--prompt_text "Reference audio transcript" \
|
||||||
|
--seed 42 \
|
||||||
--output lora_clone.wav
|
--output lora_clone.wav
|
||||||
|
|
||||||
Note: The script reads base_model path and lora_config from lora_config.json
|
Note: The script reads base_model path and lora_config from lora_config.json
|
||||||
@@ -94,6 +95,12 @@ def parse_args():
|
|||||||
action="store_true",
|
action="store_true",
|
||||||
help="Enable text normalization",
|
help="Enable text normalization",
|
||||||
)
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--seed",
|
||||||
|
type=int,
|
||||||
|
default=None,
|
||||||
|
help="Random seed for generation (default: None)",
|
||||||
|
)
|
||||||
return parser.parse_args()
|
return parser.parse_args()
|
||||||
|
|
||||||
|
|
||||||
@@ -130,6 +137,8 @@ def main():
|
|||||||
print(
|
print(
|
||||||
f" LoRA config: r={lora_cfg.r}, alpha={lora_cfg.alpha}" if lora_cfg else " LoRA config: None", file=sys.stderr
|
f" LoRA config: r={lora_cfg.r}, alpha={lora_cfg.alpha}" if lora_cfg else " LoRA config: None", file=sys.stderr
|
||||||
)
|
)
|
||||||
|
if args.seed is not None:
|
||||||
|
print(f" Seed: {args.seed}", file=sys.stderr)
|
||||||
|
|
||||||
# 3. Load model with LoRA (no denoiser)
|
# 3. Load model with LoRA (no denoiser)
|
||||||
print(f"\n[1/2] Loading model with LoRA: {pretrained_path}", file=sys.stderr)
|
print(f"\n[1/2] Loading model with LoRA: {pretrained_path}", file=sys.stderr)
|
||||||
@@ -161,6 +170,7 @@ def main():
|
|||||||
max_len=args.max_len,
|
max_len=args.max_len,
|
||||||
normalize=args.normalize,
|
normalize=args.normalize,
|
||||||
denoise=False,
|
denoise=False,
|
||||||
|
seed=args.seed,
|
||||||
)
|
)
|
||||||
lora_output = out_path.with_stem(out_path.stem + "_with_lora")
|
lora_output = out_path.with_stem(out_path.stem + "_with_lora")
|
||||||
sf.write(str(lora_output), audio_np, model.tts_model.sample_rate)
|
sf.write(str(lora_output), audio_np, model.tts_model.sample_rate)
|
||||||
@@ -181,6 +191,7 @@ def main():
|
|||||||
max_len=args.max_len,
|
max_len=args.max_len,
|
||||||
normalize=args.normalize,
|
normalize=args.normalize,
|
||||||
denoise=False,
|
denoise=False,
|
||||||
|
seed=args.seed,
|
||||||
)
|
)
|
||||||
disabled_output = out_path.with_stem(out_path.stem + "_lora_disabled")
|
disabled_output = out_path.with_stem(out_path.stem + "_lora_disabled")
|
||||||
sf.write(str(disabled_output), audio_np, model.tts_model.sample_rate)
|
sf.write(str(disabled_output), audio_np, model.tts_model.sample_rate)
|
||||||
@@ -201,6 +212,7 @@ def main():
|
|||||||
max_len=args.max_len,
|
max_len=args.max_len,
|
||||||
normalize=args.normalize,
|
normalize=args.normalize,
|
||||||
denoise=False,
|
denoise=False,
|
||||||
|
seed=args.seed,
|
||||||
)
|
)
|
||||||
reenabled_output = out_path.with_stem(out_path.stem + "_lora_reenabled")
|
reenabled_output = out_path.with_stem(out_path.stem + "_lora_reenabled")
|
||||||
sf.write(str(reenabled_output), audio_np, model.tts_model.sample_rate)
|
sf.write(str(reenabled_output), audio_np, model.tts_model.sample_rate)
|
||||||
@@ -221,6 +233,7 @@ def main():
|
|||||||
max_len=args.max_len,
|
max_len=args.max_len,
|
||||||
normalize=args.normalize,
|
normalize=args.normalize,
|
||||||
denoise=False,
|
denoise=False,
|
||||||
|
seed=args.seed,
|
||||||
)
|
)
|
||||||
reset_output = out_path.with_stem(out_path.stem + "_lora_reset")
|
reset_output = out_path.with_stem(out_path.stem + "_lora_reset")
|
||||||
sf.write(str(reset_output), audio_np, model.tts_model.sample_rate)
|
sf.write(str(reset_output), audio_np, model.tts_model.sample_rate)
|
||||||
@@ -242,6 +255,7 @@ def main():
|
|||||||
max_len=args.max_len,
|
max_len=args.max_len,
|
||||||
normalize=args.normalize,
|
normalize=args.normalize,
|
||||||
denoise=False,
|
denoise=False,
|
||||||
|
seed=args.seed,
|
||||||
)
|
)
|
||||||
reload_output = out_path.with_stem(out_path.stem + "_lora_reloaded")
|
reload_output = out_path.with_stem(out_path.stem + "_lora_reloaded")
|
||||||
sf.write(str(reload_output), audio_np, model.tts_model.sample_rate)
|
sf.write(str(reload_output), audio_np, model.tts_model.sample_rate)
|
||||||
|
|||||||
@@ -599,7 +599,9 @@ def generate_sample_audio(
|
|||||||
)
|
)
|
||||||
with torch.no_grad():
|
with torch.no_grad():
|
||||||
with autocast_ctx:
|
with autocast_ctx:
|
||||||
generated = unwrapped_model.generate(target_text=text, inference_timesteps=10, cfg_value=2.0)
|
generated = unwrapped_model.generate(
|
||||||
|
target_text=text, inference_timesteps=10, cfg_value=2.0, seed=42
|
||||||
|
)
|
||||||
|
|
||||||
# Restore training setup
|
# Restore training setup
|
||||||
# unwrapped_model.to(torch.float32)
|
# unwrapped_model.to(torch.float32)
|
||||||
|
|||||||
@@ -248,6 +248,7 @@ def _run_single(args, parser, *, text: str, output: str, prompt_text: str | None
|
|||||||
inference_timesteps=args.inference_timesteps,
|
inference_timesteps=args.inference_timesteps,
|
||||||
normalize=args.normalize,
|
normalize=args.normalize,
|
||||||
denoise=args.denoise and (args.prompt_audio is not None or args.reference_audio is not None),
|
denoise=args.denoise and (args.prompt_audio is not None or args.reference_audio is not None),
|
||||||
|
seed=args.seed,
|
||||||
)
|
)
|
||||||
|
|
||||||
import soundfile as sf
|
import soundfile as sf
|
||||||
@@ -332,6 +333,7 @@ def cmd_batch(args, parser):
|
|||||||
inference_timesteps=args.inference_timesteps,
|
inference_timesteps=args.inference_timesteps,
|
||||||
normalize=args.normalize,
|
normalize=args.normalize,
|
||||||
denoise=args.denoise and (prompt_audio_path is not None or reference_audio_path is not None),
|
denoise=args.denoise and (prompt_audio_path is not None or reference_audio_path is not None),
|
||||||
|
seed=args.seed,
|
||||||
)
|
)
|
||||||
|
|
||||||
output_file = output_dir / f"output_{i:03d}.wav"
|
output_file = output_dir / f"output_{i:03d}.wav"
|
||||||
@@ -413,6 +415,12 @@ def _add_common_generation_args(parser):
|
|||||||
help="Inference steps (int, recommended 4–30, default: 10)",
|
help="Inference steps (int, recommended 4–30, default: 10)",
|
||||||
)
|
)
|
||||||
parser.add_argument("--normalize", action="store_true", help="Enable text normalization")
|
parser.add_argument("--normalize", action="store_true", help="Enable text normalization")
|
||||||
|
parser.add_argument(
|
||||||
|
"--seed",
|
||||||
|
type=int,
|
||||||
|
default=None,
|
||||||
|
help="Random seed for generation (default: None)",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def _add_prompt_reference_args(parser):
|
def _add_prompt_reference_args(parser):
|
||||||
@@ -587,6 +595,12 @@ Examples:
|
|||||||
help="Inference steps (int, recommended 4–30, default: 10)",
|
help="Inference steps (int, recommended 4–30, default: 10)",
|
||||||
)
|
)
|
||||||
batch_parser.add_argument("--normalize", action="store_true", help="Enable text normalization")
|
batch_parser.add_argument("--normalize", action="store_true", help="Enable text normalization")
|
||||||
|
batch_parser.add_argument(
|
||||||
|
"--seed",
|
||||||
|
type=int,
|
||||||
|
default=None,
|
||||||
|
help="Random seed for generation (default: None)",
|
||||||
|
)
|
||||||
_add_model_args(batch_parser)
|
_add_model_args(batch_parser)
|
||||||
_add_lora_args(batch_parser)
|
_add_lora_args(batch_parser)
|
||||||
_add_timestamp_args(batch_parser, include_output=False)
|
_add_timestamp_args(batch_parser, include_output=False)
|
||||||
|
|||||||
@@ -196,6 +196,7 @@ class VoxCPM:
|
|||||||
retry_badcase_max_times: int = 3,
|
retry_badcase_max_times: int = 3,
|
||||||
retry_badcase_ratio_threshold: float = 6.0,
|
retry_badcase_ratio_threshold: float = 6.0,
|
||||||
streaming: bool = False,
|
streaming: bool = False,
|
||||||
|
seed: Optional[int] = None,
|
||||||
) -> Generator[np.ndarray, None, None]:
|
) -> Generator[np.ndarray, None, None]:
|
||||||
"""Synthesize speech for the given text and return a single waveform.
|
"""Synthesize speech for the given text and return a single waveform.
|
||||||
|
|
||||||
@@ -218,6 +219,7 @@ class VoxCPM:
|
|||||||
retry_badcase_max_times: Maximum number of times to retry badcase.
|
retry_badcase_max_times: Maximum number of times to retry badcase.
|
||||||
retry_badcase_ratio_threshold: Threshold for audio-to-text ratio.
|
retry_badcase_ratio_threshold: Threshold for audio-to-text ratio.
|
||||||
streaming: Whether to return a generator of audio chunks.
|
streaming: Whether to return a generator of audio chunks.
|
||||||
|
seed: Optional random seed for reproducibility.
|
||||||
Returns:
|
Returns:
|
||||||
Generator of numpy.ndarray: 1D waveform array (float32) on CPU.
|
Generator of numpy.ndarray: 1D waveform array (float32) on CPU.
|
||||||
Yields audio chunks for each generation step if ``streaming=True``,
|
Yields audio chunks for each generation step if ``streaming=True``,
|
||||||
@@ -294,6 +296,7 @@ class VoxCPM:
|
|||||||
retry_badcase_max_times=retry_badcase_max_times,
|
retry_badcase_max_times=retry_badcase_max_times,
|
||||||
retry_badcase_ratio_threshold=retry_badcase_ratio_threshold,
|
retry_badcase_ratio_threshold=retry_badcase_ratio_threshold,
|
||||||
streaming=streaming,
|
streaming=streaming,
|
||||||
|
seed=seed,
|
||||||
)
|
)
|
||||||
|
|
||||||
if streaming:
|
if streaming:
|
||||||
|
|||||||
@@ -5,9 +5,12 @@ from transformers import PreTrainedTokenizer
|
|||||||
|
|
||||||
_LOW_PRECISION_DTYPES = {"bfloat16", "bf16", "float16", "fp16"}
|
_LOW_PRECISION_DTYPES = {"bfloat16", "bf16", "float16", "fp16"}
|
||||||
_VALID_DTYPE_OVERRIDES = {
|
_VALID_DTYPE_OVERRIDES = {
|
||||||
"bfloat16", "bf16",
|
"bfloat16",
|
||||||
"float16", "fp16",
|
"bf16",
|
||||||
"float32", "fp32",
|
"float16",
|
||||||
|
"fp16",
|
||||||
|
"float32",
|
||||||
|
"fp32",
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@@ -21,6 +24,19 @@ def next_and_close(gen):
|
|||||||
gen.close()
|
gen.close()
|
||||||
|
|
||||||
|
|
||||||
|
def materialize_generation_seed(seed: Optional[int]) -> int:
|
||||||
|
"""Return a concrete seed for a generation request."""
|
||||||
|
if seed is not None:
|
||||||
|
return int(seed)
|
||||||
|
return int(torch.seed() & 0xFFFFFFFF)
|
||||||
|
|
||||||
|
|
||||||
|
def apply_generation_seed(seed: int) -> None:
|
||||||
|
torch.manual_seed(seed)
|
||||||
|
if torch.cuda.is_available():
|
||||||
|
torch.cuda.manual_seed_all(seed)
|
||||||
|
|
||||||
|
|
||||||
def mask_multichar_chinese_tokens(tokenizer: PreTrainedTokenizer):
|
def mask_multichar_chinese_tokens(tokenizer: PreTrainedTokenizer):
|
||||||
"""Create a tokenizer wrapper that converts multi-character Chinese tokens to single characters.
|
"""Create a tokenizer wrapper that converts multi-character Chinese tokens to single characters.
|
||||||
|
|
||||||
@@ -160,10 +176,7 @@ def pick_runtime_dtype(device: str, configured_dtype: str) -> str:
|
|||||||
override = os.environ.get("VOXCPM_MPS_DTYPE", "").strip().lower()
|
override = os.environ.get("VOXCPM_MPS_DTYPE", "").strip().lower()
|
||||||
if override:
|
if override:
|
||||||
if override not in _VALID_DTYPE_OVERRIDES:
|
if override not in _VALID_DTYPE_OVERRIDES:
|
||||||
raise ValueError(
|
raise ValueError(f"VOXCPM_MPS_DTYPE='{override}' is not one of " f"{sorted(_VALID_DTYPE_OVERRIDES)}")
|
||||||
f"VOXCPM_MPS_DTYPE='{override}' is not one of "
|
|
||||||
f"{sorted(_VALID_DTYPE_OVERRIDES)}"
|
|
||||||
)
|
|
||||||
return override
|
return override
|
||||||
|
|
||||||
if (configured_dtype or "").lower() in _LOW_PRECISION_DTYPES:
|
if (configured_dtype or "").lower() in _LOW_PRECISION_DTYPES:
|
||||||
@@ -211,15 +224,13 @@ def resolve_runtime_device(device: Optional[str], configured_device: str = "cuda
|
|||||||
if explicit.startswith("cuda"):
|
if explicit.startswith("cuda"):
|
||||||
if not torch.cuda.is_available():
|
if not torch.cuda.is_available():
|
||||||
raise ValueError(
|
raise ValueError(
|
||||||
f"Requested device '{device}', but CUDA is not available. "
|
f"Requested device '{device}', but CUDA is not available. " "Use device='auto' for automatic fallback."
|
||||||
"Use device='auto' for automatic fallback."
|
|
||||||
)
|
)
|
||||||
return explicit
|
return explicit
|
||||||
if explicit == "mps":
|
if explicit == "mps":
|
||||||
if not _has_mps():
|
if not _has_mps():
|
||||||
raise ValueError(
|
raise ValueError(
|
||||||
"Requested device 'mps', but MPS is not available. "
|
"Requested device 'mps', but MPS is not available. " "Use device='auto' for automatic fallback."
|
||||||
"Use device='auto' for automatic fallback."
|
|
||||||
)
|
)
|
||||||
return "mps"
|
return "mps"
|
||||||
if explicit == "cpu":
|
if explicit == "cpu":
|
||||||
|
|||||||
@@ -45,7 +45,9 @@ from ..modules.locdit import CfmConfig, UnifiedCFM, VoxCPMLocDiT
|
|||||||
from ..modules.locenc import VoxCPMLocEnc
|
from ..modules.locenc import VoxCPMLocEnc
|
||||||
from ..modules.minicpm4 import MiniCPM4Config, MiniCPMModel
|
from ..modules.minicpm4 import MiniCPM4Config, MiniCPMModel
|
||||||
from .utils import (
|
from .utils import (
|
||||||
|
apply_generation_seed,
|
||||||
get_dtype,
|
get_dtype,
|
||||||
|
materialize_generation_seed,
|
||||||
mask_multichar_chinese_tokens,
|
mask_multichar_chinese_tokens,
|
||||||
next_and_close,
|
next_and_close,
|
||||||
pick_runtime_dtype,
|
pick_runtime_dtype,
|
||||||
@@ -140,6 +142,7 @@ class VoxCPMModel(nn.Module):
|
|||||||
self.text_tokenizer = mask_multichar_chinese_tokens(tokenizer)
|
self.text_tokenizer = mask_multichar_chinese_tokens(tokenizer)
|
||||||
self.audio_start_token = 101
|
self.audio_start_token = 101
|
||||||
self.audio_end_token = 102
|
self.audio_end_token = 102
|
||||||
|
self.last_successful_seed = None
|
||||||
|
|
||||||
# Residual Acoustic LM
|
# Residual Acoustic LM
|
||||||
residual_lm_config = config.lm_config.model_copy(deep=True)
|
residual_lm_config = config.lm_config.model_copy(deep=True)
|
||||||
@@ -367,6 +370,7 @@ class VoxCPMModel(nn.Module):
|
|||||||
retry_badcase_max_times: int = 3,
|
retry_badcase_max_times: int = 3,
|
||||||
retry_badcase_ratio_threshold: float = 6.0, # setting acceptable ratio of audio length to text length (for badcase detection)
|
retry_badcase_ratio_threshold: float = 6.0, # setting acceptable ratio of audio length to text length (for badcase detection)
|
||||||
streaming: bool = False,
|
streaming: bool = False,
|
||||||
|
seed: Optional[int] = None,
|
||||||
) -> Generator[torch.Tensor, None, None]:
|
) -> Generator[torch.Tensor, None, None]:
|
||||||
if retry_badcase and streaming:
|
if retry_badcase and streaming:
|
||||||
warnings.warn("Retry on bad cases is not supported in streaming mode, setting retry_badcase=False.")
|
warnings.warn("Retry on bad cases is not supported in streaming mode, setting retry_badcase=False.")
|
||||||
@@ -452,7 +456,12 @@ class VoxCPMModel(nn.Module):
|
|||||||
target_text_length = len(self.text_tokenizer(target_text))
|
target_text_length = len(self.text_tokenizer(target_text))
|
||||||
|
|
||||||
retry_badcase_times = 0
|
retry_badcase_times = 0
|
||||||
|
current_seed = materialize_generation_seed(seed)
|
||||||
|
last_attempt_seed = current_seed
|
||||||
while retry_badcase_times < retry_badcase_max_times:
|
while retry_badcase_times < retry_badcase_max_times:
|
||||||
|
last_attempt_seed = current_seed
|
||||||
|
apply_generation_seed(last_attempt_seed)
|
||||||
|
|
||||||
inference_result = self._inference(
|
inference_result = self._inference(
|
||||||
text_token,
|
text_token,
|
||||||
text_mask,
|
text_mask,
|
||||||
@@ -471,6 +480,7 @@ class VoxCPMModel(nn.Module):
|
|||||||
for latent_pred, _ in inference_result:
|
for latent_pred, _ in inference_result:
|
||||||
decode_audio = self.audio_vae.decode(latent_pred.to(torch.float32))
|
decode_audio = self.audio_vae.decode(latent_pred.to(torch.float32))
|
||||||
decode_audio = decode_audio[..., -patch_len:].squeeze(1).cpu()
|
decode_audio = decode_audio[..., -patch_len:].squeeze(1).cpu()
|
||||||
|
self.last_successful_seed = last_attempt_seed
|
||||||
yield decode_audio
|
yield decode_audio
|
||||||
break
|
break
|
||||||
else:
|
else:
|
||||||
@@ -482,6 +492,7 @@ class VoxCPMModel(nn.Module):
|
|||||||
file=sys.stderr,
|
file=sys.stderr,
|
||||||
)
|
)
|
||||||
retry_badcase_times += 1
|
retry_badcase_times += 1
|
||||||
|
current_seed += 1
|
||||||
continue
|
continue
|
||||||
else:
|
else:
|
||||||
break
|
break
|
||||||
@@ -489,6 +500,7 @@ class VoxCPMModel(nn.Module):
|
|||||||
break
|
break
|
||||||
|
|
||||||
if not streaming:
|
if not streaming:
|
||||||
|
self.last_successful_seed = last_attempt_seed
|
||||||
decode_audio = self.audio_vae.decode(latent_pred.to(torch.float32)).squeeze(1).cpu()
|
decode_audio = self.audio_vae.decode(latent_pred.to(torch.float32)).squeeze(1).cpu()
|
||||||
yield decode_audio
|
yield decode_audio
|
||||||
|
|
||||||
@@ -603,6 +615,7 @@ class VoxCPMModel(nn.Module):
|
|||||||
retry_badcase_ratio_threshold: float = 6.0,
|
retry_badcase_ratio_threshold: float = 6.0,
|
||||||
streaming: bool = False,
|
streaming: bool = False,
|
||||||
streaming_prefix_len: int = 3,
|
streaming_prefix_len: int = 3,
|
||||||
|
seed: Optional[int] = None,
|
||||||
) -> Generator[Tuple[torch.Tensor, torch.Tensor, Union[torch.Tensor, List[torch.Tensor]]], None, None]:
|
) -> Generator[Tuple[torch.Tensor, torch.Tensor, Union[torch.Tensor, List[torch.Tensor]]], None, None]:
|
||||||
"""
|
"""
|
||||||
Generate audio using pre-built prompt cache.
|
Generate audio using pre-built prompt cache.
|
||||||
@@ -678,7 +691,12 @@ class VoxCPMModel(nn.Module):
|
|||||||
# run inference
|
# run inference
|
||||||
target_text_length = len(self.text_tokenizer(target_text))
|
target_text_length = len(self.text_tokenizer(target_text))
|
||||||
retry_badcase_times = 0
|
retry_badcase_times = 0
|
||||||
|
current_seed = materialize_generation_seed(seed)
|
||||||
|
last_attempt_seed = current_seed
|
||||||
while retry_badcase_times < retry_badcase_max_times:
|
while retry_badcase_times < retry_badcase_max_times:
|
||||||
|
last_attempt_seed = current_seed
|
||||||
|
apply_generation_seed(last_attempt_seed)
|
||||||
|
|
||||||
inference_result = self._inference(
|
inference_result = self._inference(
|
||||||
text_token,
|
text_token,
|
||||||
text_mask,
|
text_mask,
|
||||||
@@ -698,6 +716,7 @@ class VoxCPMModel(nn.Module):
|
|||||||
for latent_pred, pred_audio_feat in inference_result:
|
for latent_pred, pred_audio_feat in inference_result:
|
||||||
decode_audio = self.audio_vae.decode(latent_pred.to(torch.float32))
|
decode_audio = self.audio_vae.decode(latent_pred.to(torch.float32))
|
||||||
decode_audio = decode_audio[..., -patch_len:].squeeze(1).cpu()
|
decode_audio = decode_audio[..., -patch_len:].squeeze(1).cpu()
|
||||||
|
self.last_successful_seed = last_attempt_seed
|
||||||
yield (decode_audio, target_text_token, pred_audio_feat)
|
yield (decode_audio, target_text_token, pred_audio_feat)
|
||||||
break
|
break
|
||||||
else:
|
else:
|
||||||
@@ -709,12 +728,14 @@ class VoxCPMModel(nn.Module):
|
|||||||
file=sys.stderr,
|
file=sys.stderr,
|
||||||
)
|
)
|
||||||
retry_badcase_times += 1
|
retry_badcase_times += 1
|
||||||
|
current_seed += 1
|
||||||
continue
|
continue
|
||||||
else:
|
else:
|
||||||
break
|
break
|
||||||
else:
|
else:
|
||||||
break
|
break
|
||||||
if not streaming:
|
if not streaming:
|
||||||
|
self.last_successful_seed = last_attempt_seed
|
||||||
decode_audio = self.audio_vae.decode(latent_pred.to(torch.float32))
|
decode_audio = self.audio_vae.decode(latent_pred.to(torch.float32))
|
||||||
patch_len = self.patch_size * self.chunk_size
|
patch_len = self.patch_size * self.chunk_size
|
||||||
if audio_mask.sum().item() > 0:
|
if audio_mask.sum().item() > 0:
|
||||||
|
|||||||
@@ -46,7 +46,9 @@ from ..modules.locdit import CfmConfig, UnifiedCFM, VoxCPMLocDiTV2
|
|||||||
from ..modules.locenc import VoxCPMLocEnc
|
from ..modules.locenc import VoxCPMLocEnc
|
||||||
from ..modules.minicpm4 import MiniCPM4Config, MiniCPMModel
|
from ..modules.minicpm4 import MiniCPM4Config, MiniCPMModel
|
||||||
from .utils import (
|
from .utils import (
|
||||||
|
apply_generation_seed,
|
||||||
get_dtype,
|
get_dtype,
|
||||||
|
materialize_generation_seed,
|
||||||
mask_multichar_chinese_tokens,
|
mask_multichar_chinese_tokens,
|
||||||
next_and_close,
|
next_and_close,
|
||||||
pick_runtime_dtype,
|
pick_runtime_dtype,
|
||||||
@@ -55,7 +57,9 @@ from .utils import (
|
|||||||
|
|
||||||
|
|
||||||
# A simple function to trim audio silence using VAD, not used default
|
# A simple function to trim audio silence using VAD, not used default
|
||||||
def _trim_audio_silence_vad(audio: torch.Tensor, sample_rate: int, max_silence_ms: float = 200.0, top_db: float = 35.0) -> torch.Tensor:
|
def _trim_audio_silence_vad(
|
||||||
|
audio: torch.Tensor, sample_rate: int, max_silence_ms: float = 200.0, top_db: float = 35.0
|
||||||
|
) -> torch.Tensor:
|
||||||
if audio.numel() == 0:
|
if audio.numel() == 0:
|
||||||
return audio
|
return audio
|
||||||
y = audio.squeeze(0).numpy()
|
y = audio.squeeze(0).numpy()
|
||||||
@@ -184,6 +188,7 @@ class VoxCPM2Model(nn.Module):
|
|||||||
self.audio_end_token = 102
|
self.audio_end_token = 102
|
||||||
self.ref_audio_start_token = 103
|
self.ref_audio_start_token = 103
|
||||||
self.ref_audio_end_token = 104
|
self.ref_audio_end_token = 104
|
||||||
|
self.last_successful_seed = None
|
||||||
|
|
||||||
# Residual Acoustic LM
|
# Residual Acoustic LM
|
||||||
residual_lm_config = config.lm_config.model_copy(deep=True)
|
residual_lm_config = config.lm_config.model_copy(deep=True)
|
||||||
@@ -476,6 +481,7 @@ class VoxCPM2Model(nn.Module):
|
|||||||
trim_silence_vad: bool = False,
|
trim_silence_vad: bool = False,
|
||||||
streaming: bool = False,
|
streaming: bool = False,
|
||||||
streaming_prefix_len: int = 4,
|
streaming_prefix_len: int = 4,
|
||||||
|
seed: Optional[int] = None,
|
||||||
) -> Generator[torch.Tensor, None, None]:
|
) -> Generator[torch.Tensor, None, None]:
|
||||||
if retry_badcase and streaming:
|
if retry_badcase and streaming:
|
||||||
warnings.warn("Retry on bad cases is not supported in streaming mode, setting retry_badcase=False.")
|
warnings.warn("Retry on bad cases is not supported in streaming mode, setting retry_badcase=False.")
|
||||||
@@ -633,7 +639,12 @@ class VoxCPM2Model(nn.Module):
|
|||||||
target_text_length = len(self.text_tokenizer(target_text))
|
target_text_length = len(self.text_tokenizer(target_text))
|
||||||
|
|
||||||
retry_badcase_times = 0
|
retry_badcase_times = 0
|
||||||
|
current_seed = materialize_generation_seed(seed)
|
||||||
|
last_attempt_seed = current_seed
|
||||||
while retry_badcase_times < retry_badcase_max_times:
|
while retry_badcase_times < retry_badcase_max_times:
|
||||||
|
last_attempt_seed = current_seed
|
||||||
|
apply_generation_seed(last_attempt_seed)
|
||||||
|
|
||||||
inference_result = self._inference(
|
inference_result = self._inference(
|
||||||
text_token,
|
text_token,
|
||||||
text_mask,
|
text_mask,
|
||||||
@@ -651,6 +662,7 @@ class VoxCPM2Model(nn.Module):
|
|||||||
for latent_pred, _, _ctx in inference_result:
|
for latent_pred, _, _ctx in inference_result:
|
||||||
decode_audio = vae_dec.decode_chunk(latent_pred.to(torch.float32))
|
decode_audio = vae_dec.decode_chunk(latent_pred.to(torch.float32))
|
||||||
decode_audio = decode_audio.squeeze(1).cpu()
|
decode_audio = decode_audio.squeeze(1).cpu()
|
||||||
|
self.last_successful_seed = last_attempt_seed
|
||||||
yield decode_audio
|
yield decode_audio
|
||||||
break
|
break
|
||||||
else:
|
else:
|
||||||
@@ -662,6 +674,7 @@ class VoxCPM2Model(nn.Module):
|
|||||||
file=sys.stderr,
|
file=sys.stderr,
|
||||||
)
|
)
|
||||||
retry_badcase_times += 1
|
retry_badcase_times += 1
|
||||||
|
current_seed += 1
|
||||||
continue
|
continue
|
||||||
else:
|
else:
|
||||||
break
|
break
|
||||||
@@ -669,10 +682,11 @@ class VoxCPM2Model(nn.Module):
|
|||||||
break
|
break
|
||||||
|
|
||||||
if not streaming:
|
if not streaming:
|
||||||
|
self.last_successful_seed = last_attempt_seed
|
||||||
decode_audio = self.audio_vae.decode(latent_pred.to(torch.float32))
|
decode_audio = self.audio_vae.decode(latent_pred.to(torch.float32))
|
||||||
decode_patch_len = self.patch_size * self._decode_chunk_size
|
decode_patch_len = self.patch_size * self._decode_chunk_size
|
||||||
if context_len > 0:
|
if context_len > 0:
|
||||||
decode_audio = decode_audio[..., decode_patch_len * context_len:].squeeze(1).cpu()
|
decode_audio = decode_audio[..., decode_patch_len * context_len :].squeeze(1).cpu()
|
||||||
else:
|
else:
|
||||||
decode_audio = decode_audio.squeeze(1).cpu()
|
decode_audio = decode_audio.squeeze(1).cpu()
|
||||||
yield decode_audio
|
yield decode_audio
|
||||||
@@ -793,6 +807,7 @@ class VoxCPM2Model(nn.Module):
|
|||||||
retry_badcase_ratio_threshold: float = 6.0,
|
retry_badcase_ratio_threshold: float = 6.0,
|
||||||
streaming: bool = False,
|
streaming: bool = False,
|
||||||
streaming_prefix_len: int = 4,
|
streaming_prefix_len: int = 4,
|
||||||
|
seed: Optional[int] = None,
|
||||||
) -> Generator[Tuple[torch.Tensor, torch.Tensor, Union[torch.Tensor, List[torch.Tensor]]], None, None]:
|
) -> Generator[Tuple[torch.Tensor, torch.Tensor, Union[torch.Tensor, List[torch.Tensor]]], None, None]:
|
||||||
"""
|
"""
|
||||||
Generate audio using pre-built prompt cache.
|
Generate audio using pre-built prompt cache.
|
||||||
@@ -920,7 +935,12 @@ class VoxCPM2Model(nn.Module):
|
|||||||
# run inference
|
# run inference
|
||||||
target_text_length = len(self.text_tokenizer(target_text))
|
target_text_length = len(self.text_tokenizer(target_text))
|
||||||
retry_badcase_times = 0
|
retry_badcase_times = 0
|
||||||
|
current_seed = materialize_generation_seed(seed)
|
||||||
|
last_attempt_seed = current_seed
|
||||||
while retry_badcase_times < retry_badcase_max_times:
|
while retry_badcase_times < retry_badcase_max_times:
|
||||||
|
last_attempt_seed = current_seed
|
||||||
|
apply_generation_seed(last_attempt_seed)
|
||||||
|
|
||||||
inference_result = self._inference(
|
inference_result = self._inference(
|
||||||
text_token,
|
text_token,
|
||||||
text_mask,
|
text_mask,
|
||||||
@@ -938,6 +958,7 @@ class VoxCPM2Model(nn.Module):
|
|||||||
for latent_pred, pred_audio_feat, _ctx in inference_result:
|
for latent_pred, pred_audio_feat, _ctx in inference_result:
|
||||||
decode_audio = vae_dec.decode_chunk(latent_pred.to(torch.float32))
|
decode_audio = vae_dec.decode_chunk(latent_pred.to(torch.float32))
|
||||||
decode_audio = decode_audio.squeeze(1).cpu()
|
decode_audio = decode_audio.squeeze(1).cpu()
|
||||||
|
self.last_successful_seed = last_attempt_seed
|
||||||
yield (decode_audio, target_text_token, pred_audio_feat)
|
yield (decode_audio, target_text_token, pred_audio_feat)
|
||||||
break
|
break
|
||||||
else:
|
else:
|
||||||
@@ -949,16 +970,18 @@ class VoxCPM2Model(nn.Module):
|
|||||||
file=sys.stderr,
|
file=sys.stderr,
|
||||||
)
|
)
|
||||||
retry_badcase_times += 1
|
retry_badcase_times += 1
|
||||||
|
current_seed += 1
|
||||||
continue
|
continue
|
||||||
else:
|
else:
|
||||||
break
|
break
|
||||||
else:
|
else:
|
||||||
break
|
break
|
||||||
if not streaming:
|
if not streaming:
|
||||||
|
self.last_successful_seed = last_attempt_seed
|
||||||
decode_audio = self.audio_vae.decode(latent_pred.to(torch.float32))
|
decode_audio = self.audio_vae.decode(latent_pred.to(torch.float32))
|
||||||
decode_patch_len = self.patch_size * self._decode_chunk_size
|
decode_patch_len = self.patch_size * self._decode_chunk_size
|
||||||
if context_len > 0:
|
if context_len > 0:
|
||||||
decode_audio = decode_audio[..., decode_patch_len * context_len:].squeeze(1).cpu()
|
decode_audio = decode_audio[..., decode_patch_len * context_len :].squeeze(1).cpu()
|
||||||
else:
|
else:
|
||||||
decode_audio = decode_audio.squeeze(1).cpu()
|
decode_audio = decode_audio.squeeze(1).cpu()
|
||||||
yield (decode_audio, target_text_token, pred_audio_feat)
|
yield (decode_audio, target_text_token, pred_audio_feat)
|
||||||
|
|||||||
@@ -551,8 +551,7 @@ class StreamingVAEDecoder:
|
|||||||
if x.shape[-1] >= _p:
|
if x.shape[-1] >= _p:
|
||||||
states[_k] = x[:, :, -_p:].detach()
|
states[_k] = x[:, :, -_p:].detach()
|
||||||
else:
|
else:
|
||||||
prev = states.get(_k, torch.zeros(x.shape[0], x.shape[1], _p,
|
prev = states.get(_k, torch.zeros(x.shape[0], x.shape[1], _p, device=x.device, dtype=x.dtype))
|
||||||
device=x.device, dtype=x.dtype))
|
|
||||||
states[_k] = torch.cat([prev, x], dim=-1)[:, :, -_p:].detach()
|
states[_k] = torch.cat([prev, x], dim=-1)[:, :, -_p:].detach()
|
||||||
return nn.Conv1d.forward(_m, x_pad)
|
return nn.Conv1d.forward(_m, x_pad)
|
||||||
|
|
||||||
|
|||||||
@@ -350,45 +350,80 @@ class AudioFeatureProcessingPacker:
|
|||||||
|
|
||||||
# -- text token track --
|
# -- text token track --
|
||||||
# [103, 0×R, 104, text_ids, 101, 0×A, 102]
|
# [103, 0×R, 104, text_ids, 101, 0×A, 102]
|
||||||
text_token_info = torch.cat([
|
text_token_info = torch.cat(
|
||||||
_tok([self.audio_prompt_start_id]),
|
[
|
||||||
torch.zeros(ref_len, dtype=torch.int32, device=device),
|
_tok([self.audio_prompt_start_id]),
|
||||||
_tok([self.audio_prompt_end_id]),
|
torch.zeros(ref_len, dtype=torch.int32, device=device),
|
||||||
text_token,
|
_tok([self.audio_prompt_end_id]),
|
||||||
_tok([self.audio_start_id]),
|
text_token,
|
||||||
torch.zeros(tgt_len, dtype=torch.int32, device=device),
|
_tok([self.audio_start_id]),
|
||||||
_tok([self.audio_end_id]),
|
torch.zeros(tgt_len, dtype=torch.int32, device=device),
|
||||||
])
|
_tok([self.audio_end_id]),
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
# -- audio feature track --
|
# -- audio feature track --
|
||||||
zero_1 = torch.zeros((1,) + feat_shape, dtype=torch.float32, device=device)
|
zero_1 = torch.zeros((1,) + feat_shape, dtype=torch.float32, device=device)
|
||||||
zero_txt = torch.zeros((txt_len,) + feat_shape, dtype=torch.float32, device=device)
|
zero_txt = torch.zeros((txt_len,) + feat_shape, dtype=torch.float32, device=device)
|
||||||
audio_feat_info = torch.cat([
|
audio_feat_info = torch.cat(
|
||||||
zero_1, ref_feats, zero_1, # 103, ref, 104
|
[
|
||||||
zero_txt, # text
|
zero_1,
|
||||||
zero_1, tgt_feats, zero_1, # 101, target, 102
|
ref_feats,
|
||||||
], dim=0)
|
zero_1, # 103, ref, 104
|
||||||
|
zero_txt, # text
|
||||||
|
zero_1,
|
||||||
|
tgt_feats,
|
||||||
|
zero_1, # 101, target, 102
|
||||||
|
],
|
||||||
|
dim=0,
|
||||||
|
)
|
||||||
|
|
||||||
# -- masks --
|
# -- masks --
|
||||||
text_mask = torch.cat([
|
text_mask = (
|
||||||
torch.ones(1), torch.zeros(ref_len), torch.ones(1),
|
torch.cat(
|
||||||
torch.ones(txt_len),
|
[
|
||||||
torch.ones(1), torch.zeros(tgt_len), torch.ones(1),
|
torch.ones(1),
|
||||||
]).to(torch.int32).to(device)
|
torch.zeros(ref_len),
|
||||||
|
torch.ones(1),
|
||||||
|
torch.ones(txt_len),
|
||||||
|
torch.ones(1),
|
||||||
|
torch.zeros(tgt_len),
|
||||||
|
torch.ones(1),
|
||||||
|
]
|
||||||
|
)
|
||||||
|
.to(torch.int32)
|
||||||
|
.to(device)
|
||||||
|
)
|
||||||
|
|
||||||
audio_mask = torch.cat([
|
audio_mask = (
|
||||||
torch.zeros(1), torch.ones(ref_len), torch.zeros(1),
|
torch.cat(
|
||||||
torch.zeros(txt_len),
|
[
|
||||||
torch.zeros(1), torch.ones(tgt_len), torch.zeros(1),
|
torch.zeros(1),
|
||||||
]).to(torch.int32).to(device)
|
torch.ones(ref_len),
|
||||||
|
torch.zeros(1),
|
||||||
|
torch.zeros(txt_len),
|
||||||
|
torch.zeros(1),
|
||||||
|
torch.ones(tgt_len),
|
||||||
|
torch.zeros(1),
|
||||||
|
]
|
||||||
|
)
|
||||||
|
.to(torch.int32)
|
||||||
|
.to(device)
|
||||||
|
)
|
||||||
|
|
||||||
loss_mask = torch.cat([
|
loss_mask = (
|
||||||
torch.zeros(1 + ref_len + 1), # ref part: no loss
|
torch.cat(
|
||||||
torch.zeros(txt_len), # text: no loss
|
[
|
||||||
torch.zeros(1), # 101: no loss
|
torch.zeros(1 + ref_len + 1), # ref part: no loss
|
||||||
torch.ones(tgt_len), # target audio: LOSS
|
torch.zeros(txt_len), # text: no loss
|
||||||
torch.zeros(1), # 102: no loss
|
torch.zeros(1), # 101: no loss
|
||||||
]).to(torch.int32).to(device)
|
torch.ones(tgt_len), # target audio: LOSS
|
||||||
|
torch.zeros(1), # 102: no loss
|
||||||
|
]
|
||||||
|
)
|
||||||
|
.to(torch.int32)
|
||||||
|
.to(device)
|
||||||
|
)
|
||||||
|
|
||||||
total_len = 1 + ref_len + 1 + txt_len + 1 + tgt_len + 1
|
total_len = 1 + ref_len + 1 + txt_len + 1 + tgt_len + 1
|
||||||
labels = torch.zeros(total_len, dtype=torch.int32, device=device)
|
labels = torch.zeros(total_len, dtype=torch.int32, device=device)
|
||||||
|
|||||||
@@ -44,10 +44,7 @@ def _check_audio_file(audio_path: str, sample_rate: int) -> Optional[str]:
|
|||||||
if info.frames == 0:
|
if info.frames == 0:
|
||||||
return f"Audio file is empty: {audio_path}"
|
return f"Audio file is empty: {audio_path}"
|
||||||
if info.samplerate != sample_rate:
|
if info.samplerate != sample_rate:
|
||||||
return (
|
return f"Sample rate mismatch in {audio_path}: " f"expected {sample_rate} Hz, got {info.samplerate} Hz"
|
||||||
f"Sample rate mismatch in {audio_path}: "
|
|
||||||
f"expected {sample_rate} Hz, got {info.samplerate} Hz"
|
|
||||||
)
|
|
||||||
return None
|
return None
|
||||||
except ImportError:
|
except ImportError:
|
||||||
# soundfile not available; just check existence
|
# soundfile not available; just check existence
|
||||||
@@ -187,13 +184,9 @@ def validate_manifest(
|
|||||||
if duration is not None:
|
if duration is not None:
|
||||||
result.audio_durations.append(duration)
|
result.audio_durations.append(duration)
|
||||||
if duration < 0.3:
|
if duration < 0.3:
|
||||||
result.warnings.append(
|
result.warnings.append(f"Line {i + 1}: Very short audio ({duration:.2f}s)")
|
||||||
f"Line {i + 1}: Very short audio ({duration:.2f}s)"
|
|
||||||
)
|
|
||||||
elif duration > 30.0:
|
elif duration > 30.0:
|
||||||
result.warnings.append(
|
result.warnings.append(f"Line {i + 1}: Very long audio ({duration:.1f}s), may cause OOM")
|
||||||
f"Line {i + 1}: Very long audio ({duration:.1f}s), may cause OOM"
|
|
||||||
)
|
|
||||||
else:
|
else:
|
||||||
result.errors.append(f"Line {i + 1}: Invalid audio path")
|
result.errors.append(f"Line {i + 1}: Invalid audio path")
|
||||||
has_error = True
|
has_error = True
|
||||||
@@ -209,9 +202,7 @@ def validate_manifest(
|
|||||||
if os.path.isfile(ref_path):
|
if os.path.isfile(ref_path):
|
||||||
result.has_ref_audio += 1
|
result.has_ref_audio += 1
|
||||||
else:
|
else:
|
||||||
result.warnings.append(
|
result.warnings.append(f"Line {i + 1}: ref_audio file not found: {ref_path}")
|
||||||
f"Line {i + 1}: ref_audio file not found: {ref_path}"
|
|
||||||
)
|
|
||||||
|
|
||||||
if not has_error:
|
if not has_error:
|
||||||
result.valid_samples += 1
|
result.valid_samples += 1
|
||||||
@@ -222,14 +213,10 @@ def validate_manifest(
|
|||||||
# Summarize truncated errors
|
# Summarize truncated errors
|
||||||
if missing_audio_count > 5:
|
if missing_audio_count > 5:
|
||||||
result.errors.append(
|
result.errors.append(
|
||||||
f"... and {missing_audio_count - 5} more missing audio files "
|
f"... and {missing_audio_count - 5} more missing audio files " f"({missing_audio_count} total)"
|
||||||
f"({missing_audio_count} total)"
|
|
||||||
)
|
)
|
||||||
if empty_text_count > 5:
|
if empty_text_count > 5:
|
||||||
result.warnings.append(
|
result.warnings.append(f"... and {empty_text_count - 5} more empty text entries " f"({empty_text_count} total)")
|
||||||
f"... and {empty_text_count - 5} more empty text entries "
|
|
||||||
f"({empty_text_count} total)"
|
|
||||||
)
|
|
||||||
|
|
||||||
return result
|
return result
|
||||||
|
|
||||||
|
|||||||
@@ -676,3 +676,61 @@ def test_detect_model_architecture_uses_local_configs():
|
|||||||
|
|
||||||
assert cli.detect_model_architecture(v1_args) == "voxcpm"
|
assert cli.detect_model_architecture(v1_args) == "voxcpm"
|
||||||
assert cli.detect_model_architecture(v2_args) == "voxcpm2"
|
assert cli.detect_model_architecture(v2_args) == "voxcpm2"
|
||||||
|
|
||||||
|
|
||||||
|
def test_parser_accepts_seed():
|
||||||
|
parser = cli._build_parser()
|
||||||
|
# Default seed should be None
|
||||||
|
args = parser.parse_args(["design", "--text", "hello", "--output", "out.wav"])
|
||||||
|
assert args.seed is None
|
||||||
|
|
||||||
|
# Custom seed should be parsed as int
|
||||||
|
args = parser.parse_args(["design", "--text", "hello", "--output", "out.wav", "--seed", "42"])
|
||||||
|
assert args.seed == 42
|
||||||
|
|
||||||
|
|
||||||
|
def test_design_subcommand_passes_seed(monkeypatch, tmp_path):
|
||||||
|
dummy_model = DummyModel()
|
||||||
|
monkeypatch.setattr(cli, "load_model", lambda args: dummy_model)
|
||||||
|
patch_soundfile_write(monkeypatch)
|
||||||
|
|
||||||
|
run_main(
|
||||||
|
monkeypatch,
|
||||||
|
[
|
||||||
|
"design",
|
||||||
|
"--text",
|
||||||
|
"hello",
|
||||||
|
"--seed",
|
||||||
|
"123",
|
||||||
|
"--output",
|
||||||
|
str(tmp_path / "out.wav"),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|
||||||
|
assert dummy_model.calls[0]["seed"] == 123
|
||||||
|
|
||||||
|
|
||||||
|
def test_batch_subcommand_passes_seed(monkeypatch, tmp_path):
|
||||||
|
dummy_model = DummyModel()
|
||||||
|
input_file = tmp_path / "texts.txt"
|
||||||
|
input_file.write_text("hello\nworld\n", encoding="utf-8")
|
||||||
|
|
||||||
|
monkeypatch.setattr(cli, "load_model", lambda args: dummy_model)
|
||||||
|
patch_soundfile_write(monkeypatch)
|
||||||
|
|
||||||
|
run_main(
|
||||||
|
monkeypatch,
|
||||||
|
[
|
||||||
|
"batch",
|
||||||
|
"--input",
|
||||||
|
str(input_file),
|
||||||
|
"--output-dir",
|
||||||
|
str(tmp_path / "outs"),
|
||||||
|
"--seed",
|
||||||
|
"999",
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|
||||||
|
assert len(dummy_model.calls) == 2
|
||||||
|
assert dummy_model.calls[0]["seed"] == 999
|
||||||
|
assert dummy_model.calls[1]["seed"] == 999
|
||||||
|
|||||||
@@ -47,3 +47,25 @@ def test_resolve_runtime_device_rejects_unavailable_explicit_cuda(monkeypatch):
|
|||||||
|
|
||||||
with pytest.raises(ValueError, match="CUDA is not available"):
|
with pytest.raises(ValueError, match="CUDA is not available"):
|
||||||
utils.resolve_runtime_device("cuda:0", "cuda")
|
utils.resolve_runtime_device("cuda:0", "cuda")
|
||||||
|
|
||||||
|
|
||||||
|
def test_materialize_generation_seed_preserves_explicit_seed():
|
||||||
|
assert utils.materialize_generation_seed(42) == 42
|
||||||
|
|
||||||
|
|
||||||
|
def test_materialize_generation_seed_creates_concrete_seed_for_none(monkeypatch):
|
||||||
|
monkeypatch.setattr(utils.torch, "seed", lambda: 0x123456789)
|
||||||
|
|
||||||
|
assert utils.materialize_generation_seed(None) == 0x23456789
|
||||||
|
|
||||||
|
|
||||||
|
def test_apply_generation_seed_sets_cpu_and_cuda_rng(monkeypatch):
|
||||||
|
calls = []
|
||||||
|
|
||||||
|
monkeypatch.setattr(utils.torch, "manual_seed", lambda seed: calls.append(("cpu", seed)))
|
||||||
|
monkeypatch.setattr(utils.torch.cuda, "is_available", lambda: True)
|
||||||
|
monkeypatch.setattr(utils.torch.cuda, "manual_seed_all", lambda seed: calls.append(("cuda", seed)))
|
||||||
|
|
||||||
|
utils.apply_generation_seed(123)
|
||||||
|
|
||||||
|
assert calls == [("cpu", 123), ("cuda", 123)]
|
||||||
|
|||||||
@@ -207,6 +207,7 @@ class TestValidateManifest:
|
|||||||
|
|
||||||
audio = tmp_dir / "audio_8k.wav"
|
audio = tmp_dir / "audio_8k.wav"
|
||||||
import numpy as np
|
import numpy as np
|
||||||
|
|
||||||
samples = np.zeros(8000, dtype=np.float32)
|
samples = np.zeros(8000, dtype=np.float32)
|
||||||
sf.write(str(audio), samples, 8000)
|
sf.write(str(audio), samples, 8000)
|
||||||
|
|
||||||
@@ -240,6 +241,7 @@ class TestValidateManifest:
|
|||||||
def test_cli_validate_exit_code(self, tmp_dir):
|
def test_cli_validate_exit_code(self, tmp_dir):
|
||||||
"""validate subcommand must exit 1 on validation error (missing audio)."""
|
"""validate subcommand must exit 1 on validation error (missing audio)."""
|
||||||
import subprocess
|
import subprocess
|
||||||
|
|
||||||
manifest = tmp_dir / "bad.jsonl"
|
manifest = tmp_dir / "bad.jsonl"
|
||||||
_write_manifest(manifest, [{"text": "hi", "audio": "/nonexistent/x.wav"}])
|
_write_manifest(manifest, [{"text": "hi", "audio": "/nonexistent/x.wav"}])
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user