mirror of
https://github.com/OpenBMB/VoxCPM.git
synced 2026-09-20 10:53:32 +08:00
fix: formatting/trailing whitespace
This commit is contained in:
21
app.py
21
app.py
@@ -45,8 +45,8 @@ _EXAMPLES_FOOTER_EN = (
|
|||||||
"**Example 1 — Gentle & Melancholic Girl** \n"
|
"**Example 1 — Gentle & Melancholic Girl** \n"
|
||||||
'`Control Instruction`: *"A young girl with a soft, sweet voice. '
|
'`Control Instruction`: *"A young girl with a soft, sweet voice. '
|
||||||
'Speaks slowly with a melancholic, slightly tsundere tone."* \n'
|
'Speaks slowly with a melancholic, slightly tsundere tone."* \n'
|
||||||
'`Target Text`: *"I never asked you to stay… It\'s not like I care or anything. '
|
"`Target Text`: *\"I never asked you to stay… It's not like I care or anything. "
|
||||||
'But… why does it still hurt so much now that you\'re gone?"* \n\n'
|
"But… why does it still hurt so much now that you're gone?\"* \n\n"
|
||||||
"**Example 2 — Laid-Back Surfer Dude** \n"
|
"**Example 2 — Laid-Back Surfer Dude** \n"
|
||||||
'`Control Instruction`: *"Relaxed young male voice, slightly nasal, '
|
'`Control Instruction`: *"Relaxed young male voice, slightly nasal, '
|
||||||
'lazy drawl, very casual and chill."* \n'
|
'lazy drawl, very casual and chill."* \n'
|
||||||
@@ -147,7 +147,7 @@ _I18N_TRANSLATIONS = {
|
|||||||
"examples_footer": _EXAMPLES_FOOTER_ZH,
|
"examples_footer": _EXAMPLES_FOOTER_ZH,
|
||||||
},
|
},
|
||||||
"zh-Hans": None, # alias, filled below
|
"zh-Hans": None, # alias, filled below
|
||||||
"zh": None, # alias, filled below
|
"zh": None, # alias, filled below
|
||||||
}
|
}
|
||||||
_I18N_TRANSLATIONS["zh-Hans"] = _I18N_TRANSLATIONS["zh-CN"]
|
_I18N_TRANSLATIONS["zh-Hans"] = _I18N_TRANSLATIONS["zh-CN"]
|
||||||
_I18N_TRANSLATIONS["zh"] = _I18N_TRANSLATIONS["zh-CN"]
|
_I18N_TRANSLATIONS["zh"] = _I18N_TRANSLATIONS["zh-CN"]
|
||||||
@@ -160,8 +160,7 @@ for _d in _I18N_TRANSLATIONS.values():
|
|||||||
I18N = gr.I18n(**_I18N_TRANSLATIONS)
|
I18N = gr.I18n(**_I18N_TRANSLATIONS)
|
||||||
|
|
||||||
DEFAULT_TARGET_TEXT = (
|
DEFAULT_TARGET_TEXT = (
|
||||||
"VoxCPM2 is a creative multilingual TTS model from ModelBest, "
|
"VoxCPM2 is a creative multilingual TTS model from ModelBest, " "designed to generate highly realistic speech."
|
||||||
"designed to generate highly realistic speech."
|
|
||||||
)
|
)
|
||||||
|
|
||||||
_CUSTOM_CSS = """
|
_CUSTOM_CSS = """
|
||||||
@@ -224,6 +223,7 @@ _APP_THEME = gr.themes.Soft(
|
|||||||
|
|
||||||
# ---------- Model ----------
|
# ---------- Model ----------
|
||||||
|
|
||||||
|
|
||||||
class VoxCPMDemo:
|
class VoxCPMDemo:
|
||||||
def __init__(self, model_id: str = "openbmb/VoxCPM2", device: str = "auto") -> None:
|
def __init__(self, model_id: str = "openbmb/VoxCPM2", device: str = "auto") -> None:
|
||||||
self.device = resolve_runtime_device(device, "cuda")
|
self.device = resolve_runtime_device(device, "cuda")
|
||||||
@@ -252,9 +252,7 @@ class VoxCPMDemo:
|
|||||||
def get_or_load_asr_model(self) -> AutoModel:
|
def get_or_load_asr_model(self) -> AutoModel:
|
||||||
if self.asr_model is not None:
|
if self.asr_model is not None:
|
||||||
return self.asr_model
|
return self.asr_model
|
||||||
logger.info(
|
logger.info(f"Loading ASR model: {self.asr_model_id} on device: {self.asr_device}")
|
||||||
f"Loading ASR model: {self.asr_model_id} on device: {self.asr_device}"
|
|
||||||
)
|
|
||||||
self.asr_model = AutoModel(
|
self.asr_model = AutoModel(
|
||||||
model=self.asr_model_id,
|
model=self.asr_model_id,
|
||||||
disable_update=True,
|
disable_update=True,
|
||||||
@@ -352,6 +350,7 @@ class VoxCPMDemo:
|
|||||||
|
|
||||||
# ---------- UI ----------
|
# ---------- UI ----------
|
||||||
|
|
||||||
|
|
||||||
def create_demo_interface(demo: VoxCPMDemo):
|
def create_demo_interface(demo: VoxCPMDemo):
|
||||||
gr.set_static_paths(paths=[Path.cwd().absolute() / "assets"])
|
gr.set_static_paths(paths=[Path.cwd().absolute() / "assets"])
|
||||||
|
|
||||||
@@ -554,6 +553,7 @@ def create_demo_interface(demo: VoxCPMDemo):
|
|||||||
|
|
||||||
return interface
|
return interface
|
||||||
|
|
||||||
|
|
||||||
def run_demo(
|
def run_demo(
|
||||||
server_name: str = "0.0.0.0",
|
server_name: str = "0.0.0.0",
|
||||||
server_port: int = 8808,
|
server_port: int = 8808,
|
||||||
@@ -575,9 +575,12 @@ def run_demo(
|
|||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
import argparse
|
import argparse
|
||||||
|
|
||||||
parser = argparse.ArgumentParser()
|
parser = argparse.ArgumentParser()
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
"--model-id", type=str, default="openbmb/VoxCPM2",
|
"--model-id",
|
||||||
|
type=str,
|
||||||
|
default="openbmb/VoxCPM2",
|
||||||
help="Local path or HuggingFace repo ID (default: openbmb/VoxCPM2)",
|
help="Local path or HuggingFace repo ID (default: openbmb/VoxCPM2)",
|
||||||
)
|
)
|
||||||
parser.add_argument("--port", type=int, default=8808, help="Server port")
|
parser.add_argument("--port", type=int, default=8808, help="Server port")
|
||||||
|
|||||||
36
app_old.py
36
app_old.py
@@ -6,6 +6,7 @@ import gradio as gr
|
|||||||
from typing import Optional, Tuple
|
from typing import Optional, Tuple
|
||||||
from funasr import AutoModel
|
from funasr import AutoModel
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
os.environ["TOKENIZERS_PARALLELISM"] = "false"
|
os.environ["TOKENIZERS_PARALLELISM"] = "false"
|
||||||
if os.environ.get("HF_REPO_ID", "").strip() == "":
|
if os.environ.get("HF_REPO_ID", "").strip() == "":
|
||||||
os.environ["HF_REPO_ID"] = "openbmb/VoxCPM1.5"
|
os.environ["HF_REPO_ID"] = "openbmb/VoxCPM1.5"
|
||||||
@@ -23,7 +24,7 @@ class VoxCPMDemo:
|
|||||||
self.asr_model: Optional[AutoModel] = AutoModel(
|
self.asr_model: Optional[AutoModel] = AutoModel(
|
||||||
model=self.asr_model_id,
|
model=self.asr_model_id,
|
||||||
disable_update=True,
|
disable_update=True,
|
||||||
log_level='DEBUG',
|
log_level="DEBUG",
|
||||||
device="cuda:0" if self.device == "cuda" else "cpu",
|
device="cuda:0" if self.device == "cuda" else "cpu",
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -48,6 +49,7 @@ class VoxCPMDemo:
|
|||||||
if not os.path.isdir(target_dir):
|
if not os.path.isdir(target_dir):
|
||||||
try:
|
try:
|
||||||
from huggingface_hub import snapshot_download # type: ignore
|
from huggingface_hub import snapshot_download # type: ignore
|
||||||
|
|
||||||
os.makedirs(target_dir, exist_ok=True)
|
os.makedirs(target_dir, exist_ok=True)
|
||||||
print(f"Downloading model from HF repo '{repo_id}' to '{target_dir}' ...", file=sys.stderr)
|
print(f"Downloading model from HF repo '{repo_id}' to '{target_dir}' ...", file=sys.stderr)
|
||||||
snapshot_download(repo_id=repo_id, local_dir=target_dir, local_dir_use_symlinks=False)
|
snapshot_download(repo_id=repo_id, local_dir=target_dir, local_dir_use_symlinks=False)
|
||||||
@@ -72,7 +74,7 @@ class VoxCPMDemo:
|
|||||||
if prompt_wav is None:
|
if prompt_wav is None:
|
||||||
return ""
|
return ""
|
||||||
res = self.asr_model.generate(input=prompt_wav, language="auto", use_itn=True)
|
res = self.asr_model.generate(input=prompt_wav, language="auto", use_itn=True)
|
||||||
text = res[0]["text"].split('|>')[-1]
|
text = res[0]["text"].split("|>")[-1]
|
||||||
return text
|
return text
|
||||||
|
|
||||||
def generate_tts_audio(
|
def generate_tts_audio(
|
||||||
@@ -149,11 +151,13 @@ _CUSTOM_CSS = """
|
|||||||
|
|
||||||
def create_demo_interface(demo: VoxCPMDemo):
|
def create_demo_interface(demo: VoxCPMDemo):
|
||||||
"""Build the Gradio UI for VoxCPM demo."""
|
"""Build the Gradio UI for VoxCPM demo."""
|
||||||
gr.set_static_paths(paths=[Path.cwd().absolute()/"assets"])
|
gr.set_static_paths(paths=[Path.cwd().absolute() / "assets"])
|
||||||
|
|
||||||
with gr.Blocks() as interface:
|
with gr.Blocks() as interface:
|
||||||
# Header logo
|
# Header logo
|
||||||
gr.HTML('<div class="logo-container"><img src="/gradio_api/file=assets/voxcpm_logo.png" alt="VoxCPM Logo"></div>')
|
gr.HTML(
|
||||||
|
'<div class="logo-container"><img src="/gradio_api/file=assets/voxcpm_logo.png" alt="VoxCPM Logo"></div>'
|
||||||
|
)
|
||||||
|
|
||||||
# Quick Start
|
# Quick Start
|
||||||
with gr.Accordion("📋 Quick Start Guide |快速入门", open=False, elem_id="acc_quick"):
|
with gr.Accordion("📋 Quick Start Guide |快速入门", open=False, elem_id="acc_quick"):
|
||||||
@@ -201,7 +205,7 @@ def create_demo_interface(demo: VoxCPMDemo):
|
|||||||
with gr.Row():
|
with gr.Row():
|
||||||
with gr.Column():
|
with gr.Column():
|
||||||
prompt_wav = gr.Audio(
|
prompt_wav = gr.Audio(
|
||||||
sources=["upload", 'microphone'],
|
sources=["upload", "microphone"],
|
||||||
type="filepath",
|
type="filepath",
|
||||||
label="Prompt Speech (Optional, or let VoxCPM improvise)",
|
label="Prompt Speech (Optional, or let VoxCPM improvise)",
|
||||||
value="./examples/example.wav",
|
value="./examples/example.wav",
|
||||||
@@ -210,13 +214,13 @@ def create_demo_interface(demo: VoxCPMDemo):
|
|||||||
value=False,
|
value=False,
|
||||||
label="Prompt Speech Enhancement",
|
label="Prompt Speech Enhancement",
|
||||||
elem_id="chk_denoise",
|
elem_id="chk_denoise",
|
||||||
info="We use ZipEnhancer model to denoise the prompt audio."
|
info="We use ZipEnhancer model to denoise the prompt audio.",
|
||||||
)
|
)
|
||||||
with gr.Row():
|
with gr.Row():
|
||||||
prompt_text = gr.Textbox(
|
prompt_text = gr.Textbox(
|
||||||
value="Just by listening a few minutes a day, you'll be able to eliminate negative thoughts by conditioning your mind to be more positive.",
|
value="Just by listening a few minutes a day, you'll be able to eliminate negative thoughts by conditioning your mind to be more positive.",
|
||||||
label="Prompt Text",
|
label="Prompt Text",
|
||||||
placeholder="Please enter the prompt text. Automatic recognition is supported, and you can correct the results yourself..."
|
placeholder="Please enter the prompt text. Automatic recognition is supported, and you can correct the results yourself...",
|
||||||
)
|
)
|
||||||
run_btn = gr.Button("Generate Speech", variant="primary")
|
run_btn = gr.Button("Generate Speech", variant="primary")
|
||||||
|
|
||||||
@@ -227,7 +231,7 @@ def create_demo_interface(demo: VoxCPMDemo):
|
|||||||
value=2.0,
|
value=2.0,
|
||||||
step=0.1,
|
step=0.1,
|
||||||
label="CFG Value (Guidance Scale)",
|
label="CFG Value (Guidance Scale)",
|
||||||
info="Higher values increase adherence to prompt, lower values allow more creativity"
|
info="Higher values increase adherence to prompt, lower values allow more creativity",
|
||||||
)
|
)
|
||||||
inference_timesteps = gr.Slider(
|
inference_timesteps = gr.Slider(
|
||||||
minimum=4,
|
minimum=4,
|
||||||
@@ -235,7 +239,7 @@ def create_demo_interface(demo: VoxCPMDemo):
|
|||||||
value=10,
|
value=10,
|
||||||
step=1,
|
step=1,
|
||||||
label="Inference Timesteps",
|
label="Inference Timesteps",
|
||||||
info="Number of inference timesteps for generation (higher values may improve quality but slower)"
|
info="Number of inference timesteps for generation (higher values may improve quality but slower)",
|
||||||
)
|
)
|
||||||
with gr.Row():
|
with gr.Row():
|
||||||
text = gr.Textbox(
|
text = gr.Textbox(
|
||||||
@@ -247,14 +251,22 @@ def create_demo_interface(demo: VoxCPMDemo):
|
|||||||
value=False,
|
value=False,
|
||||||
label="Text Normalization",
|
label="Text Normalization",
|
||||||
elem_id="chk_normalize",
|
elem_id="chk_normalize",
|
||||||
info="We use wetext library to normalize the input text."
|
info="We use wetext library to normalize the input text.",
|
||||||
)
|
)
|
||||||
audio_output = gr.Audio(label="Output Audio")
|
audio_output = gr.Audio(label="Output Audio")
|
||||||
|
|
||||||
# Wiring
|
# Wiring
|
||||||
run_btn.click(
|
run_btn.click(
|
||||||
fn=demo.generate_tts_audio,
|
fn=demo.generate_tts_audio,
|
||||||
inputs=[text, prompt_wav, prompt_text, cfg_value, inference_timesteps, DoNormalizeText, DoDenoisePromptAudio],
|
inputs=[
|
||||||
|
text,
|
||||||
|
prompt_wav,
|
||||||
|
prompt_text,
|
||||||
|
cfg_value,
|
||||||
|
inference_timesteps,
|
||||||
|
DoNormalizeText,
|
||||||
|
DoDenoisePromptAudio,
|
||||||
|
],
|
||||||
outputs=[audio_output],
|
outputs=[audio_output],
|
||||||
show_progress=True,
|
show_progress=True,
|
||||||
api_name="generate",
|
api_name="generate",
|
||||||
@@ -277,4 +289,4 @@ def run_demo(server_name: str = "localhost", server_port: int = 7860, show_error
|
|||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
run_demo()
|
run_demo()
|
||||||
|
|||||||
@@ -308,7 +308,8 @@ def run_inference(text, prompt_wav, prompt_text, lora_selection, cfg_scale, step
|
|||||||
if new_r is not None and current_r is not None and new_r != current_r:
|
if new_r is not None and current_r is not None and new_r != current_r:
|
||||||
print(f"LoRA rank mismatch (model r={current_r}, checkpoint r={new_r}), reloading...", file=sys.stderr)
|
print(f"LoRA rank mismatch (model r={current_r}, checkpoint r={new_r}), reloading...", file=sys.stderr)
|
||||||
reload_base = (
|
reload_base = (
|
||||||
new_base_model if new_base_model and os.path.exists(new_base_model)
|
new_base_model
|
||||||
|
if new_base_model and os.path.exists(new_base_model)
|
||||||
else (pretrained_path if pretrained_path and pretrained_path.strip() else default_pretrained_path)
|
else (pretrained_path if pretrained_path and pretrained_path.strip() else default_pretrained_path)
|
||||||
)
|
)
|
||||||
try:
|
try:
|
||||||
@@ -982,9 +983,7 @@ with gr.Blocks(title="VoxCPM LoRA WebUI", theme=gr.themes.Soft(), css=custom_css
|
|||||||
|
|
||||||
gr.Markdown("#### 分发选项 (Distribution)")
|
gr.Markdown("#### 分发选项 (Distribution)")
|
||||||
with gr.Row():
|
with gr.Row():
|
||||||
hf_model_id = gr.Textbox(
|
hf_model_id = gr.Textbox(label="HuggingFace Model ID (e.g., openbmb/VoxCPM2)", value="")
|
||||||
label="HuggingFace Model ID (e.g., openbmb/VoxCPM2)", value=""
|
|
||||||
)
|
|
||||||
distribute = gr.Checkbox(label="分发模式 (distribute)", value=False)
|
distribute = gr.Checkbox(label="分发模式 (distribute)", value=False)
|
||||||
|
|
||||||
with gr.Column(scale=2, elem_classes="form-section"):
|
with gr.Column(scale=2, elem_classes="form-section"):
|
||||||
|
|||||||
@@ -3,6 +3,7 @@
|
|||||||
Loads src/voxcpm/model/utils.py directly to avoid the heavy voxcpm package
|
Loads src/voxcpm/model/utils.py directly to avoid the heavy voxcpm package
|
||||||
init. Run with: `python scripts/test_pick_runtime_dtype.py`.
|
init. Run with: `python scripts/test_pick_runtime_dtype.py`.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import importlib.util
|
import importlib.util
|
||||||
import os
|
import os
|
||||||
import pathlib
|
import pathlib
|
||||||
|
|||||||
@@ -143,4 +143,4 @@ def main():
|
|||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
main()
|
main()
|
||||||
|
|||||||
@@ -273,4 +273,4 @@ def main():
|
|||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
main()
|
main()
|
||||||
|
|||||||
@@ -599,7 +599,9 @@ def generate_sample_audio(
|
|||||||
)
|
)
|
||||||
with torch.no_grad():
|
with torch.no_grad():
|
||||||
with autocast_ctx:
|
with autocast_ctx:
|
||||||
generated = unwrapped_model.generate(target_text=text, inference_timesteps=10, cfg_value=2.0, seed=42 )
|
generated = unwrapped_model.generate(
|
||||||
|
target_text=text, inference_timesteps=10, cfg_value=2.0, seed=42
|
||||||
|
)
|
||||||
|
|
||||||
# Restore training setup
|
# Restore training setup
|
||||||
# unwrapped_model.to(torch.float32)
|
# unwrapped_model.to(torch.float32)
|
||||||
|
|||||||
@@ -86,9 +86,7 @@ def resolve_prompt_text(args, parser) -> str | None:
|
|||||||
|
|
||||||
|
|
||||||
def detect_model_architecture(args) -> str | None:
|
def detect_model_architecture(args) -> str | None:
|
||||||
model_location = getattr(args, "model_path", None) or getattr(
|
model_location = getattr(args, "model_path", None) or getattr(args, "hf_model_id", None)
|
||||||
args, "hf_model_id", None
|
|
||||||
)
|
|
||||||
if not model_location:
|
if not model_location:
|
||||||
return None
|
return None
|
||||||
|
|
||||||
@@ -103,11 +101,7 @@ def detect_model_architecture(args) -> str | None:
|
|||||||
model_hint = str(model_location).lower()
|
model_hint = str(model_location).lower()
|
||||||
if "voxcpm2" in model_hint:
|
if "voxcpm2" in model_hint:
|
||||||
return "voxcpm2"
|
return "voxcpm2"
|
||||||
if (
|
if "voxcpm1.5" in model_hint or "voxcpm-1.5" in model_hint or "voxcpm_1.5" in model_hint:
|
||||||
"voxcpm1.5" in model_hint
|
|
||||||
or "voxcpm-1.5" in model_hint
|
|
||||||
or "voxcpm_1.5" in model_hint
|
|
||||||
):
|
|
||||||
return "voxcpm"
|
return "voxcpm"
|
||||||
|
|
||||||
return None
|
return None
|
||||||
@@ -121,9 +115,7 @@ def validate_prompt_related_args(args, parser, prompt_text: str | None):
|
|||||||
parser.error("--prompt-audio requires --prompt-text or --prompt-file.")
|
parser.error("--prompt-audio requires --prompt-text or --prompt-file.")
|
||||||
|
|
||||||
if args.control and prompt_text:
|
if args.control and prompt_text:
|
||||||
parser.error(
|
parser.error("--control cannot be used together with --prompt-text or --prompt-file.")
|
||||||
"--control cannot be used together with --prompt-text or --prompt-file."
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def validate_reference_support(args, parser):
|
def validate_reference_support(args, parser):
|
||||||
@@ -138,9 +130,7 @@ def validate_reference_support(args, parser):
|
|||||||
def validate_design_args(args, parser):
|
def validate_design_args(args, parser):
|
||||||
prompt_text = resolve_prompt_text(args, parser)
|
prompt_text = resolve_prompt_text(args, parser)
|
||||||
if args.prompt_audio or args.reference_audio or prompt_text:
|
if args.prompt_audio or args.reference_audio or prompt_text:
|
||||||
parser.error(
|
parser.error("`design` does not accept prompt/reference audio. Use `clone` instead.")
|
||||||
"`design` does not accept prompt/reference audio. Use `clone` instead."
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def validate_clone_args(args, parser):
|
def validate_clone_args(args, parser):
|
||||||
@@ -149,9 +139,7 @@ def validate_clone_args(args, parser):
|
|||||||
validate_reference_support(args, parser)
|
validate_reference_support(args, parser)
|
||||||
|
|
||||||
if not args.prompt_audio and not args.reference_audio:
|
if not args.prompt_audio and not args.reference_audio:
|
||||||
parser.error(
|
parser.error("`clone` requires --reference-audio, or --prompt-audio with --prompt-text/--prompt-file.")
|
||||||
"`clone` requires --reference-audio, or --prompt-audio with --prompt-text/--prompt-file."
|
|
||||||
)
|
|
||||||
|
|
||||||
return prompt_text
|
return prompt_text
|
||||||
|
|
||||||
@@ -173,9 +161,7 @@ def load_model(args):
|
|||||||
|
|
||||||
print("Loading VoxCPM model...", file=sys.stderr)
|
print("Loading VoxCPM model...", file=sys.stderr)
|
||||||
|
|
||||||
zipenhancer_path = getattr(args, "zipenhancer_path", None) or os.environ.get(
|
zipenhancer_path = getattr(args, "zipenhancer_path", None) or os.environ.get("ZIPENHANCER_MODEL_PATH", None)
|
||||||
"ZIPENHANCER_MODEL_PATH", None
|
|
||||||
)
|
|
||||||
|
|
||||||
# Build LoRA config if provided
|
# Build LoRA config if provided
|
||||||
lora_config = None
|
lora_config = None
|
||||||
@@ -259,8 +245,7 @@ def _run_single(args, parser, *, text: str, output: str, prompt_text: str | None
|
|||||||
cfg_value=args.cfg_value,
|
cfg_value=args.cfg_value,
|
||||||
inference_timesteps=args.inference_timesteps,
|
inference_timesteps=args.inference_timesteps,
|
||||||
normalize=args.normalize,
|
normalize=args.normalize,
|
||||||
denoise=args.denoise
|
denoise=args.denoise and (args.prompt_audio is not None or args.reference_audio is not None),
|
||||||
and (args.prompt_audio is not None or args.reference_audio is not None),
|
|
||||||
seed=args.seed,
|
seed=args.seed,
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -275,17 +260,13 @@ def _run_single(args, parser, *, text: str, output: str, prompt_text: str | None
|
|||||||
def cmd_design(args, parser):
|
def cmd_design(args, parser):
|
||||||
validate_design_args(args, parser)
|
validate_design_args(args, parser)
|
||||||
final_text = build_final_text(args.text, args.control)
|
final_text = build_final_text(args.text, args.control)
|
||||||
return _run_single(
|
return _run_single(args, parser, text=final_text, output=args.output, prompt_text=None)
|
||||||
args, parser, text=final_text, output=args.output, prompt_text=None
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def cmd_clone(args, parser):
|
def cmd_clone(args, parser):
|
||||||
prompt_text = validate_clone_args(args, parser)
|
prompt_text = validate_clone_args(args, parser)
|
||||||
final_text = build_final_text(args.text, args.control)
|
final_text = build_final_text(args.text, args.control)
|
||||||
return _run_single(
|
return _run_single(args, parser, text=final_text, output=args.output, prompt_text=prompt_text)
|
||||||
args, parser, text=final_text, output=args.output, prompt_text=prompt_text
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def cmd_validate(args, parser):
|
def cmd_validate(args, parser):
|
||||||
@@ -324,15 +305,11 @@ def cmd_batch(args, parser):
|
|||||||
|
|
||||||
prompt_audio_path = None
|
prompt_audio_path = None
|
||||||
if args.prompt_audio:
|
if args.prompt_audio:
|
||||||
prompt_audio_path = str(
|
prompt_audio_path = str(require_file_exists(args.prompt_audio, parser, "prompt audio file"))
|
||||||
require_file_exists(args.prompt_audio, parser, "prompt audio file")
|
|
||||||
)
|
|
||||||
|
|
||||||
reference_audio_path = None
|
reference_audio_path = None
|
||||||
if args.reference_audio:
|
if args.reference_audio:
|
||||||
reference_audio_path = str(
|
reference_audio_path = str(require_file_exists(args.reference_audio, parser, "reference audio file"))
|
||||||
require_file_exists(args.reference_audio, parser, "reference audio file")
|
|
||||||
)
|
|
||||||
|
|
||||||
success_count = 0
|
success_count = 0
|
||||||
|
|
||||||
@@ -347,8 +324,7 @@ def cmd_batch(args, parser):
|
|||||||
cfg_value=args.cfg_value,
|
cfg_value=args.cfg_value,
|
||||||
inference_timesteps=args.inference_timesteps,
|
inference_timesteps=args.inference_timesteps,
|
||||||
normalize=args.normalize,
|
normalize=args.normalize,
|
||||||
denoise=args.denoise
|
denoise=args.denoise and (prompt_audio_path is not None or reference_audio_path is not None),
|
||||||
and (prompt_audio_path is not None or reference_audio_path is not None),
|
|
||||||
seed=args.seed,
|
seed=args.seed,
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -389,9 +365,7 @@ def _add_common_generation_args(parser):
|
|||||||
default=10,
|
default=10,
|
||||||
help="Inference steps (int, recommended 4–30, default: 10)",
|
help="Inference steps (int, recommended 4–30, default: 10)",
|
||||||
)
|
)
|
||||||
parser.add_argument(
|
parser.add_argument("--normalize", action="store_true", help="Enable text normalization")
|
||||||
"--normalize", action="store_true", help="Enable text normalization"
|
|
||||||
)
|
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
"--seed",
|
"--seed",
|
||||||
type=int,
|
type=int,
|
||||||
@@ -406,12 +380,8 @@ def _add_prompt_reference_args(parser):
|
|||||||
"-pa",
|
"-pa",
|
||||||
help="Prompt audio file path (continuation mode, requires --prompt-text or --prompt-file)",
|
help="Prompt audio file path (continuation mode, requires --prompt-text or --prompt-file)",
|
||||||
)
|
)
|
||||||
parser.add_argument(
|
parser.add_argument("--prompt-text", "-pt", help="Text corresponding to the prompt audio")
|
||||||
"--prompt-text", "-pt", help="Text corresponding to the prompt audio"
|
parser.add_argument("--prompt-file", type=str, help="Text file corresponding to the prompt audio")
|
||||||
)
|
|
||||||
parser.add_argument(
|
|
||||||
"--prompt-file", type=str, help="Text file corresponding to the prompt audio"
|
|
||||||
)
|
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
"--reference-audio",
|
"--reference-audio",
|
||||||
"-ra",
|
"-ra",
|
||||||
@@ -438,15 +408,9 @@ def _add_model_args(parser):
|
|||||||
default="auto",
|
default="auto",
|
||||||
help="Runtime device: auto, cpu, mps, cuda, or cuda:N (default: auto)",
|
help="Runtime device: auto, cpu, mps, cuda, or cuda:N (default: auto)",
|
||||||
)
|
)
|
||||||
parser.add_argument(
|
parser.add_argument("--cache-dir", type=str, help="Cache directory for Hub downloads")
|
||||||
"--cache-dir", type=str, help="Cache directory for Hub downloads"
|
parser.add_argument("--local-files-only", action="store_true", help="Disable network access")
|
||||||
)
|
parser.add_argument("--no-denoiser", action="store_true", help="Disable denoiser model loading")
|
||||||
parser.add_argument(
|
|
||||||
"--local-files-only", action="store_true", help="Disable network access"
|
|
||||||
)
|
|
||||||
parser.add_argument(
|
|
||||||
"--no-denoiser", action="store_true", help="Disable denoiser model loading"
|
|
||||||
)
|
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
"--no-optimize",
|
"--no-optimize",
|
||||||
action="store_true",
|
action="store_true",
|
||||||
@@ -461,9 +425,7 @@ def _add_model_args(parser):
|
|||||||
|
|
||||||
def _add_lora_args(parser):
|
def _add_lora_args(parser):
|
||||||
parser.add_argument("--lora-path", type=str, help="Path to LoRA weights")
|
parser.add_argument("--lora-path", type=str, help="Path to LoRA weights")
|
||||||
parser.add_argument(
|
parser.add_argument("--lora-r", type=int, default=32, help="LoRA rank (positive int, default: 32)")
|
||||||
"--lora-r", type=int, default=32, help="LoRA rank (positive int, default: 32)"
|
|
||||||
)
|
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
"--lora-alpha",
|
"--lora-alpha",
|
||||||
type=int,
|
type=int,
|
||||||
@@ -476,12 +438,8 @@ def _add_lora_args(parser):
|
|||||||
default=0.0,
|
default=0.0,
|
||||||
help="LoRA dropout rate (0.0–1.0, default: 0.0)",
|
help="LoRA dropout rate (0.0–1.0, default: 0.0)",
|
||||||
)
|
)
|
||||||
parser.add_argument(
|
parser.add_argument("--lora-disable-lm", action="store_true", help="Disable LoRA on LM layers")
|
||||||
"--lora-disable-lm", action="store_true", help="Disable LoRA on LM layers"
|
parser.add_argument("--lora-disable-dit", action="store_true", help="Disable LoRA on DiT layers")
|
||||||
)
|
|
||||||
parser.add_argument(
|
|
||||||
"--lora-disable-dit", action="store_true", help="Disable LoRA on DiT layers"
|
|
||||||
)
|
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
"--lora-enable-proj",
|
"--lora-enable-proj",
|
||||||
action="store_true",
|
action="store_true",
|
||||||
@@ -504,37 +462,23 @@ Examples:
|
|||||||
|
|
||||||
subparsers = parser.add_subparsers(dest="command")
|
subparsers = parser.add_subparsers(dest="command")
|
||||||
|
|
||||||
design_parser = subparsers.add_parser(
|
design_parser = subparsers.add_parser("design", help="Generate speech with VoxCPM2-first voice design")
|
||||||
"design", help="Generate speech with VoxCPM2-first voice design"
|
|
||||||
)
|
|
||||||
_add_common_generation_args(design_parser)
|
_add_common_generation_args(design_parser)
|
||||||
_add_prompt_reference_args(design_parser)
|
_add_prompt_reference_args(design_parser)
|
||||||
_add_model_args(design_parser)
|
_add_model_args(design_parser)
|
||||||
_add_lora_args(design_parser)
|
_add_lora_args(design_parser)
|
||||||
design_parser.add_argument(
|
design_parser.add_argument("--output", "-o", required=True, help="Output audio file path")
|
||||||
"--output", "-o", required=True, help="Output audio file path"
|
|
||||||
)
|
|
||||||
|
|
||||||
clone_parser = subparsers.add_parser(
|
clone_parser = subparsers.add_parser("clone", help="Clone a voice with reference/prompt audio")
|
||||||
"clone", help="Clone a voice with reference/prompt audio"
|
|
||||||
)
|
|
||||||
_add_common_generation_args(clone_parser)
|
_add_common_generation_args(clone_parser)
|
||||||
_add_prompt_reference_args(clone_parser)
|
_add_prompt_reference_args(clone_parser)
|
||||||
_add_model_args(clone_parser)
|
_add_model_args(clone_parser)
|
||||||
_add_lora_args(clone_parser)
|
_add_lora_args(clone_parser)
|
||||||
clone_parser.add_argument(
|
clone_parser.add_argument("--output", "-o", required=True, help="Output audio file path")
|
||||||
"--output", "-o", required=True, help="Output audio file path"
|
|
||||||
)
|
|
||||||
|
|
||||||
batch_parser = subparsers.add_parser(
|
batch_parser = subparsers.add_parser("batch", help="Batch-generate one line per output file")
|
||||||
"batch", help="Batch-generate one line per output file"
|
batch_parser.add_argument("--input", "-i", required=True, help="Input text file (one text per line)")
|
||||||
)
|
batch_parser.add_argument("--output-dir", "-od", required=True, help="Output directory")
|
||||||
batch_parser.add_argument(
|
|
||||||
"--input", "-i", required=True, help="Input text file (one text per line)"
|
|
||||||
)
|
|
||||||
batch_parser.add_argument(
|
|
||||||
"--output-dir", "-od", required=True, help="Output directory"
|
|
||||||
)
|
|
||||||
batch_parser.add_argument(
|
batch_parser.add_argument(
|
||||||
"--control",
|
"--control",
|
||||||
type=str,
|
type=str,
|
||||||
@@ -553,9 +497,7 @@ Examples:
|
|||||||
default=10,
|
default=10,
|
||||||
help="Inference steps (int, recommended 4–30, default: 10)",
|
help="Inference steps (int, recommended 4–30, default: 10)",
|
||||||
)
|
)
|
||||||
batch_parser.add_argument(
|
batch_parser.add_argument("--normalize", action="store_true", help="Enable text normalization")
|
||||||
"--normalize", action="store_true", help="Enable text normalization"
|
|
||||||
)
|
|
||||||
batch_parser.add_argument(
|
batch_parser.add_argument(
|
||||||
"--seed",
|
"--seed",
|
||||||
type=int,
|
type=int,
|
||||||
@@ -570,9 +512,7 @@ Examples:
|
|||||||
"validate",
|
"validate",
|
||||||
help="Validate a training data manifest (JSONL) before fine-tuning",
|
help="Validate a training data manifest (JSONL) before fine-tuning",
|
||||||
)
|
)
|
||||||
validate_parser.add_argument(
|
validate_parser.add_argument("--manifest", "-m", required=True, help="Path to JSONL training manifest")
|
||||||
"--manifest", "-m", required=True, help="Path to JSONL training manifest"
|
|
||||||
)
|
|
||||||
validate_parser.add_argument(
|
validate_parser.add_argument(
|
||||||
"--sample-rate",
|
"--sample-rate",
|
||||||
type=int,
|
type=int,
|
||||||
@@ -585,19 +525,13 @@ Examples:
|
|||||||
default=0,
|
default=0,
|
||||||
help="Maximum number of samples to validate (0 = all, default: 0)",
|
help="Maximum number of samples to validate (0 = all, default: 0)",
|
||||||
)
|
)
|
||||||
validate_parser.add_argument(
|
validate_parser.add_argument("--verbose", "-v", action="store_true", help="Print per-sample progress")
|
||||||
"--verbose", "-v", action="store_true", help="Print per-sample progress"
|
|
||||||
)
|
|
||||||
|
|
||||||
# Legacy root arguments
|
# Legacy root arguments
|
||||||
parser.add_argument("--input", "-i", help="Input text file (batch mode only)")
|
parser.add_argument("--input", "-i", help="Input text file (batch mode only)")
|
||||||
parser.add_argument(
|
parser.add_argument("--output-dir", "-od", help="Output directory (batch mode only)")
|
||||||
"--output-dir", "-od", help="Output directory (batch mode only)"
|
|
||||||
)
|
|
||||||
_add_common_generation_args(parser)
|
_add_common_generation_args(parser)
|
||||||
parser.add_argument(
|
parser.add_argument("--output", "-o", help="Output audio file path (single or clone mode)")
|
||||||
"--output", "-o", help="Output audio file path (single or clone mode)"
|
|
||||||
)
|
|
||||||
_add_prompt_reference_args(parser)
|
_add_prompt_reference_args(parser)
|
||||||
_add_model_args(parser)
|
_add_model_args(parser)
|
||||||
_add_lora_args(parser)
|
_add_lora_args(parser)
|
||||||
@@ -609,9 +543,7 @@ def _dispatch_legacy(args, parser):
|
|||||||
warn_legacy_mode()
|
warn_legacy_mode()
|
||||||
|
|
||||||
if args.input and args.text:
|
if args.input and args.text:
|
||||||
parser.error(
|
parser.error("Use either batch mode (--input) or single mode (--text), not both.")
|
||||||
"Use either batch mode (--input) or single mode (--text), not both."
|
|
||||||
)
|
|
||||||
|
|
||||||
if args.input:
|
if args.input:
|
||||||
if not args.output_dir:
|
if not args.output_dir:
|
||||||
@@ -621,12 +553,7 @@ def _dispatch_legacy(args, parser):
|
|||||||
if not args.text or not args.output:
|
if not args.text or not args.output:
|
||||||
parser.error("Single-sample legacy mode requires --text and --output")
|
parser.error("Single-sample legacy mode requires --text and --output")
|
||||||
|
|
||||||
if (
|
if args.prompt_audio or args.prompt_text or args.prompt_file or args.reference_audio:
|
||||||
args.prompt_audio
|
|
||||||
or args.prompt_text
|
|
||||||
or args.prompt_file
|
|
||||||
or args.reference_audio
|
|
||||||
):
|
|
||||||
return cmd_clone(args, parser)
|
return cmd_clone(args, parser)
|
||||||
|
|
||||||
return cmd_design(args, parser)
|
return cmd_design(args, parser)
|
||||||
@@ -663,4 +590,4 @@ def main():
|
|||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
main()
|
main()
|
||||||
|
|||||||
@@ -293,7 +293,7 @@ class VoxCPM:
|
|||||||
retry_badcase_max_times=retry_badcase_max_times,
|
retry_badcase_max_times=retry_badcase_max_times,
|
||||||
retry_badcase_ratio_threshold=retry_badcase_ratio_threshold,
|
retry_badcase_ratio_threshold=retry_badcase_ratio_threshold,
|
||||||
streaming=streaming,
|
streaming=streaming,
|
||||||
seed=seed
|
seed=seed,
|
||||||
)
|
)
|
||||||
|
|
||||||
if streaming:
|
if streaming:
|
||||||
|
|||||||
@@ -5,9 +5,12 @@ from transformers import PreTrainedTokenizer
|
|||||||
|
|
||||||
_LOW_PRECISION_DTYPES = {"bfloat16", "bf16", "float16", "fp16"}
|
_LOW_PRECISION_DTYPES = {"bfloat16", "bf16", "float16", "fp16"}
|
||||||
_VALID_DTYPE_OVERRIDES = {
|
_VALID_DTYPE_OVERRIDES = {
|
||||||
"bfloat16", "bf16",
|
"bfloat16",
|
||||||
"float16", "fp16",
|
"bf16",
|
||||||
"float32", "fp32",
|
"float16",
|
||||||
|
"fp16",
|
||||||
|
"float32",
|
||||||
|
"fp32",
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@@ -173,10 +176,7 @@ def pick_runtime_dtype(device: str, configured_dtype: str) -> str:
|
|||||||
override = os.environ.get("VOXCPM_MPS_DTYPE", "").strip().lower()
|
override = os.environ.get("VOXCPM_MPS_DTYPE", "").strip().lower()
|
||||||
if override:
|
if override:
|
||||||
if override not in _VALID_DTYPE_OVERRIDES:
|
if override not in _VALID_DTYPE_OVERRIDES:
|
||||||
raise ValueError(
|
raise ValueError(f"VOXCPM_MPS_DTYPE='{override}' is not one of " f"{sorted(_VALID_DTYPE_OVERRIDES)}")
|
||||||
f"VOXCPM_MPS_DTYPE='{override}' is not one of "
|
|
||||||
f"{sorted(_VALID_DTYPE_OVERRIDES)}"
|
|
||||||
)
|
|
||||||
return override
|
return override
|
||||||
|
|
||||||
if (configured_dtype or "").lower() in _LOW_PRECISION_DTYPES:
|
if (configured_dtype or "").lower() in _LOW_PRECISION_DTYPES:
|
||||||
@@ -224,15 +224,13 @@ def resolve_runtime_device(device: Optional[str], configured_device: str = "cuda
|
|||||||
if explicit.startswith("cuda"):
|
if explicit.startswith("cuda"):
|
||||||
if not torch.cuda.is_available():
|
if not torch.cuda.is_available():
|
||||||
raise ValueError(
|
raise ValueError(
|
||||||
f"Requested device '{device}', but CUDA is not available. "
|
f"Requested device '{device}', but CUDA is not available. " "Use device='auto' for automatic fallback."
|
||||||
"Use device='auto' for automatic fallback."
|
|
||||||
)
|
)
|
||||||
return explicit
|
return explicit
|
||||||
if explicit == "mps":
|
if explicit == "mps":
|
||||||
if not _has_mps():
|
if not _has_mps():
|
||||||
raise ValueError(
|
raise ValueError(
|
||||||
"Requested device 'mps', but MPS is not available. "
|
"Requested device 'mps', but MPS is not available. " "Use device='auto' for automatic fallback."
|
||||||
"Use device='auto' for automatic fallback."
|
|
||||||
)
|
)
|
||||||
return "mps"
|
return "mps"
|
||||||
if explicit == "cpu":
|
if explicit == "cpu":
|
||||||
|
|||||||
@@ -57,7 +57,9 @@ from .utils import (
|
|||||||
|
|
||||||
|
|
||||||
# A simple function to trim audio silence using VAD, not used default
|
# A simple function to trim audio silence using VAD, not used default
|
||||||
def _trim_audio_silence_vad(audio: torch.Tensor, sample_rate: int, max_silence_ms: float = 200.0, top_db: float = 35.0) -> torch.Tensor:
|
def _trim_audio_silence_vad(
|
||||||
|
audio: torch.Tensor, sample_rate: int, max_silence_ms: float = 200.0, top_db: float = 35.0
|
||||||
|
) -> torch.Tensor:
|
||||||
if audio.numel() == 0:
|
if audio.numel() == 0:
|
||||||
return audio
|
return audio
|
||||||
y = audio.squeeze(0).numpy()
|
y = audio.squeeze(0).numpy()
|
||||||
@@ -684,7 +686,7 @@ class VoxCPM2Model(nn.Module):
|
|||||||
decode_audio = self.audio_vae.decode(latent_pred.to(torch.float32))
|
decode_audio = self.audio_vae.decode(latent_pred.to(torch.float32))
|
||||||
decode_patch_len = self.patch_size * self._decode_chunk_size
|
decode_patch_len = self.patch_size * self._decode_chunk_size
|
||||||
if context_len > 0:
|
if context_len > 0:
|
||||||
decode_audio = decode_audio[..., decode_patch_len * context_len:].squeeze(1).cpu()
|
decode_audio = decode_audio[..., decode_patch_len * context_len :].squeeze(1).cpu()
|
||||||
else:
|
else:
|
||||||
decode_audio = decode_audio.squeeze(1).cpu()
|
decode_audio = decode_audio.squeeze(1).cpu()
|
||||||
yield decode_audio
|
yield decode_audio
|
||||||
@@ -979,7 +981,7 @@ class VoxCPM2Model(nn.Module):
|
|||||||
decode_audio = self.audio_vae.decode(latent_pred.to(torch.float32))
|
decode_audio = self.audio_vae.decode(latent_pred.to(torch.float32))
|
||||||
decode_patch_len = self.patch_size * self._decode_chunk_size
|
decode_patch_len = self.patch_size * self._decode_chunk_size
|
||||||
if context_len > 0:
|
if context_len > 0:
|
||||||
decode_audio = decode_audio[..., decode_patch_len * context_len:].squeeze(1).cpu()
|
decode_audio = decode_audio[..., decode_patch_len * context_len :].squeeze(1).cpu()
|
||||||
else:
|
else:
|
||||||
decode_audio = decode_audio.squeeze(1).cpu()
|
decode_audio = decode_audio.squeeze(1).cpu()
|
||||||
yield (decode_audio, target_text_token, pred_audio_feat)
|
yield (decode_audio, target_text_token, pred_audio_feat)
|
||||||
|
|||||||
@@ -551,8 +551,7 @@ class StreamingVAEDecoder:
|
|||||||
if x.shape[-1] >= _p:
|
if x.shape[-1] >= _p:
|
||||||
states[_k] = x[:, :, -_p:].detach()
|
states[_k] = x[:, :, -_p:].detach()
|
||||||
else:
|
else:
|
||||||
prev = states.get(_k, torch.zeros(x.shape[0], x.shape[1], _p,
|
prev = states.get(_k, torch.zeros(x.shape[0], x.shape[1], _p, device=x.device, dtype=x.dtype))
|
||||||
device=x.device, dtype=x.dtype))
|
|
||||||
states[_k] = torch.cat([prev, x], dim=-1)[:, :, -_p:].detach()
|
states[_k] = torch.cat([prev, x], dim=-1)[:, :, -_p:].detach()
|
||||||
return nn.Conv1d.forward(_m, x_pad)
|
return nn.Conv1d.forward(_m, x_pad)
|
||||||
|
|
||||||
|
|||||||
@@ -350,45 +350,80 @@ class AudioFeatureProcessingPacker:
|
|||||||
|
|
||||||
# -- text token track --
|
# -- text token track --
|
||||||
# [103, 0×R, 104, text_ids, 101, 0×A, 102]
|
# [103, 0×R, 104, text_ids, 101, 0×A, 102]
|
||||||
text_token_info = torch.cat([
|
text_token_info = torch.cat(
|
||||||
_tok([self.audio_prompt_start_id]),
|
[
|
||||||
torch.zeros(ref_len, dtype=torch.int32, device=device),
|
_tok([self.audio_prompt_start_id]),
|
||||||
_tok([self.audio_prompt_end_id]),
|
torch.zeros(ref_len, dtype=torch.int32, device=device),
|
||||||
text_token,
|
_tok([self.audio_prompt_end_id]),
|
||||||
_tok([self.audio_start_id]),
|
text_token,
|
||||||
torch.zeros(tgt_len, dtype=torch.int32, device=device),
|
_tok([self.audio_start_id]),
|
||||||
_tok([self.audio_end_id]),
|
torch.zeros(tgt_len, dtype=torch.int32, device=device),
|
||||||
])
|
_tok([self.audio_end_id]),
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
# -- audio feature track --
|
# -- audio feature track --
|
||||||
zero_1 = torch.zeros((1,) + feat_shape, dtype=torch.float32, device=device)
|
zero_1 = torch.zeros((1,) + feat_shape, dtype=torch.float32, device=device)
|
||||||
zero_txt = torch.zeros((txt_len,) + feat_shape, dtype=torch.float32, device=device)
|
zero_txt = torch.zeros((txt_len,) + feat_shape, dtype=torch.float32, device=device)
|
||||||
audio_feat_info = torch.cat([
|
audio_feat_info = torch.cat(
|
||||||
zero_1, ref_feats, zero_1, # 103, ref, 104
|
[
|
||||||
zero_txt, # text
|
zero_1,
|
||||||
zero_1, tgt_feats, zero_1, # 101, target, 102
|
ref_feats,
|
||||||
], dim=0)
|
zero_1, # 103, ref, 104
|
||||||
|
zero_txt, # text
|
||||||
|
zero_1,
|
||||||
|
tgt_feats,
|
||||||
|
zero_1, # 101, target, 102
|
||||||
|
],
|
||||||
|
dim=0,
|
||||||
|
)
|
||||||
|
|
||||||
# -- masks --
|
# -- masks --
|
||||||
text_mask = torch.cat([
|
text_mask = (
|
||||||
torch.ones(1), torch.zeros(ref_len), torch.ones(1),
|
torch.cat(
|
||||||
torch.ones(txt_len),
|
[
|
||||||
torch.ones(1), torch.zeros(tgt_len), torch.ones(1),
|
torch.ones(1),
|
||||||
]).to(torch.int32).to(device)
|
torch.zeros(ref_len),
|
||||||
|
torch.ones(1),
|
||||||
|
torch.ones(txt_len),
|
||||||
|
torch.ones(1),
|
||||||
|
torch.zeros(tgt_len),
|
||||||
|
torch.ones(1),
|
||||||
|
]
|
||||||
|
)
|
||||||
|
.to(torch.int32)
|
||||||
|
.to(device)
|
||||||
|
)
|
||||||
|
|
||||||
audio_mask = torch.cat([
|
audio_mask = (
|
||||||
torch.zeros(1), torch.ones(ref_len), torch.zeros(1),
|
torch.cat(
|
||||||
torch.zeros(txt_len),
|
[
|
||||||
torch.zeros(1), torch.ones(tgt_len), torch.zeros(1),
|
torch.zeros(1),
|
||||||
]).to(torch.int32).to(device)
|
torch.ones(ref_len),
|
||||||
|
torch.zeros(1),
|
||||||
|
torch.zeros(txt_len),
|
||||||
|
torch.zeros(1),
|
||||||
|
torch.ones(tgt_len),
|
||||||
|
torch.zeros(1),
|
||||||
|
]
|
||||||
|
)
|
||||||
|
.to(torch.int32)
|
||||||
|
.to(device)
|
||||||
|
)
|
||||||
|
|
||||||
loss_mask = torch.cat([
|
loss_mask = (
|
||||||
torch.zeros(1 + ref_len + 1), # ref part: no loss
|
torch.cat(
|
||||||
torch.zeros(txt_len), # text: no loss
|
[
|
||||||
torch.zeros(1), # 101: no loss
|
torch.zeros(1 + ref_len + 1), # ref part: no loss
|
||||||
torch.ones(tgt_len), # target audio: LOSS
|
torch.zeros(txt_len), # text: no loss
|
||||||
torch.zeros(1), # 102: no loss
|
torch.zeros(1), # 101: no loss
|
||||||
]).to(torch.int32).to(device)
|
torch.ones(tgt_len), # target audio: LOSS
|
||||||
|
torch.zeros(1), # 102: no loss
|
||||||
|
]
|
||||||
|
)
|
||||||
|
.to(torch.int32)
|
||||||
|
.to(device)
|
||||||
|
)
|
||||||
|
|
||||||
total_len = 1 + ref_len + 1 + txt_len + 1 + tgt_len + 1
|
total_len = 1 + ref_len + 1 + txt_len + 1 + tgt_len + 1
|
||||||
labels = torch.zeros(total_len, dtype=torch.int32, device=device)
|
labels = torch.zeros(total_len, dtype=torch.int32, device=device)
|
||||||
|
|||||||
@@ -44,10 +44,7 @@ def _check_audio_file(audio_path: str, sample_rate: int) -> Optional[str]:
|
|||||||
if info.frames == 0:
|
if info.frames == 0:
|
||||||
return f"Audio file is empty: {audio_path}"
|
return f"Audio file is empty: {audio_path}"
|
||||||
if info.samplerate != sample_rate:
|
if info.samplerate != sample_rate:
|
||||||
return (
|
return f"Sample rate mismatch in {audio_path}: " f"expected {sample_rate} Hz, got {info.samplerate} Hz"
|
||||||
f"Sample rate mismatch in {audio_path}: "
|
|
||||||
f"expected {sample_rate} Hz, got {info.samplerate} Hz"
|
|
||||||
)
|
|
||||||
return None
|
return None
|
||||||
except ImportError:
|
except ImportError:
|
||||||
# soundfile not available; just check existence
|
# soundfile not available; just check existence
|
||||||
@@ -187,13 +184,9 @@ def validate_manifest(
|
|||||||
if duration is not None:
|
if duration is not None:
|
||||||
result.audio_durations.append(duration)
|
result.audio_durations.append(duration)
|
||||||
if duration < 0.3:
|
if duration < 0.3:
|
||||||
result.warnings.append(
|
result.warnings.append(f"Line {i + 1}: Very short audio ({duration:.2f}s)")
|
||||||
f"Line {i + 1}: Very short audio ({duration:.2f}s)"
|
|
||||||
)
|
|
||||||
elif duration > 30.0:
|
elif duration > 30.0:
|
||||||
result.warnings.append(
|
result.warnings.append(f"Line {i + 1}: Very long audio ({duration:.1f}s), may cause OOM")
|
||||||
f"Line {i + 1}: Very long audio ({duration:.1f}s), may cause OOM"
|
|
||||||
)
|
|
||||||
else:
|
else:
|
||||||
result.errors.append(f"Line {i + 1}: Invalid audio path")
|
result.errors.append(f"Line {i + 1}: Invalid audio path")
|
||||||
has_error = True
|
has_error = True
|
||||||
@@ -209,9 +202,7 @@ def validate_manifest(
|
|||||||
if os.path.isfile(ref_path):
|
if os.path.isfile(ref_path):
|
||||||
result.has_ref_audio += 1
|
result.has_ref_audio += 1
|
||||||
else:
|
else:
|
||||||
result.warnings.append(
|
result.warnings.append(f"Line {i + 1}: ref_audio file not found: {ref_path}")
|
||||||
f"Line {i + 1}: ref_audio file not found: {ref_path}"
|
|
||||||
)
|
|
||||||
|
|
||||||
if not has_error:
|
if not has_error:
|
||||||
result.valid_samples += 1
|
result.valid_samples += 1
|
||||||
@@ -222,14 +213,10 @@ def validate_manifest(
|
|||||||
# Summarize truncated errors
|
# Summarize truncated errors
|
||||||
if missing_audio_count > 5:
|
if missing_audio_count > 5:
|
||||||
result.errors.append(
|
result.errors.append(
|
||||||
f"... and {missing_audio_count - 5} more missing audio files "
|
f"... and {missing_audio_count - 5} more missing audio files " f"({missing_audio_count} total)"
|
||||||
f"({missing_audio_count} total)"
|
|
||||||
)
|
)
|
||||||
if empty_text_count > 5:
|
if empty_text_count > 5:
|
||||||
result.warnings.append(
|
result.warnings.append(f"... and {empty_text_count - 5} more empty text entries " f"({empty_text_count} total)")
|
||||||
f"... and {empty_text_count - 5} more empty text entries "
|
|
||||||
f"({empty_text_count} total)"
|
|
||||||
)
|
|
||||||
|
|
||||||
return result
|
return result
|
||||||
|
|
||||||
|
|||||||
@@ -455,10 +455,7 @@ def test_clone_rejects_prompt_audio_without_transcript(monkeypatch, tmp_path, ca
|
|||||||
with pytest.raises(SystemExit):
|
with pytest.raises(SystemExit):
|
||||||
cli.main()
|
cli.main()
|
||||||
|
|
||||||
assert (
|
assert "--prompt-audio requires --prompt-text or --prompt-file" in capsys.readouterr().err
|
||||||
"--prompt-audio requires --prompt-text or --prompt-file"
|
|
||||||
in capsys.readouterr().err
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def test_clone_rejects_transcript_without_prompt_audio(monkeypatch, tmp_path, capsys):
|
def test_clone_rejects_transcript_without_prompt_audio(monkeypatch, tmp_path, capsys):
|
||||||
@@ -480,9 +477,7 @@ def test_clone_rejects_transcript_without_prompt_audio(monkeypatch, tmp_path, ca
|
|||||||
with pytest.raises(SystemExit):
|
with pytest.raises(SystemExit):
|
||||||
cli.main()
|
cli.main()
|
||||||
|
|
||||||
assert (
|
assert "--prompt-text/--prompt-file requires --prompt-audio" in capsys.readouterr().err
|
||||||
"--prompt-text/--prompt-file requires --prompt-audio" in capsys.readouterr().err
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def test_batch_rejects_control_with_prompt_transcript(monkeypatch, tmp_path, capsys):
|
def test_batch_rejects_control_with_prompt_transcript(monkeypatch, tmp_path, capsys):
|
||||||
@@ -603,4 +598,4 @@ def test_batch_subcommand_passes_seed(monkeypatch, tmp_path):
|
|||||||
|
|
||||||
assert len(dummy_model.calls) == 2
|
assert len(dummy_model.calls) == 2
|
||||||
assert dummy_model.calls[0]["seed"] == 999
|
assert dummy_model.calls[0]["seed"] == 999
|
||||||
assert dummy_model.calls[1]["seed"] == 999
|
assert dummy_model.calls[1]["seed"] == 999
|
||||||
|
|||||||
@@ -207,6 +207,7 @@ class TestValidateManifest:
|
|||||||
|
|
||||||
audio = tmp_dir / "audio_8k.wav"
|
audio = tmp_dir / "audio_8k.wav"
|
||||||
import numpy as np
|
import numpy as np
|
||||||
|
|
||||||
samples = np.zeros(8000, dtype=np.float32)
|
samples = np.zeros(8000, dtype=np.float32)
|
||||||
sf.write(str(audio), samples, 8000)
|
sf.write(str(audio), samples, 8000)
|
||||||
|
|
||||||
@@ -240,6 +241,7 @@ class TestValidateManifest:
|
|||||||
def test_cli_validate_exit_code(self, tmp_dir):
|
def test_cli_validate_exit_code(self, tmp_dir):
|
||||||
"""validate subcommand must exit 1 on validation error (missing audio)."""
|
"""validate subcommand must exit 1 on validation error (missing audio)."""
|
||||||
import subprocess
|
import subprocess
|
||||||
|
|
||||||
manifest = tmp_dir / "bad.jsonl"
|
manifest = tmp_dir / "bad.jsonl"
|
||||||
_write_manifest(manifest, [{"text": "hi", "audio": "/nonexistent/x.wav"}])
|
_write_manifest(manifest, [{"text": "hi", "audio": "/nonexistent/x.wav"}])
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user