diff --git a/.gitignore b/.gitignore index d397292..f7fa981 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ launch.json +.venv/ __pycache__ voxcpm.egg-info .DS_Store ./pretrained_models/ -app_local.py \ No newline at end of file +app_local.py diff --git a/README.md b/README.md index cea9b3c..dc1a544 100644 --- a/README.md +++ b/README.md @@ -242,6 +242,14 @@ voxcpm --help python app.py --port 8808 # then open in browser: http://localhost:8808 ``` +Use `--device` to choose the runtime device: + +```bash +python app.py --device auto +``` + +Supported values are `auto`, `cpu`, `mps`, `cuda`, and `cuda:N`. On Apple Silicon Macs, `auto` uses MPS when available. + ### 🚢 Production Deployment (Nano-vLLM) For high-throughput serving, use [**Nano-vLLM-VoxCPM**](https://github.com/a710128/nanovllm-voxcpm) — a dedicated inference engine built on Nano-vLLM with concurrent request support and an async API. diff --git a/README_zh.md b/README_zh.md index 497554d..ec781e4 100644 --- a/README_zh.md +++ b/README_zh.md @@ -241,6 +241,14 @@ voxcpm --help python app.py --port 8808 # 然后在浏览器打开 http://localhost:8808 ``` +使用 `--device` 选择运行设备： + +```bash +python app.py --device auto +``` + +支持的取值包括 `auto`、`cpu`、`mps`、`cuda` 和 `cuda:N`。在 Apple Silicon Mac 上，`auto` 会在可用时使用 MPS。 + ### 🚢 生产部署（Nano-vLLM） 如需高吞吐量部署，使用 [**Nano-vLLM-VoxCPM**](https://github.com/a710128/nanovllm-voxcpm) — 基于 Nano-vLLM 构建的专用推理引擎，支持并发请求和异步 API。 diff --git a/app.py b/app.py index dba6fe3..95eac94 100644 --- a/app.py +++ b/app.py @@ -3,7 +3,6 @@ import re import sys import logging import numpy as np -import torch import gradio as gr from typing import Optional, Tuple from funasr import AutoModel @@ -12,6 +11,7 @@ from pathlib import Path os.environ["TOKENIZERS_PARALLELISM"] = "false" import voxcpm +from voxcpm.model.utils import resolve_runtime_device logging.basicConfig( level=logging.INFO, @@ -220,17 +220,14 @@ _APP_THEME = gr.themes.Soft( # ---------- Model ---------- class VoxCPMDemo: - def __init__(self, model_id: str = "openbmb/VoxCPM2") -> None: - self.device = "cuda" if torch.cuda.is_available() else "cpu" - logger.info(f"Running on device: {self.device}") + def __init__(self, model_id: str = "openbmb/VoxCPM2", device: str = "auto") -> None: + self.device = resolve_runtime_device(device, "cuda") + logger.info(f"Running VoxCPM on device: {self.device}") + self.optimize = self.device.startswith("cuda") self.asr_model_id = "iic/SenseVoiceSmall" - self.asr_model: Optional[AutoModel] = AutoModel( - model=self.asr_model_id, - disable_update=True, - log_level="DEBUG", - device="cuda:0" if self.device == "cuda" else "cpu", - ) + self.asr_device = "cuda:0" if self.device.startswith("cuda") else "cpu" + self.asr_model: Optional[AutoModel] = None self.voxcpm_model: Optional[voxcpm.VoxCPM] = None self._model_id = model_id @@ -239,14 +236,37 @@ class VoxCPMDemo: if self.voxcpm_model is not None: return self.voxcpm_model logger.info(f"Loading model: {self._model_id}") - self.voxcpm_model = voxcpm.VoxCPM.from_pretrained(self._model_id, optimize=True) + self.voxcpm_model = voxcpm.VoxCPM.from_pretrained( + self._model_id, + optimize=self.optimize, + device=self.device, + ) logger.info("Model loaded successfully.") return self.voxcpm_model + def get_or_load_asr_model(self) -> AutoModel: + if self.asr_model is not None: + return self.asr_model + logger.info( + f"Loading ASR model: {self.asr_model_id} on device: {self.asr_device}" + ) + self.asr_model = AutoModel( + model=self.asr_model_id, + disable_update=True, + log_level="DEBUG", + device=self.asr_device, + ) + logger.info("ASR model loaded successfully.") + return self.asr_model + def prompt_wav_recognition(self, prompt_wav: Optional[str]) -> str: if prompt_wav is None: return "" - res = self.asr_model.generate(input=prompt_wav, language="auto", use_itn=True) + res = self.get_or_load_asr_model().generate( + input=prompt_wav, + language="auto", + use_itn=True, + ) return res[0]["text"].split("|>")[-1] def _build_generate_kwargs( @@ -487,8 +507,9 @@ def run_demo( server_port: int = 8808, show_error: bool = True, model_id: str = "openbmb/VoxCPM2", + device: str = "auto", ): - demo = VoxCPMDemo(model_id=model_id) + demo = VoxCPMDemo(model_id=model_id, device=device) interface = create_demo_interface(demo) interface.queue(max_size=10, default_concurrency_limit=1).launch( server_name=server_name, @@ -508,5 +529,11 @@ if __name__ == "__main__": help="Local path or HuggingFace repo ID (default: openbmb/VoxCPM2)", ) parser.add_argument("--port", type=int, default=8808, help="Server port") + parser.add_argument( + "--device", + type=str, + default="auto", + help="Runtime device: auto, cpu, mps, cuda, or cuda:N (default: auto)", + ) args = parser.parse_args() - run_demo(model_id=args.model_id, server_port=args.port) + run_demo(model_id=args.model_id, server_port=args.port, device=args.device) diff --git a/tests/test_cli.py b/tests/test_cli.py index cae8ec8..2b568b1 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -54,6 +54,11 @@ def run_main(monkeypatch, argv): cli.main() +def patch_soundfile_write(monkeypatch): + soundfile_stub = types.SimpleNamespace(write=lambda *args, **kwargs: None) + monkeypatch.setitem(sys.modules, "soundfile", soundfile_stub) + + def test_parser_defaults_to_voxcpm2(): parser = cli._build_parser() args = parser.parse_args(["design", "--text", "hello", "--output", "out.wav"]) @@ -70,7 +75,7 @@ def test_load_model_respects_no_optimize_for_local_model(monkeypatch): calls["kwargs"] = kwargs self.tts_model = DummyTTSModel() - monkeypatch.setattr(cli, "VoxCPM", FakeVoxCPM) + monkeypatch.setattr(core_stub, "VoxCPM", FakeVoxCPM) args = cli._build_parser().parse_args( [ "design", @@ -99,7 +104,7 @@ def test_load_model_defaults_optimize_for_hf(monkeypatch): calls["kwargs"] = kwargs return DummyModel() - monkeypatch.setattr(cli, "VoxCPM", FakeVoxCPM) + monkeypatch.setattr(core_stub, "VoxCPM", FakeVoxCPM) args = cli._build_parser().parse_args( [ "design", @@ -125,7 +130,7 @@ def test_load_model_respects_no_optimize_for_hf(monkeypatch): calls["kwargs"] = kwargs return DummyModel() - monkeypatch.setattr(cli, "VoxCPM", FakeVoxCPM) + monkeypatch.setattr(core_stub, "VoxCPM", FakeVoxCPM) args = cli._build_parser().parse_args( [ "design", @@ -152,7 +157,7 @@ def test_load_model_passes_explicit_device_to_hf(monkeypatch): calls["kwargs"] = kwargs return DummyModel() - monkeypatch.setattr(cli, "VoxCPM", FakeVoxCPM) + monkeypatch.setattr(core_stub, "VoxCPM", FakeVoxCPM) args = cli._build_parser().parse_args( [ "design", @@ -173,7 +178,7 @@ def test_load_model_passes_explicit_device_to_hf(monkeypatch): def test_design_subcommand_applies_control(monkeypatch, tmp_path): dummy_model = DummyModel() monkeypatch.setattr(cli, "load_model", lambda args: dummy_model) - monkeypatch.setattr(cli.sf, "write", lambda *args, **kwargs: None) + patch_soundfile_write(monkeypatch) run_main( monkeypatch, @@ -201,7 +206,7 @@ def test_clone_subcommand_reads_prompt_file(monkeypatch, tmp_path): prompt_file.write_text("prompt transcript\n", encoding="utf-8") monkeypatch.setattr(cli, "load_model", lambda args: dummy_model) - monkeypatch.setattr(cli.sf, "write", lambda *args, **kwargs: None) + patch_soundfile_write(monkeypatch) run_main( monkeypatch, @@ -273,7 +278,7 @@ def test_clone_rejects_reference_audio_for_v1_hf_model_id(monkeypatch, tmp_path) def test_legacy_root_args_still_work_and_warn(monkeypatch, tmp_path, capsys): dummy_model = DummyModel() monkeypatch.setattr(cli, "load_model", lambda args: dummy_model) - monkeypatch.setattr(cli.sf, "write", lambda *args, **kwargs: None) + patch_soundfile_write(monkeypatch) run_main( monkeypatch, @@ -296,7 +301,7 @@ def test_batch_subcommand_applies_control(monkeypatch, tmp_path): input_file.write_text("hello\nworld\n", encoding="utf-8") monkeypatch.setattr(cli, "load_model", lambda args: dummy_model) - monkeypatch.setattr(cli.sf, "write", lambda *args, **kwargs: None) + patch_soundfile_write(monkeypatch) run_main( monkeypatch, @@ -325,7 +330,7 @@ def test_legacy_clone_with_prompt_file_still_works(monkeypatch, tmp_path, capsys prompt_file.write_text("legacy transcript", encoding="utf-8") monkeypatch.setattr(cli, "load_model", lambda args: dummy_model) - monkeypatch.setattr(cli.sf, "write", lambda *args, **kwargs: None) + patch_soundfile_write(monkeypatch) run_main( monkeypatch,