diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..ea5c99b --- /dev/null +++ b/.dockerignore @@ -0,0 +1,26 @@ +# Model weights (mounted at runtime via volumes) +models/ +data/ +lora/ +output/ + +# Git history +.git/ + +# Python cache +__pycache__/ +*.pyc +*.pyo +*.egg-info/ +.venv/ +venv/ +.venv-bench/ + +# Docker config (not needed inside image) +docker/docker-compose.yml +docker/nginx.conf +docker/README.md + +# IDE / OS +.DS_Store +.vscode/ diff --git a/.gitignore b/.gitignore index f7fa981..61bac30 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,9 @@ voxcpm.egg-info .DS_Store ./pretrained_models/ app_local.py + +# Docker volume mount directories (large files, user-specific) +models/ +data/ +lora/ +output/ diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 0000000..b37d20c --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,52 @@ +# ───────────────────────────────────────────────────────────────────── +# VoxCPM Training WebUI — Docker image +# ───────────────────────────────────────────────────────────────────── +# Base: PyTorch with CUDA for GPU-accelerated LoRA fine-tuning. +# Build context should be the project root: +# +# docker build -f docker/Dockerfile -t voxcpm-training . +# +# ───────────────────────────────────────────────────────────────────── +FROM pytorch/pytorch:2.5.1-cuda12.4-cudnn9-devel + +LABEL maintainer="OpenBMB " +LABEL description="VoxCPM LoRA Training WebUI with GPU support" + +# Avoid interactive prompts during package installation +ENV DEBIAN_FRONTEND=noninteractive + +# System deps required by Python packages: +# git — setuptools_scm needs it to resolve version in pyproject.toml +# libsndfile1 — C library backing the 'soundfile' Python package +# ffmpeg — audio codec support for torchaudio/librosa +RUN apt-get update && apt-get install -y --no-install-recommends \ + git \ + libsndfile1 \ + ffmpeg \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +# Layer 1: Install dependencies only (cached unless pyproject.toml changes) +# Create a minimal package stub so pip can resolve deps without real source. +COPY pyproject.toml /app/ +RUN mkdir -p /app/src/voxcpm && echo '__version__ = "0.0.0"' > /app/src/voxcpm/__init__.py +ENV SETUPTOOLS_SCM_PRETEND_VERSION=0.0.0 +RUN pip install --no-cache-dir -e . + +# Layer 2: Copy full project source (cheap rebuild on code changes) +COPY . /app/ + +# Create default directories and declare volumes +RUN mkdir -p /app/lora /app/models /app/output /app/data +VOLUME ["/app/models", "/app/lora", "/app/output", "/app/data"] + +EXPOSE 7860 + +# Environment variables for configuration +ENV GRADIO_SERVER_PORT=7860 +ENV GRADIO_ROOT_PATH="" +ENV HF_HOME=/app/models + +# Default: launch training WebUI +CMD ["python", "lora_ft_webui.py"] diff --git a/docker/README.md b/docker/README.md new file mode 100644 index 0000000..5d0f261 --- /dev/null +++ b/docker/README.md @@ -0,0 +1,155 @@ +# Docker Support for VoxCPM Training WebUI + +Run the VoxCPM LoRA fine-tuning WebUI in a Docker container with full GPU support and nginx reverse proxy. + +## Prerequisites + +- Docker Engine 19.03+ with [NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/install-guide.html) +- NVIDIA GPU with CUDA 12.4+ compatible drivers +- At least 16 GB GPU VRAM (24 GB+ recommended for larger models) + +## Quick Start + +```bash +# From the project root directory: +docker compose -f docker/docker-compose.yml up --build +``` + +This starts: +- **training-webui** — the Gradio-based training interface on port 7860 +- **nginx** — reverse proxy serving the WebUI at `http://localhost/webui/` + +Access the WebUI at **http://localhost/webui/**. + +## Volume Mounts + +The compose file maps host directories to container paths. Create these directories at the project root before starting: + +``` +VoxCPM/ +├── docker/ +│ ├── docker-compose.yml +│ ├── Dockerfile +│ └── nginx.conf +├── models/ ← Pretrained model weights (or auto-downloaded via HF) +│ ├── openbmb__VoxCPM2/ +│ └── openbmb__VoxCPM1.5/ +├── data/ ← Training manifests + audio files +│ ├── train.jsonl +│ ├── val.jsonl (optional) +│ └── audio/ +│ ├── speaker1_001.wav +│ └── ... +├── lora/ ← LoRA training output (created automatically) +│ └── my-voice-2024/ +│ ├── checkpoints/ +│ ├── logs/ +│ └── train_config.yaml +└── output/ ← Additional training artifacts +``` + +### Mount Reference + +| Host Path | Container Path | Purpose | +|-----------|---------------|---------| +| `./models/` | `/app/models` | Pretrained model weights and HF cache (`HF_HOME`). Pre-populate with model dirs (e.g., `openbmb__VoxCPM2/`) or leave empty — models auto-download on first run and persist here. | +| `./data/` | `/app/data` | Training data. Put JSONL manifests and audio files here. In the WebUI, reference paths as `/app/data/train.jsonl`. | +| `./lora/` | `/app/lora` | LoRA checkpoint output. After training, find results in `lora//checkpoints/`. Also used to resume training from existing checkpoints. | +| `./output/` | `/app/output` | Miscellaneous training artifacts. | + +### Training Data Format + +The train manifest is a JSONL file where each line references an audio file: + +```json +{"audio_path": "/app/data/audio/speaker1_001.wav", "text": "Hello world", "speaker": "speaker1"} +``` + +Use absolute container paths (`/app/data/...`) in your manifest so the container can find the files. + +### Models + +If `models/openbmb__VoxCPM2/` exists on the host, the app loads directly from that path — no network access needed. If the directory is empty or missing, `from_pretrained` falls back to `snapshot_download` from HuggingFace Hub. + +The Dockerfile sets `HF_HOME=/app/models` so any Hub downloads land in the same mounted volume. This means models persist across container restarts regardless of whether they were pre-populated or auto-downloaded. + +**Recommended:** Pre-populate to avoid first-run download delay: + +```bash +huggingface-cli download openbmb/VoxCPM2 --local-dir ./models/openbmb__VoxCPM2 +``` + +The Dockerfile creates empty `/app/models`, `/app/lora`, `/app/output` directories, but the volume mounts override them with your host directories. + +## Health Check + +The nginx proxy forwards `GET /` to the training-webui backend, so load balancer health checks (AWS ALB, etc.) reflect real application health — returning 502 when the backend is down. This is separate from the WebUI at `/webui/`. + +```bash +curl http://localhost/ +``` + +## Direct Access (no proxy) + +If you want to bypass nginx and access Gradio directly: + +```bash +docker compose -f docker/docker-compose.yml up --build training-webui +``` + +Set `GRADIO_ROOT_PATH=` (empty) in the compose file when running without the proxy, then access at `http://localhost:7860`. + +## Building Manually + +```bash +# Build the image +docker build -f docker/Dockerfile -t voxcpm-training . + +# Run with GPU access (no reverse proxy) +docker run --gpus all -p 7860:7860 \ + -v ./models:/app/models \ + -v ./data:/app/data \ + -v ./lora:/app/lora \ + -v ./output:/app/output \ + voxcpm-training +``` + +## Environment Variables + +| Variable | Default | Description | +|----------|---------|-------------| +| `GRADIO_SERVER_PORT` | `7860` | Port for the WebUI server | +| `GRADIO_ROOT_PATH` | `""` | URL prefix when behind a reverse proxy (e.g., `/webui`) | + +## Reverse Proxy + +The included `docker-compose.yml` ships with an nginx reverse proxy that serves the WebUI at `/webui/`. The `GRADIO_ROOT_PATH=/webui` env var ensures Gradio generates correct URLs for assets and WebSocket connections. + +### Custom nginx config + +Edit `docker/nginx.conf` to change the location prefix or add TLS. + +### Traefik Example (labels) + +```yaml +labels: + - "traefik.http.routers.voxcpm.rule=PathPrefix(`/webui`)" + - "traefik.http.services.voxcpm.loadbalancer.server.port=7860" +``` + +## Viewing Training Logs + +Training subprocess output is streamed to stdout, visible via: + +```bash +docker compose -f docker/docker-compose.yml logs -f training-webui +``` + +## Troubleshooting + +- **"no NVIDIA GPU detected"**: Ensure the NVIDIA Container Toolkit is installed and `docker run --gpus all nvidia-smi` works. +- **OOM errors**: Reduce batch size in the WebUI or use a GPU with more VRAM. +- **WebUI not accessible**: Check that port 80 (nginx) or 7860 (direct) isn't blocked by a firewall. +- **WebSocket errors behind proxy**: Ensure your proxy forwards `Upgrade` and `Connection` headers (the included nginx.conf handles this). +- **Health check failing**: Ensure the training-webui container is running — `curl http://localhost/` proxies to the backend and returns 502 if it's unreachable. +- **Mixed-content / audio not playing over HTTPS**: The nginx config uses `map $http_x_forwarded_proto` to pass the correct protocol through to Gradio. This ensures `https://` file URLs are generated when accessed via HTTPS through a load balancer. diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml new file mode 100644 index 0000000..5e99f04 --- /dev/null +++ b/docker/docker-compose.yml @@ -0,0 +1,43 @@ +services: + training-webui: + build: + context: .. + dockerfile: docker/Dockerfile + ports: + - "7860:7860" + volumes: + # Pretrained model weights + HF cache (HF_HOME=/app/models in Dockerfile). + # Pre-populate with model dirs, or leave empty — auto-downloads on first run. + - ../models:/app/models + + # Training data: JSONL manifests and audio files. + # Reference paths inside the container as /app/data/train.jsonl etc. + - ../data:/app/data + + # LoRA training output — checkpoints, configs, logs. + # Results appear in lora//checkpoints/ after training. + - ../lora:/app/lora + + # Additional training artifacts. + - ../output:/app/output + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: 1 + capabilities: [gpu] + environment: + - GRADIO_SERVER_PORT=7860 + - GRADIO_ROOT_PATH=/webui # Matches nginx location block + restart: unless-stopped + + nginx: + image: nginx:alpine + ports: + - "80:80" + volumes: + - ./nginx.conf:/etc/nginx/conf.d/default.conf:ro + depends_on: + - training-webui + restart: unless-stopped diff --git a/docker/nginx.conf b/docker/nginx.conf new file mode 100644 index 0000000..5af3f9c --- /dev/null +++ b/docker/nginx.conf @@ -0,0 +1,51 @@ +# Preserve X-Forwarded-Proto from upstream load balancer (e.g. AWS ALB). +# If ALB already set it to "https", pass that through instead of $scheme +# (which is "http" since ALB→nginx is unencrypted). Falls back to $scheme +# when accessed directly (no upstream proxy). +map $http_x_forwarded_proto $forwarded_proto { + default $http_x_forwarded_proto; + "" $scheme; +} + +server { + listen 80; + server_name _; + + absolute_redirect off; + + # Health check for load balancers (AWS ALB, etc.) + location = / { + proxy_pass http://training-webui:7860/; + proxy_set_header Host $host; + proxy_read_timeout 5s; + proxy_connect_timeout 3s; + access_log off; + } + + location = /manifest.json { + return 200 '{"name":"VoxCPM Training","short_name":"VoxCPM","start_url":"/webui/"}'; + default_type application/json; + } + + location = /favicon.ico { + return 204; + access_log off; + } + + location /webui/ { + proxy_pass http://training-webui:7860/; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $forwarded_proto; + + # WebSocket support (required for Gradio) + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + + # Increase timeouts for long-running training operations + proxy_read_timeout 300s; + proxy_send_timeout 300s; + } +} diff --git a/lora_ft_webui.py b/lora_ft_webui.py index 3d91c3d..69f272a 100644 --- a/lora_ft_webui.py +++ b/lora_ft_webui.py @@ -500,6 +500,7 @@ def start_training( assert training_process.stdout is not None for line in training_process.stdout: + print(line, end="", flush=True) # Stream to stdout (Docker logs) training_log += line # Keep log size manageable if len(training_log) > 100000: @@ -1322,6 +1323,9 @@ with gr.Blocks(title="VoxCPM LoRA WebUI", theme=gr.themes.Soft(), css=custom_css ) if __name__ == "__main__": - # Ensure lora directory exists os.makedirs("lora", exist_ok=True) - app.queue().launch(server_name="0.0.0.0", server_port=7860) + port = int(os.environ.get("GRADIO_SERVER_PORT", "7860")) + root_path = os.environ.get("GRADIO_ROOT_PATH", "") + + print(f"\U0001f399\ufe0f VoxCPM Training WebUI: http://0.0.0.0:{port}{root_path}", flush=True) + app.queue().launch(server_name="0.0.0.0", server_port=port, root_path=root_path)