fix: optimize concurrency settings and update documentation for performance improvements

app/config/settings.py

@@ -25,8 +25,11 @@ class Settings(BaseSettings):
     api_key_enabled: bool = False
     api_key: str = ""
 
-    # Concurrency
-    max_concurrent_syntheses: int = Field(default=2, ge=1)
+    # Concurrency. Safe hardware-blind floor: 1 = never oversubscribe CPU. Each
+    # synthesis uses ~2 cores (pocket-tts's generate→decode pipeline), so a 2-core box
+    # thrashes at 2 under mixed-language load. configure.sh derives a higher value from
+    # nproc/RAM for bigger hardware.
+    max_concurrent_syntheses: int = Field(default=1, ge=1)
 
     # Providers
     enabled_providers: str = "pocket"

app/core/synthesizer.py

@@ -1,5 +1,6 @@
 import logging
 import struct
+import time
 
 from app.api.errors import AppError, PayloadTooLarge, RequestValidationFailed, ServiceNotReady
 from app.config.settings import settings
@@ -69,12 +70,11 @@ async def synthesize(
 
         out = request.output
         logger.info(
-            "synthesize voice=%s provider=%s format=%s chars=%d boundary=%s",
+            "synth start voice=%s provider=%s fmt=%s chars=%d",
             voice.identifier,
             provider.id,
             out.format.value,
             len(text),
-            request.boundary,
         )
         params = SynthesisParams(
             text=text,
@@ -91,6 +91,7 @@ async def synthesize(
         if breaker is not None and not breaker.allow():
             raise ServiceNotReady(f"Provider '{provider.id}' is temporarily unavailable")
 
+        t0 = time.monotonic()
         try:
             result: AudioResult = await provider.synthesize(params)
         except AppError:
@@ -100,23 +101,35 @@ async def synthesize(
         else:
             if breaker is not None:
                 breaker.record_success()
+        gen_ms = round((time.monotonic() - t0) * 1000, 1)
 
         if out.format == AudioFormat.WAV:
             audio = _pcm_to_wav(result.pcm, result.sample_rate)
-            logger.info("generated wav pcm_bytes=%d output_bytes=%d", len(result.pcm), len(audio))
+            logger.info(
+                "synth done voice=%s fmt=wav gen_ms=%.1f total_ms=%.1f bytes=%d",
+                voice.identifier,
+                gen_ms,
+                round((time.monotonic() - t0) * 1000, 1),
+                len(audio),
+            )
             return (audio, "audio/wav", result.boundaries, boundaries_supported)
 
+        enc_t = time.monotonic()
         encoded = await ffmpeg_driver.encode(
             result.pcm,
             result.sample_rate,
             out.format.value,
             ffmpeg_bin=settings.ffmpeg_bin,
             bitrate=out.bitrate,
         )
+        enc_ms = round((time.monotonic() - enc_t) * 1000, 1)
         logger.info(
-            "generated %s pcm_bytes=%d output_bytes=%d",
+            "synth done voice=%s fmt=%s gen_ms=%.1f enc_ms=%.1f total_ms=%.1f bytes=%d",
+            voice.identifier,
             out.format.value,
-            len(result.pcm),
+            gen_ms,
+            enc_ms,
+            round((time.monotonic() - t0) * 1000, 1),
             len(encoded),
         )
         return (

app/providers/pocket_tts.py

@@ -23,13 +23,17 @@
 
 _VOICES_PATH = Path(__file__).parent.parent / "data" / "voices" / "pocket" / "voices.json"
 
-# BCP-47 prefix → pocket-tts language identifier
+# BCP-47 prefix → pocket-tts language identifier. Default to the fast/small 6-layer model for
+# every language that has one. French is the exception: pocket-tts ships only a 24-layer French
+# (`french_24l`), no 6-layer variant. de/es/it/pt/en also have 24-layer (`_24l`) variants that are
+# higher quality but ~3× larger and ~4× slower — not used here. Making this selectable is deferred
+# (see issue); flip a value to its `_24l` form to opt a language into the heavier model.
 _LANG_MODEL: dict[str, str] = {
     "en": "english",
     "fr": "french_24l",
     "it": "italian",
-    "de": "german_24l",
-    "es": "spanish_24l",
+    "de": "german",
+    "es": "spanish",
     "pt": "portuguese",
 }
 
@@ -59,10 +63,13 @@ async def load(self) -> None:
         self._ready = await anyio.to_thread.run_sync(self._load_sync)
 
     def _load_sync(self) -> bool:
-        import torch
         from pocket_tts import TTSModel
 
-        torch.set_num_threads(1)
+        # Do NOT set torch.set_num_threads() here. pocket-tts pins it to 1 itself on
+        # import (models/tts_model.py) by design: each model runs a 2-thread
+        # generate→decode pipeline that already saturates its documented "2 CPU cores"
+        # ceiling. Raising intra-op threads oversubscribes those pipeline threads and
+        # makes inference SLOWER, not faster. Scale throughput with WORKERS, not threads.
 
         enabled = self.active_languages()
         if not enabled:

docs/configuration.md

@@ -70,7 +70,7 @@ only ever *adds* cross-language support:
 |---|---|---|
 | `HF_TOKEN` | _(empty)_ | HuggingFace token. Optional — prevents rate-limiting on first-run model downloads. Shared across any provider that pulls models from HF |
 | `WORKERS` | `1` | Uvicorn worker processes. Each loads a full copy of every active language model |
-| `MAX_CONCURRENT_SYNTHESES` | `2` | Max parallel CPU inference jobs per worker |
+| `MAX_CONCURRENT_SYNTHESES` | `1` | Max parallel CPU inference jobs per worker. Each job uses ~2 cores; `configure.sh` derives this from `nproc`/RAM so cores ≈ workers × concurrency × 2 (no oversubscription) |
 | `CIRCUIT_BREAKER_ENABLED` | `true` | Trip a provider's circuit breaker after repeated `synthesize()` failures, returning `503` immediately instead of hammering a broken provider |
 | `CIRCUIT_BREAKER_FAILURE_THRESHOLD` | `5` | Consecutive failures before a provider's breaker opens |
 | `CIRCUIT_BREAKER_RECOVERY_SECONDS` | `30` | How long an open breaker waits before allowing one trial call |
@@ -88,7 +88,7 @@ only ever *adds* cross-language support:
 
 | Variable | Default | Description |
 |---|---|---|
-| `LANGUAGES` | _(empty)_ | Comma-separated BCP-47 language codes to **load as base models**. No hardcoded fallback — env-driven; unset means no base models load (no voices served). Size per language is *not* uniform: the 6-layer `en`/`it`/`pt` models are ~219 MB download / ~438 MB RAM; the 24-layer `fr`/`de`/`es` models are ~672 MB download / ~1344 MB RAM (loaded size ~2x the download, measured at startup). Supported: `en fr it de es pt`. This is the ceiling — nothing below can exceed it |
+| `LANGUAGES` | _(empty)_ | Comma-separated BCP-47 language codes to **load as base models**. No hardcoded fallback — env-driven; unset means no base models load (no voices served). Size per language is *not* uniform: the 6-layer `en`/`it`/`de`/`es`/`pt` models are ~219 MB download / ~438 MB RAM; the 24-layer `fr` model is ~672 MB download / ~1344 MB RAM (loaded size ~2x the download, measured at startup). Supported: `en fr it de es pt`. This is the ceiling — nothing below can exceed it |
 | `VOICE_LANGUAGES` | _(empty)_ | Which voices get warmed against which of those loaded models, beyond each voice's own primary — see [Per-voice language overrides](#per-voice-language-overrides) |
 | `POCKET_DEFAULT_VOICE` | _(empty)_ | Default voice when none is specified. No hardcoded fallback — env-driven; empty means the setting is unset |
 
@@ -144,8 +144,8 @@ are approximate.
 
 | Languages | Layers | Download (disk, once) | RAM (per worker) |
 |---|---|---|---|
-| `en` `it` `pt` | 6 | ~219 MB | ~438 MB |
-| `fr` `de` `es` | 24 (`_24l`) | ~672 MB | ~1344 MB |
+| `en` `it` `de` `es` `pt` | 6 | ~219 MB | ~438 MB |
+| `fr` | 24 (`_24l`) | ~672 MB | ~1344 MB |
 
 RAM ≈ 2× the download (measured at startup). This is the dominant cost.
 

docs/providers/pocket.md

@@ -51,5 +51,24 @@ re-encoding (the default, and fastest — no ffmpeg). The server can transcode o
 
 ## Constraints
 
-- CPU only; torch ≥ 2.5 CPU build. `en`/`it`/`pt` are 6-layer models; `fr`/`de`/`es` are larger
-  24-layer models (slower, higher quality). Throughput scales by worker processes.
+- CPU only; torch ≥ 2.5 CPU build. Throughput scales by worker processes.
+- **Model variant:** most languages ship both a fast/small **6-layer** model and a slower/bigger
+  higher-quality **24-layer** (`_24l`) one. We default to **6-layer** everywhere it exists.
+  French is the exception — pocket-tts ships only a 24-layer French, so `fr` is always 24-layer.
+
+  | Lang | 6-layer | 24-layer | Used |
+  |---|---|---|---|
+  | `en` | `english` | — | 6-layer |
+  | `fr` | — | `french_24l` | **24-layer (only option)** |
+  | `it` | `italian` | `italian_24l` | 6-layer |
+  | `de` | `german` | `german_24l` | 6-layer |
+  | `es` | `spanish` | `spanish_24l` | 6-layer |
+  | `pt` | `portuguese` | `portuguese_24l` | 6-layer |
+
+  (This is why you see `french_24l` but plain `spanish`/`german` — not an inconsistency; French
+  has no 6-layer.) The mapping lives in `_LANG_MODEL` (`app/providers/pocket_tts.py`). Making the
+  6l/24l choice user-selectable per language is **deferred** pending need — see the follow-up
+  issue.
+- Each model uses ~2 CPU cores via an internal generate→decode pipeline and pins torch to a
+  single intra-op thread itself (on import) — so we don't set `torch.set_num_threads()`. Give
+  the box more cores by running more workers, not more threads per model.

docs/voices.md

@@ -52,7 +52,7 @@ languages without repeating it per voice.
 
 PocketTTS pre-computes voice embeddings per voice per language (stored as `.safetensors` files
 under `kyutai/pocket-tts-without-voice-cloning`), sized by the target language's model:
-~5–8 MB into the 6-layer `en`/`it`/`pt` models, ~24–33 MB into the 24-layer `fr`/`de`/`es`. The embedding is warmed once at
+~5–8 MB into the 6-layer `en`/`it`/`de`/`es`/`pt` models, ~24–33 MB into the 24-layer `fr`. The embedding is warmed once at
 model-load time — no per-request cloning overhead — but only for the (voice, language) pairs
 `VOICE_LANGUAGES` and `LANGUAGES` actually call for. Unused language weights can be reclaimed
 from the Docker volume with `scripts/prune_weights.py` — see [Configuration → disk space](configuration.md).

scripts/configure.sh

@@ -53,6 +53,36 @@ set_env()  { _set_kv "$ENV_FILE" "$1" "$2"; }
 get_penv() { _get_kv "$POCKET_ENV_FILE" "$1"; }
 set_penv() { _set_kv "$POCKET_ENV_FILE" "$1" "$2"; }
 
+# Hardware-derived defaults. pocket-tts uses ~2 CPU cores per synthesis (its
+# generate→decode pipeline; docs: "Uses only 2 CPU cores") and each worker holds a
+# full model copy (~2 GB for en+fr). So 1 worker ≈ 2 cores + ~2 GB RAM. Derive
+# non-thrashing WORKERS / MAX_CONCURRENT_SYNTHESES from the box instead of shipping a
+# static default that only fits one machine. Sets DETECTED_* and SUGGESTED_* globals.
+_detect_hw() {
+    local mem_bytes w_by_core w_by_ram
+    if [[ "$(uname)" == "Darwin" ]]; then
+        DETECTED_CORES=$(sysctl -n hw.ncpu 2>/dev/null || echo 2)
+        mem_bytes=$(sysctl -n hw.memsize 2>/dev/null || echo 0)
+        DETECTED_RAM_GB=$(( mem_bytes / 1024 / 1024 / 1024 ))
+    else
+        DETECTED_CORES=$(nproc 2>/dev/null || echo 2)
+        DETECTED_RAM_GB=$(awk '/MemTotal/ {print int($2/1024/1024)}' /proc/meminfo 2>/dev/null || echo 0)
+    fi
+    [[ "$DETECTED_CORES"  -ge 1 ]] || DETECTED_CORES=1
+    [[ "$DETECTED_RAM_GB" -ge 1 ]] || DETECTED_RAM_GB=2   # unknown RAM → assume one worker
+
+    w_by_core=$(( DETECTED_CORES / 2 )); [[ "$w_by_core" -ge 1 ]] || w_by_core=1
+    w_by_ram=$(( DETECTED_RAM_GB / 2 )); [[ "$w_by_ram"  -ge 1 ]] || w_by_ram=1
+    SUGGESTED_WORKERS=$(( w_by_core < w_by_ram ? w_by_core : w_by_ram ))
+}
+
+# Per-worker concurrent syntheses that fit the box's cores without oversubscribing:
+# total 2-core jobs (cores/2) spread across the chosen worker count. Args: <workers>.
+_concurrency_for() {
+    local c=$(( DETECTED_CORES / (2 * $1) ))
+    [[ "$c" -ge 1 ]] && echo "$c" || echo 1
+}
+
 # ── Migrations (one-time, idempotent — safe to call every run) ─────────────────
 _migrate_pocket_env() {
     [[ -f "$ENV_FILE" ]] || return 0
@@ -382,15 +412,22 @@ do_first_setup() {
     hdr "Step 2/5 — Voice coverage"
     _set_coverage
 
-    hdr "Step 3/5 — Workers"
-    WORKERS=$(gum input --placeholder "1" \
-        --header "Uvicorn worker processes (throughput; RAM scales with count — see docs)") || true
-    WORKERS="${WORKERS:-1}"
+    hdr "Step 3/5 — Workers & concurrency"
+    _detect_hw
+    ok "Detected: ${DETECTED_CORES} cores, ~${DETECTED_RAM_GB} GB RAM"
+    WORKERS=$(gum input --placeholder "$SUGGESTED_WORKERS" \
+        --header "Uvicorn workers (each ≈2 cores + ~2 GB RAM; suggested $SUGGESTED_WORKERS)") || true
+    WORKERS="${WORKERS:-$SUGGESTED_WORKERS}"
     case "$WORKERS" in
-        ''|*[!0-9]*|0) warn "Invalid — using 1"; WORKERS=1 ;;
+        ''|*[!0-9]*|0) warn "Invalid — using $SUGGESTED_WORKERS"; WORKERS=$SUGGESTED_WORKERS ;;
     esac
     ok "Workers: $WORKERS"
 
+    # Derive per-worker concurrency for the ACTUAL worker count chosen, so cores stay
+    # ≈ workers × concurrency × 2 and never oversubscribe (which slows every request).
+    MAX_CONCURRENT=$(_concurrency_for "$WORKERS")
+    ok "Max concurrent syntheses/worker: $MAX_CONCURRENT (${DETECTED_CORES} cores ÷ 2 ÷ ${WORKERS} workers)"
+
     hdr "Step 4/5 — HuggingFace Token"
     HF_TOKEN=$(gum input --password --placeholder "hf_..." \
         --header "Get a free read-only token at: https://huggingface.co/settings/tokens") || true
@@ -448,6 +485,7 @@ MAX_TEXT_LENGTH=2000
 EOF
     chmod 600 "$ENV_FILE"
     set_env WORKERS "$WORKERS"
+    set_env MAX_CONCURRENT_SYNTHESES "$MAX_CONCURRENT"
     set_env HF_TOKEN "$HF_TOKEN"
     set_env DOMAIN "$DOMAIN_INPUT"