Feat/sglang image vllm asr engines

doc/source/models/model_abilities/audio.rst

@@ -69,6 +69,20 @@ For Mac M-series chips only:
 * :ref:`whisper-large-v3-mlx <models_builtin_whisper-large-v3-mlx>`
 * :ref:`whisper-large-v3-turbo-mlx <models_builtin_whisper-large-v3-turbo-mlx>`
 
+Audio engines
+~~~~~~~~~~~~~
+
+``Qwen3-ASR-0.6B`` and ``Qwen3-ASR-1.7B`` support engine selection. The default
+``transformers`` engine works on all platforms; on Linux with NVIDIA GPUs they
+can also run on the ``vLLM`` engine for faster transcriptions. To use it,
+install the vLLM backend of `qwen-asr <https://pypi.org/project/qwen-asr/>`_ via
+``pip install 'qwen-asr[vllm]'``, then launch the model with
+``--model-engine vLLM``, for example:
+
+.. code-block:: bash
+
+    xinference launch --model-name Qwen3-ASR-1.7B --model-type audio --model-engine vLLM
+
 
 Text to audio (TTS)
 ~~~~~~~~~~~~~~~~~~~~~

doc/source/models/model_abilities/image.rst

@@ -58,6 +58,32 @@ Image-to-image supported models:
 * Qwen-Image-Edit
 
 
+Image engines
+-------------------
+
+Text-to-image models run on the ``diffusers`` engine by default. On Linux with
+NVIDIA GPUs, the following models can also run on the ``SGLang`` engine
+(powered by `sglang-diffusion <https://docs.sglang.io/docs/sglang-diffusion/installation>`_)
+for faster inference:
+
+* FLUX.1-dev
+* Qwen-Image
+* Qwen-Image-2512
+* Z-Image
+* Z-Image-Turbo
+
+To use it, install SGLang with diffusion support via
+``pip install 'sglang[diffusion]'``, then launch the model with
+``--model-engine SGLang``, for example:
+
+.. code-block:: bash
+
+    xinference launch --model-name Z-Image-Turbo --model-type image --model-engine SGLang
+
+Note that GGUF quantization, Lightning acceleration, LoRA and controlnet are
+only available on the ``diffusers`` engine.
+
+
 Quickstart
 ===================
 

doc/source/user_guide/backends.rst

@@ -172,6 +172,11 @@ SGLang
 It significantly accelerates the execution of complex LLM programs by automatic KV cache reuse across multiple calls.
 And it also supports other common techniques like continuous batching and tensor parallelism.
 
+Besides LLMs, SGLang can also serve as the engine for some text-to-image models
+(e.g. ``Qwen-Image``, ``Z-Image-Turbo``) on Linux with NVIDIA GPUs, see
+:ref:`image engines <image>`. This requires SGLang with diffusion support:
+``pip install 'sglang[diffusion]'``.
+
 .. _mlx_backend:
 
 MLX

xinference/model/audio/__init__.py

@@ -34,6 +34,8 @@
     register_audio,
     unregister_audio,
 )
+from .engine import register_builtin_audio_engines
+from .engine_family import generate_engine_config_by_model_name
 
 BUILTIN_AUDIO_MODELS: Dict[str, List["AudioModelFamilyV2"]] = {}
 
@@ -87,11 +89,17 @@ def _install():
         if model_spec.model_name not in AUDIO_MODEL_DESCRIPTIONS:
             AUDIO_MODEL_DESCRIPTIONS.update(generate_audio_description(model_spec))
 
+    register_builtin_audio_engines()
+    for model_specs in BUILTIN_AUDIO_MODELS.values():
+        for model_spec in model_specs:
+            generate_engine_config_by_model_name(model_spec)
+
     register_custom_model()
 
     # register model description
     for ud_audio in get_user_defined_audios():
         AUDIO_MODEL_DESCRIPTIONS.update(generate_audio_description(ud_audio))
+        generate_engine_config_by_model_name(ud_audio)
 
 
 def register_builtin_model():

xinference/model/audio/core.py

@@ -19,6 +19,7 @@
 from ..utils import ModelInstanceInfoMixin
 from .chattts import ChatTTSModel
 from .cosyvoice import CosyVoiceModel
+from .engine_family import AudioEngineModel
 from .f5tts import F5TTSModel
 from .f5tts_mlx import F5TTSMLXModel
 from .fish_speech import FishSpeechModel
@@ -142,6 +143,7 @@ def create_audio_model_instance(
         Literal["huggingface", "modelscope", "openmind_hub", "csghub"]
     ] = None,
     model_path: Optional[str] = None,
+    model_engine: Optional[str] = None,
     **kwargs,
 ) -> Union[
     WhisperModel,
@@ -161,14 +163,55 @@ def create_audio_model_instance(
     Qwen3ASRModel,
     Qwen3TTSModel,
     VoxCPMModel,
+    AudioEngineModel,
 ]:
     from ..cache_manager import CacheManager
+    from .engine_family import AUDIO_ENGINES
 
-    kwargs.pop("enable_virtual_env", None)
+    enable_virtual_env = kwargs.pop("enable_virtual_env", None)
     model_spec = match_audio(model_name, download_hub)
     if model_path is None:
         cache_manager = CacheManager(model_spec)
         model_path = cache_manager.cache()
+
+    # Engine-aware dispatch for model families with multiple engines
+    # (e.g. qwen3_asr on transformers or vLLM). Families without registered
+    # engines keep the legacy dispatch below.
+    if model_spec.model_name in AUDIO_ENGINES or model_engine is not None:
+        from .engine_family import (
+            check_engine_by_model_name_and_engine,
+            check_engine_by_model_name_and_engine_with_virtual_env,
+        )
+
+        if model_engine is None:
+            # the first registered engine is the default one
+            model_engine = next(iter(AUDIO_ENGINES[model_spec.model_name]))
+
+        if model_spec.model_name not in AUDIO_ENGINES:
+            logger.warning(
+                "Audio model %s does not support engine selection, "
+                "`model_engine=%s` is ignored.",
+                model_spec.model_name,
+                model_engine,
+            )
+        else:
+            if enable_virtual_env is None:
+                from ...constants import XINFERENCE_ENABLE_VIRTUAL_ENV
+
+                enable_virtual_env = XINFERENCE_ENABLE_VIRTUAL_ENV
+            if enable_virtual_env:
+                audio_cls = check_engine_by_model_name_and_engine_with_virtual_env(
+                    model_engine,
+                    model_spec.model_name,
+                    model_family=model_spec,
+                )
+            else:
+                audio_cls = check_engine_by_model_name_and_engine(
+                    model_engine,
+                    model_spec.model_name,
+                )
+            return audio_cls(model_uid, model_path, model_spec, **kwargs)  # type: ignore
+
     model: Union[
         WhisperModel,
         WhisperMLXModel,

xinference/model/audio/custom.py

@@ -95,10 +95,17 @@ def register_audio(model_spec: CustomAudioModelFamilyV2, persist: bool):
 
     registry = RegistryManager.get_registry("audio")
     registry.register(model_spec, persist)
+    from .engine_family import generate_engine_config_by_model_name
+
+    generate_engine_config_by_model_name(model_spec)
 
 
 def unregister_audio(model_name: str, raise_error: bool = True):
     from ..custom import RegistryManager
 
     registry = RegistryManager.get_registry("audio")
     registry.unregister(model_name, raise_error)
+    from .engine_family import AUDIO_ENGINES
+
+    if model_name in AUDIO_ENGINES:
+        del AUDIO_ENGINES[model_name]

xinference/model/audio/engine.py

@@ -0,0 +1,50 @@
+# Copyright 2022-2026 XProbe Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import platform
+from typing import TYPE_CHECKING
+
+from ..utils import has_cuda_device
+from .engine_family import SUPPORTED_ENGINES, AudioEngineModel
+from .qwen3_asr import Qwen3ASRModel
+from .vllm import VLLMQwen3ASRModel
+
+if TYPE_CHECKING:
+    from .core import AudioModelFamilyV2
+
+
+class TransformersQwen3ASRAudioModel(Qwen3ASRModel, AudioEngineModel):
+    required_libs = ("qwen_asr",)
+
+    @classmethod
+    def match(cls, model_family: "AudioModelFamilyV2") -> bool:
+        return model_family.model_family == "qwen3_asr"
+
+
+class VLLMQwen3ASRAudioModel(VLLMQwen3ASRModel, AudioEngineModel):
+    required_libs = ("qwen_asr", "vllm")
+
+    @classmethod
+    def match(cls, model_family: "AudioModelFamilyV2") -> bool:
+        if platform.system() != "Linux":
+            return False
+        if not has_cuda_device():
+            return False
+        return model_family.model_family == "qwen3_asr"
+
+
+def register_builtin_audio_engines() -> None:
+    # the first registered engine is the default one for a model
+    SUPPORTED_ENGINES["transformers"] = [TransformersQwen3ASRAudioModel]
+    SUPPORTED_ENGINES["vLLM"] = [VLLMQwen3ASRAudioModel]

xinference/model/audio/engine_family.py

@@ -0,0 +1,130 @@
+# Copyright 2022-2026 XProbe Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import importlib.util
+import logging
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Type, Union
+
+if TYPE_CHECKING:
+    from .core import AudioModelFamilyV2
+
+logger = logging.getLogger(__name__)
+
+
+class AudioEngineModel:
+    required_libs: Tuple[str, ...] = ()
+
+    @classmethod
+    def match(cls, model_family: "AudioModelFamilyV2") -> bool:
+        raise NotImplementedError
+
+    @classmethod
+    def check_lib(cls) -> Union[bool, Tuple[bool, str]]:
+        for lib in cls.required_libs:
+            if importlib.util.find_spec(lib) is None:
+                return False, f"Library '{lib}' is not installed"
+        return True
+
+
+# { audio model name -> { engine name -> engine params } }
+AUDIO_ENGINES: Dict[str, Dict[str, List[Dict[str, Any]]]] = {}
+SUPPORTED_ENGINES: Dict[str, List[Type[AudioEngineModel]]] = {}
+
+
+def check_engine_by_model_name_and_engine(
+    model_engine: str,
+    model_name: str,
+) -> Type[AudioEngineModel]:
+    def get_model_engine_from_spell(engine_str: str) -> str:
+        for engine in AUDIO_ENGINES[model_name].keys():
+            if engine.lower() == engine_str.lower():
+                return engine
+        return engine_str
+
+    if model_name not in AUDIO_ENGINES:
+        raise ValueError(f"Audio model {model_name} not found.")
+    model_engine = get_model_engine_from_spell(model_engine)
+    if model_engine not in AUDIO_ENGINES[model_name]:
+        raise ValueError(
+            f"Audio model {model_name} cannot be run on engine {model_engine}."
+        )
+    match_params = AUDIO_ENGINES[model_name][model_engine]
+    for param in match_params:
+        if model_name == param["model_name"]:
+            return param["audio_class"]
+    raise ValueError(
+        f"Audio model {model_name} cannot be run on engine {model_engine}."
+    )
+
+
+def check_engine_by_model_name_and_engine_with_virtual_env(
+    model_engine: str,
+    model_name: str,
+    model_family: Optional["AudioModelFamilyV2"] = None,
+) -> Type[AudioEngineModel]:
+    from ..utils import _collect_virtualenv_engine_markers
+
+    if model_family is None:
+        raise ValueError(f"Audio model {model_name} not found.")
+
+    engine_markers = _collect_virtualenv_engine_markers(model_family)
+
+    def _engine_class_by_marker() -> Optional[Type[AudioEngineModel]]:
+        if model_engine.lower() in engine_markers:
+            for engine, engine_classes in SUPPORTED_ENGINES.items():
+                if engine.lower() == model_engine.lower() and engine_classes:
+                    logger.warning(
+                        "Bypassing engine compatibility checks for %s due to "
+                        "virtualenv marker.",
+                        model_engine,
+                    )
+                    return engine_classes[0]
+        return None
+
+    if model_name not in AUDIO_ENGINES:
+        engine_cls = _engine_class_by_marker()
+        if engine_cls is not None:
+            return engine_cls
+        raise ValueError(f"Audio model {model_name} not found.")
+
+    try:
+        return check_engine_by_model_name_and_engine(model_engine, model_name)
+    except ValueError:
+        engine_cls = _engine_class_by_marker()
+        if engine_cls is not None:
+            return engine_cls
+        raise
+
+
+def generate_engine_config_by_model_name(model_family: "AudioModelFamilyV2") -> None:
+    model_name = model_family.model_name
+    engines: Dict[str, List[Dict[str, Any]]] = AUDIO_ENGINES.get(model_name, {})
+    for engine, classes in SUPPORTED_ENGINES.items():
+        for cls in classes:
+            if cls.match(model_family):
+                engine_params = engines.get(engine, [])
+                already_exists = any(
+                    param["model_name"] == model_name for param in engine_params
+                )
+                if not already_exists:
+                    engine_params.append(
+                        {
+                            "model_name": model_name,
+                            "audio_class": cls,
+                        }
+                    )
+                engines[engine] = engine_params
+                break
+    if engines:
+        AUDIO_ENGINES[model_name] = engines

xinference/model/audio/model_spec.json

@@ -1170,6 +1170,7 @@
     "virtualenv": {
       "packages": [
         "qwen-asr",
+        "qwen-asr[vllm] ; #engine# == \"vLLM\"",
         "#system_torch#",
         "#system_numpy#",
         "#system_pandas#"
@@ -1204,6 +1205,7 @@
     "virtualenv": {
       "packages": [
         "qwen-asr",
+        "qwen-asr[vllm] ; #engine# == \"vLLM\"",
         "#system_torch#",
         "#system_numpy#",
         "#system_pandas#"