From 2944c8bfad4e4aeb2ac1bef745b92bee71a940c0 Mon Sep 17 00:00:00 2001 From: OliverBryant <2713999266@qq.com> Date: Thu, 2 Jul 2026 17:46:32 +0800 Subject: [PATCH 1/3] FEAT: [image] support SGLang engine for Qwen-Image / Z-Image / FLUX.1-dev Implement the SGLangImageModel stub with sglang-diffusion offline API (sglang.multimodal_gen.DiffGenerator) so text-to-image models can run on the SGLang engine on Linux with NVIDIA GPUs. Supported models: Qwen-Image, Qwen-Image-2512, Z-Image, Z-Image-Turbo and FLUX.1-dev, matching the sglang-diffusion compatibility matrix. Virtualenv specs install sglang[diffusion] when the SGLang engine is selected. Ref #4896 Co-Authored-By: Claude Fable 5 --- doc/source/models/model_abilities/image.rst | 26 +++ doc/source/user_guide/backends.rst | 5 + xinference/model/image/engine.py | 23 +- xinference/model/image/model_spec.json | 5 + xinference/model/image/sglang/__init__.py | 13 ++ xinference/model/image/sglang/core.py | 220 ++++++++++++++++++ .../model/image/tests/test_sglang_engine.py | 182 +++++++++++++++ xinference/model/utils.py | 19 ++ 8 files changed, 483 insertions(+), 10 deletions(-) create mode 100644 xinference/model/image/sglang/__init__.py create mode 100644 xinference/model/image/sglang/core.py create mode 100644 xinference/model/image/tests/test_sglang_engine.py diff --git a/doc/source/models/model_abilities/image.rst b/doc/source/models/model_abilities/image.rst index 89d450e639..331494b847 100644 --- a/doc/source/models/model_abilities/image.rst +++ b/doc/source/models/model_abilities/image.rst @@ -58,6 +58,32 @@ Image-to-image supported models: * Qwen-Image-Edit +Image engines +------------------- + +Text-to-image models run on the ``diffusers`` engine by default. On Linux with +NVIDIA GPUs, the following models can also run on the ``SGLang`` engine +(powered by `sglang-diffusion `_) +for faster inference: + +* FLUX.1-dev +* Qwen-Image +* Qwen-Image-2512 +* Z-Image +* Z-Image-Turbo + +To use it, install SGLang with diffusion support via +``pip install 'sglang[diffusion]'``, then launch the model with +``--model-engine SGLang``, for example: + +.. code-block:: bash + + xinference launch --model-name Z-Image-Turbo --model-type image --model-engine SGLang + +Note that GGUF quantization, Lightning acceleration, LoRA and controlnet are +only available on the ``diffusers`` engine. + + Quickstart =================== diff --git a/doc/source/user_guide/backends.rst b/doc/source/user_guide/backends.rst index 71c5b18cf9..9ebb4ebf1c 100644 --- a/doc/source/user_guide/backends.rst +++ b/doc/source/user_guide/backends.rst @@ -172,6 +172,11 @@ SGLang It significantly accelerates the execution of complex LLM programs by automatic KV cache reuse across multiple calls. And it also supports other common techniques like continuous batching and tensor parallelism. +Besides LLMs, SGLang can also serve as the engine for some text-to-image models +(e.g. ``Qwen-Image``, ``Z-Image-Turbo``) on Linux with NVIDIA GPUs, see +:ref:`image engines `. This requires SGLang with diffusion support: +``pip install 'sglang[diffusion]'``. + .. _mlx_backend: MLX diff --git a/xinference/model/image/engine.py b/xinference/model/image/engine.py index 60c4232cc5..9029a10b14 100644 --- a/xinference/model/image/engine.py +++ b/xinference/model/image/engine.py @@ -12,9 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. +import platform from typing import TYPE_CHECKING +from ..utils import has_cuda_device from .engine_family import SUPPORTED_ENGINES, ImageEngineModel +from .sglang.core import SGLANG_SUPPORTED_IMAGE_MODELS, SGLangDiffusionModel from .stable_diffusion.core import DiffusionModel if TYPE_CHECKING: @@ -45,18 +48,18 @@ def check_lib(cls): ) -class SGLangImageModel(ImageEngineModel): - @classmethod - def match(cls, model_family: "ImageModelFamilyV2") -> bool: - _ = model_family - return False +class SGLangImageModel(SGLangDiffusionModel, ImageEngineModel): + engine_model_format = "diffusers" + engine_quantization = "none" + required_libs = ("sglang",) @classmethod - def check_lib(cls): - return ( - False, - "Engine SGLang is not compatible with current image model or environment", - ) + def match(cls, model_family: "ImageModelFamilyV2") -> bool: + if platform.system() != "Linux": + return False + if not has_cuda_device(): + return False + return model_family.model_name in SGLANG_SUPPORTED_IMAGE_MODELS def register_builtin_image_engines() -> None: diff --git a/xinference/model/image/model_spec.json b/xinference/model/image/model_spec.json index 747e8c5277..400fe415c6 100644 --- a/xinference/model/image/model_spec.json +++ b/xinference/model/image/model_spec.json @@ -129,6 +129,7 @@ "virtualenv": { "packages": [ "#diffusers_dependencies# ; #engine# == \"diffusers\"", + "sglang[diffusion] ; #engine# == \"SGLang\"", "transformers>=4.51.0", "#system_torch#", "#system_numpy#" @@ -291,6 +292,7 @@ "packages": [ "diffusers==0.35.1 ; #engine# == \"diffusers\"", "huggingface-hub<1.0 ; #engine# == \"diffusers\"", + "sglang[diffusion] ; #engine# == \"SGLang\"", "peft>=0.17.0", "#system_torch#", "#system_numpy#" @@ -1349,6 +1351,7 @@ "virtualenv": { "packages": [ "#diffusers_dependencies# ; #engine# == \"diffusers\"", + "sglang[diffusion] ; #engine# == \"SGLang\"", "transformers>=4.51.0", "#system_torch#", "#system_numpy#" @@ -1605,6 +1608,7 @@ "packages": [ "diffusers==0.35.1 ; #engine# == \"diffusers\"", "huggingface-hub<1.0 ; #engine# == \"diffusers\"", + "sglang[diffusion] ; #engine# == \"SGLang\"", "peft>=0.17.0", "#system_torch#", "#system_numpy#" @@ -1632,6 +1636,7 @@ "virtualenv": { "packages": [ "#diffusers_dependencies# ; #engine# == \"diffusers\"", + "sglang[diffusion] ; #engine# == \"SGLang\"", "transformers>=4.51.0", "#system_torch#", "#system_numpy#" diff --git a/xinference/model/image/sglang/__init__.py b/xinference/model/image/sglang/__init__.py new file mode 100644 index 0000000000..546dcb6ac3 --- /dev/null +++ b/xinference/model/image/sglang/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2022-2026 XProbe Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/xinference/model/image/sglang/core.py b/xinference/model/image/sglang/core.py new file mode 100644 index 0000000000..8d37952e5c --- /dev/null +++ b/xinference/model/image/sglang/core.py @@ -0,0 +1,220 @@ +# Copyright 2022-2026 XProbe Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import asyncio +import logging +import random +import re +from typing import TYPE_CHECKING, Any, Dict, List, Optional + +from ....types import LoRA +from ..utils import handle_image_result + +if TYPE_CHECKING: + from ..core import ImageModelFamilyV2 + +logger = logging.getLogger(__name__) + +# Image models verified against the sglang-diffusion compatibility matrix: +# https://docs.sglang.io/docs/sglang-diffusion/compatibility_matrix +SGLANG_SUPPORTED_IMAGE_MODELS = ( + "Qwen-Image", + "Qwen-Image-2512", + "Z-Image", + "Z-Image-Turbo", + "FLUX.1-dev", +) + + +def _filter_kwargs_by_dataclass_fields( + kwargs: Dict[str, Any], dataclass_type: Any, purpose: str +) -> Dict[str, Any]: + import dataclasses + + valid_keys = {f.name for f in dataclasses.fields(dataclass_type)} + dropped = sorted(set(kwargs) - valid_keys) + if dropped: + logger.info("Dropping args unsupported by SGLang %s: %s", purpose, dropped) + return {k: v for k, v in kwargs.items() if k in valid_keys} + + +class SGLangDiffusionModel: + def __init__( + self, + model_uid: str, + model_path: Optional[str] = None, + device: Optional[str] = None, + lora_model: Optional[List[LoRA]] = None, + lora_load_kwargs: Optional[Dict] = None, + lora_fuse_kwargs: Optional[Dict] = None, + model_spec: Optional["ImageModelFamilyV2"] = None, + gguf_model_path: Optional[str] = None, + lightning_model_path: Optional[str] = None, + **kwargs, + ): + if gguf_model_path: + raise ValueError( + "GGUF quantization is not supported by the SGLang image engine, " + "please use the diffusers engine instead" + ) + if lightning_model_path: + raise ValueError( + "Lightning LoRA acceleration is not supported by the SGLang image " + "engine, please use the diffusers engine instead" + ) + if lora_model: + raise ValueError( + "LoRA is not supported by the SGLang image engine yet, " + "please use the diffusers engine instead" + ) + if kwargs.get("controlnet"): + raise ValueError( + "Controlnet is not supported by the SGLang image engine, " + "please use the diffusers engine instead" + ) + self.model_family = model_spec + self._model_uid = model_uid + self._model_path = model_path + self._device = device + self._model = None + self._model_spec = model_spec + self._abilities = model_spec.model_ability or [] # type: ignore + self._kwargs = kwargs + + @property + def model_ability(self): + return self._abilities + + def load(self): + try: + from sglang.multimodal_gen import DiffGenerator + from sglang.multimodal_gen.runtime.server_args import ServerArgs + except ImportError: + error_message = "Failed to import module 'sglang.multimodal_gen'" + installation_guide = [ + "Please make sure 'sglang' with diffusion support is installed. ", + "You can install it by `pip install 'sglang[diffusion]'`\n", + ] + raise ImportError(f"{error_message}\n\n{''.join(installation_guide)}") + + engine_kwargs = _filter_kwargs_by_dataclass_fields( + self._kwargs, ServerArgs, "server args" + ) + logger.debug( + "Loading SGLang diffusion model from %s, kwargs: %s", + self._model_path, + engine_kwargs, + ) + self._model = DiffGenerator.from_pretrained( + model_path=self._model_path, + **engine_kwargs, + ) + + def stop(self): + if self._model is not None: + try: + self._model.shutdown() + except Exception: + logger.warning( + "Failed to shutdown SGLang diffusion model", exc_info=True + ) + self._model = None + + def _build_sampling_params( + self, + prompt: str, + n: int, + width: int, + height: int, + generate_config: Dict[str, Any], + ) -> Dict[str, Any]: + from sglang.multimodal_gen.configs.sample.sampling_params import SamplingParams + + seed = generate_config.pop("seed", None) + if seed is None or seed == -1: + # SamplingParams uses a fixed default seed; keep the default + # xinference behavior of generating a new image every call + seed = random.randint(0, 2**31 - 1) + params = dict(generate_config) + params.update( + prompt=prompt, + width=width, + height=height, + num_outputs_per_prompt=n, + seed=seed, + save_output=False, + return_frames=True, + ) + return _filter_kwargs_by_dataclass_fields( + params, SamplingParams, "sampling params" + ) + + @staticmethod + def _extract_images(result: Any) -> List[Any]: + import numpy as np + from PIL import Image + + if result is None: + raise RuntimeError( + "SGLang image generation failed, see server logs for details" + ) + results = result if isinstance(result, list) else [result] + images = [] + for res in results: + frames = getattr(res, "frames", None) + if frames is None: + continue + if not isinstance(frames, (list, tuple)): + frames = [frames] + for frame in frames: + if isinstance(frame, Image.Image): + images.append(frame) + elif isinstance(frame, np.ndarray): + if frame.ndim == 3 and frame.shape[-1] == 1: + frame = frame[..., 0] + images.append(Image.fromarray(frame)) + else: + raise RuntimeError( + f"Unexpected frame type from SGLang: {type(frame)}" + ) + if not images: + raise RuntimeError( + "SGLang image generation returned no images, " + "see server logs for details" + ) + return images + + async def text_to_image( + self, + prompt: str, + n: int = 1, + size: str = "1024*1024", + response_format: str = "url", + **kwargs, + ): + assert self._model is not None + width, height = map(int, re.split(r"[^\d]+", size)) + generate_config: Dict[str, Any] = ( + self._model_spec.default_generate_config or {} # type: ignore + ).copy() + generate_config.update({k: v for k, v in kwargs.items() if v is not None}) + sampling_params_kwargs = self._build_sampling_params( + prompt, n, width, height, generate_config + ) + result = await asyncio.to_thread( + self._model.generate, + sampling_params_kwargs=sampling_params_kwargs, + ) + images = self._extract_images(result) + return handle_image_result(response_format, images) diff --git a/xinference/model/image/tests/test_sglang_engine.py b/xinference/model/image/tests/test_sglang_engine.py new file mode 100644 index 0000000000..8218b44e31 --- /dev/null +++ b/xinference/model/image/tests/test_sglang_engine.py @@ -0,0 +1,182 @@ +# Copyright 2022-2026 XProbe Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import dataclasses +import sys +import types +from typing import Any, Optional +from unittest.mock import patch + +import numpy as np +import pytest + +from .. import BUILTIN_IMAGE_MODELS, _install +from ..engine import SGLangImageModel +from ..engine import platform as engine_platform +from ..sglang.core import SGLANG_SUPPORTED_IMAGE_MODELS, SGLangDiffusionModel + + +@pytest.fixture(scope="module", autouse=True) +def setup_builtin_models(): + _install() + + +def _get_spec(model_name: str): + return BUILTIN_IMAGE_MODELS[model_name][0] + + +def test_match_on_linux_with_cuda(): + with patch.object(engine_platform, "system", return_value="Linux"), patch.object( + sys.modules[SGLangImageModel.__module__], "has_cuda_device", return_value=True + ): + for model_name in SGLANG_SUPPORTED_IMAGE_MODELS: + assert SGLangImageModel.match(_get_spec(model_name)) is True + assert SGLangImageModel.match(_get_spec("sd3-medium")) is False + + +def test_match_rejects_non_linux(): + with patch.object(engine_platform, "system", return_value="Darwin"), patch.object( + sys.modules[SGLangImageModel.__module__], "has_cuda_device", return_value=True + ): + assert SGLangImageModel.match(_get_spec("Qwen-Image")) is False + + +def test_match_rejects_without_cuda(): + with patch.object(engine_platform, "system", return_value="Linux"), patch.object( + sys.modules[SGLangImageModel.__module__], "has_cuda_device", return_value=False + ): + assert SGLangImageModel.match(_get_spec("Qwen-Image")) is False + + +def test_constructor_rejects_unsupported_features(): + spec = _get_spec("Qwen-Image") + with pytest.raises(ValueError, match="GGUF"): + SGLangDiffusionModel( + "uid", "/path", model_spec=spec, gguf_model_path="/gguf/path" + ) + with pytest.raises(ValueError, match="Lightning"): + SGLangDiffusionModel( + "uid", "/path", model_spec=spec, lightning_model_path="/lightning/path" + ) + with pytest.raises(ValueError, match="LoRA"): + SGLangDiffusionModel("uid", "/path", model_spec=spec, lora_model=[object()]) + with pytest.raises(ValueError, match="Controlnet"): + SGLangDiffusionModel("uid", "/path", model_spec=spec, controlnet=("canny",)) + + +@pytest.fixture +def fake_sglang_sampling_params(): + @dataclasses.dataclass + class FakeSamplingParams: + prompt: Optional[str] = None + negative_prompt: Optional[str] = None + width: Optional[int] = None + height: Optional[int] = None + num_inference_steps: Optional[int] = None + guidance_scale: float = 1.0 + seed: int = 42 + num_outputs_per_prompt: int = 1 + save_output: bool = True + return_frames: bool = False + + mod = types.ModuleType("sglang.multimodal_gen.configs.sample.sampling_params") + mod.SamplingParams = FakeSamplingParams + fake_modules = { + "sglang": types.ModuleType("sglang"), + "sglang.multimodal_gen": types.ModuleType("sglang.multimodal_gen"), + "sglang.multimodal_gen.configs": types.ModuleType( + "sglang.multimodal_gen.configs" + ), + "sglang.multimodal_gen.configs.sample": types.ModuleType( + "sglang.multimodal_gen.configs.sample" + ), + "sglang.multimodal_gen.configs.sample.sampling_params": mod, + } + to_restore = {name: sys.modules.get(name) for name in fake_modules} + sys.modules.update(fake_modules) + try: + yield FakeSamplingParams + finally: + for name, orig in to_restore.items(): + if orig is None: + sys.modules.pop(name, None) + else: + sys.modules[name] = orig + + +def test_build_sampling_params(fake_sglang_sampling_params): + spec = _get_spec("Qwen-Image") + model = SGLangDiffusionModel("uid", "/path", model_spec=spec) + + params = model._build_sampling_params( + "a cat", + 2, + 512, + 768, + { + "guidance_scale": 1, + "true_cfg_scale": 1, # diffusers-only, should be dropped + "num_inference_steps": 20, + "seed": 123, + }, + ) + assert params["prompt"] == "a cat" + assert params["width"] == 512 + assert params["height"] == 768 + assert params["num_outputs_per_prompt"] == 2 + assert params["seed"] == 123 + assert params["num_inference_steps"] == 20 + assert params["save_output"] is False + assert params["return_frames"] is True + assert "true_cfg_scale" not in params + + +@pytest.mark.parametrize("seed", [None, -1]) +def test_build_sampling_params_random_seed(fake_sglang_sampling_params, seed): + spec = _get_spec("Qwen-Image") + model = SGLangDiffusionModel("uid", "/path", model_spec=spec) + params = model._build_sampling_params("a cat", 1, 512, 512, {"seed": seed}) + assert isinstance(params["seed"], int) + assert params["seed"] >= 0 + + +def test_extract_images(): + @dataclasses.dataclass + class FakeResult: + frames: Any = None + + frame = np.zeros((8, 8, 3), dtype=np.uint8) + images = SGLangDiffusionModel._extract_images(FakeResult(frames=[frame])) + assert len(images) == 1 + assert images[0].size == (8, 8) + + images = SGLangDiffusionModel._extract_images( + [FakeResult(frames=[frame]), FakeResult(frames=[frame])] + ) + assert len(images) == 2 + + with pytest.raises(RuntimeError, match="failed"): + SGLangDiffusionModel._extract_images(None) + + with pytest.raises(RuntimeError, match="no images"): + SGLangDiffusionModel._extract_images(FakeResult(frames=[])) + + +def test_builtin_specs_have_sglang_virtualenv_marker(): + for model_name in SGLANG_SUPPORTED_IMAGE_MODELS: + for spec in BUILTIN_IMAGE_MODELS[model_name]: + packages = spec.virtualenv.packages if spec.virtualenv else [] + assert any( + "sglang" in pkg and '#engine# == "SGLang"' in pkg for pkg in packages + ), f"{model_name} misses sglang virtualenv marker" diff --git a/xinference/model/utils.py b/xinference/model/utils.py index 26da505144..5aff2d42ae 100644 --- a/xinference/model/utils.py +++ b/xinference/model/utils.py @@ -407,6 +407,25 @@ def check_dependency_available( return True +@functools.lru_cache +def has_cuda_device() -> bool: + # use pynvml rather than torch to avoid initializing CUDA on import + device_count = 0 + try: + from pynvml import nvmlDeviceGetCount, nvmlInit, nvmlShutdown + + nvmlInit() + device_count = nvmlDeviceGetCount() + except Exception: + pass + finally: + try: + nvmlShutdown() + except Exception: + pass + return device_count > 0 + + def is_locale_chinese_simplified() -> bool: import locale From c3a40b9a6c8c5138ffea4a5e32881d1d784a6682 Mon Sep 17 00:00:00 2001 From: OliverBryant <2713999266@qq.com> Date: Thu, 2 Jul 2026 18:06:59 +0800 Subject: [PATCH 2/3] FEAT: [audio] engine abstraction with vLLM engine for Qwen3-ASR Add an engine abstraction layer for audio models (mirroring the image module design): AUDIO_ENGINES registry, engine matching and virtualenv marker support. Families without registered engines keep the legacy hardcoded dispatch, so existing behavior is unchanged. Qwen3-ASR-0.6B/1.7B register two engines: the default transformers engine and, on Linux with NVIDIA GPUs, a vLLM engine backed by qwen_asr's Qwen3ASRModel.LLM which shares the same transcribe API. /v1/engines/audio/ now returns engine params for audio models. Ref #4896 Co-Authored-By: Claude Fable 5 --- doc/source/models/model_abilities/audio.rst | 14 ++ xinference/model/audio/__init__.py | 8 + xinference/model/audio/core.py | 45 +++++- xinference/model/audio/custom.py | 7 + xinference/model/audio/engine.py | 50 ++++++ xinference/model/audio/engine_family.py | 130 +++++++++++++++ xinference/model/audio/model_spec.json | 2 + .../model/audio/tests/test_audio_engine.py | 148 ++++++++++++++++++ xinference/model/audio/vllm.py | 78 +++++++++ xinference/model/core.py | 1 + xinference/model/utils.py | 66 +++++++- 11 files changed, 547 insertions(+), 2 deletions(-) create mode 100644 xinference/model/audio/engine.py create mode 100644 xinference/model/audio/engine_family.py create mode 100644 xinference/model/audio/tests/test_audio_engine.py create mode 100644 xinference/model/audio/vllm.py diff --git a/doc/source/models/model_abilities/audio.rst b/doc/source/models/model_abilities/audio.rst index 92aa9470cd..676995eb03 100644 --- a/doc/source/models/model_abilities/audio.rst +++ b/doc/source/models/model_abilities/audio.rst @@ -69,6 +69,20 @@ For Mac M-series chips only: * :ref:`whisper-large-v3-mlx ` * :ref:`whisper-large-v3-turbo-mlx ` +Audio engines +~~~~~~~~~~~~~ + +``Qwen3-ASR-0.6B`` and ``Qwen3-ASR-1.7B`` support engine selection. The default +``transformers`` engine works on all platforms; on Linux with NVIDIA GPUs they +can also run on the ``vLLM`` engine for faster transcriptions. To use it, +install the vLLM backend of `qwen-asr `_ via +``pip install 'qwen-asr[vllm]'``, then launch the model with +``--model-engine vLLM``, for example: + +.. code-block:: bash + + xinference launch --model-name Qwen3-ASR-1.7B --model-type audio --model-engine vLLM + Text to audio (TTS) ~~~~~~~~~~~~~~~~~~~~~ diff --git a/xinference/model/audio/__init__.py b/xinference/model/audio/__init__.py index 8cf675e75f..751849f730 100644 --- a/xinference/model/audio/__init__.py +++ b/xinference/model/audio/__init__.py @@ -34,6 +34,8 @@ register_audio, unregister_audio, ) +from .engine import register_builtin_audio_engines +from .engine_family import generate_engine_config_by_model_name BUILTIN_AUDIO_MODELS: Dict[str, List["AudioModelFamilyV2"]] = {} @@ -87,11 +89,17 @@ def _install(): if model_spec.model_name not in AUDIO_MODEL_DESCRIPTIONS: AUDIO_MODEL_DESCRIPTIONS.update(generate_audio_description(model_spec)) + register_builtin_audio_engines() + for model_specs in BUILTIN_AUDIO_MODELS.values(): + for model_spec in model_specs: + generate_engine_config_by_model_name(model_spec) + register_custom_model() # register model description for ud_audio in get_user_defined_audios(): AUDIO_MODEL_DESCRIPTIONS.update(generate_audio_description(ud_audio)) + generate_engine_config_by_model_name(ud_audio) def register_builtin_model(): diff --git a/xinference/model/audio/core.py b/xinference/model/audio/core.py index 777efced95..ad7624a836 100644 --- a/xinference/model/audio/core.py +++ b/xinference/model/audio/core.py @@ -19,6 +19,7 @@ from ..utils import ModelInstanceInfoMixin from .chattts import ChatTTSModel from .cosyvoice import CosyVoiceModel +from .engine_family import AudioEngineModel from .f5tts import F5TTSModel from .f5tts_mlx import F5TTSMLXModel from .fish_speech import FishSpeechModel @@ -142,6 +143,7 @@ def create_audio_model_instance( Literal["huggingface", "modelscope", "openmind_hub", "csghub"] ] = None, model_path: Optional[str] = None, + model_engine: Optional[str] = None, **kwargs, ) -> Union[ WhisperModel, @@ -161,14 +163,55 @@ def create_audio_model_instance( Qwen3ASRModel, Qwen3TTSModel, VoxCPMModel, + AudioEngineModel, ]: from ..cache_manager import CacheManager + from .engine_family import AUDIO_ENGINES - kwargs.pop("enable_virtual_env", None) + enable_virtual_env = kwargs.pop("enable_virtual_env", None) model_spec = match_audio(model_name, download_hub) if model_path is None: cache_manager = CacheManager(model_spec) model_path = cache_manager.cache() + + # Engine-aware dispatch for model families with multiple engines + # (e.g. qwen3_asr on transformers or vLLM). Families without registered + # engines keep the legacy dispatch below. + if model_spec.model_name in AUDIO_ENGINES or model_engine is not None: + from .engine_family import ( + check_engine_by_model_name_and_engine, + check_engine_by_model_name_and_engine_with_virtual_env, + ) + + if model_engine is None: + # the first registered engine is the default one + model_engine = next(iter(AUDIO_ENGINES[model_spec.model_name])) + + if model_spec.model_name not in AUDIO_ENGINES: + logger.warning( + "Audio model %s does not support engine selection, " + "`model_engine=%s` is ignored.", + model_spec.model_name, + model_engine, + ) + else: + if enable_virtual_env is None: + from ...constants import XINFERENCE_ENABLE_VIRTUAL_ENV + + enable_virtual_env = XINFERENCE_ENABLE_VIRTUAL_ENV + if enable_virtual_env: + audio_cls = check_engine_by_model_name_and_engine_with_virtual_env( + model_engine, + model_spec.model_name, + model_family=model_spec, + ) + else: + audio_cls = check_engine_by_model_name_and_engine( + model_engine, + model_spec.model_name, + ) + return audio_cls(model_uid, model_path, model_spec, **kwargs) # type: ignore + model: Union[ WhisperModel, WhisperMLXModel, diff --git a/xinference/model/audio/custom.py b/xinference/model/audio/custom.py index 9c1dea3c6f..7a92f55898 100644 --- a/xinference/model/audio/custom.py +++ b/xinference/model/audio/custom.py @@ -95,6 +95,9 @@ def register_audio(model_spec: CustomAudioModelFamilyV2, persist: bool): registry = RegistryManager.get_registry("audio") registry.register(model_spec, persist) + from .engine_family import generate_engine_config_by_model_name + + generate_engine_config_by_model_name(model_spec) def unregister_audio(model_name: str, raise_error: bool = True): @@ -102,3 +105,7 @@ def unregister_audio(model_name: str, raise_error: bool = True): registry = RegistryManager.get_registry("audio") registry.unregister(model_name, raise_error) + from .engine_family import AUDIO_ENGINES + + if model_name in AUDIO_ENGINES: + del AUDIO_ENGINES[model_name] diff --git a/xinference/model/audio/engine.py b/xinference/model/audio/engine.py new file mode 100644 index 0000000000..3e56dfffb2 --- /dev/null +++ b/xinference/model/audio/engine.py @@ -0,0 +1,50 @@ +# Copyright 2022-2026 XProbe Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import platform +from typing import TYPE_CHECKING + +from ..utils import has_cuda_device +from .engine_family import SUPPORTED_ENGINES, AudioEngineModel +from .qwen3_asr import Qwen3ASRModel +from .vllm import VLLMQwen3ASRModel + +if TYPE_CHECKING: + from .core import AudioModelFamilyV2 + + +class TransformersQwen3ASRAudioModel(Qwen3ASRModel, AudioEngineModel): + required_libs = ("qwen_asr",) + + @classmethod + def match(cls, model_family: "AudioModelFamilyV2") -> bool: + return model_family.model_family == "qwen3_asr" + + +class VLLMQwen3ASRAudioModel(VLLMQwen3ASRModel, AudioEngineModel): + required_libs = ("qwen_asr", "vllm") + + @classmethod + def match(cls, model_family: "AudioModelFamilyV2") -> bool: + if platform.system() != "Linux": + return False + if not has_cuda_device(): + return False + return model_family.model_family == "qwen3_asr" + + +def register_builtin_audio_engines() -> None: + # the first registered engine is the default one for a model + SUPPORTED_ENGINES["transformers"] = [TransformersQwen3ASRAudioModel] + SUPPORTED_ENGINES["vLLM"] = [VLLMQwen3ASRAudioModel] diff --git a/xinference/model/audio/engine_family.py b/xinference/model/audio/engine_family.py new file mode 100644 index 0000000000..040ed4b7f5 --- /dev/null +++ b/xinference/model/audio/engine_family.py @@ -0,0 +1,130 @@ +# Copyright 2022-2026 XProbe Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import importlib.util +import logging +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Type, Union + +if TYPE_CHECKING: + from .core import AudioModelFamilyV2 + +logger = logging.getLogger(__name__) + + +class AudioEngineModel: + required_libs: Tuple[str, ...] = () + + @classmethod + def match(cls, model_family: "AudioModelFamilyV2") -> bool: + raise NotImplementedError + + @classmethod + def check_lib(cls) -> Union[bool, Tuple[bool, str]]: + for lib in cls.required_libs: + if importlib.util.find_spec(lib) is None: + return False, f"Library '{lib}' is not installed" + return True + + +# { audio model name -> { engine name -> engine params } } +AUDIO_ENGINES: Dict[str, Dict[str, List[Dict[str, Any]]]] = {} +SUPPORTED_ENGINES: Dict[str, List[Type[AudioEngineModel]]] = {} + + +def check_engine_by_model_name_and_engine( + model_engine: str, + model_name: str, +) -> Type[AudioEngineModel]: + def get_model_engine_from_spell(engine_str: str) -> str: + for engine in AUDIO_ENGINES[model_name].keys(): + if engine.lower() == engine_str.lower(): + return engine + return engine_str + + if model_name not in AUDIO_ENGINES: + raise ValueError(f"Audio model {model_name} not found.") + model_engine = get_model_engine_from_spell(model_engine) + if model_engine not in AUDIO_ENGINES[model_name]: + raise ValueError( + f"Audio model {model_name} cannot be run on engine {model_engine}." + ) + match_params = AUDIO_ENGINES[model_name][model_engine] + for param in match_params: + if model_name == param["model_name"]: + return param["audio_class"] + raise ValueError( + f"Audio model {model_name} cannot be run on engine {model_engine}." + ) + + +def check_engine_by_model_name_and_engine_with_virtual_env( + model_engine: str, + model_name: str, + model_family: Optional["AudioModelFamilyV2"] = None, +) -> Type[AudioEngineModel]: + from ..utils import _collect_virtualenv_engine_markers + + if model_family is None: + raise ValueError(f"Audio model {model_name} not found.") + + engine_markers = _collect_virtualenv_engine_markers(model_family) + + def _engine_class_by_marker() -> Optional[Type[AudioEngineModel]]: + if model_engine.lower() in engine_markers: + for engine, engine_classes in SUPPORTED_ENGINES.items(): + if engine.lower() == model_engine.lower() and engine_classes: + logger.warning( + "Bypassing engine compatibility checks for %s due to " + "virtualenv marker.", + model_engine, + ) + return engine_classes[0] + return None + + if model_name not in AUDIO_ENGINES: + engine_cls = _engine_class_by_marker() + if engine_cls is not None: + return engine_cls + raise ValueError(f"Audio model {model_name} not found.") + + try: + return check_engine_by_model_name_and_engine(model_engine, model_name) + except ValueError: + engine_cls = _engine_class_by_marker() + if engine_cls is not None: + return engine_cls + raise + + +def generate_engine_config_by_model_name(model_family: "AudioModelFamilyV2") -> None: + model_name = model_family.model_name + engines: Dict[str, List[Dict[str, Any]]] = AUDIO_ENGINES.get(model_name, {}) + for engine, classes in SUPPORTED_ENGINES.items(): + for cls in classes: + if cls.match(model_family): + engine_params = engines.get(engine, []) + already_exists = any( + param["model_name"] == model_name for param in engine_params + ) + if not already_exists: + engine_params.append( + { + "model_name": model_name, + "audio_class": cls, + } + ) + engines[engine] = engine_params + break + if engines: + AUDIO_ENGINES[model_name] = engines diff --git a/xinference/model/audio/model_spec.json b/xinference/model/audio/model_spec.json index 004f171d20..7d72b0b518 100644 --- a/xinference/model/audio/model_spec.json +++ b/xinference/model/audio/model_spec.json @@ -1170,6 +1170,7 @@ "virtualenv": { "packages": [ "qwen-asr", + "qwen-asr[vllm] ; #engine# == \"vLLM\"", "#system_torch#", "#system_numpy#", "#system_pandas#" @@ -1204,6 +1205,7 @@ "virtualenv": { "packages": [ "qwen-asr", + "qwen-asr[vllm] ; #engine# == \"vLLM\"", "#system_torch#", "#system_numpy#", "#system_pandas#" diff --git a/xinference/model/audio/tests/test_audio_engine.py b/xinference/model/audio/tests/test_audio_engine.py new file mode 100644 index 0000000000..6369519672 --- /dev/null +++ b/xinference/model/audio/tests/test_audio_engine.py @@ -0,0 +1,148 @@ +# Copyright 2022-2026 XProbe Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from unittest.mock import patch + +import pytest + +from .. import BUILTIN_AUDIO_MODELS, _install +from ..core import create_audio_model_instance +from ..engine import TransformersQwen3ASRAudioModel, VLLMQwen3ASRAudioModel +from ..engine import platform as engine_platform +from ..engine import register_builtin_audio_engines +from ..engine_family import ( + AUDIO_ENGINES, + check_engine_by_model_name_and_engine, + generate_engine_config_by_model_name, +) +from ..funasr import FunASRModel +from ..whisper import WhisperModel + + +@pytest.fixture(scope="module", autouse=True) +def setup_builtin_models(): + _install() + + +def _get_spec(model_name: str): + return BUILTIN_AUDIO_MODELS[model_name][0] + + +def _register_all_engines(): + register_builtin_audio_engines() + for model_specs in BUILTIN_AUDIO_MODELS.values(): + for model_spec in model_specs: + generate_engine_config_by_model_name(model_spec) + + +@pytest.fixture +def linux_cuda_engines(): + engine_mod = __import__( + VLLMQwen3ASRAudioModel.__module__, fromlist=["has_cuda_device"] + ) + old_engines = {k: dict(v) for k, v in AUDIO_ENGINES.items()} + with patch.object(engine_platform, "system", return_value="Linux"), patch.object( + engine_mod, "has_cuda_device", return_value=True + ): + AUDIO_ENGINES.clear() + _register_all_engines() + yield + AUDIO_ENGINES.clear() + AUDIO_ENGINES.update(old_engines) + + +def test_qwen3_asr_registers_transformers_engine(): + assert "Qwen3-ASR-0.6B" in AUDIO_ENGINES + assert "transformers" in AUDIO_ENGINES["Qwen3-ASR-0.6B"] + # default engine is the first registered one + assert next(iter(AUDIO_ENGINES["Qwen3-ASR-0.6B"])) == "transformers" + + +def test_non_engine_families_not_registered(): + assert "whisper-large-v3" not in AUDIO_ENGINES + + +def test_qwen3_asr_vllm_engine_on_linux_cuda(linux_cuda_engines): + for model_name in ("Qwen3-ASR-0.6B", "Qwen3-ASR-1.7B"): + assert sorted(AUDIO_ENGINES[model_name]) == ["transformers", "vLLM"] + cls = check_engine_by_model_name_and_engine("vLLM", model_name) + assert cls is VLLMQwen3ASRAudioModel + # engine name is case-insensitive + cls = check_engine_by_model_name_and_engine("vllm", model_name) + assert cls is VLLMQwen3ASRAudioModel + + +def test_vllm_engine_not_matched_without_cuda(): + engine_mod = __import__( + VLLMQwen3ASRAudioModel.__module__, fromlist=["has_cuda_device"] + ) + with patch.object(engine_platform, "system", return_value="Linux"), patch.object( + engine_mod, "has_cuda_device", return_value=False + ): + assert VLLMQwen3ASRAudioModel.match(_get_spec("Qwen3-ASR-0.6B")) is False + with patch.object(engine_platform, "system", return_value="Darwin"), patch.object( + engine_mod, "has_cuda_device", return_value=True + ): + assert VLLMQwen3ASRAudioModel.match(_get_spec("Qwen3-ASR-0.6B")) is False + + +def test_create_audio_model_instance_default_engine(): + model = create_audio_model_instance( + "uid", + "Qwen3-ASR-0.6B", + model_path="/fake/path", + enable_virtual_env=False, + ) + assert isinstance(model, TransformersQwen3ASRAudioModel) + + +def test_create_audio_model_instance_vllm_engine(linux_cuda_engines): + model = create_audio_model_instance( + "uid", + "Qwen3-ASR-0.6B", + model_path="/fake/path", + model_engine="vLLM", + enable_virtual_env=False, + ) + assert isinstance(model, VLLMQwen3ASRAudioModel) + + +def test_create_audio_model_instance_legacy_dispatch(): + model = create_audio_model_instance( + "uid", + "whisper-large-v3", + model_path="/fake/path", + enable_virtual_env=False, + ) + assert isinstance(model, WhisperModel) + + # model_engine on a legacy family is ignored with a warning + model = create_audio_model_instance( + "uid", + "SenseVoiceSmall", + model_path="/fake/path", + model_engine="transformers", + enable_virtual_env=False, + ) + assert isinstance(model, FunASRModel) + + +def test_builtin_specs_have_vllm_virtualenv_marker(): + for model_name in ("Qwen3-ASR-0.6B", "Qwen3-ASR-1.7B"): + for spec in BUILTIN_AUDIO_MODELS[model_name]: + packages = spec.virtualenv.packages if spec.virtualenv else [] + assert any( + "qwen-asr[vllm]" in pkg and '#engine# == "vLLM"' in pkg + for pkg in packages + ), f"{model_name} misses qwen-asr[vllm] virtualenv marker" diff --git a/xinference/model/audio/vllm.py b/xinference/model/audio/vllm.py new file mode 100644 index 0000000000..c5db8eb8a1 --- /dev/null +++ b/xinference/model/audio/vllm.py @@ -0,0 +1,78 @@ +# Copyright 2022-2026 XProbe Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import os + +from ...device_utils import get_available_device, get_device_preferred_dtype +from .qwen3_asr import Qwen3ASRModel + +logger = logging.getLogger(__name__) + + +class VLLMQwen3ASRModel(Qwen3ASRModel): + """Qwen3-ASR served by the vLLM backend of the ``qwen_asr`` package. + + ``qwen_asr`` exposes the same ``transcribe`` API for both its + transformers backend (``Qwen3ASRModel.from_pretrained``) and its vLLM + backend (``Qwen3ASRModel.LLM``), so only model loading differs here. + """ + + def load(self): + try: + from qwen_asr import Qwen3ASRModel as QwenASR + + QwenASR.LLM + except ImportError: + error_message = ( + "Failed to import module 'qwen_asr' with vLLM backend support" + ) + installation_guide = [ + "Please make sure 'qwen-asr' with vLLM support is installed. ", + "You can install it by `pip install 'qwen-asr[vllm]'`\n", + ] + raise ImportError(f"{error_message}\n\n{''.join(installation_guide)}") + except AttributeError: + raise RuntimeError( + "The installed 'qwen-asr' does not support the vLLM backend, " + "please upgrade it by `pip install -U 'qwen-asr[vllm]'`" + ) + + init_kwargs = ( + self._model_spec.default_model_config.copy() + if getattr(self._model_spec, "default_model_config", None) + else {} + ) + init_kwargs.update(self._kwargs) + # transformers-only args that must not reach vllm.LLM + for key in ("device_map", "dtype", "torch_dtype"): + init_kwargs.pop(key, None) + + forced_aligner = init_kwargs.get("forced_aligner") + if isinstance(forced_aligner, str) and not os.path.exists(forced_aligner): + resolved = self._resolve_forced_aligner(forced_aligner) + if resolved: + init_kwargs["forced_aligner"] = resolved + if "forced_aligner" in init_kwargs: + # the forced aligner still runs on transformers + device = self._device or get_available_device() + forced_aligner_kwargs = init_kwargs.get("forced_aligner_kwargs") or {} + forced_aligner_kwargs.setdefault("device_map", device) + forced_aligner_kwargs.setdefault( + "dtype", get_device_preferred_dtype(device) + ) + init_kwargs["forced_aligner_kwargs"] = forced_aligner_kwargs + + logger.debug("Loading Qwen3-ASR model with vLLM, kwargs: %s", init_kwargs) + self._model = QwenASR.LLM(model=self._model_path, **init_kwargs) diff --git a/xinference/model/core.py b/xinference/model/core.py index 43d1a60741..bf3faaba59 100644 --- a/xinference/model/core.py +++ b/xinference/model/core.py @@ -103,6 +103,7 @@ def create_model_instance( model_name, download_hub, model_path, + model_engine=model_engine, **kwargs, ) elif model_type == "video": diff --git a/xinference/model/utils.py b/xinference/model/utils.py index 5aff2d42ae..34426f5c37 100644 --- a/xinference/model/utils.py +++ b/xinference/model/utils.py @@ -1233,6 +1233,32 @@ def _get_image_families(model_name: str, is_ocr: bool) -> List[Any]: ) return engine_params + if model_type == "audio": + from .audio import BUILTIN_AUDIO_MODELS + from .audio.custom import get_user_defined_audios + from .audio.engine_family import AUDIO_ENGINES + from .audio.engine_family import SUPPORTED_ENGINES as AUDIO_SUPPORTED_ENGINES + + if model_name not in AUDIO_ENGINES: + return None + + available_engines = deepcopy(AUDIO_ENGINES[model_name]) + for engine, params in available_engines.items(): + _append_available_engine(engine, params, "audio_class") + audio_families: List[Any] = list(BUILTIN_AUDIO_MODELS.get(model_name, [])) + audio_families.extend( + f for f in get_user_defined_audios() if f.model_name == model_name + ) + _validate_available_image_engines( + audio_families, + AUDIO_SUPPORTED_ENGINES, + "audio", + ) + _collect_supported_image_engines( + audio_families, AUDIO_SUPPORTED_ENGINES, "audio" + ) + return engine_params + return None @@ -1716,9 +1742,47 @@ def _get_image_families(model_name: str, is_ocr: bool) -> List[Any]: return engine_params + elif model_type == "audio": + from .audio import BUILTIN_AUDIO_MODELS + from .audio.custom import get_user_defined_audios + from .audio.engine_family import AUDIO_ENGINES + from .audio.engine_family import SUPPORTED_ENGINES as AUDIO_SUPPORTED_ENGINES + + if model_name not in AUDIO_ENGINES: + return None + + available_engines = deepcopy(AUDIO_ENGINES[model_name]) + for engine, params in available_engines.items(): + _append_available_engine(engine, params, "audio_class") + audio_families: List[Any] = list(BUILTIN_AUDIO_MODELS.get(model_name, [])) + audio_families.extend( + f for f in get_user_defined_audios() if f.model_name == model_name + ) + audio_engine_markers: Set[str] = set() + for family in audio_families: + audio_engine_markers |= _collect_virtualenv_engine_markers(family) + _validate_available_image_engines( + audio_families, + AUDIO_SUPPORTED_ENGINES, + "audio", + audio_engine_markers, + enable_virtual_env, + ) + _collect_supported_image_engines( + audio_families, AUDIO_SUPPORTED_ENGINES, "audio" + ) + _apply_virtualenv_engine_overrides( + engine_params, + AUDIO_SUPPORTED_ENGINES, + audio_engine_markers, + enable_virtual_env, + ) + + return engine_params + raise ValueError( "Cannot support model_engine for " - f"{model_type}, only available for LLM, embedding, rerank, image" + f"{model_type}, only available for LLM, embedding, rerank, image, audio" ) From 7e346eefec6bc7973a9ce06fb4e02b25df13b76a Mon Sep 17 00:00:00 2001 From: OliverBryant <2713999266@qq.com> Date: Thu, 2 Jul 2026 18:11:02 +0800 Subject: [PATCH 3/3] FEAT: [webui] engine selector for audio models Show the engine dropdown in the launch drawer for audio models that support engine selection (e.g. Qwen3-ASR on transformers/vLLM). Audio models without registered engines keep the previous form, and launch history entries without a model_engine still apply for them. Ref #4896 Co-Authored-By: Claude Fable 5 --- .../launch_model/components/launchModelDrawer.js | 14 ++++++++++---- .../launch_model/components/modelFormConfig.js | 8 ++++++++ 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/xinference/ui/web/ui/src/scenes/launch_model/components/launchModelDrawer.js b/xinference/ui/web/ui/src/scenes/launch_model/components/launchModelDrawer.js index 8b1cbc3dfb..6b11b625de 100644 --- a/xinference/ui/web/ui/src/scenes/launch_model/components/launchModelDrawer.js +++ b/xinference/ui/web/ui/src/scenes/launch_model/components/launchModelDrawer.js @@ -44,7 +44,7 @@ import Progress from './progress' import SelectField from './selectField' const enginesWithNWorker = ['SGLang', 'vLLM', 'MLX'] -const modelEngineType = ['LLM', 'embedding', 'rerank', 'image'] +const modelEngineType = ['LLM', 'embedding', 'rerank', 'image', 'audio'] const historyStorageKey = 'historyArr' const defaultHistoryUID = '__default_model_uid__' @@ -485,7 +485,9 @@ const LaunchModelDrawer = ({ } const isHistoryEngineValid = (historyData) => { - if (!historyData?.model_engine) return false + // models without engine selection (e.g. most audio models) have no + // model_engine in history entries + if (!historyData?.model_engine) return !engineOptions.length const engineData = enginesObj[historyData.model_engine] return ( @@ -888,8 +890,12 @@ const LaunchModelDrawer = ({ if (!open || !hasFetchedEngines) return if (!pendingHistory.model_engine) { - setFormData({}) - setCollapseState({}) + if (engineOptions.length) { + setFormData({}) + setCollapseState({}) + } else { + applyHistory(pendingHistory) + } setPendingHistory(null) return } diff --git a/xinference/ui/web/ui/src/scenes/launch_model/components/modelFormConfig.js b/xinference/ui/web/ui/src/scenes/launch_model/components/modelFormConfig.js index 7263b75a5b..8b0cd6ef6e 100644 --- a/xinference/ui/web/ui/src/scenes/launch_model/components/modelFormConfig.js +++ b/xinference/ui/web/ui/src/scenes/launch_model/components/modelFormConfig.js @@ -849,6 +849,14 @@ export default function getModelFormConfig({ type: 'input', visible: true, }, + { + name: 'model_engine', + label: t('launchModel.modelEngine'), + type: 'select', + options: engineItems, + visible: !!engineItems?.length, + required: false, + }, { name: 'replica', label: t('launchModel.replica'),