From b3a3ed04e2491b17b40d60580557efe4308348e1 Mon Sep 17 00:00:00 2001 From: Brian Dilinila Date: Wed, 25 Feb 2026 20:51:48 -0800 Subject: [PATCH 1/4] Add renderer_type from Hydra arg --- .../overview/core-concepts/sensors/camera.rst | 5 +++++ .../isaaclab/sensors/camera/camera_cfg.py | 9 +++++++++ .../isaaclab/sensors/camera/tiled_camera.py | 17 ++++++++++++++++- 3 files changed, 30 insertions(+), 1 deletion(-) diff --git a/docs/source/overview/core-concepts/sensors/camera.rst b/docs/source/overview/core-concepts/sensors/camera.rst index 6a34c6fab30c..a2021cfdc204 100644 --- a/docs/source/overview/core-concepts/sensors/camera.rst +++ b/docs/source/overview/core-concepts/sensors/camera.rst @@ -23,6 +23,11 @@ The Tiled Rendering APIs provide a vectorized interface for collecting data from Isaac Lab provides tiled rendering APIs for RGB, depth, along with other annotators through the :class:`~sensors.TiledCamera` class. Configurations for the tiled rendering APIs can be defined through the :class:`~sensors.TiledCameraCfg` class, specifying parameters such as the regex expression for all camera paths, the transform for the cameras, the desired data type, the type of cameras to add to the scene, and the camera resolution. +The renderer backend (Isaac RTX vs. Newton Warp) can be selected at run time via the config's ``renderer_type`` +(``"rtx"`` or ``"warp_renderer"``). When using Hydra (e.g. in ``train.py``), override the camera config path your +task uses—e.g. ``env.scene.base_camera.renderer_type=rtx`` when the scene exposes ``base_camera``, or +``env.tiled_camera.renderer_type=rtx`` when the camera is on the env config. See **Hydra Configuration System** (Features) for override paths and examples. + .. code-block:: python tiled_camera: TiledCameraCfg = TiledCameraCfg( diff --git a/source/isaaclab/isaaclab/sensors/camera/camera_cfg.py b/source/isaaclab/isaaclab/sensors/camera/camera_cfg.py index 49e1c5a8dddd..03b297364560 100644 --- a/source/isaaclab/isaaclab/sensors/camera/camera_cfg.py +++ b/source/isaaclab/isaaclab/sensors/camera/camera_cfg.py @@ -147,5 +147,14 @@ class OffsetCfg: """ + renderer_type: Literal["rtx", "warp_renderer"] = "rtx" + """Hydra-friendly renderer selector. When set, overrides :attr:`renderer_cfg` for the backend. + + - ``"rtx"``: Use Isaac RTX (Replicator) renderer (default). + - ``"warp_renderer"``: Use Newton Warp renderer (when available). + + Can be overridden at train time via e.g. ``env.scene.base_camera.renderer_type=warp_renderer``. + """ + renderer_cfg: RendererCfg = field(default_factory=IsaacRtxRendererCfg) """Renderer configuration for camera sensor.""" diff --git a/source/isaaclab/isaaclab/sensors/camera/tiled_camera.py b/source/isaaclab/isaaclab/sensors/camera/tiled_camera.py index b07b5d20b648..a08b9322a069 100644 --- a/source/isaaclab/isaaclab/sensors/camera/tiled_camera.py +++ b/source/isaaclab/isaaclab/sensors/camera/tiled_camera.py @@ -185,7 +185,9 @@ def _initialize_impl(self): self._sensor_prims.append(UsdGeom.Camera(cam_prim)) # Create renderer after scene is ready (post-cloning) so world_count is correct - self.renderer = Renderer(self.cfg.renderer_cfg) + # Resolve Hydra-friendly renderer_type to concrete renderer_cfg if set + renderer_cfg = self._get_effective_renderer_cfg() + self.renderer = Renderer(renderer_cfg) logger.info("Using renderer: %s", type(self.renderer).__name__) self.render_data = self.renderer.create_render_data(self) @@ -222,6 +224,19 @@ def _update_buffers_impl(self, env_mask: wp.array): Private Helpers """ + def _get_effective_renderer_cfg(self): + """Resolve renderer_cfg from optional renderer_type (Hydra override).""" + rt = getattr(self.cfg, "renderer_type", None) + if rt == "rtx": + from isaaclab_physx.renderers import IsaacRtxRendererCfg + + return IsaacRtxRendererCfg() + if rt == "warp_renderer": + from isaaclab_newton.renderers import NewtonWarpRendererCfg + + return NewtonWarpRendererCfg() + return self.cfg.renderer_cfg + def _check_supported_data_types(self, cfg: TiledCameraCfg): """Checks if the data types are supported by the ray-caster camera.""" # check if there is any intersection in unsupported types From 5c407d68000ffd3ff5a949e93a70e23582f7752a Mon Sep 17 00:00:00 2001 From: Brian Dilinila Date: Wed, 25 Feb 2026 20:55:01 -0800 Subject: [PATCH 2/4] Hydra-based render selection --- docs/source/features/hydra.rst | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/docs/source/features/hydra.rst b/docs/source/features/hydra.rst index c728ecd0979e..92d97dda70c5 100644 --- a/docs/source/features/hydra.rst +++ b/docs/source/features/hydra.rst @@ -83,6 +83,37 @@ To set parameters to None, use the ``null`` keyword, which is a special keyword In the above example, we could also disable the ``joint_pos_rel`` observation by setting it to None with ``env.observations.policy.joint_pos_rel=null``. +TiledCamera renderer type +^^^^^^^^^^^^^^^^^^^^^^^^^ + +You can select the :class:`~isaaclab.sensors.TiledCamera` backend (RTX vs. Warp) at train time via Hydra by +overriding the camera config's ``renderer_type``. + +**Override path:** It must match where the task puts the TiledCamera in the env config: + +- **Scene has ``base_camera``:** If the task uses a scene config that exposes the camera as ``scene.base_camera`` + (e.g. a scene class like ``KukaAllegroSingleTiledCameraSceneCfg`` with a ``base_camera: TiledCameraCfg`` field), + use: + + .. code-block:: shell + + env.scene.base_camera.renderer_type=rtx + # or + env.scene.base_camera.renderer_type=warp_renderer + +- **Camera on env config:** If the task puts the camera elsewhere (e.g. ``env.tiled_camera`` on the env config), + override that path instead: + + .. code-block:: shell + + env.tiled_camera.renderer_type=rtx + # or + env.tiled_camera.renderer_type=warp_renderer + +**Values:** ``rtx`` selects the Isaac RTX (Replicator) renderer; ``warp_renderer`` selects the Newton Warp renderer +(when the ``isaaclab_newton`` package is available). No change to env or env_cfg code is required—only the +config hierarchy must expose the camera at the path you override. + Dictionaries ^^^^^^^^^^^^ Elements in dictionaries are handled as a parameters in the hierarchy. For example, in the Cartpole environment: From c6341fd448485214317e527f7e5652751280fb3e Mon Sep 17 00:00:00 2001 From: Brian Dilinila Date: Fri, 27 Feb 2026 10:28:20 -0800 Subject: [PATCH 3/4] Fix rebase --- .../isaaclab/sensors/camera/camera_cfg.py | 9 --- .../isaaclab/sensors/camera/tiled_camera.py | 19 +---- .../isaaclab_tasks/utils/hydra.py | 66 +++++++++++---- source/isaaclab_tasks/test/test_hydra.py | 81 ++++++++++++++----- 4 files changed, 111 insertions(+), 64 deletions(-) diff --git a/source/isaaclab/isaaclab/sensors/camera/camera_cfg.py b/source/isaaclab/isaaclab/sensors/camera/camera_cfg.py index 03b297364560..49e1c5a8dddd 100644 --- a/source/isaaclab/isaaclab/sensors/camera/camera_cfg.py +++ b/source/isaaclab/isaaclab/sensors/camera/camera_cfg.py @@ -147,14 +147,5 @@ class OffsetCfg: """ - renderer_type: Literal["rtx", "warp_renderer"] = "rtx" - """Hydra-friendly renderer selector. When set, overrides :attr:`renderer_cfg` for the backend. - - - ``"rtx"``: Use Isaac RTX (Replicator) renderer (default). - - ``"warp_renderer"``: Use Newton Warp renderer (when available). - - Can be overridden at train time via e.g. ``env.scene.base_camera.renderer_type=warp_renderer``. - """ - renderer_cfg: RendererCfg = field(default_factory=IsaacRtxRendererCfg) """Renderer configuration for camera sensor.""" diff --git a/source/isaaclab/isaaclab/sensors/camera/tiled_camera.py b/source/isaaclab/isaaclab/sensors/camera/tiled_camera.py index a08b9322a069..e872e7798b8e 100644 --- a/source/isaaclab/isaaclab/sensors/camera/tiled_camera.py +++ b/source/isaaclab/isaaclab/sensors/camera/tiled_camera.py @@ -11,7 +11,7 @@ from typing import TYPE_CHECKING, Any import torch -import warp as wp +import warp as wp from pxr import UsdGeom @@ -185,9 +185,7 @@ def _initialize_impl(self): self._sensor_prims.append(UsdGeom.Camera(cam_prim)) # Create renderer after scene is ready (post-cloning) so world_count is correct - # Resolve Hydra-friendly renderer_type to concrete renderer_cfg if set - renderer_cfg = self._get_effective_renderer_cfg() - self.renderer = Renderer(renderer_cfg) + self.renderer = Renderer(self.cfg.renderer_cfg) logger.info("Using renderer: %s", type(self.renderer).__name__) self.render_data = self.renderer.create_render_data(self) @@ -224,19 +222,6 @@ def _update_buffers_impl(self, env_mask: wp.array): Private Helpers """ - def _get_effective_renderer_cfg(self): - """Resolve renderer_cfg from optional renderer_type (Hydra override).""" - rt = getattr(self.cfg, "renderer_type", None) - if rt == "rtx": - from isaaclab_physx.renderers import IsaacRtxRendererCfg - - return IsaacRtxRendererCfg() - if rt == "warp_renderer": - from isaaclab_newton.renderers import NewtonWarpRendererCfg - - return NewtonWarpRendererCfg() - return self.cfg.renderer_cfg - def _check_supported_data_types(self, cfg: TiledCameraCfg): """Checks if the data types are supported by the ray-caster camera.""" # check if there is any intersection in unsupported types diff --git a/source/isaaclab_tasks/isaaclab_tasks/utils/hydra.py b/source/isaaclab_tasks/isaaclab_tasks/utils/hydra.py index 525b425917fa..485f85195c4a 100644 --- a/source/isaaclab_tasks/isaaclab_tasks/utils/hydra.py +++ b/source/isaaclab_tasks/isaaclab_tasks/utils/hydra.py @@ -20,6 +20,46 @@ from isaaclab.utils import replace_slices_with_strings, replace_strings_with_slices from isaaclab_tasks.utils.parse_cfg import load_cfg_from_registry +from isaaclab_tasks.utils.render_config_store import register_render_configs + + +def process_hydra_config( + hydra_cfg: DictConfig | dict, + env_cfg: ManagerBasedRLEnvCfg | DirectRLEnvCfg, + agent_cfg: dict | object, +) -> tuple[ManagerBasedRLEnvCfg | DirectRLEnvCfg, dict | object]: + """Process composed Hydra config and update env/agent configs in place. + + Shared by hydra_task_config and tests. Applies render config to cameras, + updates env/agent from dict, restores gymnasium spaces. + """ + if not isinstance(hydra_cfg, dict): + hydra_cfg = OmegaConf.to_container(hydra_cfg, resolve=True) + hydra_cfg = replace_strings_with_slices(hydra_cfg) + + if "render" in hydra_cfg and hydra_cfg["render"]: + renderer_dict = hydra_cfg["render"] + if isinstance(renderer_dict, dict): + env_dict = hydra_cfg.get("env", {}) + + def apply_to_cameras(d: dict) -> None: + for v in d.values(): + if isinstance(v, dict): + if "renderer_cfg" in v: + v["renderer_cfg"] = renderer_dict + apply_to_cameras(v) + + apply_to_cameras(env_dict) + + env_cfg.from_dict(hydra_cfg["env"]) + env_cfg = replace_strings_with_env_cfg_spaces(env_cfg) + + if isinstance(agent_cfg, dict) or agent_cfg is None: + agent_cfg = hydra_cfg["agent"] + else: + agent_cfg.from_dict(hydra_cfg["agent"]) + + return env_cfg, agent_cfg def register_task_to_hydra( @@ -51,10 +91,15 @@ def register_task_to_hydra( agent_cfg_dict = agent_cfg else: agent_cfg_dict = agent_cfg.to_dict() - cfg_dict = {"env": env_cfg_dict, "agent": agent_cfg_dict} + cfg_dict = { + "defaults": ["_self_", {"render": "isaac_rtx"}], + "env": env_cfg_dict, + "agent": agent_cfg_dict, + } # replace slices with strings because OmegaConf does not support slices cfg_dict = replace_slices_with_strings(cfg_dict) - # store the configuration to Hydra + # register render config presets and store the configuration to Hydra + register_render_configs() ConfigStore.instance().store(name=task_name, node=cfg_dict) return env_cfg, agent_cfg @@ -82,21 +127,7 @@ def wrapper(*args, **kwargs): # define the new Hydra main function @hydra.main(config_path=None, config_name=task_name.split(":")[-1], version_base="1.3") def hydra_main(hydra_env_cfg: DictConfig, env_cfg=env_cfg, agent_cfg=agent_cfg): - # convert to a native dictionary - hydra_env_cfg = OmegaConf.to_container(hydra_env_cfg, resolve=True) - # replace string with slices because OmegaConf does not support slices - hydra_env_cfg = replace_strings_with_slices(hydra_env_cfg) - # update the configs with the Hydra command line arguments - env_cfg.from_dict(hydra_env_cfg["env"]) - # replace strings that represent gymnasium spaces because OmegaConf does not support them. - # this must be done after converting the env configs from dictionary to avoid internal reinterpretations - env_cfg = replace_strings_with_env_cfg_spaces(env_cfg) - # get agent configs - if isinstance(agent_cfg, dict) or agent_cfg is None: - agent_cfg = hydra_env_cfg["agent"] - else: - agent_cfg.from_dict(hydra_env_cfg["agent"]) - # call the original function + env_cfg, agent_cfg = process_hydra_config(hydra_env_cfg, env_cfg, agent_cfg) func(env_cfg, agent_cfg, *args, **kwargs) # call the new Hydra main function @@ -105,3 +136,4 @@ def hydra_main(hydra_env_cfg: DictConfig, env_cfg=env_cfg, agent_cfg=agent_cfg): return wrapper return decorator + diff --git a/source/isaaclab_tasks/test/test_hydra.py b/source/isaaclab_tasks/test/test_hydra.py index 5c81cb3e650f..5d36edc99f1f 100644 --- a/source/isaaclab_tasks/test/test_hydra.py +++ b/source/isaaclab_tasks/test/test_hydra.py @@ -21,39 +21,23 @@ import hydra from hydra import compose, initialize -from omegaconf import OmegaConf - -from isaaclab.utils import replace_strings_with_slices +import pytest import isaaclab_tasks # noqa: F401 -from isaaclab_tasks.utils.hydra import register_task_to_hydra +from isaaclab_tasks.utils.hydra import process_hydra_config, register_task_to_hydra +from isaaclab_tasks.utils.render_config_store import NEWTON_WARP_AVAILABLE def hydra_task_config_test(task_name: str, agent_cfg_entry_point: str) -> Callable: - """Copied from hydra.py hydra_task_config, since hydra.main requires a single point of entry, - which will not work with multiple tests. Here, we replace hydra.main with hydra initialize - and compose.""" + """Mirrors hydra_task_config: register task, compose, process_hydra_config, then run test.""" def decorator(func): @functools.wraps(func) def wrapper(*args, **kwargs): - # register the task to Hydra env_cfg, agent_cfg = register_task_to_hydra(task_name, agent_cfg_entry_point) - - # replace hydra.main with initialize and compose with initialize(config_path=None, version_base="1.3"): hydra_env_cfg = compose(config_name=task_name, overrides=sys.argv[1:]) - # convert to a native dictionary - hydra_env_cfg = OmegaConf.to_container(hydra_env_cfg, resolve=True) - # replace string with slices because OmegaConf does not support slices - hydra_env_cfg = replace_strings_with_slices(hydra_env_cfg) - # update the configs with the Hydra command line arguments - env_cfg.from_dict(hydra_env_cfg["env"]) - if isinstance(agent_cfg, dict): - agent_cfg = hydra_env_cfg["agent"] - else: - agent_cfg.from_dict(hydra_env_cfg["agent"]) - # call the original function + env_cfg, agent_cfg = process_hydra_config(hydra_env_cfg, env_cfg, agent_cfg) func(env_cfg, agent_cfg, *args, **kwargs) return wrapper @@ -103,3 +87,58 @@ def main(env_cfg, agent_cfg): # clean up sys.argv = [sys.argv[0]] hydra.core.global_hydra.GlobalHydra.instance().clear() + + +def test_render_config_default(): + """Test that render config defaults to isaac_rtx when no override is passed.""" + sys.argv = [sys.argv[0]] + + @hydra_task_config_test("Isaac-Cartpole-RGB-Camera-Direct-v0", "rl_games_cfg_entry_point") + def main(env_cfg, agent_cfg): + assert hasattr(env_cfg, "tiled_camera") + assert env_cfg.tiled_camera.renderer_cfg.renderer_type == "isaac_rtx" + + main() + sys.argv = [sys.argv[0]] + hydra.core.global_hydra.GlobalHydra.instance().clear() + + +def test_render_config_override(): + """Test that render config group override is applied to cameras.""" + sys.argv = [sys.argv[0], "render=isaac_rtx"] + + @hydra_task_config_test("Isaac-Cartpole-RGB-Camera-Direct-v0", "rl_games_cfg_entry_point") + def main(env_cfg, agent_cfg): + assert hasattr(env_cfg, "tiled_camera") + assert env_cfg.tiled_camera.renderer_cfg.renderer_type == "isaac_rtx" + + main() + sys.argv = [sys.argv[0]] + hydra.core.global_hydra.GlobalHydra.instance().clear() + + +@pytest.mark.skipif(not NEWTON_WARP_AVAILABLE, reason="isaaclab_newton not installed") +def test_render_config_override_newton_warp(): + """Test that render=newton_warp override is applied to cameras (requires isaaclab_newton).""" + sys.argv = [sys.argv[0], "render=newton_warp"] + + @hydra_task_config_test("Isaac-Cartpole-RGB-Camera-Direct-v0", "rl_games_cfg_entry_point") + def main(env_cfg, agent_cfg): + assert hasattr(env_cfg, "tiled_camera") + assert env_cfg.tiled_camera.renderer_cfg.renderer_type == "newton_warp" + + main() + sys.argv = [sys.argv[0]] + hydra.core.global_hydra.GlobalHydra.instance().clear() + + +def test_render_config_invalid_raises(): + """Test that invalid render config raises an error.""" + sys.argv = [sys.argv[0], "render=invalid_renderer"] + + with pytest.raises(Exception, match="invalid_renderer|Could not find|No match"): + env_cfg, agent_cfg = register_task_to_hydra("Isaac-Cartpole-RGB-Camera-Direct-v0", "rl_games_cfg_entry_point") + with initialize(config_path=None, version_base="1.3"): + compose(config_name="Isaac-Cartpole-RGB-Camera-Direct-v0", overrides=sys.argv[1:]) + sys.argv = [sys.argv[0]] + hydra.core.global_hydra.GlobalHydra.instance().clear() From 5e2736759bd190b468c9892be4837e0bb30c6691 Mon Sep 17 00:00:00 2001 From: Brian Dilinila Date: Fri, 27 Feb 2026 10:30:33 -0800 Subject: [PATCH 4/4] Missed file in rebase --- .../isaaclab/sensors/camera/tiled_camera.py | 2 +- .../utils/render_config_store.py | 31 +++++++++++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) create mode 100644 source/isaaclab_tasks/isaaclab_tasks/utils/render_config_store.py diff --git a/source/isaaclab/isaaclab/sensors/camera/tiled_camera.py b/source/isaaclab/isaaclab/sensors/camera/tiled_camera.py index e872e7798b8e..b07b5d20b648 100644 --- a/source/isaaclab/isaaclab/sensors/camera/tiled_camera.py +++ b/source/isaaclab/isaaclab/sensors/camera/tiled_camera.py @@ -11,7 +11,7 @@ from typing import TYPE_CHECKING, Any import torch -import warp as wp +import warp as wp from pxr import UsdGeom diff --git a/source/isaaclab_tasks/isaaclab_tasks/utils/render_config_store.py b/source/isaaclab_tasks/isaaclab_tasks/utils/render_config_store.py new file mode 100644 index 000000000000..4b2896f569fe --- /dev/null +++ b/source/isaaclab_tasks/isaaclab_tasks/utils/render_config_store.py @@ -0,0 +1,31 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Renderer config presets for Hydra ConfigStore. + +Register renderer backend configs that can be selected via the ``render`` config +group (e.g. ``render=isaac_rtx`` or ``render=newton_warp``). The selected config +is applied to all cameras in the scene. +""" + +from hydra.core.config_store import ConfigStore + +from isaaclab_physx.renderers.isaac_rtx_renderer_cfg import IsaacRtxRendererCfg + +try: + from isaaclab_newton.renderers.newton_warp_renderer_cfg import NewtonWarpRendererCfg + + NEWTON_WARP_AVAILABLE = True +except ImportError: + NewtonWarpRendererCfg = None + NEWTON_WARP_AVAILABLE = False + + +def register_render_configs() -> None: + """Register renderer config presets in Hydra ConfigStore.""" + cs = ConfigStore.instance() + cs.store(name="isaac_rtx", group="render", node=IsaacRtxRendererCfg) + if NewtonWarpRendererCfg is not None: + cs.store(name="newton_warp", group="render", node=NewtonWarpRendererCfg)