diff --git a/docs/source/features/hydra.rst b/docs/source/features/hydra.rst index c728ecd0979..92d97dda70c 100644 --- a/docs/source/features/hydra.rst +++ b/docs/source/features/hydra.rst @@ -83,6 +83,37 @@ To set parameters to None, use the ``null`` keyword, which is a special keyword In the above example, we could also disable the ``joint_pos_rel`` observation by setting it to None with ``env.observations.policy.joint_pos_rel=null``. +TiledCamera renderer type +^^^^^^^^^^^^^^^^^^^^^^^^^ + +You can select the :class:`~isaaclab.sensors.TiledCamera` backend (RTX vs. Warp) at train time via Hydra by +overriding the camera config's ``renderer_type``. + +**Override path:** It must match where the task puts the TiledCamera in the env config: + +- **Scene has ``base_camera``:** If the task uses a scene config that exposes the camera as ``scene.base_camera`` + (e.g. a scene class like ``KukaAllegroSingleTiledCameraSceneCfg`` with a ``base_camera: TiledCameraCfg`` field), + use: + + .. code-block:: shell + + env.scene.base_camera.renderer_type=rtx + # or + env.scene.base_camera.renderer_type=warp_renderer + +- **Camera on env config:** If the task puts the camera elsewhere (e.g. ``env.tiled_camera`` on the env config), + override that path instead: + + .. code-block:: shell + + env.tiled_camera.renderer_type=rtx + # or + env.tiled_camera.renderer_type=warp_renderer + +**Values:** ``rtx`` selects the Isaac RTX (Replicator) renderer; ``warp_renderer`` selects the Newton Warp renderer +(when the ``isaaclab_newton`` package is available). No change to env or env_cfg code is required—only the +config hierarchy must expose the camera at the path you override. + Dictionaries ^^^^^^^^^^^^ Elements in dictionaries are handled as a parameters in the hierarchy. For example, in the Cartpole environment: diff --git a/docs/source/overview/core-concepts/sensors/camera.rst b/docs/source/overview/core-concepts/sensors/camera.rst index 6a34c6fab30..a2021cfdc20 100644 --- a/docs/source/overview/core-concepts/sensors/camera.rst +++ b/docs/source/overview/core-concepts/sensors/camera.rst @@ -23,6 +23,11 @@ The Tiled Rendering APIs provide a vectorized interface for collecting data from Isaac Lab provides tiled rendering APIs for RGB, depth, along with other annotators through the :class:`~sensors.TiledCamera` class. Configurations for the tiled rendering APIs can be defined through the :class:`~sensors.TiledCameraCfg` class, specifying parameters such as the regex expression for all camera paths, the transform for the cameras, the desired data type, the type of cameras to add to the scene, and the camera resolution. +The renderer backend (Isaac RTX vs. Newton Warp) can be selected at run time via the config's ``renderer_type`` +(``"rtx"`` or ``"warp_renderer"``). When using Hydra (e.g. in ``train.py``), override the camera config path your +task uses—e.g. ``env.scene.base_camera.renderer_type=rtx`` when the scene exposes ``base_camera``, or +``env.tiled_camera.renderer_type=rtx`` when the camera is on the env config. See **Hydra Configuration System** (Features) for override paths and examples. + .. code-block:: python tiled_camera: TiledCameraCfg = TiledCameraCfg( diff --git a/source/isaaclab_tasks/isaaclab_tasks/utils/hydra.py b/source/isaaclab_tasks/isaaclab_tasks/utils/hydra.py index 525b425917f..485f85195c4 100644 --- a/source/isaaclab_tasks/isaaclab_tasks/utils/hydra.py +++ b/source/isaaclab_tasks/isaaclab_tasks/utils/hydra.py @@ -20,6 +20,46 @@ from isaaclab.utils import replace_slices_with_strings, replace_strings_with_slices from isaaclab_tasks.utils.parse_cfg import load_cfg_from_registry +from isaaclab_tasks.utils.render_config_store import register_render_configs + + +def process_hydra_config( + hydra_cfg: DictConfig | dict, + env_cfg: ManagerBasedRLEnvCfg | DirectRLEnvCfg, + agent_cfg: dict | object, +) -> tuple[ManagerBasedRLEnvCfg | DirectRLEnvCfg, dict | object]: + """Process composed Hydra config and update env/agent configs in place. + + Shared by hydra_task_config and tests. Applies render config to cameras, + updates env/agent from dict, restores gymnasium spaces. + """ + if not isinstance(hydra_cfg, dict): + hydra_cfg = OmegaConf.to_container(hydra_cfg, resolve=True) + hydra_cfg = replace_strings_with_slices(hydra_cfg) + + if "render" in hydra_cfg and hydra_cfg["render"]: + renderer_dict = hydra_cfg["render"] + if isinstance(renderer_dict, dict): + env_dict = hydra_cfg.get("env", {}) + + def apply_to_cameras(d: dict) -> None: + for v in d.values(): + if isinstance(v, dict): + if "renderer_cfg" in v: + v["renderer_cfg"] = renderer_dict + apply_to_cameras(v) + + apply_to_cameras(env_dict) + + env_cfg.from_dict(hydra_cfg["env"]) + env_cfg = replace_strings_with_env_cfg_spaces(env_cfg) + + if isinstance(agent_cfg, dict) or agent_cfg is None: + agent_cfg = hydra_cfg["agent"] + else: + agent_cfg.from_dict(hydra_cfg["agent"]) + + return env_cfg, agent_cfg def register_task_to_hydra( @@ -51,10 +91,15 @@ def register_task_to_hydra( agent_cfg_dict = agent_cfg else: agent_cfg_dict = agent_cfg.to_dict() - cfg_dict = {"env": env_cfg_dict, "agent": agent_cfg_dict} + cfg_dict = { + "defaults": ["_self_", {"render": "isaac_rtx"}], + "env": env_cfg_dict, + "agent": agent_cfg_dict, + } # replace slices with strings because OmegaConf does not support slices cfg_dict = replace_slices_with_strings(cfg_dict) - # store the configuration to Hydra + # register render config presets and store the configuration to Hydra + register_render_configs() ConfigStore.instance().store(name=task_name, node=cfg_dict) return env_cfg, agent_cfg @@ -82,21 +127,7 @@ def wrapper(*args, **kwargs): # define the new Hydra main function @hydra.main(config_path=None, config_name=task_name.split(":")[-1], version_base="1.3") def hydra_main(hydra_env_cfg: DictConfig, env_cfg=env_cfg, agent_cfg=agent_cfg): - # convert to a native dictionary - hydra_env_cfg = OmegaConf.to_container(hydra_env_cfg, resolve=True) - # replace string with slices because OmegaConf does not support slices - hydra_env_cfg = replace_strings_with_slices(hydra_env_cfg) - # update the configs with the Hydra command line arguments - env_cfg.from_dict(hydra_env_cfg["env"]) - # replace strings that represent gymnasium spaces because OmegaConf does not support them. - # this must be done after converting the env configs from dictionary to avoid internal reinterpretations - env_cfg = replace_strings_with_env_cfg_spaces(env_cfg) - # get agent configs - if isinstance(agent_cfg, dict) or agent_cfg is None: - agent_cfg = hydra_env_cfg["agent"] - else: - agent_cfg.from_dict(hydra_env_cfg["agent"]) - # call the original function + env_cfg, agent_cfg = process_hydra_config(hydra_env_cfg, env_cfg, agent_cfg) func(env_cfg, agent_cfg, *args, **kwargs) # call the new Hydra main function @@ -105,3 +136,4 @@ def hydra_main(hydra_env_cfg: DictConfig, env_cfg=env_cfg, agent_cfg=agent_cfg): return wrapper return decorator + diff --git a/source/isaaclab_tasks/isaaclab_tasks/utils/render_config_store.py b/source/isaaclab_tasks/isaaclab_tasks/utils/render_config_store.py new file mode 100644 index 00000000000..4b2896f569f --- /dev/null +++ b/source/isaaclab_tasks/isaaclab_tasks/utils/render_config_store.py @@ -0,0 +1,31 @@ +# Copyright (c) 2022-2026, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +"""Renderer config presets for Hydra ConfigStore. + +Register renderer backend configs that can be selected via the ``render`` config +group (e.g. ``render=isaac_rtx`` or ``render=newton_warp``). The selected config +is applied to all cameras in the scene. +""" + +from hydra.core.config_store import ConfigStore + +from isaaclab_physx.renderers.isaac_rtx_renderer_cfg import IsaacRtxRendererCfg + +try: + from isaaclab_newton.renderers.newton_warp_renderer_cfg import NewtonWarpRendererCfg + + NEWTON_WARP_AVAILABLE = True +except ImportError: + NewtonWarpRendererCfg = None + NEWTON_WARP_AVAILABLE = False + + +def register_render_configs() -> None: + """Register renderer config presets in Hydra ConfigStore.""" + cs = ConfigStore.instance() + cs.store(name="isaac_rtx", group="render", node=IsaacRtxRendererCfg) + if NewtonWarpRendererCfg is not None: + cs.store(name="newton_warp", group="render", node=NewtonWarpRendererCfg) diff --git a/source/isaaclab_tasks/test/test_hydra.py b/source/isaaclab_tasks/test/test_hydra.py index 5c81cb3e650..5d36edc99f1 100644 --- a/source/isaaclab_tasks/test/test_hydra.py +++ b/source/isaaclab_tasks/test/test_hydra.py @@ -21,39 +21,23 @@ import hydra from hydra import compose, initialize -from omegaconf import OmegaConf - -from isaaclab.utils import replace_strings_with_slices +import pytest import isaaclab_tasks # noqa: F401 -from isaaclab_tasks.utils.hydra import register_task_to_hydra +from isaaclab_tasks.utils.hydra import process_hydra_config, register_task_to_hydra +from isaaclab_tasks.utils.render_config_store import NEWTON_WARP_AVAILABLE def hydra_task_config_test(task_name: str, agent_cfg_entry_point: str) -> Callable: - """Copied from hydra.py hydra_task_config, since hydra.main requires a single point of entry, - which will not work with multiple tests. Here, we replace hydra.main with hydra initialize - and compose.""" + """Mirrors hydra_task_config: register task, compose, process_hydra_config, then run test.""" def decorator(func): @functools.wraps(func) def wrapper(*args, **kwargs): - # register the task to Hydra env_cfg, agent_cfg = register_task_to_hydra(task_name, agent_cfg_entry_point) - - # replace hydra.main with initialize and compose with initialize(config_path=None, version_base="1.3"): hydra_env_cfg = compose(config_name=task_name, overrides=sys.argv[1:]) - # convert to a native dictionary - hydra_env_cfg = OmegaConf.to_container(hydra_env_cfg, resolve=True) - # replace string with slices because OmegaConf does not support slices - hydra_env_cfg = replace_strings_with_slices(hydra_env_cfg) - # update the configs with the Hydra command line arguments - env_cfg.from_dict(hydra_env_cfg["env"]) - if isinstance(agent_cfg, dict): - agent_cfg = hydra_env_cfg["agent"] - else: - agent_cfg.from_dict(hydra_env_cfg["agent"]) - # call the original function + env_cfg, agent_cfg = process_hydra_config(hydra_env_cfg, env_cfg, agent_cfg) func(env_cfg, agent_cfg, *args, **kwargs) return wrapper @@ -103,3 +87,58 @@ def main(env_cfg, agent_cfg): # clean up sys.argv = [sys.argv[0]] hydra.core.global_hydra.GlobalHydra.instance().clear() + + +def test_render_config_default(): + """Test that render config defaults to isaac_rtx when no override is passed.""" + sys.argv = [sys.argv[0]] + + @hydra_task_config_test("Isaac-Cartpole-RGB-Camera-Direct-v0", "rl_games_cfg_entry_point") + def main(env_cfg, agent_cfg): + assert hasattr(env_cfg, "tiled_camera") + assert env_cfg.tiled_camera.renderer_cfg.renderer_type == "isaac_rtx" + + main() + sys.argv = [sys.argv[0]] + hydra.core.global_hydra.GlobalHydra.instance().clear() + + +def test_render_config_override(): + """Test that render config group override is applied to cameras.""" + sys.argv = [sys.argv[0], "render=isaac_rtx"] + + @hydra_task_config_test("Isaac-Cartpole-RGB-Camera-Direct-v0", "rl_games_cfg_entry_point") + def main(env_cfg, agent_cfg): + assert hasattr(env_cfg, "tiled_camera") + assert env_cfg.tiled_camera.renderer_cfg.renderer_type == "isaac_rtx" + + main() + sys.argv = [sys.argv[0]] + hydra.core.global_hydra.GlobalHydra.instance().clear() + + +@pytest.mark.skipif(not NEWTON_WARP_AVAILABLE, reason="isaaclab_newton not installed") +def test_render_config_override_newton_warp(): + """Test that render=newton_warp override is applied to cameras (requires isaaclab_newton).""" + sys.argv = [sys.argv[0], "render=newton_warp"] + + @hydra_task_config_test("Isaac-Cartpole-RGB-Camera-Direct-v0", "rl_games_cfg_entry_point") + def main(env_cfg, agent_cfg): + assert hasattr(env_cfg, "tiled_camera") + assert env_cfg.tiled_camera.renderer_cfg.renderer_type == "newton_warp" + + main() + sys.argv = [sys.argv[0]] + hydra.core.global_hydra.GlobalHydra.instance().clear() + + +def test_render_config_invalid_raises(): + """Test that invalid render config raises an error.""" + sys.argv = [sys.argv[0], "render=invalid_renderer"] + + with pytest.raises(Exception, match="invalid_renderer|Could not find|No match"): + env_cfg, agent_cfg = register_task_to_hydra("Isaac-Cartpole-RGB-Camera-Direct-v0", "rl_games_cfg_entry_point") + with initialize(config_path=None, version_base="1.3"): + compose(config_name="Isaac-Cartpole-RGB-Camera-Direct-v0", overrides=sys.argv[1:]) + sys.argv = [sys.argv[0]] + hydra.core.global_hydra.GlobalHydra.instance().clear()