feat(eval): add agent memory settings to eval sets and CLI override

packages/uipath/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "uipath"
-version = "2.13.6"
+version = "2.13.7"
 description = "Python SDK and CLI for UiPath Platform, enabling programmatic interaction with automation services, process management, and deployment tools."
 readme = { file = "README.md", content-type = "text/markdown" }
 requires-python = ">=3.11"

packages/uipath/src/uipath/_cli/cli_eval.py

@@ -119,6 +119,57 @@ def _resolve_model_settings_override(
     return override if override else None
 
 
+def _resolve_agent_memory_settings_override(
+    agent_memory_settings_id: str, evaluation_set: EvaluationSet
+) -> dict[str, Any]:
+    """Resolve agent memory settings override from evaluation set.
+
+    Evaluation runs never consume agent memory unless the evaluation set
+    explicitly enables it (agentMemoryEnabled), mirroring the Agents
+    backend behavior for eval runs.
+
+    Returns:
+        Memory override dict passed to the factory via the
+        agent_memory_settings kwarg. ``{"enabled": False}`` disables agent
+        memory for the run; ``{"enabled": True, ...}`` force-enables it with
+        the selected settings ("same-as-agent" values preserve the agent's
+        own configuration).
+    """
+    if not evaluation_set.agent_memory_enabled:
+        return {"enabled": False}
+
+    memory_settings = evaluation_set.agent_memory_settings
+    target = None
+    if agent_memory_settings_id:
+        # "default" is looked up like any other id: the eval-set editor persists a
+        # "default" entry (all fields "same-as-agent") alongside user-defined ones.
+        target = next(
+            (ms for ms in memory_settings if ms.id == agent_memory_settings_id),
+            None,
+        )
+        if not target and agent_memory_settings_id != "default":
+            logger.warning(
+                f"Agent memory settings ID '{agent_memory_settings_id}' not found in evaluation set"
+            )
+    if target is None:
+        target = memory_settings[0] if memory_settings else None
+
+    if target is None:
+        # Memory enabled but no settings configured: keep the agent's own configuration
+        return {"enabled": True}
+
+    logger.info(
+        f"Applying agent memory settings override: searchMode={target.search_mode}, "
+        f"resultCount={target.result_count}, threshold={target.threshold}"
+    )
+    return {
+        "enabled": True,
+        "resultCount": target.result_count,
+        "searchMode": target.search_mode,
+        "threshold": target.threshold,
+    }
+
+
 class _EvalDiscoveryError(EntrypointDiscoveryException):
     """Raised when auto-discovery of entrypoint or eval set fails."""
 
@@ -203,6 +254,12 @@ def _discover_eval_sets() -> list[Path]:
     default="default",
     help="Model settings ID from evaluation set to override agent settings (default: 'default')",
 )
+@click.option(
+    "--agent-memory-settings-id",
+    type=str,
+    default="default",
+    help="Agent memory settings ID from evaluation set to override agent memory settings (default: 'default')",
+)
 @click.option(
     "--trace-file",
     required=False,
@@ -244,6 +301,7 @@ def eval(
     enable_mocker_cache: bool,
     report_coverage: bool,
     model_settings_id: str,
+    agent_memory_settings_id: str,
     trace_file: str | None,
     max_llm_concurrency: int,
     input_overrides: dict[str, Any],
@@ -262,6 +320,7 @@ def eval(
         enable_mocker_cache: Enable caching for LLM mocker responses
         report_coverage: Report evaluation coverage
         model_settings_id: Model settings ID to override agent settings
+        agent_memory_settings_id: Agent memory settings ID to override agent memory settings
         trace_file: File path where traces will be written in JSONL format
         max_llm_concurrency: Maximum concurrent LLM requests
         input_overrides: Input field overrides mapping (direct field override with deep merge)
@@ -403,11 +462,17 @@ async def execute_eval():
                         settings_override = _resolve_model_settings_override(
                             model_settings_id, eval_context.evaluation_set
                         )
+                        agent_memory_settings_override = (
+                            _resolve_agent_memory_settings_override(
+                                agent_memory_settings_id, eval_context.evaluation_set
+                            )
+                        )
 
                         runtime = await runtime_factory.new_runtime(
                             entrypoint=eval_context.entrypoint or "",
                             runtime_id=eval_context.execution_id,
                             settings=settings_override,
+                            agent_memory_settings=agent_memory_settings_override,
                         )
 
                         eval_context.runtime_schema = await runtime.get_schema()

packages/uipath/src/uipath/eval/helpers.py

@@ -212,6 +212,8 @@ def migrate_evaluation_item(
                         for evaluation in eval_set.evaluations
                     ],
                     model_settings=eval_set.model_settings,
+                    agent_memory_enabled=eval_set.agent_memory_enabled,
+                    agent_memory_settings=eval_set.agent_memory_settings,
                 )
         except ValidationError as e:
             raise ValueError(

packages/uipath/src/uipath/eval/models/evaluation_set.py

@@ -89,6 +89,22 @@ class EvaluationSetModelSettings(BaseModel):
     temperature: float | str | None = Field(default=None, alias="temperature")
 
 
+class EvaluationSetAgentMemorySettings(BaseModel):
+    """Agent memory setting overrides within evaluation sets with ID.
+
+    Values are stored as strings; the literal "same-as-agent" preserves the
+    agent's own memory configuration for that field. Matches the Agents
+    eval-set storage schema (agentMemorySettings).
+    """
+
+    model_config = ConfigDict(populate_by_name=True)
+
+    id: str = Field(..., alias="id")
+    result_count: str = Field(default="same-as-agent", alias="resultCount")
+    search_mode: str = Field(default="same-as-agent", alias="searchMode")
+    threshold: str = Field(default="same-as-agent", alias="threshold")
+
+
 class EvaluationItem(BaseModel):
     """Individual evaluation item within an evaluation set."""
 
@@ -177,6 +193,10 @@ class EvaluationSet(BaseModel):
     model_settings: list[EvaluationSetModelSettings] = Field(
         default_factory=list, alias="modelSettings"
     )
+    agent_memory_enabled: bool = Field(default=False, alias="agentMemoryEnabled")
+    agent_memory_settings: list[EvaluationSetAgentMemorySettings] = Field(
+        default_factory=list, alias="agentMemorySettings"
+    )
 
     def extract_selected_evals(self, eval_ids) -> None:
         """Filter evaluations to only include those with specified IDs."""
@@ -209,6 +229,10 @@ class LegacyEvaluationSet(BaseModel):
     model_settings: list[EvaluationSetModelSettings] = Field(
         default_factory=list, alias="modelSettings"
     )
+    agent_memory_enabled: bool = Field(default=False, alias="agentMemoryEnabled")
+    agent_memory_settings: list[EvaluationSetAgentMemorySettings] = Field(
+        default_factory=list, alias="agentMemorySettings"
+    )
     created_at: str = Field(alias="createdAt")
     updated_at: str = Field(alias="updatedAt")
 

packages/uipath/tests/cli/eval/test_agent_memory_settings.py

@@ -0,0 +1,207 @@
+"""Tests for agent memory settings in evaluation sets and the eval CLI override."""
+
+import json
+import tempfile
+from pathlib import Path
+
+from uipath._cli.cli_eval import _resolve_agent_memory_settings_override
+from uipath.eval.helpers import EvalHelpers
+from uipath.eval.models.evaluation_set import (
+    EvaluationSet,
+    EvaluationSetAgentMemorySettings,
+)
+
+
+def make_eval_set(**overrides) -> EvaluationSet:
+    data = {
+        "id": "eval-set-1",
+        "name": "Test Eval Set",
+        "version": "1.0",
+        "evaluatorConfigs": [],
+        "evaluations": [],
+        **overrides,
+    }
+    return EvaluationSet.model_validate(data)
+
+
+class TestEvaluationSetSchema:
+    def test_parses_agent_memory_fields(self):
+        eval_set = make_eval_set(
+            agentMemoryEnabled=True,
+            agentMemorySettings=[
+                {
+                    "id": "primary-memory",
+                    "resultCount": "5",
+                    "searchMode": "hybrid",
+                    "threshold": "0.8",
+                }
+            ],
+        )
+
+        assert eval_set.agent_memory_enabled is True
+        assert len(eval_set.agent_memory_settings) == 1
+        setting = eval_set.agent_memory_settings[0]
+        assert setting.id == "primary-memory"
+        assert setting.result_count == "5"
+        assert setting.search_mode == "hybrid"
+        assert setting.threshold == "0.8"
+
+    def test_defaults_preserve_existing_eval_sets(self):
+        eval_set = make_eval_set()
+
+        assert eval_set.agent_memory_enabled is False
+        assert eval_set.agent_memory_settings == []
+
+    def test_memory_settings_default_to_same_as_agent(self):
+        setting = EvaluationSetAgentMemorySettings(id="s1")
+
+        assert setting.result_count == "same-as-agent"
+        assert setting.search_mode == "same-as-agent"
+        assert setting.threshold == "same-as-agent"
+
+    def test_serializes_with_camel_case_aliases(self):
+        eval_set = make_eval_set(
+            agentMemoryEnabled=True,
+            agentMemorySettings=[{"id": "s1", "searchMode": "semantic"}],
+        )
+
+        dumped = eval_set.model_dump(by_alias=True)
+        assert dumped["agentMemoryEnabled"] is True
+        assert dumped["agentMemorySettings"][0]["searchMode"] == "semantic"
+
+
+class TestLegacyEvalSetMigration:
+    def test_legacy_migration_preserves_agent_memory_fields(self):
+        legacy_eval_set = {
+            "fileName": "test-eval.json",
+            "id": "test-eval-set-id",
+            "name": "Test Eval Set",
+            "batchSize": 10,
+            "evaluatorRefs": ["evaluator1"],
+            "evaluations": [],
+            "modelSettings": [],
+            "agentMemoryEnabled": True,
+            "agentMemorySettings": [
+                {
+                    "id": "primary-memory",
+                    "resultCount": "5",
+                    "searchMode": "hybrid",
+                    "threshold": "0.8",
+                }
+            ],
+            "createdAt": "2025-01-26T00:00:00.000Z",
+            "updatedAt": "2025-01-26T00:00:00.000Z",
+        }
+
+        with tempfile.TemporaryDirectory() as tmpdir:
+            eval_file = Path(tmpdir) / "eval-set.json"
+            eval_file.write_text(json.dumps(legacy_eval_set))
+
+            loaded_eval_set, _ = EvalHelpers.load_eval_set(str(eval_file))
+
+        assert loaded_eval_set.agent_memory_enabled is True
+        assert len(loaded_eval_set.agent_memory_settings) == 1
+        assert loaded_eval_set.agent_memory_settings[0].id == "primary-memory"
+
+
+class TestResolveAgentMemorySettingsOverride:
+    def test_memory_disabled_returns_disabled_override(self):
+        eval_set = make_eval_set(agentMemoryEnabled=False)
+
+        override = _resolve_agent_memory_settings_override("default", eval_set)
+
+        assert override == {"enabled": False}
+
+    def test_memory_enabled_default_id_uses_first_setting(self):
+        eval_set = make_eval_set(
+            agentMemoryEnabled=True,
+            agentMemorySettings=[
+                {
+                    "id": "s1",
+                    "resultCount": "5",
+                    "searchMode": "hybrid",
+                    "threshold": "0.8",
+                },
+                {
+                    "id": "s2",
+                    "resultCount": "10",
+                    "searchMode": "semantic",
+                    "threshold": "0.5",
+                },
+            ],
+        )
+
+        override = _resolve_agent_memory_settings_override("default", eval_set)
+
+        assert override == {
+            "enabled": True,
+            "resultCount": "5",
+            "searchMode": "hybrid",
+            "threshold": "0.8",
+        }
+
+    def test_memory_enabled_selects_setting_by_id(self):
+        eval_set = make_eval_set(
+            agentMemoryEnabled=True,
+            agentMemorySettings=[
+                {"id": "s1", "searchMode": "hybrid"},
+                {
+                    "id": "s2",
+                    "resultCount": "10",
+                    "searchMode": "semantic",
+                    "threshold": "0.5",
+                },
+            ],
+        )
+
+        override = _resolve_agent_memory_settings_override("s2", eval_set)
+
+        assert override == {
+            "enabled": True,
+            "resultCount": "10",
+            "searchMode": "semantic",
+            "threshold": "0.5",
+        }
+
+    def test_default_id_matches_persisted_default_entry(self):
+        # The eval-set editor persists a "default" entry (all same-as-agent)
+        # alongside user-defined settings; it may not be first in the list.
+        eval_set = make_eval_set(
+            agentMemoryEnabled=True,
+            agentMemorySettings=[
+                {
+                    "id": "s1",
+                    "resultCount": "10",
+                    "searchMode": "semantic",
+                    "threshold": "0.5",
+                },
+                {"id": "default"},
+            ],
+        )
+
+        override = _resolve_agent_memory_settings_override("default", eval_set)
+
+        assert override == {
+            "enabled": True,
+            "resultCount": "same-as-agent",
+            "searchMode": "same-as-agent",
+            "threshold": "same-as-agent",
+        }
+
+    def test_unknown_id_falls_back_to_first_setting(self):
+        eval_set = make_eval_set(
+            agentMemoryEnabled=True,
+            agentMemorySettings=[{"id": "s1", "searchMode": "hybrid"}],
+        )
+
+        override = _resolve_agent_memory_settings_override("missing", eval_set)
+
+        assert override["enabled"] is True
+        assert override["searchMode"] == "hybrid"
+
+    def test_memory_enabled_without_settings_keeps_agent_configuration(self):
+        eval_set = make_eval_set(agentMemoryEnabled=True)
+
+        override = _resolve_agent_memory_settings_override("default", eval_set)
+
+        assert override == {"enabled": True}