From 48448f85d925a13ce6128137b5b49a4c37b3dd42 Mon Sep 17 00:00:00 2001 From: Mayank Jha Date: Tue, 7 Jul 2026 13:17:13 -0700 Subject: [PATCH 1/3] feat(eval): add agent memory settings to eval sets and CLI override Port of the old-world eval memory support (UiPath/Agents #3560/#3603) to the eval-set schema and CLI: - EvaluationSetAgentMemorySettings model plus agentMemoryEnabled / agentMemorySettings fields on EvaluationSet and LegacyEvaluationSet, carried through the legacy eval-set migration. - New --agent-memory-settings-id option on `uipath eval`; the resolved override is passed to the runtime factory via the agent_memory_settings kwarg. Eval runs disable agent memory unless the eval set explicitly enables it, matching the Agents backend behavior for eval runs. Co-Authored-By: Claude Fable 5 --- packages/uipath/src/uipath/_cli/cli_eval.py | 63 ++++++ packages/uipath/src/uipath/eval/helpers.py | 2 + .../src/uipath/eval/models/evaluation_set.py | 24 +++ .../cli/eval/test_agent_memory_settings.py | 182 ++++++++++++++++++ 4 files changed, 271 insertions(+) create mode 100644 packages/uipath/tests/cli/eval/test_agent_memory_settings.py diff --git a/packages/uipath/src/uipath/_cli/cli_eval.py b/packages/uipath/src/uipath/_cli/cli_eval.py index 12ae1f062..37bb4ef99 100644 --- a/packages/uipath/src/uipath/_cli/cli_eval.py +++ b/packages/uipath/src/uipath/_cli/cli_eval.py @@ -119,6 +119,55 @@ def _resolve_model_settings_override( return override if override else None +def _resolve_agent_memory_settings_override( + agent_memory_settings_id: str, evaluation_set: EvaluationSet +) -> dict[str, Any]: + """Resolve agent memory settings override from evaluation set. + + Evaluation runs never consume agent memory unless the evaluation set + explicitly enables it (agentMemoryEnabled), mirroring the Agents + backend behavior for eval runs. + + Returns: + Memory override dict passed to the factory via the + agent_memory_settings kwarg. ``{"enabled": False}`` disables agent + memory for the run; ``{"enabled": True, ...}`` force-enables it with + the selected settings ("same-as-agent" values preserve the agent's + own configuration). + """ + if not evaluation_set.agent_memory_enabled: + return {"enabled": False} + + memory_settings = evaluation_set.agent_memory_settings + target = None + if agent_memory_settings_id and agent_memory_settings_id != "default": + target = next( + (ms for ms in memory_settings if ms.id == agent_memory_settings_id), + None, + ) + if not target: + logger.warning( + f"Agent memory settings ID '{agent_memory_settings_id}' not found in evaluation set" + ) + if target is None: + target = memory_settings[0] if memory_settings else None + + if target is None: + # Memory enabled but no settings configured: keep the agent's own configuration + return {"enabled": True} + + logger.info( + f"Applying agent memory settings override: searchMode={target.search_mode}, " + f"resultCount={target.result_count}, threshold={target.threshold}" + ) + return { + "enabled": True, + "resultCount": target.result_count, + "searchMode": target.search_mode, + "threshold": target.threshold, + } + + class _EvalDiscoveryError(EntrypointDiscoveryException): """Raised when auto-discovery of entrypoint or eval set fails.""" @@ -203,6 +252,12 @@ def _discover_eval_sets() -> list[Path]: default="default", help="Model settings ID from evaluation set to override agent settings (default: 'default')", ) +@click.option( + "--agent-memory-settings-id", + type=str, + default="default", + help="Agent memory settings ID from evaluation set to override agent memory settings (default: 'default')", +) @click.option( "--trace-file", required=False, @@ -244,6 +299,7 @@ def eval( enable_mocker_cache: bool, report_coverage: bool, model_settings_id: str, + agent_memory_settings_id: str, trace_file: str | None, max_llm_concurrency: int, input_overrides: dict[str, Any], @@ -262,6 +318,7 @@ def eval( enable_mocker_cache: Enable caching for LLM mocker responses report_coverage: Report evaluation coverage model_settings_id: Model settings ID to override agent settings + agent_memory_settings_id: Agent memory settings ID to override agent memory settings trace_file: File path where traces will be written in JSONL format max_llm_concurrency: Maximum concurrent LLM requests input_overrides: Input field overrides mapping (direct field override with deep merge) @@ -403,11 +460,17 @@ async def execute_eval(): settings_override = _resolve_model_settings_override( model_settings_id, eval_context.evaluation_set ) + agent_memory_settings_override = ( + _resolve_agent_memory_settings_override( + agent_memory_settings_id, eval_context.evaluation_set + ) + ) runtime = await runtime_factory.new_runtime( entrypoint=eval_context.entrypoint or "", runtime_id=eval_context.execution_id, settings=settings_override, + agent_memory_settings=agent_memory_settings_override, ) eval_context.runtime_schema = await runtime.get_schema() diff --git a/packages/uipath/src/uipath/eval/helpers.py b/packages/uipath/src/uipath/eval/helpers.py index 8405e4a7a..167858813 100644 --- a/packages/uipath/src/uipath/eval/helpers.py +++ b/packages/uipath/src/uipath/eval/helpers.py @@ -212,6 +212,8 @@ def migrate_evaluation_item( for evaluation in eval_set.evaluations ], model_settings=eval_set.model_settings, + agent_memory_enabled=eval_set.agent_memory_enabled, + agent_memory_settings=eval_set.agent_memory_settings, ) except ValidationError as e: raise ValueError( diff --git a/packages/uipath/src/uipath/eval/models/evaluation_set.py b/packages/uipath/src/uipath/eval/models/evaluation_set.py index c80da8e14..889f0f0e9 100644 --- a/packages/uipath/src/uipath/eval/models/evaluation_set.py +++ b/packages/uipath/src/uipath/eval/models/evaluation_set.py @@ -89,6 +89,22 @@ class EvaluationSetModelSettings(BaseModel): temperature: float | str | None = Field(default=None, alias="temperature") +class EvaluationSetAgentMemorySettings(BaseModel): + """Agent memory setting overrides within evaluation sets with ID. + + Values are stored as strings; the literal "same-as-agent" preserves the + agent's own memory configuration for that field. Matches the Agents + eval-set storage schema (agentMemorySettings). + """ + + model_config = ConfigDict(populate_by_name=True) + + id: str = Field(..., alias="id") + result_count: str = Field(default="same-as-agent", alias="resultCount") + search_mode: str = Field(default="same-as-agent", alias="searchMode") + threshold: str = Field(default="same-as-agent", alias="threshold") + + class EvaluationItem(BaseModel): """Individual evaluation item within an evaluation set.""" @@ -177,6 +193,10 @@ class EvaluationSet(BaseModel): model_settings: list[EvaluationSetModelSettings] = Field( default_factory=list, alias="modelSettings" ) + agent_memory_enabled: bool = Field(default=False, alias="agentMemoryEnabled") + agent_memory_settings: list[EvaluationSetAgentMemorySettings] = Field( + default_factory=list, alias="agentMemorySettings" + ) def extract_selected_evals(self, eval_ids) -> None: """Filter evaluations to only include those with specified IDs.""" @@ -209,6 +229,10 @@ class LegacyEvaluationSet(BaseModel): model_settings: list[EvaluationSetModelSettings] = Field( default_factory=list, alias="modelSettings" ) + agent_memory_enabled: bool = Field(default=False, alias="agentMemoryEnabled") + agent_memory_settings: list[EvaluationSetAgentMemorySettings] = Field( + default_factory=list, alias="agentMemorySettings" + ) created_at: str = Field(alias="createdAt") updated_at: str = Field(alias="updatedAt") diff --git a/packages/uipath/tests/cli/eval/test_agent_memory_settings.py b/packages/uipath/tests/cli/eval/test_agent_memory_settings.py new file mode 100644 index 000000000..ad39c7bd6 --- /dev/null +++ b/packages/uipath/tests/cli/eval/test_agent_memory_settings.py @@ -0,0 +1,182 @@ +"""Tests for agent memory settings in evaluation sets and the eval CLI override.""" + +import json +import tempfile +from pathlib import Path + +from uipath._cli.cli_eval import _resolve_agent_memory_settings_override +from uipath.eval.helpers import EvalHelpers +from uipath.eval.models.evaluation_set import ( + EvaluationSet, + EvaluationSetAgentMemorySettings, +) + + +def make_eval_set(**overrides) -> EvaluationSet: + data = { + "id": "eval-set-1", + "name": "Test Eval Set", + "version": "1.0", + "evaluatorConfigs": [], + "evaluations": [], + **overrides, + } + return EvaluationSet.model_validate(data) + + +class TestEvaluationSetSchema: + def test_parses_agent_memory_fields(self): + eval_set = make_eval_set( + agentMemoryEnabled=True, + agentMemorySettings=[ + { + "id": "primary-memory", + "resultCount": "5", + "searchMode": "hybrid", + "threshold": "0.8", + } + ], + ) + + assert eval_set.agent_memory_enabled is True + assert len(eval_set.agent_memory_settings) == 1 + setting = eval_set.agent_memory_settings[0] + assert setting.id == "primary-memory" + assert setting.result_count == "5" + assert setting.search_mode == "hybrid" + assert setting.threshold == "0.8" + + def test_defaults_preserve_existing_eval_sets(self): + eval_set = make_eval_set() + + assert eval_set.agent_memory_enabled is False + assert eval_set.agent_memory_settings == [] + + def test_memory_settings_default_to_same_as_agent(self): + setting = EvaluationSetAgentMemorySettings(id="s1") + + assert setting.result_count == "same-as-agent" + assert setting.search_mode == "same-as-agent" + assert setting.threshold == "same-as-agent" + + def test_serializes_with_camel_case_aliases(self): + eval_set = make_eval_set( + agentMemoryEnabled=True, + agentMemorySettings=[{"id": "s1", "searchMode": "semantic"}], + ) + + dumped = eval_set.model_dump(by_alias=True) + assert dumped["agentMemoryEnabled"] is True + assert dumped["agentMemorySettings"][0]["searchMode"] == "semantic" + + +class TestLegacyEvalSetMigration: + def test_legacy_migration_preserves_agent_memory_fields(self): + legacy_eval_set = { + "fileName": "test-eval.json", + "id": "test-eval-set-id", + "name": "Test Eval Set", + "batchSize": 10, + "evaluatorRefs": ["evaluator1"], + "evaluations": [], + "modelSettings": [], + "agentMemoryEnabled": True, + "agentMemorySettings": [ + { + "id": "primary-memory", + "resultCount": "5", + "searchMode": "hybrid", + "threshold": "0.8", + } + ], + "createdAt": "2025-01-26T00:00:00.000Z", + "updatedAt": "2025-01-26T00:00:00.000Z", + } + + with tempfile.TemporaryDirectory() as tmpdir: + eval_file = Path(tmpdir) / "eval-set.json" + eval_file.write_text(json.dumps(legacy_eval_set)) + + loaded_eval_set, _ = EvalHelpers.load_eval_set(str(eval_file)) + + assert loaded_eval_set.agent_memory_enabled is True + assert len(loaded_eval_set.agent_memory_settings) == 1 + assert loaded_eval_set.agent_memory_settings[0].id == "primary-memory" + + +class TestResolveAgentMemorySettingsOverride: + def test_memory_disabled_returns_disabled_override(self): + eval_set = make_eval_set(agentMemoryEnabled=False) + + override = _resolve_agent_memory_settings_override("default", eval_set) + + assert override == {"enabled": False} + + def test_memory_enabled_default_id_uses_first_setting(self): + eval_set = make_eval_set( + agentMemoryEnabled=True, + agentMemorySettings=[ + { + "id": "s1", + "resultCount": "5", + "searchMode": "hybrid", + "threshold": "0.8", + }, + { + "id": "s2", + "resultCount": "10", + "searchMode": "semantic", + "threshold": "0.5", + }, + ], + ) + + override = _resolve_agent_memory_settings_override("default", eval_set) + + assert override == { + "enabled": True, + "resultCount": "5", + "searchMode": "hybrid", + "threshold": "0.8", + } + + def test_memory_enabled_selects_setting_by_id(self): + eval_set = make_eval_set( + agentMemoryEnabled=True, + agentMemorySettings=[ + {"id": "s1", "searchMode": "hybrid"}, + { + "id": "s2", + "resultCount": "10", + "searchMode": "semantic", + "threshold": "0.5", + }, + ], + ) + + override = _resolve_agent_memory_settings_override("s2", eval_set) + + assert override == { + "enabled": True, + "resultCount": "10", + "searchMode": "semantic", + "threshold": "0.5", + } + + def test_unknown_id_falls_back_to_first_setting(self): + eval_set = make_eval_set( + agentMemoryEnabled=True, + agentMemorySettings=[{"id": "s1", "searchMode": "hybrid"}], + ) + + override = _resolve_agent_memory_settings_override("missing", eval_set) + + assert override["enabled"] is True + assert override["searchMode"] == "hybrid" + + def test_memory_enabled_without_settings_keeps_agent_configuration(self): + eval_set = make_eval_set(agentMemoryEnabled=True) + + override = _resolve_agent_memory_settings_override("default", eval_set) + + assert override == {"enabled": True} From eb03b424750cad6a6c8e18a713175b1a87a68e48 Mon Sep 17 00:00:00 2001 From: Mayank Jha Date: Tue, 7 Jul 2026 15:27:25 -0700 Subject: [PATCH 2/3] fix(eval): resolve "default" agent memory settings id by lookup The eval-set editor persists a "default" entry (all same-as-agent) alongside user-defined settings, so resolve it by id like any other before falling back to the first configured entry. Co-Authored-By: Claude Fable 5 --- packages/uipath/src/uipath/_cli/cli_eval.py | 6 +++-- .../cli/eval/test_agent_memory_settings.py | 25 +++++++++++++++++++ 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/packages/uipath/src/uipath/_cli/cli_eval.py b/packages/uipath/src/uipath/_cli/cli_eval.py index 37bb4ef99..547266da7 100644 --- a/packages/uipath/src/uipath/_cli/cli_eval.py +++ b/packages/uipath/src/uipath/_cli/cli_eval.py @@ -140,12 +140,14 @@ def _resolve_agent_memory_settings_override( memory_settings = evaluation_set.agent_memory_settings target = None - if agent_memory_settings_id and agent_memory_settings_id != "default": + if agent_memory_settings_id: + # "default" is looked up like any other id: the eval-set editor persists a + # "default" entry (all fields "same-as-agent") alongside user-defined ones. target = next( (ms for ms in memory_settings if ms.id == agent_memory_settings_id), None, ) - if not target: + if not target and agent_memory_settings_id != "default": logger.warning( f"Agent memory settings ID '{agent_memory_settings_id}' not found in evaluation set" ) diff --git a/packages/uipath/tests/cli/eval/test_agent_memory_settings.py b/packages/uipath/tests/cli/eval/test_agent_memory_settings.py index ad39c7bd6..fa550327a 100644 --- a/packages/uipath/tests/cli/eval/test_agent_memory_settings.py +++ b/packages/uipath/tests/cli/eval/test_agent_memory_settings.py @@ -163,6 +163,31 @@ def test_memory_enabled_selects_setting_by_id(self): "threshold": "0.5", } + def test_default_id_matches_persisted_default_entry(self): + # The eval-set editor persists a "default" entry (all same-as-agent) + # alongside user-defined settings; it may not be first in the list. + eval_set = make_eval_set( + agentMemoryEnabled=True, + agentMemorySettings=[ + { + "id": "s1", + "resultCount": "10", + "searchMode": "semantic", + "threshold": "0.5", + }, + {"id": "default"}, + ], + ) + + override = _resolve_agent_memory_settings_override("default", eval_set) + + assert override == { + "enabled": True, + "resultCount": "same-as-agent", + "searchMode": "same-as-agent", + "threshold": "same-as-agent", + } + def test_unknown_id_falls_back_to_first_setting(self): eval_set = make_eval_set( agentMemoryEnabled=True, From e28c239df8aeab6f3eb624d98a6e8b4844265233 Mon Sep 17 00:00:00 2001 From: Mayank Jha Date: Tue, 7 Jul 2026 15:30:17 -0700 Subject: [PATCH 3/3] chore: bump uipath to 2.13.7 Co-Authored-By: Claude Fable 5 --- packages/uipath/pyproject.toml | 2 +- packages/uipath/uv.lock | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/uipath/pyproject.toml b/packages/uipath/pyproject.toml index 035dac2d1..2ca26e087 100644 --- a/packages/uipath/pyproject.toml +++ b/packages/uipath/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "uipath" -version = "2.13.6" +version = "2.13.7" description = "Python SDK and CLI for UiPath Platform, enabling programmatic interaction with automation services, process management, and deployment tools." readme = { file = "README.md", content-type = "text/markdown" } requires-python = ">=3.11" diff --git a/packages/uipath/uv.lock b/packages/uipath/uv.lock index 19e974d12..b26e1984a 100644 --- a/packages/uipath/uv.lock +++ b/packages/uipath/uv.lock @@ -2598,7 +2598,7 @@ wheels = [ [[package]] name = "uipath" -version = "2.13.6" +version = "2.13.7" source = { editable = "." } dependencies = [ { name = "applicationinsights" },