From 3c3640d1c7cf8c02ae5e2406add580c7f7cfb9cf Mon Sep 17 00:00:00 2001 From: Chibi Vikram Date: Tue, 7 Jul 2026 13:43:20 -0700 Subject: [PATCH 1/3] fix(eval): stop stripping -community-agents suffix before calling LLM Gateway LLM-judge evaluators (current-gen and legacy) removed the "-community-agents" suffix from the configured model name before calling UiPathLlmChatService. That suffix is not cosmetic: Community/EU tenants' LLM Gateway routing rules are keyed on the suffixed model id -- the same id AgentHub sends unmodified when it runs the agent itself. Stripping it sends a bare model id the Gateway has no EU routing rule for, causing a 417 "No llm routing rule found for product agentsplaygroundfallback in EU using model ..." on every Community/EU evaluation run, even though the identical model id works for the agent. Removes the strip from llm_as_judge_evaluator.py (covers both the output-similarity and trajectory current-gen judges), legacy_llm_as_judge_evaluator.py, legacy_trajectory_evaluator.py, and the shared clean_model_name() helper used by legacy_context_precision and legacy_faithfulness evaluators. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01P5DrPWsvEMA5UdzVtnsvuM --- .../legacy_context_precision_evaluator.py | 5 +- .../eval/evaluators/legacy_evaluator_utils.py | 16 -- .../legacy_faithfulness_evaluator.py | 4 +- .../legacy_llm_as_judge_evaluator.py | 7 +- .../evaluators/legacy_trajectory_evaluator.py | 6 +- .../eval/evaluators/llm_as_judge_evaluator.py | 7 +- .../evaluators/test_llm_judge_model_suffix.py | 209 ++++++++++++++++++ 7 files changed, 221 insertions(+), 33 deletions(-) create mode 100644 packages/uipath/tests/evaluators/test_llm_judge_model_suffix.py diff --git a/packages/uipath/src/uipath/eval/evaluators/legacy_context_precision_evaluator.py b/packages/uipath/src/uipath/eval/evaluators/legacy_context_precision_evaluator.py index f0ea9c3a5..e13776950 100644 --- a/packages/uipath/src/uipath/eval/evaluators/legacy_context_precision_evaluator.py +++ b/packages/uipath/src/uipath/eval/evaluators/legacy_context_precision_evaluator.py @@ -18,7 +18,7 @@ LegacyEvaluatorConfig, track_evaluation_metrics, ) -from .legacy_evaluator_utils import clean_model_name, serialize_object +from .legacy_evaluator_utils import serialize_object class LegacyContextPrecisionEvaluatorConfig(LegacyEvaluatorConfig): @@ -326,8 +326,7 @@ async def _get_structured_llm_response( ToolParametersDefinition, ) - # Remove community-agents suffix from llm model name - model = clean_model_name(self.model) + model = self.model # Create tool definition for context precision evaluation # Note: We pass the array schema as a raw dict because ToolPropertyDefinition diff --git a/packages/uipath/src/uipath/eval/evaluators/legacy_evaluator_utils.py b/packages/uipath/src/uipath/eval/evaluators/legacy_evaluator_utils.py index 76d517cea..b8ee81906 100644 --- a/packages/uipath/src/uipath/eval/evaluators/legacy_evaluator_utils.py +++ b/packages/uipath/src/uipath/eval/evaluators/legacy_evaluator_utils.py @@ -3,22 +3,6 @@ import json from typing import Any, Optional -from uipath.platform.constants import COMMUNITY_agents_SUFFIX - - -def clean_model_name(model: str) -> str: - """Remove community-agents suffix from model name. - - Args: - model: Model name that may have the community suffix - - Returns: - Model name without the community suffix - """ - if model.endswith(COMMUNITY_agents_SUFFIX): - return model.replace(COMMUNITY_agents_SUFFIX, "") - return model - def serialize_object( content: Any, diff --git a/packages/uipath/src/uipath/eval/evaluators/legacy_faithfulness_evaluator.py b/packages/uipath/src/uipath/eval/evaluators/legacy_faithfulness_evaluator.py index 20d33a87e..b17ec7f9f 100644 --- a/packages/uipath/src/uipath/eval/evaluators/legacy_faithfulness_evaluator.py +++ b/packages/uipath/src/uipath/eval/evaluators/legacy_faithfulness_evaluator.py @@ -18,7 +18,6 @@ track_evaluation_metrics, ) from .legacy_evaluator_utils import ( - clean_model_name, serialize_object, ) @@ -513,8 +512,7 @@ async def _get_structured_llm_response( ToolParametersDefinition, ) - # Remove community-agents suffix from llm model name - model = clean_model_name(self.model) + model = self.model # Create a dynamic tool definition based on the schema tool = ToolDefinition( diff --git a/packages/uipath/src/uipath/eval/evaluators/legacy_llm_as_judge_evaluator.py b/packages/uipath/src/uipath/eval/evaluators/legacy_llm_as_judge_evaluator.py index 280e3ee98..78a5fb207 100644 --- a/packages/uipath/src/uipath/eval/evaluators/legacy_llm_as_judge_evaluator.py +++ b/packages/uipath/src/uipath/eval/evaluators/legacy_llm_as_judge_evaluator.py @@ -8,7 +8,6 @@ from uipath.platform import UiPath from uipath.platform.chat import UiPathLlmChatService from uipath.platform.chat.llm_gateway import RequiredToolChoice -from uipath.platform.constants import COMMUNITY_agents_SUFFIX from .._execution_context import eval_set_run_id_context from .._helpers.helpers import is_empty_value @@ -193,10 +192,10 @@ async def _get_llm_response(self, evaluation_prompt: str) -> LLMResponse: Returns: LLMResponse with score and justification """ - # remove community-agents suffix from llm model name + # Send the model id exactly as configured -- AgentHub routes the + # same "-community-agents"-suffixed id for the agent's own LLM + # calls, and Community/EU Gateway routing rules are keyed on it. model = self.model - if model.endswith(COMMUNITY_agents_SUFFIX): - model = model.replace(COMMUNITY_agents_SUFFIX, "") # Create evaluation tool for function calling (works across all models) evaluation_tool = create_evaluation_tool() diff --git a/packages/uipath/src/uipath/eval/evaluators/legacy_trajectory_evaluator.py b/packages/uipath/src/uipath/eval/evaluators/legacy_trajectory_evaluator.py index 680c41be0..366c49303 100644 --- a/packages/uipath/src/uipath/eval/evaluators/legacy_trajectory_evaluator.py +++ b/packages/uipath/src/uipath/eval/evaluators/legacy_trajectory_evaluator.py @@ -9,7 +9,6 @@ from uipath.platform import UiPath from uipath.platform.chat import UiPathLlmChatService from uipath.platform.chat.llm_gateway import RequiredToolChoice -from uipath.platform.constants import COMMUNITY_agents_SUFFIX from .._execution_context import eval_set_run_id_context from .._helpers.evaluators_helpers import trace_to_str @@ -162,9 +161,10 @@ async def _get_llm_response(self, evaluation_prompt: str) -> LLMResponse: """ assert self.llm, "LLM should be initialized before calling this method." + # Send the model id exactly as configured -- AgentHub routes the + # same "-community-agents"-suffixed id for the agent's own LLM + # calls, and Community/EU Gateway routing rules are keyed on it. model = self.model - if model.endswith(COMMUNITY_agents_SUFFIX): - model = model.replace(COMMUNITY_agents_SUFFIX, "") # Create evaluation tool for function calling (works across all models) evaluation_tool = create_evaluation_tool() diff --git a/packages/uipath/src/uipath/eval/evaluators/llm_as_judge_evaluator.py b/packages/uipath/src/uipath/eval/evaluators/llm_as_judge_evaluator.py index 4285f7965..4b56ca8f7 100644 --- a/packages/uipath/src/uipath/eval/evaluators/llm_as_judge_evaluator.py +++ b/packages/uipath/src/uipath/eval/evaluators/llm_as_judge_evaluator.py @@ -11,7 +11,6 @@ from uipath.platform import UiPath from uipath.platform.chat import UiPathLlmChatService -from uipath.platform.constants import COMMUNITY_agents_SUFFIX from .._execution_context import eval_set_run_id_context from ..models import ( @@ -211,10 +210,10 @@ async def _get_llm_response(self, evaluation_prompt: str) -> LLMResponse: ToolPropertyDefinition, ) - # Remove community-agents suffix from llm model name + # Send the model id exactly as configured -- AgentHub routes the + # same "-community-agents"-suffixed id for the agent's own LLM + # calls, and Community/EU Gateway routing rules are keyed on it. model = self.evaluator_config.model - if model.endswith(COMMUNITY_agents_SUFFIX): - model = model.replace(COMMUNITY_agents_SUFFIX, "") # Define function/tool for structured output (works for ALL models via Normalized API) evaluation_tool = ToolDefinition( diff --git a/packages/uipath/tests/evaluators/test_llm_judge_model_suffix.py b/packages/uipath/tests/evaluators/test_llm_judge_model_suffix.py new file mode 100644 index 000000000..b17e20ea5 --- /dev/null +++ b/packages/uipath/tests/evaluators/test_llm_judge_model_suffix.py @@ -0,0 +1,209 @@ +"""Regression tests: LLM-judge evaluators must send the model name to the LLM +Gateway exactly as configured, including a "-community-agents" suffix. + +Community/EU tenants' LLM Gateway routing rules are keyed on the suffixed +model id -- the same id AgentHub sends when it runs the agent itself. +Stripping the suffix before calling the Gateway causes a 417 "No llm routing +rule found for product agentsplaygroundfallback in EU using model ..." for +every Community/EU evaluation run, even though the identical model id works +fine for the agent. +""" + +import uuid +from types import SimpleNamespace +from unittest.mock import AsyncMock, patch + +import pytest + +from uipath.eval.evaluators import ( + LegacyContextPrecisionEvaluator, + LegacyFaithfulnessEvaluator, +) +from uipath.eval.evaluators.base_legacy_evaluator import LegacyEvaluationCriteria +from uipath.eval.evaluators.legacy_llm_as_judge_evaluator import ( + LegacyLlmAsAJudgeEvaluator, + LegacyLlmAsAJudgeEvaluatorConfig, +) +from uipath.eval.evaluators.legacy_trajectory_evaluator import ( + LegacyTrajectoryEvaluator, + LegacyTrajectoryEvaluatorConfig, +) +from uipath.eval.evaluators.llm_judge_output_evaluator import LLMJudgeOutputEvaluator +from uipath.eval.evaluators.llm_judge_trajectory_evaluator import ( + LLMJudgeTrajectoryEvaluator, +) +from uipath.eval.models.models import LegacyEvaluatorCategory, LegacyEvaluatorType + +COMMUNITY_MODEL = "gpt-5.4-2026-03-05-community-agents" + + +def _fake_tool_call_response(score: float = 90, justification: str = "ok"): + tool_call = SimpleNamespace( + arguments={"score": score, "justification": justification} + ) + message = SimpleNamespace(tool_calls=[tool_call]) + choice = SimpleNamespace(message=message) + return SimpleNamespace(choices=[choice]) + + +def _legacy_context_precision_evaluator() -> LegacyContextPrecisionEvaluator: + return LegacyContextPrecisionEvaluator( + id="context-precision", + category=LegacyEvaluatorCategory.LlmAsAJudge, + type=LegacyEvaluatorType.ContextPrecision, + name="Context Precision", + description="Evaluates context chunk relevance", + createdAt="2025-01-01T00:00:00Z", + updatedAt="2025-01-01T00:00:00Z", + targetOutputKey="*", + model=COMMUNITY_MODEL, + ) + + +class TestLegacyContextPrecisionEvaluatorSendsConfiguredModel: + @pytest.mark.asyncio + async def test_get_structured_llm_response_sends_full_model_name(self): + evaluator = _legacy_context_precision_evaluator() + mock_chat_completions = AsyncMock(return_value=_fake_tool_call_response()) + evaluator.llm = SimpleNamespace(chat_completions=mock_chat_completions) + + await evaluator._get_structured_llm_response("some evaluation prompt") + + sent_model = mock_chat_completions.call_args.kwargs["model"] + assert sent_model == COMMUNITY_MODEL + + +def _legacy_faithfulness_evaluator() -> LegacyFaithfulnessEvaluator: + return LegacyFaithfulnessEvaluator( + id="faithfulness", + category=LegacyEvaluatorCategory.LlmAsAJudge, + type=LegacyEvaluatorType.Faithfulness, + name="Faithfulness", + description="Evaluates faithfulness of claims against context", + createdAt="2025-01-01T00:00:00Z", + updatedAt="2025-01-01T00:00:00Z", + targetOutputKey="*", + model=COMMUNITY_MODEL, + ) + + +class TestLegacyFaithfulnessEvaluatorSendsConfiguredModel: + @pytest.mark.asyncio + async def test_get_structured_llm_response_sends_full_model_name(self): + evaluator = _legacy_faithfulness_evaluator() + mock_chat_completions = AsyncMock(return_value=_fake_tool_call_response()) + evaluator.llm = SimpleNamespace(chat_completions=mock_chat_completions) + + await evaluator._get_structured_llm_response( + "some evaluation prompt", "submit_result", {"type": "object"} + ) + + sent_model = mock_chat_completions.call_args.kwargs["model"] + assert sent_model == COMMUNITY_MODEL + + +class TestLLMJudgeOutputEvaluatorSendsConfiguredModel: + """Covers LLMJudgeMixin._get_llm_response -- the code path hit by + 'uipath-llm-judge-output-semantic-similarity' in production.""" + + @pytest.mark.asyncio + async def test_get_llm_response_sends_full_model_name_to_gateway(self): + config = { + "name": "TestEvaluator", + "prompt": "Evaluate {{ActualOutput}} against {{ExpectedOutput}}", + "model": COMMUNITY_MODEL, + } + with patch("uipath.platform.UiPath"): + evaluator = LLMJudgeOutputEvaluator.model_validate( + {"evaluatorConfig": config, "id": str(uuid.uuid4())} + ) + mock_llm_service = AsyncMock(return_value=_fake_tool_call_response()) + evaluator.llm_service = mock_llm_service + + await evaluator._get_llm_response("some evaluation prompt") + + sent_model = mock_llm_service.call_args.kwargs["model"] + assert sent_model == COMMUNITY_MODEL + + +class TestLLMJudgeTrajectoryEvaluatorSendsConfiguredModel: + """Covers the same LLMJudgeMixin._get_llm_response via the trajectory + evaluator -- this is the exact evaluator type from the reported + production trace ('uipath-llm-judge-trajectory-similarity').""" + + @pytest.mark.asyncio + async def test_get_llm_response_sends_full_model_name_to_gateway(self): + config = { + "name": "TestEvaluator", + "prompt": "Judge {{AgentRunHistory}} against {{ExpectedAgentBehavior}}", + "model": COMMUNITY_MODEL, + } + with patch("uipath.platform.UiPath"): + evaluator = LLMJudgeTrajectoryEvaluator.model_validate( + {"evaluatorConfig": config, "id": str(uuid.uuid4())} + ) + mock_llm_service = AsyncMock(return_value=_fake_tool_call_response()) + evaluator.llm_service = mock_llm_service + + await evaluator._get_llm_response("some evaluation prompt") + + sent_model = mock_llm_service.call_args.kwargs["model"] + assert sent_model == COMMUNITY_MODEL + + +def _legacy_trajectory_evaluator() -> LegacyTrajectoryEvaluator: + return LegacyTrajectoryEvaluator( + id=str(uuid.uuid4()), + name="Legacy trajectory", + config_type=LegacyTrajectoryEvaluatorConfig, + evaluation_criteria_type=LegacyEvaluationCriteria, + justification_type=str, + category=LegacyEvaluatorCategory.Trajectory, + type=LegacyEvaluatorType.Trajectory, + prompt="History:\n{{AgentRunHistory}}\nExpected:\n{{ExpectedAgentBehavior}}", + model=COMMUNITY_MODEL, + createdAt="2026-05-14T00:00:00Z", + updatedAt="2026-05-14T00:00:00Z", + ) + + +class TestLegacyTrajectoryEvaluatorSendsConfiguredModel: + @pytest.mark.asyncio + async def test_get_llm_response_sends_full_model_name_to_gateway(self): + evaluator = _legacy_trajectory_evaluator() + mock_chat_completions = AsyncMock(return_value=_fake_tool_call_response()) + evaluator.llm = SimpleNamespace(chat_completions=mock_chat_completions) + + await evaluator._get_llm_response("some evaluation prompt") + + sent_model = mock_chat_completions.call_args.kwargs["model"] + assert sent_model == COMMUNITY_MODEL + + +def _legacy_llm_as_judge_evaluator() -> LegacyLlmAsAJudgeEvaluator: + return LegacyLlmAsAJudgeEvaluator( + id=str(uuid.uuid4()), + name="Legacy LLM judge", + config_type=LegacyLlmAsAJudgeEvaluatorConfig, + evaluation_criteria_type=LegacyEvaluationCriteria, + justification_type=str, + category=LegacyEvaluatorCategory.LlmAsAJudge, + type=LegacyEvaluatorType.Factuality, + prompt="Compare {{ActualOutput}} to {{ExpectedOutput}}", + model=COMMUNITY_MODEL, + createdAt="2026-05-14T00:00:00Z", + updatedAt="2026-05-14T00:00:00Z", + ) + + +class TestLegacyLlmAsAJudgeEvaluatorSendsConfiguredModel: + @pytest.mark.asyncio + async def test_get_llm_response_sends_full_model_name_to_gateway(self): + evaluator = _legacy_llm_as_judge_evaluator() + mock_chat_completions = AsyncMock(return_value=_fake_tool_call_response()) + evaluator.llm = SimpleNamespace(chat_completions=mock_chat_completions) + + await evaluator._get_llm_response("some evaluation prompt") + + sent_model = mock_chat_completions.call_args.kwargs["model"] + assert sent_model == COMMUNITY_MODEL From 142dbba12ae40747cbb972ba2aeac2cd9a4257bc Mon Sep 17 00:00:00 2001 From: Chibi Vikram Date: Tue, 7 Jul 2026 13:56:11 -0700 Subject: [PATCH 2/3] chore(uipath): bump version to 2.13.7 Patch release for the -community-agents suffix fix so it can publish to PyPI once merged. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01P5DrPWsvEMA5UdzVtnsvuM --- packages/uipath/pyproject.toml | 2 +- packages/uipath/uv.lock | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/uipath/pyproject.toml b/packages/uipath/pyproject.toml index 035dac2d1..2ca26e087 100644 --- a/packages/uipath/pyproject.toml +++ b/packages/uipath/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "uipath" -version = "2.13.6" +version = "2.13.7" description = "Python SDK and CLI for UiPath Platform, enabling programmatic interaction with automation services, process management, and deployment tools." readme = { file = "README.md", content-type = "text/markdown" } requires-python = ">=3.11" diff --git a/packages/uipath/uv.lock b/packages/uipath/uv.lock index 19e974d12..b26e1984a 100644 --- a/packages/uipath/uv.lock +++ b/packages/uipath/uv.lock @@ -2598,7 +2598,7 @@ wheels = [ [[package]] name = "uipath" -version = "2.13.6" +version = "2.13.7" source = { editable = "." } dependencies = [ { name = "applicationinsights" }, From e7078694153bc27fc45059b701e0363fc10d06ba Mon Sep 17 00:00:00 2001 From: Chibi Vikram Date: Tue, 7 Jul 2026 14:09:58 -0700 Subject: [PATCH 3/3] fix(eval): satisfy CI lint/type-check, trim comments - Fix mypy errors in the new suffix test: missing config_type / evaluation_criteria_type / justification_type on the legacy context-precision and faithfulness constructors, and use AsyncMock instead of SimpleNamespace for the .llm attribute (typed as UiPathLlmChatService | None). - Drop the explanatory comments left over the four fixed call sites -- the code (model = self.model) is self-evident on its own. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01P5DrPWsvEMA5UdzVtnsvuM --- .../legacy_llm_as_judge_evaluator.py | 3 --- .../evaluators/legacy_trajectory_evaluator.py | 3 --- .../eval/evaluators/llm_as_judge_evaluator.py | 3 --- .../evaluators/test_llm_judge_model_suffix.py | 22 ++++++++++++++----- 4 files changed, 16 insertions(+), 15 deletions(-) diff --git a/packages/uipath/src/uipath/eval/evaluators/legacy_llm_as_judge_evaluator.py b/packages/uipath/src/uipath/eval/evaluators/legacy_llm_as_judge_evaluator.py index 78a5fb207..ef2ab8a34 100644 --- a/packages/uipath/src/uipath/eval/evaluators/legacy_llm_as_judge_evaluator.py +++ b/packages/uipath/src/uipath/eval/evaluators/legacy_llm_as_judge_evaluator.py @@ -192,9 +192,6 @@ async def _get_llm_response(self, evaluation_prompt: str) -> LLMResponse: Returns: LLMResponse with score and justification """ - # Send the model id exactly as configured -- AgentHub routes the - # same "-community-agents"-suffixed id for the agent's own LLM - # calls, and Community/EU Gateway routing rules are keyed on it. model = self.model # Create evaluation tool for function calling (works across all models) diff --git a/packages/uipath/src/uipath/eval/evaluators/legacy_trajectory_evaluator.py b/packages/uipath/src/uipath/eval/evaluators/legacy_trajectory_evaluator.py index 366c49303..1368d9339 100644 --- a/packages/uipath/src/uipath/eval/evaluators/legacy_trajectory_evaluator.py +++ b/packages/uipath/src/uipath/eval/evaluators/legacy_trajectory_evaluator.py @@ -161,9 +161,6 @@ async def _get_llm_response(self, evaluation_prompt: str) -> LLMResponse: """ assert self.llm, "LLM should be initialized before calling this method." - # Send the model id exactly as configured -- AgentHub routes the - # same "-community-agents"-suffixed id for the agent's own LLM - # calls, and Community/EU Gateway routing rules are keyed on it. model = self.model # Create evaluation tool for function calling (works across all models) diff --git a/packages/uipath/src/uipath/eval/evaluators/llm_as_judge_evaluator.py b/packages/uipath/src/uipath/eval/evaluators/llm_as_judge_evaluator.py index 4b56ca8f7..a38bbde64 100644 --- a/packages/uipath/src/uipath/eval/evaluators/llm_as_judge_evaluator.py +++ b/packages/uipath/src/uipath/eval/evaluators/llm_as_judge_evaluator.py @@ -210,9 +210,6 @@ async def _get_llm_response(self, evaluation_prompt: str) -> LLMResponse: ToolPropertyDefinition, ) - # Send the model id exactly as configured -- AgentHub routes the - # same "-community-agents"-suffixed id for the agent's own LLM - # calls, and Community/EU Gateway routing rules are keyed on it. model = self.evaluator_config.model # Define function/tool for structured output (works for ALL models via Normalized API) diff --git a/packages/uipath/tests/evaluators/test_llm_judge_model_suffix.py b/packages/uipath/tests/evaluators/test_llm_judge_model_suffix.py index b17e20ea5..de533a963 100644 --- a/packages/uipath/tests/evaluators/test_llm_judge_model_suffix.py +++ b/packages/uipath/tests/evaluators/test_llm_judge_model_suffix.py @@ -15,11 +15,15 @@ import pytest -from uipath.eval.evaluators import ( +from uipath.eval.evaluators.base_legacy_evaluator import LegacyEvaluationCriteria +from uipath.eval.evaluators.legacy_context_precision_evaluator import ( LegacyContextPrecisionEvaluator, + LegacyContextPrecisionEvaluatorConfig, +) +from uipath.eval.evaluators.legacy_faithfulness_evaluator import ( LegacyFaithfulnessEvaluator, + LegacyFaithfulnessEvaluatorConfig, ) -from uipath.eval.evaluators.base_legacy_evaluator import LegacyEvaluationCriteria from uipath.eval.evaluators.legacy_llm_as_judge_evaluator import ( LegacyLlmAsAJudgeEvaluator, LegacyLlmAsAJudgeEvaluatorConfig, @@ -49,6 +53,9 @@ def _fake_tool_call_response(score: float = 90, justification: str = "ok"): def _legacy_context_precision_evaluator() -> LegacyContextPrecisionEvaluator: return LegacyContextPrecisionEvaluator( id="context-precision", + config_type=LegacyContextPrecisionEvaluatorConfig, + evaluation_criteria_type=LegacyEvaluationCriteria, + justification_type=str, category=LegacyEvaluatorCategory.LlmAsAJudge, type=LegacyEvaluatorType.ContextPrecision, name="Context Precision", @@ -65,7 +72,7 @@ class TestLegacyContextPrecisionEvaluatorSendsConfiguredModel: async def test_get_structured_llm_response_sends_full_model_name(self): evaluator = _legacy_context_precision_evaluator() mock_chat_completions = AsyncMock(return_value=_fake_tool_call_response()) - evaluator.llm = SimpleNamespace(chat_completions=mock_chat_completions) + evaluator.llm = AsyncMock(chat_completions=mock_chat_completions) await evaluator._get_structured_llm_response("some evaluation prompt") @@ -76,6 +83,9 @@ async def test_get_structured_llm_response_sends_full_model_name(self): def _legacy_faithfulness_evaluator() -> LegacyFaithfulnessEvaluator: return LegacyFaithfulnessEvaluator( id="faithfulness", + config_type=LegacyFaithfulnessEvaluatorConfig, + evaluation_criteria_type=LegacyEvaluationCriteria, + justification_type=str, category=LegacyEvaluatorCategory.LlmAsAJudge, type=LegacyEvaluatorType.Faithfulness, name="Faithfulness", @@ -92,7 +102,7 @@ class TestLegacyFaithfulnessEvaluatorSendsConfiguredModel: async def test_get_structured_llm_response_sends_full_model_name(self): evaluator = _legacy_faithfulness_evaluator() mock_chat_completions = AsyncMock(return_value=_fake_tool_call_response()) - evaluator.llm = SimpleNamespace(chat_completions=mock_chat_completions) + evaluator.llm = AsyncMock(chat_completions=mock_chat_completions) await evaluator._get_structured_llm_response( "some evaluation prompt", "submit_result", {"type": "object"} @@ -172,7 +182,7 @@ class TestLegacyTrajectoryEvaluatorSendsConfiguredModel: async def test_get_llm_response_sends_full_model_name_to_gateway(self): evaluator = _legacy_trajectory_evaluator() mock_chat_completions = AsyncMock(return_value=_fake_tool_call_response()) - evaluator.llm = SimpleNamespace(chat_completions=mock_chat_completions) + evaluator.llm = AsyncMock(chat_completions=mock_chat_completions) await evaluator._get_llm_response("some evaluation prompt") @@ -201,7 +211,7 @@ class TestLegacyLlmAsAJudgeEvaluatorSendsConfiguredModel: async def test_get_llm_response_sends_full_model_name_to_gateway(self): evaluator = _legacy_llm_as_judge_evaluator() mock_chat_completions = AsyncMock(return_value=_fake_tool_call_response()) - evaluator.llm = SimpleNamespace(chat_completions=mock_chat_completions) + evaluator.llm = AsyncMock(chat_completions=mock_chat_completions) await evaluator._get_llm_response("some evaluation prompt")