diff --git a/packages/uipath/pyproject.toml b/packages/uipath/pyproject.toml index 035dac2d1..2ca26e087 100644 --- a/packages/uipath/pyproject.toml +++ b/packages/uipath/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "uipath" -version = "2.13.6" +version = "2.13.7" description = "Python SDK and CLI for UiPath Platform, enabling programmatic interaction with automation services, process management, and deployment tools." readme = { file = "README.md", content-type = "text/markdown" } requires-python = ">=3.11" diff --git a/packages/uipath/src/uipath/eval/evaluators/legacy_context_precision_evaluator.py b/packages/uipath/src/uipath/eval/evaluators/legacy_context_precision_evaluator.py index f0ea9c3a5..e13776950 100644 --- a/packages/uipath/src/uipath/eval/evaluators/legacy_context_precision_evaluator.py +++ b/packages/uipath/src/uipath/eval/evaluators/legacy_context_precision_evaluator.py @@ -18,7 +18,7 @@ LegacyEvaluatorConfig, track_evaluation_metrics, ) -from .legacy_evaluator_utils import clean_model_name, serialize_object +from .legacy_evaluator_utils import serialize_object class LegacyContextPrecisionEvaluatorConfig(LegacyEvaluatorConfig): @@ -326,8 +326,7 @@ async def _get_structured_llm_response( ToolParametersDefinition, ) - # Remove community-agents suffix from llm model name - model = clean_model_name(self.model) + model = self.model # Create tool definition for context precision evaluation # Note: We pass the array schema as a raw dict because ToolPropertyDefinition diff --git a/packages/uipath/src/uipath/eval/evaluators/legacy_evaluator_utils.py b/packages/uipath/src/uipath/eval/evaluators/legacy_evaluator_utils.py index 76d517cea..b8ee81906 100644 --- a/packages/uipath/src/uipath/eval/evaluators/legacy_evaluator_utils.py +++ b/packages/uipath/src/uipath/eval/evaluators/legacy_evaluator_utils.py @@ -3,22 +3,6 @@ import json from typing import Any, Optional -from uipath.platform.constants import COMMUNITY_agents_SUFFIX - - -def clean_model_name(model: str) -> str: - """Remove community-agents suffix from model name. - - Args: - model: Model name that may have the community suffix - - Returns: - Model name without the community suffix - """ - if model.endswith(COMMUNITY_agents_SUFFIX): - return model.replace(COMMUNITY_agents_SUFFIX, "") - return model - def serialize_object( content: Any, diff --git a/packages/uipath/src/uipath/eval/evaluators/legacy_faithfulness_evaluator.py b/packages/uipath/src/uipath/eval/evaluators/legacy_faithfulness_evaluator.py index 20d33a87e..b17ec7f9f 100644 --- a/packages/uipath/src/uipath/eval/evaluators/legacy_faithfulness_evaluator.py +++ b/packages/uipath/src/uipath/eval/evaluators/legacy_faithfulness_evaluator.py @@ -18,7 +18,6 @@ track_evaluation_metrics, ) from .legacy_evaluator_utils import ( - clean_model_name, serialize_object, ) @@ -513,8 +512,7 @@ async def _get_structured_llm_response( ToolParametersDefinition, ) - # Remove community-agents suffix from llm model name - model = clean_model_name(self.model) + model = self.model # Create a dynamic tool definition based on the schema tool = ToolDefinition( diff --git a/packages/uipath/src/uipath/eval/evaluators/legacy_llm_as_judge_evaluator.py b/packages/uipath/src/uipath/eval/evaluators/legacy_llm_as_judge_evaluator.py index 280e3ee98..ef2ab8a34 100644 --- a/packages/uipath/src/uipath/eval/evaluators/legacy_llm_as_judge_evaluator.py +++ b/packages/uipath/src/uipath/eval/evaluators/legacy_llm_as_judge_evaluator.py @@ -8,7 +8,6 @@ from uipath.platform import UiPath from uipath.platform.chat import UiPathLlmChatService from uipath.platform.chat.llm_gateway import RequiredToolChoice -from uipath.platform.constants import COMMUNITY_agents_SUFFIX from .._execution_context import eval_set_run_id_context from .._helpers.helpers import is_empty_value @@ -193,10 +192,7 @@ async def _get_llm_response(self, evaluation_prompt: str) -> LLMResponse: Returns: LLMResponse with score and justification """ - # remove community-agents suffix from llm model name model = self.model - if model.endswith(COMMUNITY_agents_SUFFIX): - model = model.replace(COMMUNITY_agents_SUFFIX, "") # Create evaluation tool for function calling (works across all models) evaluation_tool = create_evaluation_tool() diff --git a/packages/uipath/src/uipath/eval/evaluators/legacy_trajectory_evaluator.py b/packages/uipath/src/uipath/eval/evaluators/legacy_trajectory_evaluator.py index 680c41be0..1368d9339 100644 --- a/packages/uipath/src/uipath/eval/evaluators/legacy_trajectory_evaluator.py +++ b/packages/uipath/src/uipath/eval/evaluators/legacy_trajectory_evaluator.py @@ -9,7 +9,6 @@ from uipath.platform import UiPath from uipath.platform.chat import UiPathLlmChatService from uipath.platform.chat.llm_gateway import RequiredToolChoice -from uipath.platform.constants import COMMUNITY_agents_SUFFIX from .._execution_context import eval_set_run_id_context from .._helpers.evaluators_helpers import trace_to_str @@ -163,8 +162,6 @@ async def _get_llm_response(self, evaluation_prompt: str) -> LLMResponse: assert self.llm, "LLM should be initialized before calling this method." model = self.model - if model.endswith(COMMUNITY_agents_SUFFIX): - model = model.replace(COMMUNITY_agents_SUFFIX, "") # Create evaluation tool for function calling (works across all models) evaluation_tool = create_evaluation_tool() diff --git a/packages/uipath/src/uipath/eval/evaluators/llm_as_judge_evaluator.py b/packages/uipath/src/uipath/eval/evaluators/llm_as_judge_evaluator.py index 4285f7965..a38bbde64 100644 --- a/packages/uipath/src/uipath/eval/evaluators/llm_as_judge_evaluator.py +++ b/packages/uipath/src/uipath/eval/evaluators/llm_as_judge_evaluator.py @@ -11,7 +11,6 @@ from uipath.platform import UiPath from uipath.platform.chat import UiPathLlmChatService -from uipath.platform.constants import COMMUNITY_agents_SUFFIX from .._execution_context import eval_set_run_id_context from ..models import ( @@ -211,10 +210,7 @@ async def _get_llm_response(self, evaluation_prompt: str) -> LLMResponse: ToolPropertyDefinition, ) - # Remove community-agents suffix from llm model name model = self.evaluator_config.model - if model.endswith(COMMUNITY_agents_SUFFIX): - model = model.replace(COMMUNITY_agents_SUFFIX, "") # Define function/tool for structured output (works for ALL models via Normalized API) evaluation_tool = ToolDefinition( diff --git a/packages/uipath/tests/evaluators/test_llm_judge_model_suffix.py b/packages/uipath/tests/evaluators/test_llm_judge_model_suffix.py new file mode 100644 index 000000000..de533a963 --- /dev/null +++ b/packages/uipath/tests/evaluators/test_llm_judge_model_suffix.py @@ -0,0 +1,219 @@ +"""Regression tests: LLM-judge evaluators must send the model name to the LLM +Gateway exactly as configured, including a "-community-agents" suffix. + +Community/EU tenants' LLM Gateway routing rules are keyed on the suffixed +model id -- the same id AgentHub sends when it runs the agent itself. +Stripping the suffix before calling the Gateway causes a 417 "No llm routing +rule found for product agentsplaygroundfallback in EU using model ..." for +every Community/EU evaluation run, even though the identical model id works +fine for the agent. +""" + +import uuid +from types import SimpleNamespace +from unittest.mock import AsyncMock, patch + +import pytest + +from uipath.eval.evaluators.base_legacy_evaluator import LegacyEvaluationCriteria +from uipath.eval.evaluators.legacy_context_precision_evaluator import ( + LegacyContextPrecisionEvaluator, + LegacyContextPrecisionEvaluatorConfig, +) +from uipath.eval.evaluators.legacy_faithfulness_evaluator import ( + LegacyFaithfulnessEvaluator, + LegacyFaithfulnessEvaluatorConfig, +) +from uipath.eval.evaluators.legacy_llm_as_judge_evaluator import ( + LegacyLlmAsAJudgeEvaluator, + LegacyLlmAsAJudgeEvaluatorConfig, +) +from uipath.eval.evaluators.legacy_trajectory_evaluator import ( + LegacyTrajectoryEvaluator, + LegacyTrajectoryEvaluatorConfig, +) +from uipath.eval.evaluators.llm_judge_output_evaluator import LLMJudgeOutputEvaluator +from uipath.eval.evaluators.llm_judge_trajectory_evaluator import ( + LLMJudgeTrajectoryEvaluator, +) +from uipath.eval.models.models import LegacyEvaluatorCategory, LegacyEvaluatorType + +COMMUNITY_MODEL = "gpt-5.4-2026-03-05-community-agents" + + +def _fake_tool_call_response(score: float = 90, justification: str = "ok"): + tool_call = SimpleNamespace( + arguments={"score": score, "justification": justification} + ) + message = SimpleNamespace(tool_calls=[tool_call]) + choice = SimpleNamespace(message=message) + return SimpleNamespace(choices=[choice]) + + +def _legacy_context_precision_evaluator() -> LegacyContextPrecisionEvaluator: + return LegacyContextPrecisionEvaluator( + id="context-precision", + config_type=LegacyContextPrecisionEvaluatorConfig, + evaluation_criteria_type=LegacyEvaluationCriteria, + justification_type=str, + category=LegacyEvaluatorCategory.LlmAsAJudge, + type=LegacyEvaluatorType.ContextPrecision, + name="Context Precision", + description="Evaluates context chunk relevance", + createdAt="2025-01-01T00:00:00Z", + updatedAt="2025-01-01T00:00:00Z", + targetOutputKey="*", + model=COMMUNITY_MODEL, + ) + + +class TestLegacyContextPrecisionEvaluatorSendsConfiguredModel: + @pytest.mark.asyncio + async def test_get_structured_llm_response_sends_full_model_name(self): + evaluator = _legacy_context_precision_evaluator() + mock_chat_completions = AsyncMock(return_value=_fake_tool_call_response()) + evaluator.llm = AsyncMock(chat_completions=mock_chat_completions) + + await evaluator._get_structured_llm_response("some evaluation prompt") + + sent_model = mock_chat_completions.call_args.kwargs["model"] + assert sent_model == COMMUNITY_MODEL + + +def _legacy_faithfulness_evaluator() -> LegacyFaithfulnessEvaluator: + return LegacyFaithfulnessEvaluator( + id="faithfulness", + config_type=LegacyFaithfulnessEvaluatorConfig, + evaluation_criteria_type=LegacyEvaluationCriteria, + justification_type=str, + category=LegacyEvaluatorCategory.LlmAsAJudge, + type=LegacyEvaluatorType.Faithfulness, + name="Faithfulness", + description="Evaluates faithfulness of claims against context", + createdAt="2025-01-01T00:00:00Z", + updatedAt="2025-01-01T00:00:00Z", + targetOutputKey="*", + model=COMMUNITY_MODEL, + ) + + +class TestLegacyFaithfulnessEvaluatorSendsConfiguredModel: + @pytest.mark.asyncio + async def test_get_structured_llm_response_sends_full_model_name(self): + evaluator = _legacy_faithfulness_evaluator() + mock_chat_completions = AsyncMock(return_value=_fake_tool_call_response()) + evaluator.llm = AsyncMock(chat_completions=mock_chat_completions) + + await evaluator._get_structured_llm_response( + "some evaluation prompt", "submit_result", {"type": "object"} + ) + + sent_model = mock_chat_completions.call_args.kwargs["model"] + assert sent_model == COMMUNITY_MODEL + + +class TestLLMJudgeOutputEvaluatorSendsConfiguredModel: + """Covers LLMJudgeMixin._get_llm_response -- the code path hit by + 'uipath-llm-judge-output-semantic-similarity' in production.""" + + @pytest.mark.asyncio + async def test_get_llm_response_sends_full_model_name_to_gateway(self): + config = { + "name": "TestEvaluator", + "prompt": "Evaluate {{ActualOutput}} against {{ExpectedOutput}}", + "model": COMMUNITY_MODEL, + } + with patch("uipath.platform.UiPath"): + evaluator = LLMJudgeOutputEvaluator.model_validate( + {"evaluatorConfig": config, "id": str(uuid.uuid4())} + ) + mock_llm_service = AsyncMock(return_value=_fake_tool_call_response()) + evaluator.llm_service = mock_llm_service + + await evaluator._get_llm_response("some evaluation prompt") + + sent_model = mock_llm_service.call_args.kwargs["model"] + assert sent_model == COMMUNITY_MODEL + + +class TestLLMJudgeTrajectoryEvaluatorSendsConfiguredModel: + """Covers the same LLMJudgeMixin._get_llm_response via the trajectory + evaluator -- this is the exact evaluator type from the reported + production trace ('uipath-llm-judge-trajectory-similarity').""" + + @pytest.mark.asyncio + async def test_get_llm_response_sends_full_model_name_to_gateway(self): + config = { + "name": "TestEvaluator", + "prompt": "Judge {{AgentRunHistory}} against {{ExpectedAgentBehavior}}", + "model": COMMUNITY_MODEL, + } + with patch("uipath.platform.UiPath"): + evaluator = LLMJudgeTrajectoryEvaluator.model_validate( + {"evaluatorConfig": config, "id": str(uuid.uuid4())} + ) + mock_llm_service = AsyncMock(return_value=_fake_tool_call_response()) + evaluator.llm_service = mock_llm_service + + await evaluator._get_llm_response("some evaluation prompt") + + sent_model = mock_llm_service.call_args.kwargs["model"] + assert sent_model == COMMUNITY_MODEL + + +def _legacy_trajectory_evaluator() -> LegacyTrajectoryEvaluator: + return LegacyTrajectoryEvaluator( + id=str(uuid.uuid4()), + name="Legacy trajectory", + config_type=LegacyTrajectoryEvaluatorConfig, + evaluation_criteria_type=LegacyEvaluationCriteria, + justification_type=str, + category=LegacyEvaluatorCategory.Trajectory, + type=LegacyEvaluatorType.Trajectory, + prompt="History:\n{{AgentRunHistory}}\nExpected:\n{{ExpectedAgentBehavior}}", + model=COMMUNITY_MODEL, + createdAt="2026-05-14T00:00:00Z", + updatedAt="2026-05-14T00:00:00Z", + ) + + +class TestLegacyTrajectoryEvaluatorSendsConfiguredModel: + @pytest.mark.asyncio + async def test_get_llm_response_sends_full_model_name_to_gateway(self): + evaluator = _legacy_trajectory_evaluator() + mock_chat_completions = AsyncMock(return_value=_fake_tool_call_response()) + evaluator.llm = AsyncMock(chat_completions=mock_chat_completions) + + await evaluator._get_llm_response("some evaluation prompt") + + sent_model = mock_chat_completions.call_args.kwargs["model"] + assert sent_model == COMMUNITY_MODEL + + +def _legacy_llm_as_judge_evaluator() -> LegacyLlmAsAJudgeEvaluator: + return LegacyLlmAsAJudgeEvaluator( + id=str(uuid.uuid4()), + name="Legacy LLM judge", + config_type=LegacyLlmAsAJudgeEvaluatorConfig, + evaluation_criteria_type=LegacyEvaluationCriteria, + justification_type=str, + category=LegacyEvaluatorCategory.LlmAsAJudge, + type=LegacyEvaluatorType.Factuality, + prompt="Compare {{ActualOutput}} to {{ExpectedOutput}}", + model=COMMUNITY_MODEL, + createdAt="2026-05-14T00:00:00Z", + updatedAt="2026-05-14T00:00:00Z", + ) + + +class TestLegacyLlmAsAJudgeEvaluatorSendsConfiguredModel: + @pytest.mark.asyncio + async def test_get_llm_response_sends_full_model_name_to_gateway(self): + evaluator = _legacy_llm_as_judge_evaluator() + mock_chat_completions = AsyncMock(return_value=_fake_tool_call_response()) + evaluator.llm = AsyncMock(chat_completions=mock_chat_completions) + + await evaluator._get_llm_response("some evaluation prompt") + + sent_model = mock_chat_completions.call_args.kwargs["model"] + assert sent_model == COMMUNITY_MODEL diff --git a/packages/uipath/uv.lock b/packages/uipath/uv.lock index 19e974d12..b26e1984a 100644 --- a/packages/uipath/uv.lock +++ b/packages/uipath/uv.lock @@ -2598,7 +2598,7 @@ wheels = [ [[package]] name = "uipath" -version = "2.13.6" +version = "2.13.7" source = { editable = "." } dependencies = [ { name = "applicationinsights" },