From 9c8f31ce144ecc797b0e2b8b0b65491a4415f636 Mon Sep 17 00:00:00 2001 From: erik Date: Wed, 23 Jul 2025 14:42:26 +0300 Subject: [PATCH 1/3] Fix outcome detection after converter change. Remove dependency. --- .../agentune/simulate/outcome_detection/rag/prompt.py | 2 +- .../agentune/simulate/outcome_detection/rag/rag.py | 2 +- .../agentune/simulate/rag/indexing_and_retrieval.py | 11 ++++++++++- .../streamlit/pages/Agentune_Simulate_Runner.py | 2 +- .../tests/simulate/rag/test_indexing_retrieval.py | 5 ++++- 5 files changed, 17 insertions(+), 5 deletions(-) diff --git a/agentune_simulate/agentune/simulate/outcome_detection/rag/prompt.py b/agentune_simulate/agentune/simulate/outcome_detection/rag/prompt.py index d2835afc..b081ed04 100644 --- a/agentune_simulate/agentune/simulate/outcome_detection/rag/prompt.py +++ b/agentune_simulate/agentune/simulate/outcome_detection/rag/prompt.py @@ -108,7 +108,7 @@ def format_examples(examples: list[tuple[Document, float]]) -> str: for i, (doc, _) in enumerate(examples): messages_data = doc.metadata.get('full_conversation', '') - messages = converter.structure(messages_data, list[Message]) + messages = converter.loads(messages_data, list[Message]) conversation_text = indexing_and_retrieval.format_conversation(messages) outcome_info = doc.metadata.get('outcome', 'No outcome information available') diff --git a/agentune_simulate/agentune/simulate/outcome_detection/rag/rag.py b/agentune_simulate/agentune/simulate/outcome_detection/rag/rag.py index e4747c1b..abbd1aef 100644 --- a/agentune_simulate/agentune/simulate/outcome_detection/rag/rag.py +++ b/agentune_simulate/agentune/simulate/outcome_detection/rag/rag.py @@ -94,7 +94,7 @@ async def detect_outcomes( async def prepare_prompt_template_params(instance_idx): instance = instances[instance_idx] # Retrieve similar conversations for this instance - few_shot_examples = await indexing_and_retrieval.get_similar_finished_conversations(self.vector_store, instance.conversation, k=self.num_examples) + few_shot_examples = await indexing_and_retrieval.get_similar_finished_conversations(self.vector_store, instance.conversation, k=self.num_examples, instance=instance) formatted_conversation = indexing_and_retrieval.format_conversation(instance.conversation.messages) formatted_examples = format_examples(few_shot_examples) diff --git a/agentune_simulate/agentune/simulate/rag/indexing_and_retrieval.py b/agentune_simulate/agentune/simulate/rag/indexing_and_retrieval.py index ff9ebaa8..6c96d299 100644 --- a/agentune_simulate/agentune/simulate/rag/indexing_and_retrieval.py +++ b/agentune_simulate/agentune/simulate/rag/indexing_and_retrieval.py @@ -5,6 +5,7 @@ from langchain_core.vectorstores import VectorStore from ..models import Conversation, Message +from ..outcome_detection.base import OutcomeDetectionTest from ..util.structure import converter from .filtered_retriever import VectorStoreSearcher @@ -122,7 +123,8 @@ def conversations_to_langchain_documents( async def get_similar_finished_conversations( vector_store: VectorStore, conversation: Conversation, - k: int + k: int, + instance: OutcomeDetectionTest ) -> list[tuple[Document, float]]: """Retrieve similar finished conversation examples from the vector store. @@ -134,6 +136,7 @@ async def get_similar_finished_conversations( vector_store: The vector store to search for similar conversations conversation: Current conversation to find examples for k: Number of similar conversations to retrieve + instance: Optional instance to exclude exact matches (if provided) Returns: List of similar conversations as (Document, score) tuples, sorted by relevance @@ -152,6 +155,12 @@ async def get_similar_finished_conversations( # Sort by similarity score (highest first) retrieved_docs.sort(key=lambda x: x[1], reverse=True) + # If there is an exact match between the current conversation and one of the examples, remove it + retrieved_docs = [ + (doc, score) for doc, score in retrieved_docs + if doc.metadata.get('conversation_hash') != hash(instance.conversation.messages) + ] + return retrieved_docs diff --git a/agentune_simulate/streamlit/pages/Agentune_Simulate_Runner.py b/agentune_simulate/streamlit/pages/Agentune_Simulate_Runner.py index 8ed0920c..22f63b54 100644 --- a/agentune_simulate/streamlit/pages/Agentune_Simulate_Runner.py +++ b/agentune_simulate/streamlit/pages/Agentune_Simulate_Runner.py @@ -337,7 +337,7 @@ async def run_simulation( """Run the conversation simulation.""" # Get logging callback tracer - callbacks = get_llm_callbacks(config['session_name']) + callbacks = get_llm_callbacks() # Initialize models with logging callbacks agent_model = ChatOpenAI(**config['agent_model_kwargs'], callbacks=callbacks) diff --git a/agentune_simulate/tests/simulate/rag/test_indexing_retrieval.py b/agentune_simulate/tests/simulate/rag/test_indexing_retrieval.py index f129548d..5e17f725 100644 --- a/agentune_simulate/tests/simulate/rag/test_indexing_retrieval.py +++ b/agentune_simulate/tests/simulate/rag/test_indexing_retrieval.py @@ -12,7 +12,9 @@ from langchain_core.vectorstores import VectorStore, InMemoryVectorStore from langchain_core.embeddings import DeterministicFakeEmbedding +from agentune.simulate import Intent from agentune.simulate.models import Conversation, Message, ParticipantRole +from agentune.simulate.outcome_detection.base import OutcomeDetectionTest from agentune.simulate.rag.indexing_and_retrieval import ( conversations_to_langchain_documents, get_similar_finished_conversations @@ -136,7 +138,8 @@ async def test_filtering_finished_conversations(self): docs_with_scores = await get_similar_finished_conversations( vector_store=self.vector_store, conversation=query_conversation, - k=k + k=k, + instance=OutcomeDetectionTest(query_conversation, intent=Intent(role=ParticipantRole.CUSTOMER, description="No description")) # No specific intent for this test, ) # Check that we got expected number of results From ff58b3f91d62011634c7e8a868f18b014c565ba7 Mon Sep 17 00:00:00 2001 From: erik Date: Wed, 23 Jul 2025 16:00:56 +0300 Subject: [PATCH 2/3] Fix comments --- .../agentune/simulate/outcome_detection/rag/rag.py | 2 +- .../agentune/simulate/rag/indexing_and_retrieval.py | 6 ++---- .../tests/simulate/rag/test_indexing_retrieval.py | 5 +---- 3 files changed, 4 insertions(+), 9 deletions(-) diff --git a/agentune_simulate/agentune/simulate/outcome_detection/rag/rag.py b/agentune_simulate/agentune/simulate/outcome_detection/rag/rag.py index abbd1aef..e4747c1b 100644 --- a/agentune_simulate/agentune/simulate/outcome_detection/rag/rag.py +++ b/agentune_simulate/agentune/simulate/outcome_detection/rag/rag.py @@ -94,7 +94,7 @@ async def detect_outcomes( async def prepare_prompt_template_params(instance_idx): instance = instances[instance_idx] # Retrieve similar conversations for this instance - few_shot_examples = await indexing_and_retrieval.get_similar_finished_conversations(self.vector_store, instance.conversation, k=self.num_examples, instance=instance) + few_shot_examples = await indexing_and_retrieval.get_similar_finished_conversations(self.vector_store, instance.conversation, k=self.num_examples) formatted_conversation = indexing_and_retrieval.format_conversation(instance.conversation.messages) formatted_examples = format_examples(few_shot_examples) diff --git a/agentune_simulate/agentune/simulate/rag/indexing_and_retrieval.py b/agentune_simulate/agentune/simulate/rag/indexing_and_retrieval.py index 6c96d299..56c42d76 100644 --- a/agentune_simulate/agentune/simulate/rag/indexing_and_retrieval.py +++ b/agentune_simulate/agentune/simulate/rag/indexing_and_retrieval.py @@ -5,7 +5,6 @@ from langchain_core.vectorstores import VectorStore from ..models import Conversation, Message -from ..outcome_detection.base import OutcomeDetectionTest from ..util.structure import converter from .filtered_retriever import VectorStoreSearcher @@ -123,8 +122,7 @@ def conversations_to_langchain_documents( async def get_similar_finished_conversations( vector_store: VectorStore, conversation: Conversation, - k: int, - instance: OutcomeDetectionTest + k: int ) -> list[tuple[Document, float]]: """Retrieve similar finished conversation examples from the vector store. @@ -158,7 +156,7 @@ async def get_similar_finished_conversations( # If there is an exact match between the current conversation and one of the examples, remove it retrieved_docs = [ (doc, score) for doc, score in retrieved_docs - if doc.metadata.get('conversation_hash') != hash(instance.conversation.messages) + if doc.metadata.get('conversation_hash') != hash(conversation.messages) ] return retrieved_docs diff --git a/agentune_simulate/tests/simulate/rag/test_indexing_retrieval.py b/agentune_simulate/tests/simulate/rag/test_indexing_retrieval.py index 5e17f725..f129548d 100644 --- a/agentune_simulate/tests/simulate/rag/test_indexing_retrieval.py +++ b/agentune_simulate/tests/simulate/rag/test_indexing_retrieval.py @@ -12,9 +12,7 @@ from langchain_core.vectorstores import VectorStore, InMemoryVectorStore from langchain_core.embeddings import DeterministicFakeEmbedding -from agentune.simulate import Intent from agentune.simulate.models import Conversation, Message, ParticipantRole -from agentune.simulate.outcome_detection.base import OutcomeDetectionTest from agentune.simulate.rag.indexing_and_retrieval import ( conversations_to_langchain_documents, get_similar_finished_conversations @@ -138,8 +136,7 @@ async def test_filtering_finished_conversations(self): docs_with_scores = await get_similar_finished_conversations( vector_store=self.vector_store, conversation=query_conversation, - k=k, - instance=OutcomeDetectionTest(query_conversation, intent=Intent(role=ParticipantRole.CUSTOMER, description="No description")) # No specific intent for this test, + k=k ) # Check that we got expected number of results From f31de88ad57abce0708b7fa2f9d0fbe84449229c Mon Sep 17 00:00:00 2001 From: erik Date: Wed, 23 Jul 2025 16:23:22 +0300 Subject: [PATCH 3/3] Fix trailing docstring --- .../agentune/simulate/rag/indexing_and_retrieval.py | 1 - 1 file changed, 1 deletion(-) diff --git a/agentune_simulate/agentune/simulate/rag/indexing_and_retrieval.py b/agentune_simulate/agentune/simulate/rag/indexing_and_retrieval.py index 56c42d76..17aec66e 100644 --- a/agentune_simulate/agentune/simulate/rag/indexing_and_retrieval.py +++ b/agentune_simulate/agentune/simulate/rag/indexing_and_retrieval.py @@ -134,7 +134,6 @@ async def get_similar_finished_conversations( vector_store: The vector store to search for similar conversations conversation: Current conversation to find examples for k: Number of similar conversations to retrieve - instance: Optional instance to exclude exact matches (if provided) Returns: List of similar conversations as (Document, score) tuples, sorted by relevance