diff --git a/README.md b/README.md index a5fb0959..215609f7 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) [![Twitter Follow](https://img.shields.io/twitter/follow/agentune_sb?style=social)](https://x.com/agentune_sb) -[![Discord](https://img.shields.io/discord/1375004885845807114?color=7289da&label=discord&logo=discord&logoColor=white)](https://discord.gg/Hx5YYAaebz) +[![Discord](https://img.shields.io/badge/discord-join-blue?logo=discord&logoColor=white)](https://discord.gg/Hx5YYAaebz) --- diff --git a/agentune_simulate/README.md b/agentune_simulate/README.md index 848fb00f..4d4dc282 100644 --- a/agentune_simulate/README.md +++ b/agentune_simulate/README.md @@ -4,7 +4,7 @@ [![PyPI version](https://badge.fury.io/py/agentune-simulate.svg)](https://pypi.org/project/agentune-simulate/) [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) [![Twitter Follow](https://img.shields.io/twitter/follow/agentune_sb?style=social)](https://x.com/agentune_sb) -[![Discord](https://img.shields.io/discord/1375004885845807114?color=7289da&label=discord&logo=discord&logoColor=white)](https://discord.gg/Hx5YYAaebz) +[![Discord](https://img.shields.io/badge/discord-join-blue?logo=discord&logoColor=white)](https://discord.gg/Hx5YYAaebz) --- @@ -62,7 +62,7 @@ Running a simulation with Agentune Simulate generates realistic conversations be 1. **Quick Start** - [`getting_started.ipynb`](https://github.com/SparkBeyond/agentune/blob/main/agentune_simulate/examples/getting_started.ipynb) for a quick getting started example 2. **Production Setup** - [`persistent_storage_example.ipynb`](https://github.com/SparkBeyond/agentune/blob/main/agentune_simulate/examples/persistent_storage_example.ipynb) for a closer to real life, scalable, persistent example 3. **Validate _Your_ Data** - Adapt the 2nd example to load _your_ conversations data and validate the simulation. -Here is an example of how to load conversations from tabular data: [`load_conversations_from_csv.ipynb`](https://github.com/SparkBeyond/agentune/blob/main/agentune_simulate/examples/load_conversations_from_csv.ipynb) +Here is an example of how to load conversations from tabular data: [`loading_conversations.ipynb`](https://github.com/SparkBeyond/agentune/blob/main/agentune_simulate/examples/loading_conversations.ipynb) 4. **Connect Real Agent** - [`real_agent_integration.ipynb`](https://github.com/SparkBeyond/agentune/blob/main/agentune_simulate/examples/real_agent_integration.ipynb) for integrating your existing agent systems šŸ“§ **Need help? Have feedback?** Contact us at [agentune-dev@sparkbeyond.com](mailto:agentune-dev@sparkbeyond.com) diff --git a/agentune_simulate/agentune/simulate/models/results.py b/agentune_simulate/agentune/simulate/models/results.py index 050af1cb..b386ce2a 100644 --- a/agentune_simulate/agentune/simulate/models/results.py +++ b/agentune_simulate/agentune/simulate/models/results.py @@ -143,6 +143,15 @@ def generate_summary(self) -> str: lines.append(f"{outcome:<20} {orig_count:>3} ({orig_pct:>4.1f}%) {sim_count:>3} ({sim_pct:>4.1f}%)") + # Add row for conversations without outcome if any exist + if orig_dist.conversations_without_outcome > 0 or sim_dist.conversations_without_outcome > 0: + orig_no_outcome = orig_dist.conversations_without_outcome + sim_no_outcome = sim_dist.conversations_without_outcome + orig_no_outcome_pct = orig_dist.no_outcome_percentage + sim_no_outcome_pct = sim_dist.no_outcome_percentage + + lines.append(f"{'No outcome':<20} {orig_no_outcome:>3} ({orig_no_outcome_pct:>4.1f}%) {sim_no_outcome:>3} ({sim_no_outcome_pct:>4.1f}%)") + # Show a sample conversation if self.simulated_conversations: sample_conv = self.simulated_conversations[0].conversation diff --git a/agentune_simulate/examples/persistent_storage_example.ipynb b/agentune_simulate/examples/persistent_storage_example.ipynb index 00788966..c15adfb5 100644 --- a/agentune_simulate/examples/persistent_storage_example.ipynb +++ b/agentune_simulate/examples/persistent_storage_example.ipynb @@ -39,8 +39,8 @@ "cell_type": "code", "metadata": { "ExecuteTime": { - "end_time": "2025-07-23T07:19:52.750523Z", - "start_time": "2025-07-23T07:19:48.896707Z" + "end_time": "2025-07-23T14:03:27.619459Z", + "start_time": "2025-07-23T14:03:23.931925Z" } }, "source": [ @@ -61,8 +61,8 @@ "cell_type": "code", "metadata": { "ExecuteTime": { - "end_time": "2025-07-23T07:19:54.091900Z", - "start_time": "2025-07-23T07:19:52.853485Z" + "end_time": "2025-07-23T14:03:29.772500Z", + "start_time": "2025-07-23T14:03:27.775396Z" } }, "source": [ @@ -93,8 +93,8 @@ { "metadata": { "ExecuteTime": { - "end_time": "2025-07-23T07:19:56.049115Z", - "start_time": "2025-07-23T07:19:54.276165Z" + "end_time": "2025-07-23T14:03:38.533472Z", + "start_time": "2025-07-23T14:03:29.782917Z" } }, "cell_type": "code", @@ -122,8 +122,8 @@ { "metadata": { "ExecuteTime": { - "end_time": "2025-07-23T07:19:56.060520Z", - "start_time": "2025-07-23T07:19:56.058656Z" + "end_time": "2025-07-23T14:03:38.581039Z", + "start_time": "2025-07-23T14:03:38.577880Z" } }, "cell_type": "code", @@ -158,8 +158,8 @@ "cell_type": "code", "metadata": { "ExecuteTime": { - "end_time": "2025-07-23T07:19:57.374782Z", - "start_time": "2025-07-23T07:19:57.297830Z" + "end_time": "2025-07-23T14:03:38.905803Z", + "start_time": "2025-07-23T14:03:38.653397Z" } }, "source": [ @@ -189,8 +189,8 @@ "cell_type": "code", "metadata": { "ExecuteTime": { - "end_time": "2025-07-23T07:19:57.579752Z", - "start_time": "2025-07-23T07:19:57.566833Z" + "end_time": "2025-07-23T14:03:38.950116Z", + "start_time": "2025-07-23T14:03:38.925006Z" } }, "source": [ @@ -338,8 +338,8 @@ "cell_type": "code", "metadata": { "ExecuteTime": { - "end_time": "2025-07-23T07:19:58.769910Z", - "start_time": "2025-07-23T07:19:58.767878Z" + "end_time": "2025-07-23T14:03:38.999734Z", + "start_time": "2025-07-23T14:03:38.995622Z" } }, "source": [ @@ -386,8 +386,8 @@ { "metadata": { "ExecuteTime": { - "end_time": "2025-07-23T07:20:00.941262Z", - "start_time": "2025-07-23T07:20:00.828130Z" + "end_time": "2025-07-23T14:03:39.310715Z", + "start_time": "2025-07-23T14:03:39.044874Z" } }, "cell_type": "code", @@ -416,8 +416,8 @@ { "metadata": { "ExecuteTime": { - "end_time": "2025-07-23T07:20:04.180391Z", - "start_time": "2025-07-23T07:20:01.398536Z" + "end_time": "2025-07-23T14:03:43.677868Z", + "start_time": "2025-07-23T14:03:39.327150Z" } }, "cell_type": "code", @@ -440,7 +440,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "2025-07-23 10:20:01,411 - Anonymized telemetry enabled. See https://docs.trychroma.com/telemetry for more information.\n" + "2025-07-23 17:03:39,346 - Anonymized telemetry enabled. See https://docs.trychroma.com/telemetry for more information.\n" ] }, { @@ -462,8 +462,8 @@ "cell_type": "code", "metadata": { "ExecuteTime": { - "end_time": "2025-07-23T07:20:53.891715Z", - "start_time": "2025-07-23T07:20:04.188558Z" + "end_time": "2025-07-23T14:04:31.597482Z", + "start_time": "2025-07-23T14:03:43.699526Z" } }, "source": [ @@ -485,17 +485,16 @@ "name": "stderr", "output_type": "stream", "text": [ - "2025-07-23 10:20:04,189 - Starting intent extraction on 5 conversations\n", - "2025-07-23 10:20:07,182 - Finished extracting original intents; generated 5 scenarios\n", - "2025-07-23 10:20:07,182 - Starting conversation simulations (self.max_concurrent_conversations=20)\n", - "2025-07-23 10:20:12,184 - Progress: 0/5 scenarios completed\n", - "2025-07-23 10:20:22,187 - Progress: 1/5 scenarios completed\n", - "2025-07-23 10:20:27,188 - Progress: 2/5 scenarios completed\n", - "2025-07-23 10:20:37,191 - Progress: 3/5 scenarios completed\n", - "2025-07-23 10:20:47,193 - Progress: 5/5 scenarios completed\n", - "2025-07-23 10:20:47,194 - Finished simulating conversations; simulated 5 conversations, with 0 failures\n", - "2025-07-23 10:20:47,194 - Starting analysis of simulation results\n", - "2025-07-23 10:20:53,890 - Finished analyzing results\n" + "2025-07-23 17:03:43,702 - Starting intent extraction on 5 conversations\n", + "2025-07-23 17:03:52,403 - Finished extracting original intents; generated 5 scenarios\n", + "2025-07-23 17:03:52,404 - Starting conversation simulations (self.max_concurrent_conversations=20)\n", + "2025-07-23 17:03:57,406 - Progress: 0/5 scenarios completed\n", + "2025-07-23 17:04:07,408 - Progress: 2/5 scenarios completed\n", + "2025-07-23 17:04:12,410 - Progress: 3/5 scenarios completed\n", + "2025-07-23 17:04:22,413 - Progress: 5/5 scenarios completed\n", + "2025-07-23 17:04:22,414 - Finished simulating conversations; simulated 5 conversations, with 0 failures\n", + "2025-07-23 17:04:22,415 - Starting analysis of simulation results\n", + "2025-07-23 17:04:31,594 - Finished analyzing results\n" ] }, { @@ -512,8 +511,8 @@ "cell_type": "code", "metadata": { "ExecuteTime": { - "end_time": "2025-07-23T07:20:53.909318Z", - "start_time": "2025-07-23T07:20:53.901864Z" + "end_time": "2025-07-23T14:04:31.627201Z", + "start_time": "2025-07-23T14:04:31.614360Z" } }, "source": [ @@ -524,7 +523,7 @@ "# Save results to file using built-in method\n", "output_file = \"chroma_simulation_results.json\"\n", "chroma_result.save_to_file(output_file)\n", - "print(f\"\\nāœ“ Results saved to {Path(output_file).absolute()}\")" + "print(f\"\\nāœ“ Results saved to {Path(output_file).resolve().relative_to(Path('../../').resolve())}\")" ], "outputs": [ { @@ -541,23 +540,22 @@ "\n", "Average messages per conversation:\n", " Original: 4.4\n", - " Simulated: 4.2\n", + " Simulated: 2.8\n", "\n", "Outcome distribution comparison:\n", "Outcome Original Simulated \n", "--------------------------------------------------\n", - "resolved 4 (80.0%) 4 (80.0%)\n", - "unresolved 1 (20.0%) 0 ( 0.0%)\n", + "resolved 4 (80.0%) 3 (60.0%)\n", + "unresolved 1 (20.0%) 2 (40.0%)\n", "\n", - "Sample conversation (6 messages):\n", + "Sample conversation (4 messages):\n", " 1. customer: Last night, I waited in line for 2 hours in the business office, but because I only had a copy of my...\n", - " 2. agent: I'm sorry to hear about the inconvenience you've experienced. It sounds frustrating to have to make ...\n", - " 3. customer: I have the set-top box with me now. Can you confirm if this is all I need to proceed with the cancel...\n", - " 4. agent: Thank you for confirming that you have the set-top box with you. To proceed with the cancellation, p...\n", - " ... and 2 more messages\n", + " 2. agent: I apologize for the inconvenience and frustration you've experienced with the cancellation process. ...\n", + " 3. customer: I understand the need for the original ID and the set-top box now, but my issue is with the lack of ...\n", + " 4. agent: I completely understand your frustration, and I apologize for the inconvenience this has caused. I w...\n", "========================================\n", "\n", - "āœ“ Results saved to /Users/erik/Dev/agentune/agentune_simulate/examples/chroma_simulation_results.json\n" + "āœ“ Results saved to agentune_simulate/examples/chroma_simulation_results.json\n" ] } ],