diff --git a/backend/app/services/archetype_library.py b/backend/app/services/archetype_library.py index 76df7fb..7b6a8b0 100644 --- a/backend/app/services/archetype_library.py +++ b/backend/app/services/archetype_library.py @@ -618,6 +618,31 @@ # ── Helper Functions ────────────────────────────────────────────────────────── +def _distribute_population(total_agents: int, archetype_keys: List[str]) -> Dict[str, int]: + """Split a population across archetypes by ``population_pct``. + + Uses the largest-remainder method so the counts sum to exactly + ``total_agents`` — never overshooting or producing negative counts — even + when the requested population is smaller than the number of archetypes. + """ + if total_agents <= 0: + return {key: 0 for key in archetype_keys} + + weights = [max(0.0, ARCHETYPE_DEFINITIONS[key]["population_pct"]) for key in archetype_keys] + weight_sum = sum(weights) or float(len(archetype_keys)) + + raw = [total_agents * w / weight_sum for w in weights] + counts = [int(value) for value in raw] # floor (all values are non-negative) + + # Hand the leftover units to the largest fractional remainders. + leftover = total_agents - sum(counts) + order = sorted(range(len(archetype_keys)), key=lambda i: raw[i] - counts[i], reverse=True) + for offset in range(leftover): + counts[order[offset % len(order)]] += 1 + + return {key: count for key, count in zip(archetype_keys, counts)} + + def expand_archetypes( total_agents: int, product_name: str = "the product", @@ -639,19 +664,9 @@ def expand_archetypes( agent_id = 0 archetype_map: List[Dict[str, Any]] = [] - # Compute per-archetype counts - counts: Dict[str, int] = {} - remaining = total_agents + # Compute per-archetype counts that sum to exactly total_agents. archetype_keys = list(ARCHETYPE_DEFINITIONS.keys()) - - for i, key in enumerate(archetype_keys): - definition = ARCHETYPE_DEFINITIONS[key] - if i == len(archetype_keys) - 1: - counts[key] = remaining - else: - n = max(1, round(total_agents * definition["population_pct"])) - counts[key] = n - remaining -= n + counts = _distribute_population(total_agents, archetype_keys) today = datetime.now().strftime("%Y-%m-%d") name_counter: Dict[str, int] = {k: 0 for k in archetype_keys} diff --git a/backend/tests/test_archetype_library.py b/backend/tests/test_archetype_library.py new file mode 100644 index 0000000..ed5a93f --- /dev/null +++ b/backend/tests/test_archetype_library.py @@ -0,0 +1,118 @@ +"""Tests for the archetype library: definitions, expansion, mapping (issue #26).""" + +import pytest + +from app.services.archetype_library import ( + ARCHETYPE_DEFINITIONS, + archetype_to_activity_config, + expand_archetypes, + load_archetype_map, + save_archetype_map, +) + +_REQUIRED_KEYS = { + "name", + "population_pct", + "decision_trigger", + "activity_level", + "sentiment_bias", + "influence_weight", + "persona_template", + "interests", + "age_range", + "karma_range", + "follower_range", +} + + +# --- Definitions integrity --------------------------------------------------- + +def test_definitions_have_required_keys(): + assert ARCHETYPE_DEFINITIONS, "archetype library should not be empty" + for key, defn in ARCHETYPE_DEFINITIONS.items(): + missing = _REQUIRED_KEYS - defn.keys() + assert not missing, f"{key} missing keys: {missing}" + + +def test_definition_ranges_are_ordered_pairs(): + for key, defn in ARCHETYPE_DEFINITIONS.items(): + for range_key in ("age_range", "karma_range", "follower_range"): + lo, hi = defn[range_key] + assert lo <= hi, f"{key}.{range_key} is inverted" + + +def test_population_percentages_are_plausible(): + total = sum(d["population_pct"] for d in ARCHETYPE_DEFINITIONS.values()) + # Should roughly partition the population (allow rounding slack). + assert 0.8 <= total <= 1.2 + + +# --- expand_archetypes ------------------------------------------------------- + +def test_expand_archetypes_produces_exact_count(): + profiles, archetype_map = expand_archetypes(50, product_name="NoShowGuard") + assert len(profiles) == 50 + assert len(archetype_map) == 50 + + +def test_expand_archetypes_assigns_unique_sequential_ids(): + profiles, archetype_map = expand_archetypes(25) + ids = [p.user_id for p in profiles] + assert ids == list(range(25)) + assert [m["agent_id"] for m in archetype_map] == list(range(25)) + + +def test_expand_archetypes_maps_known_archetypes(): + _, archetype_map = expand_archetypes(30) + for entry in archetype_map: + assert entry["archetype"] in ARCHETYPE_DEFINITIONS + + +def test_expand_archetypes_injects_seed_product_context(): + seed = {"competitors": [{"name": "Calendly"}], "features": [{"name": "SMS reminders"}]} + profiles, _ = expand_archetypes(5, product_name="NoShowGuard", seed_data=seed) + assert any("Calendly" in p.persona for p in profiles) + + +@pytest.mark.parametrize("total", [3, 7, 19, 25, 50, 137]) +def test_expand_honors_exact_count(total): + # Regression: small/odd populations must produce exactly `total` agents + # (no overshoot, no negative intermediate counts). + profiles, archetype_map = expand_archetypes(total) + assert len(profiles) == total + assert len(archetype_map) == total + + +def test_distribute_population_sums_to_total(): + from app.services.archetype_library import _distribute_population, ARCHETYPE_DEFINITIONS + keys = list(ARCHETYPE_DEFINITIONS.keys()) + for total in (0, 1, 3, 19, 50, 200): + counts = _distribute_population(total, keys) + assert sum(counts.values()) == total + assert all(c >= 0 for c in counts.values()) + + +# --- activity config + map round-trip ---------------------------------------- + +def test_archetype_to_activity_config_known_key(): + key = next(iter(ARCHETYPE_DEFINITIONS)) + config = archetype_to_activity_config(key, agent_id=7) + assert config["agent_id"] == 7 + assert config["entity_type"] == key + assert "activity_level" in config and "influence_weight" in config + + +def test_archetype_to_activity_config_unknown_key_returns_empty(): + assert archetype_to_activity_config("not_a_real_archetype", agent_id=1) == {} + + +def test_save_and_load_archetype_map_round_trip(tmp_path): + sim_dir = tmp_path / "sim" + sim_dir.mkdir() + mapping = [{"agent_id": 0, "archetype": "early_adopter"}] + save_archetype_map(str(sim_dir), mapping) + assert load_archetype_map(str(sim_dir)) == mapping + + +def test_load_archetype_map_missing_returns_empty(tmp_path): + assert load_archetype_map(str(tmp_path)) == [] diff --git a/backend/tests/test_file_parser.py b/backend/tests/test_file_parser.py new file mode 100644 index 0000000..82f08b6 --- /dev/null +++ b/backend/tests/test_file_parser.py @@ -0,0 +1,81 @@ +"""Tests for file parsing + chunking utilities (issue #26).""" + +import pytest + +from app.utils.file_parser import FileParser, split_text_into_chunks + + +# --- split_text_into_chunks -------------------------------------------------- + +def test_short_text_is_single_chunk(): + assert split_text_into_chunks("hello world", chunk_size=500) == ["hello world"] + + +def test_blank_text_yields_no_chunks(): + assert split_text_into_chunks(" ", chunk_size=500) == [] + + +def test_long_text_splits_into_multiple_chunks(): + text = "x" * 1200 + chunks = split_text_into_chunks(text, chunk_size=500, overlap=50) + assert len(chunks) >= 3 + assert all(chunks) + + +def test_splits_prefer_sentence_boundaries(): + # Two long sentences separated by a period+space; the first chunk should end + # at the sentence boundary rather than mid-word. + first = "a" * 400 + ". " + second = "b" * 400 + "." + chunks = split_text_into_chunks(first + second, chunk_size=500, overlap=10) + assert chunks[0].endswith(".") + + +def test_chunks_overlap_preserves_content_order(): + text = "".join(chr(ord("a") + (i % 26)) for i in range(1500)) + chunks = split_text_into_chunks(text, chunk_size=400, overlap=50) + # Reconstructed (de-overlapped) text should still contain the start and end. + assert text[:10] in chunks[0] + assert text[-10:] in chunks[-1] + + +# --- FileParser.extract_text ------------------------------------------------- + +def test_extract_text_reads_txt(tmp_path): + path = tmp_path / "note.txt" + path.write_text("plain text body", encoding="utf-8") + assert FileParser.extract_text(str(path)) == "plain text body" + + +def test_extract_text_reads_markdown(tmp_path): + path = tmp_path / "doc.md" + path.write_text("# Title\n\nbody", encoding="utf-8") + assert "Title" in FileParser.extract_text(str(path)) + + +def test_extract_text_missing_file_raises(tmp_path): + with pytest.raises(FileNotFoundError): + FileParser.extract_text(str(tmp_path / "nope.txt")) + + +def test_extract_text_unsupported_extension_raises(tmp_path): + path = tmp_path / "data.csv" + path.write_text("a,b", encoding="utf-8") + with pytest.raises(ValueError): + FileParser.extract_text(str(path)) + + +def test_extract_text_handles_non_utf8_encoding(tmp_path): + path = tmp_path / "latin.txt" + path.write_bytes("café résumé".encode("latin-1")) + # Falls back to a detected encoding rather than raising. + result = FileParser.extract_text(str(path)) + assert "caf" in result and "sum" in result + + +def test_extract_from_multiple_merges_and_tolerates_failures(tmp_path): + good = tmp_path / "a.txt" + good.write_text("first doc", encoding="utf-8") + merged = FileParser.extract_from_multiple([str(good), str(tmp_path / "missing.txt")]) + assert "first doc" in merged + assert "extraction failed" in merged # missing file reported, not raised diff --git a/backend/tests/test_report_agent_tool_parsing.py b/backend/tests/test_report_agent_tool_parsing.py new file mode 100644 index 0000000..5f534ed --- /dev/null +++ b/backend/tests/test_report_agent_tool_parsing.py @@ -0,0 +1,66 @@ +"""Tests for ReportAgent tool-call parsing logic (issue #26). + +These exercise the pure parsing/validation helpers without constructing the full +agent (which needs LLM + Zep clients), via ``__new__``. +""" + +import pytest + +from app.services.report_agent import ReportAgent + + +@pytest.fixture +def agent(): + # Bare instance: __init__ (LLM/Zep wiring) is skipped; the methods under test + # only rely on class attributes (VALID_TOOL_NAMES) and other pure methods. + return ReportAgent.__new__(ReportAgent) + + +# --- _is_valid_tool_call ----------------------------------------------------- + +def test_valid_tool_call_accepts_known_name(agent): + assert agent._is_valid_tool_call({"name": "quick_search", "parameters": {}}) is True + + +def test_valid_tool_call_rejects_unknown_name(agent): + assert agent._is_valid_tool_call({"name": "rm_rf_slash"}) is False + + +def test_valid_tool_call_normalises_tool_and_params_keys(agent): + data = {"tool": "panorama_search", "params": {"q": "x"}} + assert agent._is_valid_tool_call(data) is True + # Keys are normalised in place to name/parameters. + assert data["name"] == "panorama_search" + assert data["parameters"] == {"q": "x"} + + +# --- _parse_tool_calls ------------------------------------------------------- + +def test_parse_xml_tool_call(agent): + response = '{"name": "quick_search"}' + calls = agent._parse_tool_calls(response) + assert calls == [{"name": "quick_search"}] + + +def test_parse_bare_json_with_nested_parameters(agent): + response = '{"name": "insight_forge", "parameters": {"query": "pricing"}}' + calls = agent._parse_tool_calls(response) + assert len(calls) == 1 + assert calls[0]["name"] == "insight_forge" + assert calls[0]["parameters"] == {"query": "pricing"} + + +def test_parse_bare_json_invalid_tool_is_ignored(agent): + assert agent._parse_tool_calls('{"name": "danger", "parameters": {}}') == [] + + +def test_parse_trailing_json_after_reasoning_text(agent): + response = 'Let me search the graph.\n{"tool": "panorama_search", "params": {"q": 1}}' + calls = agent._parse_tool_calls(response) + assert len(calls) == 1 + assert calls[0]["name"] == "panorama_search" + assert calls[0]["parameters"] == {"q": 1} + + +def test_parse_plain_prose_yields_no_tool_calls(agent): + assert agent._parse_tool_calls("Here is the final report, no tools needed.") == [] diff --git a/backend/tests/test_sentiment_analyzer_cache.py b/backend/tests/test_sentiment_analyzer_cache.py index 259f150..7f6daa7 100644 --- a/backend/tests/test_sentiment_analyzer_cache.py +++ b/backend/tests/test_sentiment_analyzer_cache.py @@ -38,7 +38,8 @@ def test_sentiment_analysis_cache_hits_and_invalidates(monkeypatch, tmp_path, te def fake_classify(self, posts): classify_calls['count'] += 1 - return [0.6 for _ in posts], [['features'] for _ in posts] + # _classify_batch returns (scores, topics, success) + return [0.6 for _ in posts], [['features'] for _ in posts], True monkeypatch.setattr(SentimentAnalyzer, '_classify_batch', fake_classify) monkeypatch.setattr(SentimentAnalyzer, '_extract_top_objections', lambda self, posts: []) diff --git a/backend/tests/test_text_processor.py b/backend/tests/test_text_processor.py new file mode 100644 index 0000000..ff0c413 --- /dev/null +++ b/backend/tests/test_text_processor.py @@ -0,0 +1,33 @@ +"""Tests for TextProcessor whitespace/normalisation helpers (issue #26).""" + +from app.services.text_processor import TextProcessor + + +def test_preprocess_normalises_line_endings_and_trims(): + text = " hello \r\n world \r\n\r\n\r\n end " + result = TextProcessor.preprocess_text(text) + # CRLF/CR -> LF, per-line trim, 3+ blank lines collapsed to one blank line. + assert "\r" not in result + assert result == "hello\nworld\n\nend" + + +def test_preprocess_collapses_excess_blank_lines(): + assert TextProcessor.preprocess_text("a\n\n\n\n\nb") == "a\n\nb" + + +def test_split_text_returns_whole_text_when_short(): + assert TextProcessor.split_text("short", chunk_size=500) == ["short"] + + +def test_split_text_chunks_long_text(): + text = "word " * 400 # 2000 chars, no sentence separators + chunks = TextProcessor.split_text(text, chunk_size=500, overlap=50) + assert len(chunks) > 1 + assert all(chunk.strip() for chunk in chunks) + + +def test_get_text_stats_counts(): + stats = TextProcessor.get_text_stats("one two\nthree") + assert stats["total_words"] == 3 + assert stats["total_lines"] == 2 + assert stats["total_chars"] == len("one two\nthree") diff --git a/backend/uv.lock b/backend/uv.lock index f1ce4b6..0181b72 100644 --- a/backend/uv.lock +++ b/backend/uv.lock @@ -475,6 +475,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/07/6c/aa3f2f849e01cb6a001cd8554a88d4c77c5c1a31c95bdf1cf9301e6d9ef4/defusedxml-0.7.1-py2.py3-none-any.whl", hash = "sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61", size = 25604, upload-time = "2021-03-08T10:59:24.45Z" }, ] +[[package]] +name = "deprecated" +version = "1.3.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "wrapt" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/49/85/12f0a49a7c4ffb70572b6c2ef13c90c88fd190debda93b23f026b25f9634/deprecated-1.3.1.tar.gz", hash = "sha256:b1b50e0ff0c1fddaa5708a2c6b0a6588bb09b892825ab2b214ac9ea9d92a5223", size = 2932523, upload-time = "2025-10-30T08:19:02.757Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/84/d0/205d54408c08b13550c733c4b85429e7ead111c7f0014309637425520a9a/deprecated-1.3.1-py2.py3-none-any.whl", hash = "sha256:597bfef186b6f60181535a29fbe44865ce137a5079f295b479886c82729d5f3f", size = 11298, upload-time = "2025-10-30T08:19:00.758Z" }, +] + [[package]] name = "distlib" version = "0.4.0" @@ -583,6 +595,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/4f/af/72ad54402e599152de6d067324c46fe6a4f531c7c65baf7e96c63db55eaf/flask_cors-6.0.2-py3-none-any.whl", hash = "sha256:e57544d415dfd7da89a9564e1e3a9e515042df76e12130641ca6f3f2f03b699a", size = 13257, upload-time = "2025-12-12T20:31:41.3Z" }, ] +[[package]] +name = "flask-limiter" +version = "4.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "flask" }, + { name = "limits" }, + { name = "ordered-set" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/4e/98/71780be5d1afb941219c4b48d241d4e246e3062b017caa4e79c4dc71314c/flask_limiter-4.1.1.tar.gz", hash = "sha256:ca11608fc7eec43dcea606964ca07c3bd4ec1ae89043a0f67f717899a4f48106", size = 403198, upload-time = "2025-12-06T17:39:00.575Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/75/7c/9fe9ffc83be199011bb0c6deb82cdcbc5a355601e380581de9dbc30490dd/flask_limiter-4.1.1-py3-none-any.whl", hash = "sha256:e1ae13e06e6b3e39a4902e7d240b901586b25932c2add7bd5f5eeb4bdc11111b", size = 30554, upload-time = "2025-12-06T17:38:59.162Z" }, +] + [[package]] name = "fsspec" version = "2025.12.0" @@ -1012,6 +1039,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/41/a0/b91504515c1f9a299fc157967ffbd2f0321bce0516a3d5b89f6f4cad0355/lazy_object_proxy-1.12.0-pp39.pp310.pp311.graalpy311-none-any.whl", hash = "sha256:c3b2e0af1f7f77c4263759c4824316ce458fabe0fceadcd24ef8ca08b2d1e402", size = 15072, upload-time = "2025-08-22T13:50:05.498Z" }, ] +[[package]] +name = "limits" +version = "5.8.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "deprecated" }, + { name = "packaging" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/71/69/826a5d1f45426c68d8f6539f8d275c0e4fcaa57f0c017ec3100986558a41/limits-5.8.0.tar.gz", hash = "sha256:c9e0d74aed837e8f6f50d1fcebcf5fd8130957287206bc3799adaee5092655da", size = 226104, upload-time = "2026-02-05T07:17:35.859Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b9/98/cb5ca20618d205a09d5bec7591fbc4130369c7e6308d9a676a28ff3ab22c/limits-5.8.0-py3-none-any.whl", hash = "sha256:ae1b008a43eb43073c3c579398bd4eb4c795de60952532dc24720ab45e1ac6b8", size = 60954, upload-time = "2026-02-05T07:17:34.425Z" }, +] + [[package]] name = "lxml" version = "6.0.2" @@ -1248,6 +1289,7 @@ dependencies = [ { name = "charset-normalizer" }, { name = "flask" }, { name = "flask-cors" }, + { name = "flask-limiter" }, { name = "openai" }, { name = "pydantic" }, { name = "pymupdf" }, @@ -1276,6 +1318,7 @@ requires-dist = [ { name = "charset-normalizer", specifier = ">=3.0.0" }, { name = "flask", specifier = ">=3.0.0" }, { name = "flask-cors", specifier = ">=6.0.0" }, + { name = "flask-limiter", specifier = ">=3.5.0" }, { name = "openai", specifier = ">=1.0.0" }, { name = "pipreqs", marker = "extra == 'dev'", specifier = ">=0.5.0" }, { name = "pydantic", specifier = ">=2.0.0" }, @@ -1692,6 +1735,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2b/4d/e744fff95aaf3aeafc968d5ba7297c8cda0d1ecb8e3acd21b25adae4d835/openapi_spec_validator-0.7.1-py3-none-any.whl", hash = "sha256:3c81825043f24ccbcd2f4b149b11e8231abce5ba84f37065e14ec947d8f4e959", size = 38998, upload-time = "2023-10-13T11:43:38.371Z" }, ] +[[package]] +name = "ordered-set" +version = "4.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/4c/ca/bfac8bc689799bcca4157e0e0ced07e70ce125193fc2e166d2e685b7e2fe/ordered-set-4.1.0.tar.gz", hash = "sha256:694a8e44c87657c59292ede72891eb91d34131f6531463aab3009191c77364a8", size = 12826, upload-time = "2022-01-26T14:38:56.6Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/33/55/af02708f230eb77084a299d7b08175cff006dea4f2721074b92cdb0296c0/ordered_set-4.1.0-py3-none-any.whl", hash = "sha256:046e1132c71fcf3330438a539928932caf51ddbc582496833e23de611de14562", size = 7634, upload-time = "2022-01-26T14:38:48.677Z" }, +] + [[package]] name = "packaging" version = "25.0"