test: cover core utils/services + fix population-sizing bug (closes #26)

backend/app/services/archetype_library.py

@@ -618,6 +618,31 @@
 
 # ── Helper Functions ──────────────────────────────────────────────────────────
 
+def _distribute_population(total_agents: int, archetype_keys: List[str]) -> Dict[str, int]:
+    """Split a population across archetypes by ``population_pct``.
+
+    Uses the largest-remainder method so the counts sum to exactly
+    ``total_agents`` — never overshooting or producing negative counts — even
+    when the requested population is smaller than the number of archetypes.
+    """
+    if total_agents <= 0:
+        return {key: 0 for key in archetype_keys}
+
+    weights = [max(0.0, ARCHETYPE_DEFINITIONS[key]["population_pct"]) for key in archetype_keys]
+    weight_sum = sum(weights) or float(len(archetype_keys))
+
+    raw = [total_agents * w / weight_sum for w in weights]
+    counts = [int(value) for value in raw]  # floor (all values are non-negative)
+
+    # Hand the leftover units to the largest fractional remainders.
+    leftover = total_agents - sum(counts)
+    order = sorted(range(len(archetype_keys)), key=lambda i: raw[i] - counts[i], reverse=True)
+    for offset in range(leftover):
+        counts[order[offset % len(order)]] += 1
+
+    return {key: count for key, count in zip(archetype_keys, counts)}
+
+
 def expand_archetypes(
     total_agents: int,
     product_name: str = "the product",
@@ -639,19 +664,9 @@ def expand_archetypes(
     agent_id = 0
     archetype_map: List[Dict[str, Any]] = []
 
-    # Compute per-archetype counts
-    counts: Dict[str, int] = {}
-    remaining = total_agents
+    # Compute per-archetype counts that sum to exactly total_agents.
     archetype_keys = list(ARCHETYPE_DEFINITIONS.keys())
-
-    for i, key in enumerate(archetype_keys):
-        definition = ARCHETYPE_DEFINITIONS[key]
-        if i == len(archetype_keys) - 1:
-            counts[key] = remaining
-        else:
-            n = max(1, round(total_agents * definition["population_pct"]))
-            counts[key] = n
-            remaining -= n
+    counts = _distribute_population(total_agents, archetype_keys)
 
     today = datetime.now().strftime("%Y-%m-%d")
     name_counter: Dict[str, int] = {k: 0 for k in archetype_keys}

backend/tests/test_archetype_library.py

@@ -0,0 +1,118 @@
+"""Tests for the archetype library: definitions, expansion, mapping (issue #26)."""
+
+import pytest
+
+from app.services.archetype_library import (
+    ARCHETYPE_DEFINITIONS,
+    archetype_to_activity_config,
+    expand_archetypes,
+    load_archetype_map,
+    save_archetype_map,
+)
+
+_REQUIRED_KEYS = {
+    "name",
+    "population_pct",
+    "decision_trigger",
+    "activity_level",
+    "sentiment_bias",
+    "influence_weight",
+    "persona_template",
+    "interests",
+    "age_range",
+    "karma_range",
+    "follower_range",
+}
+
+
+# --- Definitions integrity ---------------------------------------------------
+
+def test_definitions_have_required_keys():
+    assert ARCHETYPE_DEFINITIONS, "archetype library should not be empty"
+    for key, defn in ARCHETYPE_DEFINITIONS.items():
+        missing = _REQUIRED_KEYS - defn.keys()
+        assert not missing, f"{key} missing keys: {missing}"
+
+
+def test_definition_ranges_are_ordered_pairs():
+    for key, defn in ARCHETYPE_DEFINITIONS.items():
+        for range_key in ("age_range", "karma_range", "follower_range"):
+            lo, hi = defn[range_key]
+            assert lo <= hi, f"{key}.{range_key} is inverted"
+
+
+def test_population_percentages_are_plausible():
+    total = sum(d["population_pct"] for d in ARCHETYPE_DEFINITIONS.values())
+    # Should roughly partition the population (allow rounding slack).
+    assert 0.8 <= total <= 1.2
+
+
+# --- expand_archetypes -------------------------------------------------------
+
+def test_expand_archetypes_produces_exact_count():
+    profiles, archetype_map = expand_archetypes(50, product_name="NoShowGuard")
+    assert len(profiles) == 50
+    assert len(archetype_map) == 50
+
+
+def test_expand_archetypes_assigns_unique_sequential_ids():
+    profiles, archetype_map = expand_archetypes(25)
+    ids = [p.user_id for p in profiles]
+    assert ids == list(range(25))
+    assert [m["agent_id"] for m in archetype_map] == list(range(25))
+
+
+def test_expand_archetypes_maps_known_archetypes():
+    _, archetype_map = expand_archetypes(30)
+    for entry in archetype_map:
+        assert entry["archetype"] in ARCHETYPE_DEFINITIONS
+
+
+def test_expand_archetypes_injects_seed_product_context():
+    seed = {"competitors": [{"name": "Calendly"}], "features": [{"name": "SMS reminders"}]}
+    profiles, _ = expand_archetypes(5, product_name="NoShowGuard", seed_data=seed)
+    assert any("Calendly" in p.persona for p in profiles)
+
+
+@pytest.mark.parametrize("total", [3, 7, 19, 25, 50, 137])
+def test_expand_honors_exact_count(total):
+    # Regression: small/odd populations must produce exactly `total` agents
+    # (no overshoot, no negative intermediate counts).
+    profiles, archetype_map = expand_archetypes(total)
+    assert len(profiles) == total
+    assert len(archetype_map) == total
+
+
+def test_distribute_population_sums_to_total():
+    from app.services.archetype_library import _distribute_population, ARCHETYPE_DEFINITIONS
+    keys = list(ARCHETYPE_DEFINITIONS.keys())
+    for total in (0, 1, 3, 19, 50, 200):
+        counts = _distribute_population(total, keys)
+        assert sum(counts.values()) == total
+        assert all(c >= 0 for c in counts.values())
+
+
+# --- activity config + map round-trip ----------------------------------------
+
+def test_archetype_to_activity_config_known_key():
+    key = next(iter(ARCHETYPE_DEFINITIONS))
+    config = archetype_to_activity_config(key, agent_id=7)
+    assert config["agent_id"] == 7
+    assert config["entity_type"] == key
+    assert "activity_level" in config and "influence_weight" in config
+
+
+def test_archetype_to_activity_config_unknown_key_returns_empty():
+    assert archetype_to_activity_config("not_a_real_archetype", agent_id=1) == {}
+
+
+def test_save_and_load_archetype_map_round_trip(tmp_path):
+    sim_dir = tmp_path / "sim"
+    sim_dir.mkdir()
+    mapping = [{"agent_id": 0, "archetype": "early_adopter"}]
+    save_archetype_map(str(sim_dir), mapping)
+    assert load_archetype_map(str(sim_dir)) == mapping
+
+
+def test_load_archetype_map_missing_returns_empty(tmp_path):
+    assert load_archetype_map(str(tmp_path)) == []

backend/tests/test_file_parser.py

@@ -0,0 +1,81 @@
+"""Tests for file parsing + chunking utilities (issue #26)."""
+
+import pytest
+
+from app.utils.file_parser import FileParser, split_text_into_chunks
+
+
+# --- split_text_into_chunks --------------------------------------------------
+
+def test_short_text_is_single_chunk():
+    assert split_text_into_chunks("hello world", chunk_size=500) == ["hello world"]
+
+
+def test_blank_text_yields_no_chunks():
+    assert split_text_into_chunks("   ", chunk_size=500) == []
+
+
+def test_long_text_splits_into_multiple_chunks():
+    text = "x" * 1200
+    chunks = split_text_into_chunks(text, chunk_size=500, overlap=50)
+    assert len(chunks) >= 3
+    assert all(chunks)
+
+
+def test_splits_prefer_sentence_boundaries():
+    # Two long sentences separated by a period+space; the first chunk should end
+    # at the sentence boundary rather than mid-word.
+    first = "a" * 400 + ". "
+    second = "b" * 400 + "."
+    chunks = split_text_into_chunks(first + second, chunk_size=500, overlap=10)
+    assert chunks[0].endswith(".")
+
+
+def test_chunks_overlap_preserves_content_order():
+    text = "".join(chr(ord("a") + (i % 26)) for i in range(1500))
+    chunks = split_text_into_chunks(text, chunk_size=400, overlap=50)
+    # Reconstructed (de-overlapped) text should still contain the start and end.
+    assert text[:10] in chunks[0]
+    assert text[-10:] in chunks[-1]
+
+
+# --- FileParser.extract_text -------------------------------------------------
+
+def test_extract_text_reads_txt(tmp_path):
+    path = tmp_path / "note.txt"
+    path.write_text("plain text body", encoding="utf-8")
+    assert FileParser.extract_text(str(path)) == "plain text body"
+
+
+def test_extract_text_reads_markdown(tmp_path):
+    path = tmp_path / "doc.md"
+    path.write_text("# Title\n\nbody", encoding="utf-8")
+    assert "Title" in FileParser.extract_text(str(path))
+
+
+def test_extract_text_missing_file_raises(tmp_path):
+    with pytest.raises(FileNotFoundError):
+        FileParser.extract_text(str(tmp_path / "nope.txt"))
+
+
+def test_extract_text_unsupported_extension_raises(tmp_path):
+    path = tmp_path / "data.csv"
+    path.write_text("a,b", encoding="utf-8")
+    with pytest.raises(ValueError):
+        FileParser.extract_text(str(path))
+
+
+def test_extract_text_handles_non_utf8_encoding(tmp_path):
+    path = tmp_path / "latin.txt"
+    path.write_bytes("café résumé".encode("latin-1"))
+    # Falls back to a detected encoding rather than raising.
+    result = FileParser.extract_text(str(path))
+    assert "caf" in result and "sum" in result
+
+
+def test_extract_from_multiple_merges_and_tolerates_failures(tmp_path):
+    good = tmp_path / "a.txt"
+    good.write_text("first doc", encoding="utf-8")
+    merged = FileParser.extract_from_multiple([str(good), str(tmp_path / "missing.txt")])
+    assert "first doc" in merged
+    assert "extraction failed" in merged  # missing file reported, not raised

backend/tests/test_report_agent_tool_parsing.py

@@ -0,0 +1,66 @@
+"""Tests for ReportAgent tool-call parsing logic (issue #26).
+
+These exercise the pure parsing/validation helpers without constructing the full
+agent (which needs LLM + Zep clients), via ``__new__``.
+"""
+
+import pytest
+
+from app.services.report_agent import ReportAgent
+
+
+@pytest.fixture
+def agent():
+    # Bare instance: __init__ (LLM/Zep wiring) is skipped; the methods under test
+    # only rely on class attributes (VALID_TOOL_NAMES) and other pure methods.
+    return ReportAgent.__new__(ReportAgent)
+
+
+# --- _is_valid_tool_call -----------------------------------------------------
+
+def test_valid_tool_call_accepts_known_name(agent):
+    assert agent._is_valid_tool_call({"name": "quick_search", "parameters": {}}) is True
+
+
+def test_valid_tool_call_rejects_unknown_name(agent):
+    assert agent._is_valid_tool_call({"name": "rm_rf_slash"}) is False
+
+
+def test_valid_tool_call_normalises_tool_and_params_keys(agent):
+    data = {"tool": "panorama_search", "params": {"q": "x"}}
+    assert agent._is_valid_tool_call(data) is True
+    # Keys are normalised in place to name/parameters.
+    assert data["name"] == "panorama_search"
+    assert data["parameters"] == {"q": "x"}
+
+
+# --- _parse_tool_calls -------------------------------------------------------
+
+def test_parse_xml_tool_call(agent):
+    response = '<tool_call>{"name": "quick_search"}</tool_call>'
+    calls = agent._parse_tool_calls(response)
+    assert calls == [{"name": "quick_search"}]
+
+
+def test_parse_bare_json_with_nested_parameters(agent):
+    response = '{"name": "insight_forge", "parameters": {"query": "pricing"}}'
+    calls = agent._parse_tool_calls(response)
+    assert len(calls) == 1
+    assert calls[0]["name"] == "insight_forge"
+    assert calls[0]["parameters"] == {"query": "pricing"}
+
+
+def test_parse_bare_json_invalid_tool_is_ignored(agent):
+    assert agent._parse_tool_calls('{"name": "danger", "parameters": {}}') == []
+
+
+def test_parse_trailing_json_after_reasoning_text(agent):
+    response = 'Let me search the graph.\n{"tool": "panorama_search", "params": {"q": 1}}'
+    calls = agent._parse_tool_calls(response)
+    assert len(calls) == 1
+    assert calls[0]["name"] == "panorama_search"
+    assert calls[0]["parameters"] == {"q": 1}
+
+
+def test_parse_plain_prose_yields_no_tool_calls(agent):
+    assert agent._parse_tool_calls("Here is the final report, no tools needed.") == []

backend/tests/test_sentiment_analyzer_cache.py

@@ -38,7 +38,8 @@ def test_sentiment_analysis_cache_hits_and_invalidates(monkeypatch, tmp_path, te
 
     def fake_classify(self, posts):
         classify_calls['count'] += 1
-        return [0.6 for _ in posts], [['features'] for _ in posts]
+        # _classify_batch returns (scores, topics, success)
+        return [0.6 for _ in posts], [['features'] for _ in posts], True
 
     monkeypatch.setattr(SentimentAnalyzer, '_classify_batch', fake_classify)
     monkeypatch.setattr(SentimentAnalyzer, '_extract_top_objections', lambda self, posts: [])

backend/tests/test_text_processor.py

@@ -0,0 +1,33 @@
+"""Tests for TextProcessor whitespace/normalisation helpers (issue #26)."""
+
+from app.services.text_processor import TextProcessor
+
+
+def test_preprocess_normalises_line_endings_and_trims():
+    text = "  hello \r\n world  \r\n\r\n\r\n end  "
+    result = TextProcessor.preprocess_text(text)
+    # CRLF/CR -> LF, per-line trim, 3+ blank lines collapsed to one blank line.
+    assert "\r" not in result
+    assert result == "hello\nworld\n\nend"
+
+
+def test_preprocess_collapses_excess_blank_lines():
+    assert TextProcessor.preprocess_text("a\n\n\n\n\nb") == "a\n\nb"
+
+
+def test_split_text_returns_whole_text_when_short():
+    assert TextProcessor.split_text("short", chunk_size=500) == ["short"]
+
+
+def test_split_text_chunks_long_text():
+    text = "word " * 400  # 2000 chars, no sentence separators
+    chunks = TextProcessor.split_text(text, chunk_size=500, overlap=50)
+    assert len(chunks) > 1
+    assert all(chunk.strip() for chunk in chunks)
+
+
+def test_get_text_stats_counts():
+    stats = TextProcessor.get_text_stats("one two\nthree")
+    assert stats["total_words"] == 3
+    assert stats["total_lines"] == 2
+    assert stats["total_chars"] == len("one two\nthree")