From 718d03a577bde4568bb9a3331489ab08a9f8768c Mon Sep 17 00:00:00 2001 From: chris-colinsky Date: Fri, 26 Jun 2026 12:45:06 -0700 Subject: [PATCH 1/2] Add OTel LLM-parent fan-out instance fallback Align OTel's _resolve_llm_parent with the spec ruling (review-nested-fan-out-lineage): an orphaned LLM span (calling node has no open span) inside a fan-out instance now parents under the per-instance fan-out dispatch span, matching the Langfuse observer, instead of falling through to the subgraph or invocation span. Top-level instance parity only. The generalized nearest-open-ancestor walk -- nested instances at any depth (where the parity key can mis-resolve to a sibling top-level instance) and the instance-vs-deeper-subgraph ordering -- rides the pending spec observability fallback clause and fixture. --- CHANGELOG.md | 1 + .../observability/otel/observer.py | 22 +++++++- tests/unit/test_observability_otel.py | 55 +++++++++++++++++++ 3 files changed, 75 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 285acaed..07001d05 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). The ### Fixed +- **OTel: an orphaned LLM span inside a fan-out instance now parents under the per-instance dispatch span** (observability §5.5). An LLM provider span whose calling node has no open span (for example a provider call originating from middleware or a wrapper) and which fires inside a fan-out instance now parents under the per-instance fan-out dispatch span, matching the Langfuse observer, instead of falling through to the subgraph or invocation span. This resolves a divergence between the two observers. The generalized nearest-open-ancestor fallback (nested instances at any depth, and the instance-vs-deeper-subgraph ordering) is pending a spec clause and fixture; this aligns the top-level instance case now. - **`current_fan_out_index()` inside fan-out instance middleware** now returns the executing instance's index (and `current_fan_out_index_chain()` its lineage) instead of `None`. The engine set the fan-out lineage ContextVars per-node, inside the inner subgraph, which left them unset in `instance_middleware` that wraps the subgraph from outside; they are now set around the instance-middleware chain. The documented `instance_middleware` use (`RetryMiddleware`) does not read the index, so no shipped behavior changes. This corrects the value seen by custom instance middleware that reads the index or calls `set_invocation_metadata`. - **Langfuse per-branch dispatch-span observation** (observability §4.3 / §8.4.2, proposals 0042 / 0044). The Langfuse observer now synthesizes a per-branch Span observation under a `parallel_branches` dispatcher node, so each branch's inner observations nest under their own branch span (a three-level dispatcher / per-branch-span / inner-nodes tree) instead of parenting directly under the dispatcher. The per-branch observation carries the OA-emitted `branch_name` alongside the caller baseline metadata and any per-branch augmentation, and the Generation observation now carries `branch_name` too. The OTel observer already produced this shape (proposal 0044 shipped OTel-only in v0.11.0); this brings the Langfuse mapping into line. Callable branches (proposal 0075) are unchanged. - **Augmentation no longer lands on a shared-parent fan-out / parallel-branches node** (observability §3.4, proposal 0045). A key set via `set_invocation_metadata` inside a fan-out instance or a parallel-branches branch was incorrectly applied to the shared fan-out / dispatcher NODE span (the fork point) when the augmenting context executed at that node's own namespace, in addition to the per-instance / per-branch dispatch span where it belongs. Both the OTel and Langfuse observers now skip the shared-parent node in that case, matching the behavior already applied to strict-ancestor shared parents. The per-instance / per-branch dispatch spans and the lineage ancestors that carry the augmentation are unaffected. diff --git a/src/openarmature/observability/otel/observer.py b/src/openarmature/observability/otel/observer.py index d0d701fd..b64f570c 100644 --- a/src/openarmature/observability/otel/observer.py +++ b/src/openarmature/observability/otel/observer.py @@ -1764,7 +1764,23 @@ def _resolve_llm_parent( calling = inv_state.open_spans.get(calling_key) if calling is not None: return set_span_in_context(calling.span) - # 2. Walk up the calling namespace prefix for a synthetic + # 2. Per-instance fan-out dispatch (spec ruling, + # review-nested-fan-out-lineage): an orphaned LLM span inside a + # fan-out instance parents under the per-instance dispatch span, not + # the subgraph / invocation span. Mirrors the LangfuseObserver + # fallback. Top-level instance ONLY (namespace[:1] + the scalar + # fan_out_index): for a NESTED fan-out instance the innermost index + # can coincide with a sibling top-level instance's, so this may + # mis-resolve to that sibling rather than miss. The nearest-open- + # ancestor-at-any-depth generalization (which also reorders this ahead + # of the subgraph walk below) fixes that and rides the spec §5.5 + # fixture. + if calling_fan_out_index is not None and calling_namespace_prefix: + instance_key = _dispatch_key(calling_namespace_prefix[:1], (calling_fan_out_index,), (None,)) + dispatch = inv_state.fan_out_instance_spans.get(instance_key) + if dispatch is not None: + return set_span_in_context(dispatch.span) + # 3. Walk up the calling namespace prefix for a synthetic # subgraph dispatch span at any ancestor — covers LLM # calls from inside subgraph wrapper middleware. for plen in range(len(calling_namespace_prefix), 0, -1): @@ -1775,12 +1791,12 @@ def _resolve_llm_parent( dr = inv_state.detached_roots.get(ancestor) if dr is not None: return set_span_in_context(dr.span) - # 3. Invocation span — ``complete()`` called outside any + # 4. Invocation span — ``complete()`` called outside any # node body but inside an ``invoke()``. inv = self._invocation_span.get(invocation_id) if inv is not None: return set_span_in_context(inv.span) - # 4. No invocation in scope — return a fresh empty Context. + # 5. No invocation in scope — return a fresh empty Context. # The span will live in its own trace. return otel_context.Context() diff --git a/tests/unit/test_observability_otel.py b/tests/unit/test_observability_otel.py index c88ddf5d..76f74574 100644 --- a/tests/unit/test_observability_otel.py +++ b/tests/unit/test_observability_otel.py @@ -559,6 +559,61 @@ async def test_active_prompt_propagates_to_llm_span_attributes() -> None: assert attrs.get("openarmature.prompt.group_name") == "classifier_chain" +async def test_llm_span_parents_under_fan_out_instance_dispatch() -> None: + # Gap 2 (review-nested-fan-out-lineage): an LLM span whose calling node has + # no open span and fires inside a top-level fan-out instance MUST parent + # under the per-instance fan-out dispatch span (matching the Langfuse + # observer), not fall through to the subgraph / invocation span. Before this + # OTel had no fan-out-instance fallback in _resolve_llm_parent. + from openarmature.observability.correlation import ( + _reset_invocation_id, + _set_invocation_id, + ) + from openarmature.observability.otel.observer import ( + _dispatch_key, + _InvState, + _OpenSpan, + ) + from tests._helpers.typed_event import make_retry_attempt_event + + exporter = InMemorySpanExporter() + observer = OTelObserver(span_processor=SimpleSpanProcessor(exporter)) + invocation_id = "inv-fanout-llm" + token = _set_invocation_id(invocation_id) + try: + # Per-instance dispatch span for top-level fan-out "fan", instance 0, + # keyed by the lineage-aware _dispatch_key. No open_spans entry for the + # calling node ("fan", "ask"), so the resolver must reach the fan-out + # dispatch fallback. + observer._inv_states[invocation_id] = _InvState() # noqa: SLF001 + dispatch_span = observer._tracer.start_span("fan") # noqa: SLF001 + instance_key = _dispatch_key(("fan",), (0,), (None,)) + observer._inv_states[invocation_id].fan_out_instance_spans[instance_key] = _OpenSpan( # noqa: SLF001 + span=dispatch_span + ) + await observer( + make_retry_attempt_event( + invocation_id=invocation_id, + node_name="ask", + namespace=("fan", "ask"), + attempt_index=0, + fan_out_index=0, + branch_name=None, + ) + ) + dispatch_span.end() + finally: + _reset_invocation_id(token) + + observer.shutdown() + llm_spans = [s for s in exporter.get_finished_spans() if s.name == "openarmature.llm.complete"] + assert len(llm_spans) == 1 + assert llm_spans[0].parent is not None, "LLM span must have a parent, not be a trace root" + assert cast("Any", llm_spans[0].parent).span_id == dispatch_span.get_span_context().span_id, ( + "LLM span must parent under the per-instance fan-out dispatch span" + ) + + async def test_llm_span_has_no_prompt_attributes_when_no_active_prompt() -> None: """Without ``with_active_prompt``, the LLM-call span MUST NOT carry ``openarmature.prompt.*`` attributes.""" From 6df29f785c13d73115fc24df88d39e33573ac74f Mon Sep 17 00:00:00 2001 From: chris-colinsky Date: Fri, 26 Jun 2026 13:02:29 -0700 Subject: [PATCH 2/2] Update resolver docstring, drop test double-end PR review follow-ups: - _resolve_llm_parent docstring now lists the per-instance fan-out dispatch fallback added in this PR. - Drop the manual dispatch_span.end() in the new test: shutdown() drains fan_out_instance_spans and ends the span, so the manual end double-ended it. --- src/openarmature/observability/otel/observer.py | 4 ++-- tests/unit/test_observability_otel.py | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/openarmature/observability/otel/observer.py b/src/openarmature/observability/otel/observer.py index b64f570c..5a5b89b6 100644 --- a/src/openarmature/observability/otel/observer.py +++ b/src/openarmature/observability/otel/observer.py @@ -1752,8 +1752,8 @@ def _resolve_llm_parent( calling_branch_name: str | None, ) -> object: """Look up the calling node's span using the calling-node - identity, fall back through subgraph dispatch / invocation - span.""" + identity, falling back through the per-instance fan-out dispatch + span, subgraph dispatch / detached root, and the invocation span.""" # 1. Direct match on the calling node's ``_StackKey``. calling_key: _StackKey = ( calling_namespace_prefix, diff --git a/tests/unit/test_observability_otel.py b/tests/unit/test_observability_otel.py index 76f74574..370bafdd 100644 --- a/tests/unit/test_observability_otel.py +++ b/tests/unit/test_observability_otel.py @@ -601,7 +601,8 @@ async def test_llm_span_parents_under_fan_out_instance_dispatch() -> None: branch_name=None, ) ) - dispatch_span.end() + # dispatch_span is ended by observer.shutdown() below (it drains + # fan_out_instance_spans); ending it here too would double-end it. finally: _reset_invocation_id(token)