diff --git a/conformance.toml b/conformance.toml index d84fbf8..b71f1b1 100644 --- a/conformance.toml +++ b/conformance.toml @@ -32,7 +32,7 @@ [manifest] implementation = "openarmature-python" -spec_pin = "v0.84.0" +spec_pin = "v0.88.0" # Status values: # implemented — shipped behavior matches the proposal's contract @@ -855,7 +855,41 @@ status = "not-yet" # Spec v0.84.0 (proposal 0089). Embedding / rerank typed-event output # (graph-engine §6 EmbeddingEvent.output_vectors / RerankEvent. -# output_results + observability output mappings). Not-yet; -# observability fixtures 137/138 defer with it (and re-source 083/108). +# output_results + observability output mappings). [proposals."0089"] +status = "partial" +since = "0.16.0" +note = "The embedding half shipped in 0059b (#198): EmbeddingEvent.output_vectors is populated by OpenAIEmbeddingProvider and rendered as the Langfuse Embedding observation output (observability §8.4.5), sourced event-side per 0089; observability fixtures 083 (re-sourced) + 137 pass. partial because the rerank half is unshipped -- RerankEvent.output_results and the rerank observation output rendering ride the rerank capability (proposal 0060, not-yet); observability fixtures 108 + 138 defer with it." + +# Spec v0.85.0 (proposal 0090). Cohere rerank wire mapping +# (retrieval-provider §8.4, POST /v2/rerank; backs 0060's Cohere-shaped +# reference reranker). Not-yet; the rerank capability (0060) is +# unshipped, so its wire fixtures 028-031 defer with it. +[proposals."0090"] +status = "not-yet" + +# Spec v0.86.0 (proposal 0091). Cohere embeddings wire mapping +# (retrieval-provider §8.4, POST /v2/embed) extending §8.4 to both +# Cohere endpoints. Not-yet; wire fixtures 032-037 defer with it (no +# harness wire-capture primitive; the Cohere provider is unshipped). +[proposals."0091"] +status = "not-yet" + +# Spec v0.87.0 (proposal 0092). Embedding-mapping batch chunking, the +# general §8 rule (consecutive <=cap chunks, one request each, vectors +# concatenated in input order, usage summed). Not-yet; the TEI /embed +# over-cap fixture 038 defers with it (no embedding wire mapping is +# shipped). +[proposals."0092"] +status = "not-yet" + +# Spec v0.88.0 (proposal 0093). Nullable provider usage records: +# EmbeddingResponse.usage / RerankResponse.usage become record|null and +# the embedding usage emission (OTel §5.5.8 gen_ai.usage.input_tokens, +# Langfuse §8.4.5 usageDetails.input) becomes conditional. Not-yet; the +# no-usage fixtures 139-143 defer with it. Follow-up when they un-defer: +# widen the shipped EmbeddingResponse.usage to nullable + make its +# emission conditional; 0060 builds RerankResponse.usage nullable from +# the start. +[proposals."0093"] status = "not-yet" diff --git a/openarmature-spec b/openarmature-spec index 7a974e0..398cdaf 160000 --- a/openarmature-spec +++ b/openarmature-spec @@ -1 +1 @@ -Subproject commit 7a974e0b11d0a485b76a34041366612ea1e65b77 +Subproject commit 398cdaf16c88decd7df14f2216f273ec3e3069b0 diff --git a/pyproject.toml b/pyproject.toml index e4fe4b0..5753893 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -63,7 +63,7 @@ Specification = "https://github.com/LunarCommand/openarmature-spec" openarmature = "openarmature.cli:main" [tool.openarmature] -spec_version = "0.84.0" +spec_version = "0.88.0" [dependency-groups] dev = [ diff --git a/scripts/build_agents_md.py b/scripts/build_agents_md.py index dd9a8e7..54f1ae0 100644 --- a/scripts/build_agents_md.py +++ b/scripts/build_agents_md.py @@ -73,7 +73,7 @@ # Spec capability directory names under ``openarmature-spec/spec/``, # in the order they appear in the bundle's "Capability contracts" # section. The order matches the order capabilities were introduced -# (graph-engine first, prompt-management most recent) so an agent +# (graph-engine first, retrieval-provider most recent) so an agent # reading top-down sees the foundational layer before the layers # built on top. CAPABILITIES = ( @@ -82,6 +82,7 @@ "llm-provider", "observability", "prompt-management", + "retrieval-provider", ) diff --git a/src/openarmature/AGENTS.md b/src/openarmature/AGENTS.md index 5be11e3..fede9a9 100644 --- a/src/openarmature/AGENTS.md +++ b/src/openarmature/AGENTS.md @@ -1,6 +1,6 @@ # OpenArmature — Agent documentation -*This is the agent guide bundled with the openarmature Python package, version 0.15.0 (spec v0.84.0). For the full docs site see [openarmature.ai](https://openarmature.ai). For the canonical spec text see [openarmature.org/capabilities](https://openarmature.org/capabilities/). For project-specific conventions for the code you're editing, see the host project's `AGENTS.md` or `CLAUDE.md`.* +*This is the agent guide bundled with the openarmature Python package, version 0.15.0 (spec v0.88.0). For the full docs site see [openarmature.ai](https://openarmature.ai). For the canonical spec text see [openarmature.org/capabilities](https://openarmature.org/capabilities/). For project-specific conventions for the code you're editing, see the host project's `AGENTS.md` or `CLAUDE.md`.* ## TL;DR @@ -10,7 +10,7 @@ OpenArmature is a workflow framework for LLM pipelines and tool-calling agents: ## Capability contracts -_Sourced from openarmature-spec v0.84.0. Each entry below reproduces §1 (Purpose) and §2 (Concepts) of the capability's `spec.md` verbatim — including additions from accepted proposals that this Python implementation may not yet ship. For per-proposal implementation status (implemented / partial / textual-only / not-yet), see the `conformance.toml` manifest at the repo root. For the full spec text (execution model, error semantics, determinism, observer hooks, etc.) see the linked docs site._ +_Sourced from openarmature-spec v0.88.0. Each entry below reproduces §1 (Purpose) and §2 (Concepts) of the capability's `spec.md` verbatim — including additions from accepted proposals that this Python implementation may not yet ship. For per-proposal implementation status (implemented / partial / textual-only / not-yet), see the `conformance.toml` manifest at the repo root. For the full spec text (execution model, error semantics, determinism, observer hooks, etc.) see the linked docs site._ ### Capability: `graph-engine` @@ -460,6 +460,97 @@ variables. Splitting the two operations lets users: A convenience operation that combines fetch + render is permitted (see §6) but the spec treats fetch and render as separable. +### Capability: `retrieval-provider` + +#### 1. Purpose + +The retrieval-provider capability is the home for retrieval-primitive provider operations that sit +alongside LLM completion. The first protocol surface this specification defines is **embedding** — +turning a list of input strings into a list of vectors via an `EmbeddingProvider`. A sibling +`RerankProvider` protocol (§5) re-scores a list of candidate documents against a query, returning +them sorted by relevance — the second retrieval-primitive surface on the same capability. + +Retrieval-provider is a sibling capability to `llm-provider` (per proposal 0006), +not a subtype of it. Embedding and LLM completion both bind per-instance to a model identifier per +the llm-provider §5 contract, but embedding model identifiers (`text-embedding-3-small`, +`voyage-3`, `embed-multilingual-v3.0`, etc.) live in disjoint namespaces from completion model +identifiers (`gpt-4o-mini`, `claude-3-5-sonnet`, etc.). A single Provider abstraction bundling both +surfaces would either contradict the per-model-binding contract OR carve a different contract shape +for the same protocol — both bad. Separate protocols preserve per-model-binding while opening a +path to observable embedding calls. + +Retrieval-provider is one of a planned family of `-provider` capabilities (`llm-provider`, +`retrieval-provider`, plus future siblings as downstream demand surfaces). Each domain capability +covers related-shape provider operations under a narrow protocol surface; new domains land as new +capabilities rather than as extensions to existing ones. This keeps per-capability protocol surface +narrow and per-domain evolution independent. + +The substrate is intentionally narrow, matching llm-provider's posture: + +- An `EmbeddingProvider` is **stateless**. It does not cache vectors; the caller passes the full + input list on every call. +- An `EmbeddingProvider` does **not** handle retry, rate limiting, fallback, or routing. Those are + pipeline-utilities concerns and compose above the provider via middleware. +- An `EmbeddingProvider` is **bound to a single model identifier**. Switching models means + constructing a new provider, not passing a different argument per call. + +**Transparency.** Per charter §3.1 principle 8 ("Transparency over abstraction"), the embedding +abstraction surfaces a normalized shape — `EmbeddingResponse`, `EmbeddingUsage` — without hiding +what the underlying provider returned. The `EmbeddingResponse.raw` field carries the parsed provider +response verbatim alongside the normalized fields, and the error categories preserve the underlying +provider exception as cause. + +#### 2. Concepts + +**RetrievalProvider.** The umbrella term covering both `EmbeddingProvider` and `RerankProvider`. Not +a concrete protocol itself; used as the capability-level descriptor when discussing cross-protocol +concerns (observability, error semantics, per-model binding). + +**EmbeddingProvider.** An object that, given a sequence of input strings, returns a sequence of +vectors wrapped in an `EmbeddingResponse`. Bound to a specific embedding model identifier per +instance. + +**EmbeddingResponse.** The result of an `embed()` call: the vectors, the model identifier, the +parsed raw response, and — when present — usage information and the provider-returned request +identifier. + +**EmbeddingUsage.** A usage record carrying `input_tokens` only — embedding has no output tokens +(vectors aren't tokens). + +**Embedding runtime config.** A `RuntimeConfig`-shaped record (parallel to llm-provider §6) carrying +embedding-specific caller-supplied request parameters: an optional `dimensions` field (for callers +controlling output vector size on providers that support it), an optional **`input_type`** field, and +the extras-pass-through bag for vendor-specific knobs. `input_type` (`"query"` / `"document"`, an +extensible string) declares what the embedded text is *for*, so a provider bound to an asymmetric model +applies the model-appropriate query/document treatment per its §8 wire mapping; **absent ⇒ symmetric +embedding** (a symmetric model ignores it). Additional well-known values (`"classification"`, +`"clustering"`, …) MAY be recognized by mappings whose backend supports them; an unrecognized value is +`provider_invalid_request` (§7) at a mapping that declares a closed set, or passed through by a mapping +that accepts arbitrary types. Free-form per-model instruction overrides ride the extras-pass-through +bag (no second declared field). + +**RerankProvider.** An object that, given a query string and a list of candidate documents, returns +the documents sorted by query-relevance with provider-specific scores. Bound to a specific rerank +model identifier per instance. + +**RerankResponse.** The result of a `rerank()` call: the sorted scored documents, the model +identifier, the parsed raw response, and — when present — usage information and the +provider-returned response identifier. + +**RerankUsage.** A usage record with optional `search_units` and optional `input_tokens`, reflecting +the messy provider landscape where rerank pricing surfaces vary widely. + +**ScoredDocument.** A single result entry carrying the document's original index in the input list, +the provider-assigned relevance score, and (optionally) an echo of the document text. + +**Rerank runtime config.** A `RuntimeConfig`-shaped record (parallel to llm-provider §6) carrying +rerank-specific caller-supplied request parameters. Initially minimal: one declared field, +`return_documents` (boolean, default `False`), controlling whether the provider echoes document text +on each `ScoredDocument` in the response. The field name + default match the major rerank vendors' +wire-shape parameter (Voyage AI exposes `return_documents` defaulting `False`; Jina AI's wire default is `True`, so the §8.2 Jina mapping sends the OA value explicitly; Cohere's `/v2/rerank` has no `return_documents` field, so the §8.4 mapping treats the OA knob as a silent no-op); +per-vendor wire-format mappings pin the source-side translation where a vendor diverges. Plus the +extras-pass-through bag for vendor-specific knobs. + ## Patterns _Recipes that compose the primitives. Not framework contracts — these are how to do common things idiomatically._ diff --git a/src/openarmature/__init__.py b/src/openarmature/__init__.py index 851813c..a0470f8 100644 --- a/src/openarmature/__init__.py +++ b/src/openarmature/__init__.py @@ -25,7 +25,7 @@ """ __version__ = "0.15.0" -__spec_version__ = "0.84.0" +__spec_version__ = "0.88.0" # Proposal 0052 (spec observability §5.1 / §8.4.1): canonical # package-registry name for this implementation. Surfaces on every # OTel invocation span as ``openarmature.implementation.name`` and on diff --git a/tests/conformance/test_fixture_parsing.py b/tests/conformance/test_fixture_parsing.py index afa1f92..a457696 100644 --- a/tests/conformance/test_fixture_parsing.py +++ b/tests/conformance/test_fixture_parsing.py @@ -633,6 +633,17 @@ def _id(case: tuple[str, Path]) -> str: "prompt-management/036-prompt-manager-default-cache-ttl": ( "Proposal 0086 default cache_ttl_seconds; not implemented" ), + # Proposal 0093 (nullable provider usage records, v0.88.0) -- the rerank + # no-usage fixtures use the calls_rerank directive (rerank capability + # unshipped, proposal 0060), so they defer at the parse layer like + # 099-109 / 138. The embedding no-usage fixtures (139/140/143) parse via + # the modelled calls_embed directive. + "observability/141-otel-rerank-no-usage-attributes-omitted": ( + "Proposal 0093 rerank no-usage; rerank capability not implemented (0060)" + ), + "observability/142-langfuse-rerank-no-usage-usagedetails-omitted": ( + "Proposal 0093 rerank no-usage; rerank capability not implemented (0060)" + ), } diff --git a/tests/conformance/test_observability.py b/tests/conformance/test_observability.py index 7f24789..9142432 100644 --- a/tests/conformance/test_observability.py +++ b/tests/conformance/test_observability.py @@ -386,6 +386,37 @@ def _reset_otel_global_tracer_provider(restore_to: object) -> None: "136-langfuse-parallel-branches-dispatch-span": ( "Langfuse parallel-branches dispatch-span parity (proposal 0088) not-yet implemented" ), + # ---- v0.88.0 spec-pin bump (v0.84.0 -> v0.88.0): proposal 0093 + # nullable provider usage records. The no-usage rendering fixtures defer + # with the conditional-emission follow-up (widen the shipped + # EmbeddingResponse.usage to nullable + omit the usage attribute / + # usageDetails when the provider reports none). ---- + # 139/140 -- embedding no-usage (OTel span + Langfuse observation). + **{ + fixture_id: "nullable provider usage records (proposal 0093) not-yet implemented" + for fixture_id in ( + "139-otel-embedding-no-usage-input-tokens-omitted", + "140-langfuse-embedding-no-usage-usagedetails-omitted", + ) + }, + # 141/142 -- rerank no-usage; also blocked on the unshipped rerank + # capability (proposal 0060). + **{ + fixture_id: ( + "rerank no-usage rendering (proposal 0093); blocked on the unshipped " + "rerank capability (proposal 0060)" + ) + for fixture_id in ( + "141-otel-rerank-no-usage-attributes-omitted", + "142-langfuse-rerank-no-usage-usagedetails-omitted", + ) + }, + # 143 -- embedding no-usage metric; rides the §11 embedding-metrics path + # (proposal 0067), which is itself deferred (cf. 089). + "143-embedding-metrics-no-usage-no-token-observation": ( + "embedding no-usage metric (proposal 0093) rides the deferred §11 " + "embedding-metrics path (proposal 0067; cf. 089)" + ), } diff --git a/tests/conformance/test_retrieval_provider.py b/tests/conformance/test_retrieval_provider.py index 85eb943..2131d64 100644 --- a/tests/conformance/test_retrieval_provider.py +++ b/tests/conformance/test_retrieval_provider.py @@ -40,14 +40,20 @@ # Rerank fixtures (006-012) ride the RerankProvider, which lands with # proposal 0060. Deferred so a green run means "what we implement passes." # -# The v0.84.0 pin also pulls in the retrieval-provider WIRE-MAPPING fixtures -# (013-027) for proposals 0077 (TEI) / 0078 (Jina) / 0079 (OpenAI-compatible), -# none of which python has shipped. This runner drives only the bundled +# The v0.84.0 pin bump also introduced the retrieval-provider WIRE-MAPPING +# fixtures (013-027) for proposals 0077 (TEI) / 0078 (Jina) / 0079 +# (OpenAI-compatible), still present at the current pin, none of which python +# has shipped. This runner drives only the bundled # OpenAIEmbeddingProvider's embed() and asserts the response, so it cannot # exercise their request-side mappings (TEI server-side prompt_name, Jina's # task, the client-side query/document prefix) -- the contracts those # fixtures exist to verify. Defer the whole batch until each wire-mapping # impl PR (v0.16.0+). +# +# The v0.88.0 pin pulls in the Cohere wire mappings + the general embed +# batch-chunking rule: 028-031 (0090 Cohere rerank), 032-037 (0091 Cohere +# embed), 038 (0092 TEI /embed over-cap chunk-and-stitch). None are shipped +# -- same deferral rationale as 013-027. _DEFERRED_FIXTURES: dict[str, str] = { **{ p.stem: "RerankProvider not implemented (proposal 0060 not-yet)" @@ -74,6 +80,25 @@ for p in CONFORMANCE_DIR.glob("[0-9][0-9][0-9]-*.yaml") if 23 <= int(p.stem[:3]) <= 27 }, + **{ + p.stem: "Cohere rerank wire mapping (proposal 0090) not implemented" + for p in CONFORMANCE_DIR.glob("[0-9][0-9][0-9]-*.yaml") + if 28 <= int(p.stem[:3]) <= 31 + }, + **{ + p.stem: "Cohere embed wire mapping (proposal 0091) not implemented" + for p in CONFORMANCE_DIR.glob("[0-9][0-9][0-9]-*.yaml") + if 32 <= int(p.stem[:3]) <= 37 + }, + **{ + p.stem: ( + "embedding batch-chunking general rule (proposal 0092); the TEI " + "/embed over-cap chunk-and-stitch fixture rides the unshipped TEI " + "embed wire mapping (proposal 0077)" + ) + for p in CONFORMANCE_DIR.glob("[0-9][0-9][0-9]-*.yaml") + if int(p.stem[:3]) == 38 + }, } diff --git a/tests/test_smoke.py b/tests/test_smoke.py index e673231..0d7a42e 100644 --- a/tests/test_smoke.py +++ b/tests/test_smoke.py @@ -9,7 +9,7 @@ def test_package_versions() -> None: assert openarmature.__version__ == "0.15.0" - assert openarmature.__spec_version__ == "0.84.0" + assert openarmature.__spec_version__ == "0.88.0" def test_spec_version_matches_pyproject() -> None: