From 7296dd468ba8e42eacb2a5e348dd4a7815963641 Mon Sep 17 00:00:00 2001 From: Ryuta KOBAYASHI Date: Sat, 20 Jun 2026 15:21:45 +0900 Subject: [PATCH 1/2] feat(model-runtime): thread invocation_context through to model runtimes `invocation_context` already existed in graphon but was only wired to the LLM observability callbacks. It was never forwarded to the model runtime (`model_runtime.invoke_*`), so per-invocation metadata (e.g. the originating Dify `app_id`) could not reach a runtime/plugin for provider-side cost attribution. This threads the existing `invocation_context: Mapping[str, object] | None` from the public `invoke()` surface down to the runtime for all six model types, and adds it to the corresponding runtime protocols: - LLM: `invoke()` -> `_invoke_llm_via_runtime()` -> `model_runtime.invoke_llm`, and to the before/after/error/stream callbacks. - text embedding, rerank, tts, speech2text, moderation: `invoke()` -> `model_runtime.invoke_*`. The parameter is keyword-only with a default of `None`, so existing callers and the standard invoke paths are unaffected. The structured-output runtime entry point is intentionally out of scope and can follow separately. Enables langgenius/dify#35859 (pass app_id to model plugins). --- .../base/large_language_model.py | 8 ++ .../model_providers/base/moderation_model.py | 10 +- .../model_providers/base/rerank_model.py | 6 + .../model_providers/base/speech2text_model.py | 10 +- .../base/text_embedding_model.py | 4 + .../model_providers/base/tts_model.py | 4 +- .../model_runtime/protocols/llm_runtime.py | 5 +- .../protocols/moderation_runtime.py | 2 + .../model_runtime/protocols/rerank_runtime.py | 3 + .../protocols/speech_to_text_runtime.py | 2 + .../protocols/text_embedding_runtime.py | 3 + .../model_runtime/protocols/tts_runtime.py | 3 +- tests/model_runtime/test_model_dispatch.py | 105 ++++++++++++++++-- 13 files changed, 152 insertions(+), 13 deletions(-) diff --git a/src/graphon/model_runtime/model_providers/base/large_language_model.py b/src/graphon/model_runtime/model_providers/base/large_language_model.py index 70e063e..c24bed5 100644 --- a/src/graphon/model_runtime/model_providers/base/large_language_model.py +++ b/src/graphon/model_runtime/model_providers/base/large_language_model.py @@ -284,6 +284,7 @@ def _invoke_llm_via_runtime( tools: list[PromptMessageTool] | None, stop: Sequence[str] | None, stream: bool, + invocation_context: Mapping[str, object] | None, ) -> LLMResult | Generator[LLMResultChunk, None, None]: return llm_model.model_runtime.invoke_llm( provider=provider, @@ -294,6 +295,7 @@ def _invoke_llm_via_runtime( tools=tools, stop=stop, stream=stream, + invocation_context=invocation_context, ) @@ -312,6 +314,7 @@ def invoke( stop: list[str] | None = None, stream: bool = True, callbacks: list[Callback] | None = None, + invocation_context: Mapping[str, object] | None = None, ) -> LLMResult | Generator[LLMResultChunk, None, None]: """Invoke the large language model and optionally stream result chunks.""" # validate and filter model parameters @@ -334,6 +337,7 @@ def invoke( tools=tools, stop=stop, stream=stream, + invocation_context=invocation_context, callbacks=callbacks, ) @@ -350,6 +354,7 @@ def invoke( tools=tools, stop=stop, stream=stream, + invocation_context=invocation_context, ) if not stream: @@ -368,6 +373,7 @@ def invoke( tools=tools, stop=stop, stream=stream, + invocation_context=invocation_context, callbacks=callbacks, ) @@ -383,6 +389,7 @@ def invoke( tools=tools, stop=stop, stream=stream, + invocation_context=invocation_context, callbacks=callbacks, ) if isinstance(result, LLMResult): @@ -395,6 +402,7 @@ def invoke( tools=tools, stop=stop, stream=stream, + invocation_context=invocation_context, callbacks=callbacks, ) # Following https://github.com/langgenius/dify/issues/17799, diff --git a/src/graphon/model_runtime/model_providers/base/moderation_model.py b/src/graphon/model_runtime/model_providers/base/moderation_model.py index afde654..c51e9cf 100644 --- a/src/graphon/model_runtime/model_providers/base/moderation_model.py +++ b/src/graphon/model_runtime/model_providers/base/moderation_model.py @@ -1,4 +1,5 @@ import time +from collections.abc import Mapping from graphon.model_runtime.entities.model_entities import ModelType from graphon.model_runtime.model_providers.base.ai_model import AIModel @@ -12,7 +13,13 @@ class ModerationModel(AIModel[ModerationModelRuntime]): model_type: ModelType = ModelType.MODERATION - def invoke(self, model: str, credentials: dict, text: str) -> bool: + def invoke( + self, + model: str, + credentials: dict, + text: str, + invocation_context: Mapping[str, object] | None = None, + ) -> bool: """Invoke the moderation model and return whether the text is unsafe.""" self.started_at = time.perf_counter() @@ -22,6 +29,7 @@ def invoke(self, model: str, credentials: dict, text: str) -> bool: model=model, credentials=credentials, text=text, + invocation_context=invocation_context, ) except Exception as e: raise self._transform_invoke_error(e) from e diff --git a/src/graphon/model_runtime/model_providers/base/rerank_model.py b/src/graphon/model_runtime/model_providers/base/rerank_model.py index f33df42..40d9b01 100644 --- a/src/graphon/model_runtime/model_providers/base/rerank_model.py +++ b/src/graphon/model_runtime/model_providers/base/rerank_model.py @@ -1,3 +1,5 @@ +from collections.abc import Mapping + from graphon.model_runtime.entities.model_entities import ModelType from graphon.model_runtime.entities.rerank_entities import ( MultimodalRerankInput, @@ -20,6 +22,7 @@ def invoke( docs: list[str], score_threshold: float | None = None, top_n: int | None = None, + invocation_context: Mapping[str, object] | None = None, ) -> RerankResult: """Invoke the rerank model for text inputs.""" try: @@ -31,6 +34,7 @@ def invoke( docs=docs, score_threshold=score_threshold, top_n=top_n, + invocation_context=invocation_context, ) except Exception as e: raise self._transform_invoke_error(e) from e @@ -43,6 +47,7 @@ def invoke_multimodal_rerank( docs: list[MultimodalRerankInput], score_threshold: float | None = None, top_n: int | None = None, + invocation_context: Mapping[str, object] | None = None, ) -> RerankResult: """Invoke the rerank model for multimodal inputs.""" try: @@ -54,6 +59,7 @@ def invoke_multimodal_rerank( docs=docs, score_threshold=score_threshold, top_n=top_n, + invocation_context=invocation_context, ) except Exception as e: raise self._transform_invoke_error(e) from e diff --git a/src/graphon/model_runtime/model_providers/base/speech2text_model.py b/src/graphon/model_runtime/model_providers/base/speech2text_model.py index 2459755..71de36b 100644 --- a/src/graphon/model_runtime/model_providers/base/speech2text_model.py +++ b/src/graphon/model_runtime/model_providers/base/speech2text_model.py @@ -1,3 +1,4 @@ +from collections.abc import Mapping from typing import IO from graphon.model_runtime.entities.model_entities import ModelType @@ -12,7 +13,13 @@ class Speech2TextModel(AIModel[SpeechToTextModelRuntime]): model_type: ModelType = ModelType.SPEECH2TEXT - def invoke(self, model: str, credentials: dict, file: IO[bytes]) -> str: + def invoke( + self, + model: str, + credentials: dict, + file: IO[bytes], + invocation_context: Mapping[str, object] | None = None, + ) -> str: """Invoke the speech-to-text model and return the transcribed text.""" try: return self.model_runtime.invoke_speech_to_text( @@ -20,6 +27,7 @@ def invoke(self, model: str, credentials: dict, file: IO[bytes]) -> str: model=model, credentials=credentials, file=file, + invocation_context=invocation_context, ) except Exception as e: raise self._transform_invoke_error(e) from e diff --git a/src/graphon/model_runtime/model_providers/base/text_embedding_model.py b/src/graphon/model_runtime/model_providers/base/text_embedding_model.py index 7f842c7..5c7ae78 100644 --- a/src/graphon/model_runtime/model_providers/base/text_embedding_model.py +++ b/src/graphon/model_runtime/model_providers/base/text_embedding_model.py @@ -1,3 +1,4 @@ +from collections.abc import Mapping from typing import Any from graphon.model_runtime.entities.model_entities import ModelPropertyKey, ModelType @@ -23,6 +24,7 @@ def invoke( texts: list[str] | None = None, multimodel_documents: list[dict[str, Any]] | None = None, input_type: EmbeddingInputType = EmbeddingInputType.DOCUMENT, + invocation_context: Mapping[str, object] | None = None, ) -> EmbeddingResult: """Invoke text or multimodal embedding generation for the provided inputs.""" if not texts and not multimodel_documents: @@ -37,6 +39,7 @@ def invoke( credentials=credentials, texts=texts, input_type=input_type, + invocation_context=invocation_context, ) except Exception as e: raise self._transform_invoke_error(e) from e @@ -52,6 +55,7 @@ def invoke( credentials=credentials, documents=multimodel_documents, input_type=input_type, + invocation_context=invocation_context, ) except Exception as e: raise self._transform_invoke_error(e) from e diff --git a/src/graphon/model_runtime/model_providers/base/tts_model.py b/src/graphon/model_runtime/model_providers/base/tts_model.py index 43ffad6..696c1cb 100644 --- a/src/graphon/model_runtime/model_providers/base/tts_model.py +++ b/src/graphon/model_runtime/model_providers/base/tts_model.py @@ -1,5 +1,5 @@ import logging -from collections.abc import Iterable +from collections.abc import Iterable, Mapping from typing import Any from graphon.model_runtime.entities.model_entities import ModelType @@ -20,6 +20,7 @@ def invoke( credentials: dict, content_text: str, voice: str, + invocation_context: Mapping[str, object] | None = None, ) -> Iterable[bytes]: """Invoke the TTS model and return an audio byte stream.""" try: @@ -29,6 +30,7 @@ def invoke( credentials=credentials, content_text=content_text, voice=voice, + invocation_context=invocation_context, ) except Exception as e: raise self._transform_invoke_error(e) from e diff --git a/src/graphon/model_runtime/protocols/llm_runtime.py b/src/graphon/model_runtime/protocols/llm_runtime.py index 7fad95c..6c58afd 100644 --- a/src/graphon/model_runtime/protocols/llm_runtime.py +++ b/src/graphon/model_runtime/protocols/llm_runtime.py @@ -1,7 +1,7 @@ from __future__ import annotations from abc import abstractmethod -from collections.abc import Generator, Sequence +from collections.abc import Generator, Mapping, Sequence from typing import Any, Literal, Protocol, overload, runtime_checkable from graphon.model_runtime.entities.llm_entities import ( @@ -34,6 +34,7 @@ def invoke_llm( tools: list[PromptMessageTool] | None, stop: Sequence[str] | None, stream: Literal[False], + invocation_context: Mapping[str, object] | None = None, ) -> LLMResult: ... @overload @@ -48,6 +49,7 @@ def invoke_llm( tools: list[PromptMessageTool] | None, stop: Sequence[str] | None, stream: Literal[True], + invocation_context: Mapping[str, object] | None = None, ) -> Generator[LLMResultChunk, None, None]: ... @abstractmethod @@ -62,6 +64,7 @@ def invoke_llm( tools: list[PromptMessageTool] | None, stop: Sequence[str] | None, stream: bool, + invocation_context: Mapping[str, object] | None = None, ) -> LLMResult | Generator[LLMResultChunk, None, None]: ... @overload diff --git a/src/graphon/model_runtime/protocols/moderation_runtime.py b/src/graphon/model_runtime/protocols/moderation_runtime.py index 15e0371..20c3c09 100644 --- a/src/graphon/model_runtime/protocols/moderation_runtime.py +++ b/src/graphon/model_runtime/protocols/moderation_runtime.py @@ -1,6 +1,7 @@ from __future__ import annotations from abc import abstractmethod +from collections.abc import Mapping from typing import Any, Protocol, runtime_checkable from graphon.model_runtime.protocols.provider_runtime import ModelProviderRuntime @@ -18,4 +19,5 @@ def invoke_moderation( model: str, credentials: dict[str, Any], text: str, + invocation_context: Mapping[str, object] | None = None, ) -> bool: ... diff --git a/src/graphon/model_runtime/protocols/rerank_runtime.py b/src/graphon/model_runtime/protocols/rerank_runtime.py index 66f70ff..60323a8 100644 --- a/src/graphon/model_runtime/protocols/rerank_runtime.py +++ b/src/graphon/model_runtime/protocols/rerank_runtime.py @@ -1,6 +1,7 @@ from __future__ import annotations from abc import abstractmethod +from collections.abc import Mapping from typing import Any, Protocol, runtime_checkable from graphon.model_runtime.entities.rerank_entities import ( @@ -25,6 +26,7 @@ def invoke_rerank( docs: list[str], score_threshold: float | None, top_n: int | None, + invocation_context: Mapping[str, object] | None = None, ) -> RerankResult: ... @abstractmethod @@ -38,4 +40,5 @@ def invoke_multimodal_rerank( docs: list[MultimodalRerankInput], score_threshold: float | None, top_n: int | None, + invocation_context: Mapping[str, object] | None = None, ) -> RerankResult: ... diff --git a/src/graphon/model_runtime/protocols/speech_to_text_runtime.py b/src/graphon/model_runtime/protocols/speech_to_text_runtime.py index ddcae2b..3a4c940 100644 --- a/src/graphon/model_runtime/protocols/speech_to_text_runtime.py +++ b/src/graphon/model_runtime/protocols/speech_to_text_runtime.py @@ -1,6 +1,7 @@ from __future__ import annotations from abc import abstractmethod +from collections.abc import Mapping from typing import IO, Any, Protocol, runtime_checkable from graphon.model_runtime.protocols.provider_runtime import ModelProviderRuntime @@ -18,4 +19,5 @@ def invoke_speech_to_text( model: str, credentials: dict[str, Any], file: IO[bytes], + invocation_context: Mapping[str, object] | None = None, ) -> str: ... diff --git a/src/graphon/model_runtime/protocols/text_embedding_runtime.py b/src/graphon/model_runtime/protocols/text_embedding_runtime.py index d94f80a..5c8a461 100644 --- a/src/graphon/model_runtime/protocols/text_embedding_runtime.py +++ b/src/graphon/model_runtime/protocols/text_embedding_runtime.py @@ -1,6 +1,7 @@ from __future__ import annotations from abc import abstractmethod +from collections.abc import Mapping from typing import Any, Protocol, runtime_checkable from graphon.model_runtime.entities.text_embedding_entities import ( @@ -23,6 +24,7 @@ def invoke_text_embedding( credentials: dict[str, Any], texts: list[str], input_type: EmbeddingInputType, + invocation_context: Mapping[str, object] | None = None, ) -> EmbeddingResult: ... @abstractmethod @@ -34,6 +36,7 @@ def invoke_multimodal_embedding( credentials: dict[str, Any], documents: list[dict[str, Any]], input_type: EmbeddingInputType, + invocation_context: Mapping[str, object] | None = None, ) -> EmbeddingResult: ... @abstractmethod diff --git a/src/graphon/model_runtime/protocols/tts_runtime.py b/src/graphon/model_runtime/protocols/tts_runtime.py index a1a2388..a7b7245 100644 --- a/src/graphon/model_runtime/protocols/tts_runtime.py +++ b/src/graphon/model_runtime/protocols/tts_runtime.py @@ -1,7 +1,7 @@ from __future__ import annotations from abc import abstractmethod -from collections.abc import Iterable +from collections.abc import Iterable, Mapping from typing import Any, Protocol, runtime_checkable from graphon.model_runtime.protocols.provider_runtime import ModelProviderRuntime @@ -20,6 +20,7 @@ def invoke_tts( credentials: dict[str, Any], content_text: str, voice: str, + invocation_context: Mapping[str, object] | None = None, ) -> Iterable[bytes]: ... @abstractmethod diff --git a/tests/model_runtime/test_model_dispatch.py b/tests/model_runtime/test_model_dispatch.py index 3e9450c..2b8cff9 100644 --- a/tests/model_runtime/test_model_dispatch.py +++ b/tests/model_runtime/test_model_dispatch.py @@ -158,8 +158,10 @@ def invoke_llm( tools: list[PromptMessageTool] | None, stop: Sequence[str] | None, stream: bool, + invocation_context: Mapping[str, object] | None = None, ) -> LLMResult | Generator[LLMResultChunk, None, None]: _ = provider, credentials, model_parameters, tools, stop, stream + _ = invocation_context return LLMResult( model=model, prompt_messages=list(prompt_messages), @@ -227,9 +229,10 @@ def invoke_llm( tools: list[PromptMessageTool] | None, stop: Sequence[str] | None, stream: bool, + invocation_context: Mapping[str, object] | None = None, ) -> Generator[LLMResultChunk, None, None]: _ = provider, model, credentials, model_parameters, prompt_messages - _ = tools, stop, stream + _ = tools, stop, stream, invocation_context yield from self._chunks if self._fail_after_chunks: msg = "stream failed" @@ -378,8 +381,9 @@ def invoke_text_embedding( credentials: dict[str, Any], texts: list[str], input_type: EmbeddingInputType, + invocation_context: Mapping[str, object] | None = None, ) -> EmbeddingResult: - _ = provider, model, credentials, texts, input_type + _ = provider, model, credentials, texts, input_type, invocation_context return EmbeddingResult( model=model, embeddings=[[0.1, 0.2]], @@ -402,8 +406,9 @@ def invoke_multimodal_embedding( credentials: dict[str, Any], documents: list[dict[str, Any]], input_type: EmbeddingInputType, + invocation_context: Mapping[str, object] | None = None, ) -> EmbeddingResult: - _ = provider, model, credentials, documents, input_type + _ = provider, model, credentials, documents, input_type, invocation_context return EmbeddingResult( model=model, embeddings=[[0.1, 0.2]], @@ -439,8 +444,9 @@ def invoke_tts( credentials: dict[str, Any], content_text: str, voice: str, + invocation_context: Mapping[str, object] | None = None, ) -> list[bytes]: - _ = provider, model, credentials, content_text, voice + _ = provider, model, credentials, content_text, voice, invocation_context return [b"audio"] def get_tts_model_voices( @@ -463,8 +469,9 @@ def invoke_moderation( model: str, credentials: dict[str, Any], text: str, + invocation_context: Mapping[str, object] | None = None, ) -> bool: - _ = provider, model, credentials, text + _ = provider, model, credentials, text, invocation_context return True @@ -479,8 +486,9 @@ def invoke_rerank( docs: list[str], score_threshold: float | None, top_n: int | None, + invocation_context: Mapping[str, object] | None = None, ) -> RerankResult: - _ = provider, credentials, query, score_threshold, top_n + _ = provider, credentials, query, score_threshold, top_n, invocation_context return RerankResult( model=model, docs=[RerankDocument(index=0, text=docs[0], score=0.9)], @@ -496,8 +504,9 @@ def invoke_multimodal_rerank( docs: list[MultimodalRerankInput], score_threshold: float | None, top_n: int | None, + invocation_context: Mapping[str, object] | None = None, ) -> RerankResult: - _ = provider, credentials, query, score_threshold, top_n + _ = provider, credentials, query, score_threshold, top_n, invocation_context return RerankResult( model=model, docs=[RerankDocument(index=0, text=docs[0]["content"], score=0.9)], @@ -512,8 +521,9 @@ def invoke_speech_to_text( model: str, credentials: dict[str, Any], file: IO[bytes], + invocation_context: Mapping[str, object] | None = None, ) -> str: - _ = provider, model, credentials, file + _ = provider, model, credentials, file, invocation_context return "transcript" @@ -841,3 +851,82 @@ def test_speech_to_text_model_accepts_speech_only_runtime_surface() -> None: ) == "transcript" ) + + +class _InvocationContextRecordingLLMRuntimeStub(_LLMRuntimeStub): + def __init__(self) -> None: + super().__init__() + self.received_invocation_contexts: list[Mapping[str, object] | None] = [] + + def invoke_llm( + self, + *, + provider: str, + model: str, + credentials: dict[str, Any], + model_parameters: dict[str, Any], + prompt_messages: Sequence[PromptMessage], + tools: list[PromptMessageTool] | None, + stop: Sequence[str] | None, + stream: bool, + invocation_context: Mapping[str, object] | None = None, + ) -> LLMResult | Generator[LLMResultChunk, None, None]: + self.received_invocation_contexts.append(invocation_context) + return super().invoke_llm( + provider=provider, + model=model, + credentials=credentials, + model_parameters=model_parameters, + prompt_messages=prompt_messages, + tools=tools, + stop=stop, + stream=stream, + invocation_context=invocation_context, + ) + + +def test_large_language_model_forwards_invocation_context_to_runtime() -> None: + provider = ProviderEntity( + provider="test-provider", + label=I18nObject(en_US="Test Provider"), + supported_model_types=[ModelType.LLM], + configurate_methods=[], + ) + stub = _InvocationContextRecordingLLMRuntimeStub() + model = LargeLanguageModel( + provider_schema=provider, + model_runtime=cast("LLMModelRuntime", stub), + ) + + model.invoke( + model="fake-chat", + credentials={}, + prompt_messages=[UserPromptMessage(content="hello")], + stream=False, + invocation_context={"app_id": "app-123"}, + ) + + assert stub.received_invocation_contexts == [{"app_id": "app-123"}] + + +def test_large_language_model_defaults_invocation_context_to_none() -> None: + provider = ProviderEntity( + provider="test-provider", + label=I18nObject(en_US="Test Provider"), + supported_model_types=[ModelType.LLM], + configurate_methods=[], + ) + stub = _InvocationContextRecordingLLMRuntimeStub() + model = LargeLanguageModel( + provider_schema=provider, + model_runtime=cast("LLMModelRuntime", stub), + ) + + model.invoke( + model="fake-chat", + credentials={}, + prompt_messages=[UserPromptMessage(content="hello")], + stream=False, + ) + + assert stub.received_invocation_contexts == [None] From f57d5257ee9369fe287718cdcc16ee474c9a02d1 Mon Sep 17 00:00:00 2001 From: WH-2099 Date: Fri, 26 Jun 2026 12:45:18 +0800 Subject: [PATCH 2/2] refactor(model-runtime): rename invocation context to request metadata --- .../model_runtime/callbacks/base_callback.py | 16 ++-- .../callbacks/logging_callback.py | 26 +++--- .../base/large_language_model.py | 51 +++++----- .../model_providers/base/moderation_model.py | 5 +- .../model_providers/base/rerank_model.py | 10 +- .../model_providers/base/speech2text_model.py | 5 +- .../base/text_embedding_model.py | 7 +- .../model_providers/base/tts_model.py | 5 +- .../model_runtime/protocols/llm_runtime.py | 6 +- .../protocols/moderation_runtime.py | 2 +- .../model_runtime/protocols/rerank_runtime.py | 4 +- .../protocols/speech_to_text_runtime.py | 2 +- .../protocols/text_embedding_runtime.py | 4 +- .../model_runtime/protocols/tts_runtime.py | 2 +- tests/model_runtime/test_model_dispatch.py | 93 +++++++++++-------- 15 files changed, 131 insertions(+), 107 deletions(-) diff --git a/src/graphon/model_runtime/callbacks/base_callback.py b/src/graphon/model_runtime/callbacks/base_callback.py index 3c05a56..8179b38 100644 --- a/src/graphon/model_runtime/callbacks/base_callback.py +++ b/src/graphon/model_runtime/callbacks/base_callback.py @@ -37,7 +37,7 @@ def on_before_invoke( stop: Sequence[str] | None = None, stream: bool = True, user: str | None = None, - invocation_context: Mapping[str, object] | None = None, + request_metadata: Mapping[str, object] | None = None, ) -> None: """Before invoke callback @@ -50,7 +50,7 @@ def on_before_invoke( :param stop: stop words :param stream: is stream response :param user: optional end-user identifier for the invocation - :param invocation_context: opaque request metadata for the current invocation + :param request_metadata: opaque metadata for the current request """ raise NotImplementedError @@ -67,7 +67,7 @@ def on_new_chunk( stop: Sequence[str] | None = None, stream: bool = True, user: str | None = None, - invocation_context: Mapping[str, object] | None = None, + request_metadata: Mapping[str, object] | None = None, ) -> None: """On new chunk callback @@ -81,7 +81,7 @@ def on_new_chunk( :param stop: stop words :param stream: is stream response :param user: optional end-user identifier for the invocation - :param invocation_context: opaque request metadata for the current invocation + :param request_metadata: opaque metadata for the current request """ raise NotImplementedError @@ -98,7 +98,7 @@ def on_after_invoke( stop: Sequence[str] | None = None, stream: bool = True, user: str | None = None, - invocation_context: Mapping[str, object] | None = None, + request_metadata: Mapping[str, object] | None = None, ) -> None: """After invoke callback @@ -112,7 +112,7 @@ def on_after_invoke( :param stop: stop words :param stream: is stream response :param user: optional end-user identifier for the invocation - :param invocation_context: opaque request metadata for the current invocation + :param request_metadata: opaque metadata for the current request """ raise NotImplementedError @@ -129,7 +129,7 @@ def on_invoke_error( stop: Sequence[str] | None = None, stream: bool = True, user: str | None = None, - invocation_context: Mapping[str, object] | None = None, + request_metadata: Mapping[str, object] | None = None, ) -> None: """Invoke error callback @@ -143,7 +143,7 @@ def on_invoke_error( :param stop: stop words :param stream: is stream response :param user: optional end-user identifier for the invocation - :param invocation_context: opaque request metadata for the current invocation + :param request_metadata: opaque metadata for the current request """ raise NotImplementedError diff --git a/src/graphon/model_runtime/callbacks/logging_callback.py b/src/graphon/model_runtime/callbacks/logging_callback.py index 005de89..28fd0bc 100644 --- a/src/graphon/model_runtime/callbacks/logging_callback.py +++ b/src/graphon/model_runtime/callbacks/logging_callback.py @@ -28,7 +28,7 @@ def on_before_invoke( stop: Sequence[str] | None = None, stream: bool = True, user: str | None = None, - invocation_context: Mapping[str, object] | None = None, + request_metadata: Mapping[str, object] | None = None, ) -> None: """Before invoke callback @@ -41,7 +41,7 @@ def on_before_invoke( :param stop: stop words :param stream: is stream response :param user: optional end-user identifier for the invocation - :param invocation_context: opaque request metadata for the current invocation + :param request_metadata: opaque metadata for the current request """ self.print_text("\n[on_llm_before_invoke]\n", color="blue") self.print_text(f"Model: {model}\n", color="blue") @@ -61,9 +61,9 @@ def on_before_invoke( if user: self.print_text(f"User: {user}\n", color="blue") - if invocation_context: + if request_metadata: self.print_text( - f"Invocation context: {dict(invocation_context)}\n", + f"Request metadata: {dict(request_metadata)}\n", color="blue", ) @@ -91,7 +91,7 @@ def on_new_chunk( stop: Sequence[str] | None = None, stream: bool = True, user: str | None = None, - invocation_context: Mapping[str, object] | None = None, + request_metadata: Mapping[str, object] | None = None, ) -> None: """On new chunk callback @@ -104,9 +104,9 @@ def on_new_chunk( :param tools: tools for tool calling :param stop: stop words :param stream: is stream response - :param invocation_context: opaque request metadata for the current invocation + :param request_metadata: opaque metadata for the current request """ - _ = user, invocation_context + _ = user, request_metadata if isinstance(chunk.delta.message.content, str): sys.stdout.write(chunk.delta.message.content) sys.stdout.flush() @@ -124,7 +124,7 @@ def on_after_invoke( stop: Sequence[str] | None = None, stream: bool = True, user: str | None = None, - invocation_context: Mapping[str, object] | None = None, + request_metadata: Mapping[str, object] | None = None, ) -> None: """After invoke callback @@ -137,9 +137,9 @@ def on_after_invoke( :param tools: tools for tool calling :param stop: stop words :param stream: is stream response - :param invocation_context: opaque request metadata for the current invocation + :param request_metadata: opaque metadata for the current request """ - _ = user, invocation_context + _ = user, request_metadata self.print_text("\n[on_llm_after_invoke]\n", color="yellow") self.print_text(f"Content: {result.message.content}\n", color="yellow") @@ -173,7 +173,7 @@ def on_invoke_error( stop: Sequence[str] | None = None, stream: bool = True, user: str | None = None, - invocation_context: Mapping[str, object] | None = None, + request_metadata: Mapping[str, object] | None = None, ) -> None: """Invoke error callback @@ -186,8 +186,8 @@ def on_invoke_error( :param tools: tools for tool calling :param stop: stop words :param stream: is stream response - :param invocation_context: opaque request metadata for the current invocation + :param request_metadata: opaque metadata for the current request """ - _ = user, invocation_context + _ = user, request_metadata self.print_text("\n[on_llm_invoke_error]\n", color="red") logger.error("LLM invoke failed: %s", ex, exc_info=ex) diff --git a/src/graphon/model_runtime/model_providers/base/large_language_model.py b/src/graphon/model_runtime/model_providers/base/large_language_model.py index c24bed5..deae9fb 100644 --- a/src/graphon/model_runtime/model_providers/base/large_language_model.py +++ b/src/graphon/model_runtime/model_providers/base/large_language_model.py @@ -284,7 +284,7 @@ def _invoke_llm_via_runtime( tools: list[PromptMessageTool] | None, stop: Sequence[str] | None, stream: bool, - invocation_context: Mapping[str, object] | None, + request_metadata: Mapping[str, object] | None, ) -> LLMResult | Generator[LLMResultChunk, None, None]: return llm_model.model_runtime.invoke_llm( provider=provider, @@ -295,7 +295,7 @@ def _invoke_llm_via_runtime( tools=tools, stop=stop, stream=stream, - invocation_context=invocation_context, + request_metadata=request_metadata, ) @@ -314,7 +314,8 @@ def invoke( stop: list[str] | None = None, stream: bool = True, callbacks: list[Callback] | None = None, - invocation_context: Mapping[str, object] | None = None, + *, + request_metadata: Mapping[str, object] | None = None, ) -> LLMResult | Generator[LLMResultChunk, None, None]: """Invoke the large language model and optionally stream result chunks.""" # validate and filter model parameters @@ -337,7 +338,7 @@ def invoke( tools=tools, stop=stop, stream=stream, - invocation_context=invocation_context, + request_metadata=request_metadata, callbacks=callbacks, ) @@ -354,7 +355,7 @@ def invoke( tools=tools, stop=stop, stream=stream, - invocation_context=invocation_context, + request_metadata=request_metadata, ) if not stream: @@ -373,7 +374,7 @@ def invoke( tools=tools, stop=stop, stream=stream, - invocation_context=invocation_context, + request_metadata=request_metadata, callbacks=callbacks, ) @@ -389,7 +390,7 @@ def invoke( tools=tools, stop=stop, stream=stream, - invocation_context=invocation_context, + request_metadata=request_metadata, callbacks=callbacks, ) if isinstance(result, LLMResult): @@ -402,7 +403,7 @@ def invoke( tools=tools, stop=stop, stream=stream, - invocation_context=invocation_context, + request_metadata=request_metadata, callbacks=callbacks, ) # Following https://github.com/langgenius/dify/issues/17799, @@ -423,7 +424,7 @@ def _invoke_result_generator( tools: list[PromptMessageTool] | None = None, stop: Sequence[str] | None = None, stream: bool = True, - invocation_context: Mapping[str, object] | None = None, + request_metadata: Mapping[str, object] | None = None, callbacks: list[Callback] | None = None, ) -> Generator[LLMResultChunk, None, None]: """Stream runtime result chunks through callbacks and bookkeeping hooks.""" @@ -441,7 +442,7 @@ def _invoke_result_generator( tools=tools, stop=stop, stream=stream, - invocation_context=invocation_context, + request_metadata=request_metadata, callbacks=callbacks, ) except Exception as e: @@ -458,7 +459,7 @@ def _invoke_result_generator( tools=tools, stop=stop, stream=stream, - invocation_context=invocation_context, + request_metadata=request_metadata, callbacks=callbacks, ) @@ -474,7 +475,7 @@ def _stream_invoke_result_chunks( tools: list[PromptMessageTool] | None, stop: Sequence[str] | None, stream: bool, - invocation_context: Mapping[str, object] | None, + request_metadata: Mapping[str, object] | None, callbacks: list[Callback], ) -> Generator[LLMResultChunk, None, None]: for chunk in result: @@ -494,7 +495,7 @@ def _stream_invoke_result_chunks( tools=tools, stop=stop, stream=stream, - invocation_context=invocation_context, + request_metadata=request_metadata, callbacks=callbacks, ) accumulator.consume(chunk) @@ -566,7 +567,7 @@ def _trigger_before_invoke_callbacks( tools: list[PromptMessageTool] | None = None, stop: Sequence[str] | None = None, stream: bool = True, - invocation_context: Mapping[str, object] | None = None, + request_metadata: Mapping[str, object] | None = None, callbacks: list[Callback] | None = None, ) -> None: """Trigger before invoke callbacks @@ -578,7 +579,7 @@ def _trigger_before_invoke_callbacks( :param tools: tools for tool calling :param stop: stop words :param stream: is stream response - :param invocation_context: opaque request metadata for the current invocation + :param request_metadata: opaque metadata for the current request :param callbacks: callbacks """ _run_callbacks( @@ -593,7 +594,7 @@ def _trigger_before_invoke_callbacks( tools=tools, stop=stop, stream=stream, - invocation_context=invocation_context, + request_metadata=request_metadata, ), ) @@ -607,7 +608,7 @@ def _trigger_new_chunk_callbacks( tools: list[PromptMessageTool] | None = None, stop: Sequence[str] | None = None, stream: bool = True, - invocation_context: Mapping[str, object] | None = None, + request_metadata: Mapping[str, object] | None = None, callbacks: list[Callback] | None = None, ) -> None: """Trigger new chunk callbacks @@ -620,7 +621,7 @@ def _trigger_new_chunk_callbacks( :param tools: tools for tool calling :param stop: stop words :param stream: is stream response - :param invocation_context: opaque request metadata for the current invocation + :param request_metadata: opaque metadata for the current request """ _run_callbacks( callbacks, @@ -635,7 +636,7 @@ def _trigger_new_chunk_callbacks( tools=tools, stop=stop, stream=stream, - invocation_context=invocation_context, + request_metadata=request_metadata, ), ) @@ -649,7 +650,7 @@ def _trigger_after_invoke_callbacks( tools: list[PromptMessageTool] | None = None, stop: Sequence[str] | None = None, stream: bool = True, - invocation_context: Mapping[str, object] | None = None, + request_metadata: Mapping[str, object] | None = None, callbacks: list[Callback] | None = None, ) -> None: """Trigger after invoke callbacks @@ -662,7 +663,7 @@ def _trigger_after_invoke_callbacks( :param tools: tools for tool calling :param stop: stop words :param stream: is stream response - :param invocation_context: opaque request metadata for the current invocation + :param request_metadata: opaque metadata for the current request :param callbacks: callbacks """ _run_callbacks( @@ -678,7 +679,7 @@ def _trigger_after_invoke_callbacks( tools=tools, stop=stop, stream=stream, - invocation_context=invocation_context, + request_metadata=request_metadata, ), ) @@ -692,7 +693,7 @@ def _trigger_invoke_error_callbacks( tools: list[PromptMessageTool] | None = None, stop: Sequence[str] | None = None, stream: bool = True, - invocation_context: Mapping[str, object] | None = None, + request_metadata: Mapping[str, object] | None = None, callbacks: list[Callback] | None = None, ) -> None: """Trigger invoke error callbacks @@ -705,7 +706,7 @@ def _trigger_invoke_error_callbacks( :param tools: tools for tool calling :param stop: stop words :param stream: is stream response - :param invocation_context: opaque request metadata for the current invocation + :param request_metadata: opaque metadata for the current request :param callbacks: callbacks """ _run_callbacks( @@ -721,6 +722,6 @@ def _trigger_invoke_error_callbacks( tools=tools, stop=stop, stream=stream, - invocation_context=invocation_context, + request_metadata=request_metadata, ), ) diff --git a/src/graphon/model_runtime/model_providers/base/moderation_model.py b/src/graphon/model_runtime/model_providers/base/moderation_model.py index c51e9cf..75f3146 100644 --- a/src/graphon/model_runtime/model_providers/base/moderation_model.py +++ b/src/graphon/model_runtime/model_providers/base/moderation_model.py @@ -18,7 +18,8 @@ def invoke( model: str, credentials: dict, text: str, - invocation_context: Mapping[str, object] | None = None, + *, + request_metadata: Mapping[str, object] | None = None, ) -> bool: """Invoke the moderation model and return whether the text is unsafe.""" self.started_at = time.perf_counter() @@ -29,7 +30,7 @@ def invoke( model=model, credentials=credentials, text=text, - invocation_context=invocation_context, + request_metadata=request_metadata, ) except Exception as e: raise self._transform_invoke_error(e) from e diff --git a/src/graphon/model_runtime/model_providers/base/rerank_model.py b/src/graphon/model_runtime/model_providers/base/rerank_model.py index 40d9b01..cefbf0c 100644 --- a/src/graphon/model_runtime/model_providers/base/rerank_model.py +++ b/src/graphon/model_runtime/model_providers/base/rerank_model.py @@ -22,7 +22,8 @@ def invoke( docs: list[str], score_threshold: float | None = None, top_n: int | None = None, - invocation_context: Mapping[str, object] | None = None, + *, + request_metadata: Mapping[str, object] | None = None, ) -> RerankResult: """Invoke the rerank model for text inputs.""" try: @@ -34,7 +35,7 @@ def invoke( docs=docs, score_threshold=score_threshold, top_n=top_n, - invocation_context=invocation_context, + request_metadata=request_metadata, ) except Exception as e: raise self._transform_invoke_error(e) from e @@ -47,7 +48,8 @@ def invoke_multimodal_rerank( docs: list[MultimodalRerankInput], score_threshold: float | None = None, top_n: int | None = None, - invocation_context: Mapping[str, object] | None = None, + *, + request_metadata: Mapping[str, object] | None = None, ) -> RerankResult: """Invoke the rerank model for multimodal inputs.""" try: @@ -59,7 +61,7 @@ def invoke_multimodal_rerank( docs=docs, score_threshold=score_threshold, top_n=top_n, - invocation_context=invocation_context, + request_metadata=request_metadata, ) except Exception as e: raise self._transform_invoke_error(e) from e diff --git a/src/graphon/model_runtime/model_providers/base/speech2text_model.py b/src/graphon/model_runtime/model_providers/base/speech2text_model.py index 71de36b..4c64aa9 100644 --- a/src/graphon/model_runtime/model_providers/base/speech2text_model.py +++ b/src/graphon/model_runtime/model_providers/base/speech2text_model.py @@ -18,7 +18,8 @@ def invoke( model: str, credentials: dict, file: IO[bytes], - invocation_context: Mapping[str, object] | None = None, + *, + request_metadata: Mapping[str, object] | None = None, ) -> str: """Invoke the speech-to-text model and return the transcribed text.""" try: @@ -27,7 +28,7 @@ def invoke( model=model, credentials=credentials, file=file, - invocation_context=invocation_context, + request_metadata=request_metadata, ) except Exception as e: raise self._transform_invoke_error(e) from e diff --git a/src/graphon/model_runtime/model_providers/base/text_embedding_model.py b/src/graphon/model_runtime/model_providers/base/text_embedding_model.py index 5c7ae78..2c3837c 100644 --- a/src/graphon/model_runtime/model_providers/base/text_embedding_model.py +++ b/src/graphon/model_runtime/model_providers/base/text_embedding_model.py @@ -24,7 +24,8 @@ def invoke( texts: list[str] | None = None, multimodel_documents: list[dict[str, Any]] | None = None, input_type: EmbeddingInputType = EmbeddingInputType.DOCUMENT, - invocation_context: Mapping[str, object] | None = None, + *, + request_metadata: Mapping[str, object] | None = None, ) -> EmbeddingResult: """Invoke text or multimodal embedding generation for the provided inputs.""" if not texts and not multimodel_documents: @@ -39,7 +40,7 @@ def invoke( credentials=credentials, texts=texts, input_type=input_type, - invocation_context=invocation_context, + request_metadata=request_metadata, ) except Exception as e: raise self._transform_invoke_error(e) from e @@ -55,7 +56,7 @@ def invoke( credentials=credentials, documents=multimodel_documents, input_type=input_type, - invocation_context=invocation_context, + request_metadata=request_metadata, ) except Exception as e: raise self._transform_invoke_error(e) from e diff --git a/src/graphon/model_runtime/model_providers/base/tts_model.py b/src/graphon/model_runtime/model_providers/base/tts_model.py index 696c1cb..695b36c 100644 --- a/src/graphon/model_runtime/model_providers/base/tts_model.py +++ b/src/graphon/model_runtime/model_providers/base/tts_model.py @@ -20,7 +20,8 @@ def invoke( credentials: dict, content_text: str, voice: str, - invocation_context: Mapping[str, object] | None = None, + *, + request_metadata: Mapping[str, object] | None = None, ) -> Iterable[bytes]: """Invoke the TTS model and return an audio byte stream.""" try: @@ -30,7 +31,7 @@ def invoke( credentials=credentials, content_text=content_text, voice=voice, - invocation_context=invocation_context, + request_metadata=request_metadata, ) except Exception as e: raise self._transform_invoke_error(e) from e diff --git a/src/graphon/model_runtime/protocols/llm_runtime.py b/src/graphon/model_runtime/protocols/llm_runtime.py index 6c58afd..c0a1403 100644 --- a/src/graphon/model_runtime/protocols/llm_runtime.py +++ b/src/graphon/model_runtime/protocols/llm_runtime.py @@ -34,7 +34,7 @@ def invoke_llm( tools: list[PromptMessageTool] | None, stop: Sequence[str] | None, stream: Literal[False], - invocation_context: Mapping[str, object] | None = None, + request_metadata: Mapping[str, object] | None = None, ) -> LLMResult: ... @overload @@ -49,7 +49,7 @@ def invoke_llm( tools: list[PromptMessageTool] | None, stop: Sequence[str] | None, stream: Literal[True], - invocation_context: Mapping[str, object] | None = None, + request_metadata: Mapping[str, object] | None = None, ) -> Generator[LLMResultChunk, None, None]: ... @abstractmethod @@ -64,7 +64,7 @@ def invoke_llm( tools: list[PromptMessageTool] | None, stop: Sequence[str] | None, stream: bool, - invocation_context: Mapping[str, object] | None = None, + request_metadata: Mapping[str, object] | None = None, ) -> LLMResult | Generator[LLMResultChunk, None, None]: ... @overload diff --git a/src/graphon/model_runtime/protocols/moderation_runtime.py b/src/graphon/model_runtime/protocols/moderation_runtime.py index 20c3c09..c3046dc 100644 --- a/src/graphon/model_runtime/protocols/moderation_runtime.py +++ b/src/graphon/model_runtime/protocols/moderation_runtime.py @@ -19,5 +19,5 @@ def invoke_moderation( model: str, credentials: dict[str, Any], text: str, - invocation_context: Mapping[str, object] | None = None, + request_metadata: Mapping[str, object] | None = None, ) -> bool: ... diff --git a/src/graphon/model_runtime/protocols/rerank_runtime.py b/src/graphon/model_runtime/protocols/rerank_runtime.py index 60323a8..91adbba 100644 --- a/src/graphon/model_runtime/protocols/rerank_runtime.py +++ b/src/graphon/model_runtime/protocols/rerank_runtime.py @@ -26,7 +26,7 @@ def invoke_rerank( docs: list[str], score_threshold: float | None, top_n: int | None, - invocation_context: Mapping[str, object] | None = None, + request_metadata: Mapping[str, object] | None = None, ) -> RerankResult: ... @abstractmethod @@ -40,5 +40,5 @@ def invoke_multimodal_rerank( docs: list[MultimodalRerankInput], score_threshold: float | None, top_n: int | None, - invocation_context: Mapping[str, object] | None = None, + request_metadata: Mapping[str, object] | None = None, ) -> RerankResult: ... diff --git a/src/graphon/model_runtime/protocols/speech_to_text_runtime.py b/src/graphon/model_runtime/protocols/speech_to_text_runtime.py index 3a4c940..d34d468 100644 --- a/src/graphon/model_runtime/protocols/speech_to_text_runtime.py +++ b/src/graphon/model_runtime/protocols/speech_to_text_runtime.py @@ -19,5 +19,5 @@ def invoke_speech_to_text( model: str, credentials: dict[str, Any], file: IO[bytes], - invocation_context: Mapping[str, object] | None = None, + request_metadata: Mapping[str, object] | None = None, ) -> str: ... diff --git a/src/graphon/model_runtime/protocols/text_embedding_runtime.py b/src/graphon/model_runtime/protocols/text_embedding_runtime.py index 5c8a461..1799795 100644 --- a/src/graphon/model_runtime/protocols/text_embedding_runtime.py +++ b/src/graphon/model_runtime/protocols/text_embedding_runtime.py @@ -24,7 +24,7 @@ def invoke_text_embedding( credentials: dict[str, Any], texts: list[str], input_type: EmbeddingInputType, - invocation_context: Mapping[str, object] | None = None, + request_metadata: Mapping[str, object] | None = None, ) -> EmbeddingResult: ... @abstractmethod @@ -36,7 +36,7 @@ def invoke_multimodal_embedding( credentials: dict[str, Any], documents: list[dict[str, Any]], input_type: EmbeddingInputType, - invocation_context: Mapping[str, object] | None = None, + request_metadata: Mapping[str, object] | None = None, ) -> EmbeddingResult: ... @abstractmethod diff --git a/src/graphon/model_runtime/protocols/tts_runtime.py b/src/graphon/model_runtime/protocols/tts_runtime.py index a7b7245..7661080 100644 --- a/src/graphon/model_runtime/protocols/tts_runtime.py +++ b/src/graphon/model_runtime/protocols/tts_runtime.py @@ -20,7 +20,7 @@ def invoke_tts( credentials: dict[str, Any], content_text: str, voice: str, - invocation_context: Mapping[str, object] | None = None, + request_metadata: Mapping[str, object] | None = None, ) -> Iterable[bytes]: ... @abstractmethod diff --git a/tests/model_runtime/test_model_dispatch.py b/tests/model_runtime/test_model_dispatch.py index 2b8cff9..38639f0 100644 --- a/tests/model_runtime/test_model_dispatch.py +++ b/tests/model_runtime/test_model_dispatch.py @@ -1,3 +1,4 @@ +import inspect from collections.abc import Generator, Mapping, Sequence from decimal import Decimal from io import BytesIO @@ -158,10 +159,10 @@ def invoke_llm( tools: list[PromptMessageTool] | None, stop: Sequence[str] | None, stream: bool, - invocation_context: Mapping[str, object] | None = None, + request_metadata: Mapping[str, object] | None = None, ) -> LLMResult | Generator[LLMResultChunk, None, None]: _ = provider, credentials, model_parameters, tools, stop, stream - _ = invocation_context + _ = request_metadata return LLMResult( model=model, prompt_messages=list(prompt_messages), @@ -229,10 +230,10 @@ def invoke_llm( tools: list[PromptMessageTool] | None, stop: Sequence[str] | None, stream: bool, - invocation_context: Mapping[str, object] | None = None, + request_metadata: Mapping[str, object] | None = None, ) -> Generator[LLMResultChunk, None, None]: _ = provider, model, credentials, model_parameters, prompt_messages - _ = tools, stop, stream, invocation_context + _ = tools, stop, stream, request_metadata yield from self._chunks if self._fail_after_chunks: msg = "stream failed" @@ -254,7 +255,7 @@ def on_before_invoke( stop: Sequence[str] | None = None, stream: bool = True, user: str | None = None, - invocation_context: Mapping[str, object] | None = None, + request_metadata: Mapping[str, object] | None = None, ) -> None: _ = ( llm_instance, @@ -266,7 +267,7 @@ def on_before_invoke( stop, stream, user, - invocation_context, + request_metadata, ) def on_new_chunk( @@ -281,7 +282,7 @@ def on_new_chunk( stop: Sequence[str] | None = None, stream: bool = True, user: str | None = None, - invocation_context: Mapping[str, object] | None = None, + request_metadata: Mapping[str, object] | None = None, ) -> None: _ = ( llm_instance, @@ -294,7 +295,7 @@ def on_new_chunk( stop, stream, user, - invocation_context, + request_metadata, ) def on_after_invoke( @@ -309,7 +310,7 @@ def on_after_invoke( stop: Sequence[str] | None = None, stream: bool = True, user: str | None = None, - invocation_context: Mapping[str, object] | None = None, + request_metadata: Mapping[str, object] | None = None, ) -> None: _ = ( llm_instance, @@ -321,7 +322,7 @@ def on_after_invoke( stop, stream, user, - invocation_context, + request_metadata, ) self.after_results.append(result) @@ -337,7 +338,7 @@ def on_invoke_error( stop: Sequence[str] | None = None, stream: bool = True, user: str | None = None, - invocation_context: Mapping[str, object] | None = None, + request_metadata: Mapping[str, object] | None = None, ) -> None: _ = ( llm_instance, @@ -350,7 +351,7 @@ def on_invoke_error( stop, stream, user, - invocation_context, + request_metadata, ) @@ -381,9 +382,9 @@ def invoke_text_embedding( credentials: dict[str, Any], texts: list[str], input_type: EmbeddingInputType, - invocation_context: Mapping[str, object] | None = None, + request_metadata: Mapping[str, object] | None = None, ) -> EmbeddingResult: - _ = provider, model, credentials, texts, input_type, invocation_context + _ = provider, model, credentials, texts, input_type, request_metadata return EmbeddingResult( model=model, embeddings=[[0.1, 0.2]], @@ -406,9 +407,9 @@ def invoke_multimodal_embedding( credentials: dict[str, Any], documents: list[dict[str, Any]], input_type: EmbeddingInputType, - invocation_context: Mapping[str, object] | None = None, + request_metadata: Mapping[str, object] | None = None, ) -> EmbeddingResult: - _ = provider, model, credentials, documents, input_type, invocation_context + _ = provider, model, credentials, documents, input_type, request_metadata return EmbeddingResult( model=model, embeddings=[[0.1, 0.2]], @@ -444,9 +445,9 @@ def invoke_tts( credentials: dict[str, Any], content_text: str, voice: str, - invocation_context: Mapping[str, object] | None = None, + request_metadata: Mapping[str, object] | None = None, ) -> list[bytes]: - _ = provider, model, credentials, content_text, voice, invocation_context + _ = provider, model, credentials, content_text, voice, request_metadata return [b"audio"] def get_tts_model_voices( @@ -469,9 +470,9 @@ def invoke_moderation( model: str, credentials: dict[str, Any], text: str, - invocation_context: Mapping[str, object] | None = None, + request_metadata: Mapping[str, object] | None = None, ) -> bool: - _ = provider, model, credentials, text, invocation_context + _ = provider, model, credentials, text, request_metadata return True @@ -486,9 +487,9 @@ def invoke_rerank( docs: list[str], score_threshold: float | None, top_n: int | None, - invocation_context: Mapping[str, object] | None = None, + request_metadata: Mapping[str, object] | None = None, ) -> RerankResult: - _ = provider, credentials, query, score_threshold, top_n, invocation_context + _ = provider, credentials, query, score_threshold, top_n, request_metadata return RerankResult( model=model, docs=[RerankDocument(index=0, text=docs[0], score=0.9)], @@ -504,9 +505,9 @@ def invoke_multimodal_rerank( docs: list[MultimodalRerankInput], score_threshold: float | None, top_n: int | None, - invocation_context: Mapping[str, object] | None = None, + request_metadata: Mapping[str, object] | None = None, ) -> RerankResult: - _ = provider, credentials, query, score_threshold, top_n, invocation_context + _ = provider, credentials, query, score_threshold, top_n, request_metadata return RerankResult( model=model, docs=[RerankDocument(index=0, text=docs[0]["content"], score=0.9)], @@ -521,9 +522,9 @@ def invoke_speech_to_text( model: str, credentials: dict[str, Any], file: IO[bytes], - invocation_context: Mapping[str, object] | None = None, + request_metadata: Mapping[str, object] | None = None, ) -> str: - _ = provider, model, credentials, file, invocation_context + _ = provider, model, credentials, file, request_metadata return "transcript" @@ -853,10 +854,10 @@ def test_speech_to_text_model_accepts_speech_only_runtime_surface() -> None: ) -class _InvocationContextRecordingLLMRuntimeStub(_LLMRuntimeStub): +class _RequestMetadataRecordingLLMRuntimeStub(_LLMRuntimeStub): def __init__(self) -> None: super().__init__() - self.received_invocation_contexts: list[Mapping[str, object] | None] = [] + self.received_request_metadata: list[Mapping[str, object] | None] = [] def invoke_llm( self, @@ -869,9 +870,9 @@ def invoke_llm( tools: list[PromptMessageTool] | None, stop: Sequence[str] | None, stream: bool, - invocation_context: Mapping[str, object] | None = None, + request_metadata: Mapping[str, object] | None = None, ) -> LLMResult | Generator[LLMResultChunk, None, None]: - self.received_invocation_contexts.append(invocation_context) + self.received_request_metadata.append(request_metadata) return super().invoke_llm( provider=provider, model=model, @@ -881,18 +882,18 @@ def invoke_llm( tools=tools, stop=stop, stream=stream, - invocation_context=invocation_context, + request_metadata=request_metadata, ) -def test_large_language_model_forwards_invocation_context_to_runtime() -> None: +def test_large_language_model_forwards_request_metadata_to_runtime() -> None: provider = ProviderEntity( provider="test-provider", label=I18nObject(en_US="Test Provider"), supported_model_types=[ModelType.LLM], configurate_methods=[], ) - stub = _InvocationContextRecordingLLMRuntimeStub() + stub = _RequestMetadataRecordingLLMRuntimeStub() model = LargeLanguageModel( provider_schema=provider, model_runtime=cast("LLMModelRuntime", stub), @@ -903,20 +904,20 @@ def test_large_language_model_forwards_invocation_context_to_runtime() -> None: credentials={}, prompt_messages=[UserPromptMessage(content="hello")], stream=False, - invocation_context={"app_id": "app-123"}, + request_metadata={"app_id": "app-123"}, ) - assert stub.received_invocation_contexts == [{"app_id": "app-123"}] + assert stub.received_request_metadata == [{"app_id": "app-123"}] -def test_large_language_model_defaults_invocation_context_to_none() -> None: +def test_large_language_model_defaults_request_metadata_to_none() -> None: provider = ProviderEntity( provider="test-provider", label=I18nObject(en_US="Test Provider"), supported_model_types=[ModelType.LLM], configurate_methods=[], ) - stub = _InvocationContextRecordingLLMRuntimeStub() + stub = _RequestMetadataRecordingLLMRuntimeStub() model = LargeLanguageModel( provider_schema=provider, model_runtime=cast("LLMModelRuntime", stub), @@ -929,4 +930,20 @@ def test_large_language_model_defaults_invocation_context_to_none() -> None: stream=False, ) - assert stub.received_invocation_contexts == [None] + assert stub.received_request_metadata == [None] + + +def test_request_metadata_is_keyword_only_on_public_model_surfaces() -> None: + surfaces = ( + LargeLanguageModel.invoke, + TextEmbeddingModel.invoke, + RerankModel.invoke, + RerankModel.invoke_multimodal_rerank, + TTSModel.invoke, + Speech2TextModel.invoke, + ModerationModel.invoke, + ) + + for surface in surfaces: + parameter = inspect.signature(surface).parameters["request_metadata"] + assert parameter.kind is inspect.Parameter.KEYWORD_ONLY