Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 13 additions & 13 deletions sdks/python/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ Python SDK for Learning Commons educational text evaluators.
- **`schemas/`** – Pydantic schemas for inputs, outputs, config, metadata, and errors.
- **`providers/`** – LangChain-based LLM provider factory (OpenAI, Google, Anthropic).
- **`settings/`** – TOML settings loader for evaluator configuration.
- **`config.py`** – Re-exports config types (`EvaluatorConfig`, `PromptProviderConfig`, factory functions).
- **`config.py`** – Re-exports config types (`EvaluatorConfig`, `LLMProviderConfig`, factory functions).
- **`errors.py`** – Re-exports error types.
- **`logger.py`** – Standard Python logging utilities.

Expand Down Expand Up @@ -138,7 +138,7 @@ import logging
from learning_commons_evaluators import (
ConventionalityEvaluator,
ConventionalityEvaluationInput,
GooglePromptProviderConfig,
GoogleLLMProviderConfig,
create_config,
)

Expand All @@ -147,7 +147,7 @@ logging.basicConfig(level=logging.INFO)

# Create config with provider credentials
config = create_config(
google_prompt_provider_config=GooglePromptProviderConfig(api_key="your-google-key"),
google_llm_provider_config=GoogleLLMProviderConfig(api_key="your-google-key"),
telemetry_partner_id="your-telemetry-id",
)

Expand Down Expand Up @@ -175,12 +175,12 @@ Evaluates text for conventionality of language (idioms, metaphors, implied meani
from learning_commons_evaluators import (
ConventionalityEvaluator,
ConventionalityEvaluationInput,
GooglePromptProviderConfig,
GoogleLLMProviderConfig,
create_config,
)

config = create_config(
google_prompt_provider_config=GooglePromptProviderConfig(api_key="..."),
google_llm_provider_config=GoogleLLMProviderConfig(api_key="..."),
telemetry_partner_id="your-telemetry-id",
)
evaluator = ConventionalityEvaluator(config)
Expand All @@ -205,14 +205,14 @@ Each LLM provider requires its own config with an API key:

```python
from learning_commons_evaluators import (
GooglePromptProviderConfig,
OpenAIPromptProviderConfig,
AnthropicPromptProviderConfig,
GoogleLLMProviderConfig,
OpenAILLMProviderConfig,
AnthropicLLMProviderConfig,
)

google_config = GooglePromptProviderConfig(api_key="...")
openai_config = OpenAIPromptProviderConfig(api_key="...")
anthropic_config = AnthropicPromptProviderConfig(api_key="...")
google_config = GoogleLLMProviderConfig(api_key="...")
openai_config = OpenAILLMProviderConfig(api_key="...")
anthropic_config = AnthropicLLMProviderConfig(api_key="...")
```

### EvaluatorConfig
Expand All @@ -223,8 +223,8 @@ Use `create_config()` to create an `EvaluatorConfig`:
from learning_commons_evaluators import create_config

config = create_config(
google_prompt_provider_config=google_config,
openai_prompt_provider_config=openai_config,
google_llm_provider_config=google_config,
openai_llm_provider_config=openai_config,
telemetry_partner_id="your-telemetry-id",
logger=my_logger, # Optional: any standard logging.Logger (default: package logger)
)
Expand Down
20 changes: 10 additions & 10 deletions sdks/python/src/learning_commons_evaluators/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,11 @@

# Config
from learning_commons_evaluators.config import (
AnthropicPromptProviderConfig,
AnthropicLLMProviderConfig,
EvaluatorConfig,
GooglePromptProviderConfig,
OpenAIPromptProviderConfig,
PromptProviderConfig,
GoogleLLMProviderConfig,
LLMProviderConfig,
OpenAILLMProviderConfig,
PromptSettings,
TelemetryConfig,
create_config,
Expand Down Expand Up @@ -55,7 +55,7 @@
GradeInputField,
TextInputField,
)
from learning_commons_evaluators.schemas.config import EvaluationSettings, LlmProvider
from learning_commons_evaluators.schemas.config import EvaluationSettings, LLMProvider
from learning_commons_evaluators.schemas.conventionality import (
ConventionalityEvaluationSettings,
ConventionalityOutput,
Expand Down Expand Up @@ -95,7 +95,7 @@
"__description__",
"__version__",
"APIError",
"AnthropicPromptProviderConfig",
"AnthropicLLMProviderConfig",
"AuthenticationError",
"BaseEvaluator",
"ConfigurationError",
Expand All @@ -115,20 +115,20 @@
"EvaluatorMetadata",
"EvaluatorRetryableError",
"EvaluatorTimeoutError",
"GooglePromptProviderConfig",
"GoogleLLMProviderConfig",
"AnyInputSpec",
"GradeInputField",
"GradeInputSpec",
"InputField",
"InputSpec",
"InputT",
"TextInputSpec",
"LlmProvider",
"LLMProvider",
"LLMProviderConfig",
"Logger",
"NetworkError",
"OpenAIPromptProviderConfig",
"OpenAILLMProviderConfig",
"OutputT",
"PromptProviderConfig",
"PromptSettings",
"RateLimitError",
"SDK_LOGGER_NAME",
Expand Down
16 changes: 8 additions & 8 deletions sdks/python/src/learning_commons_evaluators/config.py
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
"""Re-export evaluator config from schemas.config for package-level imports."""

from learning_commons_evaluators.schemas.config import (
AnthropicPromptProviderConfig,
AnthropicLLMProviderConfig,
EvaluatorConfig,
GooglePromptProviderConfig,
OpenAIPromptProviderConfig,
PromptProviderConfig,
GoogleLLMProviderConfig,
LLMProviderConfig,
OpenAILLMProviderConfig,
PromptSettings,
TelemetryConfig,
create_config,
Expand All @@ -14,11 +14,11 @@
)

__all__ = [
"AnthropicPromptProviderConfig",
"AnthropicLLMProviderConfig",
"EvaluatorConfig",
"GooglePromptProviderConfig",
"OpenAIPromptProviderConfig",
"PromptProviderConfig",
"GoogleLLMProviderConfig",
"OpenAILLMProviderConfig",
"LLMProviderConfig",
"PromptSettings",
"TelemetryConfig",
"create_config",
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
"""Evaluator implementations.

Config types (``EvaluatorConfig``, ``GooglePromptProviderConfig``, etc.) are
Config types (``EvaluatorConfig``, ``GoogleLLMProviderConfig``, etc.) are
exported from the top-level :mod:`learning_commons_evaluators` package, not
from here. Import evaluator classes directly from this sub-package only when
you want to be explicit about the source.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,8 @@

create_provider() returns a langchain_core BaseChatModel (OpenAI, Google, Anthropic).

Config types (PromptProviderConfig and provider-specific configs:
AnthropicPromptProviderConfig, GooglePromptProviderConfig, OpenAIPromptProviderConfig)
Config types (LLMProviderConfig and provider-specific configs:
AnthropicLLMProviderConfig, GoogleLLMProviderConfig, OpenAILLMProviderConfig)
live in learning_commons_evaluators.schemas.config.
"""

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
from learning_commons_evaluators.errors import ConfigurationError
from learning_commons_evaluators.schemas.config import (
EvaluatorConfig,
LlmProvider,
LLMProvider,
PromptSettings,
)
from learning_commons_evaluators.schemas.metadata import TokenUsage
Expand Down Expand Up @@ -48,7 +48,7 @@ def _create_openai_llm(
) -> BaseChatModel:
from langchain_openai import ChatOpenAI

openai_config = evaluator_config.openai_prompt_provider_config
openai_config = evaluator_config.openai_llm_provider_config
if openai_config is None:
raise ConfigurationError("OpenAI provider config is not set on EvaluatorConfig")
kwargs: dict[str, Any] = {
Expand All @@ -64,7 +64,7 @@ def _create_google_llm(
) -> BaseChatModel:
from langchain_google_genai import ChatGoogleGenerativeAI

google_config = evaluator_config.google_prompt_provider_config
google_config = evaluator_config.google_llm_provider_config
if google_config is None:
raise ConfigurationError("Google provider config is not set on EvaluatorConfig")
return ChatGoogleGenerativeAI(
Expand All @@ -79,7 +79,7 @@ def _create_anthropic_llm(
) -> BaseChatModel:
from langchain_anthropic import ChatAnthropic

anthropic_config = evaluator_config.anthropic_prompt_provider_config
anthropic_config = evaluator_config.anthropic_llm_provider_config
if anthropic_config is None:
raise ConfigurationError("Anthropic provider config is not set on EvaluatorConfig")
return ChatAnthropic(
Expand All @@ -93,11 +93,11 @@ def create_provider(
prompt_settings: PromptSettings, evaluator_config: EvaluatorConfig
) -> BaseChatModel:
"""Create a LangChain chat model from a PromptSettings for use in a chain."""
if prompt_settings.provider_type == LlmProvider.OPENAI:
if prompt_settings.provider_type == LLMProvider.OPENAI:
return _create_openai_llm(prompt_settings, evaluator_config)
if prompt_settings.provider_type == LlmProvider.GOOGLE:
if prompt_settings.provider_type == LLMProvider.GOOGLE:
return _create_google_llm(prompt_settings, evaluator_config)
if prompt_settings.provider_type == LlmProvider.ANTHROPIC:
if prompt_settings.provider_type == LLMProvider.ANTHROPIC:
return _create_anthropic_llm(prompt_settings, evaluator_config)
raise ConfigurationError(
f"Unsupported LLM provider type: {prompt_settings.provider_type!r}. "
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
)
from learning_commons_evaluators.schemas.config import (
EvaluationSettings,
LlmProvider,
LLMProvider,
PromptSettings,
)
from learning_commons_evaluators.schemas.conventionality import (
Expand Down Expand Up @@ -61,7 +61,7 @@
"GradeInputField",
"InputField",
"InputMetadata",
"LlmProvider",
"LLMProvider",
"PromptSettings",
"PROMPT_STEP_EXTRA_PROMPT_SETTINGS",
"PROMPT_STEP_EXTRA_TOKEN_USAGE",
Expand Down
Loading