diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 0000000..74bb0c0 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,42 @@ +--- +name: Bug report +about: Create a report to help us improve Ruby Skill Bench +title: '[BUG] ' +labels: bug +--- + +## Description + +A clear and concise description of what the bug is. + +## Steps to Reproduce + +1. Go to '...' +2. Run command '...' +3. See error + +## Expected Behavior + +A clear and concise description of what you expected to happen. + +## Actual Behavior + +A clear and concise description of what actually happened. + +## Environment + +- Ruby version: +- Ruby Skill Bench version: +- Operating system: + +## Logs / Error Messages + +Please paste any error messages and stack traces below. If the output is long, consider using a `
` block. + +```text +Paste logs here +``` + +## Additional Context + +Add any other context about the problem here. diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 0000000..a65c24b --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,5 @@ +blank_issues_enabled: false +contact_links: + - name: Questions and discussions + url: https://github.com/igmarin/ruby-skill-bench/discussions + about: Please use GitHub Discussions for questions, support, or general discussion. diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 0000000..2525e4d --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,22 @@ +--- +name: Feature request +about: Suggest an idea for Ruby Skill Bench +title: '[FEATURE] ' +labels: enhancement +--- + +## Problem / Motivation + +A clear and concise description of the problem or limitation you're facing. Ex. I'm always frustrated when [...] + +## Proposed Solution + +A clear and concise description of what you want to happen. + +## Alternatives Considered + +A clear and concise description of any alternative solutions or features you've considered. + +## Additional Context + +Add any other context, screenshots, or examples about the feature request here. diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000..4f9ea34 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,25 @@ +## Summary + +Brief description of the changes in this pull request. + +## Related Issue + +Closes # + +## Type of Change + +- [ ] Bug fix +- [ ] New feature +- [ ] Breaking change +- [ ] Documentation update +- [ ] Refactoring / code cleanup + +## Checklist + +- [ ] Tests added or updated +- [ ] `bundle exec rake test` passes +- [ ] `bundle exec rubocop` passes +- [ ] `bundle exec reek` passes +- [ ] `bundle exec rake yard:coverage` passes +- [ ] CHANGELOG.md has been updated +- [ ] Documentation has been updated diff --git a/.reek.yml b/.reek.yml index 7d3605c..4117043 100644 --- a/.reek.yml +++ b/.reek.yml @@ -51,6 +51,7 @@ detectors: enabled: false ControlParameter: exclude: + - "SkillBench::Cli::BatchResultPrinter#self.batch_output" - "SkillBench::Commands::Run#validate_api_key" - "SkillBench::Execution::ContextHydrator#initialize" - "SkillBench::EvaluateCommand#print_parse_error" @@ -128,6 +129,7 @@ detectors: - "SkillBench::Criteria#assign_attributes" - "SkillBench::Dimension#==" - "SkillBench::Evaluation::Runner#compute_deltas" + - "SkillBench::Services::SummaryFormatter#total_cost" # Reduces a collected costs array — array manipulation is the domain - "SkillBench::Judge::Prompt#criteria_section" - "SkillBench::Judge::Response#extract_dimensions" - "SkillBench::CLI#handle_eval" @@ -173,6 +175,7 @@ detectors: - "SkillBench::Services::RunnerService#build_client_params" # Builder method — naturally manipulates params and provider hashes BooleanParameter: exclude: + - "SkillBench::Cli::BatchResultPrinter#self.call" # summary: toggles the JSON gate — part of the documented CI signature - "SkillBench::Commands::Init#self.run" - "SkillBench::Clients::BaseUrlValidator#self.call" # Named transport flags (has_credential/allow_insecure) — security policy inputs UtilityFunction: @@ -256,4 +259,6 @@ detectors: - "SkillBench::Clients::RetryHandler#extract_status" # Duck-type dispatch across Faraday error classes with different APIs - "SkillBench::OutputFormatter#self.format_junit" # Duck-type dispatch for backward compatibility with legacy result format - "SkillBench::Services::FeedbackGenerator#self.feedback_applicable?" # Duck-type guard for DeltaReport vs plain objects - - "SkillBench::Services::JUnitFormatter#self.format" # Duck-type dispatch for backward compatibility with legacy result format + - "SkillBench::Services::JUnitFormatter#self.passing?" # Duck-type dispatch for backward compatibility with legacy result format + - "SkillBench::Services::JUnitFormatter#self.score_for" # Duck-type dispatch for backward compatibility with legacy result format + - "SkillBench::Services::SummaryFormatter#delta_entry" # Duck-type guard for DeltaReport vs results without a report diff --git a/.rubocop.yml b/.rubocop.yml index 43f6f80..b52b894 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -19,6 +19,7 @@ Metrics/MethodLength: Metrics/ParameterLists: Max: 6 + CountKeywordArgs: false Metrics/AbcSize: Max: 50 diff --git a/CHANGELOG.md b/CHANGELOG.md index 93ece7b..56520c2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,16 +12,30 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 > In progress — the v1.2.0 quality program (security, performance, documentation, examples). The release date is set when the version is tagged. ### Added +- `skill-bench validate` (alias `doctor`): a pre-flight check that validates `criteria.json` via `CriteriaValidator`, schema-checks `skill-bench.json`, and reports missing provider API keys — all without running an eval or hitting the network (#45). +- `--format html`: a self-contained HTML report rendering the delta-score table and the baseline/context iteration timelines, with all dynamic text escaped (#44). +- Per-run token & cost accounting: agent token usage is threaded through the run, an estimated USD cost is shown in the human report (`Tokens: N | Est. Cost: $X.XXXX`) and exposed in JSON output; new `Services::CostCalculator` with a per-model price table (#40). +- Opt-in content-addressed response caching via `--cache` / `SKILL_BENCH_CACHE`: identical agent/judge calls — notably `compare`'s twice-run skill-less baseline — reuse a cached response instead of hitting the network. Default off (#42). +- Batch CI surfaces: a `--summary` JSON gate (aggregate pass/fail counts, summed tokens/cost, worst-delta eval) and per-eval `` aggregation in JUnit output, plus a top-level composite `action.yml` so downstream repos can gate skill changes with `uses: igmarin/ruby-skill-bench@v1` (#46). +- Mistral provider, an OpenAI-compatible client subclass (#47). +- `skill-bench init --mock` scaffolds a ready-to-run offline `mock` config (`{"provider":"mock","max_execution_time":30}`) (#53). +- Community-health files: `CODE_OF_CONDUCT.md` (Contributor Covenant 2.1), issue templates, and a pull-request template (#38). - Runnable, fully offline `examples/offline-quickstart/` that demonstrates a complete eval with the built-in `mock` provider — no API keys, no network (#49). - Example files are now tracked: removed the blanket `examples/` rule from `.gitignore` (#48). +### Security +- Documented that the command allowlist authorizes only the base token, so any allowlisted wrapper binary (`rake`, `rspec`, `make`, `find`, `git`) is equivalent to arbitrary host execution, and tied this to the fail-closed host-execution model. Added an optional, default-off `command_argument_constraints` hook to constrain high-risk allowed commands (#21). + ### Performance +- Provider resolution now parses `skill-bench.json` at most once per run via an mtime-memoized load, removing a redundant per-run file read + parse (#31). - Baseline and context agent runs now execute concurrently, roughly halving the dominant agent phase of a run. The previously unused `parallel` dependency is now wired into the active `RunnerService` path (#26). ### Fixed +- An explicit `{"provider":"mock"}` config no longer prints a misleading "Config load failed" warning on a clean offline run; genuine load failures (missing/broken config) still warn (#54). - `Judge::Prompt` no longer emits an empty `## Skill Context` section on baseline (skill-less) runs. The empty header had caused the `mock` provider to score baseline and context identically (delta 0); baseline prompts are now cleaner and offline scoring is correct (#58). ### Documentation +- `docs/architecture.md` now shows the correct `.skill-bench-trends.json` filename and notes the `.skill-bench-trends.json.bak` auto-backup (#39). - Corrected the evaluation-history filename throughout the README and docs: the engine writes `.skill-bench-trends.json`, not the previously-documented `.skill-bench-history.json` (#34). ## [1.1.0] - 2026-06-23 diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..c6791f3 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,133 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +We as members, contributors, and leaders pledge to make participation in our +community a harassment-free experience for everyone, regardless of age, body +size, visible or invisible disability, ethnicity, sex characteristics, gender +identity and expression, level of experience, education, socio-economic status, +nationality, personal appearance, race, religion, or sexual identity +and orientation. + +We pledge to act and interact in ways that contribute to an open, welcoming, +diverse, inclusive, and healthy community. + +## Our Standards + +Examples of behavior that contributes to a positive environment for our +community include: + +* Demonstrating empathy and kindness toward other people +* Being respectful of differing opinions, viewpoints, and experiences +* Giving and gracefully accepting constructive feedback +* Accepting responsibility and apologizing to those affected by our mistakes, + and learning from the experience +* Focusing on what is best not just for us as individuals, but for the + overall community + +Examples of unacceptable behavior include: + +* The use of sexualized language or imagery, and sexual attention or + advances of any kind +* Trolling, insulting or derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or email + address, without their explicit permission +* Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Enforcement Responsibilities + +Community leaders are responsible for clarifying and enforcing our standards of +acceptable behavior and will take appropriate and fair corrective action in +response to any behavior that they deem inappropriate, threatening, offensive, +or harmful. + +Community leaders have the right and responsibility to remove, edit, or reject +comments, commits, code, wiki edits, issues, and other contributions that are +not aligned to this Code of Conduct, and will communicate reasons for moderation +decisions when appropriate. + +## Scope + +This Code of Conduct applies within all community spaces, and also applies when +an individual is officially representing the community in public spaces. +Examples of representing our community include using an official e-mail address, +posting via an official social media account, or acting as an appointed +representative at an online or offline event. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported to the community leaders responsible for enforcement at +[ismael.marin@gmail.com](mailto:ismael.marin@gmail.com). + +All complaints will be reviewed and investigated promptly and fairly. + +All community leaders are obligated to respect the privacy and security of the +reporter of any incident. + +## Enforcement Guidelines + +Community leaders will follow these Community Impact Guidelines in determining +the consequences for any action they deem in violation of this Code of Conduct: + +### 1. Correction + +**Community Impact**: Use of inappropriate language or other behavior deemed +unprofessional or unwelcome in the community. + +**Consequence**: A private, written warning from community leaders, providing +clarity around the nature of the violation and an explanation of why the +behavior was inappropriate. A public apology may be requested. + +### 2. Warning + +**Community Impact**: A violation through a single incident or series +of actions. + +**Consequence**: A warning with consequences for continued behavior. No +interaction with the people involved, including unsolicited interaction with +those enforcing the Code of Conduct, for a specified period of time. This +includes avoiding interactions in community spaces as well as external channels +like social media. Violating these terms may lead to a temporary or +permanent ban. + +### 3. Temporary Ban + +**Community Impact**: A serious violation of community standards, including +sustained inappropriate behavior. + +**Consequence**: A temporary ban from any sort of interaction or public +communication with the community for a specified period of time. No public or +private interaction with the people involved, including unsolicited interaction +with those enforcing the Code of Conduct, is allowed during this period. +Violating these terms may lead to a permanent ban. + +### 4. Permanent Ban + +**Community Impact**: Demonstrating a pattern of violation of community +standards, including sustained inappropriate behavior, harassment of an +individual, or aggression toward or disparagement of classes of individuals. + +**Consequence**: A permanent ban from any sort of public interaction within +the community. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], +version 2.1, available at +[https://www.contributor-covenant.org/version/2/1/code_of_conduct.html][v2.1]. + +Community Impact Guidelines were inspired by +[Mozilla's code of conduct enforcement ladder][Mozilla CoC]. + +For answers to common questions about this code of conduct, see the FAQ at +[https://www.contributor-covenant.org/faq][FAQ]. Translations are available +at [https://www.contributor-covenant.org/translations][translations]. + +[homepage]: https://www.contributor-covenant.org +[v2.1]: https://www.contributor-covenant.org/version/2/1/code_of_conduct.html +[Mozilla CoC]: https://github.com/mozilla/diversity +[FAQ]: https://www.contributor-covenant.org/faq +[translations]: https://www.contributor-covenant.org/translations diff --git a/README.md b/README.md index f0985c2..0f0e891 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,7 @@ See the [Ecosystem Overview](https://github.com/igmarin/agent-mcp-runtime/blob/m - **Isolated Git Sandboxes**: Every run operates in a temporary repo. Clean diffs, zero side-effects, 100% reproducibility. - **Blind Judging with Dimensions**: LLM judge scores baseline and context independently across 5 canonical dimensions (Correctness, Skill Adherence, Code Quality, Test Coverage, Documentation). Eval authors configure weights and thresholds via `criteria.json`. - **Sophisticated ReAct Loop**: Employs a robust `Thought → Tool → Observation` loop to handle complex, multi-step engineering tasks. -- **Multi-Provider Ecosystem**: Native support for **OpenAI**, **Anthropic**, **Google Gemini**, **Azure OpenAI**, **Ollama**, **Groq**, **DeepSeek**, and **OpenCode**. +- **Multi-Provider Ecosystem**: Native support for **OpenAI**, **Anthropic**, **Google Gemini**, **Azure OpenAI**, **Ollama**, **Groq**, **DeepSeek**, **Mistral**, and **OpenCode**. - **Standardized Intelligence**: Consistent reporting format regardless of the underlying LLM provider. --- @@ -64,11 +64,14 @@ CLI / API → RunnerService → Sandbox + ReAct Agent → LLM Client Layer → P | **Ollama** | — | `:ollama` | | **Groq** | `SKILL_BENCH_GROQ_API_KEY` | `:groq` | | **DeepSeek** | `SKILL_BENCH_DEEPSEEK_API_KEY` | `:deepseek` | +| **Mistral** | `SKILL_BENCH_MISTRAL_API_KEY` | `:mistral` | | **OpenCode** | `SKILL_BENCH_OPENCODE_API_KEY`, `SKILL_BENCH_OPENCODE_BASE_URL` | `:opencode` | > **Note:** Environment variables are loaded automatically. You can also configure provider settings in `skill-bench.json` (created by `skill-bench init`). > > **OpenCode requires a custom `base_url`:** OpenCode does not host a public LLM API. You must provide your own OpenAI-compatible endpoint (e.g. a LiteLLM proxy, self-hosted vLLM, or company gateway) via the `base_url` config key. Without it, the provider will fail with "Base URL not set for Opencode". +> +> **Mistral** uses Mistral's OpenAI-compatible chat completions API (default model `mistral-large-latest`). Set `SKILL_BENCH_MISTRAL_API_KEY` and scaffold it with `skill-bench init --mistral`. ### Command Allowlist @@ -140,7 +143,9 @@ skill-bench init --openai } ``` -**Available providers:** `--openai`, `--anthropic`, `--gemini`, `--ollama`, `--azure`, `--groq`, `--deepseek`, `--opencode` +**Available providers:** `--openai`, `--anthropic`, `--gemini`, `--ollama`, `--azure`, `--groq`, `--deepseek`, `--mistral`, `--opencode` + +**Zero-config offline path:** `skill-bench init --mock` scaffolds a minimal offline config that needs no API key and no network — `{"provider":"mock","max_execution_time":30}`. Use it to try the full flow (and run the bundled examples) before wiring up a real provider. Use `--force` to overwrite an existing config. @@ -353,11 +358,54 @@ skill-bench run my-first-eval --skill=skill-a --skill=skill-b Both skill contexts are concatenated and sent to the agent. The judge evaluates whether the combined context improves results. -**Output Formats:** +**Output Formats:** `--format human` (default), `json`, `junit`, or `html`. -- Human-readable (default) -- JSON: `--format json` -- JUnit XML: `--format junit` +- Human-readable (default) — full delta table, iteration timeline, and a `Tokens: N | Est. Cost: $X.XXXX` line. +- JSON: `--format json` — machine-readable, including top-level `tokens` and `cost` fields. +- JUnit XML: `--format junit` — for CI test reporting. +- HTML: `--format html` — a self-contained, shareable report (styles inlined, no external assets) with the delta table and iteration timeline. Redirect it to a file: + + ```bash + skill-bench run my-first-eval --skill=my-service --format html > report.html + ``` + +--- + +## Pre-flight Checks: `validate` / `doctor` + +Before spending tokens on a run, sanity-check your setup. `skill-bench validate` (aliased as `doctor`) runs read-only pre-flight checks — it never runs an eval and never makes a network call: + +```bash +skill-bench validate +# or, identically: +skill-bench doctor +``` + +It runs three checks and prints a `PASS` / `FAIL` / `SKIP` line for each: + +1. **criteria** — validates the criteria JSON (default `criteria.json`, override with `--criteria PATH`). Skipped if the default file is absent. +2. **config** — schema-checks `skill-bench.json` (default, override with `--config PATH`): `provider` is required and must be a known provider, `max_execution_time` must be a positive integer, and `config` (when present) must be an object. +3. **provider key** — reports whether the configured provider's API key is present (the `mock` provider needs none). + +A passing report exits `0`: + +```text +skill-bench validate + +[PASS] criteria criteria.json is valid +[PASS] config skill-bench.json matches the expected shape +[PASS] provider key openai credentials present + +All checks passed. +``` + +A failure exits non-zero and names what is wrong: + +```text +[FAIL] provider key openai is missing: api_key + +1 check(s) failed. +``` --- @@ -430,6 +478,22 @@ The `--variant` spec supports two forms: - `pack:` — resolve via registry manifest - `/absolute/path` or `relative/path` — use a direct path +### Response Caching (opt-in, `--cache`) + +LLM responses can be cached so identical requests reuse a previous result instead of calling the provider again. Caching is **off by default**. Enable it per run with `--cache`, or set the `SKILL_BENCH_CACHE` environment variable to a truthy value (`1`, `true`, `yes`, or `on`): + +```bash +# Per-run flag +skill-bench run my-first-eval --skill=my-service --cache + +# Or via the environment +SKILL_BENCH_CACHE=1 skill-bench run my-first-eval --skill=my-service +``` + +The cache is in-memory (process-lifetime) and content-addressed: the key is a SHA-256 digest of the provider, model, system prompt, messages, tools, and temperature, so only truly identical requests share an entry. The `mock` and null providers are never cached. + +This pays off most with `compare`, which runs the skill-less baseline twice with identical inputs — with caching enabled, the repeated baseline reuses the cached response instead of making a second call. + --- ## File Reference: What Lives on Disk @@ -734,6 +798,7 @@ These 5 dimensions are **mandatory** in every `criteria.json`. You can add custo Eval: my-first-eval Skill: my-service Provider: openai + Tokens: 18432 | Est. Cost: $0.0934 ═══════════════════════════════════════════════════════ === BASELINE ITERATIONS === @@ -781,6 +846,7 @@ These 5 dimensions are **mandatory** in every `criteria.json`. You can add custo - **TOTAL:** Sum of all dimension scores. Max possible is 100. - **TREND:** Comparison against the previous run of the same eval + skill (from `.skill-bench-trends.json`). Shows whether scores are improving over time. - **VERDICT:** `PASS` only if `CONTEXT >= pass_threshold` AND `DELTA >= minimum_delta`. +- **Tokens / Est. Cost:** The header shows total tokens used across the run and an estimated USD cost as `Tokens: N | Est. Cost: $X.XXXX`. The cost is approximate — it comes from a built-in per-model price table (`Services::CostCalculator`) and shows `—` when the model isn't in that table. JSON output (`--format json`) exposes the same data as top-level `tokens` and `cost` fields. **Iteration timeline:** @@ -902,6 +968,10 @@ Ruby Skill Bench is designed with security as a primary concern. The system exec - **Shell Tokenization:** Commands are tokenized before execution to prevent shell injection - **Fail-Closed Host Execution:** Container isolation is not yet active, so commands run on the host inside a temporary sandbox directory. To match this reality, `run_command` refuses to execute unless `allow_host_execution: true` is set; it is **disabled by default**. +> **The allowlist is the only real authorization control — and it only checks the base command.** `run_command` authorizes by the first token of the command (`rake`, `find`, `git`, …); it does **not** inspect arguments. Shell tokenization stops metacharacter injection, but it does **not** sandbox what an allowlisted binary can do. Because many common tools are general-purpose execution wrappers, **allowlisting any one of them is equivalent to granting arbitrary host code execution** — for example `rake -e '...'`, `rspec -e`, `make` (arbitrary recipes), `find . -exec ...`, or `git` (hooks, `-c core.fsmonitor=...`, `! ...` aliases). Combined with the fail-closed model above (`run_command` refuses to run on the host unless `allow_host_execution` is explicitly enabled — see `HOST_EXECUTION_REFUSED` in `run_command.rb`), the practical guidance is: **keep `allowed_commands` as minimal as possible — empty for untrusted skills** — and treat every entry as if you were handing the skill a shell. +> +> An **optional, default-off** `command_argument_constraints` setting can refuse commands whose arguments contain configured substrings/flags (for example blocking `-e` or `-exec`). It is a defense-in-depth speed bump, **not** a sandbox, and is unset by default; the allowlist remains the control that matters. + #### Docker Security Hardening (Planned — Not Yet Active) > **Status:** The container isolation model described below is **planned, not shipped**. No Docker build context is packaged, so containers are never launched today — `run_command` runs on the host gated by the allowlist and `allow_host_execution`. The settings below document the intended hardened model for when container isolation lands. @@ -1020,15 +1090,62 @@ If you encounter issues not covered here: ## CI/CD Integration -GitHub Actions workflow included (`.github/workflows/ci.yml`): +### Batch Runs + +Run every eval at once instead of one at a time: + +```bash +# Every eval under the default evals/ directory +skill-bench run --all --skill=my-service + +# Or point at a specific directory +skill-bench run --evals-dir path/to/evals --skill=my-service +``` + +A batch run exits `0` only when **every** eval passes and non-zero if any fail, so the process exit code is itself a CI gate. Two formats are built for batch consumption: + +- `--summary` emits an aggregate JSON gate — `passed` / `failed` / `total` counts, summed `tokens` and `cost`, and the `worst_delta` eval (the smallest context-minus-baseline delta in the batch). Archive it as a single machine-readable artifact: + + ```bash + skill-bench run --all --skill=my-service --summary + ``` + +- `--format junit` aggregates the batch into one JUnit document with **one `` per eval** (a `` child for each failing eval), so test reporters show per-eval results: + + ```bash + skill-bench run --all --skill=my-service --format junit > junit.xml + ``` + +### GitHub Action + +Downstream repos can gate a skill change on every push or PR with the bundled composite action. Add a step that references `igmarin/ruby-skill-bench@v1`: + +```yaml +# .github/workflows/skill-bench.yml +name: skill-bench +on: [pull_request] + +jobs: + skill-bench: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: igmarin/ruby-skill-bench@v1 + with: + evals-dir: evals # directory scanned for evals (default: evals) + skill: skills/my-service # skill applied to every eval (default: "") + format: junit # human | json | junit | html (default: junit) + ruby-version: "3.3" # Ruby for ruby/setup-ruby (default: 3.3) + args: --summary # extra flags appended verbatim (e.g. --summary, --pack NAME) +``` + +The action installs the gem and runs `skill-bench run --all --evals-dir --format ` (adding `--skill` when set and appending `args` verbatim). The run step's exit code is the gate. For a full copy-paste workflow template, see [`examples/ci/`](examples/ci/). + +> The gem's own repository CI (`.github/workflows/ci.yml`) runs the test suite — rubocop, reek, and minitest against Ruby 3.3 and 3.4, on push and pull requests — and is separate from the reusable action above. -- Runs on push and pull requests -- Tests against Ruby 3.3 and 3.4 -- Executes rubocop, reek, and minitest -- Outputs JUnit XML for test reporting +To preview the machine-readable output locally: ```bash -# Run locally with CI output skill-bench run my-eval --skill=my-skill --format json ``` diff --git a/action.yml b/action.yml new file mode 100644 index 0000000..7e8a972 --- /dev/null +++ b/action.yml @@ -0,0 +1,80 @@ +# --------------------------------------------------------------------------- +# Ruby Skill Bench — reusable GitHub composite action +# --------------------------------------------------------------------------- +# Add this to a downstream repo to gate a skill change on every run: +# +# - uses: igmarin/ruby-skill-bench@v1 +# with: +# evals-dir: evals +# skill: skills/greeting-skill +# format: junit # human | json | junit (CI gate) | html +# +# The action installs the `ruby-skill-bench` gem and runs +# `skill-bench run --all` over `evals-dir`. The gem exits 0 when every eval +# passes and non-zero when any fails, so the run step's exit code IS the gate. +# +# This is the reusable action definition. For a full copy-paste *workflow* +# template, see `examples/ci/github-action.yml`. +# --------------------------------------------------------------------------- +name: "Ruby Skill Bench" +description: "Run ruby-skill-bench evals over a directory and gate CI on the results." +author: "igmarin" + +branding: + icon: "check-circle" + color: "purple" + +inputs: + evals-dir: + description: "Directory to scan for evals (passed to --evals-dir)." + required: false + default: "evals" + skill: + description: "Skill to apply to every eval (passed to --skill). Leave blank when using --pack via `args`." + required: false + default: "" + format: + description: "Output format: human, json, junit, or html. junit/json are machine-readable CI gates." + required: false + default: "junit" + ruby-version: + description: "Ruby version for ruby/setup-ruby." + required: false + default: "3.3" + args: + description: "Extra arguments appended verbatim to the `skill-bench run` command (e.g. --summary, --pack NAME)." + required: false + default: "" + +runs: + using: "composite" + steps: + - name: Set up Ruby + uses: ruby/setup-ruby@v1 + with: + ruby-version: ${{ inputs.ruby-version }} + + - name: Install ruby-skill-bench + shell: bash + run: gem install ruby-skill-bench + + - name: Run skill-bench + shell: bash + # Inputs are passed through env (not interpolated into the script) so a + # crafted input cannot inject shell before bash parses it. A non-zero exit + # here fails the step — that is the CI gate. + env: + EVALS_DIR: ${{ inputs.evals-dir }} + SKILL: ${{ inputs.skill }} + FORMAT: ${{ inputs.format }} + EXTRA_ARGS: ${{ inputs.args }} + run: | + run_args=(run --all --evals-dir "$EVALS_DIR" --format "$FORMAT") + if [ -n "$SKILL" ]; then + run_args+=(--skill "$SKILL") + fi + if [ -n "$EXTRA_ARGS" ]; then + # Intentionally unquoted so multiple extra flags split into words. + run_args+=($EXTRA_ARGS) + fi + skill-bench "${run_args[@]}" diff --git a/docs/architecture.md b/docs/architecture.md index f8c77f4..71695a2 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -175,6 +175,8 @@ project-root/ └── .skill-bench-trends.json # Benchmark history (auto-generated) ``` +A `.skill-bench-trends.json.bak` file is created automatically as a backup of the trend file. + ### Skill Discovery Skills are discovered recursively. These are all valid: diff --git a/examples/README.md b/examples/README.md index 082697f..48adc92 100644 --- a/examples/README.md +++ b/examples/README.md @@ -4,6 +4,10 @@ Runnable, copy-paste examples for `ruby-skill-bench`. Each one is offline by default — they use the built-in `mock` provider, so they need **no API key and no network access**. +> Tip: run `skill-bench init --mock` to scaffold that offline mock config +> (`{"provider":"mock","max_execution_time":30}`) automatically. As a fallback, +> you can write that same JSON to `skill-bench.json` by hand. + ## Index ### `offline-quickstart/` diff --git a/lib/skill_bench.rb b/lib/skill_bench.rb index a93d6ab..3a025da 100644 --- a/lib/skill_bench.rb +++ b/lib/skill_bench.rb @@ -74,6 +74,7 @@ # Services require_relative 'skill_bench/services/runner_service' require_relative 'skill_bench/services/batch_runner_service' +require_relative 'skill_bench/services/summary_formatter' require_relative 'skill_bench/services/template_registry' # Tools diff --git a/lib/skill_bench/cli.rb b/lib/skill_bench/cli.rb index f59f1d4..87e0191 100644 --- a/lib/skill_bench/cli.rb +++ b/lib/skill_bench/cli.rb @@ -5,6 +5,7 @@ require_relative 'cli/compare_command' require_relative 'cli/skill_command' require_relative 'cli/eval_command' +require_relative 'cli/validate_command' require_relative 'cli/help_printer' require_relative 'cli/result_printer' require_relative 'cli/batch_result_printer' @@ -43,6 +44,7 @@ def call when 'compare' then Cli::CompareCommand.call(@argv) when 'skill' then Cli::SkillCommand.call(@argv) when 'eval' then Cli::EvalCommand.call(@argv) + when 'validate', 'doctor' then Cli::ValidateCommand.call(@argv) when '-h', '--help', 'help' help.call else diff --git a/lib/skill_bench/cli/batch_result_printer.rb b/lib/skill_bench/cli/batch_result_printer.rb index a37f4aa..c3b8fc8 100644 --- a/lib/skill_bench/cli/batch_result_printer.rb +++ b/lib/skill_bench/cli/batch_result_printer.rb @@ -1,19 +1,45 @@ # frozen_string_literal: true require_relative '../output_formatter' +require_relative '../services/summary_formatter' module SkillBench module Cli # Prints the aggregate result of a batch `skill-bench run --all` command. + # + # Defaults to the human-readable batch summary, but can instead emit a + # JUnit document (`format: :junit`) or a JSON gate (`summary: true`). The + # returned exit code is always {OutputFormatter.batch_exit_code}, so CI + # gating works identically across every output mode. class BatchResultPrinter # Prints the aggregate summary and returns the appropriate exit code. # - # @param aggregate [Hash] Aggregate envelope from BatchRunnerService - # @return [Integer] Exit code (0 when all pass, 1 when any fails) - def self.call(aggregate) - puts OutputFormatter.format_batch(aggregate) + # @param aggregate [Hash] Aggregate envelope from BatchRunnerService. + # @param format [Symbol, nil] Output format (:junit for JUnit XML, else human). + # @param summary [Boolean] When true, print the JSON summary gate instead. + # @return [Integer] Exit code (0 when all pass, 1 when any fails). + def self.call(aggregate, format: nil, summary: false) + puts batch_output(aggregate, format: format, summary: summary) OutputFormatter.batch_exit_code(aggregate) end + + # Selects the rendered batch output for the requested mode. + # + # `:junit` and `:json` produce machine-readable batch output; `:json` maps + # to the same JSON gate as `summary: true`. `:html` (and any other format) + # falls back to the human batch summary, since there is no batch HTML report. + # + # @param aggregate [Hash] Aggregate envelope from BatchRunnerService. + # @param format [Symbol, nil] Output format (:junit, :json, else human). + # @param summary [Boolean] When true, render the JSON summary gate. + # @return [String] The formatted batch output. + def self.batch_output(aggregate, format:, summary:) + return Services::SummaryFormatter.format(aggregate) if summary || format == :json + return Services::JUnitFormatter.format_batch(aggregate) if format == :junit + + OutputFormatter.format_batch(aggregate) + end + private_class_method :batch_output end end end diff --git a/lib/skill_bench/cli/help_printer.rb b/lib/skill_bench/cli/help_printer.rb index c6e9c12..ed957bd 100644 --- a/lib/skill_bench/cli/help_printer.rb +++ b/lib/skill_bench/cli/help_printer.rb @@ -20,11 +20,14 @@ def self.call --force Overwrite existing config file run --skill [--skill ] [--format FORMAT] [--pack NAME] - Run an evaluation + Run an evaluation (single eval, or a whole directory with --all) --skill Skill to use (can be specified multiple times) --pack Pack context for registry-based skill resolution --registry-manifest PATH Path to registry.json manifest - --format Output format: human, json, junit (default: human) + --format Output format: human, json, junit, html (default: human) + --all Run every eval under evals/ (batch mode) + --evals-dir DIR Run every eval under DIR (batch mode) + --summary Emit a JSON summary gate for a batch run (batch mode) compare --variant-a SPEC --variant-b SPEC --eval PATH Compare the same skill across two pack variants @@ -45,6 +48,11 @@ def self.call Auto-generate an eval from a skill --name Name for the generated eval (optional) + validate (alias: doctor) [--criteria PATH] [--config PATH] + Run read-only pre-flight checks (no eval, no network) + --criteria Criteria JSON to validate (default: criteria.json) + --config Config file to validate (default: skill-bench.json) + Global Options: -h, --help Show this help message USAGE diff --git a/lib/skill_bench/cli/init_command.rb b/lib/skill_bench/cli/init_command.rb index 4716b61..3bd2204 100644 --- a/lib/skill_bench/cli/init_command.rb +++ b/lib/skill_bench/cli/init_command.rb @@ -45,6 +45,7 @@ def build_parser(options) OptionParser.new do |opts| opts.banner = 'Usage: skill-bench init -- [options]' register_provider_options(opts, options) + opts.on('--mock', 'Generate offline mock config (no API key required)') { options[:provider] = :mock } opts.on('--force', 'Overwrite existing config file') { options[:force] = true } opts.on('-h', '--help', 'Prints this help') do puts opts @@ -60,7 +61,7 @@ def register_provider_options(parser, options) end def error_missing_provider - providers = SkillBench::Clients::ProviderSchemas.names.map { |provider_name| "--#{provider_name}" }.join(', ') + providers = (SkillBench::Clients::ProviderSchemas.names.map { |provider_name| "--#{provider_name}" } + ['--mock']).join(', ') warn "Error: provider is required. Use one of: #{providers}" 1 end diff --git a/lib/skill_bench/cli/result_printer.rb b/lib/skill_bench/cli/result_printer.rb index e324bd4..34a9a5f 100644 --- a/lib/skill_bench/cli/result_printer.rb +++ b/lib/skill_bench/cli/result_printer.rb @@ -9,7 +9,7 @@ class ResultPrinter # Prints the result and returns the appropriate exit code. # # @param result [Hash] Result from ScoringService - # @param format [Symbol] Output format (:human, :json, :junit) + # @param format [Symbol] Output format (:human, :json, :junit, :html) # @return [Integer] Exit code (0 for pass, 1 for fail) def self.call(result, format: :human) puts OutputFormatter.format(result, format: format) diff --git a/lib/skill_bench/cli/run_command.rb b/lib/skill_bench/cli/run_command.rb index dfa1585..1f26e9f 100644 --- a/lib/skill_bench/cli/run_command.rb +++ b/lib/skill_bench/cli/run_command.rb @@ -57,7 +57,7 @@ def run_single(options) return error_missing_skill if options[:skill_names].empty? && !options[:pack] options[:eval_name] = eval_name - exec_options = options.reject { |key| key == :format } + exec_options = options.reject { |key| %i[format summary all evals_dir].include?(key) } result = Commands::Run.run(**exec_options) ResultPrinter.call(result, format: options[:format] || :human) end @@ -75,7 +75,7 @@ def run_batch(options) pack: options[:pack], registry_manifest: options[:registry_manifest] ) - BatchResultPrinter.call(aggregate) + BatchResultPrinter.call(aggregate, format: options[:format], summary: options[:summary]) end def build_parser(options) @@ -84,9 +84,11 @@ def build_parser(options) opts.on('--skill NAME', 'Skill to use (can be specified multiple times)') { |v| options[:skill_names] << v } opts.on('--pack NAME', 'Pack context for skill resolution') { |v| options[:pack] = v } opts.on('--registry-manifest PATH', 'Path to registry.json manifest') { |v| options[:registry_manifest] = v } - opts.on('--format FORMAT', 'Output format (human, json, junit)') { |v| options[:format] = v.to_sym } + opts.on('--format FORMAT', 'Output format (human, json, junit, html)') { |v| options[:format] = v.to_sym } opts.on('--all', 'Run every eval under the default evals/ directory') { options[:all] = true } opts.on('--evals-dir DIR', 'Run every eval under DIR') { |v| options[:evals_dir] = v } + opts.on('--summary', 'Emit a JSON summary gate for a batch run') { options[:summary] = true } + opts.on('--cache', 'Enable content-addressed response caching') { ENV['SKILL_BENCH_CACHE'] = '1' } opts.on('-h', '--help', 'Prints this help') do puts opts raise SkillBench::HelpRequested diff --git a/lib/skill_bench/cli/validate_command.rb b/lib/skill_bench/cli/validate_command.rb new file mode 100644 index 0000000..0b6229c --- /dev/null +++ b/lib/skill_bench/cli/validate_command.rb @@ -0,0 +1,242 @@ +# frozen_string_literal: true + +require 'json' +require 'optparse' + +module SkillBench + module Cli + # Handles the `skill-bench validate` / `doctor` subcommand. + # + # Runs read-only pre-flight checks and prints a PASS/FAIL report: + # 1. Criteria JSON structure (via {Models::CriteriaValidator}). + # 2. skill-bench.json shape (hand-rolled, lightweight schema check). + # 3. Provider credentials for the configured non-mock provider. + # + # It never runs an eval and never makes a network call. + class ValidateCommand + # Default criteria file validated when --criteria is not given. + DEFAULT_CRITERIA = 'criteria.json' + + # @param argv [Array] Raw CLI arguments + # @return [Integer] Exit code + def self.call(argv) + new(argv).call + end + + # @param argv [Array] Raw CLI arguments + def initialize(argv) + @argv = argv + end + + # Parses options, runs the pre-flight checks, and prints the report. + # + # @return [Integer] Exit code (0 when all checks pass, 1 otherwise) + def call + options = parse_options + config_path = options[:config] || SkillBench::Config::CONFIG_FILENAME + config_data = load_config_data(config_path) + results = [ + check_criteria(options), + check_config(config_path, config_data), + check_provider_key(config_data) + ] + print_report(results) + results.any? { |result| result[:status] == :fail } ? 1 : 0 + rescue HelpRequested + 0 + rescue StandardError => e + warn "Error: #{e.message}" + 1 + end + + private + + def parse_options + options = {} + build_parser(options).parse!(@argv) + options + end + + def build_parser(options) + OptionParser.new do |opts| + opts.banner = 'Usage: skill-bench validate [options]' + opts.on('--criteria PATH', 'Criteria JSON file to validate (default: criteria.json)') { |v| options[:criteria] = v } + opts.on('--config PATH', 'Config file to validate (default: skill-bench.json)') { |v| options[:config] = v } + opts.on('-h', '--help', 'Prints this help') do + puts opts + raise SkillBench::HelpRequested + end + end + end + + # --- Check (a): criteria ------------------------------------------------ + + def check_criteria(options) + path = options[:criteria] || DEFAULT_CRITERIA + unless File.exist?(path) + return fail_result('criteria', "criteria file not found: #{path}") if options[:criteria] + + return skip_result('criteria', "no #{DEFAULT_CRITERIA} found (skipped)") + end + + result = Models::CriteriaValidator.call(path:) + return pass_result('criteria', "#{path} is valid") if result[:success] + + fail_result('criteria', "#{path}: #{criteria_error(result)}") + end + + def criteria_error(result) + result.dig(:response, :error, :message) || 'invalid criteria' + end + + # --- Check (b): config shape ------------------------------------------- + + def check_config(path, config_data) + case config_data[:status] + when :missing + fail_result('config', "#{path} not found") + when :invalid_json + fail_result('config', "#{path} is not valid JSON: #{config_data[:message]}") + else + validate_config_shape(path, config_data[:data]) + end + end + + def validate_config_shape(path, data) + return fail_result('config', "#{path} must contain a JSON object") unless data.is_a?(Hash) + + errors = config_shape_errors(data) + return fail_result('config', errors.join('; ')) if errors.any? + + pass_result('config', "#{path} matches the expected shape") + end + + def config_shape_errors(data) + errors = provider_errors(data[:provider]) + errors.concat(max_execution_time_errors(data[:max_execution_time])) + errors << "'config' must be an object" if data.key?(:config) && !data[:config].is_a?(Hash) + errors + end + + def provider_errors(provider) + return ["'provider' is required"] if provider.nil? + return ["'provider' must be a string"] unless provider.is_a?(String) + + allowed = Models::Provider::ALLOWED_PROVIDERS + return [] if allowed.include?(provider) + + ["'provider' '#{provider}' is not one of: #{allowed.join(', ')}"] + end + + def max_execution_time_errors(value) + return [] if value.nil? + return [] if value.is_a?(Integer) && value.positive? + + ["'max_execution_time' must be a positive integer"] + end + + # --- Check (c): provider key ------------------------------------------- + + def check_provider_key(config_data) + return skip_result('provider key', 'skipped (no usable config)') unless config_data[:status] == :ok + + provider = config_provider(config_data[:data]) + return skip_result('provider key', 'skipped (provider invalid)') unless provider + return pass_result('provider key', 'mock provider requires no API key') if provider == 'mock' + + missing = missing_provider_keys(provider, config_data[:data][:config]) + return pass_result('provider key', "#{provider} credentials present") if missing.empty? + + fail_result('provider key', "#{provider} is missing: #{missing.join(', ')}") + rescue StandardError => e + # Building the client can raise on unrelated config (e.g. base_url + # validation); surface that as a structured FAIL rather than crashing. + fail_result('provider key', "#{provider} config is invalid: #{e.message}") + end + + def config_provider(data) + return nil unless data.is_a?(Hash) + + provider = data[:provider] + return nil unless provider.is_a?(String) && Models::Provider::ALLOWED_PROVIDERS.include?(provider) + + provider + end + + def missing_provider_keys(provider, provider_config) + provider_sym = provider.to_sym + options = provider_client_options(provider_sym, provider_config) + client = Clients::ProviderRegistry.for(provider_sym).new(options) + return [] unless client.respond_to?(:missing_config_keys, true) + + client.send(:missing_config_keys) + end + + def provider_client_options(provider_sym, provider_config) + options = provider_config.is_a?(Hash) ? provider_config.dup : {} + Models::Provider::ENV_OVERRIDABLE_SETTINGS.each do |setting| + value = env_setting(provider_sym, setting) + options[setting] = value unless value.nil? + end + options + end + + def env_setting(provider_sym, setting) + provider = provider_sym.to_s.upcase + name = setting.to_s.upcase + ["SKILL_BENCH_#{provider}_#{name}", "#{provider}_#{name}"].each do |var| + value = ENV.fetch(var, nil) + return value if value && !value.empty? + end + nil + end + + # --- Config loading ---------------------------------------------------- + + def load_config_data(path) + return { status: :missing } unless File.exist?(path) + + { status: :ok, data: JSON.parse(File.read(path), symbolize_names: true) } + rescue JSON::ParserError => e + { status: :invalid_json, message: e.message } + end + + # --- Reporting --------------------------------------------------------- + + def print_report(results) + puts 'skill-bench validate' + puts + results.each { |result| puts format_result(result) } + puts + puts summary_line(results) + end + + def format_result(result) + "[#{label(result[:status])}] #{result[:name].ljust(13)} #{result[:message]}" + end + + def label(status) + { pass: 'PASS', fail: 'FAIL', skip: 'SKIP' }.fetch(status) + end + + def summary_line(results) + failed = results.count { |result| result[:status] == :fail } + return "#{failed} check(s) failed." if failed.positive? + + 'All checks passed.' + end + + def pass_result(name, message) + { name:, status: :pass, message: } + end + + def fail_result(name, message) + { name:, status: :fail, message: } + end + + def skip_result(name, message) + { name:, status: :skip, message: } + end + end + end +end diff --git a/lib/skill_bench/client.rb b/lib/skill_bench/client.rb index d87ad87..e303bd0 100644 --- a/lib/skill_bench/client.rb +++ b/lib/skill_bench/client.rb @@ -1,13 +1,27 @@ # frozen_string_literal: true require_relative 'clients/all' +require_relative 'services/response_cache' module SkillBench # Facade for calling LLM clients. # Delegates to the configured provider. class Client + # Provider clients that must never be cached: their results either signal a + # configuration error (NullClient) or are cheap, deterministic test doubles + # (Mock). Caching them would provide no benefit and could mask errors. + UNCACHEABLE_CLIENTS = [ + Clients::Providers::NullClient, + Clients::Providers::Mock + ].freeze + # Calls the configured LLM provider with the given parameters. # + # When response caching is enabled (see {Services::ResponseCache.enabled?}) + # and the resolved provider is cacheable, identical requests reuse a cached + # response instead of calling the provider again. When caching is disabled + # (the default), the provider is always invoked, leaving behavior unchanged. + # # @param system_prompt [String] System prompt for the LLM # @param messages [Array] Conversation messages # @param provider [Symbol, nil] Override the configured LLM provider (e.g., :deepseek, :openai) @@ -17,7 +31,35 @@ def self.call(system_prompt:, messages:, provider: nil, **options) resolved = provider || Config.current_llm_provider || :openai client_class = Clients::ProviderRegistry.for(resolved) warn "WARNING: LLM provider '#{resolved}' is not configured. Falling back to null client." if client_class == Clients::Providers::NullClient - client_class.call(system_prompt: system_prompt, messages: messages, **options) + + invoke = -> { client_class.call(system_prompt: system_prompt, messages: messages, **options) } + return invoke.call unless cache_eligible?(client_class) + + cache_key = Services::ResponseCache.key( + provider: resolved, + model: options[:model], + system_prompt: system_prompt, + messages: messages, + tools: options[:tools], + temperature: options[:temperature], + provider_config: options.slice(:base_url, :request_path, :endpoint, :location, :project_id, :api_version) + ) + Services::ResponseCache.fetch(cache_key, &invoke) + end + + # Whether a resolved provider client may be served from the cache. + # + # Requires caching to be enabled and the client to not be one of the + # {UNCACHEABLE_CLIENTS} (null/mock), so disabling the cache restores the + # original, uncached behavior exactly. + # + # @param client_class [Class] The resolved provider client class + # @return [Boolean] true when the call should go through the cache + def self.cache_eligible?(client_class) + return false unless Services::ResponseCache.enabled? + + !UNCACHEABLE_CLIENTS.include?(client_class) end + private_class_method :cache_eligible? end end diff --git a/lib/skill_bench/clients/all.rb b/lib/skill_bench/clients/all.rb index d52bc11..933a85a 100644 --- a/lib/skill_bench/clients/all.rb +++ b/lib/skill_bench/clients/all.rb @@ -18,5 +18,6 @@ require_relative 'providers/opencode' require_relative 'providers/groq' require_relative 'providers/deepseek' +require_relative 'providers/mistral' require_relative 'providers/openrouter' require_relative 'providers/mock' diff --git a/lib/skill_bench/clients/provider_schemas.rb b/lib/skill_bench/clients/provider_schemas.rb index af04113..8e13e30 100644 --- a/lib/skill_bench/clients/provider_schemas.rb +++ b/lib/skill_bench/clients/provider_schemas.rb @@ -39,6 +39,10 @@ class ProviderSchemas api_key: nil, model: 'deepseek-chat' }.freeze, + mistral: { + api_key: nil, + model: 'mistral-large-latest' + }.freeze, opencode: { api_key: nil, model: 'opencode-model', diff --git a/lib/skill_bench/clients/providers/mistral.rb b/lib/skill_bench/clients/providers/mistral.rb new file mode 100644 index 0000000..ad68ae5 --- /dev/null +++ b/lib/skill_bench/clients/providers/mistral.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +require_relative '../base_client' +require_relative '../provider_registry' + +module SkillBench + module Clients + module Providers + # Mistral (la Plateforme) LLM client. + # Uses Mistral's OpenAI-compatible chat completions API with bearer-token auth. + # + # NOTE: AWS Bedrock access to Mistral models (which requires SigV4 request + # signing rather than a static bearer token) is intentionally not handled + # here and is left as a follow-up. + class Mistral < BaseClient + SkillBench::Clients::ProviderRegistry.register(:mistral, self) + + # Returns the provider identifier. + # + # @return [Symbol] + def provider_name + :mistral + end + + protected + + # Returns the base URL for the Mistral API. + # + # The Mistral API base is https://api.mistral.ai/v1; the version segment + # lives in {#request_path} so Faraday does not drop it (an absolute + # request path replaces any path component of the connection base URL). + # + # @return [String] + def base_url + @base_url_config || 'https://api.mistral.ai' + end + + # Returns the request path for chat completions. + # + # @return [String] + def request_path + @request_path_config || '/v1/chat/completions' + end + end + end + end +end diff --git a/lib/skill_bench/commands/init.rb b/lib/skill_bench/commands/init.rb index 7da6487..b9d8c77 100644 --- a/lib/skill_bench/commands/init.rb +++ b/lib/skill_bench/commands/init.rb @@ -24,10 +24,15 @@ def self.run(provider:, force: false) # Generates configuration hash for a specific provider. # + # The built-in `:mock` provider needs no credentials, so it produces a + # minimal offline config without a nested `config:` block. + # # @param provider [Symbol] LLM provider name # @return [Hash] Single-provider configuration # @raise [ArgumentError] if provider is not registered def self.config_for_provider(provider) + return { provider: :mock, max_execution_time: 30 } if provider == :mock + { provider: provider, max_execution_time: 30, diff --git a/lib/skill_bench/config.rb b/lib/skill_bench/config.rb index bc49813..6936e86 100644 --- a/lib/skill_bench/config.rb +++ b/lib/skill_bench/config.rb @@ -103,6 +103,16 @@ def allow_host_execution store.allow_host_execution || false end + # Returns the optional per-command argument constraints. + # + # When unconfigured, returns an empty Hash meaning no argument constraints + # apply (the allowlist remains the only command-authorization control). + # + # @return [Hash] base command => disallowed argument substrings/flags + def command_argument_constraints + store.command_argument_constraints || {} + end + # Returns max execution time from configuration. # # @return [Integer] Maximum execution time in seconds diff --git a/lib/skill_bench/config/applier.rb b/lib/skill_bench/config/applier.rb index 8c34d90..2cfb527 100644 --- a/lib/skill_bench/config/applier.rb +++ b/lib/skill_bench/config/applier.rb @@ -42,6 +42,7 @@ def apply_scalar_values @store.assign_max_execution_time(@data[:max_execution_time]) if @data.key?(:max_execution_time) @store.assign_allowed_commands(@data[:allowed_commands]) if @data.key?(:allowed_commands) @store.assign_allow_host_execution(@data[:allow_host_execution]) if @data.key?(:allow_host_execution) + @store.assign_command_argument_constraints(@data[:command_argument_constraints]) if @data.key?(:command_argument_constraints) @store.skill_sources = @data[:skill_sources] if @data.key?(:skill_sources) end diff --git a/lib/skill_bench/config/defaults.rb b/lib/skill_bench/config/defaults.rb index 40492b7..86ff913 100644 --- a/lib/skill_bench/config/defaults.rb +++ b/lib/skill_bench/config/defaults.rb @@ -20,6 +20,7 @@ def self.config max_execution_time: 30, allowed_commands: nil, allow_host_execution: false, + command_argument_constraints: {}, skill_sources: {}, llm_providers_config: { openai: { api_key: nil, model: 'gpt-4o' }, diff --git a/lib/skill_bench/config/facade_writers.rb b/lib/skill_bench/config/facade_writers.rb index ee9602b..dfe9ff4 100644 --- a/lib/skill_bench/config/facade_writers.rb +++ b/lib/skill_bench/config/facade_writers.rb @@ -110,6 +110,15 @@ def allow_host_execution=(value) store.assign_allow_host_execution(value) end + # Sets the optional per-command argument constraints. + # + # @param value [Hash, nil] base command => disallowed argument + # substrings/flags + # @return [Hash, nil] assigned constraints + def command_argument_constraints=(value) + store.assign_command_argument_constraints(value) + end + # Replaces provider configuration. # # @param value [Hash] provider configuration diff --git a/lib/skill_bench/config/json_loader.rb b/lib/skill_bench/config/json_loader.rb index 7a52998..7e5471a 100644 --- a/lib/skill_bench/config/json_loader.rb +++ b/lib/skill_bench/config/json_loader.rb @@ -29,7 +29,7 @@ def call data = JSON.parse(File.read(@path), symbolize_names: true) return warn_invalid_config unless data.is_a?(Hash) - success_data = data.slice(:current_llm_provider, :max_execution_time, :allowed_commands, :allow_host_execution, :skill_sources).compact + success_data = data.slice(:current_llm_provider, :max_execution_time, :allowed_commands, :allow_host_execution, :command_argument_constraints, :skill_sources).compact success_data[:current_llm_provider] ||= data[:provider] if data.key?(:provider) success(success_data.merge(providers: normalized_providers(data[:providers]))) rescue JSON::ParserError => e diff --git a/lib/skill_bench/config/store.rb b/lib/skill_bench/config/store.rb index 0756cb0..e5b3c5a 100644 --- a/lib/skill_bench/config/store.rb +++ b/lib/skill_bench/config/store.rb @@ -25,6 +25,12 @@ class Store # @return [Boolean, nil] true when host execution is explicitly allowed attr_reader :allow_host_execution + # Returns the optional per-command argument constraints. + # + # @return [Hash, nil] base command => disallowed argument + # substrings/flags, or nil when unconfigured + attr_reader :command_argument_constraints + # Returns provider configuration. # # @return [Hash, nil] provider configuration by provider name @@ -123,6 +129,15 @@ def assign_allow_host_execution(value) @allow_host_execution = value end + # Sets the optional per-command argument constraints. + # + # @param value [Hash, nil] base command => disallowed argument + # substrings/flags + # @return [Hash, nil] assigned constraints + def assign_command_argument_constraints(value) + @command_argument_constraints = value + end + # Sets provider configuration. # # @param value [Hash] provider configuration diff --git a/lib/skill_bench/models/config.rb b/lib/skill_bench/models/config.rb index 87881f6..8cf7dc5 100644 --- a/lib/skill_bench/models/config.rb +++ b/lib/skill_bench/models/config.rb @@ -24,6 +24,30 @@ def self.load(path = 'skill-bench.json') new(raw_data) end + # Returns the configuration for a path, memoizing the parse per run. + # + # Hot paths such as {SkillBench::Services::ProviderResolver} resolve the + # provider on every run, yet skill-bench.json is stable within a single + # run. The parse is cached per absolute path and invalidated when the + # file's mtime changes, so the file is parsed at most once per run while + # a rewritten file (for example between tests) is still re-read. Reset by + # setting the @loaded ivar to nil. + # + # @param path [String] Path to config file (default: skill-bench.json) + # @return [SkillBench::Models::Config] Memoized config instance + # @raise [Errno::ENOENT] if config file not found + def self.loaded(path = 'skill-bench.json') + key = File.expand_path(path) + mtime = File.mtime(key) + cache = (@loaded ||= {}) + entry = cache[key] + return entry[:config] if entry && entry[:mtime] == mtime + + config = load(path) + cache[key] = { mtime: mtime, config: config } + config + end + # Returns the configured provider name # @return [String, nil] Provider name def provider_name @@ -36,6 +60,14 @@ def provider_config @data[:config] || {} end + # Indicates whether the config explicitly selects the built-in mock + # provider, as opposed to having no provider configured at all. + # + # @return [Boolean] true when the configured provider is 'mock' + def mock? + provider_name == 'mock' + end + # Returns max execution time # @return [Integer] Max execution time in seconds def max_execution_time diff --git a/lib/skill_bench/output_formatter.rb b/lib/skill_bench/output_formatter.rb index 55f4cdf..0a9b23d 100644 --- a/lib/skill_bench/output_formatter.rb +++ b/lib/skill_bench/output_formatter.rb @@ -5,6 +5,7 @@ require_relative 'services/feedback_generator' require_relative 'services/json_formatter' require_relative 'services/junit_formatter' +require_relative 'services/html_formatter' module SkillBench # Handles formatting output for different use cases (human, CI, etc.). @@ -14,7 +15,7 @@ class OutputFormatter # Format the eval result for output. # # @param result [Hash] Eval result with keys like :eval_name, :pass, :score, etc. - # @param format [Symbol] Output format (:human, :json, :junit) + # @param format [Symbol] Output format (:human, :json, :junit, :html) # @return [String] Formatted output string def self.format(result, format: :human) case format @@ -22,6 +23,8 @@ def self.format(result, format: :human) Services::JsonFormatter.format(result) when :junit Services::JUnitFormatter.format(result) + when :html + Services::HtmlFormatter.format(result) else format_human(result) end diff --git a/lib/skill_bench/package_verifier.rb b/lib/skill_bench/package_verifier.rb index 7380c64..ab07b13 100644 --- a/lib/skill_bench/package_verifier.rb +++ b/lib/skill_bench/package_verifier.rb @@ -25,7 +25,7 @@ class Error < StandardError; end lib/skill_bench/config/json_loader.rb lib/skill_bench/config/store.rb lib/skill_bench/package_verifier.rb - lib/skill_bench/source_path_resolver.rb + lib/skill_bench/execution/source_path_resolver.rb lib/skill_bench/runner.rb ].freeze diff --git a/lib/skill_bench/services/compare_option_parser.rb b/lib/skill_bench/services/compare_option_parser.rb index 05f54b0..c6228e2 100644 --- a/lib/skill_bench/services/compare_option_parser.rb +++ b/lib/skill_bench/services/compare_option_parser.rb @@ -44,6 +44,7 @@ def build_parser(options) opts.on('--variant-b SPEC', 'Second variant (e.g., "pack:hanami" or "/path/to/skill")') { |v| options[:variant_b] = v } opts.on('--eval PATH', 'Path to the eval directory') { |v| options[:eval] = v } opts.on('--format FORMAT', 'Output format (human, json)') { |v| options[:format] = v.to_sym } + opts.on('--cache', 'Enable content-addressed response caching') { ENV['SKILL_BENCH_CACHE'] = '1' } opts.on('-h', '--help', 'Prints this help') do puts opts raise SkillBench::HelpRequested diff --git a/lib/skill_bench/services/html_formatter.rb b/lib/skill_bench/services/html_formatter.rb new file mode 100644 index 0000000..a39f01d --- /dev/null +++ b/lib/skill_bench/services/html_formatter.rb @@ -0,0 +1,289 @@ +# frozen_string_literal: true + +require 'cgi' +require_relative 'formatting_helpers' +require_relative '../delta_report' + +module SkillBench + module Services + # Formats evaluation results as a complete, self-contained HTML document. + # + # The output embeds all styling inline (no external assets) and escapes every + # dynamic, user-derived value with {CGI.escapeHTML} to prevent HTML injection. + # Both the modern DeltaReport shape and the legacy result shape are supported. + class HtmlFormatter + extend FormattingHelpers + + # Inline stylesheet embedded in every generated document. + STYLE = <<~CSS + body { font-family: -apple-system, Segoe UI, Roboto, sans-serif; margin: 2rem; color: #1a1a1a; background: #fafafa; } + main { max-width: 960px; margin: 0 auto; } + header { border-bottom: 2px solid #ddd; padding-bottom: 1rem; margin-bottom: 1.5rem; } + h1 { margin: 0 0 0.5rem; font-size: 1.6rem; } + dl.meta { display: grid; grid-template-columns: max-content 1fr; gap: 0.2rem 1rem; margin: 0.5rem 0; } + dl.meta dt { font-weight: 600; color: #555; } + dl.meta dd { margin: 0; } + p.usage { color: #555; font-variant-numeric: tabular-nums; } + table { border-collapse: collapse; width: 100%; margin: 1rem 0; } + th, td { padding: 0.4rem 0.75rem; text-align: right; border-bottom: 1px solid #e2e2e2; } + th:first-child, td:first-child { text-align: left; } + tr.total td { font-weight: 700; border-top: 2px solid #bbb; } + p.verdict { font-weight: 700; padding: 0.5rem 0.75rem; border-radius: 4px; display: inline-block; } + p.verdict.pass { background: #e6f4ea; color: #1e7e34; } + p.verdict.fail { background: #fde8e8; color: #c0392b; } + p.error { color: #c0392b; } + section.iterations h3 { margin-bottom: 0.25rem; } + ol { margin: 0.25rem 0 1rem; } + li { margin: 0.2rem 0; } + span.tools, span.observation { color: #555; } + CSS + + # Format an eval result as a full HTML document. + # + # @param result [Hash] Eval result envelope (DeltaReport or legacy shape). + # @return [String] A complete HTML document string. + def self.format(result) + report = result.dig(:response, :report) + body = report.is_a?(SkillBench::DeltaReport) ? delta_body(result, report) : legacy_section(result) + build_document(result, body) + end + + # Builds the body for a DeltaReport result (table plus iteration timeline). + # + # @param result [Hash] Eval result envelope. + # @param report [SkillBench::DeltaReport] The delta report. + # @return [String] HTML for the report and iteration sections. + def self.delta_body(result, report) + "#{report_section(report)}\n#{iterations_section(result)}" + end + private_class_method :delta_body + + # Wraps body HTML in a complete, styled HTML document. + # + # @param result [Hash] Eval result envelope (used for the header/title). + # @param body [String] Pre-rendered body HTML. + # @return [String] A complete HTML document string. + def self.build_document(result, body) + title = escape(result[:eval_name] || 'Report') + <<~HTML + + + + + SkillBench Report — #{title} + + + +
+ #{header_html(result)} + #{body} +
+ + + HTML + end + private_class_method :build_document + + # Builds the header with eval/skill/provider names and the usage line. + # + # @param result [Hash] Eval result envelope. + # @return [String] HTML for the document header. + def self.header_html(result) + <<~HTML.chomp +
+

SkillBench Report

+
+
Eval
#{escape(result[:eval_name])}
+
Skill
#{escape(result[:skill_name])}
+
Provider
#{escape(result[:provider_name])}
+
+

#{usage_line(result)}

+
+ HTML + end + private_class_method :header_html + + # Builds the token/cost summary line for the header. + # + # @param result [Hash] Eval result envelope; reads :tokens and :cost. + # @return [String] An escaped "Tokens / Est. Cost" line. + def self.usage_line(result) + tokens = result[:tokens] || {} + total = tokens[:total_tokens] || tokens['total_tokens'] || 0 + cost = result[:cost] + cost_label = cost ? Kernel.format('$%.4f', cost) : '—' + "Tokens: #{escape(total)} | Est. Cost: #{escape(cost_label)}" + end + private_class_method :usage_line + + # Builds the scoring table and verdict for a DeltaReport. + # + # @param report [SkillBench::DeltaReport] The delta report. + # @return [String] HTML for the report section. + def self.report_section(report) + <<~HTML.chomp +
+

Delta Report

+ + + + #{dimension_rows(report)} + #{total_row(report)} + +
DimensionBaselineContextDelta
+ #{verdict_html(report)} +
+ HTML + end + private_class_method :report_section + + # Builds one table row per scored dimension. + # + # @param report [SkillBench::DeltaReport] The delta report. + # @return [String] HTML table rows joined by newlines. + def self.dimension_rows(report) + report.deltas.map { |name, delta| dimension_row(name, delta, report) }.join("\n") + end + private_class_method :dimension_rows + + # Builds a single dimension table row. + # + # @param name [String] Dimension name. + # @param delta [Numeric] Context-minus-baseline delta. + # @param report [SkillBench::DeltaReport] The delta report. + # @return [String] An HTML table row. + def self.dimension_row(name, delta, report) + dim = report.criteria.dimensions.find { |candidate| candidate.name == name } + humanized = humanize(name) + label = dim ? "#{humanized} (#{dim.max_score})" : humanized + baseline = report.baseline_scores[name] + context = report.context_scores[name] + row_cells('dimension', label, baseline, context, delta_str(delta)) + end + private_class_method :dimension_row + + # Builds the totals table row. + # + # @param report [SkillBench::DeltaReport] The delta report. + # @return [String] An HTML table row for the totals. + def self.total_row(report) + total_delta = report.deltas.values.sum + row_cells('total', 'Total', "#{report.baseline_total}/100", + "#{report.context_total}/100", delta_str(total_delta)) + end + private_class_method :total_row + + # Builds an HTML table row from escaped cell values. + # + # @param css_class [String] CSS class for the row. + # @param label [String] First-column label. + # @param baseline [Object] Baseline score cell. + # @param context [Object] Context score cell. + # @param delta [String] Delta cell. + # @return [String] An HTML table row. + def self.row_cells(css_class, label, baseline, context, delta) + "#{escape(label)}#{escape(baseline)}" \ + "#{escape(context)}#{escape(delta)}" + end + private_class_method :row_cells + + # Builds the verdict paragraph. + # + # @param report [SkillBench::DeltaReport] The delta report. + # @return [String] An HTML verdict paragraph. + def self.verdict_html(report) + verdict = report.verdict + criteria = report.criteria + status = verdict ? 'PASS' : 'FAIL' + css = verdict ? 'pass' : 'fail' + threshold = escape(criteria.pass_threshold) + minimum_delta = escape(criteria.minimum_delta) + %(

Verdict: #{status} (threshold: #{threshold}, minimum delta: #{minimum_delta})

) + end + private_class_method :verdict_html + + # Builds the baseline/context iteration timeline section. + # + # @param result [Hash] Eval result envelope. + # @return [String] HTML for the iterations section, or empty string. + def self.iterations_section(result) + baseline = result.dig(:response, :baseline_iterations) || [] + context = result.dig(:response, :context_iterations) || [] + baseline_empty = baseline.empty? + context_empty = context.empty? + return '' if baseline_empty && context_empty + + blocks = [] + blocks << iteration_block('Baseline Iterations', baseline) unless baseline_empty + blocks << iteration_block('Context Iterations', context) unless context_empty + %(
\n

Iteration Timeline

\n#{blocks.join("\n")}\n
) + end + private_class_method :iterations_section + + # Builds one named iteration timeline block. + # + # @param title [String] Section title. + # @param iterations [Array] Iteration metadata entries. + # @return [String] HTML for the timeline block. + def self.iteration_block(title, iterations) + items = iterations.map { |iteration| iteration_item(iteration) }.join("\n") + %(

#{escape(title)}

    \n#{items}\n
) + end + private_class_method :iteration_block + + # Builds one list item for a single iteration step. + # + # @param iteration [Hash] Iteration metadata with :step_number, :thought, + # :tools_used, and :observation_summary keys. + # @return [String] An HTML list item. + def self.iteration_item(iteration) + tools = iteration[:tools_used] || [] + tools_html = tools.empty? ? '' : %( Tools: #{escape(tools.join(', '))}) + observation = iteration[:observation_summary].to_s + observation_html = observation.empty? ? '' : %( Observation: #{escape(observation)}) + step = "Step #{escape(iteration[:step_number])}: #{escape(iteration[:thought])}" + %(
  • #{step}#{tools_html}#{observation_html}
    • ) + end + private_class_method :iteration_item + + # Builds the body for a legacy (non-DeltaReport) result. + # + # @param result [Hash] Legacy eval result envelope. + # @return [String] HTML for the legacy status section. + def self.legacy_section(result) + passed = result[:pass] + status = passed ? 'PASSED' : 'FAILED' + css = passed ? 'pass' : 'fail' + score = result[:score]&.round(2) + <<~HTML.chomp +
    +

    Result

    +

    Status: #{status}

    +

    Score: #{escape(score || 'N/A')}

    + #{legacy_error(result)} +
    + HTML + end + private_class_method :legacy_section + + # Builds the optional error paragraph for a legacy result. + # + # @param result [Hash] Legacy eval result envelope. + # @return [String] An HTML error paragraph, or empty string. + def self.legacy_error(result) + message = result.dig(:response, :error, :message) + message ? %(

    Error: #{escape(message)}

    ) : '' + end + private_class_method :legacy_error + + # Escapes any value for safe HTML embedding. + # + # @param value [Object] The value to escape (coerced via #to_s). + # @return [String] HTML-escaped text. + def self.escape(value) + CGI.escapeHTML(value.to_s) + end + private_class_method :escape + end + end +end diff --git a/lib/skill_bench/services/json_formatter.rb b/lib/skill_bench/services/json_formatter.rb index 99a039e..3df5b49 100644 --- a/lib/skill_bench/services/json_formatter.rb +++ b/lib/skill_bench/services/json_formatter.rb @@ -27,11 +27,9 @@ def self.format(result) def self.with_usage_fields(result) return result unless result.is_a?(Hash) - defaults = { - tokens: result[:tokens] || result.dig(:response, :tokens) || EMPTY_USAGE, - cost: result.key?(:cost) ? result[:cost] : result.dig(:response, :cost) - } - defaults.merge(result) + tokens = result[:tokens] || result.dig(:response, :tokens) || EMPTY_USAGE + cost = result.key?(:cost) ? result[:cost] : result.dig(:response, :cost) + result.merge(tokens: tokens, cost: cost) end end end diff --git a/lib/skill_bench/services/junit_formatter.rb b/lib/skill_bench/services/junit_formatter.rb index af099e5..ba5dc5c 100644 --- a/lib/skill_bench/services/junit_formatter.rb +++ b/lib/skill_bench/services/junit_formatter.rb @@ -4,39 +4,89 @@ module SkillBench module Services - # Formats evaluation results as JUnit XML. + # Formats evaluation results as JUnit XML for CI consumption. + # + # Two entry points share the same per-result verdict/score logic: + # {.format} emits a single-result suite (one ), while + # {.format_batch} aggregates many results into one suite so a batch + # `skill-bench run --all` produces a single JUnit artifact. class JUnitFormatter - # Format result as JUnit XML. + # classname attribute applied to every emitted . + CLASSNAME = 'SkillBench' + + # Format a single result as a JUnit XML document. # # Supports both legacy format (result[:pass]) and modern DeltaReport format. # # @param result [Hash] Eval result. # @return [String] JUnit XML-formatted string. def self.format(result) + suite([result]) + end + + # Format an aggregate batch envelope as one JUnit XML document. + # + # Emits a single with one per result, adding a + # child for every failing eval. + # + # @param aggregate [Hash] Aggregate envelope with a :results array. + # @return [String] JUnit XML-formatted string. + def self.format_batch(aggregate) + suite(aggregate[:results] || []) + end + + # Builds a wrapping one per result. + # + # @param results [Array] Per-eval result envelopes. + # @return [String] JUnit XML-formatted string. + def self.suite(results) + failures = results.count { |result| !passing?(result) } + cases = results.map { |result| testcase(result) }.join("\n") + <<~XML + + + #{cases} + + XML + end + private_class_method :suite + + # Renders one element (indented two spaces) for a result. + # + # @param result [Hash] A single-eval result envelope. + # @return [String] A XML fragment. + def self.testcase(result) + name = CGI.escapeHTML(result[:eval_name].to_s) + return %( ) if passing?(result) + + score = CGI.escapeHTML(score_for(result).to_s) + [ + %( ), + %( Eval failed), + ' ' + ].join("\n") + end + private_class_method :testcase + + # Whether a result passed (DeltaReport verdict or legacy :pass). + # + # @param result [Hash] A single-eval result envelope. + # @return [Boolean] true when the eval passed. + def self.passing?(result) + report = result.dig(:response, :report) + report.respond_to?(:verdict) ? report.verdict : result[:pass] + end + private_class_method :passing? + + # The score reported for a failing eval. + # + # @param result [Hash] A single-eval result envelope. + # @return [Object] DeltaReport context_total or legacy :score. + def self.score_for(result) report = result.dig(:response, :report) - verdict = report.respond_to?(:verdict) ? report.verdict : result[:pass] - eval_name = CGI.escapeHTML(result[:eval_name].to_s) - - if verdict - <<~XML - - - - - XML - else - score = report.respond_to?(:context_total) ? report.context_total : result[:score] - escaped_score = CGI.escapeHTML(score.to_s) - <<~XML - - - - Eval failed - - - XML - end + report.respond_to?(:context_total) ? report.context_total : result[:score] end + private_class_method :score_for end end end diff --git a/lib/skill_bench/services/provider_resolver.rb b/lib/skill_bench/services/provider_resolver.rb index 4c57012..7a40754 100644 --- a/lib/skill_bench/services/provider_resolver.rb +++ b/lib/skill_bench/services/provider_resolver.rb @@ -51,11 +51,14 @@ def call private def resolve_provider - config = SkillBench::Models::Config.load + config = SkillBench::Models::Config.loaded provider = config.to_provider return provider if provider - warn 'Config load failed, using mock provider' + # Explicit `{"provider":"mock"}` is a valid choice, not a load failure, + # so it falls through to the mock provider without a warning. A missing + # provider key (genuine misconfiguration) still warns below. + warn 'Config load failed, using mock provider' unless config.mock? MOCK_PROVIDER.new('mock', 'mock', 'mock', {}) rescue JSON::ParserError, ArgumentError, Errno::ENOENT => e # Config parsing/validation errors or missing config file - fall back to mock diff --git a/lib/skill_bench/services/response_cache.rb b/lib/skill_bench/services/response_cache.rb new file mode 100644 index 0000000..34a1e52 --- /dev/null +++ b/lib/skill_bench/services/response_cache.rb @@ -0,0 +1,130 @@ +# frozen_string_literal: true + +require 'digest' +require 'json' + +module SkillBench + module Services + # Content-addressed, in-memory cache for LLM responses. + # + # The cache is opt-in and disabled by default. When enabled it lets repeated, + # identical LLM requests reuse a previously computed response instead of + # hitting the network again. The canonical example is `compare`, which runs + # the skill-less baseline twice with identical inputs. + # + # The backing store is a process-lifetime {Hash} keyed by a stable SHA-256 + # digest of the request, so the same logical request always maps to the same + # entry regardless of hash-key ordering. Access to the store is serialized + # with a mutex so concurrent callers (e.g. {Parallel}-driven agents) cannot + # corrupt it or double-store a key. + class ResponseCache + # Environment variable that opts caching on when set to a truthy value. + ENV_FLAG = 'SKILL_BENCH_CACHE' + + # Raw env values treated as "on". + TRUTHY_VALUES = %w[1 true yes on].freeze + + # Guards every read/write of the shared store. Concurrent agents/judges run + # on separate threads; without this, the membership check and the write in + # {fetch} could interleave and store a key more than once. + MUTEX = Mutex.new + + class << self + # Whether response caching is currently enabled. + # + # Enabled when {ENV_FLAG} is set to a truthy value (one of + # {TRUTHY_VALUES}); disabled when unset or set to anything else. + # + # @return [Boolean] true when caching is on + def enabled? + raw = ENV.fetch(ENV_FLAG, '').to_s.strip.downcase + TRUTHY_VALUES.include?(raw) + end + + # Computes a stable content-addressed cache key for a request. + # + # The inputs are assembled into a canonical structure (hash keys sorted + # and stringified recursively) and hashed, so semantically identical + # requests always produce the same digest. Request-affecting provider + # configuration (endpoint/base URL/etc.) is included so two providers that + # share a name but target different endpoints never collide. + # + # @param provider [Symbol, String] Resolved provider identifier + # @param model [String, nil] Model name + # @param system_prompt [String] System prompt + # @param messages [Array] Conversation messages + # @param tools [Array, nil] Tool definitions, when present + # @param temperature [Float, nil] Sampling temperature, when present + # @param provider_config [Hash] Request-affecting provider settings such as + # base_url, request_path, endpoint, location, project_id, api_version + # @return [String] Hex-encoded SHA-256 digest of the canonical request + def key(provider:, model:, system_prompt:, messages:, tools: nil, temperature: nil, provider_config: {}) + payload = { + provider: provider.to_s, + model: model, + system_prompt: system_prompt, + messages: messages, + tools: tools, + temperature: temperature, + provider_config: provider_config + } + Digest::SHA256.hexdigest(JSON.generate(canonicalize(payload))) + end + + # Returns the cached value for a key, computing and storing it on a miss. + # + # The value is computed outside the lock so requests for distinct keys run + # concurrently; the store read and the store write are each serialized by + # {MUTEX}, and a missing key is written exactly once (first writer wins). + # + # @param key [String] Cache key from {key} + # @yield Computes the value to cache when the key is absent + # @yieldreturn [Object] The value to cache + # @return [Object] The cached value (existing on a hit, freshly stored on a miss) + def fetch(key) + hit = MUTEX.synchronize { store[key] } + return hit unless hit.nil? + + value = yield + MUTEX.synchronize { store[key] ||= value } + end + + # Removes every cached entry. + # + # @return [void] + def clear + MUTEX.synchronize { store.clear } + end + + private + + # The process-lifetime backing store. + # + # @return [Hash{String => Object}] digest => cached response + def store + @store ||= {} + end + + # Recursively rewrites a value into a stable form for serialization. + # + # Hashes get their keys stringified and sorted so that key ordering does + # not affect the resulting digest; arrays and scalars are preserved. + # + # @param value [Object] The value to canonicalize + # @return [Object] A canonical, order-stable copy of the value + def canonicalize(value) + case value + when Hash + value + .sort_by { |entry| entry.first.to_s } + .each_with_object({}) { |(name, val), acc| acc[name.to_s] = canonicalize(val) } + when Array + value.map { |element| canonicalize(element) } + else + value + end + end + end + end + end +end diff --git a/lib/skill_bench/services/summary_formatter.rb b/lib/skill_bench/services/summary_formatter.rb new file mode 100644 index 0000000..a684e44 --- /dev/null +++ b/lib/skill_bench/services/summary_formatter.rb @@ -0,0 +1,90 @@ +# frozen_string_literal: true + +require 'json' + +module SkillBench + module Services + # Builds a compact JSON summary of a batch run for CI gating. + # + # Surfaces the aggregate pass/fail counts plus rolled-up token and cost + # usage and the single worst skill-vs-baseline delta across the batch, so + # a CI job can gate on (and archive) one machine-readable artifact. + class SummaryFormatter + # Format an aggregate batch envelope as a pretty JSON summary string. + # + # @param aggregate [Hash] Aggregate envelope with :results and :summary. + # @return [String] Pretty-printed JSON summary. + def self.format(aggregate) + new(aggregate).format + end + + # @param aggregate [Hash] Aggregate envelope with :results and :summary. + def initialize(aggregate) + @results = aggregate[:results] || [] + @summary = aggregate[:summary] || {} + end + + # Builds the JSON summary document. + # + # @return [String] Pretty-printed JSON summary. + def format + JSON.pretty_generate( + passed: summary[:passed], + failed: summary[:failed], + total: summary[:total], + tokens: total_tokens, + cost: total_cost, + worst_delta: worst_delta + ) + end + + private + + attr_reader :results, :summary + + # Sums total_tokens across every result, treating missing usage as 0. + # + # @return [Integer] Aggregate token count. + def total_tokens + results.sum { |result| tokens_for(result) } + end + + # Reads a single result's total token count. + # + # @param result [Hash] A single-eval result envelope. + # @return [Integer] total_tokens, or 0 when absent. + def tokens_for(result) + tokens = result[:tokens] || {} + tokens[:total_tokens] || tokens['total_tokens'] || 0 + end + + # Sums non-nil per-result costs. + # + # @return [Float, nil] Total cost, or nil when no result reports a cost. + def total_cost + costs = results.filter_map { |result| result[:cost] } + costs.empty? ? nil : costs.sum + end + + # Finds the eval with the smallest skill-vs-baseline delta. + # + # @return [Hash, nil] {:eval_name, :delta} for the worst eval, or nil + # when no result carries a delta report. + def worst_delta + scored = results.filter_map { |result| delta_entry(result) } + scored.min_by { |entry| entry[:delta] } + end + + # Builds a {eval_name, delta} entry for a result with a delta report. + # + # @param result [Hash] A single-eval result envelope. + # @return [Hash, nil] Entry hash, or nil when the report lacks deltas. + def delta_entry(result) + report = result.dig(:response, :report) + return nil unless report.respond_to?(:context_total) && report.respond_to?(:baseline_total) + + { eval_name: result[:eval_name], delta: report.context_total - report.baseline_total } + end + end + end +end diff --git a/lib/skill_bench/tools/run_command.rb b/lib/skill_bench/tools/run_command.rb index 6460374..b3d40d7 100644 --- a/lib/skill_bench/tools/run_command.rb +++ b/lib/skill_bench/tools/run_command.rb @@ -75,12 +75,40 @@ def self.call(command, working_dir_path, container_id = nil) return 'Error: No allowed commands configured. Set allowed_commands in skill-bench.json or use --mode mock.' if allowed.nil? return "Error: Command '#{base_cmd}' is not permitted." unless allowed.include?(base_cmd) + return "Error: Command '#{base_cmd}' arguments are not permitted by the configured argument constraints." unless arguments_permitted?(base_cmd, argv) + return HOST_EXECUTION_REFUSED unless container_id || SkillBench::Config.allow_host_execution warn_unisolated_host_execution unless container_id execute(argv, working_dir_path, container_id) end + # Checks the command's arguments against the optional, per-command + # argument constraints from configuration. + # + # This is a default-off seam: the command allowlist remains the primary + # authorization control, and any allowlisted wrapper binary still grants + # broad host execution. When no constraints are configured (the default), + # or none apply to +base_cmd+, every argument is permitted so behavior is + # unchanged. When a constraint exists for +base_cmd+, the command is + # refused if any argument contains a disallowed substring/flag. + # + # @param base_cmd [String] The base command (first token of the command). + # @param argv [Array] The tokenized command and arguments. + # @return [Boolean] true when the arguments are permitted to run. + def self.arguments_permitted?(base_cmd, argv) + constraints = SkillBench::Config.command_argument_constraints + return true if constraints.nil? || constraints.empty? + + # Constraint keys may be strings (facade API) or symbols (loaded from + # JSON via symbolize_names), so look the command up under both. + disallowed = constraints[base_cmd] || constraints[base_cmd.to_sym] + return true if disallowed.nil? || disallowed.empty? + + argv.drop(1).none? { |arg| disallowed.any? { |bad| arg.include?(bad.to_s) } } + end + private_class_method :arguments_permitted? + # Runs the resolved command and formats its result, enforcing the # configured execution timeout. # diff --git a/test/agent_eval/commands/init_test.rb b/test/agent_eval/commands/init_test.rb index 1f3caac..8d82373 100644 --- a/test/agent_eval/commands/init_test.rb +++ b/test/agent_eval/commands/init_test.rb @@ -52,6 +52,22 @@ def test_run_overwrites_when_force_true refute_equal({ 'invalid' => true }, config) assert config.key?('provider') end + + def test_run_with_mock_provider_writes_minimal_config + Init.run(provider: :mock, force: false) + + content = File.read(SkillBench::Config::CONFIG_FILENAME) + config = JSON.parse(content, symbolize_names: true) + + assert_equal 'mock', config[:provider] + assert_equal 30, config[:max_execution_time] + refute config.key?(:config) + refute config.key?(:api_key) + end + + def test_config_for_provider_mock_omits_config_block + assert_equal({ provider: :mock, max_execution_time: 30 }, Init.config_for_provider(:mock)) + end end end end diff --git a/test/agent_eval/models/config_test.rb b/test/agent_eval/models/config_test.rb index de60b9d..20e4b4d 100644 --- a/test/agent_eval/models/config_test.rb +++ b/test/agent_eval/models/config_test.rb @@ -71,6 +71,36 @@ def test_max_execution_time_default assert_equal 30, config.max_execution_time end + + def test_mock_predicate_true_for_mock_provider + assert_predicate Config.new({ provider: 'mock' }), :mock? + end + + def test_mock_predicate_false_for_other_or_missing_provider + refute_predicate Config.new({ provider: 'openai' }), :mock? + refute_predicate Config.new({}), :mock? + end + + def test_loaded_memoizes_and_reads_file_once + Config.instance_variable_set(:@loaded, nil) + File.write('skill-bench.json', JSON.generate({ provider: 'mock', max_execution_time: 30, config: {} })) + loads = 0 + stub_config = Config.new({ provider: 'mock', max_execution_time: 30, config: {} }) + + Config.stub(:load, lambda { |*_args| + loads += 1 + stub_config + }) do + first = Config.loaded + second = Config.loaded + + assert_same first, second + end + + assert_equal 1, loads + ensure + Config.instance_variable_set(:@loaded, nil) + end end end end diff --git a/test/agent_eval/services/provider_resolver_test.rb b/test/agent_eval/services/provider_resolver_test.rb index adeb739..062065c 100644 --- a/test/agent_eval/services/provider_resolver_test.rb +++ b/test/agent_eval/services/provider_resolver_test.rb @@ -42,8 +42,69 @@ def test_call_returns_error_when_config_fails assert result[:provider] end + def test_explicit_mock_config_does_not_warn + write_mock_config + + stderr = capture_stderr { ProviderResolver.call } + + refute_includes stderr, 'Config load failed' + end + + def test_missing_config_still_warns + Models::Config.instance_variable_set(:@loaded, nil) + + result = nil + stderr = capture_stderr { result = ProviderResolver.call } + + assert_includes stderr, 'Config load failed' + assert_equal 'mock', result[:provider].name + end + + def test_broken_config_still_warns + Models::Config.instance_variable_set(:@loaded, nil) + File.write(SkillBench::Config::CONFIG_FILENAME, '{ not valid json') + + stderr = capture_stderr { ProviderResolver.call } + + assert_includes stderr, 'Config load failed' + end + + def test_missing_provider_key_still_warns + Models::Config.instance_variable_set(:@loaded, nil) + File.write(SkillBench::Config::CONFIG_FILENAME, JSON.generate({ max_execution_time: 30 })) + + stderr = capture_stderr { ProviderResolver.call } + + assert_includes stderr, 'Config load failed, using mock provider' + end + + def test_resolve_reads_config_file_once_across_calls + write_mock_config + loads = 0 + stub_config = Models::Config.new({ provider: 'mock', max_execution_time: 30, config: {} }) + + Models::Config.stub(:load, lambda { |*_args| + loads += 1 + stub_config + }) do + ProviderResolver.call + ProviderResolver.call + end + + assert_equal 1, loads + end + private + def capture_stderr + original = $stderr + $stderr = StringIO.new + yield + $stderr.string + ensure + $stderr = original + end + def write_mock_config Models::Config.instance_variable_set(:@loaded, nil) File.write(SkillBench::Config::CONFIG_FILENAME, JSON.generate({ diff --git a/test/cli/batch_result_printer_test.rb b/test/cli/batch_result_printer_test.rb new file mode 100644 index 0000000..ed28a1f --- /dev/null +++ b/test/cli/batch_result_printer_test.rb @@ -0,0 +1,61 @@ +# frozen_string_literal: true + +require 'test_helper' +require 'json' + +module SkillBench + module Cli + class BatchResultPrinterTest < Minitest::Test + def test_human_output_is_the_default + exit_code = nil + out, = capture_io { exit_code = BatchResultPrinter.call(aggregate(failed: 0)) } + + assert_equal 0, exit_code + assert_includes out, 'PASS evals/eval-a' + assert_includes out, 'Summary: 1 passed / 0 failed (1 total)' + end + + def test_junit_format_emits_a_testsuite_and_failure_exit_code + exit_code = nil + out, = capture_io { exit_code = BatchResultPrinter.call(aggregate(failed: 1), format: :junit) } + + assert_equal 1, exit_code + assert_includes out, '' + assert_includes out, '' + end + + def test_summary_emits_json_gate_and_failure_exit_code + exit_code = nil + out, = capture_io { exit_code = BatchResultPrinter.call(aggregate(failed: 1), summary: true) } + parsed = JSON.parse(out) + + assert_equal 1, exit_code + assert_equal 1, parsed['passed'] + assert_equal 1, parsed['failed'] + assert_equal 2, parsed['total'] + end + + def test_summary_takes_precedence_over_format + out, = capture_io { BatchResultPrinter.call(aggregate(failed: 0), format: :junit, summary: true) } + + assert_includes out, '"worst_delta"' + refute_includes out, '] Messages (ignored) + # @param _options [Hash] Provider options (ignored) + # @return [Hash] A fixed success response + def call(system_prompt:, messages:, **_options) + _ = [system_prompt, messages] + self.calls = (calls || 0) + 1 + { success: true, response: { message: { content: 'ok' } } } + end + end + end + + def setup + @original_env = ENV.fetch('SKILL_BENCH_CACHE', nil) + ENV.delete('SKILL_BENCH_CACHE') + Services::ResponseCache.clear + CountingClient.calls = 0 + @original_providers = Clients::ProviderRegistry.providers.dup + Clients::ProviderRegistry.register(:counting, CountingClient) + end + + def teardown + Clients::ProviderRegistry.instance_variable_set(:@providers, @original_providers) + Services::ResponseCache.clear + if @original_env.nil? + ENV.delete('SKILL_BENCH_CACHE') + else + ENV['SKILL_BENCH_CACHE'] = @original_env + end + end + + def test_caching_disabled_invokes_provider_for_every_call + 2.times { call_counting('hi') } + + assert_equal 2, CountingClient.calls + end + + def test_caching_enabled_dedupes_identical_calls + ENV['SKILL_BENCH_CACHE'] = '1' + + 2.times { call_counting('hi') } + + assert_equal 1, CountingClient.calls + end + + def test_caching_enabled_distinguishes_different_inputs + ENV['SKILL_BENCH_CACHE'] = '1' + + call_counting('a') + call_counting('b') + + assert_equal 2, CountingClient.calls + end + + def test_caching_enabled_returns_identical_cached_response + ENV['SKILL_BENCH_CACHE'] = '1' + + first = call_counting('hi') + second = call_counting('hi') + + assert_same first, second + end + + def test_null_client_is_never_cached + ENV['SKILL_BENCH_CACHE'] = '1' + + response = Client.call(system_prompt: 'sys', messages: [{ role: 'user', content: 'hi' }], provider: :counting_unknown) + + refute response[:success] + end + + private + + def call_counting(content) + Client.call( + system_prompt: 'sys', + messages: [{ role: 'user', content: content }], + provider: :counting, + model: 'gpt-4o' + ) + end + end +end diff --git a/test/evaluator/clients/providers/mistral_test.rb b/test/evaluator/clients/providers/mistral_test.rb new file mode 100644 index 0000000..6011bd6 --- /dev/null +++ b/test/evaluator/clients/providers/mistral_test.rb @@ -0,0 +1,101 @@ +# frozen_string_literal: true + +require 'test_helper' + +module SkillBench + module Clients + module Providers + class MistralTest < Minitest::Test + def setup + Config.reset + Config.current_llm_provider = :mistral + end + + def test_call_returns_message_content_on_success + Config.setup do |config| + config.set_provider_api_key(:mistral, 'test_mistral_key') + end + + stub_request(:post, %r{api\.mistral\.ai/v1/chat/completions}) + .to_return( + status: 200, + body: { choices: [{ message: { content: 'Hello from Mistral', role: 'assistant' } }] }.to_json, + headers: { 'Content-Type' => 'application/json' } + ) + + old_stderr = $stderr + $stderr = StringIO.new + result = Mistral.call( + api_key: 'test_mistral_key', + model: 'mistral-large-latest', + system_prompt: 'System', + messages: [{ role: 'user', content: 'Hi' }] + ) + ensure + $stderr = old_stderr + + assert result[:success] + assert_equal 'Hello from Mistral', result[:response][:message]['content'] + end + + def test_call_sends_request_to_mistral_endpoint_with_bearer_auth + Config.setup do |config| + config.set_provider_api_key(:mistral, 'test_mistral_key') + end + + stub = stub_request(:post, 'https://api.mistral.ai/v1/chat/completions') + .with( + headers: { 'Authorization' => 'Bearer test_mistral_key' }, + body: hash_including('model' => 'mistral-large-latest') + ) + .to_return( + status: 200, + body: { choices: [{ message: { content: 'ok', role: 'assistant' } }] }.to_json, + headers: { 'Content-Type' => 'application/json' } + ) + + old_stderr = $stderr + $stderr = StringIO.new + Mistral.call( + api_key: 'test_mistral_key', + model: 'mistral-large-latest', + system_prompt: 'System', + messages: [{ role: 'user', content: 'Hi' }] + ) + ensure + $stderr = old_stderr + + assert_requested(stub) + end + + def test_call_returns_error_on_api_failure + Config.setup do |config| + config.set_provider_api_key(:mistral, 'test_mistral_key') + end + + stub_request(:post, %r{api\.mistral\.ai/v1/chat/completions}) + .to_return(status: 400, body: 'Bad Request') + + old_stderr = $stderr + $stderr = StringIO.new + result = Mistral.call( + api_key: 'test_mistral_key', + model: 'mistral-large-latest', + system_prompt: 'System', + messages: [{ role: 'user', content: 'Hi' }] + ) + ensure + $stderr = old_stderr + + refute result[:success] + end + + def test_provider_name + client = Mistral.new(system_prompt: '', messages: []) + + assert_equal :mistral, client.provider_name + end + end + end + end +end diff --git a/test/evaluator/tools/run_command_test.rb b/test/evaluator/tools/run_command_test.rb index 85e7f14..e4318e5 100644 --- a/test/evaluator/tools/run_command_test.rb +++ b/test/evaluator/tools/run_command_test.rb @@ -85,6 +85,53 @@ def test_call_executes_in_container end end + def test_call_runs_with_args_when_no_argument_constraints_configured + # Default (no command_argument_constraints) leaves behavior unchanged. + SkillBench::Config.allow_host_execution = true + + Dir.mktmpdir do |dir| + working_dir = Pathname.new(dir).expand_path + + result = RunCommand.call('echo hello world', working_dir) + + assert_match(/STDOUT:\nhello world/, result) + assert_match(/Exit Status: 0/, result) + end + end + + def test_call_refuses_command_whose_args_hit_configured_constraint + SkillBench::Config.allow_host_execution = true + SkillBench::Config.command_argument_constraints = { 'echo' => ['-n'] } + + Dir.mktmpdir do |dir| + working_dir = Pathname.new(dir).expand_path + + # The constraint must block before anything is executed. + Open3.expects(:capture3).never + + result = RunCommand.call('echo -n test', working_dir) + + assert_equal( + "Error: Command 'echo' arguments are not permitted by the configured argument constraints.", + result + ) + end + end + + def test_call_runs_when_constrained_command_has_clean_args + SkillBench::Config.allow_host_execution = true + SkillBench::Config.command_argument_constraints = { 'echo' => ['-n'] } + + Dir.mktmpdir do |dir| + working_dir = Pathname.new(dir).expand_path + + result = RunCommand.call('echo hello', working_dir) + + assert_match(/STDOUT:\nhello/, result) + assert_match(/Exit Status: 0/, result) + end + end + def test_call_returns_timeout_result_without_waiting_for_full_runtime SkillBench::Config.allow_host_execution = true SkillBench::Config.allowed_commands = %w[sleep] @@ -123,6 +170,34 @@ def test_call_kills_runaway_child_process_on_timeout assert_empty lingering, 'the runaway sleep child must be killed and reaped on timeout' end end + + def test_call_enforces_argument_constraints_loaded_from_json_config + config = { + provider: 'mock', + allowed_commands: %w[echo], + allow_host_execution: true, + command_argument_constraints: { 'echo' => ['-n'] } + } + + Dir.mktmpdir do |dir| + Dir.chdir(dir) do + File.write('skill-bench.json', JSON.generate(config)) + SkillBench::Config.reset + + working_dir = Pathname.new(dir).expand_path + + # The constraint loaded from skill-bench.json (symbol-keyed after + # symbolize_names) must block execution before anything is spawned. + Open3.expects(:popen3).never + result = RunCommand.call('echo -n test', working_dir) + + assert_equal( + "Error: Command 'echo' arguments are not permitted by the configured argument constraints.", + result + ) + end + end + end end end end diff --git a/test/services/html_formatter_test.rb b/test/services/html_formatter_test.rb new file mode 100644 index 0000000..add0b7a --- /dev/null +++ b/test/services/html_formatter_test.rb @@ -0,0 +1,195 @@ +# frozen_string_literal: true + +require 'test_helper' + +module SkillBench + module Services + class HtmlFormatterTest < Minitest::Test + def test_format_returns_complete_html_document + output = HtmlFormatter.format(delta_result) + + assert output.start_with?(''), 'should start with the HTML doctype' + assert_includes output, '' + assert_includes output, '' + end + + def test_format_includes_eval_skill_provider_names + output = HtmlFormatter.format(delta_result) + + assert_includes output, 'delta-eval' + assert_includes output, 'test-skill' + assert_includes output, 'mock' + end + + def test_format_renders_tokens_and_cost + output = HtmlFormatter.format(delta_result) + + assert_includes output, 'Tokens: 150' + assert_includes output, 'Est. Cost: $0.0125' + end + + def test_format_renders_dash_cost_when_unknown + result = delta_result.merge(tokens: { total_tokens: 0 }, cost: nil) + output = HtmlFormatter.format(result) + + assert_includes output, 'Est. Cost: —' + end + + def test_format_renders_per_criterion_scores_and_deltas + output = HtmlFormatter.format(delta_result) + + assert_includes output, '' + assert_includes output, 'Correctness (30)' + assert_includes output, 'Skill Adherence (25)' + assert_includes output, '' # baseline correctness + assert_includes output, '' # context correctness + assert_includes output, '' # correctness delta + assert_includes output, '' # skill_adherence delta + end + + def test_format_renders_totals_row + output = HtmlFormatter.format(delta_result) + + assert_includes output, 'class="total"' + assert_includes output, '17/100' + assert_includes output, '50/100' + end + + def test_format_renders_pass_verdict + output = HtmlFormatter.format(delta_result(verdict: true)) + + assert_includes output, 'class="verdict pass"' + assert_includes output, 'Verdict: PASS' + end + + def test_format_renders_fail_verdict + output = HtmlFormatter.format(delta_result(verdict: false)) + + assert_includes output, 'class="verdict fail"' + assert_includes output, 'Verdict: FAIL' + end + + def test_format_renders_iteration_timelines + result = delta_result + result[:response][:baseline_iterations] = [ + { step_number: 1, thought: 'Read file', tools_used: %w[read_file], observation_summary: 'content' } + ] + result[:response][:context_iterations] = [ + { step_number: 1, thought: 'Final answer', tools_used: [], observation_summary: '' } + ] + output = HtmlFormatter.format(result) + + assert_includes output, 'Iteration Timeline' + assert_includes output, 'Baseline Iterations' + assert_includes output, 'Step 1: Read file' + assert_includes output, 'read_file' + assert_includes output, 'Context Iterations' + assert_includes output, 'Step 1: Final answer' + end + + def test_format_omits_iterations_section_when_absent + output = HtmlFormatter.format(delta_result) + + refute_includes output, 'Iteration Timeline' + end + + def test_format_escapes_html_significant_characters_in_names + result = delta_result(skill_name: 'aalert(1)', tools_used: [], observation_summary: '' } + ] + output = HtmlFormatter.format(result) + + assert_includes output, '<script>alert(1)</script>' + refute_includes output, '' + end + + def test_format_legacy_result_produces_valid_html + result = { + eval_name: 'legacy-eval', + skill_name: 'legacy-skill', + provider_name: 'openai', + pass: true, + score: 0.875 + } + output = HtmlFormatter.format(result) + + assert output.start_with?('') + assert_includes output, '' + assert_includes output, 'legacy-eval' + assert_includes output, 'legacy-skill' + assert_includes output, 'openai' + assert_includes output, 'Status: PASSED' + assert_includes output, 'Score: 0.88' + refute_includes output, '
    1228+16+17
    ' + end + + def test_format_legacy_failed_result_shows_error + result = { + eval_name: 'broken-eval', + pass: false, + response: { error: { message: 'connection refused & timed out' } } + } + output = HtmlFormatter.format(result) + + assert_includes output, 'Status: FAILED' + assert_includes output, 'connection refused & timed out' + assert_includes output, 'Score: N/A' + end + + private + + def delta_result(verdict: true, skill_name: 'test-skill') + { + success: true, + eval_name: 'delta-eval', + skill_name: skill_name, + provider_name: 'mock', + tokens: { prompt_tokens: 100, completion_tokens: 50, total_tokens: 150 }, + cost: 0.0125, + response: { report: build_delta_report(verdict: verdict) } + } + end + + def build_delta_report(verdict:) + dimensions = [ + Dimension.new(name: 'correctness', description: '', max_score: 30), + Dimension.new(name: 'skill_adherence', description: '', max_score: 25) + ] + baseline = { + 'correctness' => { score: 12, max_score: 30, reasoning: '' }, + 'skill_adherence' => { score: 5, max_score: 25, reasoning: '' } + } + context = { + 'correctness' => { score: 28, max_score: 30, reasoning: '' }, + 'skill_adherence' => { score: 22, max_score: 25, reasoning: '' } + } + DeltaReport.new(baseline: baseline, context: context, criteria: build_criteria(dimensions)).tap do |report| + report.instance_variable_set(:@baseline_total, 17) + report.instance_variable_set(:@context_total, 50) + report.instance_variable_set(:@baseline_scores, { 'correctness' => 12, 'skill_adherence' => 5 }) + report.instance_variable_set(:@context_scores, { 'correctness' => 28, 'skill_adherence' => 22 }) + report.instance_variable_set(:@deltas, { 'correctness' => 16, 'skill_adherence' => 17 }) + report.instance_variable_set(:@verdict, verdict) + end + end + + def build_criteria(dimensions) + Criteria.new(path: '/dev/null').tap do |criteria| + criteria.instance_variable_set(:@context, '') + criteria.instance_variable_set(:@pass_threshold, 70) + criteria.instance_variable_set(:@minimum_delta, 10) + criteria.instance_variable_set(:@dimensions, dimensions) + end + end + end + end +end diff --git a/test/services/json_formatter_test.rb b/test/services/json_formatter_test.rb index c66e9df..64c7ceb 100644 --- a/test/services/json_formatter_test.rb +++ b/test/services/json_formatter_test.rb @@ -52,6 +52,12 @@ def test_format_defaults_tokens_and_cost_when_absent assert_equal 0, parsed['tokens']['total_tokens'] assert_nil parsed['cost'] end + + def test_format_keeps_tokens_when_explicitly_nil + parsed = JSON.parse(JsonFormatter.format({ eval_name: 'e', tokens: nil })) + + assert_equal 0, parsed['tokens']['total_tokens'] + end end end end diff --git a/test/services/junit_formatter_test.rb b/test/services/junit_formatter_test.rb index 2abe683..aaee56b 100644 --- a/test/services/junit_formatter_test.rb +++ b/test/services/junit_formatter_test.rb @@ -58,8 +58,45 @@ def test_format_escapes_html_in_eval_name assert_includes output, 'test<eval>&name' end + def test_format_batch_emits_one_testcase_per_result + output = JUnitFormatter.format_batch(batch_aggregate) + + assert_equal 3, output.scan('' + end + + def test_format_batch_adds_failure_for_failing_eval + output = JUnitFormatter.format_batch(batch_aggregate) + + assert_includes output, '' + assert_includes output, 'Eval failed' + end + + def test_format_batch_escapes_eval_names + aggregate = { results: [{ eval_name: 'a&c', pass: true }], summary: {} } + output = JUnitFormatter.format_batch(aggregate) + + refute_includes output, 'a&c' + assert_includes output, 'a<b>&c' + end + + def test_format_batch_handles_empty_results + output = JUnitFormatter.format_batch({ results: [], summary: {} }) + + assert_includes output, '' + end + private + def batch_aggregate + results = [ + { eval_name: 'pass-legacy', pass: true, score: 1.0 }, + { eval_name: 'pass-delta', success: true, response: { report: build_delta_report(verdict: true) } }, + { eval_name: 'fail-eval', pass: false, score: 0.3 } + ] + { results: results, summary: { total: 3, passed: 2, failed: 1 } } + end + def build_delta_report(verdict:) dimensions = [ Dimension.new(name: 'correctness', description: '', max_score: 30), diff --git a/test/services/response_cache_test.rb b/test/services/response_cache_test.rb new file mode 100644 index 0000000..5365568 --- /dev/null +++ b/test/services/response_cache_test.rb @@ -0,0 +1,147 @@ +# frozen_string_literal: true + +require 'test_helper' + +module SkillBench + module Services + class ResponseCacheTest < Minitest::Test + def setup + @original_env = ENV.fetch('SKILL_BENCH_CACHE', nil) + ENV.delete('SKILL_BENCH_CACHE') + ResponseCache.clear + end + + def teardown + ResponseCache.clear + if @original_env.nil? + ENV.delete('SKILL_BENCH_CACHE') + else + ENV['SKILL_BENCH_CACHE'] = @original_env + end + end + + def test_disabled_by_default + refute_predicate ResponseCache, :enabled? + end + + def test_enabled_when_env_truthy + ENV['SKILL_BENCH_CACHE'] = '1' + + assert_predicate ResponseCache, :enabled? + end + + def test_enabled_for_other_truthy_values + %w[true yes on TRUE Yes].each do |raw| + ENV['SKILL_BENCH_CACHE'] = raw + + assert_predicate ResponseCache, :enabled?, "expected #{raw.inspect} to enable caching" + end + end + + def test_disabled_when_env_falsey + ENV['SKILL_BENCH_CACHE'] = '0' + + refute_predicate ResponseCache, :enabled? + end + + def test_same_inputs_produce_same_key + first = build_key + second = build_key + + assert_equal first, second + end + + def test_key_is_independent_of_message_hash_key_order + ordered = ResponseCache.key(**base_args, messages: [{ role: 'user', content: 'hi' }]) + shuffled = ResponseCache.key(**base_args, messages: [{ content: 'hi', role: 'user' }]) + + assert_equal ordered, shuffled + end + + def test_symbol_and_string_provider_match + as_symbol = ResponseCache.key(**base_args, provider: :openai) + as_string = ResponseCache.key(**base_args, provider: 'openai') + + assert_equal as_symbol, as_string + end + + def test_different_model_produces_different_key + gpt4o = ResponseCache.key(**base_args, model: 'gpt-4o') + mini = ResponseCache.key(**base_args, model: 'gpt-4o-mini') + + refute_equal gpt4o, mini + end + + def test_different_messages_produce_different_key + first = ResponseCache.key(**base_args, messages: [{ role: 'user', content: 'a' }]) + second = ResponseCache.key(**base_args, messages: [{ role: 'user', content: 'b' }]) + + refute_equal first, second + end + + def test_different_tools_produce_different_key + without = ResponseCache.key(**base_args, tools: nil) + with = ResponseCache.key(**base_args, tools: [{ name: 'search' }]) + + refute_equal without, with + end + + def test_key_is_a_hex_sha256_digest + assert_match(/\A[0-9a-f]{64}\z/, build_key) + end + + def test_fetch_yields_and_stores_on_miss + calls = 0 + value = ResponseCache.fetch('k') do + calls += 1 + 'computed' + end + + assert_equal 'computed', value + assert_equal 1, calls + end + + def test_fetch_returns_cached_value_without_yielding_on_hit + cached = 'first' + ResponseCache.fetch('k') { cached } + + calls = 0 + value = ResponseCache.fetch('k') do + calls += 1 + 'second' + end + + assert_equal 'first', value + assert_equal 0, calls + end + + def test_clear_empties_the_store + original = 'first' + ResponseCache.fetch('k') { original } + ResponseCache.clear + + replacement = 'second' + recomputed = ResponseCache.fetch('k') { replacement } + + assert_equal 'second', recomputed + end + + private + + def base_args + { + provider: :openai, + model: 'gpt-4o', + system_prompt: 'You are helpful', + messages: [{ role: 'user', content: 'hi' }], + tools: nil, + temperature: 0.0 + } + end + + def build_key + ResponseCache.key(**base_args) + end + end + end +end diff --git a/test/services/summary_formatter_test.rb b/test/services/summary_formatter_test.rb new file mode 100644 index 0000000..557c9d4 --- /dev/null +++ b/test/services/summary_formatter_test.rb @@ -0,0 +1,96 @@ +# frozen_string_literal: true + +require 'test_helper' +require 'json' + +module SkillBench + module Services + class SummaryFormatterTest < Minitest::Test + def test_reports_counts_from_summary + parsed = JSON.parse(SummaryFormatter.format(sample_aggregate)) + + assert_equal 2, parsed['passed'] + assert_equal 1, parsed['failed'] + assert_equal 3, parsed['total'] + end + + def test_sums_total_tokens_across_results + parsed = JSON.parse(SummaryFormatter.format(sample_aggregate)) + + assert_equal 450, parsed['tokens'] + end + + def test_sums_non_nil_costs + parsed = JSON.parse(SummaryFormatter.format(sample_aggregate)) + + assert_in_delta(0.06, parsed['cost']) + end + + def test_worst_delta_picks_smallest_skill_vs_baseline_delta + parsed = JSON.parse(SummaryFormatter.format(sample_aggregate)) + + assert_equal 'evals/eval-c', parsed['worst_delta']['eval_name'] + assert_equal 5, parsed['worst_delta']['delta'] + end + + def test_cost_is_null_when_no_result_reports_a_cost + results = [result('evals/only', cost: nil, tokens: 10, report: report(40, 80))] + parsed = JSON.parse(SummaryFormatter.format(envelope(results, passed: 1, failed: 0))) + + assert_nil parsed['cost'] + end + + def test_tolerates_results_without_a_delta_report + results = [result('evals/no-report', cost: 0.01, tokens: 100, report: nil)] + parsed = JSON.parse(SummaryFormatter.format(envelope(results, passed: 1, failed: 0))) + + assert_nil parsed['worst_delta'] + assert_equal 100, parsed['tokens'] + end + + def test_treats_missing_tokens_as_zero + results = [{ eval_name: 'evals/bare', cost: 0.01, response: { report: report(40, 80) } }] + parsed = JSON.parse(SummaryFormatter.format(envelope(results, passed: 1, failed: 0))) + + assert_equal 0, parsed['tokens'] + end + + def test_output_is_pretty_printed + output = SummaryFormatter.format(sample_aggregate) + + assert_includes output, "{\n" + assert_includes output, ' "passed":' + end + + private + + def sample_aggregate + results = [ + result('evals/eval-a', cost: 0.02, tokens: 100, report: report(30, 80)), # delta 50 + result('evals/eval-b', cost: 0.04, tokens: 200, report: report(20, 60)), # delta 40 + result('evals/eval-c', cost: nil, tokens: 150, report: report(35, 40)) # delta 5 (worst) + ] + envelope(results, passed: 2, failed: 1) + end + + def envelope(results, passed:, failed:) + { results: results, summary: { total: results.size, passed: passed, failed: failed } } + end + + def result(name, tokens:, cost:, report:) + { + eval_name: name, + tokens: { total_tokens: tokens }, + cost: cost, + response: { report: report } + } + end + + def report(baseline_total, context_total) + Struct.new(:baseline_total, :context_total, keyword_init: true).new( + baseline_total: baseline_total, context_total: context_total + ) + end + end + end +end