diff --git a/.github/actions/vally-eval/action.yml b/.github/actions/vally-eval/action.yml index da26eeaef17..bdd7d7a7f07 100644 --- a/.github/actions/vally-eval/action.yml +++ b/.github/actions/vally-eval/action.yml @@ -51,6 +51,16 @@ runs: with: node-version: ${{ inputs.node-version }} + - name: Set up Go + uses: actions/setup-go@v6 + with: + go-version-file: cli/azd/test/evals/go.mod + + - name: Test report generator + shell: bash + working-directory: ${{ inputs.working-directory }} + run: go test ./... + - name: Install eval dependencies shell: bash working-directory: ${{ inputs.working-directory }} @@ -79,7 +89,12 @@ runs: # and export that value as 'vally-run-dir'. # run_dir=$(find vally-results -mindepth 1 -maxdepth 1 -type d 2>/dev/null | sort | tail -n1) - echo "vally-run-dir=$WORKING_DIRECTORY/${run_dir:-vally-results}" >> "$GITHUB_OUTPUT" + run_dir=${run_dir:-vally-results} + echo "vally-run-dir=$WORKING_DIRECTORY/$run_dir" >> "$GITHUB_OUTPUT" + + if [[ -d "$run_dir" ]]; then + npm run report -- --output-dir "$run_dir/vally_report" + fi - name: Write job summary if: always() @@ -97,6 +112,10 @@ runs: else echo "_No eval-results.md found -- see the vally-results artifact and raw output above._" fi + echo + echo "--------------------------------------------------------------------------------" + echo "Additional reports generated by vally_report.go are in \`$VALLY_RUN_DIR/vally_report/\` in the artifact." + echo "--------------------------------------------------------------------------------" } >> "$GITHUB_STEP_SUMMARY" - name: Upload vally results diff --git a/.github/workflows/eval-unit.yml b/.github/workflows/eval-unit.yml deleted file mode 100644 index a4231775ab8..00000000000 --- a/.github/workflows/eval-unit.yml +++ /dev/null @@ -1,61 +0,0 @@ -name: "Eval: Unit Tests" - -on: - pull_request: - paths: - - "cli/azd/test/eval/**" - - "cli/azd/internal/mcp/**" - - "cli/azd/cmd/mcp.go" - - "cli/azd/cmd/root.go" - -permissions: - contents: read - -jobs: - unit-tests: - runs-on: ubuntu-latest - timeout-minutes: 30 - steps: - - uses: actions/checkout@v6 - - - uses: actions/setup-go@v6 - with: - go-version-file: "cli/azd/go.mod" - - - uses: actions/setup-node@v6 - with: - node-version: "22" - - - name: Build azd - working-directory: cli/azd - run: go build -o ./azd . - - - name: Install eval dependencies - working-directory: cli/azd/test/eval - run: npm ci - - - name: Run unit tests - working-directory: cli/azd/test/eval - run: npm run test:unit -- --ci - - # Waza YAML validation is local-only (run `npm run waza:validate` locally). - # waza CLI is not available in CI. - - - uses: actions/setup-python@v6 - with: - python-version: "3.12" - - - name: Install grader test dependencies - run: pip install pytest - - - name: Run grader tests - working-directory: cli/azd/test/eval/graders - run: python -m pytest test_graders.py -v - - - name: Upload test results - if: always() - uses: actions/upload-artifact@v4 - with: - name: eval-unit-results - path: cli/azd/test/eval/reports/ - retention-days: 30 diff --git a/.github/workflows/vally-ci.yml b/.github/workflows/vally-ci.yml index 6adf093cc51..0301b279396 100644 --- a/.github/workflows/vally-ci.yml +++ b/.github/workflows/vally-ci.yml @@ -18,13 +18,13 @@ concurrency: cancel-in-progress: true jobs: - evals: - name: Run vally evals + qna: + name: Run vally evals (Q&A) runs-on: ubuntu-latest timeout-minutes: 90 steps: - name: Checkout repository - uses: actions/checkout@v7 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.0 with: persist-credentials: false @@ -33,6 +33,25 @@ jobs: uses: ./.github/actions/vally-eval with: working-directory: cli/azd/test/evals - eval-file: eval.yaml + eval-file: eval-azd-qna.yaml + artifact-name: ${{ github.job }}-vally-results + github-token: ${{ github.token }} + + deploy: + name: Run vally evals (deploy) + runs-on: ubuntu-latest + timeout-minutes: 90 + steps: + - name: Checkout repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.0 + with: + persist-credentials: false + + - name: Run vally eval + id: vally + uses: ./.github/actions/vally-eval + with: + working-directory: cli/azd/test/evals + eval-file: eval-azd-deploy.yaml artifact-name: ${{ github.job }}-vally-results github-token: ${{ github.token }} diff --git a/cli/azd/.vscode/cspell.yaml b/cli/azd/.vscode/cspell.yaml index a40517e12c6..17d8209d1da 100644 --- a/cli/azd/.vscode/cspell.yaml +++ b/cli/azd/.vscode/cspell.yaml @@ -441,28 +441,6 @@ overrides: - covcounters - covmeta - AMRD - - filename: test/eval/README.md - words: - - Waza - - waza - - urlopen - - filename: "test/eval/graders/*.py" - words: - - Waza - - waza - - hdrs - - mysite - - mydb - - filename: "test/eval/tasks/**/*.yaml" - words: - - authenticat - - idempoten - - filename: "test/eval/tests/human/*.test.ts" - words: - - compdef - - badcfg - - provison - - notacommand - filename: pkg/infra/provisioning/bicep/bicep_provider.go words: - BIZSPARK diff --git a/cli/azd/test/eval/.gitignore b/cli/azd/test/eval/.gitignore deleted file mode 100644 index 9bb11eb6cf5..00000000000 --- a/cli/azd/test/eval/.gitignore +++ /dev/null @@ -1,6 +0,0 @@ -node_modules/ -dist/ -reports/*.json -reports/*.md -reports/junit.xml -!reports/.gitkeep diff --git a/cli/azd/test/eval/README.md b/cli/azd/test/eval/README.md deleted file mode 100644 index 2a85f978954..00000000000 --- a/cli/azd/test/eval/README.md +++ /dev/null @@ -1,385 +0,0 @@ -# AZD CLI Evaluation & Testing Framework - -Measures how well GitHub Copilot CLI interacts with the Azure Developer CLI (`azd`). Inspired by the [microsoft/github-copilot-for-azure](https://github.com/microsoft/github-copilot-for-azure) testing architecture. - -## Overview - -When a user asks Copilot CLI a question like "deploy my Python app to Azure", the LLM suggests `azd` shell commands (`azd init`, `azd provision`, `azd deploy`). This framework tests: - -1. **Does the LLM suggest the right commands?** (text graders) -2. **Are the commands in the right order?** (action_sequence graders) -3. **Do the suggested commands actually succeed?** (Jest unit + human tests validate CLI behavior) -4. **Is the deployed infrastructure correct?** (Python code graders with ARM + HTTP validation, used in E2E lifecycle evals only) - -## Architecture - -| Component | Purpose | Technology | -|-----------|---------|------------| -| **Waza tasks** (`tasks/`) | LLM evaluation scenarios | [microsoft/waza](https://github.com/microsoft/waza) YAML | -| **Custom graders** (`graders/`) | Azure resource + app validation | Python | -| **Jest unit tests** (`tests/unit/`) | Command structure validation | TypeScript/Jest | -| **Jest human tests** (`tests/human/`) | Human CLI usage baselines | TypeScript/Jest | - -## Quick Start - -### Prerequisites - -- Node.js 20+ -- Go (to build azd) -- [Waza CLI](https://github.com/microsoft/waza) (`npm install -g waza`) -- Python 3.x (for custom graders) - -### Run Tests - -```bash -# Install dependencies -npm install - -# Build azd (required for CLI tests) -cd ../../ && go build -o ./azd . && cd test/eval - -# Run Jest unit tests (no LLM, no Azure) -npm run test:unit - -# Validate Waza task YAML syntax -npm run waza:validate - -# Run Waza evals with mock executor (offline, fast) -npm run waza:run:mock - -# Run Waza evals with Copilot SDK (requires COPILOT_CLI_TOKEN) -export COPILOT_CLI_TOKEN= -npm run waza:run - -# Run human usage baseline tests -npm run test:human -``` - -## How-To Guides - -### Adding a Waza LLM Eval Task - -Waza tasks are YAML files that define a prompt → grader pipeline. Each task sends a user prompt to the LLM and grades the response. - -**Step 1: Pick the right category folder** - -| Category | Directory | When to use | -|----------|-----------|-------------| -| `tasks/deploy/` | User wants to deploy something to Azure | -| `tasks/troubleshoot/` | User pastes an error and asks for help | -| `tasks/environment/` | User manages azd environments | -| `tasks/lifecycle/` | Full init→provision→deploy→down workflows | -| `tasks/negative/` | Questions where azd is the **wrong** tool | - -**Step 2: Create the YAML file** - -```bash -# Example: test that the LLM handles a "wrong SKU" error -touch tasks/troubleshoot/wrong-sku-error.yaml -``` - -**Step 3: Define the task structure** - -Every task file needs these fields: - -```yaml -id: troubleshoot-wrong-sku-001 # Unique ID (category-name-NNN) -description: > # What this task tests - User's azd provision failed because the requested VM SKU - is not available in their region. -inputs: - prompt: | # The exact prompt sent to the LLM - I ran azd provision and got this error: - ERROR: deployment failed: The requested VM size Standard_NC6 is not - available in the current region (eastus2). Available sizes: Standard_D2s_v3, - Standard_D4s_v3. - How do I fix this? - context: | # Optional background (not sent to LLM, used by graders) - User is deploying a GPU workload but picked a region without GPU SKUs. -graders: # One or more graders (weights must sum to 1.0) - - type: text - weight: 0.4 - config: - must_match: - - "(region|location|SKU|size)" - must_not_match: - - "I don't know" - - type: text - weight: 0.3 - config: - must_match_any: - - "different (region|location)" - - "Standard_D" - - "az vm list-sizes" - - type: behavior - weight: 0.3 - config: - max_tool_calls: 5 - max_tokens: 2000 -``` - -**Step 4: Validate and test** - -```bash -npm run waza:validate # Check YAML syntax -npm run waza:run:mock # Quick test with mock executor -COPILOT_CLI_TOKEN= npm run waza:run # Real LLM test -``` - -### Grader Reference - -| Grader | Purpose | Key config fields | -|--------|---------|-------------------| -| `text` | Regex matching on LLM response | `must_match` (all must hit), `must_not_match` (none can hit), `must_match_any` (at least one) | -| `action_sequence` | Verify commands appear in order | `expected_order` (list of commands/patterns) | -| `behavior` | Efficiency constraints | `max_tool_calls`, `max_tokens` | -| `code` | Custom Python validation | `script` (path to grader in `graders/`), `params` (dict passed to script) | - -**Grader weights** must sum to 1.0 across all graders in a task. Each grader returns a 0.0–1.0 score, and the weighted average is the task score. - -**Regex tips for `text` graders:** -- Patterns are Python regexes, case-insensitive by default -- Use `(a|b|c)` for alternatives: `"azd (up|provision)"` -- Use `must_match` when ALL patterns must appear -- Use `must_match_any` when at least ONE pattern must appear -- Use `must_not_match` as guardrails (e.g., don't suggest destructive commands) - -### Adding a Custom Python Grader - -Custom graders validate real Azure resources during E2E evals. They live in `graders/` and are referenced from task YAML via the `code` grader type. - -**Step 1: Create the grader** - -```python -# graders/my_validator.py -import json -import os -import sys -import urllib.request - -from azure_auth import get_access_token - -def grade(context: dict) -> dict: - """ - Called by Waza with: - - context: dict containing "params" from the task YAML, plus run metadata - - Must return: {"score": 0.0-1.0, "reason": "explanation"} - """ - params = context.get("params", {}) - subscription_id = params.get("subscription_id", os.environ.get("AZURE_SUBSCRIPTION_ID")) - resource_group = params.get("resource_group") - - try: - token = get_access_token() - except RuntimeError as e: - return {"score": 0.0, "reason": str(e)} - - # Make ARM API calls to validate resources... - url = ( - f"https://management.azure.com/subscriptions/{subscription_id}" - f"/resourceGroups/{resource_group}?api-version=2021-04-01" - ) - req = urllib.request.Request(url, headers={"Authorization": f"Bearer {token}"}) - try: - resp = urllib.request.urlopen(req) - data = json.loads(resp.read()) - return {"score": 1.0, "reason": f"Resource group '{resource_group}' exists"} - except Exception as e: - return {"score": 0.0, "reason": str(e)} - -if __name__ == "__main__": - context = json.loads(sys.argv[1]) if len(sys.argv) > 1 else {} - print(json.dumps(grade(context))) -``` - -**Step 2: Reference it from a task YAML** - -```yaml -graders: - - type: code - weight: 0.3 - config: - script: graders/my_validator.py - params: - resource_group: "rg-myapp-dev" - expected_resources: - - "Microsoft.Web/sites" -``` - -**Existing graders you can reuse:** -- `graders/infra_validator.py` — checks that a resource group and its resources exist after `azd provision` -- `graders/cleanup_validator.py` — checks that resources are deleted after `azd down` -- `graders/app_health.py` — HTTP health checks against deployed app URLs - -### Adding a Jest Unit Test - -Unit tests validate `azd` CLI behavior directly — no LLM, no Azure. They run fast and catch regressions in command structure, help text, and flags. - -**Step 1: Create or edit a test file in `tests/unit/`** - -```typescript -// tests/unit/my-new-test.test.ts -import { azd } from "../test-utils"; - -describe("my new azd tests", () => { - test("version outputs a semver string", () => { - const result = azd("version"); - expect(result.exitCode).toBe(0); - expect(result.stdout).toMatch(/\d+\.\d+\.\d+/); - }); -}); -``` - -> **Important:** The shared `azd()` helper in `tests/test-utils.ts` sets `NO_COLOR: "1"` and `AZD_FORCE_TTY: "false"` automatically. Always import from there — don't create local helpers. - -**Step 2: Run it** - -```bash -# Build azd first (tests shell out to the binary) -cd ../../ && go build && cd test/eval - -# Run just your test -npx jest tests/unit/my-new-test.test.ts - -# Run all unit tests -npm run test:unit -``` - -### Adding a Human Scenario Test - -Human tests validate CLI usability patterns — things like "can a user discover the right command?" These run against the real `azd` binary but test the experience, not the LLM. - -```typescript -// tests/human/my-scenario.test.ts -import { azd } from "../test-utils"; - -describe("deploy command discovery", () => { - test("user can find deploy command from root help", () => { - const root = azd("--help"); - expect(root.stdout).toContain("deploy"); - - const deploy = azd("deploy --help"); - expect(deploy.exitCode).toBe(0); - expect(deploy.stdout).toMatch(/\bUsage\b/); - }); -}); -``` - -### Directory Structure Reference - -``` -cli/azd/test/eval/ -├── eval.yaml # Waza eval config (model, metrics, thresholds) -├── jest.config.ts # Jest config for unit + human tests -├── package.json # Node dependencies and npm scripts -├── tsconfig.json # TypeScript config -├── graders/ # Custom Python graders for E2E validation -│ ├── app_health.py # HTTP health checks on deployed apps -│ ├── cleanup_validator.py # Validates resources deleted after azd down -│ └── infra_validator.py # Validates resources exist after azd provision -├── tasks/ # Waza LLM eval task definitions -│ ├── deploy/ # Deployment scenarios -│ ├── environment/ # Environment management scenarios -│ ├── lifecycle/ # Full init→provision→deploy→down E2E -│ ├── negative/ # Questions where azd is the wrong tool -│ └── troubleshoot/ # Error diagnosis scenarios -├── tests/ -│ ├── unit/ # Jest unit tests (no LLM, no Azure) -│ │ ├── command-registry.test.ts -│ │ ├── command-sequencing.test.ts -│ │ ├── flag-validation.test.ts -│ │ └── help-text-quality.test.ts -│ └── human/ # Human CLI usability tests -│ ├── cli-workflow.test.ts -│ ├── command-discovery.test.ts -│ └── error-recovery.test.ts -├── reports/ # Generated reports (gitignored) -``` - -## Scenario Categories - -| Category | Directory | Graders | CI Frequency | -|----------|-----------|---------|-------------| -| Deploy workflows | `tasks/deploy/` | text, action_sequence, behavior | 3x daily | -| Error troubleshooting | `tasks/troubleshoot/` | text, behavior | 3x daily | -| Environment management | `tasks/environment/` | text, action_sequence | 3x daily | -| Negative tests | `tasks/negative/` | text, behavior | 3x daily | -| Full lifecycle E2E | `tasks/lifecycle/` | text, action_sequence, code | Weekly | - -## CI/CD - -Unit tests run in GitHub Actions (no secrets needed). Workflows that require -Azure credentials or the Copilot CLI token run in internal ADO pipelines, -following the team convention for authenticated workloads. - -| Pipeline | Location | Trigger | What it does | -|----------|----------|---------|-------------| -| `eval-unit.yml` | `.github/workflows/` | On PR | Jest unit tests + Python grader tests | -| `eval-unit.yml` | `eng/pipelines/` | On PR | Same, ADO mirror | -| `eval-waza.yml` | `eng/pipelines/` | 3x daily (Tue-Sat) | Waza evals via Copilot SDK | -| `eval-e2e.yml` | `eng/pipelines/` | Weekly | Waza E2E with Azure resource validation | -| `eval-report.yml` | `eng/pipelines/` | Weekly | Aggregates results from Waza + E2E runs | - -## Authentication & Secrets - -### Test tiers and what they need - -| Tier | Command | Azure Auth | Copilot Token | Browser Popups | -|------|---------|-----------|---------------|----------------| -| Unit tests | `npm run test:unit` | ❌ None | ❌ None | Never | -| Human tests | `npm run test:human` | ❌ None | ❌ None | Never | -| Mock LLM eval | `npm run waza:run:mock` | ❌ None | ❌ None | Never | -| LLM eval | `npm run waza:run` | ❌ None | ✅ Required | Never | -| E2E lifecycle | `eng/pipelines/eval-e2e.yml` | ✅ Required | ✅ Required | Never (ADO service connection) | - -> **No test should ever open a browser.** Unit and human tests use `--no-prompt` and only test help text / error messages — they never call Azure APIs. E2E workflows use OIDC service principal auth (headless). - -### Local development - -```bash -# 1. Log in to Azure (one-time, uses your existing browser session) -az login -az account set --subscription - -# 2. Verify your auth works (no browser should open) -azd auth login # uses cached az credentials - -# 3. Run tests — unit tests never touch Azure -npm run test:unit - -# 4. For LLM evals (optional) -export COPILOT_CLI_TOKEN= -npm run waza:run -``` - -**Configuring a specific subscription for E2E graders:** - -```bash -# Set the subscription that E2E graders will validate against -export AZURE_SUBSCRIPTION_ID= - -# Or configure via azd -azd config set defaults.subscription -``` - -### CI/CD setup (Azure DevOps) - -Waza, E2E, and report pipelines run in internal ADO. Configure these -variables in the pipeline's variable group: - -| Variable | Pipelines | Purpose | -|----------|-----------|---------| -| `COPILOT_CLI_TOKEN` | `eval-waza`, `eval-e2e` | Authenticate Waza Copilot SDK executor | -| `AZURE_SUBSCRIPTION_ID` | `eval-e2e` | Target subscription for E2E deployments | -| `AZURE_TENANT_ID` | `eval-e2e` | AAD tenant ID | -| `AZURE_CLIENT_ID` | `eval-e2e` | Service principal client ID | - -The E2E pipeline uses the `azure-sdk-tests` service connection (same as -other functional test pipelines). No additional service principal setup -is needed if the existing connection has `Contributor` on the target -subscription. - -## Reports - -Generated reports are saved to `reports/` (gitignored). In CI, they're uploaded as workflow artifacts with 30-day retention. diff --git a/cli/azd/test/eval/eval.yaml b/cli/azd/test/eval/eval.yaml deleted file mode 100644 index 4bdd7dd2fb3..00000000000 --- a/cli/azd/test/eval/eval.yaml +++ /dev/null @@ -1,71 +0,0 @@ -name: azd-copilot-cli-eval -description: > - Evaluate how well GitHub Copilot CLI suggests and executes Azure Developer CLI (azd) commands. - Tests whether the LLM suggests correct commands, with right flags, in proper order, - and whether the commands produce correct outcomes when executed. - -executor: - type: copilot-sdk - config: - model: gpt-4o - system_prompt: | - You are GitHub Copilot in the terminal. You help users with software engineering - tasks including Azure development. When appropriate, suggest Azure Developer CLI (azd) - commands to help users provision infrastructure, deploy applications, and manage - Azure environments. - - Available azd commands: - - azd init: Initialize a new azd project from a template - - azd provision: Provision Azure infrastructure defined in the project - - azd deploy: Deploy application code to provisioned Azure resources - - azd up: Shorthand for init + provision + deploy (full setup) - - azd down: Delete Azure resources and clean up - - azd env new : Create a new environment - - azd env list: List all environments - - azd env set : Set an environment variable - - azd env get-values: Get all environment values - - azd env select : Switch active environment - - azd env remove : Remove an environment - - azd monitor: Open application monitoring dashboard - - azd show: Display project and environment information - - azd auth login: Authenticate with Azure - - azd config set : Set configuration - - azd restore: Restore project dependencies - - azd build: Build application services - - azd package: Package application for deployment - - azd pipeline: Configure and manage CI/CD pipeline - - Common flags: - - --environment, -e: Specify target environment - - --no-prompt: Run without interactive prompts - - --output, -o: Output format (json, table) - - --help, -h: Show help for a command - -metrics: - - name: command_accuracy - weight: 0.35 - threshold: 0.8 - description: Does the LLM suggest the correct azd commands? - - - name: sequencing_accuracy - weight: 0.2 - threshold: 0.85 - description: Are commands in the correct execution order? - - - name: response_quality - weight: 0.2 - threshold: 0.7 - description: Is the response helpful, complete, and actionable? - - - name: negative_accuracy - weight: 0.15 - threshold: 0.95 - description: Does the LLM correctly NOT suggest azd for non-azd questions? - - - name: execution_success - weight: 0.1 - threshold: 0.8 - description: Do the suggested commands actually succeed when executed? - -task_patterns: - - "tasks/**/*.yaml" diff --git a/cli/azd/test/eval/graders/app_health.py b/cli/azd/test/eval/graders/app_health.py deleted file mode 100644 index 9f00909cffa..00000000000 --- a/cli/azd/test/eval/graders/app_health.py +++ /dev/null @@ -1,129 +0,0 @@ -""" -Waza code grader: Validates a deployed application is healthy after azd deploy. - -Makes HTTP requests to specified endpoints and checks status codes and optional -response body content. Returns a proportional score based on how many health -checks pass. - -Usage in task YAML: - graders: - - type: code - config: - language: python - file: graders/app_health.py - params: - endpoints: - - url: "https://myapp.azurewebsites.net/" - expected_status: 200 - - url: "https://myapp.azurewebsites.net/api/health" - expected_status: 200 - expected_body_contains: "healthy" - - url: "https://myapp.azurewebsites.net/api/version" - expected_status: 200 - expected_body_contains: "version" - timeout: 30 - retries: 3 - retry_delay: 5 -""" -import time -from urllib.request import Request, urlopen -from urllib.error import HTTPError, URLError - - -def check_endpoint( - url: str, - expected_status: int = 200, - expected_body_contains: str = "", - timeout: int = 30, - retries: int = 3, - retry_delay: int = 5, -) -> dict: - """Check a single endpoint, with retries for transient failures.""" - last_error = None - - for attempt in range(retries): - try: - req = Request(url, method="GET") - resp = urlopen(req, timeout=timeout) - status = resp.status - body = resp.read().decode("utf-8", errors="replace") - - if status != expected_status: - last_error = f"Expected status {expected_status}, got {status}" - if attempt < retries - 1: - time.sleep(retry_delay) - continue - - if expected_body_contains and expected_body_contains not in body: - last_error = f"Response body missing expected string '{expected_body_contains}'" - if attempt < retries - 1: - time.sleep(retry_delay) - continue - - return {"passed": True, "reason": f"Status {status} OK"} - - except HTTPError as e: - if e.code == expected_status: - if expected_body_contains: - body = e.read().decode("utf-8", errors="replace") - if expected_body_contains not in body: - last_error = ( - f"Status {e.code} matched but body missing " - f"'{expected_body_contains}'" - ) - if attempt < retries - 1: - time.sleep(retry_delay) - continue - return {"passed": True, "reason": f"Status {e.code} matches expected"} - last_error = f"HTTP {e.code}: {e.reason}" - except URLError as e: - last_error = f"Connection error: {e.reason}" - except TimeoutError: - last_error = f"Request timed out after {timeout}s" - except Exception as e: - last_error = f"Unexpected error: {e}" - - if attempt < retries - 1: - time.sleep(retry_delay) - - return {"passed": False, "reason": last_error or "Unknown error"} - - -def grade(context: dict) -> dict: - """Waza grader entry point.""" - params = context.get("params", {}) - endpoints = params.get("endpoints", []) - default_timeout = params.get("timeout", 30) - default_retries = params.get("retries", 3) - default_retry_delay = params.get("retry_delay", 5) - - if not endpoints: - return {"score": 0.0, "reason": "No endpoints specified in params"} - - results = [] - for ep in endpoints: - url = ep.get("url", "") - if not url: - results.append({"url": "", "passed": False, "reason": "Empty URL"}) - continue - - result = check_endpoint( - url=url, - expected_status=ep.get("expected_status", 200), - expected_body_contains=ep.get("expected_body_contains", ""), - timeout=ep.get("timeout", default_timeout), - retries=ep.get("retries", default_retries), - retry_delay=ep.get("retry_delay", default_retry_delay), - ) - result["url"] = url - results.append(result) - - passed = sum(1 for r in results if r["passed"]) - score = passed / len(results) - - failed = [r for r in results if not r["passed"]] - if failed: - details = "; ".join(f"{r['url']}: {r['reason']}" for r in failed) - return {"score": score, "reason": f"Failed checks: {details}"} - - return {"score": 1.0, "reason": f"All {len(results)} health checks passed"} diff --git a/cli/azd/test/eval/graders/azure_auth.py b/cli/azd/test/eval/graders/azure_auth.py deleted file mode 100644 index 041fc5747b5..00000000000 --- a/cli/azd/test/eval/graders/azure_auth.py +++ /dev/null @@ -1,28 +0,0 @@ -"""Shared Azure authentication helper for eval graders.""" -import os -import subprocess - - -def get_access_token() -> str: - """Get Azure access token using Azure CLI or environment variable. - - Tries `az account get-access-token` first, then falls back to - the AZURE_ACCESS_TOKEN environment variable. - """ - try: - result = subprocess.run( - ["az", "account", "get-access-token", "--query", "accessToken", "-o", "tsv"], - capture_output=True, text=True, check=True - ) - token = result.stdout.strip() - if token: - return token - except (subprocess.CalledProcessError, FileNotFoundError) as e: - # az CLI not found or returned non-zero — try fallback - pass - - token = os.environ.get("AZURE_ACCESS_TOKEN") - if token: - return token - - raise RuntimeError("No Azure credentials available. Run 'az login' or set AZURE_ACCESS_TOKEN.") diff --git a/cli/azd/test/eval/graders/cleanup_validator.py b/cli/azd/test/eval/graders/cleanup_validator.py deleted file mode 100644 index 70f2b8b23e4..00000000000 --- a/cli/azd/test/eval/graders/cleanup_validator.py +++ /dev/null @@ -1,107 +0,0 @@ -""" -Waza code grader: Validates Azure resources are cleaned up after azd down. - -Confirms the resource group no longer exists by expecting a 404 from the -Azure ARM API. Returns 1.0 if fully cleaned up, 0.0 if resources remain. - -Usage in task YAML: - graders: - - type: code - config: - language: python - file: graders/cleanup_validator.py - params: - resource_group: "rg-myapp-dev" -""" -import os -import json -from urllib.request import Request, urlopen -from urllib.error import HTTPError - -from azure_auth import get_access_token - - -def check_resource_group_exists(subscription_id: str, resource_group: str, token: str) -> dict: - """Check resource group status. Returns dict with exists flag and details.""" - url = ( - f"https://management.azure.com/subscriptions/{subscription_id}" - f"/resourcegroups/{resource_group}?api-version=2021-04-01" - ) - req = Request(url, headers={"Authorization": f"Bearer {token}"}) - try: - resp = urlopen(req) - data = json.loads(resp.read()) - state = data.get("properties", {}).get("provisioningState", "Unknown") - return {"exists": True, "state": state} - except HTTPError as e: - if e.code == 404: - return {"exists": False, "state": "Deleted"} - raise - - -def list_remaining_resources(subscription_id: str, resource_group: str, token: str) -> list: - """List any resources still present in the resource group.""" - url = ( - f"https://management.azure.com/subscriptions/{subscription_id}" - f"/resourceGroups/{resource_group}/resources?api-version=2021-04-01" - ) - req = Request(url, headers={"Authorization": f"Bearer {token}"}) - try: - resp = urlopen(req) - data = json.loads(resp.read()) - return [ - {"name": r.get("name", ""), "type": r.get("type", "")} - for r in data.get("value", []) - ] - except HTTPError as e: - if e.code == 404: - return [] # Resource group doesn't exist — that's clean - raise # Auth failures, permission errors should surface - - -def grade(context: dict) -> dict: - """Waza grader entry point.""" - params = context.get("params", {}) - subscription_id = params.get("subscription_id", os.environ.get("AZURE_SUBSCRIPTION_ID", "")) - resource_group = params.get("resource_group", "") - - if not subscription_id or not resource_group: - return {"score": 0.0, "reason": "Missing subscription_id or resource_group parameter"} - - try: - token = get_access_token() - except RuntimeError as e: - return {"score": 0.0, "reason": str(e)} - - try: - rg_status = check_resource_group_exists(subscription_id, resource_group, token) - except HTTPError as e: - return {"score": 0.0, "reason": f"HTTP error checking resource group: {e.code} {e.reason}"} - - if not rg_status["exists"]: - return {"score": 1.0, "reason": f"Resource group '{resource_group}' successfully deleted"} - - # Resource group still exists — check if it's being deleted - if rg_status["state"] == "Deleting": - return { - "score": 0.5, - "reason": f"Resource group '{resource_group}' is still being deleted (state: Deleting)", - } - - # Resource group exists with resources remaining - try: - remaining = list_remaining_resources(subscription_id, resource_group, token) - except HTTPError as e: - return {"score": 0.0, "reason": f"HTTP error listing resources: {e.code} {e.reason}"} - if remaining: - details = ", ".join(f"{r['type']}/{r['name']}" for r in remaining[:10]) - suffix = f" (and {len(remaining) - 10} more)" if len(remaining) > 10 else "" - return { - "score": 0.0, - "reason": f"Resource group '{resource_group}' still exists with resources: {details}{suffix}", - } - - return { - "score": 0.0, - "reason": f"Resource group '{resource_group}' still exists (state: {rg_status['state']})", - } diff --git a/cli/azd/test/eval/graders/infra_validator.py b/cli/azd/test/eval/graders/infra_validator.py deleted file mode 100644 index 266ab2eea24..00000000000 --- a/cli/azd/test/eval/graders/infra_validator.py +++ /dev/null @@ -1,96 +0,0 @@ -""" -Waza code grader: Validates Azure infrastructure exists after azd provision. - -Usage in task YAML: - graders: - - type: code - config: - language: python - file: graders/infra_validator.py - params: - resource_group: "rg-myapp-dev" - expected_resources: - - "Microsoft.Web/sites" - - "Microsoft.DocumentDB/databaseAccounts" -""" -import os -import json -from urllib.request import Request, urlopen -from urllib.error import HTTPError - -from azure_auth import get_access_token - - -def check_resource_group_exists(subscription_id: str, resource_group: str, token: str) -> bool: - """Check if a resource group exists.""" - url = ( - f"https://management.azure.com/subscriptions/{subscription_id}" - f"/resourcegroups/{resource_group}?api-version=2021-04-01" - ) - req = Request(url, headers={"Authorization": f"Bearer {token}"}) - try: - urlopen(req) - return True - except HTTPError as e: - if e.code == 404: - return False - raise - - -def list_resources(subscription_id: str, resource_group: str, token: str) -> list: - """List all resources in a resource group.""" - url = ( - f"https://management.azure.com/subscriptions/{subscription_id}" - f"/resourceGroups/{resource_group}/resources?api-version=2021-04-01" - ) - req = Request(url, headers={"Authorization": f"Bearer {token}"}) - try: - resp = urlopen(req) - data = json.loads(resp.read()) - return [r["type"] for r in data.get("value", [])] - except HTTPError as e: - if e.code == 404: - return [] # Resource group doesn't exist - raise # Auth failures, permission errors should surface - - -def grade(context: dict) -> dict: - """Waza grader entry point.""" - params = context.get("params", {}) - subscription_id = params.get("subscription_id", os.environ.get("AZURE_SUBSCRIPTION_ID", "")) - resource_group = params.get("resource_group", "") - expected_resources = params.get("expected_resources", []) - - if not subscription_id or not resource_group: - return {"score": 0.0, "reason": "Missing subscription_id or resource_group parameter"} - - try: - token = get_access_token() - except RuntimeError as e: - return {"score": 0.0, "reason": str(e)} - - # Check resource group exists - try: - rg_exists = check_resource_group_exists(subscription_id, resource_group, token) - except HTTPError as e: - return {"score": 0.0, "reason": f"HTTP error checking resource group: {e.code} {e.reason}"} - - if not rg_exists: - return {"score": 0.0, "reason": f"Resource group '{resource_group}' does not exist"} - - if not expected_resources: - return {"score": 1.0, "reason": f"Resource group '{resource_group}' exists"} - - # Check expected resources - try: - actual_types = list_resources(subscription_id, resource_group, token) - except HTTPError as e: - return {"score": 0.0, "reason": f"HTTP error listing resources: {e.code} {e.reason}"} - found = sum(1 for expected in expected_resources if expected in actual_types) - score = found / len(expected_resources) - - missing = [r for r in expected_resources if r not in actual_types] - if missing: - return {"score": score, "reason": f"Missing resources: {', '.join(missing)}"} - - return {"score": 1.0, "reason": "All expected resources found"} diff --git a/cli/azd/test/eval/graders/test_graders.py b/cli/azd/test/eval/graders/test_graders.py deleted file mode 100644 index e79dab361d3..00000000000 --- a/cli/azd/test/eval/graders/test_graders.py +++ /dev/null @@ -1,341 +0,0 @@ -""" -Unit tests for eval graders. - -Tests grade() functions with mocked HTTP responses to verify scoring logic -without making real Azure API or HTTP calls. -""" -import sys -from pathlib import Path - -sys.path.insert(0, str(Path(__file__).parent)) - -import json -from unittest.mock import patch, MagicMock -from urllib.error import HTTPError, URLError - -import app_health -import cleanup_validator -import infra_validator - - -# --------------------------------------------------------------------------- -# app_health tests -# --------------------------------------------------------------------------- - -class TestAppHealthGrader: - """Tests for app_health.grade().""" - - def test_no_endpoints_returns_zero(self): - result = app_health.grade({"params": {}}) - assert result["score"] == 0.0 - assert "No endpoints" in result["reason"] - - @patch("app_health.urlopen") - def test_all_endpoints_healthy(self, mock_urlopen): - resp = MagicMock() - resp.status = 200 - resp.read.return_value = b'{"status": "healthy"}' - mock_urlopen.return_value = resp - - result = app_health.grade({ - "params": { - "endpoints": [ - {"url": "https://example.com/", "expected_status": 200}, - {"url": "https://example.com/api", "expected_status": 200}, - ], - "retries": 1, - } - }) - assert result["score"] == 1.0 - assert "All 2 health checks passed" in result["reason"] - - @patch("app_health.urlopen") - def test_partial_failure(self, mock_urlopen): - success_resp = MagicMock() - success_resp.status = 200 - success_resp.read.return_value = b"OK" - - fail_resp = MagicMock() - fail_resp.status = 500 - fail_resp.read.return_value = b"error" - - mock_urlopen.side_effect = [success_resp, fail_resp] - - result = app_health.grade({ - "params": { - "endpoints": [ - {"url": "https://example.com/ok", "expected_status": 200}, - {"url": "https://example.com/fail", "expected_status": 200}, - ], - "retries": 1, - } - }) - assert result["score"] == 0.5 - - @patch("app_health.urlopen") - def test_body_content_mismatch_retries(self, mock_urlopen): - """Body-content mismatches should retry (not fail immediately).""" - bad_resp = MagicMock() - bad_resp.status = 200 - bad_resp.read.return_value = b"loading..." - - good_resp = MagicMock() - good_resp.status = 200 - good_resp.read.return_value = b'{"status": "healthy"}' - - mock_urlopen.side_effect = [bad_resp, good_resp] - - result = app_health.grade({ - "params": { - "endpoints": [ - { - "url": "https://example.com/", - "expected_status": 200, - "expected_body_contains": "healthy", - }, - ], - "retries": 2, - "retry_delay": 0, - } - }) - assert result["score"] == 1.0 - assert mock_urlopen.call_count == 2 - - @patch("app_health.urlopen") - def test_connection_error(self, mock_urlopen): - mock_urlopen.side_effect = URLError("Connection refused") - - result = app_health.grade({ - "params": { - "endpoints": [{"url": "https://down.example.com/"}], - "retries": 1, - } - }) - assert result["score"] == 0.0 - assert "Connection error" in result["reason"] - - @patch("app_health.urlopen") - def test_non_2xx_expected_status(self, mock_urlopen): - """Non-2xx expected_status should match against HTTPError code.""" - mock_urlopen.side_effect = HTTPError( - url="", code=404, msg="Not Found", hdrs={}, fp=None - ) - - result = app_health.grade({ - "params": { - "endpoints": [ - {"url": "https://example.com/deleted", "expected_status": 404}, - ], - "retries": 1, - } - }) - assert result["score"] == 1.0 - - @patch("app_health.urlopen") - def test_non_2xx_unexpected_status(self, mock_urlopen): - """HTTPError with wrong code should fail.""" - mock_urlopen.side_effect = HTTPError( - url="", code=500, msg="Server Error", hdrs={}, fp=None - ) - - result = app_health.grade({ - "params": { - "endpoints": [ - {"url": "https://example.com/fail", "expected_status": 200}, - ], - "retries": 1, - } - }) - assert result["score"] == 0.0 - - def test_empty_url_fails(self): - result = app_health.grade({ - "params": { - "endpoints": [{"url": ""}], - } - }) - assert result["score"] == 0.0 - assert "Empty URL" in result["reason"] - - -# --------------------------------------------------------------------------- -# cleanup_validator tests -# --------------------------------------------------------------------------- - -class TestCleanupValidatorGrader: - """Tests for cleanup_validator.grade().""" - - def test_missing_params_returns_zero(self): - result = cleanup_validator.grade({"params": {}}) - assert result["score"] == 0.0 - assert "Missing" in result["reason"] - - @patch("cleanup_validator.get_access_token") - @patch("cleanup_validator.urlopen") - def test_resource_group_deleted(self, mock_urlopen, mock_token): - mock_token.return_value = "fake-token" - mock_urlopen.side_effect = HTTPError( - url="", code=404, msg="Not Found", hdrs={}, fp=None - ) - - result = cleanup_validator.grade({ - "params": { - "subscription_id": "sub-123", - "resource_group": "rg-deleted", - } - }) - assert result["score"] == 1.0 - assert "successfully deleted" in result["reason"] - - @patch("cleanup_validator.get_access_token") - @patch("cleanup_validator.urlopen") - def test_resource_group_still_exists(self, mock_urlopen, mock_token): - mock_token.return_value = "fake-token" - - rg_resp = MagicMock() - rg_resp.read.return_value = json.dumps({ - "properties": {"provisioningState": "Succeeded"} - }).encode() - - resources_resp = MagicMock() - resources_resp.read.return_value = json.dumps({ - "value": [{"name": "mysite", "type": "Microsoft.Web/sites"}] - }).encode() - - mock_urlopen.side_effect = [rg_resp, resources_resp] - - result = cleanup_validator.grade({ - "params": { - "subscription_id": "sub-123", - "resource_group": "rg-exists", - } - }) - assert result["score"] == 0.0 - assert "still exists" in result["reason"] - - @patch("cleanup_validator.get_access_token") - @patch("cleanup_validator.urlopen") - def test_resource_group_deleting(self, mock_urlopen, mock_token): - mock_token.return_value = "fake-token" - - resp = MagicMock() - resp.read.return_value = json.dumps({ - "properties": {"provisioningState": "Deleting"} - }).encode() - mock_urlopen.return_value = resp - - result = cleanup_validator.grade({ - "params": { - "subscription_id": "sub-123", - "resource_group": "rg-deleting", - } - }) - assert result["score"] == 0.5 - assert "Deleting" in result["reason"] - - -# --------------------------------------------------------------------------- -# infra_validator tests -# --------------------------------------------------------------------------- - -class TestInfraValidatorGrader: - """Tests for infra_validator.grade().""" - - def test_missing_params_returns_zero(self): - result = infra_validator.grade({"params": {}}) - assert result["score"] == 0.0 - assert "Missing" in result["reason"] - - @patch("infra_validator.get_access_token") - @patch("infra_validator.urlopen") - def test_resource_group_not_found(self, mock_urlopen, mock_token): - mock_token.return_value = "fake-token" - mock_urlopen.side_effect = HTTPError( - url="", code=404, msg="Not Found", hdrs={}, fp=None - ) - - result = infra_validator.grade({ - "params": { - "subscription_id": "sub-123", - "resource_group": "rg-missing", - } - }) - assert result["score"] == 0.0 - assert "does not exist" in result["reason"] - - @patch("infra_validator.get_access_token") - @patch("infra_validator.urlopen") - def test_all_expected_resources_found(self, mock_urlopen, mock_token): - mock_token.return_value = "fake-token" - - rg_resp = MagicMock() - rg_resp.read.return_value = b'{"properties": {"provisioningState": "Succeeded"}}' - - resources_resp = MagicMock() - resources_resp.read.return_value = json.dumps({ - "value": [ - {"type": "Microsoft.Web/sites", "name": "mysite"}, - {"type": "Microsoft.DocumentDB/databaseAccounts", "name": "mydb"}, - ] - }).encode() - - mock_urlopen.side_effect = [rg_resp, resources_resp] - - result = infra_validator.grade({ - "params": { - "subscription_id": "sub-123", - "resource_group": "rg-test", - "expected_resources": [ - "Microsoft.Web/sites", - "Microsoft.DocumentDB/databaseAccounts", - ], - } - }) - assert result["score"] == 1.0 - assert "All expected resources found" in result["reason"] - - @patch("infra_validator.get_access_token") - @patch("infra_validator.urlopen") - def test_missing_expected_resources(self, mock_urlopen, mock_token): - mock_token.return_value = "fake-token" - - rg_resp = MagicMock() - rg_resp.read.return_value = b'{"properties": {"provisioningState": "Succeeded"}}' - - resources_resp = MagicMock() - resources_resp.read.return_value = json.dumps({ - "value": [{"type": "Microsoft.Web/sites", "name": "mysite"}] - }).encode() - - mock_urlopen.side_effect = [rg_resp, resources_resp] - - result = infra_validator.grade({ - "params": { - "subscription_id": "sub-123", - "resource_group": "rg-test", - "expected_resources": [ - "Microsoft.Web/sites", - "Microsoft.DocumentDB/databaseAccounts", - ], - } - }) - assert result["score"] == 0.5 - assert "Missing resources" in result["reason"] - - @patch("infra_validator.get_access_token") - @patch("infra_validator.urlopen") - def test_rg_exists_no_expected_resources(self, mock_urlopen, mock_token): - mock_token.return_value = "fake-token" - - resp = MagicMock() - resp.read.return_value = b'{"properties": {"provisioningState": "Succeeded"}}' - mock_urlopen.return_value = resp - - result = infra_validator.grade({ - "params": { - "subscription_id": "sub-123", - "resource_group": "rg-test", - } - }) - assert result["score"] == 1.0 - assert "exists" in result["reason"] diff --git a/cli/azd/test/eval/jest.config.ts b/cli/azd/test/eval/jest.config.ts deleted file mode 100644 index 84a2281e074..00000000000 --- a/cli/azd/test/eval/jest.config.ts +++ /dev/null @@ -1,31 +0,0 @@ -import type { Config } from "jest"; - -const config: Config = { - preset: "ts-jest", - testEnvironment: "node", - roots: ["/tests"], - testMatch: ["**/*.test.ts"], - moduleFileExtensions: ["ts", "js", "json"], - transform: { - "^.+\\.ts$": "ts-jest", - }, - reporters: [ - "default", - ...(process.env.CI - ? [ - [ - "jest-junit", - { - outputDirectory: "reports", - outputName: "junit.xml", - classNameTemplate: "{classname}", - titleTemplate: "{title}", - }, - ] as [string, Record], - ] - : []), - ], - testTimeout: 30_000, // 30s default — override per-test for slow CLI workflows -}; - -export default config; diff --git a/cli/azd/test/eval/package-lock.json b/cli/azd/test/eval/package-lock.json deleted file mode 100644 index 871a9002b5d..00000000000 --- a/cli/azd/test/eval/package-lock.json +++ /dev/null @@ -1,4067 +0,0 @@ -{ - "name": "@azure/azd-eval", - "version": "0.1.0", - "lockfileVersion": 3, - "requires": true, - "packages": { - "": { - "name": "@azure/azd-eval", - "version": "0.1.0", - "devDependencies": { - "@types/jest": "^29.5.0", - "@types/node": "^20.0.0", - "jest": "^29.7.0", - "jest-junit": "^17.0.0", - "ts-jest": "^29.1.2", - "ts-node": "^10.9.2", - "typescript": "^5.4.0" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@babel/code-frame": { - "version": "7.29.7", - "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.29.7.tgz", - "integrity": "sha512-Aup7aUOfpbAUg2ROOJN6Iw5f9DMBlzu0mIkm/malLQFN/YQgO48wCj0Kxa3sEHJvPVFg7siR+qRInwXd2qhQKw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-validator-identifier": "^7.29.7", - "js-tokens": "^4.0.0", - "picocolors": "^1.1.1" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/compat-data": { - "version": "7.29.7", - "resolved": "https://registry.npmjs.org/@babel/compat-data/-/compat-data-7.29.7.tgz", - "integrity": "sha512-locTkQyKvwIEgBzVrn8693ebc97F2U8ZHjbXwDXJ5Fn2TCpNwTlKcaKLkdHop5c/icOFE7qt7Q9JC5hnKNa6Gg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/core": { - "version": "7.29.7", - "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.29.7.tgz", - "integrity": "sha512-RgHBCvtjbOK2gXSNBNIkNoEc9qoVEtau3hj8gEqKQuL3HZAibKarWFEI3Lfm6EYKkLalOh8eSrj9b+ch9H/VBA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/code-frame": "^7.29.7", - "@babel/generator": "^7.29.7", - "@babel/helper-compilation-targets": "^7.29.7", - "@babel/helper-module-transforms": "^7.29.7", - "@babel/helpers": "^7.29.7", - "@babel/parser": "^7.29.7", - "@babel/template": "^7.29.7", - "@babel/traverse": "^7.29.7", - "@babel/types": "^7.29.7", - "@jridgewell/remapping": "^2.3.5", - "convert-source-map": "^2.0.0", - "debug": "^4.1.0", - "gensync": "^1.0.0-beta.2", - "json5": "^2.2.3", - "semver": "^6.3.1" - }, - "engines": { - "node": ">=6.9.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/babel" - } - }, - "node_modules/@babel/generator": { - "version": "7.29.7", - "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.29.7.tgz", - "integrity": "sha512-DkXD5OJQaAQIdZ1bt3UZdEnHAn9Imd3IVBdX03UFe+ony9Ojw5pzr9YVKGDY1jt+Gcn/FnGkNf8r+Vj5NOJWtQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/parser": "^7.29.7", - "@babel/types": "^7.29.7", - "@jridgewell/gen-mapping": "^0.3.12", - "@jridgewell/trace-mapping": "^0.3.28", - "jsesc": "^3.0.2" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/helper-compilation-targets": { - "version": "7.29.7", - "resolved": "https://registry.npmjs.org/@babel/helper-compilation-targets/-/helper-compilation-targets-7.29.7.tgz", - "integrity": "sha512-wem6WaBj4NaVYVdNhLPPVacES6ZJ+KBBfSkTMD3YZxbP3rm3Di85tJU5ljaUNhaOynt+Aj0xruhYuzQBt8n71g==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/compat-data": "^7.29.7", - "@babel/helper-validator-option": "^7.29.7", - "browserslist": "^4.24.0", - "lru-cache": "^5.1.1", - "semver": "^6.3.1" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/helper-globals": { - "version": "7.29.7", - "resolved": "https://registry.npmjs.org/@babel/helper-globals/-/helper-globals-7.29.7.tgz", - "integrity": "sha512-3nQVUAtvkKH9zahfWgw96Jc/uFOmjACE1kQz82E2lqWmHBgjzbNlsC22nuQTfahmWeQtTq5nQ/4Nnd2A1wj4zA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/helper-module-imports": { - "version": "7.29.7", - "resolved": "https://registry.npmjs.org/@babel/helper-module-imports/-/helper-module-imports-7.29.7.tgz", - "integrity": "sha512-ejHwrQQYcm9xnTivShn2IDOlIzInN34AXskvq9QicvCtEzq1Vzclu/tKF8Jq1Cg8JG2GL6/EmjgsCT7lXepE3g==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/traverse": "^7.29.7", - "@babel/types": "^7.29.7" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/helper-module-transforms": { - "version": "7.29.7", - "resolved": "https://registry.npmjs.org/@babel/helper-module-transforms/-/helper-module-transforms-7.29.7.tgz", - "integrity": "sha512-UPUVSyXbOh627KiCIGQSgwWzGeBKLkaJ9PJEdrngIwMSzxLR4jS4+f1f1jb7VzBbg8nFLaYotvVPFCTqdrmTAg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-module-imports": "^7.29.7", - "@babel/helper-validator-identifier": "^7.29.7", - "@babel/traverse": "^7.29.7" - }, - "engines": { - "node": ">=6.9.0" - }, - "peerDependencies": { - "@babel/core": "^7.0.0" - } - }, - "node_modules/@babel/helper-plugin-utils": { - "version": "7.28.6", - "resolved": "https://registry.npmjs.org/@babel/helper-plugin-utils/-/helper-plugin-utils-7.28.6.tgz", - "integrity": "sha512-S9gzZ/bz83GRysI7gAD4wPT/AI3uCnY+9xn+Mx/KPs2JwHJIz1W8PZkg2cqyt3RNOBM8ejcXhV6y8Og7ly/Dug==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/helper-string-parser": { - "version": "7.29.7", - "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.29.7.tgz", - "integrity": "sha512-Pb5ijPrZ89GDH8223L4UP8i6QApWxs04RbPQJTeWDV0/keR2E36MeKnyr6LYmUUvqRRI+Iv87SuF1W6ErINzYw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/helper-validator-identifier": { - "version": "7.29.7", - "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.29.7.tgz", - "integrity": "sha512-qehxGkRj55h/ff8EMaJ+cYhyaKlHIxqYDn682wQD7RNp9UujOQsHog2uS0r2vzr4pW+sXf90NeeayjcNaX3fFg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/helper-validator-option": { - "version": "7.29.7", - "resolved": "https://registry.npmjs.org/@babel/helper-validator-option/-/helper-validator-option-7.29.7.tgz", - "integrity": "sha512-N9ZErrD+yW5geCDtBqnOoxmR8+tNKiGuxKlDpuJxfsqpa2dFcexaziGAE/qoHLiDDreVNMupxGmSoNlyvsA3gw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/helpers": { - "version": "7.29.7", - "resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.29.7.tgz", - "integrity": "sha512-1k2lAGRMfHTcwuNYcCNUmaUffmQv8KWMfh2iJUUeRlwlwH4FdNG7mfPI10NPfLHJFThE4Tyr4mv7kTNZOiPuBg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/template": "^7.29.7", - "@babel/types": "^7.29.7" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/parser": { - "version": "7.29.7", - "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.29.7.tgz", - "integrity": "sha512-hnORnjP/1P/zFEndoeX+n+t1RwWRJiJpM/jO7FW32Kn9r5+sJB2JWOdYo4L6k78j15eCwY3Gm/7364B1EMwtNg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/types": "^7.29.7" - }, - "bin": { - "parser": "bin/babel-parser.js" - }, - "engines": { - "node": ">=6.0.0" - } - }, - "node_modules/@babel/plugin-syntax-async-generators": { - "version": "7.8.4", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-async-generators/-/plugin-syntax-async-generators-7.8.4.tgz", - "integrity": "sha512-tycmZxkGfZaxhMRbXlPXuVFpdWlXpir2W4AMhSJgRKzk/eDlIXOhb2LHWoLpDF7TEHylV5zNhykX6KAgHJmTNw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-plugin-utils": "^7.8.0" - }, - "peerDependencies": { - "@babel/core": "^7.0.0-0" - } - }, - "node_modules/@babel/plugin-syntax-bigint": { - "version": "7.8.3", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-bigint/-/plugin-syntax-bigint-7.8.3.tgz", - "integrity": "sha512-wnTnFlG+YxQm3vDxpGE57Pj0srRU4sHE/mDkt1qv2YJJSeUAec2ma4WLUnUPeKjyrfntVwe/N6dCXpU+zL3Npg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-plugin-utils": "^7.8.0" - }, - "peerDependencies": { - "@babel/core": "^7.0.0-0" - } - }, - "node_modules/@babel/plugin-syntax-class-properties": { - "version": "7.12.13", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-class-properties/-/plugin-syntax-class-properties-7.12.13.tgz", - "integrity": "sha512-fm4idjKla0YahUNgFNLCB0qySdsoPiZP3iQE3rky0mBUtMZ23yDJ9SJdg6dXTSDnulOVqiF3Hgr9nbXvXTQZYA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-plugin-utils": "^7.12.13" - }, - "peerDependencies": { - "@babel/core": "^7.0.0-0" - } - }, - "node_modules/@babel/plugin-syntax-class-static-block": { - "version": "7.14.5", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-class-static-block/-/plugin-syntax-class-static-block-7.14.5.tgz", - "integrity": "sha512-b+YyPmr6ldyNnM6sqYeMWE+bgJcJpO6yS4QD7ymxgH34GBPNDM/THBh8iunyvKIZztiwLH4CJZ0RxTk9emgpjw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-plugin-utils": "^7.14.5" - }, - "engines": { - "node": ">=6.9.0" - }, - "peerDependencies": { - "@babel/core": "^7.0.0-0" - } - }, - "node_modules/@babel/plugin-syntax-import-attributes": { - "version": "7.28.6", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-import-attributes/-/plugin-syntax-import-attributes-7.28.6.tgz", - "integrity": "sha512-jiLC0ma9XkQT3TKJ9uYvlakm66Pamywo+qwL+oL8HJOvc6TWdZXVfhqJr8CCzbSGUAbDOzlGHJC1U+vRfLQDvw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-plugin-utils": "^7.28.6" - }, - "engines": { - "node": ">=6.9.0" - }, - "peerDependencies": { - "@babel/core": "^7.0.0-0" - } - }, - "node_modules/@babel/plugin-syntax-import-meta": { - "version": "7.10.4", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-import-meta/-/plugin-syntax-import-meta-7.10.4.tgz", - "integrity": "sha512-Yqfm+XDx0+Prh3VSeEQCPU81yC+JWZ2pDPFSS4ZdpfZhp4MkFMaDC1UqseovEKwSUpnIL7+vK+Clp7bfh0iD7g==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-plugin-utils": "^7.10.4" - }, - "peerDependencies": { - "@babel/core": "^7.0.0-0" - } - }, - "node_modules/@babel/plugin-syntax-json-strings": { - "version": "7.8.3", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-json-strings/-/plugin-syntax-json-strings-7.8.3.tgz", - "integrity": "sha512-lY6kdGpWHvjoe2vk4WrAapEuBR69EMxZl+RoGRhrFGNYVK8mOPAW8VfbT/ZgrFbXlDNiiaxQnAtgVCZ6jv30EA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-plugin-utils": "^7.8.0" - }, - "peerDependencies": { - "@babel/core": "^7.0.0-0" - } - }, - "node_modules/@babel/plugin-syntax-jsx": { - "version": "7.28.6", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-jsx/-/plugin-syntax-jsx-7.28.6.tgz", - "integrity": "sha512-wgEmr06G6sIpqr8YDwA2dSRTE3bJ+V0IfpzfSY3Lfgd7YWOaAdlykvJi13ZKBt8cZHfgH1IXN+CL656W3uUa4w==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-plugin-utils": "^7.28.6" - }, - "engines": { - "node": ">=6.9.0" - }, - "peerDependencies": { - "@babel/core": "^7.0.0-0" - } - }, - "node_modules/@babel/plugin-syntax-logical-assignment-operators": { - "version": "7.10.4", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-logical-assignment-operators/-/plugin-syntax-logical-assignment-operators-7.10.4.tgz", - "integrity": "sha512-d8waShlpFDinQ5MtvGU9xDAOzKH47+FFoney2baFIoMr952hKOLp1HR7VszoZvOsV/4+RRszNY7D17ba0te0ig==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-plugin-utils": "^7.10.4" - }, - "peerDependencies": { - "@babel/core": "^7.0.0-0" - } - }, - "node_modules/@babel/plugin-syntax-nullish-coalescing-operator": { - "version": "7.8.3", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-nullish-coalescing-operator/-/plugin-syntax-nullish-coalescing-operator-7.8.3.tgz", - "integrity": "sha512-aSff4zPII1u2QD7y+F8oDsz19ew4IGEJg9SVW+bqwpwtfFleiQDMdzA/R+UlWDzfnHFCxxleFT0PMIrR36XLNQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-plugin-utils": "^7.8.0" - }, - "peerDependencies": { - "@babel/core": "^7.0.0-0" - } - }, - "node_modules/@babel/plugin-syntax-numeric-separator": { - "version": "7.10.4", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-numeric-separator/-/plugin-syntax-numeric-separator-7.10.4.tgz", - "integrity": "sha512-9H6YdfkcK/uOnY/K7/aA2xpzaAgkQn37yzWUMRK7OaPOqOpGS1+n0H5hxT9AUw9EsSjPW8SVyMJwYRtWs3X3ug==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-plugin-utils": "^7.10.4" - }, - "peerDependencies": { - "@babel/core": "^7.0.0-0" - } - }, - "node_modules/@babel/plugin-syntax-object-rest-spread": { - "version": "7.8.3", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-object-rest-spread/-/plugin-syntax-object-rest-spread-7.8.3.tgz", - "integrity": "sha512-XoqMijGZb9y3y2XskN+P1wUGiVwWZ5JmoDRwx5+3GmEplNyVM2s2Dg8ILFQm8rWM48orGy5YpI5Bl8U1y7ydlA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-plugin-utils": "^7.8.0" - }, - "peerDependencies": { - "@babel/core": "^7.0.0-0" - } - }, - "node_modules/@babel/plugin-syntax-optional-catch-binding": { - "version": "7.8.3", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-optional-catch-binding/-/plugin-syntax-optional-catch-binding-7.8.3.tgz", - "integrity": "sha512-6VPD0Pc1lpTqw0aKoeRTMiB+kWhAoT24PA+ksWSBrFtl5SIRVpZlwN3NNPQjehA2E/91FV3RjLWoVTglWcSV3Q==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-plugin-utils": "^7.8.0" - }, - "peerDependencies": { - "@babel/core": "^7.0.0-0" - } - }, - "node_modules/@babel/plugin-syntax-optional-chaining": { - "version": "7.8.3", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-optional-chaining/-/plugin-syntax-optional-chaining-7.8.3.tgz", - "integrity": "sha512-KoK9ErH1MBlCPxV0VANkXW2/dw4vlbGDrFgz8bmUsBGYkFRcbRwMh6cIJubdPrkxRwuGdtCk0v/wPTKbQgBjkg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-plugin-utils": "^7.8.0" - }, - "peerDependencies": { - "@babel/core": "^7.0.0-0" - } - }, - "node_modules/@babel/plugin-syntax-private-property-in-object": { - "version": "7.14.5", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-private-property-in-object/-/plugin-syntax-private-property-in-object-7.14.5.tgz", - "integrity": "sha512-0wVnp9dxJ72ZUJDV27ZfbSj6iHLoytYZmh3rFcxNnvsJF3ktkzLDZPy/mA17HGsaQT3/DQsWYX1f1QGWkCoVUg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-plugin-utils": "^7.14.5" - }, - "engines": { - "node": ">=6.9.0" - }, - "peerDependencies": { - "@babel/core": "^7.0.0-0" - } - }, - "node_modules/@babel/plugin-syntax-top-level-await": { - "version": "7.14.5", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-top-level-await/-/plugin-syntax-top-level-await-7.14.5.tgz", - "integrity": "sha512-hx++upLv5U1rgYfwe1xBQUhRmU41NEvpUvrp8jkrSCdvGSnM5/qdRMtylJ6PG5OFkBaHkbTAKTnd3/YyESRHFw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-plugin-utils": "^7.14.5" - }, - "engines": { - "node": ">=6.9.0" - }, - "peerDependencies": { - "@babel/core": "^7.0.0-0" - } - }, - "node_modules/@babel/plugin-syntax-typescript": { - "version": "7.28.6", - "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-typescript/-/plugin-syntax-typescript-7.28.6.tgz", - "integrity": "sha512-+nDNmQye7nlnuuHDboPbGm00Vqg3oO8niRRL27/4LYHUsHYh0zJ1xWOz0uRwNFmM1Avzk8wZbc6rdiYhomzv/A==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-plugin-utils": "^7.28.6" - }, - "engines": { - "node": ">=6.9.0" - }, - "peerDependencies": { - "@babel/core": "^7.0.0-0" - } - }, - "node_modules/@babel/template": { - "version": "7.29.7", - "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.29.7.tgz", - "integrity": "sha512-puq+Gf35oI24FeN11LkoUQFqv9uwNeWpxXZi/Ji3rRIoKAzKnxRaZ+Gkj0vKS9ZCiTESfng1N9LyOyXvo+m+Gg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/code-frame": "^7.29.7", - "@babel/parser": "^7.29.7", - "@babel/types": "^7.29.7" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/traverse": { - "version": "7.29.7", - "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.29.7.tgz", - "integrity": "sha512-EhlfNQtZ+NK22w5BM61ciuiq1m58ed33Wr1Xan//ZRTy6hgjnwyCffRYwzsGXdASJSUJ1guZILsErh1eQcl+zw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/code-frame": "^7.29.7", - "@babel/generator": "^7.29.7", - "@babel/helper-globals": "^7.29.7", - "@babel/parser": "^7.29.7", - "@babel/template": "^7.29.7", - "@babel/types": "^7.29.7", - "debug": "^4.3.1" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/types": { - "version": "7.29.7", - "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.29.7.tgz", - "integrity": "sha512-4zBIxpPzowiZpusoFkyGVwakdRJUyuH5PxQ/PrqghfdFWWasvnCdPfQXHrenDai+gyLARulZjZowCOj6fjT4pA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-string-parser": "^7.29.7", - "@babel/helper-validator-identifier": "^7.29.7" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@bcoe/v8-coverage": { - "version": "0.2.3", - "resolved": "https://registry.npmjs.org/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz", - "integrity": "sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==", - "dev": true, - "license": "MIT" - }, - "node_modules/@cspotcode/source-map-support": { - "version": "0.8.1", - "resolved": "https://registry.npmjs.org/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz", - "integrity": "sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jridgewell/trace-mapping": "0.3.9" - }, - "engines": { - "node": ">=12" - } - }, - "node_modules/@cspotcode/source-map-support/node_modules/@jridgewell/trace-mapping": { - "version": "0.3.9", - "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.9.tgz", - "integrity": "sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jridgewell/resolve-uri": "^3.0.3", - "@jridgewell/sourcemap-codec": "^1.4.10" - } - }, - "node_modules/@istanbuljs/load-nyc-config": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@istanbuljs/load-nyc-config/-/load-nyc-config-1.1.0.tgz", - "integrity": "sha512-VjeHSlIzpv/NyD3N0YuHfXOPDIixcA1q2ZV98wsMqcYlPmv2n3Yb2lYP9XMElnaFVXg5A7YLTeLu6V84uQDjmQ==", - "dev": true, - "license": "ISC", - "dependencies": { - "camelcase": "^5.3.1", - "find-up": "^4.1.0", - "get-package-type": "^0.1.0", - "js-yaml": "^3.13.1", - "resolve-from": "^5.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/@istanbuljs/schema": { - "version": "0.1.3", - "resolved": "https://registry.npmjs.org/@istanbuljs/schema/-/schema-0.1.3.tgz", - "integrity": "sha512-ZXRY4jNvVgSVQ8DL3LTcakaAtXwTVUxE81hslsyD2AtoXW/wVob10HkOJ1X/pAlcI7D+2YoZKg5do8G/w6RYgA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/@jest/console": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/@jest/console/-/console-29.7.0.tgz", - "integrity": "sha512-5Ni4CU7XHQi32IJ398EEP4RrB8eV09sXP2ROqD4bksHrnTree52PsxvX8tpL8LvTZ3pFzXyPbNQReSN41CAhOg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/types": "^29.6.3", - "@types/node": "*", - "chalk": "^4.0.0", - "jest-message-util": "^29.7.0", - "jest-util": "^29.7.0", - "slash": "^3.0.0" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/@jest/core": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/@jest/core/-/core-29.7.0.tgz", - "integrity": "sha512-n7aeXWKMnGtDA48y8TLWJPJmLmmZ642Ceo78cYWEpiD7FzDgmNDV/GCVRorPABdXLJZ/9wzzgZAlHjXjxDHGsg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/console": "^29.7.0", - "@jest/reporters": "^29.7.0", - "@jest/test-result": "^29.7.0", - "@jest/transform": "^29.7.0", - "@jest/types": "^29.6.3", - "@types/node": "*", - "ansi-escapes": "^4.2.1", - "chalk": "^4.0.0", - "ci-info": "^3.2.0", - "exit": "^0.1.2", - "graceful-fs": "^4.2.9", - "jest-changed-files": "^29.7.0", - "jest-config": "^29.7.0", - "jest-haste-map": "^29.7.0", - "jest-message-util": "^29.7.0", - "jest-regex-util": "^29.6.3", - "jest-resolve": "^29.7.0", - "jest-resolve-dependencies": "^29.7.0", - "jest-runner": "^29.7.0", - "jest-runtime": "^29.7.0", - "jest-snapshot": "^29.7.0", - "jest-util": "^29.7.0", - "jest-validate": "^29.7.0", - "jest-watcher": "^29.7.0", - "micromatch": "^4.0.4", - "pretty-format": "^29.7.0", - "slash": "^3.0.0", - "strip-ansi": "^6.0.0" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - }, - "peerDependencies": { - "node-notifier": "^8.0.1 || ^9.0.0 || ^10.0.0" - }, - "peerDependenciesMeta": { - "node-notifier": { - "optional": true - } - } - }, - "node_modules/@jest/environment": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/@jest/environment/-/environment-29.7.0.tgz", - "integrity": "sha512-aQIfHDq33ExsN4jP1NWGXhxgQ/wixs60gDiKO+XVMd8Mn0NWPWgc34ZQDTb2jKaUWQ7MuwoitXAsN2XVXNMpAw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/fake-timers": "^29.7.0", - "@jest/types": "^29.6.3", - "@types/node": "*", - "jest-mock": "^29.7.0" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/@jest/expect": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/@jest/expect/-/expect-29.7.0.tgz", - "integrity": "sha512-8uMeAMycttpva3P1lBHB8VciS9V0XAr3GymPpipdyQXbBcuhkLQOSe8E/p92RyAdToS6ZD1tFkX+CkhoECE0dQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "expect": "^29.7.0", - "jest-snapshot": "^29.7.0" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/@jest/expect-utils": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/@jest/expect-utils/-/expect-utils-29.7.0.tgz", - "integrity": "sha512-GlsNBWiFQFCVi9QVSx7f5AgMeLxe9YCCs5PuP2O2LdjDAA8Jh9eX7lA1Jq/xdXw3Wb3hyvlFNfZIfcRetSzYcA==", - "dev": true, - "license": "MIT", - "dependencies": { - "jest-get-type": "^29.6.3" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/@jest/fake-timers": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/@jest/fake-timers/-/fake-timers-29.7.0.tgz", - "integrity": "sha512-q4DH1Ha4TTFPdxLsqDXK1d3+ioSL7yL5oCMJZgDYm6i+6CygW5E5xVr/D1HdsGxjt1ZWSfUAs9OxSB/BNelWrQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/types": "^29.6.3", - "@sinonjs/fake-timers": "^10.0.2", - "@types/node": "*", - "jest-message-util": "^29.7.0", - "jest-mock": "^29.7.0", - "jest-util": "^29.7.0" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/@jest/globals": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/@jest/globals/-/globals-29.7.0.tgz", - "integrity": "sha512-mpiz3dutLbkW2MNFubUGUEVLkTGiqW6yLVTA+JbP6fI6J5iL9Y0Nlg8k95pcF8ctKwCS7WVxteBs29hhfAotzQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/environment": "^29.7.0", - "@jest/expect": "^29.7.0", - "@jest/types": "^29.6.3", - "jest-mock": "^29.7.0" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/@jest/reporters": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/@jest/reporters/-/reporters-29.7.0.tgz", - "integrity": "sha512-DApq0KJbJOEzAFYjHADNNxAE3KbhxQB1y5Kplb5Waqw6zVbuWatSnMjE5gs8FUgEPmNsnZA3NCWl9NG0ia04Pg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@bcoe/v8-coverage": "^0.2.3", - "@jest/console": "^29.7.0", - "@jest/test-result": "^29.7.0", - "@jest/transform": "^29.7.0", - "@jest/types": "^29.6.3", - "@jridgewell/trace-mapping": "^0.3.18", - "@types/node": "*", - "chalk": "^4.0.0", - "collect-v8-coverage": "^1.0.0", - "exit": "^0.1.2", - "glob": "^7.1.3", - "graceful-fs": "^4.2.9", - "istanbul-lib-coverage": "^3.0.0", - "istanbul-lib-instrument": "^6.0.0", - "istanbul-lib-report": "^3.0.0", - "istanbul-lib-source-maps": "^4.0.0", - "istanbul-reports": "^3.1.3", - "jest-message-util": "^29.7.0", - "jest-util": "^29.7.0", - "jest-worker": "^29.7.0", - "slash": "^3.0.0", - "string-length": "^4.0.1", - "strip-ansi": "^6.0.0", - "v8-to-istanbul": "^9.0.1" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - }, - "peerDependencies": { - "node-notifier": "^8.0.1 || ^9.0.0 || ^10.0.0" - }, - "peerDependenciesMeta": { - "node-notifier": { - "optional": true - } - } - }, - "node_modules/@jest/schemas": { - "version": "29.6.3", - "resolved": "https://registry.npmjs.org/@jest/schemas/-/schemas-29.6.3.tgz", - "integrity": "sha512-mo5j5X+jIZmJQveBKeS/clAueipV7KgiX1vMgCxam1RNYiqE1w62n0/tJJnHtjW8ZHcQco5gY85jA3mi0L+nSA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@sinclair/typebox": "^0.27.8" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/@jest/source-map": { - "version": "29.6.3", - "resolved": "https://registry.npmjs.org/@jest/source-map/-/source-map-29.6.3.tgz", - "integrity": "sha512-MHjT95QuipcPrpLM+8JMSzFx6eHp5Bm+4XeFDJlwsvVBjmKNiIAvasGK2fxz2WbGRlnvqehFbh07MMa7n3YJnw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jridgewell/trace-mapping": "^0.3.18", - "callsites": "^3.0.0", - "graceful-fs": "^4.2.9" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/@jest/test-result": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/@jest/test-result/-/test-result-29.7.0.tgz", - "integrity": "sha512-Fdx+tv6x1zlkJPcWXmMDAG2HBnaR9XPSd5aDWQVsfrZmLVT3lU1cwyxLgRmXR9yrq4NBoEm9BMsfgFzTQAbJYA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/console": "^29.7.0", - "@jest/types": "^29.6.3", - "@types/istanbul-lib-coverage": "^2.0.0", - "collect-v8-coverage": "^1.0.0" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/@jest/test-sequencer": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/@jest/test-sequencer/-/test-sequencer-29.7.0.tgz", - "integrity": "sha512-GQwJ5WZVrKnOJuiYiAF52UNUJXgTZx1NHjFSEB0qEMmSZKAkdMoIzw/Cj6x6NF4AvV23AUqDpFzQkN/eYCYTxw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/test-result": "^29.7.0", - "graceful-fs": "^4.2.9", - "jest-haste-map": "^29.7.0", - "slash": "^3.0.0" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/@jest/transform": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/@jest/transform/-/transform-29.7.0.tgz", - "integrity": "sha512-ok/BTPFzFKVMwO5eOHRrvnBVHdRy9IrsrW1GpMaQ9MCnilNLXQKmAX8s1YXDFaai9xJpac2ySzV0YeRRECr2Vw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/core": "^7.11.6", - "@jest/types": "^29.6.3", - "@jridgewell/trace-mapping": "^0.3.18", - "babel-plugin-istanbul": "^6.1.1", - "chalk": "^4.0.0", - "convert-source-map": "^2.0.0", - "fast-json-stable-stringify": "^2.1.0", - "graceful-fs": "^4.2.9", - "jest-haste-map": "^29.7.0", - "jest-regex-util": "^29.6.3", - "jest-util": "^29.7.0", - "micromatch": "^4.0.4", - "pirates": "^4.0.4", - "slash": "^3.0.0", - "write-file-atomic": "^4.0.2" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/@jest/types": { - "version": "29.6.3", - "resolved": "https://registry.npmjs.org/@jest/types/-/types-29.6.3.tgz", - "integrity": "sha512-u3UPsIilWKOM3F9CXtrG8LEJmNxwoCQC/XVj4IKYXvvpx7QIi/Kg1LI5uDmDpKlac62NUtX7eLjRh+jVZcLOzw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/schemas": "^29.6.3", - "@types/istanbul-lib-coverage": "^2.0.0", - "@types/istanbul-reports": "^3.0.0", - "@types/node": "*", - "@types/yargs": "^17.0.8", - "chalk": "^4.0.0" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/@jridgewell/gen-mapping": { - "version": "0.3.13", - "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.13.tgz", - "integrity": "sha512-2kkt/7niJ6MgEPxF0bYdQ6etZaA+fQvDcLKckhy1yIQOzaoKjBBjSj63/aLVjYE3qhRt5dvM+uUyfCg6UKCBbA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jridgewell/sourcemap-codec": "^1.5.0", - "@jridgewell/trace-mapping": "^0.3.24" - } - }, - "node_modules/@jridgewell/remapping": { - "version": "2.3.5", - "resolved": "https://registry.npmjs.org/@jridgewell/remapping/-/remapping-2.3.5.tgz", - "integrity": "sha512-LI9u/+laYG4Ds1TDKSJW2YPrIlcVYOwi2fUC6xB43lueCjgxV4lffOCZCtYFiH6TNOX+tQKXx97T4IKHbhyHEQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jridgewell/gen-mapping": "^0.3.5", - "@jridgewell/trace-mapping": "^0.3.24" - } - }, - "node_modules/@jridgewell/resolve-uri": { - "version": "3.1.2", - "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", - "integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6.0.0" - } - }, - "node_modules/@jridgewell/sourcemap-codec": { - "version": "1.5.5", - "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.5.tgz", - "integrity": "sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og==", - "dev": true, - "license": "MIT" - }, - "node_modules/@jridgewell/trace-mapping": { - "version": "0.3.31", - "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.31.tgz", - "integrity": "sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jridgewell/resolve-uri": "^3.1.0", - "@jridgewell/sourcemap-codec": "^1.4.14" - } - }, - "node_modules/@sinclair/typebox": { - "version": "0.27.10", - "resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.27.10.tgz", - "integrity": "sha512-MTBk/3jGLNB2tVxv6uLlFh1iu64iYOQ2PbdOSK3NW8JZsmlaOh2q6sdtKowBhfw8QFLmYNzTW4/oK4uATIi6ZA==", - "dev": true, - "license": "MIT" - }, - "node_modules/@sinonjs/commons": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/@sinonjs/commons/-/commons-3.0.1.tgz", - "integrity": "sha512-K3mCHKQ9sVh8o1C9cxkwxaOmXoAMlDxC1mYyHrjqOWEcBjYr76t96zL2zlj5dUGZ3HSw240X1qgH3Mjf1yJWpQ==", - "dev": true, - "license": "BSD-3-Clause", - "dependencies": { - "type-detect": "4.0.8" - } - }, - "node_modules/@sinonjs/fake-timers": { - "version": "10.3.0", - "resolved": "https://registry.npmjs.org/@sinonjs/fake-timers/-/fake-timers-10.3.0.tgz", - "integrity": "sha512-V4BG07kuYSUkTCSBHG8G8TNhM+F19jXFWnQtzj+we8DrkpSBCee9Z3Ms8yiGer/dlmhe35/Xdgyo3/0rQKg7YA==", - "dev": true, - "license": "BSD-3-Clause", - "dependencies": { - "@sinonjs/commons": "^3.0.0" - } - }, - "node_modules/@tsconfig/node10": { - "version": "1.0.12", - "resolved": "https://registry.npmjs.org/@tsconfig/node10/-/node10-1.0.12.tgz", - "integrity": "sha512-UCYBaeFvM11aU2y3YPZ//O5Rhj+xKyzy7mvcIoAjASbigy8mHMryP5cK7dgjlz2hWxh1g5pLw084E0a/wlUSFQ==", - "dev": true, - "license": "MIT" - }, - "node_modules/@tsconfig/node12": { - "version": "1.0.11", - "resolved": "https://registry.npmjs.org/@tsconfig/node12/-/node12-1.0.11.tgz", - "integrity": "sha512-cqefuRsh12pWyGsIoBKJA9luFu3mRxCA+ORZvA4ktLSzIuCUtWVxGIuXigEwO5/ywWFMZ2QEGKWvkZG1zDMTag==", - "dev": true, - "license": "MIT" - }, - "node_modules/@tsconfig/node14": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/@tsconfig/node14/-/node14-1.0.3.tgz", - "integrity": "sha512-ysT8mhdixWK6Hw3i1V2AeRqZ5WfXg1G43mqoYlM2nc6388Fq5jcXyr5mRsqViLx/GJYdoL0bfXD8nmF+Zn/Iow==", - "dev": true, - "license": "MIT" - }, - "node_modules/@tsconfig/node16": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/@tsconfig/node16/-/node16-1.0.4.tgz", - "integrity": "sha512-vxhUy4J8lyeyinH7Azl1pdd43GJhZH/tP2weN8TntQblOY+A0XbT8DJk1/oCPuOOyg/Ja757rG0CgHcWC8OfMA==", - "dev": true, - "license": "MIT" - }, - "node_modules/@types/babel__core": { - "version": "7.20.5", - "resolved": "https://registry.npmjs.org/@types/babel__core/-/babel__core-7.20.5.tgz", - "integrity": "sha512-qoQprZvz5wQFJwMDqeseRXWv3rqMvhgpbXFfVyWhbx9X47POIA6i/+dXefEmZKoAgOaTdaIgNSMqMIU61yRyzA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/parser": "^7.20.7", - "@babel/types": "^7.20.7", - "@types/babel__generator": "*", - "@types/babel__template": "*", - "@types/babel__traverse": "*" - } - }, - "node_modules/@types/babel__generator": { - "version": "7.27.0", - "resolved": "https://registry.npmjs.org/@types/babel__generator/-/babel__generator-7.27.0.tgz", - "integrity": "sha512-ufFd2Xi92OAVPYsy+P4n7/U7e68fex0+Ee8gSG9KX7eo084CWiQ4sdxktvdl0bOPupXtVJPY19zk6EwWqUQ8lg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/types": "^7.0.0" - } - }, - "node_modules/@types/babel__template": { - "version": "7.4.4", - "resolved": "https://registry.npmjs.org/@types/babel__template/-/babel__template-7.4.4.tgz", - "integrity": "sha512-h/NUaSyG5EyxBIp8YRxo4RMe2/qQgvyowRwVMzhYhBCONbW8PUsg4lkFMrhgZhUe5z3L3MiLDuvyJ/CaPa2A8A==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/parser": "^7.1.0", - "@babel/types": "^7.0.0" - } - }, - "node_modules/@types/babel__traverse": { - "version": "7.28.0", - "resolved": "https://registry.npmjs.org/@types/babel__traverse/-/babel__traverse-7.28.0.tgz", - "integrity": "sha512-8PvcXf70gTDZBgt9ptxJ8elBeBjcLOAcOtoO/mPJjtji1+CdGbHgm77om1GrsPxsiE+uXIpNSK64UYaIwQXd4Q==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/types": "^7.28.2" - } - }, - "node_modules/@types/graceful-fs": { - "version": "4.1.9", - "resolved": "https://registry.npmjs.org/@types/graceful-fs/-/graceful-fs-4.1.9.tgz", - "integrity": "sha512-olP3sd1qOEe5dXTSaFvQG+02VdRXcdytWLAZsAq1PecU8uqQAhkrnbli7DagjtXKW/Bl7YJbUsa8MPcuc8LHEQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/node": "*" - } - }, - "node_modules/@types/istanbul-lib-coverage": { - "version": "2.0.6", - "resolved": "https://registry.npmjs.org/@types/istanbul-lib-coverage/-/istanbul-lib-coverage-2.0.6.tgz", - "integrity": "sha512-2QF/t/auWm0lsy8XtKVPG19v3sSOQlJe/YHZgfjb/KBBHOGSV+J2q/S671rcq9uTBrLAXmZpqJiaQbMT+zNU1w==", - "dev": true, - "license": "MIT" - }, - "node_modules/@types/istanbul-lib-report": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/@types/istanbul-lib-report/-/istanbul-lib-report-3.0.3.tgz", - "integrity": "sha512-NQn7AHQnk/RSLOxrBbGyJM/aVQ+pjj5HCgasFxc0K/KhoATfQ/47AyUl15I2yBUpihjmas+a+VJBOqecrFH+uA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/istanbul-lib-coverage": "*" - } - }, - "node_modules/@types/istanbul-reports": { - "version": "3.0.4", - "resolved": "https://registry.npmjs.org/@types/istanbul-reports/-/istanbul-reports-3.0.4.tgz", - "integrity": "sha512-pk2B1NWalF9toCRu6gjBzR69syFjP4Od8WRAX+0mmf9lAjCRicLOWc+ZrxZHx/0XRjotgkF9t6iaMJ+aXcOdZQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/istanbul-lib-report": "*" - } - }, - "node_modules/@types/jest": { - "version": "29.5.14", - "resolved": "https://registry.npmjs.org/@types/jest/-/jest-29.5.14.tgz", - "integrity": "sha512-ZN+4sdnLUbo8EVvVc2ao0GFW6oVrQRPn4K2lglySj7APvSrgzxHiNNK99us4WDMi57xxA2yggblIAMNhXOotLQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "expect": "^29.0.0", - "pretty-format": "^29.0.0" - } - }, - "node_modules/@types/node": { - "version": "20.19.37", - "resolved": "https://registry.npmjs.org/@types/node/-/node-20.19.37.tgz", - "integrity": "sha512-8kzdPJ3FsNsVIurqBs7oodNnCEVbni9yUEkaHbgptDACOPW04jimGagZ51E6+lXUwJjgnBw+hyko/lkFWCldqw==", - "dev": true, - "license": "MIT", - "dependencies": { - "undici-types": "~6.21.0" - } - }, - "node_modules/@types/stack-utils": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/@types/stack-utils/-/stack-utils-2.0.3.tgz", - "integrity": "sha512-9aEbYZ3TbYMznPdcdr3SmIrLXwC/AKZXQeCf9Pgao5CKb8CyHuEX5jzWPTkvregvhRJHcpRO6BFoGW9ycaOkYw==", - "dev": true, - "license": "MIT" - }, - "node_modules/@types/yargs": { - "version": "17.0.35", - "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-17.0.35.tgz", - "integrity": "sha512-qUHkeCyQFxMXg79wQfTtfndEC+N9ZZg76HJftDJp+qH2tV7Gj4OJi7l+PiWwJ+pWtW8GwSmqsDj/oymhrTWXjg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/yargs-parser": "*" - } - }, - "node_modules/@types/yargs-parser": { - "version": "21.0.3", - "resolved": "https://registry.npmjs.org/@types/yargs-parser/-/yargs-parser-21.0.3.tgz", - "integrity": "sha512-I4q9QU9MQv4oEOz4tAHJtNz1cwuLxn2F3xcc2iV5WdqLPpUnj30aUuxt1mAxYTG+oe8CZMV/+6rU4S4gRDzqtQ==", - "dev": true, - "license": "MIT" - }, - "node_modules/acorn": { - "version": "8.16.0", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.16.0.tgz", - "integrity": "sha512-UVJyE9MttOsBQIDKw1skb9nAwQuR5wuGD3+82K6JgJlm/Y+KI92oNsMNGZCYdDsVtRHSak0pcV5Dno5+4jh9sw==", - "dev": true, - "license": "MIT", - "bin": { - "acorn": "bin/acorn" - }, - "engines": { - "node": ">=0.4.0" - } - }, - "node_modules/acorn-walk": { - "version": "8.3.5", - "resolved": "https://registry.npmjs.org/acorn-walk/-/acorn-walk-8.3.5.tgz", - "integrity": "sha512-HEHNfbars9v4pgpW6SO1KSPkfoS0xVOM/9UzkJltjlsHZmJasxg8aXkuZa7SMf8vKGIBhpUsPluQSqhJFCqebw==", - "dev": true, - "license": "MIT", - "dependencies": { - "acorn": "^8.11.0" - }, - "engines": { - "node": ">=0.4.0" - } - }, - "node_modules/ansi-escapes": { - "version": "4.3.2", - "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-4.3.2.tgz", - "integrity": "sha512-gKXj5ALrKWQLsYG9jlTRmR/xKluxHV+Z9QEwNIgCfM1/uwPMCuzVVnh5mwTd+OuBZcwSIMbqssNWRm1lE51QaQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "type-fest": "^0.21.3" - }, - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/ansi-regex": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", - "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/ansi-styles": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", - "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", - "dev": true, - "license": "MIT", - "dependencies": { - "color-convert": "^2.0.1" - }, - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/chalk/ansi-styles?sponsor=1" - } - }, - "node_modules/anymatch": { - "version": "3.1.3", - "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.3.tgz", - "integrity": "sha512-KMReFUr0B4t+D+OBkjR3KYqvocp2XaSzO55UcB6mgQMd3KbcE+mWTyvVV7D/zsdEbNnV6acZUutkiHQXvTr1Rw==", - "dev": true, - "license": "ISC", - "dependencies": { - "normalize-path": "^3.0.0", - "picomatch": "^2.0.4" - }, - "engines": { - "node": ">= 8" - } - }, - "node_modules/arg": { - "version": "4.1.3", - "resolved": "https://registry.npmjs.org/arg/-/arg-4.1.3.tgz", - "integrity": "sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA==", - "dev": true, - "license": "MIT" - }, - "node_modules/argparse": { - "version": "1.0.10", - "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", - "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", - "dev": true, - "license": "MIT", - "dependencies": { - "sprintf-js": "~1.0.2" - } - }, - "node_modules/babel-jest": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/babel-jest/-/babel-jest-29.7.0.tgz", - "integrity": "sha512-BrvGY3xZSwEcCzKvKsCi2GgHqDqsYkOP4/by5xCgIwGXQxIEh+8ew3gmrE1y7XRR6LHZIj6yLYnUi/mm2KXKBg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/transform": "^29.7.0", - "@types/babel__core": "^7.1.14", - "babel-plugin-istanbul": "^6.1.1", - "babel-preset-jest": "^29.6.3", - "chalk": "^4.0.0", - "graceful-fs": "^4.2.9", - "slash": "^3.0.0" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - }, - "peerDependencies": { - "@babel/core": "^7.8.0" - } - }, - "node_modules/babel-plugin-istanbul": { - "version": "6.1.1", - "resolved": "https://registry.npmjs.org/babel-plugin-istanbul/-/babel-plugin-istanbul-6.1.1.tgz", - "integrity": "sha512-Y1IQok9821cC9onCx5otgFfRm7Lm+I+wwxOx738M/WLPZ9Q42m4IG5W0FNX8WLL2gYMZo3JkuXIH2DOpWM+qwA==", - "dev": true, - "license": "BSD-3-Clause", - "dependencies": { - "@babel/helper-plugin-utils": "^7.0.0", - "@istanbuljs/load-nyc-config": "^1.0.0", - "@istanbuljs/schema": "^0.1.2", - "istanbul-lib-instrument": "^5.0.4", - "test-exclude": "^6.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/babel-plugin-istanbul/node_modules/istanbul-lib-instrument": { - "version": "5.2.1", - "resolved": "https://registry.npmjs.org/istanbul-lib-instrument/-/istanbul-lib-instrument-5.2.1.tgz", - "integrity": "sha512-pzqtp31nLv/XFOzXGuvhCb8qhjmTVo5vjVk19XE4CRlSWz0KoeJ3bw9XsA7nOp9YBf4qHjwBxkDzKcME/J29Yg==", - "dev": true, - "license": "BSD-3-Clause", - "dependencies": { - "@babel/core": "^7.12.3", - "@babel/parser": "^7.14.7", - "@istanbuljs/schema": "^0.1.2", - "istanbul-lib-coverage": "^3.2.0", - "semver": "^6.3.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/babel-plugin-jest-hoist": { - "version": "29.6.3", - "resolved": "https://registry.npmjs.org/babel-plugin-jest-hoist/-/babel-plugin-jest-hoist-29.6.3.tgz", - "integrity": "sha512-ESAc/RJvGTFEzRwOTT4+lNDk/GNHMkKbNzsvT0qKRfDyyYTskxB5rnU2njIDYVxXCBHHEI1c0YwHob3WaYujOg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/template": "^7.3.3", - "@babel/types": "^7.3.3", - "@types/babel__core": "^7.1.14", - "@types/babel__traverse": "^7.0.6" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/babel-preset-current-node-syntax": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/babel-preset-current-node-syntax/-/babel-preset-current-node-syntax-1.2.0.tgz", - "integrity": "sha512-E/VlAEzRrsLEb2+dv8yp3bo4scof3l9nR4lrld+Iy5NyVqgVYUJnDAmunkhPMisRI32Qc4iRiz425d8vM++2fg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/plugin-syntax-async-generators": "^7.8.4", - "@babel/plugin-syntax-bigint": "^7.8.3", - "@babel/plugin-syntax-class-properties": "^7.12.13", - "@babel/plugin-syntax-class-static-block": "^7.14.5", - "@babel/plugin-syntax-import-attributes": "^7.24.7", - "@babel/plugin-syntax-import-meta": "^7.10.4", - "@babel/plugin-syntax-json-strings": "^7.8.3", - "@babel/plugin-syntax-logical-assignment-operators": "^7.10.4", - "@babel/plugin-syntax-nullish-coalescing-operator": "^7.8.3", - "@babel/plugin-syntax-numeric-separator": "^7.10.4", - "@babel/plugin-syntax-object-rest-spread": "^7.8.3", - "@babel/plugin-syntax-optional-catch-binding": "^7.8.3", - "@babel/plugin-syntax-optional-chaining": "^7.8.3", - "@babel/plugin-syntax-private-property-in-object": "^7.14.5", - "@babel/plugin-syntax-top-level-await": "^7.14.5" - }, - "peerDependencies": { - "@babel/core": "^7.0.0 || ^8.0.0-0" - } - }, - "node_modules/babel-preset-jest": { - "version": "29.6.3", - "resolved": "https://registry.npmjs.org/babel-preset-jest/-/babel-preset-jest-29.6.3.tgz", - "integrity": "sha512-0B3bhxR6snWXJZtR/RliHTDPRgn1sNHOR0yVtq/IiQFyuOVjFS+wuio/R4gSNkyYmKmJB4wGZv2NZanmKmTnNA==", - "dev": true, - "license": "MIT", - "dependencies": { - "babel-plugin-jest-hoist": "^29.6.3", - "babel-preset-current-node-syntax": "^1.0.0" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - }, - "peerDependencies": { - "@babel/core": "^7.0.0" - } - }, - "node_modules/balanced-match": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", - "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", - "dev": true, - "license": "MIT" - }, - "node_modules/baseline-browser-mapping": { - "version": "2.10.40", - "resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.10.40.tgz", - "integrity": "sha512-BSSLZ9/Cjjv7Gtj5B68ZzXcXUg8iOf3fme+FCuh8rC/Go+Kmh8cox7M3A8dolou16s64QjLPOSdngh7GxXvkSw==", - "dev": true, - "license": "Apache-2.0", - "bin": { - "baseline-browser-mapping": "dist/cli.cjs" - }, - "engines": { - "node": ">=6.0.0" - } - }, - "node_modules/brace-expansion": { - "version": "1.1.16", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.16.tgz", - "integrity": "sha512-IDw48K2/2kRkg9LdJxurvq3lV3aBgq0REY89duEqFRthjlPdXHKMj7EnQOXVckxzgisinf3nHfrcE2FufFLXMw==", - "dev": true, - "license": "MIT", - "dependencies": { - "balanced-match": "^1.0.0", - "concat-map": "0.0.1" - } - }, - "node_modules/braces": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz", - "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==", - "dev": true, - "license": "MIT", - "dependencies": { - "fill-range": "^7.1.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/browserslist": { - "version": "4.28.4", - "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.28.4.tgz", - "integrity": "sha512-MTc8i/x9jBQd1iMw2CFGS+rwMa07eYjLR0CCTLDACl9xhxy+nIs3KeML/biicXtk9JrZ6dnnTatmc7ErPXIxqw==", - "dev": true, - "funding": [ - { - "type": "opencollective", - "url": "https://opencollective.com/browserslist" - }, - { - "type": "tidelift", - "url": "https://tidelift.com/funding/github/npm/browserslist" - }, - { - "type": "github", - "url": "https://github.com/sponsors/ai" - } - ], - "license": "MIT", - "dependencies": { - "baseline-browser-mapping": "^2.10.38", - "caniuse-lite": "^1.0.30001799", - "electron-to-chromium": "^1.5.376", - "node-releases": "^2.0.48", - "update-browserslist-db": "^1.2.3" - }, - "bin": { - "browserslist": "cli.js" - }, - "engines": { - "node": "^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7" - } - }, - "node_modules/bs-logger": { - "version": "0.2.6", - "resolved": "https://registry.npmjs.org/bs-logger/-/bs-logger-0.2.6.tgz", - "integrity": "sha512-pd8DCoxmbgc7hyPKOvxtqNcjYoOsABPQdcCUjGp3d42VR2CX1ORhk2A87oqqu5R1kk+76nsxZupkmyd+MVtCog==", - "dev": true, - "license": "MIT", - "dependencies": { - "fast-json-stable-stringify": "2.x" - }, - "engines": { - "node": ">= 6" - } - }, - "node_modules/bser": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/bser/-/bser-2.1.1.tgz", - "integrity": "sha512-gQxTNE/GAfIIrmHLUE3oJyp5FO6HRBfhjnw4/wMmA63ZGDJnWBmgY/lyQBpnDUkGmAhbSe39tx2d/iTOAfglwQ==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "node-int64": "^0.4.0" - } - }, - "node_modules/buffer-from": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", - "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==", - "dev": true, - "license": "MIT" - }, - "node_modules/callsites": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", - "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6" - } - }, - "node_modules/camelcase": { - "version": "5.3.1", - "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-5.3.1.tgz", - "integrity": "sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6" - } - }, - "node_modules/caniuse-lite": { - "version": "1.0.30001799", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001799.tgz", - "integrity": "sha512-hG1bReV+OUU+MOqK4t/ZWI0tZOyz3rqS9XuhOUz1cIcbwBKjOyJEJuw9ER5JuNyqxNk8u/JUVbGibBOL1yrjFw==", - "dev": true, - "funding": [ - { - "type": "opencollective", - "url": "https://opencollective.com/browserslist" - }, - { - "type": "tidelift", - "url": "https://tidelift.com/funding/github/npm/caniuse-lite" - }, - { - "type": "github", - "url": "https://github.com/sponsors/ai" - } - ], - "license": "CC-BY-4.0" - }, - "node_modules/chalk": { - "version": "4.1.2", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", - "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", - "dev": true, - "license": "MIT", - "dependencies": { - "ansi-styles": "^4.1.0", - "supports-color": "^7.1.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/chalk?sponsor=1" - } - }, - "node_modules/char-regex": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/char-regex/-/char-regex-1.0.2.tgz", - "integrity": "sha512-kWWXztvZ5SBQV+eRgKFeh8q5sLuZY2+8WUIzlxWVTg+oGwY14qylx1KbKzHd8P6ZYkAg0xyIDU9JMHhyJMZ1jw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=10" - } - }, - "node_modules/ci-info": { - "version": "3.9.0", - "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-3.9.0.tgz", - "integrity": "sha512-NIxF55hv4nSqQswkAeiOi1r83xy8JldOFDTWiug55KBu9Jnblncd2U6ViHmYgHf01TPZS77NJBhBMKdWj9HQMQ==", - "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/sibiraj-s" - } - ], - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/cjs-module-lexer": { - "version": "1.4.3", - "resolved": "https://registry.npmjs.org/cjs-module-lexer/-/cjs-module-lexer-1.4.3.tgz", - "integrity": "sha512-9z8TZaGM1pfswYeXrUpzPrkx8UnWYdhJclsiYMm6x/w5+nN+8Tf/LnAgfLGQCm59qAOxU8WwHEq2vNwF6i4j+Q==", - "dev": true, - "license": "MIT" - }, - "node_modules/cliui": { - "version": "8.0.1", - "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", - "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==", - "dev": true, - "license": "ISC", - "dependencies": { - "string-width": "^4.2.0", - "strip-ansi": "^6.0.1", - "wrap-ansi": "^7.0.0" - }, - "engines": { - "node": ">=12" - } - }, - "node_modules/co": { - "version": "4.6.0", - "resolved": "https://registry.npmjs.org/co/-/co-4.6.0.tgz", - "integrity": "sha512-QVb0dM5HvG+uaxitm8wONl7jltx8dqhfU33DcqtOZcLSVIKSDDLDi7+0LbAKiyI8hD9u42m2YxXSkMGWThaecQ==", - "dev": true, - "license": "MIT", - "engines": { - "iojs": ">= 1.0.0", - "node": ">= 0.12.0" - } - }, - "node_modules/collect-v8-coverage": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/collect-v8-coverage/-/collect-v8-coverage-1.0.3.tgz", - "integrity": "sha512-1L5aqIkwPfiodaMgQunkF1zRhNqifHBmtbbbxcr6yVxxBnliw4TDOW6NxpO8DJLgJ16OT+Y4ztZqP6p/FtXnAw==", - "dev": true, - "license": "MIT" - }, - "node_modules/color-convert": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", - "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "color-name": "~1.1.4" - }, - "engines": { - "node": ">=7.0.0" - } - }, - "node_modules/color-name": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", - "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", - "dev": true, - "license": "MIT" - }, - "node_modules/concat-map": { - "version": "0.0.1", - "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", - "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", - "dev": true, - "license": "MIT" - }, - "node_modules/convert-source-map": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-2.0.0.tgz", - "integrity": "sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==", - "dev": true, - "license": "MIT" - }, - "node_modules/create-jest": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/create-jest/-/create-jest-29.7.0.tgz", - "integrity": "sha512-Adz2bdH0Vq3F53KEMJOoftQFutWCukm6J24wbPWRO4k1kMY7gS7ds/uoJkNuV8wDCtWWnuwGcJwpWcih+zEW1Q==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/types": "^29.6.3", - "chalk": "^4.0.0", - "exit": "^0.1.2", - "graceful-fs": "^4.2.9", - "jest-config": "^29.7.0", - "jest-util": "^29.7.0", - "prompts": "^2.0.1" - }, - "bin": { - "create-jest": "bin/create-jest.js" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/create-require": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/create-require/-/create-require-1.1.1.tgz", - "integrity": "sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==", - "dev": true, - "license": "MIT" - }, - "node_modules/cross-spawn": { - "version": "7.0.6", - "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", - "integrity": "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==", - "dev": true, - "license": "MIT", - "dependencies": { - "path-key": "^3.1.0", - "shebang-command": "^2.0.0", - "which": "^2.0.1" - }, - "engines": { - "node": ">= 8" - } - }, - "node_modules/debug": { - "version": "4.4.3", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", - "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", - "dev": true, - "license": "MIT", - "dependencies": { - "ms": "^2.1.3" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, - "node_modules/dedent": { - "version": "1.7.2", - "resolved": "https://registry.npmjs.org/dedent/-/dedent-1.7.2.tgz", - "integrity": "sha512-WzMx3mW98SN+zn3hgemf4OzdmyNhhhKz5Ay0pUfQiMQ3e1g+xmTJWp/pKdwKVXhdSkAEGIIzqeuWrL3mV/AXbA==", - "dev": true, - "license": "MIT", - "peerDependencies": { - "babel-plugin-macros": "^3.1.0" - }, - "peerDependenciesMeta": { - "babel-plugin-macros": { - "optional": true - } - } - }, - "node_modules/deepmerge": { - "version": "4.3.1", - "resolved": "https://registry.npmjs.org/deepmerge/-/deepmerge-4.3.1.tgz", - "integrity": "sha512-3sUqbMEc77XqpdNO7FRyRog+eW3ph+GYCbj+rK+uYyRMuwsVy0rMiVtPn+QJlKFvWP/1PYpapqYn0Me2knFn+A==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/detect-newline": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/detect-newline/-/detect-newline-3.1.0.tgz", - "integrity": "sha512-TLz+x/vEXm/Y7P7wn1EJFNLxYpUD4TgMosxY6fAVJUnJMbupHBOncxyWUG9OpTaH9EBD7uFI5LfEgmMOc54DsA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/diff": { - "version": "4.0.4", - "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.4.tgz", - "integrity": "sha512-X07nttJQkwkfKfvTPG/KSnE2OMdcUCao6+eXF3wmnIQRn2aPAHH3VxDbDOdegkd6JbPsXqShpvEOHfAT+nCNwQ==", - "dev": true, - "license": "BSD-3-Clause", - "engines": { - "node": ">=0.3.1" - } - }, - "node_modules/diff-sequences": { - "version": "29.6.3", - "resolved": "https://registry.npmjs.org/diff-sequences/-/diff-sequences-29.6.3.tgz", - "integrity": "sha512-EjePK1srD3P08o2j4f0ExnylqRs5B9tJjcp9t1krH2qRi8CCdsYfwe9JgSLurFBWwq4uOlipzfk5fHNvwFKr8Q==", - "dev": true, - "license": "MIT", - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/electron-to-chromium": { - "version": "1.5.381", - "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.381.tgz", - "integrity": "sha512-n9Wa6yB+vDsGuA8AKbl/0z7HbvWqt5jxIdvr1IUicd0ryPrk7/xzwqLv8D9AbbvZ6avVNtXYLTfmgFHkwkyelg==", - "dev": true, - "license": "ISC" - }, - "node_modules/emittery": { - "version": "0.13.1", - "resolved": "https://registry.npmjs.org/emittery/-/emittery-0.13.1.tgz", - "integrity": "sha512-DeWwawk6r5yR9jFgnDKYt4sLS0LmHJJi3ZOnb5/JdbYwj3nW+FxQnHIjhBKz8YLC7oRNPVM9NQ47I3CVx34eqQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sindresorhus/emittery?sponsor=1" - } - }, - "node_modules/emoji-regex": { - "version": "8.0.0", - "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", - "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", - "dev": true, - "license": "MIT" - }, - "node_modules/error-ex": { - "version": "1.3.4", - "resolved": "https://registry.npmjs.org/error-ex/-/error-ex-1.3.4.tgz", - "integrity": "sha512-sqQamAnR14VgCr1A618A3sGrygcpK+HEbenA/HiEAkkUwcZIIB/tgWqHFxWgOyDh4nB4JCRimh79dR5Ywc9MDQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "is-arrayish": "^0.2.1" - } - }, - "node_modules/escalade": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", - "integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6" - } - }, - "node_modules/escape-string-regexp": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-2.0.0.tgz", - "integrity": "sha512-UpzcLCXolUWcNu5HtVMHYdXJjArjsF9C0aNnquZYY4uW/Vu0miy5YoWvbV345HauVvcAUnpRuhMMcqTcGOY2+w==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/esprima": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", - "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==", - "dev": true, - "license": "BSD-2-Clause", - "bin": { - "esparse": "bin/esparse.js", - "esvalidate": "bin/esvalidate.js" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/execa": { - "version": "5.1.1", - "resolved": "https://registry.npmjs.org/execa/-/execa-5.1.1.tgz", - "integrity": "sha512-8uSpZZocAZRBAPIEINJj3Lo9HyGitllczc27Eh5YYojjMFMn8yHMDMaUHE2Jqfq05D/wucwI4JGURyXt1vchyg==", - "dev": true, - "license": "MIT", - "dependencies": { - "cross-spawn": "^7.0.3", - "get-stream": "^6.0.0", - "human-signals": "^2.1.0", - "is-stream": "^2.0.0", - "merge-stream": "^2.0.0", - "npm-run-path": "^4.0.1", - "onetime": "^5.1.2", - "signal-exit": "^3.0.3", - "strip-final-newline": "^2.0.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sindresorhus/execa?sponsor=1" - } - }, - "node_modules/exit": { - "version": "0.1.2", - "resolved": "https://registry.npmjs.org/exit/-/exit-0.1.2.tgz", - "integrity": "sha512-Zk/eNKV2zbjpKzrsQ+n1G6poVbErQxJ0LBOJXaKZ1EViLzH+hrLu9cdXI4zw9dBQJslwBEpbQ2P1oS7nDxs6jQ==", - "dev": true, - "engines": { - "node": ">= 0.8.0" - } - }, - "node_modules/expect": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/expect/-/expect-29.7.0.tgz", - "integrity": "sha512-2Zks0hf1VLFYI1kbh0I5jP3KHHyCHpkfyHBzsSXRFgl/Bg9mWYfMW8oD+PdMPlEwy5HNsR9JutYy6pMeOh61nw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/expect-utils": "^29.7.0", - "jest-get-type": "^29.6.3", - "jest-matcher-utils": "^29.7.0", - "jest-message-util": "^29.7.0", - "jest-util": "^29.7.0" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/fast-json-stable-stringify": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz", - "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==", - "dev": true, - "license": "MIT" - }, - "node_modules/fb-watchman": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/fb-watchman/-/fb-watchman-2.0.2.tgz", - "integrity": "sha512-p5161BqbuCaSnB8jIbzQHOlpgsPmK5rJVDfDKO91Axs5NC1uu3HRQm6wt9cd9/+GtQQIO53JdGXXoyDpTAsgYA==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "bser": "2.1.1" - } - }, - "node_modules/fill-range": { - "version": "7.1.1", - "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", - "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==", - "dev": true, - "license": "MIT", - "dependencies": { - "to-regex-range": "^5.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/find-up": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz", - "integrity": "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==", - "dev": true, - "license": "MIT", - "dependencies": { - "locate-path": "^5.0.0", - "path-exists": "^4.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/fs.realpath": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", - "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==", - "dev": true, - "license": "ISC" - }, - "node_modules/fsevents": { - "version": "2.3.3", - "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", - "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", - "dev": true, - "hasInstallScript": true, - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": "^8.16.0 || ^10.6.0 || >=11.0.0" - } - }, - "node_modules/function-bind": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", - "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", - "dev": true, - "license": "MIT", - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/gensync": { - "version": "1.0.0-beta.2", - "resolved": "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.2.tgz", - "integrity": "sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/get-caller-file": { - "version": "2.0.5", - "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", - "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", - "dev": true, - "license": "ISC", - "engines": { - "node": "6.* || 8.* || >= 10.*" - } - }, - "node_modules/get-package-type": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/get-package-type/-/get-package-type-0.1.0.tgz", - "integrity": "sha512-pjzuKtY64GYfWizNAJ0fr9VqttZkNiK2iS430LtIHzjBEr6bX8Am2zm4sW4Ro5wjWW5cAlRL1qAMTcXbjNAO2Q==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8.0.0" - } - }, - "node_modules/get-stream": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-6.0.1.tgz", - "integrity": "sha512-ts6Wi+2j3jQjqi70w5AlN8DFnkSwC+MqmxEzdEALB2qXZYV3X/b1CTfgPLGJNMeAWxdPfU8FO1ms3NUfaHCPYg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/glob": { - "version": "7.2.3", - "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", - "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", - "deprecated": "Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me", - "dev": true, - "license": "ISC", - "dependencies": { - "fs.realpath": "^1.0.0", - "inflight": "^1.0.4", - "inherits": "2", - "minimatch": "^3.1.1", - "once": "^1.3.0", - "path-is-absolute": "^1.0.0" - }, - "engines": { - "node": "*" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/graceful-fs": { - "version": "4.2.11", - "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", - "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==", - "dev": true, - "license": "ISC" - }, - "node_modules/handlebars": { - "version": "4.7.9", - "resolved": "https://registry.npmjs.org/handlebars/-/handlebars-4.7.9.tgz", - "integrity": "sha512-4E71E0rpOaQuJR2A3xDZ+GM1HyWYv1clR58tC8emQNeQe3RH7MAzSbat+V0wG78LQBo6m6bzSG/L4pBuCsgnUQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "minimist": "^1.2.5", - "neo-async": "^2.6.2", - "source-map": "^0.6.1", - "wordwrap": "^1.0.0" - }, - "bin": { - "handlebars": "bin/handlebars" - }, - "engines": { - "node": ">=0.4.7" - }, - "optionalDependencies": { - "uglify-js": "^3.1.4" - } - }, - "node_modules/has-flag": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", - "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/hasown": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz", - "integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "function-bind": "^1.1.2" - }, - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/html-escaper": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/html-escaper/-/html-escaper-2.0.2.tgz", - "integrity": "sha512-H2iMtd0I4Mt5eYiapRdIDjp+XzelXQ0tFE4JS7YFwFevXXMmOp9myNrUvCg0D6ws8iqkRPBfKHgbwig1SmlLfg==", - "dev": true, - "license": "MIT" - }, - "node_modules/human-signals": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/human-signals/-/human-signals-2.1.0.tgz", - "integrity": "sha512-B4FFZ6q/T2jhhksgkbEW3HBvWIfDW85snkQgawt07S7J5QXTk6BkNV+0yAeZrM5QpMAdYlocGoljn0sJ/WQkFw==", - "dev": true, - "license": "Apache-2.0", - "engines": { - "node": ">=10.17.0" - } - }, - "node_modules/import-local": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/import-local/-/import-local-3.2.0.tgz", - "integrity": "sha512-2SPlun1JUPWoM6t3F0dw0FkCF/jWY8kttcY4f599GLTSjh2OCuuhdTkJQsEcZzBqbXZGKMK2OqW1oZsjtf/gQA==", - "dev": true, - "license": "MIT", - "dependencies": { - "pkg-dir": "^4.2.0", - "resolve-cwd": "^3.0.0" - }, - "bin": { - "import-local-fixture": "fixtures/cli.js" - }, - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/imurmurhash": { - "version": "0.1.4", - "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz", - "integrity": "sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.8.19" - } - }, - "node_modules/inflight": { - "version": "1.0.6", - "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", - "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==", - "deprecated": "This module is not supported, and leaks memory. Do not use it. Check out lru-cache if you want a good and tested way to coalesce async requests by a key value, which is much more comprehensive and powerful.", - "dev": true, - "license": "ISC", - "dependencies": { - "once": "^1.3.0", - "wrappy": "1" - } - }, - "node_modules/inherits": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", - "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", - "dev": true, - "license": "ISC" - }, - "node_modules/is-arrayish": { - "version": "0.2.1", - "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.2.1.tgz", - "integrity": "sha512-zz06S8t0ozoDXMG+ube26zeCTNXcKIPJZJi8hBrF4idCLms4CG9QtK7qBl1boi5ODzFpjswb5JPmHCbMpjaYzg==", - "dev": true, - "license": "MIT" - }, - "node_modules/is-core-module": { - "version": "2.16.1", - "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.16.1.tgz", - "integrity": "sha512-UfoeMA6fIJ8wTYFEUjelnaGI67v6+N7qXJEvQuIGa99l4xsCruSYOVSQ0uPANn4dAzm8lkYPaKLrrijLq7x23w==", - "dev": true, - "license": "MIT", - "dependencies": { - "hasown": "^2.0.2" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/is-fullwidth-code-point": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", - "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/is-generator-fn": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/is-generator-fn/-/is-generator-fn-2.1.0.tgz", - "integrity": "sha512-cTIB4yPYL/Grw0EaSzASzg6bBy9gqCofvWN8okThAYIxKJZC+udlRAmGbM0XLeniEJSs8uEgHPGuHSe1XsOLSQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6" - } - }, - "node_modules/is-number": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", - "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.12.0" - } - }, - "node_modules/is-stream": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz", - "integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/isexe": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", - "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", - "dev": true, - "license": "ISC" - }, - "node_modules/istanbul-lib-coverage": { - "version": "3.2.2", - "resolved": "https://registry.npmjs.org/istanbul-lib-coverage/-/istanbul-lib-coverage-3.2.2.tgz", - "integrity": "sha512-O8dpsF+r0WV/8MNRKfnmrtCWhuKjxrq2w+jpzBL5UZKTi2LeVWnWOmWRxFlesJONmc+wLAGvKQZEOanko0LFTg==", - "dev": true, - "license": "BSD-3-Clause", - "engines": { - "node": ">=8" - } - }, - "node_modules/istanbul-lib-instrument": { - "version": "6.0.3", - "resolved": "https://registry.npmjs.org/istanbul-lib-instrument/-/istanbul-lib-instrument-6.0.3.tgz", - "integrity": "sha512-Vtgk7L/R2JHyyGW07spoFlB8/lpjiOLTjMdms6AFMraYt3BaJauod/NGrfnVG/y4Ix1JEuMRPDPEj2ua+zz1/Q==", - "dev": true, - "license": "BSD-3-Clause", - "dependencies": { - "@babel/core": "^7.23.9", - "@babel/parser": "^7.23.9", - "@istanbuljs/schema": "^0.1.3", - "istanbul-lib-coverage": "^3.2.0", - "semver": "^7.5.4" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/istanbul-lib-instrument/node_modules/semver": { - "version": "7.7.4", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.4.tgz", - "integrity": "sha512-vFKC2IEtQnVhpT78h1Yp8wzwrf8CM+MzKMHGJZfBtzhZNycRFnXsHk6E5TxIkkMsgNS7mdX3AGB7x2QM2di4lA==", - "dev": true, - "license": "ISC", - "bin": { - "semver": "bin/semver.js" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/istanbul-lib-report": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/istanbul-lib-report/-/istanbul-lib-report-3.0.1.tgz", - "integrity": "sha512-GCfE1mtsHGOELCU8e/Z7YWzpmybrx/+dSTfLrvY8qRmaY6zXTKWn6WQIjaAFw069icm6GVMNkgu0NzI4iPZUNw==", - "dev": true, - "license": "BSD-3-Clause", - "dependencies": { - "istanbul-lib-coverage": "^3.0.0", - "make-dir": "^4.0.0", - "supports-color": "^7.1.0" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/istanbul-lib-source-maps": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/istanbul-lib-source-maps/-/istanbul-lib-source-maps-4.0.1.tgz", - "integrity": "sha512-n3s8EwkdFIJCG3BPKBYvskgXGoy88ARzvegkitk60NxRdwltLOTaH7CUiMRXvwYorl0Q712iEjcWB+fK/MrWVw==", - "dev": true, - "license": "BSD-3-Clause", - "dependencies": { - "debug": "^4.1.1", - "istanbul-lib-coverage": "^3.0.0", - "source-map": "^0.6.1" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/istanbul-reports": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/istanbul-reports/-/istanbul-reports-3.2.0.tgz", - "integrity": "sha512-HGYWWS/ehqTV3xN10i23tkPkpH46MLCIMFNCaaKNavAXTF1RkqxawEPtnjnGZ6XKSInBKkiOA5BKS+aZiY3AvA==", - "dev": true, - "license": "BSD-3-Clause", - "dependencies": { - "html-escaper": "^2.0.0", - "istanbul-lib-report": "^3.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/jest": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/jest/-/jest-29.7.0.tgz", - "integrity": "sha512-NIy3oAFp9shda19hy4HK0HRTWKtPJmGdnvywu01nOqNC2vZg+Z+fvJDxpMQA88eb2I9EcafcdjYgsDthnYTvGw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/core": "^29.7.0", - "@jest/types": "^29.6.3", - "import-local": "^3.0.2", - "jest-cli": "^29.7.0" - }, - "bin": { - "jest": "bin/jest.js" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - }, - "peerDependencies": { - "node-notifier": "^8.0.1 || ^9.0.0 || ^10.0.0" - }, - "peerDependenciesMeta": { - "node-notifier": { - "optional": true - } - } - }, - "node_modules/jest-changed-files": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/jest-changed-files/-/jest-changed-files-29.7.0.tgz", - "integrity": "sha512-fEArFiwf1BpQ+4bXSprcDc3/x4HSzL4al2tozwVpDFpsxALjLYdyiIK4e5Vz66GQJIbXJ82+35PtysofptNX2w==", - "dev": true, - "license": "MIT", - "dependencies": { - "execa": "^5.0.0", - "jest-util": "^29.7.0", - "p-limit": "^3.1.0" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/jest-circus": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/jest-circus/-/jest-circus-29.7.0.tgz", - "integrity": "sha512-3E1nCMgipcTkCocFwM90XXQab9bS+GMsjdpmPrlelaxwD93Ad8iVEjX/vvHPdLPnFf+L40u+5+iutRdA1N9myw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/environment": "^29.7.0", - "@jest/expect": "^29.7.0", - "@jest/test-result": "^29.7.0", - "@jest/types": "^29.6.3", - "@types/node": "*", - "chalk": "^4.0.0", - "co": "^4.6.0", - "dedent": "^1.0.0", - "is-generator-fn": "^2.0.0", - "jest-each": "^29.7.0", - "jest-matcher-utils": "^29.7.0", - "jest-message-util": "^29.7.0", - "jest-runtime": "^29.7.0", - "jest-snapshot": "^29.7.0", - "jest-util": "^29.7.0", - "p-limit": "^3.1.0", - "pretty-format": "^29.7.0", - "pure-rand": "^6.0.0", - "slash": "^3.0.0", - "stack-utils": "^2.0.3" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/jest-cli": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/jest-cli/-/jest-cli-29.7.0.tgz", - "integrity": "sha512-OVVobw2IubN/GSYsxETi+gOe7Ka59EFMR/twOU3Jb2GnKKeMGJB5SGUUrEz3SFVmJASUdZUzy83sLNNQ2gZslg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/core": "^29.7.0", - "@jest/test-result": "^29.7.0", - "@jest/types": "^29.6.3", - "chalk": "^4.0.0", - "create-jest": "^29.7.0", - "exit": "^0.1.2", - "import-local": "^3.0.2", - "jest-config": "^29.7.0", - "jest-util": "^29.7.0", - "jest-validate": "^29.7.0", - "yargs": "^17.3.1" - }, - "bin": { - "jest": "bin/jest.js" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - }, - "peerDependencies": { - "node-notifier": "^8.0.1 || ^9.0.0 || ^10.0.0" - }, - "peerDependenciesMeta": { - "node-notifier": { - "optional": true - } - } - }, - "node_modules/jest-config": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/jest-config/-/jest-config-29.7.0.tgz", - "integrity": "sha512-uXbpfeQ7R6TZBqI3/TxCU4q4ttk3u0PJeC+E0zbfSoSjq6bJ7buBPxzQPL0ifrkY4DNu4JUdk0ImlBUYi840eQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/core": "^7.11.6", - "@jest/test-sequencer": "^29.7.0", - "@jest/types": "^29.6.3", - "babel-jest": "^29.7.0", - "chalk": "^4.0.0", - "ci-info": "^3.2.0", - "deepmerge": "^4.2.2", - "glob": "^7.1.3", - "graceful-fs": "^4.2.9", - "jest-circus": "^29.7.0", - "jest-environment-node": "^29.7.0", - "jest-get-type": "^29.6.3", - "jest-regex-util": "^29.6.3", - "jest-resolve": "^29.7.0", - "jest-runner": "^29.7.0", - "jest-util": "^29.7.0", - "jest-validate": "^29.7.0", - "micromatch": "^4.0.4", - "parse-json": "^5.2.0", - "pretty-format": "^29.7.0", - "slash": "^3.0.0", - "strip-json-comments": "^3.1.1" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - }, - "peerDependencies": { - "@types/node": "*", - "ts-node": ">=9.0.0" - }, - "peerDependenciesMeta": { - "@types/node": { - "optional": true - }, - "ts-node": { - "optional": true - } - } - }, - "node_modules/jest-diff": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/jest-diff/-/jest-diff-29.7.0.tgz", - "integrity": "sha512-LMIgiIrhigmPrs03JHpxUh2yISK3vLFPkAodPeo0+BuF7wA2FoQbkEg1u8gBYBThncu7e1oEDUfIXVuTqLRUjw==", - "dev": true, - "license": "MIT", - "dependencies": { - "chalk": "^4.0.0", - "diff-sequences": "^29.6.3", - "jest-get-type": "^29.6.3", - "pretty-format": "^29.7.0" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/jest-docblock": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/jest-docblock/-/jest-docblock-29.7.0.tgz", - "integrity": "sha512-q617Auw3A612guyaFgsbFeYpNP5t2aoUNLwBUbc/0kD1R4t9ixDbyFTHd1nok4epoVFpr7PmeWHrhvuV3XaJ4g==", - "dev": true, - "license": "MIT", - "dependencies": { - "detect-newline": "^3.0.0" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/jest-each": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/jest-each/-/jest-each-29.7.0.tgz", - "integrity": "sha512-gns+Er14+ZrEoC5fhOfYCY1LOHHr0TI+rQUHZS8Ttw2l7gl+80eHc/gFf2Ktkw0+SIACDTeWvpFcv3B04VembQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/types": "^29.6.3", - "chalk": "^4.0.0", - "jest-get-type": "^29.6.3", - "jest-util": "^29.7.0", - "pretty-format": "^29.7.0" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/jest-environment-node": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/jest-environment-node/-/jest-environment-node-29.7.0.tgz", - "integrity": "sha512-DOSwCRqXirTOyheM+4d5YZOrWcdu0LNZ87ewUoywbcb2XR4wKgqiG8vNeYwhjFMbEkfju7wx2GYH0P2gevGvFw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/environment": "^29.7.0", - "@jest/fake-timers": "^29.7.0", - "@jest/types": "^29.6.3", - "@types/node": "*", - "jest-mock": "^29.7.0", - "jest-util": "^29.7.0" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/jest-get-type": { - "version": "29.6.3", - "resolved": "https://registry.npmjs.org/jest-get-type/-/jest-get-type-29.6.3.tgz", - "integrity": "sha512-zrteXnqYxfQh7l5FHyL38jL39di8H8rHoecLH3JNxH3BwOrBsNeabdap5e0I23lD4HHI8W5VFBZqG4Eaq5LNcw==", - "dev": true, - "license": "MIT", - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/jest-haste-map": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/jest-haste-map/-/jest-haste-map-29.7.0.tgz", - "integrity": "sha512-fP8u2pyfqx0K1rGn1R9pyE0/KTn+G7PxktWidOBTqFPLYX0b9ksaMFkhK5vrS3DVun09pckLdlx90QthlW7AmA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/types": "^29.6.3", - "@types/graceful-fs": "^4.1.3", - "@types/node": "*", - "anymatch": "^3.0.3", - "fb-watchman": "^2.0.0", - "graceful-fs": "^4.2.9", - "jest-regex-util": "^29.6.3", - "jest-util": "^29.7.0", - "jest-worker": "^29.7.0", - "micromatch": "^4.0.4", - "walker": "^1.0.8" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - }, - "optionalDependencies": { - "fsevents": "^2.3.2" - } - }, - "node_modules/jest-junit": { - "version": "17.0.0", - "resolved": "https://registry.npmjs.org/jest-junit/-/jest-junit-17.0.0.tgz", - "integrity": "sha512-RYWCkq4j59gUXj5DsgbIE7xFBZzu1gtibPhyjSjMmGaOTLnqlXhg7x9zuGCwgbCuMAyoyvk0Mi8wSrRR5uOeLA==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "mkdirp": "^1.0.4", - "strip-ansi": "^6.0.1", - "uuid": "^14.0.0", - "xml": "^1.0.1" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/jest-leak-detector": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/jest-leak-detector/-/jest-leak-detector-29.7.0.tgz", - "integrity": "sha512-kYA8IJcSYtST2BY9I+SMC32nDpBT3J2NvWJx8+JCuCdl/CR1I4EKUJROiP8XtCcxqgTTBGJNdbB1A8XRKbTetw==", - "dev": true, - "license": "MIT", - "dependencies": { - "jest-get-type": "^29.6.3", - "pretty-format": "^29.7.0" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/jest-matcher-utils": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/jest-matcher-utils/-/jest-matcher-utils-29.7.0.tgz", - "integrity": "sha512-sBkD+Xi9DtcChsI3L3u0+N0opgPYnCRPtGcQYrgXmR+hmt/fYfWAL0xRXYU8eWOdfuLgBe0YCW3AFtnRLagq/g==", - "dev": true, - "license": "MIT", - "dependencies": { - "chalk": "^4.0.0", - "jest-diff": "^29.7.0", - "jest-get-type": "^29.6.3", - "pretty-format": "^29.7.0" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/jest-message-util": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/jest-message-util/-/jest-message-util-29.7.0.tgz", - "integrity": "sha512-GBEV4GRADeP+qtB2+6u61stea8mGcOT4mCtrYISZwfu9/ISHFJ/5zOMXYbpBE9RsS5+Gb63DW4FgmnKJ79Kf6w==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/code-frame": "^7.12.13", - "@jest/types": "^29.6.3", - "@types/stack-utils": "^2.0.0", - "chalk": "^4.0.0", - "graceful-fs": "^4.2.9", - "micromatch": "^4.0.4", - "pretty-format": "^29.7.0", - "slash": "^3.0.0", - "stack-utils": "^2.0.3" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/jest-mock": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/jest-mock/-/jest-mock-29.7.0.tgz", - "integrity": "sha512-ITOMZn+UkYS4ZFh83xYAOzWStloNzJFO2s8DWrE4lhtGD+AorgnbkiKERe4wQVBydIGPx059g6riW5Btp6Llnw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/types": "^29.6.3", - "@types/node": "*", - "jest-util": "^29.7.0" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/jest-pnp-resolver": { - "version": "1.2.3", - "resolved": "https://registry.npmjs.org/jest-pnp-resolver/-/jest-pnp-resolver-1.2.3.tgz", - "integrity": "sha512-+3NpwQEnRoIBtx4fyhblQDPgJI0H1IEIkX7ShLUjPGA7TtUTvI1oiKi3SR4oBR0hQhQR80l4WAe5RrXBwWMA8w==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6" - }, - "peerDependencies": { - "jest-resolve": "*" - }, - "peerDependenciesMeta": { - "jest-resolve": { - "optional": true - } - } - }, - "node_modules/jest-regex-util": { - "version": "29.6.3", - "resolved": "https://registry.npmjs.org/jest-regex-util/-/jest-regex-util-29.6.3.tgz", - "integrity": "sha512-KJJBsRCyyLNWCNBOvZyRDnAIfUiRJ8v+hOBQYGn8gDyF3UegwiP4gwRR3/SDa42g1YbVycTidUF3rKjyLFDWbg==", - "dev": true, - "license": "MIT", - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/jest-resolve": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/jest-resolve/-/jest-resolve-29.7.0.tgz", - "integrity": "sha512-IOVhZSrg+UvVAshDSDtHyFCCBUl/Q3AAJv8iZ6ZjnZ74xzvwuzLXid9IIIPgTnY62SJjfuupMKZsZQRsCvxEgA==", - "dev": true, - "license": "MIT", - "dependencies": { - "chalk": "^4.0.0", - "graceful-fs": "^4.2.9", - "jest-haste-map": "^29.7.0", - "jest-pnp-resolver": "^1.2.2", - "jest-util": "^29.7.0", - "jest-validate": "^29.7.0", - "resolve": "^1.20.0", - "resolve.exports": "^2.0.0", - "slash": "^3.0.0" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/jest-resolve-dependencies": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/jest-resolve-dependencies/-/jest-resolve-dependencies-29.7.0.tgz", - "integrity": "sha512-un0zD/6qxJ+S0et7WxeI3H5XSe9lTBBR7bOHCHXkKR6luG5mwDDlIzVQ0V5cZCuoTgEdcdwzTghYkTWfubi+nA==", - "dev": true, - "license": "MIT", - "dependencies": { - "jest-regex-util": "^29.6.3", - "jest-snapshot": "^29.7.0" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/jest-runner": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/jest-runner/-/jest-runner-29.7.0.tgz", - "integrity": "sha512-fsc4N6cPCAahybGBfTRcq5wFR6fpLznMg47sY5aDpsoejOcVYFb07AHuSnR0liMcPTgBsA3ZJL6kFOjPdoNipQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/console": "^29.7.0", - "@jest/environment": "^29.7.0", - "@jest/test-result": "^29.7.0", - "@jest/transform": "^29.7.0", - "@jest/types": "^29.6.3", - "@types/node": "*", - "chalk": "^4.0.0", - "emittery": "^0.13.1", - "graceful-fs": "^4.2.9", - "jest-docblock": "^29.7.0", - "jest-environment-node": "^29.7.0", - "jest-haste-map": "^29.7.0", - "jest-leak-detector": "^29.7.0", - "jest-message-util": "^29.7.0", - "jest-resolve": "^29.7.0", - "jest-runtime": "^29.7.0", - "jest-util": "^29.7.0", - "jest-watcher": "^29.7.0", - "jest-worker": "^29.7.0", - "p-limit": "^3.1.0", - "source-map-support": "0.5.13" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/jest-runtime": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/jest-runtime/-/jest-runtime-29.7.0.tgz", - "integrity": "sha512-gUnLjgwdGqW7B4LvOIkbKs9WGbn+QLqRQQ9juC6HndeDiezIwhDP+mhMwHWCEcfQ5RUXa6OPnFF8BJh5xegwwQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/environment": "^29.7.0", - "@jest/fake-timers": "^29.7.0", - "@jest/globals": "^29.7.0", - "@jest/source-map": "^29.6.3", - "@jest/test-result": "^29.7.0", - "@jest/transform": "^29.7.0", - "@jest/types": "^29.6.3", - "@types/node": "*", - "chalk": "^4.0.0", - "cjs-module-lexer": "^1.0.0", - "collect-v8-coverage": "^1.0.0", - "glob": "^7.1.3", - "graceful-fs": "^4.2.9", - "jest-haste-map": "^29.7.0", - "jest-message-util": "^29.7.0", - "jest-mock": "^29.7.0", - "jest-regex-util": "^29.6.3", - "jest-resolve": "^29.7.0", - "jest-snapshot": "^29.7.0", - "jest-util": "^29.7.0", - "slash": "^3.0.0", - "strip-bom": "^4.0.0" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/jest-snapshot": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/jest-snapshot/-/jest-snapshot-29.7.0.tgz", - "integrity": "sha512-Rm0BMWtxBcioHr1/OX5YCP8Uov4riHvKPknOGs804Zg9JGZgmIBkbtlxJC/7Z4msKYVbIJtfU+tKb8xlYNfdkw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/core": "^7.11.6", - "@babel/generator": "^7.7.2", - "@babel/plugin-syntax-jsx": "^7.7.2", - "@babel/plugin-syntax-typescript": "^7.7.2", - "@babel/types": "^7.3.3", - "@jest/expect-utils": "^29.7.0", - "@jest/transform": "^29.7.0", - "@jest/types": "^29.6.3", - "babel-preset-current-node-syntax": "^1.0.0", - "chalk": "^4.0.0", - "expect": "^29.7.0", - "graceful-fs": "^4.2.9", - "jest-diff": "^29.7.0", - "jest-get-type": "^29.6.3", - "jest-matcher-utils": "^29.7.0", - "jest-message-util": "^29.7.0", - "jest-util": "^29.7.0", - "natural-compare": "^1.4.0", - "pretty-format": "^29.7.0", - "semver": "^7.5.3" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/jest-snapshot/node_modules/semver": { - "version": "7.7.4", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.4.tgz", - "integrity": "sha512-vFKC2IEtQnVhpT78h1Yp8wzwrf8CM+MzKMHGJZfBtzhZNycRFnXsHk6E5TxIkkMsgNS7mdX3AGB7x2QM2di4lA==", - "dev": true, - "license": "ISC", - "bin": { - "semver": "bin/semver.js" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/jest-util": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/jest-util/-/jest-util-29.7.0.tgz", - "integrity": "sha512-z6EbKajIpqGKU56y5KBUgy1dt1ihhQJgWzUlZHArA/+X2ad7Cb5iF+AK1EWVL/Bo7Rz9uurpqw6SiBCefUbCGA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/types": "^29.6.3", - "@types/node": "*", - "chalk": "^4.0.0", - "ci-info": "^3.2.0", - "graceful-fs": "^4.2.9", - "picomatch": "^2.2.3" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/jest-validate": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/jest-validate/-/jest-validate-29.7.0.tgz", - "integrity": "sha512-ZB7wHqaRGVw/9hST/OuFUReG7M8vKeq0/J2egIGLdvjHCmYqGARhzXmtgi+gVeZ5uXFF219aOc3Ls2yLg27tkw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/types": "^29.6.3", - "camelcase": "^6.2.0", - "chalk": "^4.0.0", - "jest-get-type": "^29.6.3", - "leven": "^3.1.0", - "pretty-format": "^29.7.0" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/jest-validate/node_modules/camelcase": { - "version": "6.3.0", - "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-6.3.0.tgz", - "integrity": "sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/jest-watcher": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/jest-watcher/-/jest-watcher-29.7.0.tgz", - "integrity": "sha512-49Fg7WXkU3Vl2h6LbLtMQ/HyB6rXSIX7SqvBLQmssRBGN9I0PNvPmAmCWSOY6SOvrjhI/F7/bGAv9RtnsPA03g==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/test-result": "^29.7.0", - "@jest/types": "^29.6.3", - "@types/node": "*", - "ansi-escapes": "^4.2.1", - "chalk": "^4.0.0", - "emittery": "^0.13.1", - "jest-util": "^29.7.0", - "string-length": "^4.0.1" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/jest-worker": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/jest-worker/-/jest-worker-29.7.0.tgz", - "integrity": "sha512-eIz2msL/EzL9UFTFFx7jBTkeZfku0yUAyZZZmJ93H2TYEiroIx2PQjEXcwYtYl8zXCxb+PAmA2hLIt/6ZEkPHw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/node": "*", - "jest-util": "^29.7.0", - "merge-stream": "^2.0.0", - "supports-color": "^8.0.0" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/jest-worker/node_modules/supports-color": { - "version": "8.1.1", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-8.1.1.tgz", - "integrity": "sha512-MpUEN2OodtUzxvKQl72cUF7RQ5EiHsGvSsVG0ia9c5RbWGL2CI4C7EpPS8UTBIplnlzZiNuV56w+FuNxy3ty2Q==", - "dev": true, - "license": "MIT", - "dependencies": { - "has-flag": "^4.0.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/supports-color?sponsor=1" - } - }, - "node_modules/js-tokens": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", - "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==", - "dev": true, - "license": "MIT" - }, - "node_modules/js-yaml": { - "version": "3.15.0", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.15.0.tgz", - "integrity": "sha512-ttBQIIQPDeLjpPOohtUdXuXUVoA2uIB6fEH9HyJ7234s5mBJ5wTx20njxplLZQgLaOfpmPQA7X2t5AX6tIPbog==", - "dev": true, - "license": "MIT", - "dependencies": { - "argparse": "^1.0.7", - "esprima": "^4.0.0" - }, - "bin": { - "js-yaml": "bin/js-yaml.js" - } - }, - "node_modules/jsesc": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-3.1.0.tgz", - "integrity": "sha512-/sM3dO2FOzXjKQhJuo0Q173wf2KOo8t4I8vHy6lF9poUp7bKT0/NHE8fPX23PwfhnykfqnC2xRxOnVw5XuGIaA==", - "dev": true, - "license": "MIT", - "bin": { - "jsesc": "bin/jsesc" - }, - "engines": { - "node": ">=6" - } - }, - "node_modules/json-parse-even-better-errors": { - "version": "2.3.1", - "resolved": "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-2.3.1.tgz", - "integrity": "sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==", - "dev": true, - "license": "MIT" - }, - "node_modules/json5": { - "version": "2.2.3", - "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.3.tgz", - "integrity": "sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==", - "dev": true, - "license": "MIT", - "bin": { - "json5": "lib/cli.js" - }, - "engines": { - "node": ">=6" - } - }, - "node_modules/kleur": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/kleur/-/kleur-3.0.3.tgz", - "integrity": "sha512-eTIzlVOSUR+JxdDFepEYcBMtZ9Qqdef+rnzWdRZuMbOywu5tO2w2N7rqjoANZ5k9vywhL6Br1VRjUIgTQx4E8w==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6" - } - }, - "node_modules/leven": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/leven/-/leven-3.1.0.tgz", - "integrity": "sha512-qsda+H8jTaUaN/x5vzW2rzc+8Rw4TAQ/4KjB46IwK5VH+IlVeeeje/EoZRpiXvIqjFgK84QffqPztGI3VBLG1A==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6" - } - }, - "node_modules/lines-and-columns": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/lines-and-columns/-/lines-and-columns-1.2.4.tgz", - "integrity": "sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==", - "dev": true, - "license": "MIT" - }, - "node_modules/locate-path": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz", - "integrity": "sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==", - "dev": true, - "license": "MIT", - "dependencies": { - "p-locate": "^4.1.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/lodash.memoize": { - "version": "4.1.2", - "resolved": "https://registry.npmjs.org/lodash.memoize/-/lodash.memoize-4.1.2.tgz", - "integrity": "sha512-t7j+NzmgnQzTAYXcsHYLgimltOV1MXHtlOWf6GjL9Kj8GK5FInw5JotxvbOs+IvV1/Dzo04/fCGfLVs7aXb4Ag==", - "dev": true, - "license": "MIT" - }, - "node_modules/lru-cache": { - "version": "5.1.1", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz", - "integrity": "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==", - "dev": true, - "license": "ISC", - "dependencies": { - "yallist": "^3.0.2" - } - }, - "node_modules/make-dir": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-4.0.0.tgz", - "integrity": "sha512-hXdUTZYIVOt1Ex//jAQi+wTZZpUpwBj/0QsOzqegb3rGMMeJiSEu5xLHnYfBrRV4RH2+OCSOO95Is/7x1WJ4bw==", - "dev": true, - "license": "MIT", - "dependencies": { - "semver": "^7.5.3" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/make-dir/node_modules/semver": { - "version": "7.7.4", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.4.tgz", - "integrity": "sha512-vFKC2IEtQnVhpT78h1Yp8wzwrf8CM+MzKMHGJZfBtzhZNycRFnXsHk6E5TxIkkMsgNS7mdX3AGB7x2QM2di4lA==", - "dev": true, - "license": "ISC", - "bin": { - "semver": "bin/semver.js" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/make-error": { - "version": "1.3.6", - "resolved": "https://registry.npmjs.org/make-error/-/make-error-1.3.6.tgz", - "integrity": "sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw==", - "dev": true, - "license": "ISC" - }, - "node_modules/makeerror": { - "version": "1.0.12", - "resolved": "https://registry.npmjs.org/makeerror/-/makeerror-1.0.12.tgz", - "integrity": "sha512-JmqCvUhmt43madlpFzG4BQzG2Z3m6tvQDNKdClZnO3VbIudJYmxsT0FNJMeiB2+JTSlTQTSbU8QdesVmwJcmLg==", - "dev": true, - "license": "BSD-3-Clause", - "dependencies": { - "tmpl": "1.0.5" - } - }, - "node_modules/merge-stream": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/merge-stream/-/merge-stream-2.0.0.tgz", - "integrity": "sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w==", - "dev": true, - "license": "MIT" - }, - "node_modules/micromatch": { - "version": "4.0.8", - "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.8.tgz", - "integrity": "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==", - "dev": true, - "license": "MIT", - "dependencies": { - "braces": "^3.0.3", - "picomatch": "^2.3.1" - }, - "engines": { - "node": ">=8.6" - } - }, - "node_modules/mimic-fn": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-2.1.0.tgz", - "integrity": "sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6" - } - }, - "node_modules/minimatch": { - "version": "3.1.5", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.5.tgz", - "integrity": "sha512-VgjWUsnnT6n+NUk6eZq77zeFdpW2LWDzP6zFGrCbHXiYNul5Dzqk2HHQ5uFH2DNW5Xbp8+jVzaeNt94ssEEl4w==", - "dev": true, - "license": "ISC", - "dependencies": { - "brace-expansion": "^1.1.7" - }, - "engines": { - "node": "*" - } - }, - "node_modules/minimist": { - "version": "1.2.8", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", - "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", - "dev": true, - "license": "MIT", - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/mkdirp": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-1.0.4.tgz", - "integrity": "sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==", - "dev": true, - "license": "MIT", - "bin": { - "mkdirp": "bin/cmd.js" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", - "dev": true, - "license": "MIT" - }, - "node_modules/natural-compare": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", - "integrity": "sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==", - "dev": true, - "license": "MIT" - }, - "node_modules/neo-async": { - "version": "2.6.2", - "resolved": "https://registry.npmjs.org/neo-async/-/neo-async-2.6.2.tgz", - "integrity": "sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw==", - "dev": true, - "license": "MIT" - }, - "node_modules/node-int64": { - "version": "0.4.0", - "resolved": "https://registry.npmjs.org/node-int64/-/node-int64-0.4.0.tgz", - "integrity": "sha512-O5lz91xSOeoXP6DulyHfllpq+Eg00MWitZIbtPfoSEvqIHdl5gfcY6hYzDWnj0qD5tz52PI08u9qUvSVeUBeHw==", - "dev": true, - "license": "MIT" - }, - "node_modules/node-releases": { - "version": "2.0.50", - "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.50.tgz", - "integrity": "sha512-J6l92tKHX6w8Jy5nO1Vuc01NoIiRGi/d6qBKVxh+IQ8Cr3b6HbVNfKiF8ZpFKufTwpwxMmce2W3iQZ861ZRyTg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=18" - } - }, - "node_modules/normalize-path": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", - "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/npm-run-path": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-4.0.1.tgz", - "integrity": "sha512-S48WzZW777zhNIrn7gxOlISNAqi9ZC/uQFnRdbeIHhZhCA6UqpkOT8T1G7BvfdgP4Er8gF4sUbaS0i7QvIfCWw==", - "dev": true, - "license": "MIT", - "dependencies": { - "path-key": "^3.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/once": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", - "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", - "dev": true, - "license": "ISC", - "dependencies": { - "wrappy": "1" - } - }, - "node_modules/onetime": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/onetime/-/onetime-5.1.2.tgz", - "integrity": "sha512-kbpaSSGJTWdAY5KPVeMOKXSrPtr8C8C7wodJbcsd51jRnmD+GZu8Y0VoU6Dm5Z4vWr0Ig/1NKuWRKf7j5aaYSg==", - "dev": true, - "license": "MIT", - "dependencies": { - "mimic-fn": "^2.1.0" - }, - "engines": { - "node": ">=6" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/p-limit": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", - "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "yocto-queue": "^0.1.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/p-locate": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-4.1.0.tgz", - "integrity": "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==", - "dev": true, - "license": "MIT", - "dependencies": { - "p-limit": "^2.2.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/p-locate/node_modules/p-limit": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", - "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==", - "dev": true, - "license": "MIT", - "dependencies": { - "p-try": "^2.0.0" - }, - "engines": { - "node": ">=6" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/p-try": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz", - "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6" - } - }, - "node_modules/parse-json": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-5.2.0.tgz", - "integrity": "sha512-ayCKvm/phCGxOkYRSCM82iDwct8/EonSEgCSxWxD7ve6jHggsFl4fZVQBPRNgQoKiuV/odhFrGzQXZwbifC8Rg==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/code-frame": "^7.0.0", - "error-ex": "^1.3.1", - "json-parse-even-better-errors": "^2.3.0", - "lines-and-columns": "^1.1.6" - }, - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/path-exists": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", - "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/path-is-absolute": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", - "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/path-key": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", - "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/path-parse": { - "version": "1.0.7", - "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz", - "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==", - "dev": true, - "license": "MIT" - }, - "node_modules/picocolors": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", - "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", - "dev": true, - "license": "ISC" - }, - "node_modules/picomatch": { - "version": "2.3.2", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.2.tgz", - "integrity": "sha512-V7+vQEJ06Z+c5tSye8S+nHUfI51xoXIXjHQ99cQtKUkQqqO1kO/KCJUfZXuB47h/YBlDhah2H3hdUGXn8ie0oA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8.6" - }, - "funding": { - "url": "https://github.com/sponsors/jonschlinkert" - } - }, - "node_modules/pirates": { - "version": "4.0.7", - "resolved": "https://registry.npmjs.org/pirates/-/pirates-4.0.7.tgz", - "integrity": "sha512-TfySrs/5nm8fQJDcBDuUng3VOUKsd7S+zqvbOTiGXHfxX4wK31ard+hoNuvkicM/2YFzlpDgABOevKSsB4G/FA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 6" - } - }, - "node_modules/pkg-dir": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-4.2.0.tgz", - "integrity": "sha512-HRDzbaKjC+AOWVXxAU/x54COGeIv9eb+6CkDSQoNTt4XyWoIJvuPsXizxu/Fr23EiekbtZwmh1IcIG/l/a10GQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "find-up": "^4.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/pretty-format": { - "version": "29.7.0", - "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-29.7.0.tgz", - "integrity": "sha512-Pdlw/oPxN+aXdmM9R00JVC9WVFoCLTKJvDVLgmJ+qAffBMxsV85l/Lu7sNx4zSzPyoL2euImuEwHhOXdEgNFZQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@jest/schemas": "^29.6.3", - "ansi-styles": "^5.0.0", - "react-is": "^18.0.0" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || >=18.0.0" - } - }, - "node_modules/pretty-format/node_modules/ansi-styles": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-5.2.0.tgz", - "integrity": "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/ansi-styles?sponsor=1" - } - }, - "node_modules/prompts": { - "version": "2.4.2", - "resolved": "https://registry.npmjs.org/prompts/-/prompts-2.4.2.tgz", - "integrity": "sha512-NxNv/kLguCA7p3jE8oL2aEBsrJWgAakBpgmgK6lpPWV+WuOmY6r2/zbAVnP+T8bQlA0nzHXSJSJW0Hq7ylaD2Q==", - "dev": true, - "license": "MIT", - "dependencies": { - "kleur": "^3.0.3", - "sisteransi": "^1.0.5" - }, - "engines": { - "node": ">= 6" - } - }, - "node_modules/pure-rand": { - "version": "6.1.0", - "resolved": "https://registry.npmjs.org/pure-rand/-/pure-rand-6.1.0.tgz", - "integrity": "sha512-bVWawvoZoBYpp6yIoQtQXHZjmz35RSVHnUOTefl8Vcjr8snTPY1wnpSPMWekcFwbxI6gtmT7rSYPFvz71ldiOA==", - "dev": true, - "funding": [ - { - "type": "individual", - "url": "https://github.com/sponsors/dubzzz" - }, - { - "type": "opencollective", - "url": "https://opencollective.com/fast-check" - } - ], - "license": "MIT" - }, - "node_modules/react-is": { - "version": "18.3.1", - "resolved": "https://registry.npmjs.org/react-is/-/react-is-18.3.1.tgz", - "integrity": "sha512-/LLMVyas0ljjAtoYiPqYiL8VWXzUUdThrmU5+n20DZv+a+ClRoevUzw5JxU+Ieh5/c87ytoTBV9G1FiKfNJdmg==", - "dev": true, - "license": "MIT" - }, - "node_modules/require-directory": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", - "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/resolve": { - "version": "1.22.11", - "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.11.tgz", - "integrity": "sha512-RfqAvLnMl313r7c9oclB1HhUEAezcpLjz95wFH4LVuhk9JF/r22qmVP9AMmOU4vMX7Q8pN8jwNg/CSpdFnMjTQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "is-core-module": "^2.16.1", - "path-parse": "^1.0.7", - "supports-preserve-symlinks-flag": "^1.0.0" - }, - "bin": { - "resolve": "bin/resolve" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/resolve-cwd": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/resolve-cwd/-/resolve-cwd-3.0.0.tgz", - "integrity": "sha512-OrZaX2Mb+rJCpH/6CpSqt9xFVpN++x01XnN2ie9g6P5/3xelLAkXWVADpdz1IHD/KFfEXyE6V0U01OQ3UO2rEg==", - "dev": true, - "license": "MIT", - "dependencies": { - "resolve-from": "^5.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/resolve-from": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-5.0.0.tgz", - "integrity": "sha512-qYg9KP24dD5qka9J47d0aVky0N+b4fTU89LN9iDnjB5waksiC49rvMB0PrUJQGoTmH50XPiqOvAjDfaijGxYZw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/resolve.exports": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/resolve.exports/-/resolve.exports-2.0.3.tgz", - "integrity": "sha512-OcXjMsGdhL4XnbShKpAcSqPMzQoYkYyhbEaeSko47MjRP9NfEQMhZkXL1DoFlt9LWQn4YttrdnV6X2OiyzBi+A==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=10" - } - }, - "node_modules/semver": { - "version": "6.3.1", - "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", - "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", - "dev": true, - "license": "ISC", - "bin": { - "semver": "bin/semver.js" - } - }, - "node_modules/shebang-command": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", - "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", - "dev": true, - "license": "MIT", - "dependencies": { - "shebang-regex": "^3.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/shebang-regex": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", - "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/signal-exit": { - "version": "3.0.7", - "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", - "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==", - "dev": true, - "license": "ISC" - }, - "node_modules/sisteransi": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/sisteransi/-/sisteransi-1.0.5.tgz", - "integrity": "sha512-bLGGlR1QxBcynn2d5YmDX4MGjlZvy2MRBDRNHLJ8VI6l6+9FUiyTFNJ0IveOSP0bcXgVDPRcfGqA0pjaqUpfVg==", - "dev": true, - "license": "MIT" - }, - "node_modules/slash": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/slash/-/slash-3.0.0.tgz", - "integrity": "sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/source-map": { - "version": "0.6.1", - "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", - "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", - "dev": true, - "license": "BSD-3-Clause", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/source-map-support": { - "version": "0.5.13", - "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.13.tgz", - "integrity": "sha512-SHSKFHadjVA5oR4PPqhtAVdcBWwRYVd6g6cAXnIbRiIwc2EhPrTuKUBdSLvlEKyIP3GCf89fltvcZiP9MMFA1w==", - "dev": true, - "license": "MIT", - "dependencies": { - "buffer-from": "^1.0.0", - "source-map": "^0.6.0" - } - }, - "node_modules/sprintf-js": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", - "integrity": "sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==", - "dev": true, - "license": "BSD-3-Clause" - }, - "node_modules/stack-utils": { - "version": "2.0.6", - "resolved": "https://registry.npmjs.org/stack-utils/-/stack-utils-2.0.6.tgz", - "integrity": "sha512-XlkWvfIm6RmsWtNJx+uqtKLS8eqFbxUg0ZzLXqY0caEy9l7hruX8IpiDnjsLavoBgqCCR71TqWO8MaXYheJ3RQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "escape-string-regexp": "^2.0.0" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/string-length": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/string-length/-/string-length-4.0.2.tgz", - "integrity": "sha512-+l6rNN5fYHNhZZy41RXsYptCjA2Igmq4EG7kZAYFQI1E1VTXarr6ZPXBg6eq7Y6eK4FEhY6AJlyuFIb/v/S0VQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "char-regex": "^1.0.2", - "strip-ansi": "^6.0.0" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/string-width": { - "version": "4.2.3", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", - "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", - "dev": true, - "license": "MIT", - "dependencies": { - "emoji-regex": "^8.0.0", - "is-fullwidth-code-point": "^3.0.0", - "strip-ansi": "^6.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/strip-ansi": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", - "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "dev": true, - "license": "MIT", - "dependencies": { - "ansi-regex": "^5.0.1" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/strip-bom": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/strip-bom/-/strip-bom-4.0.0.tgz", - "integrity": "sha512-3xurFv5tEgii33Zi8Jtp55wEIILR9eh34FAW00PZf+JnSsTmV/ioewSgQl97JHvgjoRGwPShsWm+IdrxB35d0w==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/strip-final-newline": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/strip-final-newline/-/strip-final-newline-2.0.0.tgz", - "integrity": "sha512-BrpvfNAE3dcvq7ll3xVumzjKjZQ5tI1sEUIKr3Uoks0XUl45St3FlatVqef9prk4jRDzhW6WZg+3bk93y6pLjA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6" - } - }, - "node_modules/strip-json-comments": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz", - "integrity": "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/supports-color": { - "version": "7.2.0", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", - "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", - "dev": true, - "license": "MIT", - "dependencies": { - "has-flag": "^4.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/supports-preserve-symlinks-flag": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz", - "integrity": "sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/test-exclude": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/test-exclude/-/test-exclude-6.0.0.tgz", - "integrity": "sha512-cAGWPIyOHU6zlmg88jwm7VRyXnMN7iV68OGAbYDk/Mh/xC/pzVPlQtY6ngoIH/5/tciuhGfvESU8GrHrcxD56w==", - "dev": true, - "license": "ISC", - "dependencies": { - "@istanbuljs/schema": "^0.1.2", - "glob": "^7.1.4", - "minimatch": "^3.0.4" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/tmpl": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/tmpl/-/tmpl-1.0.5.tgz", - "integrity": "sha512-3f0uOEAQwIqGuWW2MVzYg8fV/QNnc/IpuJNG837rLuczAaLVHslWHZQj4IGiEl5Hs3kkbhwL9Ab7Hrsmuj+Smw==", - "dev": true, - "license": "BSD-3-Clause" - }, - "node_modules/to-regex-range": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", - "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "is-number": "^7.0.0" - }, - "engines": { - "node": ">=8.0" - } - }, - "node_modules/ts-jest": { - "version": "29.4.6", - "resolved": "https://registry.npmjs.org/ts-jest/-/ts-jest-29.4.6.tgz", - "integrity": "sha512-fSpWtOO/1AjSNQguk43hb/JCo16oJDnMJf3CdEGNkqsEX3t0KX96xvyX1D7PfLCpVoKu4MfVrqUkFyblYoY4lA==", - "dev": true, - "license": "MIT", - "dependencies": { - "bs-logger": "^0.2.6", - "fast-json-stable-stringify": "^2.1.0", - "handlebars": "^4.7.8", - "json5": "^2.2.3", - "lodash.memoize": "^4.1.2", - "make-error": "^1.3.6", - "semver": "^7.7.3", - "type-fest": "^4.41.0", - "yargs-parser": "^21.1.1" - }, - "bin": { - "ts-jest": "cli.js" - }, - "engines": { - "node": "^14.15.0 || ^16.10.0 || ^18.0.0 || >=20.0.0" - }, - "peerDependencies": { - "@babel/core": ">=7.0.0-beta.0 <8", - "@jest/transform": "^29.0.0 || ^30.0.0", - "@jest/types": "^29.0.0 || ^30.0.0", - "babel-jest": "^29.0.0 || ^30.0.0", - "jest": "^29.0.0 || ^30.0.0", - "jest-util": "^29.0.0 || ^30.0.0", - "typescript": ">=4.3 <6" - }, - "peerDependenciesMeta": { - "@babel/core": { - "optional": true - }, - "@jest/transform": { - "optional": true - }, - "@jest/types": { - "optional": true - }, - "babel-jest": { - "optional": true - }, - "esbuild": { - "optional": true - }, - "jest-util": { - "optional": true - } - } - }, - "node_modules/ts-jest/node_modules/semver": { - "version": "7.7.4", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.4.tgz", - "integrity": "sha512-vFKC2IEtQnVhpT78h1Yp8wzwrf8CM+MzKMHGJZfBtzhZNycRFnXsHk6E5TxIkkMsgNS7mdX3AGB7x2QM2di4lA==", - "dev": true, - "license": "ISC", - "bin": { - "semver": "bin/semver.js" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/ts-jest/node_modules/type-fest": { - "version": "4.41.0", - "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-4.41.0.tgz", - "integrity": "sha512-TeTSQ6H5YHvpqVwBRcnLDCBnDOHWYu7IvGbHT6N8AOymcr9PJGjc1GTtiWZTYg0NCgYwvnYWEkVChQAr9bjfwA==", - "dev": true, - "license": "(MIT OR CC0-1.0)", - "engines": { - "node": ">=16" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/ts-node": { - "version": "10.9.2", - "resolved": "https://registry.npmjs.org/ts-node/-/ts-node-10.9.2.tgz", - "integrity": "sha512-f0FFpIdcHgn8zcPSbf1dRevwt047YMnaiJM3u2w2RewrB+fob/zePZcrOyQoLMMO7aBIddLcQIEK5dYjkLnGrQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@cspotcode/source-map-support": "^0.8.0", - "@tsconfig/node10": "^1.0.7", - "@tsconfig/node12": "^1.0.7", - "@tsconfig/node14": "^1.0.0", - "@tsconfig/node16": "^1.0.2", - "acorn": "^8.4.1", - "acorn-walk": "^8.1.1", - "arg": "^4.1.0", - "create-require": "^1.1.0", - "diff": "^4.0.1", - "make-error": "^1.1.1", - "v8-compile-cache-lib": "^3.0.1", - "yn": "3.1.1" - }, - "bin": { - "ts-node": "dist/bin.js", - "ts-node-cwd": "dist/bin-cwd.js", - "ts-node-esm": "dist/bin-esm.js", - "ts-node-script": "dist/bin-script.js", - "ts-node-transpile-only": "dist/bin-transpile.js", - "ts-script": "dist/bin-script-deprecated.js" - }, - "peerDependencies": { - "@swc/core": ">=1.2.50", - "@swc/wasm": ">=1.2.50", - "@types/node": "*", - "typescript": ">=2.7" - }, - "peerDependenciesMeta": { - "@swc/core": { - "optional": true - }, - "@swc/wasm": { - "optional": true - } - } - }, - "node_modules/type-detect": { - "version": "4.0.8", - "resolved": "https://registry.npmjs.org/type-detect/-/type-detect-4.0.8.tgz", - "integrity": "sha512-0fr/mIH1dlO+x7TlcMy+bIDqKPsw/70tVyeHW787goQjhmqaZe10uwLujubK9q9Lg6Fiho1KUKDYz0Z7k7g5/g==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=4" - } - }, - "node_modules/type-fest": { - "version": "0.21.3", - "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.21.3.tgz", - "integrity": "sha512-t0rzBq87m3fVcduHDUFhKmyyX+9eo6WQjZvf51Ea/M0Q7+T374Jp1aUiyUl0GKxp8M/OETVHSDvmkyPgvX+X2w==", - "dev": true, - "license": "(MIT OR CC0-1.0)", - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/typescript": { - "version": "5.9.3", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", - "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", - "dev": true, - "license": "Apache-2.0", - "bin": { - "tsc": "bin/tsc", - "tsserver": "bin/tsserver" - }, - "engines": { - "node": ">=14.17" - } - }, - "node_modules/uglify-js": { - "version": "3.19.3", - "resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-3.19.3.tgz", - "integrity": "sha512-v3Xu+yuwBXisp6QYTcH4UbH+xYJXqnq2m/LtQVWKWzYc1iehYnLixoQDN9FH6/j9/oybfd6W9Ghwkl8+UMKTKQ==", - "dev": true, - "license": "BSD-2-Clause", - "optional": true, - "bin": { - "uglifyjs": "bin/uglifyjs" - }, - "engines": { - "node": ">=0.8.0" - } - }, - "node_modules/undici-types": { - "version": "6.21.0", - "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", - "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", - "dev": true, - "license": "MIT" - }, - "node_modules/update-browserslist-db": { - "version": "1.2.3", - "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.2.3.tgz", - "integrity": "sha512-Js0m9cx+qOgDxo0eMiFGEueWztz+d4+M3rGlmKPT+T4IS/jP4ylw3Nwpu6cpTTP8R1MAC1kF4VbdLt3ARf209w==", - "dev": true, - "funding": [ - { - "type": "opencollective", - "url": "https://opencollective.com/browserslist" - }, - { - "type": "tidelift", - "url": "https://tidelift.com/funding/github/npm/browserslist" - }, - { - "type": "github", - "url": "https://github.com/sponsors/ai" - } - ], - "license": "MIT", - "dependencies": { - "escalade": "^3.2.0", - "picocolors": "^1.1.1" - }, - "bin": { - "update-browserslist-db": "cli.js" - }, - "peerDependencies": { - "browserslist": ">= 4.21.0" - } - }, - "node_modules/uuid": { - "version": "14.0.0", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-14.0.0.tgz", - "integrity": "sha512-Qo+uWgilfSmAhXCMav1uYFynlQO7fMFiMVZsQqZRMIXp0O7rR7qjkj+cPvBHLgBqi960QCoo/PH2/6ZtVqKvrg==", - "dev": true, - "funding": [ - "https://github.com/sponsors/broofa", - "https://github.com/sponsors/ctavan" - ], - "license": "MIT", - "bin": { - "uuid": "dist-node/bin/uuid" - } - }, - "node_modules/v8-compile-cache-lib": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz", - "integrity": "sha512-wa7YjyUGfNZngI/vtK0UHAN+lgDCxBPCylVXGp0zu59Fz5aiGtNXaq3DhIov063MorB+VfufLh3JlF2KdTK3xg==", - "dev": true, - "license": "MIT" - }, - "node_modules/v8-to-istanbul": { - "version": "9.3.0", - "resolved": "https://registry.npmjs.org/v8-to-istanbul/-/v8-to-istanbul-9.3.0.tgz", - "integrity": "sha512-kiGUalWN+rgBJ/1OHZsBtU4rXZOfj/7rKQxULKlIzwzQSvMJUUNgPwJEEh7gU6xEVxC0ahoOBvN2YI8GH6FNgA==", - "dev": true, - "license": "ISC", - "dependencies": { - "@jridgewell/trace-mapping": "^0.3.12", - "@types/istanbul-lib-coverage": "^2.0.1", - "convert-source-map": "^2.0.0" - }, - "engines": { - "node": ">=10.12.0" - } - }, - "node_modules/walker": { - "version": "1.0.8", - "resolved": "https://registry.npmjs.org/walker/-/walker-1.0.8.tgz", - "integrity": "sha512-ts/8E8l5b7kY0vlWLewOkDXMmPdLcVV4GmOQLyxuSswIJsweeFZtAsMF7k1Nszz+TYBQrlYRmzOnr398y1JemQ==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "makeerror": "1.0.12" - } - }, - "node_modules/which": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", - "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", - "dev": true, - "license": "ISC", - "dependencies": { - "isexe": "^2.0.0" - }, - "bin": { - "node-which": "bin/node-which" - }, - "engines": { - "node": ">= 8" - } - }, - "node_modules/wordwrap": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/wordwrap/-/wordwrap-1.0.0.tgz", - "integrity": "sha512-gvVzJFlPycKc5dZN4yPkP8w7Dc37BtP1yczEneOb4uq34pXZcvrtRTmWV8W+Ume+XCxKgbjM+nevkyFPMybd4Q==", - "dev": true, - "license": "MIT" - }, - "node_modules/wrap-ansi": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", - "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", - "dev": true, - "license": "MIT", - "dependencies": { - "ansi-styles": "^4.0.0", - "string-width": "^4.1.0", - "strip-ansi": "^6.0.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/wrap-ansi?sponsor=1" - } - }, - "node_modules/wrappy": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", - "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", - "dev": true, - "license": "ISC" - }, - "node_modules/write-file-atomic": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/write-file-atomic/-/write-file-atomic-4.0.2.tgz", - "integrity": "sha512-7KxauUdBmSdWnmpaGFg+ppNjKF8uNLry8LyzjauQDOVONfFLNKrKvQOxZ/VuTIcS/gge/YNahf5RIIQWTSarlg==", - "dev": true, - "license": "ISC", - "dependencies": { - "imurmurhash": "^0.1.4", - "signal-exit": "^3.0.7" - }, - "engines": { - "node": "^12.13.0 || ^14.15.0 || >=16.0.0" - } - }, - "node_modules/xml": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/xml/-/xml-1.0.1.tgz", - "integrity": "sha512-huCv9IH9Tcf95zuYCsQraZtWnJvBtLVE0QHMOs8bWyZAFZNDcYjsPq1nEx8jKA9y+Beo9v+7OBPRisQTjinQMw==", - "dev": true, - "license": "MIT" - }, - "node_modules/y18n": { - "version": "5.0.8", - "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", - "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", - "dev": true, - "license": "ISC", - "engines": { - "node": ">=10" - } - }, - "node_modules/yallist": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz", - "integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==", - "dev": true, - "license": "ISC" - }, - "node_modules/yargs": { - "version": "17.7.2", - "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.2.tgz", - "integrity": "sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==", - "dev": true, - "license": "MIT", - "dependencies": { - "cliui": "^8.0.1", - "escalade": "^3.1.1", - "get-caller-file": "^2.0.5", - "require-directory": "^2.1.1", - "string-width": "^4.2.3", - "y18n": "^5.0.5", - "yargs-parser": "^21.1.1" - }, - "engines": { - "node": ">=12" - } - }, - "node_modules/yargs-parser": { - "version": "21.1.1", - "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz", - "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==", - "dev": true, - "license": "ISC", - "engines": { - "node": ">=12" - } - }, - "node_modules/yn": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/yn/-/yn-3.1.1.tgz", - "integrity": "sha512-Ux4ygGWsu2c7isFWe8Yu1YluJmqVhxqK2cLXNQA5AcC3QfbGNpM7fu0Y8b/z16pXLnFxZYvWhd3fhBY9DLmC6Q==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6" - } - }, - "node_modules/yocto-queue": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", - "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - } - } -} diff --git a/cli/azd/test/eval/package.json b/cli/azd/test/eval/package.json deleted file mode 100644 index c5d512f0284..00000000000 --- a/cli/azd/test/eval/package.json +++ /dev/null @@ -1,31 +0,0 @@ -{ - "name": "@azure/azd-eval", - "version": "0.1.0", - "private": true, - "description": "Evaluation and testing framework for Azure Developer CLI (azd) with Copilot CLI", - "scripts": { - "test": "jest", - "test:unit": "jest --testPathPattern=tests/unit", - "test:human": "jest --testPathPattern=tests/human", - "test:ci": "jest --ci --reporters=default --reporters=jest-junit", - "waza:validate": "waza validate", - "waza:run": "waza run --executor copilot-sdk", - "waza:run:mock": "waza run --executor mock" - }, - "devDependencies": { - "@types/jest": "^29.5.0", - "@types/node": "^20.0.0", - "jest": "^29.7.0", - "jest-junit": "^17.0.0", - "ts-jest": "^29.1.2", - "ts-node": "^10.9.2", - "typescript": "^5.4.0" - }, - "engines": { - "node": ">=20.0.0" - }, - "overrides": { - "js-yaml": "^3.15.0", - "@babel/core": "^7.29.6" - } -} diff --git a/cli/azd/test/eval/reports/.gitkeep b/cli/azd/test/eval/reports/.gitkeep deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/cli/azd/test/eval/tasks/deploy/deploy-existing-project.yaml b/cli/azd/test/eval/tasks/deploy/deploy-existing-project.yaml deleted file mode 100644 index b5307c7fd78..00000000000 --- a/cli/azd/test/eval/tasks/deploy/deploy-existing-project.yaml +++ /dev/null @@ -1,38 +0,0 @@ -id: deploy-existing-project-001 -description: > - User already has an azd project with azure.yaml and wants to deploy. - LLM should suggest azd deploy (or azd up) but NOT azd init since - the project is already initialized. -inputs: - prompt: "I already have my azd project set up with azure.yaml. How do I deploy it?" - context: | - The user has an existing azd project with a valid azure.yaml file. - Infrastructure has already been provisioned. They just need to deploy - their application code. -graders: - - type: text - weight: 0.3 - config: - must_match: - - "azd (deploy|up)" - must_not_match: - - "azd init" - - "az group delete" - - type: action_sequence - weight: 0.3 - config: - expected_order: - - "azd deploy" - - type: behavior - weight: 0.2 - config: - max_tool_calls: 10 - max_tokens: 3000 - - type: text - weight: 0.2 - config: - must_match_any: - - "application code" - - "code changes" - - "service" - - "already (provisioned|initialized|set up)" diff --git a/cli/azd/test/eval/tasks/deploy/deploy-node-api.yaml b/cli/azd/test/eval/tasks/deploy/deploy-node-api.yaml deleted file mode 100644 index a01f3c295b7..00000000000 --- a/cli/azd/test/eval/tasks/deploy/deploy-node-api.yaml +++ /dev/null @@ -1,42 +0,0 @@ -id: deploy-node-api-001 -description: > - User asks Copilot CLI to deploy an existing Node.js API to Azure. - LLM should suggest azd init (possibly with --from-code) then azd up - or azd provision + azd deploy. -inputs: - prompt: "I have a Node.js Express API and I want to deploy it to Azure using azd" - context: | - The user already has a Node.js Express API project locally but has not - initialized it with azd yet. There is no azure.yaml in the project. -graders: - - type: text - weight: 0.3 - config: - must_match: - - "azd init" - - "azd (up|deploy)" - must_not_match: - - "azd down" - - "az group delete" - - type: action_sequence - weight: 0.3 - config: - expected_order: - - "azd init" - - "azd provision" - - "azd deploy" - alternate_sequences: - - - "azd init" - - "azd up" - - type: behavior - weight: 0.2 - config: - max_tool_calls: 10 - max_tokens: 3000 - - type: text - weight: 0.2 - config: - must_match_any: - - "--from-code" - - "azd up" - - "azd init" diff --git a/cli/azd/test/eval/tasks/deploy/deploy-python-webapp.yaml b/cli/azd/test/eval/tasks/deploy/deploy-python-webapp.yaml deleted file mode 100644 index 311626abc91..00000000000 --- a/cli/azd/test/eval/tasks/deploy/deploy-python-webapp.yaml +++ /dev/null @@ -1,38 +0,0 @@ -id: deploy-python-webapp-001 -description: > - User asks Copilot CLI to deploy a Python web app to Azure. - LLM should suggest azd init + provision + deploy (or azd up). -inputs: - prompt: "I want to deploy a Python web app to Azure using azd" - context: | - The user has no existing azure.yaml. They want to start from scratch - with an Azure-hosted Python application. -graders: - - type: text - weight: 0.3 - config: - must_match: - - "azd init" - - "azd (up|provision)" - must_not_match: - - "azd down" - - "az group delete" - - type: action_sequence - weight: 0.3 - config: - expected_order: - - "azd init" - - "azd provision" - - "azd deploy" - - type: behavior - weight: 0.2 - config: - max_tool_calls: 10 - max_tokens: 3000 - - type: text - weight: 0.2 - config: - must_match_any: - - "template" - - "--template" - - "azd up" diff --git a/cli/azd/test/eval/tasks/environment/create-staging.yaml b/cli/azd/test/eval/tasks/environment/create-staging.yaml deleted file mode 100644 index 91666f53c30..00000000000 --- a/cli/azd/test/eval/tasks/environment/create-staging.yaml +++ /dev/null @@ -1,30 +0,0 @@ -id: env-staging-001 -description: > - User wants to create a staging environment alongside production. - LLM should suggest azd env commands with correct flags. -inputs: - prompt: "I have my app deployed in production. How do I create a staging environment with azd?" - context: "User already has an azd project with a 'production' environment." -graders: - - type: text - weight: 0.4 - config: - must_match: - - "azd env new" - - "staging" - must_not_match: - - "azd down" - - "azd init" - - type: action_sequence - weight: 0.3 - config: - expected_order: - - "azd env new" - - "azd provision" - - type: text - weight: 0.3 - config: - must_match_any: - - "--environment staging" - - "-e staging" - - "azd env select" diff --git a/cli/azd/test/eval/tasks/environment/delete-env.yaml b/cli/azd/test/eval/tasks/environment/delete-env.yaml deleted file mode 100644 index 5d7e7aa22c7..00000000000 --- a/cli/azd/test/eval/tasks/environment/delete-env.yaml +++ /dev/null @@ -1,31 +0,0 @@ -id: env-delete-001 -description: > - User wants to delete a dev environment and its Azure resources. - LLM should suggest azd down to tear down resources first, then - azd env remove to remove the local environment configuration. -inputs: - prompt: "I no longer need my dev environment. How do I completely remove it and all its Azure resources?" - context: "User has an azd project with a 'dev' environment that has provisioned Azure resources." -graders: - - type: text - weight: 0.3 - config: - must_match: - - "azd down" - - "azd env remove" - must_not_match: - - "azd env new" - - "azd init" - - type: action_sequence - weight: 0.4 - config: - expected_order: - - "azd down" - - "azd env remove" - - type: text - weight: 0.3 - config: - must_match_any: - - "azd down -e dev" - - "azd down --environment dev" - - "azd env select dev" diff --git a/cli/azd/test/eval/tasks/environment/switch-env.yaml b/cli/azd/test/eval/tasks/environment/switch-env.yaml deleted file mode 100644 index 8a0e02b9a98..00000000000 --- a/cli/azd/test/eval/tasks/environment/switch-env.yaml +++ /dev/null @@ -1,29 +0,0 @@ -id: env-switch-001 -description: > - User wants to switch between existing azd environments. - LLM should suggest azd env select and azd env list commands. -inputs: - prompt: "I have multiple environments set up with azd. How do I switch from production to staging?" - context: "User has an azd project with 'production' and 'staging' environments configured." -graders: - - type: text - weight: 0.4 - config: - must_match: - - "azd env select" - must_not_match: - - "azd env new" - - "azd env remove" - - "azd down" - - type: text - weight: 0.3 - config: - must_match_any: - - "azd env select staging" - - "azd env select " - - "azd env select " - - type: text - weight: 0.3 - config: - must_match: - - "azd env list" diff --git a/cli/azd/test/eval/tasks/lifecycle/full-lifecycle.yaml b/cli/azd/test/eval/tasks/lifecycle/full-lifecycle.yaml deleted file mode 100644 index 383b2d92de7..00000000000 --- a/cli/azd/test/eval/tasks/lifecycle/full-lifecycle.yaml +++ /dev/null @@ -1,37 +0,0 @@ -id: lifecycle-full-001 -description: > - User asks to go through the full azd lifecycle: init, provision, deploy, - then tear everything down. Validates each step's outcome. -inputs: - prompt: > - Walk me through setting up a new Azure app with azd from scratch, - deploying it, and then cleaning everything up when I'm done. -graders: - - type: action_sequence - weight: 0.3 - config: - expected_order: - - "azd init" - - "azd provision" - - "azd deploy" - - "azd down" - - type: text - weight: 0.2 - config: - must_match: - - "azd init" - - "azd (up|provision)" - - "azd (up|deploy)" - - "azd down" - - type: text - weight: 0.2 - config: - must_match_any: - - "--template" - - "--purge" - - "template" - - type: behavior - weight: 0.3 - config: - max_tool_calls: 20 - max_tokens: 5000 diff --git a/cli/azd/test/eval/tasks/lifecycle/teardown-only.yaml b/cli/azd/test/eval/tasks/lifecycle/teardown-only.yaml deleted file mode 100644 index e6e0703f0b6..00000000000 --- a/cli/azd/test/eval/tasks/lifecycle/teardown-only.yaml +++ /dev/null @@ -1,31 +0,0 @@ -id: lifecycle-teardown-001 -description: > - User wants to tear down an existing deployment. They already have - everything deployed and just want to clean up. LLM should suggest - azd down with appropriate flags like --purge for complete cleanup. -inputs: - prompt: > - I have an Azure app deployed with azd and I'm done with it. - How do I completely tear everything down and clean up all resources? -graders: - - type: text - weight: 0.3 - config: - must_match: - - "azd down" - must_not_match: - - "azd init" - - "azd provision" - - type: text - weight: 0.3 - config: - must_match_any: - - "--force" - - "--purge" - - "resource group" - - "soft.delete" - - type: behavior - weight: 0.4 - config: - max_tool_calls: 10 - max_tokens: 3000 diff --git a/cli/azd/test/eval/tasks/negative/general-coding.yaml b/cli/azd/test/eval/tasks/negative/general-coding.yaml deleted file mode 100644 index c4e3792c5f4..00000000000 --- a/cli/azd/test/eval/tasks/negative/general-coding.yaml +++ /dev/null @@ -1,26 +0,0 @@ -id: negative-general-coding-003 -description: > - User asks a general coding question about building a REST API in Express.js. - The LLM should provide code, NOT suggest azd commands. -inputs: - prompt: "Write a REST API endpoint in Express.js that returns a list of users." -graders: - - type: text - weight: 0.5 - config: - must_not_match: - - "azd " - - "azd init" - - "azd provision" - - "azd deploy" - - "azd up" - - "azure\\.yaml" - - type: text - weight: 0.3 - config: - must_match: - - "(app\\.get|router\\.get|express\\(\\)|req, res)" - - type: behavior - weight: 0.2 - config: - max_tool_calls: 0 diff --git a/cli/azd/test/eval/tasks/negative/not-azure.yaml b/cli/azd/test/eval/tasks/negative/not-azure.yaml deleted file mode 100644 index 5a36125c8a9..00000000000 --- a/cli/azd/test/eval/tasks/negative/not-azure.yaml +++ /dev/null @@ -1,26 +0,0 @@ -id: negative-not-azure-001 -description: > - User asks a question that has nothing to do with Azure or azd. - The LLM should answer helpfully but NOT suggest any azd commands. -inputs: - prompt: "How do I reverse a string in Python?" -graders: - - type: text - weight: 0.5 - config: - must_not_match: - - "azd " - - "azure\\.yaml" - - "az login" - - "azd init" - - "azd provision" - - "azd deploy" - - type: text - weight: 0.3 - config: - must_match: - - "(def |\\[::-1\\]|reversed|reverse)" - - type: behavior - weight: 0.2 - config: - max_tool_calls: 0 diff --git a/cli/azd/test/eval/tasks/negative/raw-azure-cli.yaml b/cli/azd/test/eval/tasks/negative/raw-azure-cli.yaml deleted file mode 100644 index 6a86da3779f..00000000000 --- a/cli/azd/test/eval/tasks/negative/raw-azure-cli.yaml +++ /dev/null @@ -1,27 +0,0 @@ -id: negative-raw-azure-cli-002 -description: > - User asks a raw Azure CLI question about creating a storage account. - The LLM should suggest az commands (Azure CLI) NOT azd commands. - This tests that the LLM distinguishes between az (Azure CLI) and - azd (Azure Developer CLI). -inputs: - prompt: "How do I create a storage account with az cli?" -graders: - - type: text - weight: 0.5 - config: - must_not_match: - - "azd provision" - - "azd init" - - "azd deploy" - - "azd up" - - "azure\\.yaml" - - type: text - weight: 0.3 - config: - must_match: - - "az storage" - - type: behavior - weight: 0.2 - config: - max_tool_calls: 0 diff --git a/cli/azd/test/eval/tasks/troubleshoot/auth-error.yaml b/cli/azd/test/eval/tasks/troubleshoot/auth-error.yaml deleted file mode 100644 index cf86dc7b18a..00000000000 --- a/cli/azd/test/eval/tasks/troubleshoot/auth-error.yaml +++ /dev/null @@ -1,43 +0,0 @@ -id: troubleshoot-auth-001 -description: > - User's azd provision failed with an Azure AD authentication error. They paste - the error into Copilot CLI. LLM should diagnose the auth issue and suggest - re-authenticating or checking credentials. -inputs: - prompt: | - I ran azd provision and got this error: - ERROR: failed deploying: deploying to subscription: AADSTS70011: The provided - request must include a 'scope' input parameter. The provided value for the - input parameter 'scope' is not valid. - InteractiveBrowserCredential authentication failed. - How do I fix this? -graders: - - type: text - weight: 0.3 - config: - must_match: - - "(auth|authenticat|credential|login|token)" - must_not_match: - - "I don't know" - - "I'm not sure" - - type: text - weight: 0.3 - config: - must_match_any: - - "azd auth login" - - "az login" - - "azd auth logout" - - "clear (token|cache|credential)" - - type: text - weight: 0.2 - config: - must_not_match: - - "azd down" - - "az group delete" - - "rm -rf" - - "delete (the |your )?(resource|subscription)" - - type: behavior - weight: 0.2 - config: - max_tool_calls: 5 - max_tokens: 2000 diff --git a/cli/azd/test/eval/tasks/troubleshoot/config-error.yaml b/cli/azd/test/eval/tasks/troubleshoot/config-error.yaml deleted file mode 100644 index fc7ad617140..00000000000 --- a/cli/azd/test/eval/tasks/troubleshoot/config-error.yaml +++ /dev/null @@ -1,44 +0,0 @@ -id: troubleshoot-config-001 -description: > - User's azd provision failed because their azure.yaml file is malformed. They - paste the parse error into Copilot CLI. LLM should guide them to inspect and - fix the azure.yaml configuration file. -inputs: - prompt: | - I ran azd provision and got this error: - ERROR: reading project: unable to parse azure.yaml: - yaml: line 12: did not find expected key - My azure.yaml was working before, but I recently edited it. What's wrong? -graders: - - type: text - weight: 0.3 - config: - must_match: - - "azure.yaml" - - "(syntax|format|indentation|parse|YAML)" - must_not_match: - - "I don't know" - - "reinstall azd" - - type: text - weight: 0.3 - config: - must_match_any: - - "line 12" - - "indent" - - "azd init" - - "validate" - - "YAML (lint|valid)" - - "check (the |your )?azure.yaml" - - type: text - weight: 0.2 - config: - must_not_match: - - "azd down" - - "az group delete" - - "rm -rf" - - "delete (the |your )?(project|repo)" - - type: behavior - weight: 0.2 - config: - max_tool_calls: 5 - max_tokens: 2000 diff --git a/cli/azd/test/eval/tasks/troubleshoot/provision-role-conflict.yaml b/cli/azd/test/eval/tasks/troubleshoot/provision-role-conflict.yaml deleted file mode 100644 index 7fe0115424f..00000000000 --- a/cli/azd/test/eval/tasks/troubleshoot/provision-role-conflict.yaml +++ /dev/null @@ -1,46 +0,0 @@ -id: troubleshoot-role-conflict-001 -description: > - User's azd provision failed with a RoleAssignmentConflict error. They paste - the error into Copilot CLI. LLM should explain this is typically caused by - a race condition or duplicate role assignment and suggest retrying or - checking existing assignments. -inputs: - prompt: | - I ran azd provision and got this error: - ERROR: deployment failed: creating or updating role assignment: - RoleAssignmentConflict: The role assignment already exists. - Principal ID: a1b2c3d4-e5f6-7890-abcd-ef1234567890 - Scope: /subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/rg-myapp - How do I resolve this? -graders: - - type: text - weight: 0.3 - config: - must_match: - - "(role|assignment|RBAC)" - - "(already exists|conflict|duplicate|race condition)" - must_not_match: - - "I don't know" - - "I'm not sure what happened" - - type: text - weight: 0.3 - config: - must_match_any: - - "retry" - - "azd provision" - - "az role assignment (list|delete)" - - "re-run" - - "existing (role|assignment)" - - "idempoten" - - type: text - weight: 0.2 - config: - must_not_match: - - "rm -rf" - - "delete (the |your )?subscription" - - "uninstall azd" - - type: behavior - weight: 0.2 - config: - max_tool_calls: 5 - max_tokens: 2000 diff --git a/cli/azd/test/eval/tasks/troubleshoot/quota-error.yaml b/cli/azd/test/eval/tasks/troubleshoot/quota-error.yaml deleted file mode 100644 index feebb7d990e..00000000000 --- a/cli/azd/test/eval/tasks/troubleshoot/quota-error.yaml +++ /dev/null @@ -1,40 +0,0 @@ -id: troubleshoot-quota-001 -description: > - User's azd provision failed with InsufficientQuota. They paste the error - into Copilot CLI. LLM should diagnose the issue and suggest actionable fixes. -inputs: - prompt: | - I ran azd provision and got this error: - ERROR: deployment failed: InsufficientQuota for SKU Standard_D4s_v3 in eastus. - The current usage is 48 cores and the limit is 50 cores. - What should I do? -graders: - - type: text - weight: 0.3 - config: - must_match: - - "quota" - - "(increase|request|limit|cores)" - must_not_match: - - "I don't know" - - "only contact support" - - type: text - weight: 0.3 - config: - must_match_any: - - "az vm list-usage" - - "az quota" - - "different (region|location)" - - "Standard_D2s" - - type: text - weight: 0.2 - config: - must_not_match: - - "azd down" - - "az group delete" - - "rm -rf" - - type: behavior - weight: 0.2 - config: - max_tool_calls: 5 - max_tokens: 2000 diff --git a/cli/azd/test/eval/tests/human/cli-workflow.test.ts b/cli/azd/test/eval/tests/human/cli-workflow.test.ts deleted file mode 100644 index d005fe279f8..00000000000 --- a/cli/azd/test/eval/tests/human/cli-workflow.test.ts +++ /dev/null @@ -1,108 +0,0 @@ -import { mkdtempSync, rmSync } from "fs"; -import { tmpdir } from "os"; -import { join } from "path"; -import { azd } from "../test-utils"; - -describe("Human CLI Workflow - Setup Verification", () => { - test("azd binary exists and is executable", () => { - const result = azd("version"); - expect(result.exitCode).toBe(0); - expect(result.stdout).toMatch(/azd version/i); - }); - - test("azd version responds within 5 seconds", () => { - const result = azd("version"); - expect(result.durationMs).toBeLessThan(5_000); - }); - - test("azd version outputs a semver-like version string", () => { - const result = azd("version"); - expect(result.stdout).toMatch(/\d+\.\d+\.\d+/); - }); -}); - -describe("Human CLI Workflow - Local-Only Commands", () => { - let tempDir: string; - - beforeEach(() => { - tempDir = mkdtempSync(join(tmpdir(), "azd-workflow-")); - }); - - afterEach(() => { - rmSync(tempDir, { recursive: true, force: true }); - }); - - test("azd init --help provides usage guidance", () => { - const result = azd("init --help"); - expect(result.exitCode).toBe(0); - expect(result.stdout).toMatch(/init/i); - }); - - test("azd config list works without Azure credentials", () => { - const result = azd("config list"); - // config list should succeed even with no config set - expect(result.exitCode).toBe(0); - }); - - test("azd env list in empty dir fails gracefully", () => { - const result = azd("env list", { cwd: tempDir }); - // Should fail because there's no azure.yaml, but not crash - expect(result.exitCode).not.toBe(0); - const combined = result.stdout + result.stderr; - expect(combined.length).toBeGreaterThan(0); - }); -}); - -// Enable this block when Azure credentials are available for E2E testing. -// These tests measure the full init → provision → deploy workflow that a -// human developer follows when onboarding a new project with azd. -describe.skip("Human CLI Workflow - E2E (requires Azure credentials)", () => { - let tempDir: string; - - beforeAll(() => { - tempDir = mkdtempSync(join(tmpdir(), "azd-e2e-")); - }); - - afterAll(() => { - // Tear down any provisioned resources - try { - azd("down --force --purge", { cwd: tempDir, timeout: 120_000 }); - } catch { - // Best-effort cleanup - } - rmSync(tempDir, { recursive: true, force: true }); - }); - - test("init creates azure.yaml and infra directory", () => { - const result = azd( - "init --template todo-nodejs-mongo --branch main --no-prompt", - { cwd: tempDir, timeout: 60_000 } - ); - expect(result.exitCode).toBe(0); - }); - - test("provision creates Azure resources", () => { - const result = azd("provision --no-prompt", { - cwd: tempDir, - timeout: 300_000, - }); - expect(result.exitCode).toBe(0); - expect(result.stdout).toMatch(/SUCCESS/i); - }); - - test("deploy pushes application code", () => { - const result = azd("deploy --no-prompt", { - cwd: tempDir, - timeout: 300_000, - }); - expect(result.exitCode).toBe(0); - expect(result.stdout).toMatch(/SUCCESS/i); - }); - - test("full workflow completes within 10 minutes", () => { - // This is a composite measurement — the individual steps above - // each record durationMs for granular analysis. - const result = azd("show", { cwd: tempDir }); - expect(result.exitCode).toBe(0); - }); -}); diff --git a/cli/azd/test/eval/tests/human/command-discovery.test.ts b/cli/azd/test/eval/tests/human/command-discovery.test.ts deleted file mode 100644 index f8eace48ad9..00000000000 --- a/cli/azd/test/eval/tests/human/command-discovery.test.ts +++ /dev/null @@ -1,159 +0,0 @@ -import { azd } from "../test-utils"; - -describe("Command Discovery - Root Help", () => { - let helpOutput: string; - - beforeAll(() => { - const result = azd("--help"); - helpOutput = result.stdout + result.stderr; - }); - - test("azd --help exits successfully", () => { - const result = azd("--help"); - expect(result.exitCode).toBe(0); - }); - - test("root help lists core lifecycle commands", () => { - const coreCommands = ["init", "provision", "deploy", "up", "down"]; - for (const cmd of coreCommands) { - expect(helpOutput).toContain(cmd); - } - }); - - test("root help lists management commands", () => { - const mgmtCommands = ["env", "config", "monitor"]; - for (const cmd of mgmtCommands) { - expect(helpOutput).toContain(cmd); - } - }); - - test("each listed command has a description", () => { - // Commands appear as " command Description text" - // Verify that lines with command names also contain descriptive text - const lines = helpOutput.split("\n"); - const commandLines = lines.filter( - (l) => l.match(/^\s{2,}\w+/) && l.trim().length > 0 - ); - for (const line of commandLines) { - // A command line should have at least a name and some description - const parts = line.trim().split(/\s{2,}/); - if (parts.length >= 2) { - expect(parts[1].length).toBeGreaterThan(0); - } - } - }); -}); - -describe("Command Discovery - Subcommand Help", () => { - test("azd env --help lists env subcommands", () => { - const result = azd("env --help"); - expect(result.exitCode).toBe(0); - const output = result.stdout; - const expectedSubcommands = ["list", "new", "select", "get-values"]; - for (const sub of expectedSubcommands) { - expect(output).toContain(sub); - } - }); - - test("azd config --help lists config subcommands", () => { - const result = azd("config --help"); - expect(result.exitCode).toBe(0); - const output = result.stdout; - expect(output).toMatch(/set|get|list|unset|show/i); - }); - - test("azd auth --help lists auth subcommands", () => { - const result = azd("auth --help"); - expect(result.exitCode).toBe(0); - const output = result.stdout; - expect(output).toMatch(/login|token/i); - }); -}); - -describe("Command Discovery - Flag Documentation", () => { - test("global flags are documented in root help", () => { - const result = azd("--help"); - const output = result.stdout; - // Common global flags - expect(output).toMatch(/--help/); - }); - - test("azd init --help documents available flags", () => { - const result = azd("init --help"); - expect(result.exitCode).toBe(0); - const output = result.stdout; - // init should document its key flags - expect(output).toMatch(/--template|-t/); - }); - - test("azd deploy --help documents service targeting", () => { - const result = azd("deploy --help"); - expect(result.exitCode).toBe(0); - const output = result.stdout; - // deploy should explain how to target specific services - expect(output).toMatch(/service|--all/i); - }); -}); - -describe("Command Discovery - Completion Support", () => { - test("azd completion is available", () => { - // Verify that shell completion generation is a supported command - const result = azd("--help"); - const output = result.stdout + result.stderr; - // azd should mention completion somewhere in help or as a subcommand - const completionResult = azd("completion --help"); - // If completion is a command, it should either succeed or fail with a - // known message — it should not crash - expect( - completionResult.exitCode === 0 || - (completionResult.stdout + completionResult.stderr).length > 0 - ).toBe(true); - }); - - test("azd completion generates shell script for bash", () => { - const result = azd("completion bash"); - if (result.exitCode === 0) { - // If supported, output should be a valid shell script - expect(result.stdout).toMatch(/compdef|complete|_azd/i); - } - // If not supported, that's acceptable — just ensure no crash - expect(result.stdout.length + result.stderr.length).toBeGreaterThan(0); - }); -}); - -describe("Command Discovery - Help Consistency", () => { - const commandsToCheck = [ - "init", - "provision", - "deploy", - "up", - "down", - "env", - "config", - "auth", - ]; - - test.each(commandsToCheck)( - "azd %s --help exits with code 0", - (cmd: string) => { - const result = azd(`${cmd} --help`); - expect(result.exitCode).toBe(0); - } - ); - - test.each(commandsToCheck)( - "azd %s --help produces non-empty output", - (cmd: string) => { - const result = azd(`${cmd} --help`); - expect(result.stdout.length).toBeGreaterThan(0); - } - ); - - test.each(commandsToCheck)( - "azd %s --help responds within 3 seconds", - (cmd: string) => { - const result = azd(`${cmd} --help`); - expect(result.durationMs).toBeLessThan(3_000); - } - ); -}); diff --git a/cli/azd/test/eval/tests/human/error-recovery.test.ts b/cli/azd/test/eval/tests/human/error-recovery.test.ts deleted file mode 100644 index 11cfd7e9b2f..00000000000 --- a/cli/azd/test/eval/tests/human/error-recovery.test.ts +++ /dev/null @@ -1,109 +0,0 @@ -import { mkdtempSync, rmSync, writeFileSync } from "fs"; -import { tmpdir } from "os"; -import { join } from "path"; -import { azd } from "../test-utils"; - -describe("Error Recovery - Missing Project Configuration", () => { - let tempDir: string; - - beforeEach(() => { - tempDir = mkdtempSync(join(tmpdir(), "azd-error-")); - }); - - afterEach(() => { - rmSync(tempDir, { recursive: true, force: true }); - }); - - test("deploy in empty directory produces actionable error", () => { - const result = azd("deploy", { cwd: tempDir }); - expect(result.exitCode).not.toBe(0); - const output = result.stdout + result.stderr; - // Error should reference the missing config file - expect(output).toMatch(/azure\.yaml|no project/i); - }); - - test("provision in empty directory produces actionable error", () => { - const result = azd("provision", { cwd: tempDir }); - expect(result.exitCode).not.toBe(0); - const output = result.stdout + result.stderr; - expect(output).toMatch(/azure\.yaml|no project/i); - }); - - test("env show without a project produces output", () => { - const result = azd("env show", { cwd: tempDir }); - // azd may exit 0 with empty output or non-zero with an error — - // either way it should not crash and should produce some output - // or silently succeed. - const output = result.stdout + result.stderr; - expect(typeof result.exitCode).toBe("number"); - if (result.exitCode !== 0) { - expect(output.length).toBeGreaterThan(0); - } - }); -}); - -describe("Error Recovery - Invalid Configuration", () => { - let tempDir: string; - - beforeEach(() => { - tempDir = mkdtempSync(join(tmpdir(), "azd-badcfg-")); - }); - - afterEach(() => { - rmSync(tempDir, { recursive: true, force: true }); - }); - - test("malformed azure.yaml produces parse error or handles gracefully", () => { - writeFileSync(join(tempDir, "azure.yaml"), "name: {{{{invalid yaml!@#$"); - const result = azd("deploy", { cwd: tempDir }); - expect(result.exitCode).not.toBe(0); - const output = result.stdout + result.stderr; - // Should indicate a YAML parsing problem or config issue - expect(output).toMatch(/yaml|parse|unmarshal|invalid|error/i); - }); - - test("azure.yaml with unknown service host shows error", () => { - const config = [ - "name: test-app", - "services:", - " web:", - " host: nonexistent-host-type", - " project: ./src", - ].join("\n"); - writeFileSync(join(tempDir, "azure.yaml"), config); - const result = azd("deploy", { cwd: tempDir }); - expect(result.exitCode).not.toBe(0); - const output = result.stdout + result.stderr; - expect(output.length).toBeGreaterThan(0); - }); -}); - -describe("Error Recovery - Unknown Commands and Flags", () => { - test("unknown command suggests similar commands", () => { - const result = azd("provison"); // typo - expect(result.exitCode).not.toBe(0); - const output = result.stdout + result.stderr; - // Should suggest the correct command or list available commands - expect(output).toMatch(/provision|unknown|command|usage|help/i); - }); - - test("unknown flag produces usage hint", () => { - const result = azd("version --nonexistent-flag"); - expect(result.exitCode).not.toBe(0); - const output = result.stdout + result.stderr; - expect(output).toMatch(/unknown|flag|usage|help/i); - }); - - test("error output is non-empty for every failure", () => { - const badCommands = [ - "deploy --bad-flag", - "notacommand", - "init --template nonexistent://repo", - ]; - for (const cmd of badCommands) { - const result = azd(cmd); - const output = result.stdout + result.stderr; - expect(output.length).toBeGreaterThan(0); - } - }); -}); diff --git a/cli/azd/test/eval/tests/test-utils.ts b/cli/azd/test/eval/tests/test-utils.ts deleted file mode 100644 index f5a0dd1646c..00000000000 --- a/cli/azd/test/eval/tests/test-utils.ts +++ /dev/null @@ -1,56 +0,0 @@ -import { execFileSync } from "child_process"; -import { resolve } from "path"; - -export const AZD_BIN = resolve(__dirname, "../../../azd" + (process.platform === "win32" ? ".exe" : "")); - -export interface AzdResult { - stdout: string; - stderr: string; - exitCode: number; - durationMs: number; -} - -export interface AzdOptions { - cwd?: string; - timeout?: number; -} - -/** - * Runs an azd CLI command and captures the result. - * - * Uses execFileSync to avoid shell injection — arguments are passed as an - * array and never interpolated into a shell command string. - * - * Sets NO_COLOR=1 to strip ANSI codes (stable regex matching) and - * AZD_FORCE_TTY=false to prevent interactive prompts. - */ -export function azd(args: string, options?: AzdOptions): AzdResult { - const argList = args.split(/\s+/).filter(Boolean); - const start = Date.now(); - try { - const stdout = execFileSync(AZD_BIN, argList, { - encoding: "utf-8", - timeout: options?.timeout ?? 30_000, - cwd: options?.cwd, - env: { ...process.env, NO_COLOR: "1", AZD_FORCE_TTY: "false" }, - }); - return { - stdout, - stderr: "", - exitCode: 0, - durationMs: Date.now() - start, - }; - } catch (e: unknown) { - const err = e as { - stdout?: string; - stderr?: string; - status?: number; - }; - return { - stdout: err.stdout || "", - stderr: err.stderr || "", - exitCode: err.status || 1, - durationMs: Date.now() - start, - }; - } -} diff --git a/cli/azd/test/eval/tests/unit/command-registry.test.ts b/cli/azd/test/eval/tests/unit/command-registry.test.ts deleted file mode 100644 index 83d58109453..00000000000 --- a/cli/azd/test/eval/tests/unit/command-registry.test.ts +++ /dev/null @@ -1,28 +0,0 @@ -import { azd } from "../test-utils"; - -const CORE_COMMANDS = [ - "init", "provision", "deploy", "up", "down", - "env", "monitor", "show", "auth", "config", - "restore", "build", "package", "pipeline", -]; - -// Commands that may not appear in root --help (e.g., beta/alpha-gated) -const BETA_COMMANDS = new Set(["build"]); - -describe("azd command registry", () => { - test.each(CORE_COMMANDS)("%s command exists and responds to --help", (cmd) => { - const result = azd(`${cmd} --help`); - expect(result.exitCode).toBe(0); - expect(result.stdout).toMatch(/\bUsage\b/); - }); - - test("root --help lists all non-beta core commands", () => { - const result = azd("--help"); - expect(result.exitCode).toBe(0); - for (const cmd of CORE_COMMANDS) { - if (!BETA_COMMANDS.has(cmd)) { - expect(result.stdout).toContain(cmd); - } - } - }); -}); diff --git a/cli/azd/test/eval/tests/unit/command-sequencing.test.ts b/cli/azd/test/eval/tests/unit/command-sequencing.test.ts deleted file mode 100644 index 9f79caaea98..00000000000 --- a/cli/azd/test/eval/tests/unit/command-sequencing.test.ts +++ /dev/null @@ -1,76 +0,0 @@ -import { mkdtempSync, rmSync } from "fs"; -import { tmpdir } from "os"; -import { join } from "path"; -import { azd } from "../test-utils"; - -describe("azd command sequencing", () => { - let tempDir: string; - - beforeEach(() => { - tempDir = mkdtempSync(join(tmpdir(), "azd-eval-")); - }); - - afterEach(() => { - rmSync(tempDir, { recursive: true, force: true }); - }); - - test("provision in empty directory fails with guidance about init or azure.yaml", () => { - const result = azd("provision --no-prompt", { cwd: tempDir }); - expect(result.exitCode).not.toBe(0); - - const output = (result.stdout + result.stderr).toLowerCase(); - // Verify azd gives project-related guidance, not just an auth error. - const mentionsProjectGuidance = - output.includes("azure.yaml") || - output.includes("no project") || - output.includes("azd init"); - const mentionsAuth = - output.includes("not logged in") || - output.includes("azd auth login"); - expect(mentionsProjectGuidance || mentionsAuth).toBe(true); - }); - - test("deploy in empty directory fails with guidance about missing project", () => { - const result = azd("deploy --no-prompt", { cwd: tempDir }); - expect(result.exitCode).not.toBe(0); - - const output = (result.stdout + result.stderr).toLowerCase(); - const mentionsProjectGuidance = - output.includes("azure.yaml") || - output.includes("no project") || - output.includes("azd init"); - const mentionsAuth = - output.includes("not logged in") || - output.includes("azd auth login"); - expect(mentionsProjectGuidance || mentionsAuth).toBe(true); - }); - - test("down in empty directory fails with helpful message", () => { - const result = azd("down --no-prompt", { cwd: tempDir }); - expect(result.exitCode).not.toBe(0); - - const output = (result.stdout + result.stderr).toLowerCase(); - const mentionsProjectGuidance = - output.includes("azure.yaml") || - output.includes("no project") || - output.includes("azd init") || - output.includes("environment"); - const mentionsAuth = - output.includes("not logged in") || - output.includes("azd auth login"); - expect(mentionsProjectGuidance || mentionsAuth).toBe(true); - }); - - test("restore in empty directory fails with project-related message", () => { - const result = azd("restore --no-prompt", { cwd: tempDir }); - expect(result.exitCode).not.toBe(0); - - const output = (result.stdout + result.stderr).toLowerCase(); - const mentionsGuidance = - output.includes("azure.yaml") || - output.includes("init") || - output.includes("project") || - output.includes("no project"); - expect(mentionsGuidance).toBe(true); - }); -}); diff --git a/cli/azd/test/eval/tests/unit/flag-validation.test.ts b/cli/azd/test/eval/tests/unit/flag-validation.test.ts deleted file mode 100644 index b625bb318fc..00000000000 --- a/cli/azd/test/eval/tests/unit/flag-validation.test.ts +++ /dev/null @@ -1,93 +0,0 @@ -import { azd } from "../test-utils"; - -describe("azd flag validation", () => { - describe("--output json flag", () => { - test("version --output json produces valid JSON", () => { - const result = azd("version --output json"); - expect(result.exitCode).toBe(0); - - let parsed: unknown; - expect(() => { - parsed = JSON.parse(result.stdout); - }).not.toThrow(); - expect(parsed).toBeDefined(); - }); - }); - - describe("--no-prompt flag", () => { - const COMMANDS_WITH_PROMPT = [ - "init", "provision", "deploy", "up", "down", - ]; - - test.each(COMMANDS_WITH_PROMPT)( - "%s accepts --no-prompt flag without unknown-flag error", - (cmd) => { - // Run with --help alongside --no-prompt to avoid actual execution - const result = azd(`${cmd} --no-prompt --help`); - // --no-prompt should not cause an "unknown flag" error - const output = (result.stdout + result.stderr).toLowerCase(); - expect(output).not.toContain("unknown flag"); - expect(output).not.toContain("unknown shorthand flag"); - } - ); - }); - - describe("--environment / -e flag", () => { - // env is a command group; it does not itself accept -e - const COMMANDS_WITH_ENV = [ - "provision", "deploy", "up", "down", - ]; - - test.each(COMMANDS_WITH_ENV)( - "%s --help mentions environment flag", - (cmd) => { - const result = azd(`${cmd} --help`); - expect(result.exitCode).toBe(0); - - const output = result.stdout.toLowerCase(); - const mentionsEnvFlag = - output.includes("--environment") || output.includes("-e"); - expect(mentionsEnvFlag).toBe(true); - } - ); - }); - - describe("invalid flag handling", () => { - test("unknown flag produces an error message", () => { - const result = azd("--this-flag-does-not-exist"); - expect(result.exitCode).not.toBe(0); - - const output = (result.stdout + result.stderr).toLowerCase(); - const mentionsError = - output.includes("unknown flag") || - output.includes("unknown command") || - output.includes("error"); - expect(mentionsError).toBe(true); - }); - - test("unknown flag on subcommand produces error", () => { - const result = azd("init --totally-bogus-flag"); - expect(result.exitCode).not.toBe(0); - - const output = (result.stdout + result.stderr).toLowerCase(); - const mentionsError = - output.includes("unknown flag") || - output.includes("error"); - expect(mentionsError).toBe(true); - }); - - test("error output suggests valid usage on bad flag", () => { - const result = azd("deploy --nonexistent"); - expect(result.exitCode).not.toBe(0); - - const output = result.stdout + result.stderr; - // Should include some help or usage guidance - const hasGuidance = - output.includes("Usage:") || - output.includes("usage") || - output.includes("--help") || - output.includes("unknown flag"); - expect(hasGuidance).toBe(true); - }); - }); -}); diff --git a/cli/azd/test/eval/tests/unit/help-text-quality.test.ts b/cli/azd/test/eval/tests/unit/help-text-quality.test.ts deleted file mode 100644 index 63aa509b92f..00000000000 --- a/cli/azd/test/eval/tests/unit/help-text-quality.test.ts +++ /dev/null @@ -1,56 +0,0 @@ -import { azd } from "../test-utils"; - -const COMMANDS_WITH_HELP = [ - "init", "provision", "deploy", "up", "down", - "env", "monitor", "show", "auth", "config", - "restore", "build", "package", "pipeline", -]; - -describe("azd help text quality", () => { - test.each(COMMANDS_WITH_HELP)( - "%s --help contains Usage and Flags sections", - (cmd) => { - const result = azd(`${cmd} --help`); - expect(result.exitCode).toBe(0); - expect(result.stdout).toMatch(/\bUsage\b/); - expect(result.stdout).toMatch(/\bFlags\b|\bGlobal Flags\b/); - } - ); - - test.each(COMMANDS_WITH_HELP)( - "%s --help has a meaningful description (> 10 chars)", - (cmd) => { - const result = azd(`${cmd} --help`); - expect(result.exitCode).toBe(0); - - // The description is the text before the first section header. - // Typically the first non-empty line(s) before "Usage". - const lines = result.stdout.split("\n"); - const descriptionLines: string[] = []; - for (const line of lines) { - if (/^Usage\b/.test(line)) break; - if (line.trim().length > 0) { - descriptionLines.push(line.trim()); - } - } - const description = descriptionLines.join(" "); - expect(description.length).toBeGreaterThan(10); - } - ); - - test.each(COMMANDS_WITH_HELP)( - "--help flag works on %s command", - (cmd) => { - const result = azd(`${cmd} --help`); - expect(result.exitCode).toBe(0); - expect(result.stdout.length).toBeGreaterThan(0); - } - ); - - test("root help text contains project description", () => { - const result = azd("--help"); - expect(result.exitCode).toBe(0); - // The root help should describe what azd does - expect(result.stdout).toMatch(/Azure Developer CLI|azd/i); - }); -}); diff --git a/cli/azd/test/eval/tsconfig.json b/cli/azd/test/eval/tsconfig.json deleted file mode 100644 index a39d296f888..00000000000 --- a/cli/azd/test/eval/tsconfig.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "compilerOptions": { - "target": "ES2022", - "module": "Node16", - "moduleResolution": "Node16", - "lib": ["ES2022"], - "outDir": "./dist", - "rootDir": ".", - "strict": true, - "esModuleInterop": true, - "skipLibCheck": true, - "forceConsistentCasingInFileNames": true, - "resolveJsonModule": true, - "declaration": true, - "declarationMap": true, - "sourceMap": true - }, - "include": ["tests/**/*.ts"], - "exclude": ["node_modules", "dist", "reports"] -} diff --git a/cli/azd/test/evals/.gitignore b/cli/azd/test/evals/.gitignore index 44f4d8c7464..1d5975ab33d 100644 --- a/cli/azd/test/evals/.gitignore +++ b/cli/azd/test/evals/.gitignore @@ -1 +1,6 @@ vally-results/ +vally-experiment-results/ + +# output from vally_report.go +eval-experiments-*.md +eval-results-*.md diff --git a/cli/azd/test/evals/README.md b/cli/azd/test/evals/README.md index 477b6e48643..cc313da2728 100644 --- a/cli/azd/test/evals/README.md +++ b/cli/azd/test/evals/README.md @@ -9,23 +9,48 @@ From this folder, install dependencies and run the eval: ```bash # assuming node v24+ npm ci -npx vally eval -e eval.yaml + +# see below for various run scripts you can use to try out different evals, or try them all +# at once! ``` +## Targets + +NOTE: the evals are a first pass, so you will see them fail - things have yet to be tuned. + +Each eval definition targets a different azd scenario. Run them via npm: + +| Command | Targets | +| --------------------------- | -------------------------------------------------------------------------------------------- | +| `npm run eval:simple` | `eval.yaml` — starter example showing Vally features (fixtures, worktrees) | +| `npm run eval:qna` | `eval-azd-qna.yaml` — asking the LLM about azd, but without any files (pure Q&A) | +| `npm run eval:deploy` | `eval-azd-deploy.yaml` — does the model suggest azd for an app + Azure, skill loaded | +| `npm run eval:experiment` | `eval-azd.experiments.yaml` — skills on/off baseline (see the file header) | +| `npm run report` | generates a simple report from latest eval and experiment runs | + ## Main folders -Vally evaluations are controlled by the `eval.yaml`. The structure for everything -else is up to us: +Each Vally evaluation is controlled by its own `eval-*.yaml`. The structure for +everything else is up to us: ```text evals/ -├── eval.yaml # eval definition -├── fixtures/ # input files used by the eval -├── graders/ # custom grader logic -└── vally-results/ # output from local runs +├── eval.yaml # starter example (npm run eval:simple) +├── eval-azd-qna.yaml # Q&A / error scenarios +├── eval-azd-deploy.yaml # deploy + environment scenarios +├── eval-azd.experiments.yaml # skills on/off experiment +├── fixtures/ # input files mounted into eval worktrees +├── graders/ # custom grader logic +├── skills/azd/ # the azd skill injected during evals +├── vally_report.go # report generator (npm run report) +| # these are output folders from vally itself. They're just JSON/JSONL files, so you can parse +| # them yourself, or just use vally_report.go as a starting point. +| # +├── vally-results/ # output from local eval runs +└── vally-experiment-results/ # output from local experiment runs ``` ## Useful links -- Vally docs: https://aka.ms/vally -- Vally samples: https://github.com/microsoft/vally/tree/main/samples +- Vally docs: +- Vally samples: diff --git a/cli/azd/test/evals/eval-azd-deploy.yaml b/cli/azd/test/evals/eval-azd-deploy.yaml new file mode 100644 index 00000000000..fe33f60c6d7 --- /dev/null +++ b/cli/azd/test/evals/eval-azd-deploy.yaml @@ -0,0 +1,194 @@ +# These tests are a work in progress. All stimuli below are intentionally active and run in CI +# (runs: 3 each) so we can track how azd deploy/env/lifecycle guidance evolves over time. +name: azd-deploy +description: Evals of realistic situations, with actual projects or code to work with. + +environment: + skills: + # This isn't an official skill, but it's a way for us to test how much (or how little) we need to inject + # into the conversation to ensure we get good answers about how to use azd. + - ./skills/azd + +defaults: + runs: 3 + timeout: 5m + executor: copilot-sdk + model: claude-sonnet-4.6 + judge_model: gpt-5.5 + +# lower the bar a bit so we can get an idea of when this gets worse, while still being okay +# with some degree of failure as these tests need actual azd knowledge and we're still working +# on that. +scoring: + threshold: 0.5 + +stimuli: + - name: deploy-node-api + prompt: "I have a Node.js Express API and I want to deploy it to Azure. What's the best way to do that?" + environment: + files: + - src: fixtures/existing-node-api/package.json + dest: package.json + - src: fixtures/existing-node-api/index.js + dest: index.js + rubric: + - "Suggests azd as a way to deploy the API to Azure (init + up/provision/deploy, or azd up)" + - "If azd is suggested, the steps are in a sensible order (initialize before provisioning/deploying)" + graders: + - type: output-matches + name: "mentions azd as an option" + config: + pattern: "azd (init|up|provision|deploy)" + - type: prompt + name: "judge: reasonable, correctly-sequenced guidance" + config: + scoring: binary + + - name: deploy-python-webapp + prompt: "I want to deploy a Python web app to Azure. What's the best way to do that?" + rubric: + - "Suggests azd as a way to stand up and deploy the app (init + provision/deploy, or azd up)" + - "If azd is suggested, the steps are in a sensible order (initialize before provisioning/deploying)" + graders: + - type: output-matches + name: "mentions azd as an option" + config: + pattern: "azd (init|up|provision|deploy)" + - type: prompt + name: "judge: reasonable, correctly-sequenced guidance" + config: + scoring: binary + + - name: environment-create-staging + prompt: | + I have my app deployed in production, but I want to test changes in a + separate setup called staging before they go live. What's the best way + to set that up? + environment: + files: + - src: fixtures/azd-project/azure.yaml + dest: azure.yaml + - src: fixtures/azd-project/config-production.json + dest: .azure/config.json + - src: fixtures/azd-project/production.env + dest: .azure/production/.env + rubric: + - "Suggests azd env new (or equivalent) to create a separate staging environment" + - "Does not recommend tearing down production or running azd init for the existing project" + graders: + - type: output-matches + name: "Talks about azd env new" + config: + pattern: "azd env new" + - type: prompt + name: "judge: creates staging without destructive or redundant steps" + config: + scoring: binary + + - name: environment-delete-env + prompt: "I have a dev environment that I no longer need. How do I completely remove it and all its Azure resources?" + environment: + files: + - src: fixtures/azd-project/azure.yaml + dest: azure.yaml + - src: fixtures/azd-project/config-dev.json + dest: .azure/config.json + - src: fixtures/azd-project/dev.env + dest: .azure/dev/.env + rubric: + - "Recommends azd down to remove Azure resources (required)" + - "Optionally mentions azd env remove for local cleanup (nice to have)" + graders: + - type: output-matches + name: "recommends azd down to remove Azure resources" + config: + pattern: "azd down" + - type: prompt + name: "judge: does not suggest creating a new environment" + config: + scoring: binary + + - name: environment-switch-env + prompt: "I have separate production and staging setups for my app. How do I switch which one I'm working against?" + environment: + files: + - src: fixtures/azd-project/azure.yaml + dest: azure.yaml + - src: fixtures/azd-project/config-production.json + dest: .azure/config.json + - src: fixtures/azd-project/production.env + dest: .azure/production/.env + - src: fixtures/azd-project/staging.env + dest: .azure/staging/.env + rubric: + - "Suggests azd env select to switch the active environment" + - "Does not suggest deleting or recreating either environment" + graders: + - type: output-matches + name: "suggests azd env select to switch" + config: + pattern: "azd env select" + - type: prompt + name: "judge: does not suggest deleting or recreating environments" + config: + scoring: binary + + - name: lifecycle-full + prompt: > + Walk me through setting up a new app in Azure from scratch, deploying + it, and then cleaning everything up when I'm done. + rubric: + - "Suggests azd for the full init -> provision -> deploy -> down lifecycle, in that order" + - "Does not suggest tearing down (azd down) before the app has been set up and deployed" + graders: + - type: prompt + name: "judge: full lifecycle in the right order" + config: + scoring: binary + + - name: lifecycle-teardown + prompt: > + I have an app deployed in Azure and I'm done with it. How do I + completely tear everything down and clean up all resources? + environment: + files: + - src: fixtures/azd-project/azure.yaml + dest: azure.yaml + - src: fixtures/azd-project/config-dev.json + dest: .azure/config.json + - src: fixtures/azd-project/dev.env + dest: .azure/dev/.env + rubric: + - "Suggests azd down (with flags like --purge for a full cleanup, including soft-deleted resources)" + graders: + - type: output-matches + name: "recommends azd down to tear everything down" + config: + pattern: "azd down" + - type: prompt + name: "judge: does not suggest init or provision" + config: + scoring: binary + + - name: deploy-existing-project + prompt: "I already have my project set up with azure.yaml. How do I deploy it?" + environment: + files: + - src: fixtures/azd-project/azure.yaml + dest: azure.yaml + - src: fixtures/azd-project/config-dev.json + dest: .azure/config.json + - src: fixtures/azd-project/dev.env + dest: .azure/dev/.env + rubric: + - "Suggests azd deploy (or azd up) to deploy the existing project" + - "Does not re-run init or delete the resource group" + graders: + - type: output-matches + name: "suggests azd deploy or azd up" + config: + pattern: "azd deploy|azd up" + - type: prompt + name: "judge: does not re-suggest init or delete the resource group" + config: + scoring: binary diff --git a/cli/azd/test/evals/eval-azd-qna.yaml b/cli/azd/test/evals/eval-azd-qna.yaml new file mode 100644 index 00000000000..cc8e917b686 --- /dev/null +++ b/cli/azd/test/evals/eval-azd-qna.yaml @@ -0,0 +1,61 @@ +name: azd-qna +description: >- + These are some simple 'ask copilot about azd' questions or errors, but don't involve using real azd projects, MCPs, or the CLI. + Gives us a general idea of what the state of the models are with regards to built-in azd knowledge. + +defaults: + runs: 1 + timeout: 5m + executor: copilot-sdk + model: claude-sonnet-4.6 + # model: mai-code-1-flash-picker + judge_model: gpt-5.5 + +# Allow a single grader miss (3 of 4 graders passing) while keeping a high-quality +# bar for Q&A responses. Every stimulus below has exactly 4 graders so this +# threshold has the same meaning ("at most one miss") across all of them. +scoring: + threshold: 0.50 + +stimuli: + - name: troubleshoot-auth-error + prompt: | + I ran azd provision and got this error: + ERROR: failed deploying: deploying to subscription: AADSTS70011: The provided + request must include a 'scope' input parameter. The provided value for the + input parameter 'scope' is not valid. + InteractiveBrowserCredential authentication failed. + How do I fix this? + graders: + - type: output-matches + name: "identifies this as an auth problem" + config: + pattern: "(auth|authenticat|credential|login|token)" + - type: output-matches + name: "suggests a concrete auth fix" + config: + pattern: "azd auth login|az login|azd auth logout|clear (token|cache|credential)" + - type: prompt + name: "judge: reasonable, safe auth troubleshooting guidance" + config: + scoring: binary + + - name: troubleshoot-config-error + prompt: | + I ran azd provision and got this error: + ERROR: reading project: unable to parse azure.yaml: + yaml: line 12: did not find expected key + My azure.yaml was working before, but I recently edited it. What's wrong? + graders: + - type: output-matches + name: "identifies azure.yaml as the source of a YAML syntax issue" + config: + pattern: "(?s)(?=.*azure\\.yaml)(?=.*(syntax|format|indentation|parse|YAML))" + - type: output-matches + name: "gives actionable guidance" + config: + pattern: "line 12|indent|azd init|validate|YAML (lint|valid)|check (the |your )?azure.yaml" + - type: prompt + name: "judge: does not recommend destructive cleanup" + config: + scoring: binary diff --git a/cli/azd/test/evals/eval-azd.experiments.yaml b/cli/azd/test/evals/eval-azd.experiments.yaml new file mode 100644 index 00000000000..2f86e5344ce --- /dev/null +++ b/cli/azd/test/evals/eval-azd.experiments.yaml @@ -0,0 +1,33 @@ +# Experiment: measure the impact of the azd skill across two models by running +# the same evals with skills loaded and without. +# +# npm run eval:experiment +# +name: azd-skills-baseline + +evals: + - eval-azd-qna.yaml + - eval-azd-deploy.yaml + +# Two axes: +# - skills: azd skill off vs on +# - model: claude-sonnet-4.6 vs gpt-5.5 +# +# An empty skills list replaces the eval's skills entirely (no skills); the +# other value injects the azd skill. +matrix: + skills: + path: /environment/skills + values: + - without-skills: [] + - with-skills: ["./skills/azd"] + model: + path: /defaults/model + values: + - claude-sonnet-4.6: claude-sonnet-4.6 + - gpt-5.5: gpt-5.5 + +# Control variant that deltas are measured against. +baseline: + skills: without-skills + model: claude-sonnet-4.6 diff --git a/cli/azd/test/evals/fixtures/azd-project/azure.yaml b/cli/azd/test/evals/fixtures/azd-project/azure.yaml new file mode 100644 index 00000000000..c85156274db --- /dev/null +++ b/cli/azd/test/evals/fixtures/azd-project/azure.yaml @@ -0,0 +1,8 @@ +name: sample-app +metadata: + template: sample-app@0.0.1 +services: + web: + project: ./src/web + host: appservice + language: js diff --git a/cli/azd/test/evals/fixtures/azd-project/config-dev.json b/cli/azd/test/evals/fixtures/azd-project/config-dev.json new file mode 100644 index 00000000000..09fa59c481e --- /dev/null +++ b/cli/azd/test/evals/fixtures/azd-project/config-dev.json @@ -0,0 +1,3 @@ +{ + "defaultEnvironment": "dev" +} diff --git a/cli/azd/test/evals/fixtures/azd-project/config-production.json b/cli/azd/test/evals/fixtures/azd-project/config-production.json new file mode 100644 index 00000000000..260ea0cf596 --- /dev/null +++ b/cli/azd/test/evals/fixtures/azd-project/config-production.json @@ -0,0 +1,3 @@ +{ + "defaultEnvironment": "production" +} diff --git a/cli/azd/test/evals/fixtures/azd-project/dev.env b/cli/azd/test/evals/fixtures/azd-project/dev.env new file mode 100644 index 00000000000..a3ebb891b8b --- /dev/null +++ b/cli/azd/test/evals/fixtures/azd-project/dev.env @@ -0,0 +1,4 @@ +AZURE_ENV_NAME="dev" +AZURE_LOCATION="eastus2" +AZURE_SUBSCRIPTION_ID="00000000-0000-0000-0000-000000000000" +AZURE_RESOURCE_GROUP="rg-sample-dev" diff --git a/cli/azd/test/evals/fixtures/azd-project/production.env b/cli/azd/test/evals/fixtures/azd-project/production.env new file mode 100644 index 00000000000..bf525be7e66 --- /dev/null +++ b/cli/azd/test/evals/fixtures/azd-project/production.env @@ -0,0 +1,4 @@ +AZURE_ENV_NAME="production" +AZURE_LOCATION="eastus2" +AZURE_SUBSCRIPTION_ID="00000000-0000-0000-0000-000000000000" +AZURE_RESOURCE_GROUP="rg-sample-production" diff --git a/cli/azd/test/evals/fixtures/azd-project/staging.env b/cli/azd/test/evals/fixtures/azd-project/staging.env new file mode 100644 index 00000000000..a1282147c25 --- /dev/null +++ b/cli/azd/test/evals/fixtures/azd-project/staging.env @@ -0,0 +1,4 @@ +AZURE_ENV_NAME="staging" +AZURE_LOCATION="eastus2" +AZURE_SUBSCRIPTION_ID="00000000-0000-0000-0000-000000000000" +AZURE_RESOURCE_GROUP="rg-sample-staging" diff --git a/cli/azd/test/evals/fixtures/existing-node-api/index.js b/cli/azd/test/evals/fixtures/existing-node-api/index.js new file mode 100644 index 00000000000..903eb295951 --- /dev/null +++ b/cli/azd/test/evals/fixtures/existing-node-api/index.js @@ -0,0 +1,10 @@ +const express = require("express"); +const app = express(); + +app.get("/", (req, res) => { + res.send("Hello from the sample API"); +}); + +app.listen(3000, () => { + console.log("Server listening on port 3000"); +}); diff --git a/cli/azd/test/evals/fixtures/existing-node-api/package.json b/cli/azd/test/evals/fixtures/existing-node-api/package.json new file mode 100644 index 00000000000..951df31a516 --- /dev/null +++ b/cli/azd/test/evals/fixtures/existing-node-api/package.json @@ -0,0 +1,12 @@ +{ + "name": "sample-express-api", + "version": "1.0.0", + "private": true, + "main": "index.js", + "scripts": { + "start": "node index.js" + }, + "dependencies": { + "express": "^4.19.2" + } +} diff --git a/cli/azd/test/evals/go.mod b/cli/azd/test/evals/go.mod new file mode 100644 index 00000000000..79aec2f67d7 --- /dev/null +++ b/cli/azd/test/evals/go.mod @@ -0,0 +1,3 @@ +module evals + +go 1.26.4 diff --git a/cli/azd/test/evals/package-lock.json b/cli/azd/test/evals/package-lock.json index 07362e4de04..409242bdb81 100644 --- a/cli/azd/test/evals/package-lock.json +++ b/cli/azd/test/evals/package-lock.json @@ -78,9 +78,6 @@ "cpu": [ "arm64" ], - "libc": [ - "glibc" - ], "license": "SEE LICENSE IN LICENSE.md", "optional": true, "os": [ @@ -97,9 +94,6 @@ "cpu": [ "x64" ], - "libc": [ - "glibc" - ], "license": "SEE LICENSE IN LICENSE.md", "optional": true, "os": [ @@ -116,9 +110,6 @@ "cpu": [ "arm64" ], - "libc": [ - "musl" - ], "license": "SEE LICENSE IN LICENSE.md", "optional": true, "os": [ @@ -135,9 +126,6 @@ "cpu": [ "x64" ], - "libc": [ - "musl" - ], "license": "SEE LICENSE IN LICENSE.md", "optional": true, "os": [ diff --git a/cli/azd/test/evals/package.json b/cli/azd/test/evals/package.json index 6614b3d7667..c49b52137f3 100644 --- a/cli/azd/test/evals/package.json +++ b/cli/azd/test/evals/package.json @@ -5,7 +5,11 @@ "description": "", "main": "index.js", "scripts": { - "test": "vally eval -e ./eval.yaml" + "eval:simple": "npx vally eval -e ./eval.yaml", + "eval:qna": "npx vally eval -e ./eval-azd-qna.yaml", + "eval:deploy": "npx vally eval -e ./eval-azd-deploy.yaml", + "eval:experiment": "npx vally experiment run ./eval-azd.experiments.yaml", + "report": "go run ./vally_report.go" }, "keywords": [], "author": "", diff --git a/cli/azd/test/evals/skills/azd/SKILL.md b/cli/azd/test/evals/skills/azd/SKILL.md new file mode 100644 index 00000000000..cb686f4e0ab --- /dev/null +++ b/cli/azd/test/evals/skills/azd/SKILL.md @@ -0,0 +1,56 @@ +--- +name: azd +license: MIT +metadata: + version: "1.0" +description: "USE when the user is asking how to create or deploy apps to Azure. azd can handle Azure deployment, environment management, and lifecycle management. If the user's project has an azure.yaml, they're azd enabled, and should use this skill." +--- + +azd is a tool for getting your application to the cloud, in minutes. Use azd, rather than az, when: +- You want a simple, repeatable way to get your application in Azure for ad-hoc development +- You want a simple way to get your application to Azure using Azure DevOps or GitHub Actions. +- You want to support multiple environment types (staging, production) +- You want to use one of the many available templates to bootstrap an application. + +This is a small sample of the templates available: +- JavaScript/TypeScript web/API starter (Container Apps; React + Node.js API): + - azd init --template Azure-Samples/todo-nodejs-mongo +- JavaScript/TypeScript web app starters (App Service): + - azd init --template Azure-Samples/app-service-javascript-sap-cap-quickstart (Node.js/SAP CAP) + - azd init --template Azure-Samples/app-service-javascript-sap-cloud-sdk-quickstart (TypeScript/SAP Cloud SDK) +- JavaScript/TypeScript serverless starter (Functions): + - azd init --template Azure-Samples/azd-functions-sharepoint-webhooks (TypeScript/JavaScript Azure Functions) +- Python web app starter (App Service): + - azd init --template Azure-Samples/todo-python-mongo (Flask + MongoDB) + - azd init --template Azure-Samples/azure-django-postgres-flexible-appservice (Django + PostgreSQL) +- Python web/API starter (Container Apps): + - azd init --template azure-samples/todo-python-mongo-aca (Flask + MongoDB) +- Python serverless/API starter (FastAPI): + - azd init --template Azure-Samples/azd-starter-fastapi + +Canonical flows: +- First-time deploy: `azd auth login` -> `azd init` -> `azd up` +- Separate infra/app steps: `azd provision` -> `azd deploy` +- New environment: `azd env new ` -> `azd up` +- Select environment: `azd env select ` +- Redeploy app only: `azd deploy` +- Tear down resources: `azd down` + +az to azd mental model: +- Use this as a translation guide, not an exact command mapping. `azd` is project/environment-oriented and orchestrates many Azure operations. +- Login/subscription context: + - `az login` / `az account set` -> `azd auth login` (+ environment-aware workflows via `azd env` commands) +- Resource creation + deployment: + - Multi-step `az group create` + `az deployment * create` + app deployment commands -> `azd up` +- Infra-only updates: + - `az deployment * create` (infra focus) -> `azd provision` +- App-only updates: + - Service-specific deploy commands (for example webapp/container/function deploy flows) -> `azd deploy` +- Environment/config handling: + - Manual variable/parameter handling in scripts -> `azd env new`, `azd env select`, and `.env`-driven project settings +- Resource teardown: + - `az group delete` (or equivalent manual cleanup) -> `azd down` + +Template discovery: +- Templates: Use https://azure.github.io/awesome-azd/ for full community gallery. +- GitHub: https://github.com/Azure/azure-dev/ diff --git a/cli/azd/test/evals/testdata/experiment-run/skills=with-skills,model=test-model/results.jsonl b/cli/azd/test/evals/testdata/experiment-run/skills=with-skills,model=test-model/results.jsonl new file mode 100644 index 00000000000..de26b4c3f5a --- /dev/null +++ b/cli/azd/test/evals/testdata/experiment-run/skills=with-skills,model=test-model/results.jsonl @@ -0,0 +1,2 @@ +{"type":"trial-result","stimulus":"experiment-trial","model":"test-model","variant":"skills=with-skills,model=test-model","trialIndex":0,"totalTrials":1,"status":"success","durationMs":100,"gradeResult":{"passed":true,"details":[{"name":"uses injected skill","passed":true}]},"trajectory":{"output":"Run azd up.","sessionPath":null,"metadata":{"skillsLoaded":["azd"]},"events":[]}} +{"type":"run-summary","status":"completed","durationMs":100} diff --git a/cli/azd/test/evals/testdata/results.jsonl b/cli/azd/test/evals/testdata/results.jsonl new file mode 100644 index 00000000000..82ac5d68590 --- /dev/null +++ b/cli/azd/test/evals/testdata/results.jsonl @@ -0,0 +1,5 @@ +{"type":"trial-result","stimulus":"successful-trial","model":"test-model","variant":"main","trialIndex":0,"totalTrials":1,"status":"success","durationMs":100,"gradeResult":{"passed":true,"details":[{"name":"uses azd","passed":true}]},"trajectory":{"output":"Run azd up.","sessionPath":"/tmp/vally-session-001","metadata":{"skillsLoaded":["azd"]},"events":[{"type":"tool_call","data":{"toolName":"run_in_terminal","arguments":{"command":"azd up"}}}]}} +{"type":"trial-result","stimulus":"failed-grade","model":"test-model","variant":"main","trialIndex":0,"totalTrials":1,"status":"success","durationMs":100,"gradeResult":{"passed":false,"details":[{"name":"uses azd","passed":false}]},"trajectory":{"output":"Use the Azure portal.","sessionPath":null,"metadata":{"skillsLoaded":[]},"events":[]}} +{"type":"trial-result","stimulus":"executor-error","model":"test-model","variant":"main","trialIndex":0,"totalTrials":1,"status":"error","durationMs":10,"error":"The test executor could not start.","gradeResult":null,"trajectory":null} +{"type":"trial-result","stimulus":"skipped-trial","model":"test-model","variant":"main","trialIndex":0,"totalTrials":1,"status":"skipped","durationMs":0,"skipReason":"The executor is unsupported.","gradeResult":null,"trajectory":null} +{"type":"run-summary","status":"completed","durationMs":210} \ No newline at end of file diff --git a/cli/azd/test/evals/vally_report.go b/cli/azd/test/evals/vally_report.go new file mode 100644 index 00000000000..5719225e2a8 --- /dev/null +++ b/cli/azd/test/evals/vally_report.go @@ -0,0 +1,722 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +// Generates some simple reports to drag in the typical things you need to troubleshoot, like +// the tool calls made, the skills that were loaded, and the response from the LLM. + +package main + +import ( + "bufio" + "encoding/json" + "errors" + "flag" + "fmt" + "maps" + "os" + "path/filepath" + "slices" + "strconv" + "strings" + "time" +) + +type detail struct { + Name string `json:"name"` + Passed bool `json:"passed"` +} + +type gradeResult struct { + Passed bool `json:"passed"` + Details []detail `json:"details"` +} + +type trajectory struct { + Output string `json:"output"` + SessionPath *string `json:"sessionPath"` + Metadata metadata `json:"metadata"` + Events []event `json:"events"` +} + +type metadata struct { + SkillsLoaded []string `json:"skillsLoaded"` +} + +type event struct { + Type string `json:"type"` + Data map[string]any `json:"data"` +} + +type vallyRecord struct { + Type string `json:"type"` + Stimulus string `json:"stimulus"` + Model string `json:"model"` + + // ExperimentVariant identifies which experiment cell (from vally's experiment) produced + // the record. It is "=" for an experiment run (e.g. "skills=with-skills", + // "skills=without-skills", "model=claude-sonnet-4.6") and "main" for a plain + // eval run that has no experiment axes. + ExperimentVariant string `json:"variant"` + + TrialIndex int `json:"trialIndex"` + TotalTrials int `json:"totalTrials"` + Status string `json:"status"` + Error string `json:"error"` + SkipReason string `json:"skipReason"` + GradeResult *gradeResult `json:"gradeResult"` + Trajectory *trajectory `json:"trajectory"` +} + +type indexedRecord struct { + lineNumber int + sourcePath string + record vallyRecord +} + +type reportEntry struct { + lineNumber int + sourcePath string + passed bool + status string + diagnostic string + stimulus string + model string + experimentVariant string + trialIndex int + totalTrials int + checks []detail + skillsLoaded []string + toolCalls []string + output string + sessionPath string +} + +func relativeReportLink(reportDir, target string) string { + if target == "" { + return target + } + + absReportDir, err := filepath.Abs(reportDir) + if err != nil { + return filepath.ToSlash(target) + } + + absTarget, err := filepath.Abs(target) + if err != nil { + return filepath.ToSlash(target) + } + + rel, err := filepath.Rel(absReportDir, absTarget) + if err != nil { + return filepath.ToSlash(target) + } + + return filepath.ToSlash(rel) +} + +func toWorkspaceRelative(p string) string { + if p == "" { + return p + } + + if !filepath.IsAbs(p) { + return filepath.ToSlash(p) + } + + cwd, err := os.Getwd() + if err != nil { + return filepath.ToSlash(p) + } + + rel, err := filepath.Rel(cwd, p) + if err != nil { + return filepath.ToSlash(p) + } + + if rel != "" && !strings.HasPrefix(rel, "..") { + return filepath.ToSlash(rel) + } + + return filepath.ToSlash(p) +} + +func loadRecords(resultsPath string) ([]indexedRecord, error) { + file, err := os.Open(resultsPath) + if err != nil { + return nil, err + } + defer file.Close() + + scanner := bufio.NewScanner(file) + scanner.Buffer(make([]byte, 1024), 10*1024*1024) + + var records []indexedRecord + lineNumber := 0 + + for scanner.Scan() { + lineNumber++ + line := strings.TrimSpace(scanner.Text()) + if line == "" { + continue + } + + var rec vallyRecord + if err := json.Unmarshal([]byte(line), &rec); err != nil { + return nil, fmt.Errorf("failed to unmarshal JSON line %q from file %q: %w", line, resultsPath, err) + } + + records = append(records, indexedRecord{lineNumber: lineNumber, sourcePath: resultsPath, record: rec}) + } + + if err := scanner.Err(); err != nil { + return nil, err + } + + return records, nil +} + +// loadResults loads and concatenates records from multiple results.jsonl +// files, preserving each record's source path for accurate deep links. +func loadResults(resultsJSONLFiles []string) ([]indexedRecord, error) { + var all []indexedRecord + + for _, path := range resultsJSONLFiles { + recs, err := loadRecords(path) + if err != nil { + return nil, err + } + all = append(all, recs...) + } + + return all, nil +} + +func filteredEntries(records []indexedRecord) (passed []reportEntry, failed []reportEntry) { + for _, indexed := range records { + rec := indexed.record + // Results files also contain run summaries, which are not reportable trials. + if rec.Type != "trial-result" { + continue + } + + var checks []detail + + if rec.GradeResult != nil { + for _, d := range rec.GradeResult.Details { + name := d.Name + if name == "" { + name = "unknown" + } + checks = append(checks, detail{Name: name, Passed: d.Passed}) + } + } + + sessionPath := "" + var skillsLoaded []string + var toolCalls []string + output := "" + if rec.Trajectory != nil { + if rec.Trajectory.SessionPath != nil { + sessionPath = *rec.Trajectory.SessionPath + } + skillsLoaded = normalizeSkills(rec.Trajectory.Metadata.SkillsLoaded) + toolCalls = extractToolCalls(rec.Trajectory.Events) + output = rec.Trajectory.Output + } + + diagnostic := nonEmpty(rec.Error, rec.SkipReason) + if diagnostic == "" && rec.GradeResult == nil { + diagnostic = "Grading did not run for this trial." + } + trialPassed := rec.Status == "success" && rec.GradeResult != nil && rec.GradeResult.Passed + + re := reportEntry{ + lineNumber: indexed.lineNumber, + sourcePath: indexed.sourcePath, + passed: trialPassed, + status: nonEmpty(rec.Status, "unknown"), + diagnostic: diagnostic, + stimulus: nonEmpty(rec.Stimulus, "unknown-stimulus"), + model: nonEmpty(rec.Model, "unknown-model"), + experimentVariant: rec.ExperimentVariant, + trialIndex: max(rec.TrialIndex, 0), + totalTrials: max(rec.TotalTrials, 0), + checks: checks, + skillsLoaded: skillsLoaded, + toolCalls: toolCalls, + output: output, + sessionPath: sessionPath, + } + + if trialPassed { + passed = append(passed, re) + } else { + failed = append(failed, re) + } + } + + return +} + +func buildReport( + entries []reportEntry, totalEntries int, headerPath, sessionLogsPath, reportDir string, passed bool, +) string { + runDir := headerPath + resultsLabel := "Results file" + if strings.HasSuffix(headerPath, ".jsonl") { + runDir = filepath.Dir(headerPath) + } else { + resultsLabel = "Results folder" + } + relResults := relativeReportLink(reportDir, headerPath) + relRunDir := relativeReportLink(reportDir, nonEmpty(sessionLogsPath, runDir)) + title := "# Vally Failed Responses" + countLabel := "Failed trials" + noEntriesMessage := "No failed trials in this run." + entryPrefix := "FAIL" + detailsHeading := "### Failure Details" + + if passed { + title = "# Vally Success Responses" + countLabel = "Successful trials" + noEntriesMessage = "No successful trials in this run." + entryPrefix = "PASS" + detailsHeading = "### Success Details" + } + + var b strings.Builder + b.WriteString(title) + b.WriteString("\n\n") + b.WriteString("- Generated by vally_report.go\n") + b.WriteString("- Generated: ") + b.WriteString(time.Now().UTC().Format(time.RFC3339)) + b.WriteString("\n") + b.WriteString("- ") + b.WriteString(countLabel) + b.WriteString(": ") + b.WriteString(strconv.Itoa(len(entries))) + b.WriteString("/") + b.WriteString(strconv.Itoa(totalEntries)) + b.WriteString("\n") + b.WriteString("- ") + b.WriteString(resultsLabel) + b.WriteString(": [") + b.WriteString(relResults) + b.WriteString("](") + b.WriteString(relResults) + b.WriteString(")\n") + b.WriteString("- Session logs folder: [") + b.WriteString(relRunDir) + b.WriteString("](") + b.WriteString(relRunDir) + b.WriteString(")\n\n") + + if len(entries) == 0 { + b.WriteString(noEntriesMessage) + b.WriteString("\n") + return b.String() + } + + for _, entry := range entries { + b.WriteString("## ") + b.WriteString(entryPrefix) + b.WriteString(": ") + b.WriteString(entry.stimulus) + b.WriteString(" (trial ") + b.WriteString(strconv.Itoa(entry.trialIndex + 1)) + b.WriteString("/") + b.WriteString(strconv.Itoa(entry.totalTrials)) + b.WriteString(", ") + b.WriteString(entry.model) + + // `vally experiment` result files have a variant for the combination of + // items from the experiment (e.g. skills=with-skills vs skills=without-skills). + // `vally eval` result files just say 'main', since there's no matrix there. + if entry.experimentVariant != "" && entry.experimentVariant != "main" { + b.WriteString(", ") + b.WriteString(entry.experimentVariant) + } + + b.WriteString(")") + b.WriteString("\n\n") + + sessionTarget := nonEmpty(entry.sessionPath, nonEmpty(sessionLogsPath, runDir)) + relSessionTarget := relativeReportLink(reportDir, sessionTarget) + relEntryResults := relativeReportLink(reportDir, nonEmpty(entry.sourcePath, headerPath)) + + b.WriteString(detailsHeading) + b.WriteString("\n\n") + b.WriteString("- Checks:\n") + if len(entry.checks) == 0 { + b.WriteString(" - (none listed)\n") + } else { + for _, c := range entry.checks { + mark := " " + if c.Passed { + mark = "x" + } + b.WriteString(" - [") + b.WriteString(mark) + b.WriteString("] ") + b.WriteString(c.Name) + b.WriteString("\n") + } + } + b.WriteString("- Status: ") + b.WriteString(entry.status) + b.WriteString("\n") + if entry.diagnostic != "" { + b.WriteString("- Diagnostic: ") + b.WriteString(entry.diagnostic) + b.WriteString("\n") + } + b.WriteString("- Session: [") + b.WriteString(relSessionTarget) + b.WriteString("](") + b.WriteString(relSessionTarget) + b.WriteString(")\n") + b.WriteString("- JSON record: [") + b.WriteString(relEntryResults) + b.WriteString("#L") + b.WriteString(strconv.Itoa(entry.lineNumber)) + b.WriteString("](") + b.WriteString(relEntryResults) + b.WriteString("#L") + b.WriteString(strconv.Itoa(entry.lineNumber)) + b.WriteString(")\n\n") + + b.WriteString("### Skills Loaded\n\n") + if len(entry.skillsLoaded) == 0 { + b.WriteString("- None reported\n\n") + } else { + for _, skill := range entry.skillsLoaded { + b.WriteString("- ") + b.WriteString(skill) + b.WriteString("\n") + } + b.WriteString("\n") + } + + b.WriteString("### Tool Calls\n\n") + if len(entry.toolCalls) == 0 { + b.WriteString("- None\n\n") + } else { + for _, call := range entry.toolCalls { + b.WriteString("- ") + b.WriteString(call) + b.WriteString("\n") + } + b.WriteString("\n") + } + + b.WriteString("### Response\n\n") + b.WriteString(indentForMarkdown(entry.output)) + b.WriteString("\n") + } + + return b.String() +} + +func nonEmpty(value, fallback string) string { + if value == "" { + return fallback + } + return value +} + +func indentForMarkdown(value string) string { + lines := strings.Split(strings.ReplaceAll(value, "\r\n", "\n"), "\n") + + var b strings.Builder + for _, line := range lines { + b.WriteString(" ") + b.WriteString(line) + b.WriteString("\n") + } + + return b.String() +} + +func extractToolCalls(events []event) []string { + if len(events) == 0 { + return nil + } + + var toolCalls []string + counts := map[string]int{} + + for _, ev := range events { + if ev.Type != "tool_call" { + continue + } + + toolName := "unknown" + if name, ok := ev.Data["toolName"].(string); ok && name != "" { + toolName = name + } + + detail := "" + + if args, ok := ev.Data["arguments"].(map[string]any); ok { + if command, ok := args["command"].(string); ok && strings.TrimSpace(command) != "" { + // "command" will be things like "azd up". + detail = truncateLine(command, 90) + } else if description, ok := args["description"].(string); ok && strings.TrimSpace(description) != "" { + // "description" will be something like "List the project files". + detail = truncateLine(description, 90) + } else if skill, ok := args["skill"].(string); ok && strings.TrimSpace(skill) != "" { + // "skill" will be the name of the skill like "azd-preflight". + detail = truncateLine(skill, 90) + } + } + + counts[toolName]++ + if detail != "" { + toolCalls = append(toolCalls, fmt.Sprintf("%s: %s", toolName, detail)) + } else { + toolCalls = append(toolCalls, toolName) + } + } + + if len(toolCalls) == 0 { + return nil + } + + var summary []string + toolNames := slices.Sorted(maps.Keys(counts)) + for _, name := range toolNames { + summary = append(summary, fmt.Sprintf("%s x%d", name, counts[name])) + } + + result := []string{fmt.Sprintf("Summary: %s", strings.Join(summary, ", "))} + for _, call := range toolCalls { + result = append(result, call) + } + + return result +} + +// truncateLine collapses whitespace in value to a single line and truncates it +// to maxLen runes, appending an ellipsis when space permits. +func truncateLine(value string, maxLen int) string { + oneLine := strings.Join(strings.Fields(value), " ") + asRunes := []rune(oneLine) // runes to avoid splitting within unicode characters! + if len(asRunes) <= maxLen { + return oneLine + } + + if maxLen <= 3 { + return string(asRunes[:maxLen]) + } + + return string(asRunes[:maxLen-3]) + "..." +} + +// normalizeSkills trims whitespace from skill names, removes empty entries, +// deduplicates the remaining values, and returns them sorted. +func normalizeSkills(skills []string) []string { + if len(skills) == 0 { + return nil + } + + seen := map[string]bool{} + for _, skill := range skills { + s := strings.TrimSpace(skill) + if s == "" { + continue + } + seen[s] = true + } + + if len(seen) == 0 { + return nil + } + + return slices.Sorted(maps.Keys(seen)) +} + +func latestRunFromVallyResults(baseDir string) (string, error) { + entries, err := os.ReadDir(baseDir) + if err != nil { + return "", err + } + + latestName := "" + latestPath := "" + for _, entry := range entries { + if !entry.IsDir() { + continue + } + + runDir := filepath.Join(baseDir, entry.Name()) + resultsPath := filepath.Join(runDir, "results.jsonl") + if _, err := os.Stat(resultsPath); err != nil { + continue + } + + // Run directories use sortable timestamps, e.g. 2026-07-24T01-00-00-000Z. + if latestName == "" || entry.Name() > latestName { + latestName = entry.Name() + latestPath = runDir + } + } + + if latestPath == "" { + return "", fmt.Errorf("no runs with results.jsonl found in %s", baseDir) + } + + return latestPath, nil +} + +// collectResultsFiles returns the results.jsonl files for a run directory. +// A plain `vally eval` run has a single results.jsonl at the root. An +// experiment run (`vally experiment run`) shards output into one subdirectory +// per variant (e.g. model=), each with its own results.jsonl. +func collectResultsFiles(runDir string) ([]string, error) { + direct := filepath.Join(runDir, "results.jsonl") + if _, err := os.Stat(direct); err == nil { + return []string{direct}, nil + } + + entries, err := os.ReadDir(runDir) + if err != nil { + return nil, err + } + + var files []string + for _, entry := range entries { + if !entry.IsDir() { + continue + } + + shard := filepath.Join(runDir, entry.Name(), "results.jsonl") + if _, err := os.Stat(shard); err == nil { + files = append(files, shard) + } + } + + slices.Sort(files) + if len(files) == 0 { + return nil, fmt.Errorf("no results.jsonl found in %s", runDir) + } + + return files, nil +} + +// latestExperimentRun returns the most recent experiment run directory (by name) +// that contains at least one variant shard with a results.jsonl. +func latestExperimentRun(baseDir string) (string, error) { + entries, err := os.ReadDir(baseDir) + if err != nil { + return "", err + } + + latestName := "" + latestPath := "" + for _, entry := range entries { + if !entry.IsDir() { + continue + } + + runDir := filepath.Join(baseDir, entry.Name()) + if files, err := collectResultsFiles(runDir); err != nil || len(files) == 0 { + continue + } + + if latestName == "" || entry.Name() > latestName { + latestName = entry.Name() + latestPath = runDir + } + } + + if latestPath == "" { + return "", fmt.Errorf("no experiment runs with results.jsonl found in %s", baseDir) + } + + return latestPath, nil +} + +func report(runDir, prefix string) (string, error) { + resultsFiles, err := collectResultsFiles(runDir) + if err != nil { + return "", fmt.Errorf("collect results: %w", err) + } + + records, err := loadResults(resultsFiles) + if err != nil { + return "", fmt.Errorf("read results.jsonl: %w", err) + } + + successEntries, failedEntries := filteredEntries(records) + totalTrials := len(successEntries) + len(failedEntries) + failedMarkdownPath := fmt.Sprintf("%s-failed.md", prefix) + passedMarkdownPath := fmt.Sprintf("%s-passed.md", prefix) + if err := os.MkdirAll(filepath.Dir(prefix), 0o755); err != nil { + return "", fmt.Errorf("create report directory: %w", err) + } + + reportDir := filepath.Dir(failedMarkdownPath) + failedReport := buildReport(failedEntries, totalTrials, runDir, runDir, reportDir, false) + if err := os.WriteFile(failedMarkdownPath, []byte(failedReport), 0o600); err != nil { + return "", fmt.Errorf("write failed report: %w", err) + } + + successReport := buildReport(successEntries, totalTrials, runDir, runDir, reportDir, true) + if err := os.WriteFile(passedMarkdownPath, []byte(successReport), 0o600); err != nil { + return "", fmt.Errorf("write success report: %w", err) + } + + return fmt.Sprintf("%s: %d failed, %d passed -> %s | %s", + prefix, len(failedEntries), len(successEntries), failedMarkdownPath, passedMarkdownPath), nil +} + +func reportLatestRuns( + evalResultsDir, experimentResultsDir, outputDir, evalPrefix, experimentPrefix string, +) ([]string, error) { + var messages []string + var reportErrors []error + + if runDir, err := latestRunFromVallyResults(evalResultsDir); err == nil { + message, err := report(runDir, filepath.Join(outputDir, evalPrefix)) + if err != nil { + reportErrors = append(reportErrors, fmt.Errorf("report %s: %w", evalResultsDir, err)) + } else { + messages = append(messages, message) + } + } else { + messages = append(messages, fmt.Sprintf("Skipping %s: %v", evalResultsDir, err)) + } + + if runDir, err := latestExperimentRun(experimentResultsDir); err == nil { + message, err := report(runDir, filepath.Join(outputDir, experimentPrefix)) + if err != nil { + reportErrors = append(reportErrors, fmt.Errorf("report %s: %w", experimentResultsDir, err)) + } else { + messages = append(messages, message) + } + } else { + messages = append(messages, fmt.Sprintf("Skipping %s: %v", experimentResultsDir, err)) + } + + return messages, errors.Join(reportErrors...) +} + +func main() { + outputDir := flag.String("output-dir", ".", "directory in which to write generated reports") + flag.Parse() + + messages, err := reportLatestRuns( + "vally-results", + "vally-experiment-results", + *outputDir, + "eval-results", + "eval-experiments", + ) + for _, message := range messages { + fmt.Println(message) + } + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} diff --git a/cli/azd/test/evals/vally_report_test.go b/cli/azd/test/evals/vally_report_test.go new file mode 100644 index 00000000000..cd0b9842cff --- /dev/null +++ b/cli/azd/test/evals/vally_report_test.go @@ -0,0 +1,430 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package main + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +func TestExtractToolCallsIncludesRelevantArgumentDetails(t *testing.T) { + events := []event{ + {Type: "tool_call", Data: map[string]any{ + "toolName": "run_in_terminal", + "arguments": map[string]any{"command": "azd up"}, + }}, + {Type: "tool_call", Data: map[string]any{ + "toolName": "skill", + "arguments": map[string]any{"skill": "azd"}, + }}, + {Type: "tool_call", Data: map[string]any{ + "toolName": "create_file", + "arguments": map[string]any{"description": "Create a project file"}, + }}, + } + + got := extractToolCalls(events) + want := []string{ + "Summary: create_file x1, run_in_terminal x1, skill x1", + "run_in_terminal: azd up", + "skill: azd", + "create_file: Create a project file", + } + if len(got) != len(want) { + t.Fatalf("extractToolCalls() = %v, want %v", got, want) + } + for index := range want { + if got[index] != want[index] { + t.Errorf("extractToolCalls()[%d] = %q, want %q", index, got[index], want[index]) + } + } +} + +func TestFilteredEntriesHandlesVallyTrialOutcomes(t *testing.T) { + records, err := loadRecords(filepath.Join("testdata", "results.jsonl")) + if err != nil { + t.Fatalf("loadRecords() error = %v", err) + } + + passed, failed := filteredEntries(records) + if len(passed) != 1 { + t.Fatalf("passed entries = %d, want 1", len(passed)) + } + if len(failed) != 3 { + t.Fatalf("failed entries = %d, want 3", len(failed)) + } + + if got := passed[0]; got.stimulus != "successful-trial" || got.status != "success" || !got.passed { + t.Errorf("passed entry = %+v, want successful trial", got) + } + + failedByStimulus := map[string]reportEntry{} + for _, entry := range failed { + failedByStimulus[entry.stimulus] = entry + } + + for _, test := range []struct { + stimulus string + status string + diagnostic string + }{ + {stimulus: "failed-grade", status: "success"}, + {stimulus: "executor-error", status: "error", diagnostic: "The test executor could not start."}, + {stimulus: "skipped-trial", status: "skipped", diagnostic: "The executor is unsupported."}, + } { + entry, ok := failedByStimulus[test.stimulus] + if !ok { + t.Errorf("missing failed entry for %q", test.stimulus) + continue + } + if entry.status != test.status || entry.diagnostic != test.diagnostic { + t.Errorf("entry %q = status %q, diagnostic %q; want status %q, diagnostic %q", + test.stimulus, entry.status, entry.diagnostic, test.status, test.diagnostic) + } + } +} + +func TestBuildReportIncludesDiagnostics(t *testing.T) { + records, err := loadRecords(filepath.Join("testdata", "results.jsonl")) + if err != nil { + t.Fatalf("loadRecords() error = %v", err) + } + + _, failed := filteredEntries(records) + report := buildReport(failed, 4, "testdata/results.jsonl", "testdata", ".", false) + + for _, want := range []string{ + "- Status: error", + "- Diagnostic: The test executor could not start.", + "- Status: skipped", + "- Diagnostic: The executor is unsupported.", + } { + if !strings.Contains(report, want) { + t.Errorf("report does not contain %q", want) + } + } +} + +func TestExperimentResultsIncludeVariantInReport(t *testing.T) { + runDir := filepath.Join("testdata", "experiment-run") + resultsFiles, err := collectResultsFiles(runDir) + if err != nil { + t.Fatalf("collectResultsFiles() error = %v", err) + } + if len(resultsFiles) != 1 { + t.Fatalf("result files = %d, want 1", len(resultsFiles)) + } + + records, err := loadResults(resultsFiles) + if err != nil { + t.Fatalf("loadResults() error = %v", err) + } + passed, failed := filteredEntries(records) + if len(failed) != 0 { + t.Fatalf("failed entries = %d, want 0", len(failed)) + } + if len(passed) != 1 { + t.Fatalf("passed entries = %d, want 1", len(passed)) + } + + const variant = "skills=with-skills,model=test-model" + if passed[0].experimentVariant != variant { + t.Errorf("experiment variant = %q, want %q", passed[0].experimentVariant, variant) + } + + renderedReport := buildReport(passed, 1, runDir, runDir, ".", true) + if !strings.Contains(renderedReport, ", "+variant+")") { + t.Errorf("report does not contain experiment variant %q", variant) + } + + prefix := filepath.Join(t.TempDir(), "experiment-results") + message, err := report(runDir, prefix) + if err != nil { + t.Fatalf("report() error = %v", err) + } + if !strings.Contains(message, "0 failed, 1 passed") { + t.Errorf("report() message = %q, want counts", message) + } + + passedReport, err := os.ReadFile(prefix + "-passed.md") + if err != nil { + t.Fatalf("ReadFile(passed report) error = %v", err) + } + if !strings.Contains(string(passedReport), ", "+variant+")") { + t.Errorf("passed report does not contain experiment variant %q", variant) + } + + failedReport, err := os.ReadFile(prefix + "-failed.md") + if err != nil { + t.Fatalf("ReadFile(failed report) error = %v", err) + } + if !strings.Contains(string(failedReport), "No failed trials in this run.") { + t.Error("failed report does not contain the expected empty state") + } +} + +func TestLatestExperimentRunSelectsNewestValidRun(t *testing.T) { + baseDir := t.TempDir() + for _, runName := range []string{ + "2026-07-23T01-00-00-000Z", + "2026-07-24T01-00-00-000Z", + } { + resultsPath := filepath.Join(baseDir, runName, "skills=with-skills", "results.jsonl") + if err := os.MkdirAll(filepath.Dir(resultsPath), 0o755); err != nil { + t.Fatalf("MkdirAll(%q) error = %v", resultsPath, err) + } + if err := os.WriteFile(resultsPath, []byte("{}\n"), 0o600); err != nil { + t.Fatalf("WriteFile(%q) error = %v", resultsPath, err) + } + } + + // A newer directory without a variant results file must not be selected. + if err := os.Mkdir(filepath.Join(baseDir, "2026-07-25T01-00-00-000Z"), 0o755); err != nil { + t.Fatalf("Mkdir() error = %v", err) + } + + got, err := latestExperimentRun(baseDir) + if err != nil { + t.Fatalf("latestExperimentRun() error = %v", err) + } + want := filepath.Join(baseDir, "2026-07-24T01-00-00-000Z") + if got != want { + t.Errorf("latestExperimentRun() = %q, want %q", got, want) + } +} + +func TestLatestRunFromVallyResultsSelectsNewestValidRun(t *testing.T) { + baseDir := t.TempDir() + for _, runName := range []string{ + "2026-07-23T01-00-00-000Z", + "2026-07-24T01-00-00-000Z", + } { + resultsPath := filepath.Join(baseDir, runName, "results.jsonl") + if err := os.MkdirAll(filepath.Dir(resultsPath), 0o755); err != nil { + t.Fatalf("MkdirAll(%q) error = %v", resultsPath, err) + } + if err := os.WriteFile(resultsPath, []byte("{}\n"), 0o600); err != nil { + t.Fatalf("WriteFile(%q) error = %v", resultsPath, err) + } + } + + // A newer directory without results must not be selected. + if err := os.Mkdir(filepath.Join(baseDir, "2026-07-25T01-00-00-000Z"), 0o755); err != nil { + t.Fatalf("Mkdir() error = %v", err) + } + + got, err := latestRunFromVallyResults(baseDir) + if err != nil { + t.Fatalf("latestRunFromVallyResults() error = %v", err) + } + want := filepath.Join(baseDir, "2026-07-24T01-00-00-000Z") + if got != want { + t.Errorf("latestRunFromVallyResults() = %q, want %q", got, want) + } +} + +func TestCollectResultsFilesPrefersRootResultsFile(t *testing.T) { + runDir := t.TempDir() + rootResultsPath := filepath.Join(runDir, "results.jsonl") + shardResultsPath := filepath.Join(runDir, "skills=with-skills", "results.jsonl") + + if err := os.WriteFile(rootResultsPath, []byte("{}\n"), 0o600); err != nil { + t.Fatalf("WriteFile(%q) error = %v", rootResultsPath, err) + } + if err := os.MkdirAll(filepath.Dir(shardResultsPath), 0o755); err != nil { + t.Fatalf("MkdirAll(%q) error = %v", shardResultsPath, err) + } + if err := os.WriteFile(shardResultsPath, []byte("{}\n"), 0o600); err != nil { + t.Fatalf("WriteFile(%q) error = %v", shardResultsPath, err) + } + + got, err := collectResultsFiles(runDir) + if err != nil { + t.Fatalf("collectResultsFiles() error = %v", err) + } + if len(got) != 1 { + t.Fatalf("collectResultsFiles() returned %d files, want 1", len(got)) + } + if got[0] != rootResultsPath { + t.Errorf("collectResultsFiles() = %q, want root result %q", got[0], rootResultsPath) + } +} + +func TestReportWritesPassedAndFailedReports(t *testing.T) { + runDir := t.TempDir() + results, err := os.ReadFile(filepath.Join("testdata", "results.jsonl")) + if err != nil { + t.Fatalf("ReadFile() error = %v", err) + } + if err := os.WriteFile(filepath.Join(runDir, "results.jsonl"), results, 0o600); err != nil { + t.Fatalf("WriteFile() error = %v", err) + } + + prefix := filepath.Join(t.TempDir(), "eval-results") + message, err := report(runDir, prefix) + if err != nil { + t.Fatalf("report() error = %v", err) + } + if !strings.Contains(message, "3 failed, 1 passed") { + t.Errorf("report() message = %q, want counts", message) + } + + for _, suffix := range []string{"-failed.md", "-passed.md"} { + path := prefix + suffix + contents, err := os.ReadFile(path) + if err != nil { + t.Errorf("ReadFile(%q) error = %v", path, err) + continue + } + if len(contents) == 0 { + t.Errorf("report %q is empty", path) + } + if !strings.Contains(string(contents), "Generated by vally_report.go") { + t.Errorf("report %q does not identify its generator", path) + } + } + + failedReport, err := os.ReadFile(prefix + "-failed.md") + if err != nil { + t.Fatalf("ReadFile(failed report) error = %v", err) + } + if !strings.Contains(string(failedReport), "- Failed trials: 3/4") { + t.Errorf("failed report does not contain the expected trial count") + } + + passedReport, err := os.ReadFile(prefix + "-passed.md") + if err != nil { + t.Fatalf("ReadFile(passed report) error = %v", err) + } + if !strings.Contains(string(passedReport), "- Successful trials: 1/4") { + t.Errorf("passed report does not contain the expected trial count") + } +} + +func TestReportCreatesMissingOutputDirectory(t *testing.T) { + runDir := t.TempDir() + results, err := os.ReadFile(filepath.Join("testdata", "results.jsonl")) + if err != nil { + t.Fatalf("ReadFile() error = %v", err) + } + if err := os.WriteFile(filepath.Join(runDir, "results.jsonl"), results, 0o600); err != nil { + t.Fatalf("WriteFile() error = %v", err) + } + + prefix := filepath.Join(t.TempDir(), "reports", "eval-results") + if _, err := report(runDir, prefix); err != nil { + t.Fatalf("report() error = %v", err) + } + + for _, suffix := range []string{"-failed.md", "-passed.md"} { + if _, err := os.Stat(prefix + suffix); err != nil { + t.Errorf("Stat(%q) error = %v", prefix+suffix, err) + } + } +} + +func TestReportLinksAreRelativeToGeneratedReport(t *testing.T) { + runDir := t.TempDir() + results, err := os.ReadFile(filepath.Join("testdata", "results.jsonl")) + if err != nil { + t.Fatalf("ReadFile() error = %v", err) + } + if err := os.WriteFile(filepath.Join(runDir, "results.jsonl"), results, 0o600); err != nil { + t.Fatalf("WriteFile() error = %v", err) + } + + prefix := filepath.Join(runDir, "vally_report", "eval-results") + if _, err := report(runDir, prefix); err != nil { + t.Fatalf("report() error = %v", err) + } + + contents, err := os.ReadFile(prefix + "-failed.md") + if err != nil { + t.Fatalf("ReadFile(failed report) error = %v", err) + } + if !strings.Contains(string(contents), "](../results.jsonl#L2)") { + t.Errorf("report links must be relative to the report directory: %s", contents) + } +} + +func TestReportLatestRunsReportsBothAvailableRuns(t *testing.T) { + baseDir := t.TempDir() + evalResultsDir := filepath.Join(baseDir, "vally-results") + experimentResultsDir := filepath.Join(baseDir, "vally-experiment-results") + + for _, resultsPath := range []string{ + filepath.Join(evalResultsDir, "2026-07-24T01-00-00-000Z", "results.jsonl"), + filepath.Join( + experimentResultsDir, + "2026-07-24T01-00-00-000Z", + "skills=with-skills", + "results.jsonl", + ), + } { + if err := os.MkdirAll(filepath.Dir(resultsPath), 0o755); err != nil { + t.Fatalf("MkdirAll(%q) error = %v", resultsPath, err) + } + if err := os.WriteFile(resultsPath, []byte("{}\n"), 0o600); err != nil { + t.Fatalf("WriteFile(%q) error = %v", resultsPath, err) + } + } + + messages, err := reportLatestRuns( + evalResultsDir, + experimentResultsDir, + baseDir, + "eval-results", + "eval-experiments", + ) + if err != nil { + t.Fatalf("reportLatestRuns() error = %v", err) + } + if len(messages) != 2 { + t.Fatalf("messages = %d, want 2", len(messages)) + } + + for _, prefix := range []string{"eval-results", "eval-experiments"} { + for _, suffix := range []string{"-failed.md", "-passed.md"} { + if _, err := os.Stat(filepath.Join(baseDir, prefix) + suffix); err != nil { + t.Errorf("Stat(%q) error = %v", prefix+suffix, err) + } + } + } +} + +func TestReportLatestRunsReportsAvailableRunsAndSkipsMissingRuns(t *testing.T) { + baseDir := t.TempDir() + evalResultsDir := filepath.Join(baseDir, "vally-results") + experimentResultsDir := filepath.Join(baseDir, "vally-experiment-results") + + evalResultsPath := filepath.Join(evalResultsDir, "2026-07-24T01-00-00-000Z", "results.jsonl") + if err := os.MkdirAll(filepath.Dir(evalResultsPath), 0o755); err != nil { + t.Fatalf("MkdirAll() error = %v", err) + } + if err := os.WriteFile(evalResultsPath, []byte("{}\n"), 0o600); err != nil { + t.Fatalf("WriteFile() error = %v", err) + } + + messages, err := reportLatestRuns( + evalResultsDir, + experimentResultsDir, + baseDir, + "eval-results", + "eval-experiments", + ) + if len(messages) != 2 { + t.Fatalf("messages = %d, want 2", len(messages)) + } + if !strings.Contains(messages[0], "0 failed, 0 passed") { + t.Errorf("message = %q, want empty run counts", messages[0]) + } + if err != nil { + t.Errorf("reportLatestRuns() error = %v, want nil", err) + } + if len(messages) != 2 || !strings.Contains(messages[1], "Skipping "+experimentResultsDir) { + t.Errorf("messages = %v, want skipped experiment results", messages) + } +} diff --git a/eng/pipelines/eval-e2e.yml b/eng/pipelines/eval-e2e.yml deleted file mode 100644 index 8f3be20ddfb..00000000000 --- a/eng/pipelines/eval-e2e.yml +++ /dev/null @@ -1,96 +0,0 @@ -# Eval: E2E Lifecycle -# Runs end-to-end lifecycle evaluations against live Azure resources. -# Scheduled weekly on Monday. -# -# Required variables (set in ADO pipeline variable group): -# COPILOT_CLI_TOKEN - Copilot CLI authentication token -# AZURE_SUBSCRIPTION_ID - Target Azure subscription for provisioning -# AZURE_TENANT_ID - AAD tenant ID -# AZURE_CLIENT_ID - Service principal client ID (for Azure login) - -trigger: none -pr: none - -schedules: - # 6am UTC Monday - - cron: "0 6 * * 1" - displayName: Weekly E2E Eval - branches: - include: - - main - always: true - -extends: - template: /eng/pipelines/templates/stages/1es-redirect.yml - parameters: - stages: - - stage: E2EEval - displayName: E2E Lifecycle Evaluations - variables: - - template: /eng/pipelines/templates/variables/image.yml - jobs: - - job: E2ELifecycle - displayName: E2E Lifecycle - pool: - name: $(LINUXPOOL) - image: $(LINUXVMIMAGE) - os: linux - timeoutInMinutes: 60 - steps: - - checkout: self - - - template: /eng/pipelines/templates/steps/setup-go.yml - - - task: NodeTool@0 - inputs: - versionSpec: "22.x" - displayName: Setup Node.js - - - pwsh: go build -o ./azd . - workingDirectory: cli/azd - displayName: Build azd - - - bash: echo "##vso[task.prependpath]$(Build.SourcesDirectory)/cli/azd" - displayName: Add azd to PATH - - - task: AzureCLI@2 - displayName: Azure Login - inputs: - azureSubscription: azure-sdk-tests - scriptType: bash - scriptLocation: inlineScript - inlineScript: echo "Logged in to Azure" - - - bash: npm install -g waza - displayName: Install Waza CLI - - - bash: npm ci - workingDirectory: cli/azd/test/eval - displayName: Install eval dependencies - - - bash: waza run --executor copilot-sdk --filter "tasks/lifecycle/" - workingDirectory: cli/azd/test/eval - timeoutInMinutes: 40 - displayName: Run lifecycle evaluations - env: - COPILOT_CLI_TOKEN: $(COPILOT_CLI_TOKEN) - AZURE_SUBSCRIPTION_ID: $(AZURE_SUBSCRIPTION_ID) - AZURE_ENV_NAME: eval-e2e-$(Build.BuildId) - - - task: PublishPipelineArtifact@1 - condition: always() - inputs: - targetPath: cli/azd/test/eval/reports/ - artifactName: e2e-results-$(Build.BuildId) - displayName: Upload E2E results - - - bash: | - # Clean up Azure resources created during this E2E run only. - # Scope to the current build ID to avoid deleting resources from - # parallel runs or local dev testing. - for group in $(az group list --query "[?starts_with(name, 'rg-eval-e2e-$(Build.BuildId)')].name" -o tsv); do - az group delete --name "$group" --yes --no-wait 2>/dev/null || true - done - condition: always() - continueOnError: true - displayName: Cleanup Azure resources diff --git a/eng/pipelines/eval-report.yml b/eng/pipelines/eval-report.yml deleted file mode 100644 index fbba729f50e..00000000000 --- a/eng/pipelines/eval-report.yml +++ /dev/null @@ -1,79 +0,0 @@ -# Eval: Weekly Report -# Aggregates Waza and E2E artifacts into a weekly summary report. -# Scheduled weekly on Monday after E2E completes. - -trigger: none -pr: none - -schedules: - # 8am UTC Monday, after E2E completes - - cron: "0 8 * * 1" - displayName: Weekly Eval Report - branches: - include: - - main - always: true - -extends: - template: /eng/pipelines/templates/stages/1es-redirect.yml - parameters: - stages: - - stage: EvalReport - displayName: Weekly Eval Report - variables: - - template: /eng/pipelines/templates/variables/image.yml - jobs: - - job: GenerateReport - displayName: Generate Report - pool: - name: $(LINUXPOOL) - image: $(LINUXVMIMAGE) - os: linux - timeoutInMinutes: 15 - steps: - - checkout: self - - - task: NodeTool@0 - inputs: - versionSpec: "22.x" - displayName: Setup Node.js - - - bash: npm ci - workingDirectory: cli/azd/test/eval - displayName: Install eval dependencies - - # Download artifacts from recent Waza and E2E pipeline runs. - # The DownloadPipelineArtifact task retrieves artifacts from - # the most recent successful run of the specified pipeline. - - task: DownloadPipelineArtifact@2 - displayName: Download Waza artifacts - continueOnError: true - inputs: - source: specific - project: $(System.TeamProjectId) - pipeline: eval-waza - runVersion: latestFromBranch - runBranch: refs/heads/main - path: cli/azd/test/eval/reports/waza - - - task: DownloadPipelineArtifact@2 - displayName: Download E2E artifacts - continueOnError: true - inputs: - source: specific - project: $(System.TeamProjectId) - pipeline: eval-e2e - runVersion: latestFromBranch - runBranch: refs/heads/main - path: cli/azd/test/eval/reports/e2e - - # TODO: Implement report generation script (scripts/generate-report.ts) - # that diffs Waza result JSON files and produces regression-issues.json. - - - task: PublishPipelineArtifact@1 - condition: always() - inputs: - targetPath: cli/azd/test/eval/reports/ - artifactName: eval-weekly-report-$(Build.BuildId) - artifactRetentionDays: 90 - displayName: Upload aggregated report diff --git a/eng/pipelines/eval-unit.yml b/eng/pipelines/eval-unit.yml deleted file mode 100644 index 23d54c96029..00000000000 --- a/eng/pipelines/eval-unit.yml +++ /dev/null @@ -1,67 +0,0 @@ -# Eval: Unit Tests (non-LLM) -# Runs TypeScript unit tests and Python grader tests for the eval framework. -# No secrets or Azure credentials required. - -trigger: none - -pr: - paths: - include: - - cli/azd/test/eval/** - - cli/azd/internal/mcp/** - - cli/azd/cmd/mcp.go - - cli/azd/cmd/root.go - -extends: - template: /eng/pipelines/templates/stages/1es-redirect.yml - parameters: - stages: - - stage: EvalUnit - displayName: Eval Unit Tests - variables: - - template: /eng/pipelines/templates/variables/image.yml - jobs: - - job: UnitTests - displayName: Unit Tests - pool: - name: $(LINUXPOOL) - image: $(LINUXVMIMAGE) - os: linux - timeoutInMinutes: 30 - steps: - - checkout: self - - - template: /eng/pipelines/templates/steps/setup-go.yml - - - task: NodeTool@0 - inputs: - versionSpec: "22.x" - displayName: Setup Node.js - - - task: UsePythonVersion@0 - inputs: - versionSpec: "3.12" - displayName: Setup Python - - - pwsh: go build -o ./azd . - workingDirectory: cli/azd - displayName: Build azd - - - bash: npm ci - workingDirectory: cli/azd/test/eval - displayName: Install eval dependencies - - - bash: npm run test:unit -- --ci - workingDirectory: cli/azd/test/eval - displayName: Run unit tests - - - bash: pip install pytest && python -m pytest test_graders.py -v - workingDirectory: cli/azd/test/eval/graders - displayName: Run grader tests - - - task: PublishPipelineArtifact@1 - condition: always() - inputs: - targetPath: cli/azd/test/eval/reports/ - artifactName: eval-unit-results - displayName: Upload test results diff --git a/eng/pipelines/eval-waza.yml b/eng/pipelines/eval-waza.yml deleted file mode 100644 index 9b67076572e..00000000000 --- a/eng/pipelines/eval-waza.yml +++ /dev/null @@ -1,72 +0,0 @@ -# Eval: Waza Runs -# Runs Waza LLM evaluations against a freshly built azd binary. -# Scheduled 3x daily Tue-Sat. -# -# Required variables (set in ADO pipeline variable group): -# COPILOT_CLI_TOKEN - Copilot CLI authentication token for LLM evaluations - -trigger: none -pr: none - -schedules: - # 5am, 12pm, 8pm UTC, Tuesday through Saturday - - cron: "0 5,12,20 * * 2-6" - displayName: Waza Eval Runs - branches: - include: - - main - always: true - -extends: - template: /eng/pipelines/templates/stages/1es-redirect.yml - parameters: - stages: - - stage: WazaEval - displayName: Waza Evaluations - variables: - - template: /eng/pipelines/templates/variables/image.yml - jobs: - - job: WazaRun - displayName: Waza Run - pool: - name: $(LINUXPOOL) - image: $(LINUXVMIMAGE) - os: linux - timeoutInMinutes: 45 - steps: - - checkout: self - - - template: /eng/pipelines/templates/steps/setup-go.yml - - - task: NodeTool@0 - inputs: - versionSpec: "22.x" - displayName: Setup Node.js - - - pwsh: go build -o ./azd . - workingDirectory: cli/azd - displayName: Build azd - - - bash: echo "##vso[task.prependpath]$(Build.SourcesDirectory)/cli/azd" - displayName: Add azd to PATH - - - bash: npm install -g waza - displayName: Install Waza CLI - - - bash: npm ci - workingDirectory: cli/azd/test/eval - displayName: Install eval dependencies - - - bash: waza run --executor copilot-sdk - workingDirectory: cli/azd/test/eval - timeoutInMinutes: 30 - displayName: Run Waza evaluations - env: - COPILOT_CLI_TOKEN: $(COPILOT_CLI_TOKEN) - - - task: PublishPipelineArtifact@1 - condition: always() - inputs: - targetPath: cli/azd/test/eval/reports/ - artifactName: waza-results-$(Build.BuildId) - displayName: Upload Waza results