diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 47dc852d..dbaeb1f6 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -10,7 +10,7 @@ "plugins": [ { "name": "kbagent", - "version": "0.46.1", + "version": "0.53.0", "source": "./plugins/kbagent", "description": "AI-friendly interface to Keboola Connection projects — explore configs, jobs, lineage, call MCP tools, manage dev branches, and debug SQL in workspaces", "category": "development" diff --git a/CLAUDE.md b/CLAUDE.md index 8ab8ec2e..6b5c228e 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -294,6 +294,7 @@ plugins/kbagent/ ``` # Global options: --json, --verbose, --no-color, --config-dir, --hint client|service (deprecated, use kbagent serve REST API), --deny-writes, --deny-destructive, --allow-env-manage-token +# Headless / token-only (0.50.0+): export KBAGENT_PROJECT_FROM_ENV=1 + KBC_TOKEN + KBC_STORAGE_API_URL to synthesize an in-memory `__env__` project (no `project add`, no config.json on disk; token never persisted). Use `--project __env__`. Same env setup also powers `kbagent serve`. kbagent project add --project NAME --url URL --token TOKEN kbagent project list @@ -346,7 +347,7 @@ kbagent storage bucket-detail --project NAME --bucket-id ID [--branch ID] kbagent storage tables [--project NAME ...] [--bucket-id ID] [--branch ID] kbagent storage table-detail --project NAME --table-id ID [--branch ID] kbagent storage create-bucket --project NAME --stage STAGE --name NAME [--description D] [--backend B] [--branch ID] -kbagent storage create-table --project NAME --bucket-id ID --name NAME --column COL:TYPE[(length)] [...] [--primary-key COL] [--not-null COL ...] [--default NAME=VALUE ...] [--branch ID] +kbagent storage create-table --project NAME --bucket-id ID --name NAME --column COL:TYPE[(length)] [...] [--primary-key COL] [--not-null COL ...] [--default NAME=VALUE ...] [--branch ID] [--if-not-exists] kbagent storage upload-table --project NAME --table-id ID --file PATH [--incremental] [--branch ID] kbagent storage download-table --project NAME --table-id ID [--output FILE] [--columns COL ...] [--limit N] [--branch ID] kbagent storage delete-table --project NAME --table-id ID [--table-id ...] [--force] [--dry-run] [--yes] [--branch ID] @@ -354,6 +355,7 @@ kbagent storage truncate-table --project NAME --table-id ID [--table-id ...] [-- kbagent storage delete-column --project NAME --table-id ID --column COL [--column ...] [--force] [--dry-run] [--yes] [--branch ID] kbagent storage delete-bucket --project NAME --bucket-id ID [--bucket-id ...] [--force] [--dry-run] [--yes] [--branch ID] kbagent storage swap-tables --project NAME --table-id ID --target-table-id ID --branch ID [--dry-run] [--yes] +kbagent storage clone-table --project NAME --table-id ID --branch ID [--dry-run] kbagent storage describe-bucket --project NAME --bucket-id ID [--text STR | --file PATH | --stdin] [--branch ID] kbagent storage describe-table --project NAME --table-id ID [--text STR | --file PATH | --stdin] [--branch ID] kbagent storage describe-column --project NAME --table-id ID --column NAME=DESC [--column ...] [--branch ID] @@ -367,6 +369,16 @@ kbagent storage file-tag --project NAME --file-id ID [--add TAG ...] [--remove T kbagent storage load-file --project NAME --file-id ID --table-id ID [--incremental] [--delimiter D] [--enclosure E] [--branch ID] kbagent storage unload-table --project NAME --table-id ID [--columns COL ...] [--limit N] [--tag TAG ...] [--download] [--output FILE|DIR] [--file-type csv|parquet] [--branch ID] +# stream: Data Streams (OpenTelemetry/OTLP). Storage token from config (no manage token). +# Control plane = stream. (derived from connection.); the OTLP ingest URL +# (stream-in./otlp///) is returned in source.otlp.url +# with the secret in the path -- MASKED by default, --reveal to print it. create-source --type otlp +# auto-provisions the logs/metrics/traces sinks (bucket in.c-otlp-) so data lands; --no-sinks opts out. +kbagent stream list --project NAME [--branch ID] +kbagent stream create-source --project NAME --name NAME [--type otlp|http] [--branch ID] [--if-not-exists] [--no-sinks] [--reveal] +kbagent stream detail [SOURCE_ID | --name NAME] --project NAME [--branch ID] [--reveal] +kbagent stream delete SOURCE_ID --project NAME [--branch ID] [--dry-run] [--yes|--force] + kbagent lineage build --directory PATH --output PATH [--ai] [--refresh] kbagent lineage show --load PATH [--upstream NODE] [--downstream NODE] [--column COL] [--columns] [--project ALIAS] [--depth N] [--format text|mermaid|html|er] kbagent lineage info --load PATH @@ -382,6 +394,15 @@ kbagent sharing edges [--project NAME] kbagent org setup --org-id ID --url URL [--dry-run] [--yes] [--token-description PREFIX] [--refresh] kbagent org setup --project-ids 1,2,3 --url URL [--dry-run] [--yes] [--token-description PREFIX] [--refresh] +# feature: requires a super-admin Manage API token (inline hidden prompt; never persisted; --allow-env-manage-token for CI). --project resolves the stack URL (+ project_id for project ops) from config. +kbagent feature list --project ALIAS +kbagent feature project-show --project ALIAS +kbagent feature project-add --project ALIAS --feature NAME [--dry-run] [--yes] +kbagent feature project-remove --project ALIAS --feature NAME [--dry-run] [--yes] +kbagent feature user-show --project ALIAS --email EMAIL +kbagent feature user-add --project ALIAS --email EMAIL --feature NAME [--dry-run] [--yes] +kbagent feature user-remove --project ALIAS --email EMAIL --feature NAME [--dry-run] [--yes] + kbagent tool list [--project NAME] [--branch ID] kbagent tool call TOOL_NAME [--project NAME] [--input JSON|@file|-] [--branch ID] @@ -426,12 +447,49 @@ kbagent component list [--project NAME] [--type TYPE] [--query QUERY] kbagent component detail --component-id ID [--project NAME] kbagent config new --component-id ID [--name NAME] [--project NAME] [--output-dir DIR] [--push --no-files --description D --configuration JSON|@file|- --configuration-file PATH --no-validate --branch ID --dry-run] +# sync: GitOps -- configs as local files. init/pull/push/diff are filesystem-local (no serve REST surface). +kbagent sync init --project ALIAS [--directory DIR] [--git-branching] [--adopt-existing] +kbagent sync pull --project ALIAS [--all-projects] [--force] [--dry-run] [--with-samples] [--no-storage] [--no-jobs] [--job-limit N] [--branch ID] +# `sync pull --force` is conflict-aware (since 0.53.0): locally-modified config whose remote is UNCHANGED is preserved (delta stays pushable, never silently re-stamped); a true merge conflict (local AND remote both changed since last pull) aborts (exit 1, SYNC_CONFLICT, --json lists details.conflicts); local-untouched + remote-changed takes remote. Discard local edits on purpose by deleting the file/dir then pulling. +kbagent sync status [--directory DIR] +kbagent sync diff --project ALIAS [--all-projects] [--directory DIR] [--branch ID] +kbagent sync push --project ALIAS [--all-projects] [--dry-run] [--force] [--allow-plaintext-on-encrypt-failure] [--branch ID] [--no-name-drift-warnings] +kbagent sync branch-link --project ALIAS (--branch-id ID | --branch-name NAME) [--directory DIR] +kbagent sync branch-unlink [--directory DIR] +kbagent sync branch-status [--directory DIR] + +kbagent dev-portal identity add --alias A --username U [--password P | --password-stdin] + [--role-hint vendor|admin] [--vendor V] [--portal-url URL] +kbagent dev-portal identity list +kbagent dev-portal identity remove --alias A +kbagent dev-portal identity edit --alias A [--username U] [--password P|--password-stdin] + [--role-hint H] [--vendor V] [--new-alias N] +kbagent dev-portal identity use ALIAS +kbagent dev-portal identity current +kbagent dev-portal identity verify [--identity A] + +kbagent dev-portal list --vendor V [--identity A] +kbagent dev-portal get --app VENDOR.APP_ID [--identity A] + +kbagent dev-portal create --vendor V --data FILE [--identity A] [--dry-run] +kbagent dev-portal patch --app VENDOR.APP_ID (--data FILE | --property KEY (--value V | --value-file F)) + [--identity A] [--dry-run] +kbagent dev-portal upload-icon --app VENDOR.APP_ID --file PATH [--identity A] [--dry-run] +kbagent dev-portal publish --app VENDOR.APP_ID [--identity A] [--dry-run] +kbagent dev-portal deprecate --app VENDOR.APP_ID [--identity A] [--dry-run] +# All writes require an interactive random-code TTY confirm; no --yes / no env bypass. +# Since v0.51.1: --role-hint is validated (vendor/admin) and load-bearing -- admin identities route +# `patch` to PATCH /admin/apps/{app} (permissive schema). Vendor + admin-only field => fail-fast preflight. +# --password-stdin works on TTY (hidden prompt) AND on a pipe (reads to EOF). + kbagent encrypt values --project ALIAS --component-id ID --input JSON|@file|- [--output-file PATH] kbagent semantic-layer model list --project P kbagent semantic-layer model create --project P --name N [--description D] [--sql-dialect Snowflake] kbagent semantic-layer model delete --project P --model M [--yes] kbagent semantic-layer show --project P [--model M] [--type dataset|metric|relationship|constraint|glossary] +kbagent semantic-layer search-context --project P [--pattern G ...] [--type model|dataset|metric|relationship|constraint|glossary|all] [--limit N] +kbagent semantic-layer get-context --project P --context-id ID kbagent semantic-layer validate --project P [--model M] [--deep] kbagent semantic-layer export --project P [--model M] [--output PATH] kbagent semantic-layer diff (--project-a A | --file-a PATH) (--project-b B | --file-b PATH) [--model-a M] [--model-b M] diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a66e6def..523f1028 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -328,11 +328,7 @@ When adding a new command (e.g., `kbagent storage create-foo`), you must update - [ ] **Client method** in `client.py` (or `manage_client.py`) -- HTTP layer - [ ] **Service method** in `services/` -- business logic, validation, orchestration - [ ] **Command function** in `commands/` -- Typer options, formatter, error handling -- [ ] **`--hint` support** -- every command must support `--hint client` and `--hint service` code generation: - - [ ] **Hint definition** in `hints/definitions/` -- register a `CommandHint` with `ClientCall` + `ServiceCall` (see existing files for pattern) - - [ ] **Hint short-circuit** in the command function -- add `if should_hint(ctx): emit_hint(...)` **before** the service call - - [ ] **Verify** both modes produce valid Python: `kbagent --hint client ...` and `kbagent --hint service ...` - - [ ] **Exception -- infrastructure-level commands.** `--hint` only makes sense for commands that wrap a Keboola API client (Storage / Queue / Manage / AI Service / MCP). Commands that exist purely to manage kbagent itself, or to forward HTTP to another kbagent instance, are **deliberately excluded**: `doctor`, `context`, `init`, `serve`, `version`, `update`, `changelog`, `permissions`, and `http` (the four self-call verbs `http get/post/patch/delete` pass straight through `HttpForwarderService` to a running `kbagent serve` -- there is no Keboola client to mimic, and an `httpx.Client` recipe is already the canonical generic form). If you are adding a *new* command in this infrastructure category, skip the hint definition and note the exception in the PR description so reviewers don't file a BLOCKING finding against this rule. +- [ ] ~~**`--hint` support**~~ -- **DEPRECATED, do not add.** `--hint client` / `--hint service` code generation is superseded by the `kbagent serve` REST API. Do **not** add a new `hints/definitions/` entry or a `should_hint(ctx)` short-circuit for new commands, and do not teach `--hint` in new docs -- point readers at `kbagent serve` instead. Existing hint definitions are kept for backward compatibility but are no longer extended. Reviewers must **not** flag a missing hint definition. - [ ] **Permission registration** in `permissions.py` (`OPERATION_REGISTRY` dict) - [ ] **Service wiring** in `cli.py` if adding a new service class - [ ] **HTTP API endpoint** in `src/keboola_agent_cli/server/routers/.py` -- `kbagent serve` exposes the CLI as a REST API so external applications (Web UI, scheduled AI agents, Slack bots, Streamlit dashboards, CI pipelines) can call the platform without forking CLI subprocesses. The current convention is **1:1**: every command in a group has a matching endpoint in that group's router (e.g. `commands/flow.py` has 8 commands, `server/routers/flows.py` has 8 routes). If you add a new command, add the corresponding route. **Skip allowed** only for genuinely terminal-only commands (interactive prompts, Rich-rendered output that has no useful JSON shape, `doctor`/`init`/`update`-style infrastructure that manages kbagent itself rather than Keboola). Document any skip in the PR description with a one-line reason so reviewers don't flag it. @@ -366,7 +362,7 @@ before the PR is mergeable. - [ ] **`plugins/kbagent/agents/keboola-expert.md`** -- the subagent system prompt. **Highest silent-drift risk in the repo.** Update at minimum: - [ ] **§1 Rule 6 VERSION GATE** examples (e.g. `flow update needs 0.22.0+`) when adding a command that introduces or relaxes a minimum-version requirement, or when an example version reference is now stale enough to mislead. - - [ ] **§2 Tool Selection Matrix** when adding any new write or destructive command -- give it a row with `First choice / Fallback / NEVER`. A missing row means the subagent will fall back to MCP `tool call` or refuse the task. + - [ ] **§2 Tool Selection Matrix** -- one row **per command GROUP**, not per command. When you add a new write/destructive *group* (e.g. `dev-portal`), give it a single row with `First choice / Fallback / NEVER`. Adding a command to an *existing* group needs no new row. Exhaustive per-command detail belongs in `AGENT_CONTEXT` (`kbagent context`), which is loaded dynamically on demand -- `keboola-expert.md` is a static system prompt loaded into every subagent run and carries a hard 60 KB budget, so it must stay a high-signal decision matrix, not a command catalogue. If a one-row addition would push the file over budget, trim stale content first; do **not** raise the cap. *Severity note:* authors are expected to add the group row, but `/kbagent:review` flags a missing row only **NON-BLOCKING** -- `AGENT_CONTEXT` (a BLOCKING surface above) is the authoritative command catalogue, so a missing matrix row degrades subagent ergonomics without making a command undiscoverable. Don't deprioritize it just because it's non-blocking. - [ ] **§3 Inline Gotchas** when behavior changed in a way the agent will get wrong by default (e.g. dev-branch auto-materialization, native column-type whitelisting). - [ ] **`plugins/kbagent/skills/kbagent/SKILL.md`** non-table portions -- update the `description:` trigger keywords when introducing a new topic area (so description-matching auto-triggers the skill); add a workflow row to the bottom table if you created a new `references/-workflow.md`. - [ ] **`plugins/kbagent/skills/kbagent/references/commands-reference.md`** -- add the new command bullet under the appropriate section. Hand-maintained, NOT auto-generated. (Yes, this partly duplicates the auto-generated SKILL.md table -- the reference carries denser per-command notes, the table is the at-a-glance picker.) @@ -412,7 +408,7 @@ release checklist below. | `CLAUDE.md` (`## All CLI Commands`) | Adding/removing/renaming commands | NO | | `plugins/kbagent/.claude-plugin/plugin.json` | Every release (auto-synced) | YES (`make version-check`; pre-commit auto-stages) | | `plugins/kbagent/.claude-plugin/CLAUDE.md` | Changing delegation strategy / when-to-delegate rules | NO | -| `plugins/kbagent/agents/keboola-expert.md` | New write/destructive command (matrix); new minimum-version requirement (Rule 6 VERSION GATE); behavior change (gotchas) | NO -- **highest silent-drift risk** | +| `plugins/kbagent/agents/keboola-expert.md` | New write/destructive command **group** (one matrix row per group, not per command -- file has a hard 60 KB prompt budget); new minimum-version requirement (Rule 6 VERSION GATE); behavior change (gotchas) | NO -- **highest silent-drift risk** | | `plugins/kbagent/commands/keboola.md` | `/keboola` slash-command UX change (rare) | NO | | `plugins/kbagent/skills/kbagent/SKILL.md` -- table | Auto-generated by `make skill-gen` | YES (`make skill-check`; pre-commit auto-stages) | | `plugins/kbagent/skills/kbagent/SKILL.md` -- description / rules / workflow links | New topic area in `description` triggers; new workflow file added to bottom table | NO | @@ -499,8 +495,7 @@ courtesy that: - catches the silent-drift gaps (`OPERATION_REGISTRY`, `gotchas.md` version tags, `keboola-expert.md` matrix, `commands/context.py` - `AGENT_CONTEXT`, `commands-reference.md`, `--hint` definitions) that - CI does not check; + `AGENT_CONTEXT`, `commands-reference.md`) that CI does not check; - demonstrates to the human reviewer that you have walked the [Plugin synchronization map](#plugin-synchronization-map); - saves a review round-trip when the reviewer would otherwise catch the @@ -543,7 +538,7 @@ manual safety net for the silent-drift risks summarized in the 4. **Run `make skill-gen`** -- regenerates the decision table in `SKILL.md`. Idempotent if no commands changed since the previous release. 5. **Manually review `plugins/kbagent/agents/keboola-expert.md`**: - **§1 Rule 6 VERSION GATE examples** -- if any feature this release shipped (or any feature shipped in a previous release that you missed) was previously missing-and-now-present, document it with the right minimum version. Remove stale "since X.Y.Z" mentions that no longer matter to live users. - - **§2 Tool Selection Matrix** -- did you add new write/destructive commands since last release? Are they present with `First choice / Fallback / NEVER`? A missing row means the subagent will silently fall back to MCP `tool call` or refuse the task. + - **§2 Tool Selection Matrix** -- did you add a new write/destructive command *group* since last release? Is it present with one `First choice / Fallback / NEVER` row (per group, not per command)? Mind the hard 60 KB prompt budget: trim stale content rather than raising the cap. New commands inside an existing group need no new row. - **§3 Inline Gotchas** -- new behavior the agent would get wrong by default? Add it. 6. **Manually review `plugins/kbagent/skills/kbagent/references/gotchas.md`** -- every behavior introduced or changed this release that an AI agent would not infer from `--help` should have its own `(since vX.Y.Z)` entry. The version tag is non-optional. 7. **Manually review `CLAUDE.md` `## All CLI Commands`** -- diff against `kbagent --help` output (and against `kbagent context`). Hand-maintained; CI does not catch drift here. diff --git a/Makefile b/Makefile index 78ce7491..1e529cbe 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ .DEFAULT_GOAL := help -.PHONY: help install install-mcp install-server sync test test-unit test-integration test-e2e test-e2e-local test-e2e-invite test-file lint lint-fix format format-check typecheck typecheck-warn skill-check skill-gen version-sync version-check changelog changelog-check check-error-codes check clean hooks web-install web-dev-backend web-dev-frontend web-build web-clean +.PHONY: help install install-mcp install-server sync test test-unit test-integration test-e2e test-e2e-local test-e2e-invite test-e2e-feature test-e2e-stream test-file lint lint-fix format format-check typecheck typecheck-warn skill-check skill-gen version-sync version-check changelog changelog-check check-error-codes check clean hooks web-install web-dev-backend web-dev-frontend web-build web-clean help: ## Show this help message @grep -E '^[a-zA-Z0-9_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf " \033[36m%-18s\033[0m %s\n", $$1, $$2}' @@ -36,6 +36,12 @@ test-e2e-local: ## Run E2E against a project in a local config.json (CONFIG_DIR= test-e2e-invite: ## Run project invite E2E (E2E_MANAGE_TOKEN + E2E_INVITE_PROJECT_ID required) uv run pytest tests/test_e2e.py -v -s --tb=long -m e2e_invite +test-e2e-feature: ## Run feature-flag E2E (E2E_MANAGE_TOKEN super-admin + E2E_API_TOKEN + E2E_URL required) + uv run pytest tests/test_e2e.py -v -s --tb=long -k test_feature_flags_read_e2e + +test-e2e-stream: ## Run Data Streams OTLP E2E (E2E_API_TOKEN + E2E_URL required; creates + deletes a temp source) + uv run pytest tests/test_e2e.py -v -s --tb=long -k test_stream_otlp_e2e + test-file: ## Run a specific test file (FILE=tests/test_cli.py) uv run pytest $(FILE) -v diff --git a/docs/superpowers/plans/2026-05-28-dev-portal.md b/docs/superpowers/plans/2026-05-28-dev-portal.md new file mode 100644 index 00000000..950ad0d3 --- /dev/null +++ b/docs/superpowers/plans/2026-05-28-dev-portal.md @@ -0,0 +1,3010 @@ +# `kbagent dev-portal` Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Add a `kbagent dev-portal` command group that wraps the Keboola Developer Portal API (`apps-api.keboola.com`) with multi-identity credential storage and a random-code TTY confirm safety bar that an AI agent cannot bypass. + +**Architecture:** Standard kbagent 3-layer: `commands/dev_portal.py` (Typer) → `services/dev_portal_service.py` (business logic, diff, prepare/apply) → `dev_portal_client.py` (HTTP, login + MFA). Identities persisted in `AppConfig.dev_portal_identities` next to KB projects. Every write is gated by `require_random_code_confirmation()` (extracted from `commands/permissions.py` into `commands/_helpers.py` so the primitive has one home). + +**Tech Stack:** Python 3.12, Typer, Pydantic 2.x, httpx (inherits `BaseHttpClient`), pytest + `pytest-httpx`, `typer.testing.CliRunner`. + +**Source spec:** `docs/superpowers/specs/2026-05-28-dev-portal-design.md` + +--- + +## File Structure + +### New files + +| Path | Responsibility | +|------|----------------| +| `src/keboola_agent_cli/dev_portal_client.py` | Layer 3 HTTP client. Login (token + MFA), bearer in-memory, list/get/create/patch/upload-icon (two-hop)/publish/deprecate. Inherits `BaseHttpClient`. | +| `src/keboola_agent_cli/services/dev_portal_service.py` | Layer 2 business logic. Identity CRUD, `prepare_*`/`apply` pattern with diffing, publish pre-flight validation, verify-on-add. | +| `src/keboola_agent_cli/commands/dev_portal.py` | Layer 1 Typer commands. Identity subcommands; `list`/`get`; writes with `--dry-run` and random-code confirm. | +| `tests/test_dev_portal_client.py` | `pytest-httpx` mocked tests for the client. | +| `tests/test_dev_portal_service.py` | Mocked-client tests for the service. | +| `tests/test_dev_portal_cli.py` | `CliRunner` tests for the command layer. | +| `plugins/kbagent/skills/kbagent/references/dev-portal-workflow.md` | Workflow doc for the kbagent skill. | + +### Modified files + +| Path | What changes | +|------|--------------| +| `src/keboola_agent_cli/errors.py` | Add 5 new `ErrorCode` entries (`DP_LOGIN_FAILED`, `DP_MFA_REQUIRED`, `DP_APP_NOT_FOUND`, `DP_PUBLISH_REQUIREMENTS_MISSING`, `DP_ICON_UPLOAD_FAILED`). | +| `src/keboola_agent_cli/commands/_helpers.py` | Extract `require_random_code_confirmation()` from `commands/permissions.py`; add `resolve_identity_alias()` and `get_dev_portal_service()` factories. | +| `src/keboola_agent_cli/commands/permissions.py` | Replace local `_require_interactive_confirmation` with import from `_helpers`. | +| `src/keboola_agent_cli/models.py` | Add `DeveloperPortalIdentity`; extend `AppConfig` with `dev_portal_identities` + `default_dev_portal_identity`. | +| `src/keboola_agent_cli/config_store.py` | Add 5 mirror methods; extend `CLAUDE_CONFIG_WARNING` to mention DP credentials. | +| `src/keboola_agent_cli/permissions.py` | Add 14 entries to `OPERATION_REGISTRY`. | +| `src/keboola_agent_cli/cli.py` | Register `dev_portal_app` Typer sub-app under panel "Development". | +| `src/keboola_agent_cli/commands/context.py` | Extend `AGENT_CONTEXT` with `dev-portal` section. | +| `src/keboola_agent_cli/changelog.py` | Add release entry. | +| `pyproject.toml` | Bump to next minor (e.g. `0.45.0`). | +| `tests/test_config_store.py` | Add tests for DP identity mirror methods. | +| `tests/test_permissions.py` | Add tests for the 14 new ops. | +| `tests/test_helpers.py` | Add tests for `require_random_code_confirmation()`. | +| `tests/test_e2e.py` | Add identity-list smoke + optional portal-list (gated on `E2E_DP_USERNAME`/`E2E_DP_PASSWORD`). | +| `CLAUDE.md` | Update `## All CLI Commands` section. | +| `plugins/kbagent/.claude-plugin/plugin.json` | Auto-synced via `make version-sync`. | +| `plugins/kbagent/skills/kbagent/SKILL.md` | Decision-table row for "manage portal property / register app". | +| `plugins/kbagent/skills/kbagent/references/commands-reference.md` | New section for `dev-portal`. | +| `plugins/kbagent/skills/kbagent/references/gotchas.md` | Entry tagged `(since v0.45.0)` for the no-bypass write rule. | +| `plugins/kbagent/agents/keboola-expert.md` | Rule 6 version-gate update, tool-selection matrix entry, inline gotcha. | + +--- + +## Task 1: Branch off main + smoke check + +**Files:** +- Modify: none yet (branch setup only) + +- [ ] **Step 1: Confirm clean tree on main** + +```bash +git status +git checkout main +git pull --ff-only origin main +``` + +Expected: clean working tree on `main` at latest commit. + +- [ ] **Step 2: Create feature branch** + +```bash +git checkout -b feat/dev-portal +``` + +Expected: switched to `feat/dev-portal`. (Do NOT continue on the existing `feat/job-run-mode-debug` branch — that ships an unrelated change.) + +- [ ] **Step 3: Verify dev install works** + +```bash +uv pip install -e ".[dev]" +uv run pytest tests/test_cli.py -q +``` + +Expected: all tests pass. Establishes baseline so later failures are clearly caused by this work. + +- [ ] **Step 4: Commit branch readme touch (optional, skip if your team policy is "no empty commits")** + +Skipped — no commit, the branch is created lazily. + +--- + +## Task 2: Add new ErrorCode entries + +**Files:** +- Modify: `src/keboola_agent_cli/errors.py` +- Test: `tests/test_errors.py` (extend; check if file already covers enum membership — if not, the unit test in step 1 lives in `tests/test_dev_portal_client.py` later) + +- [ ] **Step 1: Write failing test for enum membership** + +Append to `tests/test_errors.py`: + +```python +from keboola_agent_cli.errors import ErrorCode + + +def test_dev_portal_error_codes_present(): + assert ErrorCode.DP_LOGIN_FAILED == "DP_LOGIN_FAILED" + assert ErrorCode.DP_MFA_REQUIRED == "DP_MFA_REQUIRED" + assert ErrorCode.DP_APP_NOT_FOUND == "DP_APP_NOT_FOUND" + assert ErrorCode.DP_PUBLISH_REQUIREMENTS_MISSING == "DP_PUBLISH_REQUIREMENTS_MISSING" + assert ErrorCode.DP_ICON_UPLOAD_FAILED == "DP_ICON_UPLOAD_FAILED" +``` + +- [ ] **Step 2: Run, expect failure** + +Run: `uv run pytest tests/test_errors.py::test_dev_portal_error_codes_present -v` +Expected: `AttributeError: DP_LOGIN_FAILED` (or `FAILED` with `AttributeError`). + +- [ ] **Step 3: Add the 5 entries to ErrorCode** + +Edit `src/keboola_agent_cli/errors.py`. After the existing `# Sync` block (line ~89), before the closing brace of the enum (look for the last `XXX = "XXX"`), add: + +```python + # Developer Portal (since 0.45.0) + DP_LOGIN_FAILED = "DP_LOGIN_FAILED" + DP_MFA_REQUIRED = "DP_MFA_REQUIRED" + DP_APP_NOT_FOUND = "DP_APP_NOT_FOUND" + DP_PUBLISH_REQUIREMENTS_MISSING = "DP_PUBLISH_REQUIREMENTS_MISSING" + DP_ICON_UPLOAD_FAILED = "DP_ICON_UPLOAD_FAILED" +``` + +Also extend the `_ERROR_TYPE` mapping (search for `ErrorCode.INVALID_TOKEN: "authentication"` around line 183) by adding: + +```python + ErrorCode.DP_LOGIN_FAILED: "authentication", + ErrorCode.DP_MFA_REQUIRED: "authentication", + ErrorCode.DP_APP_NOT_FOUND: "not_found", + ErrorCode.DP_PUBLISH_REQUIREMENTS_MISSING: "validation", + ErrorCode.DP_ICON_UPLOAD_FAILED: "api", +``` + +- [ ] **Step 4: Run, expect pass** + +Run: `uv run pytest tests/test_errors.py -v` +Expected: all pass. + +- [ ] **Step 5: Lint + commit** + +```bash +uv run ruff check src/keboola_agent_cli/errors.py tests/test_errors.py +uv run ruff format src/keboola_agent_cli/errors.py tests/test_errors.py +git add src/keboola_agent_cli/errors.py tests/test_errors.py +git commit -m "feat(dev-portal): add ErrorCode entries for Developer Portal" +``` + +--- + +## Task 3: Extract `require_random_code_confirmation()` into `_helpers.py` + +> **CRITICAL:** This is the load-bearing safety primitive. It must exist and be tested before any DP write code is written. + +**Files:** +- Modify: `src/keboola_agent_cli/commands/_helpers.py` (add helper) +- Modify: `src/keboola_agent_cli/commands/permissions.py` (replace local helper with import) +- Test: `tests/test_helpers.py` (extend) + +- [ ] **Step 1: Write failing tests in `tests/test_helpers.py`** + +Append: + +```python +import io +from unittest.mock import patch + +import pytest +import typer + +from keboola_agent_cli.commands._helpers import require_random_code_confirmation + + +class TestRequireRandomCodeConfirmation: + def test_non_tty_exits_with_permission_denied(self, monkeypatch): + # stdin isatty -> False + monkeypatch.setattr("sys.stdin.isatty", lambda: False) + with pytest.raises(typer.Exit) as exc: + require_random_code_confirmation("delete the universe") + assert exc.value.exit_code == 6 # EXIT_PERMISSION_DENIED + + def test_correct_code_accepted(self, monkeypatch): + monkeypatch.setattr("sys.stdin.isatty", lambda: True) + monkeypatch.setattr( + "keboola_agent_cli.commands._helpers.secrets.token_hex", + lambda n: "deadbeef", + ) + monkeypatch.setattr("builtins.input", lambda: "deadbeef") + # Returns None on success + assert require_random_code_confirmation("patch app") is None + + def test_wrong_code_exits(self, monkeypatch): + monkeypatch.setattr("sys.stdin.isatty", lambda: True) + monkeypatch.setattr( + "keboola_agent_cli.commands._helpers.secrets.token_hex", + lambda n: "deadbeef", + ) + monkeypatch.setattr("builtins.input", lambda: "wrongcode") + with pytest.raises(typer.Exit) as exc: + require_random_code_confirmation("patch app") + assert exc.value.exit_code == 6 + + def test_eof_exits(self, monkeypatch): + monkeypatch.setattr("sys.stdin.isatty", lambda: True) + monkeypatch.setattr( + "keboola_agent_cli.commands._helpers.secrets.token_hex", + lambda n: "deadbeef", + ) + def raise_eof(): + raise EOFError + monkeypatch.setattr("builtins.input", raise_eof) + with pytest.raises(typer.Exit) as exc: + require_random_code_confirmation("patch app") + assert exc.value.exit_code == 6 +``` + +- [ ] **Step 2: Run, expect failure** + +Run: `uv run pytest tests/test_helpers.py::TestRequireRandomCodeConfirmation -v` +Expected: `ImportError: cannot import name 'require_random_code_confirmation'`. + +- [ ] **Step 3: Add the helper to `_helpers.py`** + +Add imports at the top of `src/keboola_agent_cli/commands/_helpers.py` if not present: + +```python +import secrets +import sys +``` + +Add at the end of the file: + +```python +_CONFIRM_CODE_LENGTH = 4 + + +def require_random_code_confirmation(action_description: str) -> None: + """Require the user to type a random hex code to confirm a high-risk action. + + Prevents AI agents from programmatically approving production-affecting + writes (Developer Portal updates, permission policy changes). The agent + cannot predict the code and cannot type it into stdin. + + Behaviour: + - No TTY -> raise typer.Exit(EXIT_PERMISSION_DENIED). + - TTY + correct code -> return None (caller proceeds). + - TTY + wrong code / EOF / interrupt -> raise typer.Exit(EXIT_PERMISSION_DENIED). + + Args: + action_description: Short verb phrase shown in the prompt + (e.g. "patch keboola.ex-foo", "update permission policy"). + """ + is_tty = hasattr(sys.stdin, "isatty") and sys.stdin.isatty() + if not is_tty: + sys.stderr.write( + f"\nRefusing to {action_description}: this action requires a " + "real terminal so a human can type the confirmation code. " + "There is no --yes bypass by design.\n" + ) + raise typer.Exit(code=EXIT_PERMISSION_DENIED) + + code = secrets.token_hex(_CONFIRM_CODE_LENGTH) + sys.stderr.write(f"\nTo {action_description}, type this code: {code}\n") + sys.stderr.write("Confirmation: ") + sys.stderr.flush() + + try: + user_input = input().strip() + except (EOFError, KeyboardInterrupt): + raise typer.Exit(code=EXIT_PERMISSION_DENIED) from None + + if user_input != code: + sys.stderr.write("Confirmation failed. Aborting.\n") + raise typer.Exit(code=EXIT_PERMISSION_DENIED) +``` + +(`EXIT_PERMISSION_DENIED` is already imported in `_helpers.py` for the existing error mapping; double-check the existing imports and add it if missing — it lives in `..constants`.) + +- [ ] **Step 4: Run, expect pass** + +Run: `uv run pytest tests/test_helpers.py::TestRequireRandomCodeConfirmation -v` +Expected: all 4 tests pass. + +- [ ] **Step 5: Replace the old helper in `commands/permissions.py`** + +In `src/keboola_agent_cli/commands/permissions.py`: + +(a) Delete the local `_require_interactive_confirmation` function (lines ~31–53 — the one that returns `bool`). +(b) Delete unused `import secrets` if it was only used by the deleted function. +(c) Add import at top: + +```python +from ._helpers import require_random_code_confirmation +``` + +(d) Replace the three call sites (`permissions_set`, `permissions_reset`) — they currently look like: + +```python + if not _require_interactive_confirmation("update permission policy"): + formatter.error( + message="Confirmation failed. Permission policy not changed.", + error_code=ErrorCode.PERMISSION_DENIED, + ) + raise typer.Exit(code=EXIT_PERMISSION_DENIED) from None +``` + +Change each to: + +```python + require_random_code_confirmation("update permission policy") # raises on failure +``` + +(Same shape for `permissions_reset` with `"remove permission policy"`.) + +- [ ] **Step 6: Run existing permissions tests to confirm no regression** + +Run: `uv run pytest tests/test_permissions.py -v` +Expected: all pass. (Some tests may have asserted on the old `_require_interactive_confirmation` return-bool behaviour — if so, update them to assert on `typer.Exit` and the user-facing prompt copy.) + +- [ ] **Step 7: Lint + commit** + +```bash +uv run ruff check src/keboola_agent_cli/commands/_helpers.py src/keboola_agent_cli/commands/permissions.py tests/test_helpers.py +uv run ruff format src/keboola_agent_cli/commands/_helpers.py src/keboola_agent_cli/commands/permissions.py tests/test_helpers.py +git add src/keboola_agent_cli/commands/_helpers.py src/keboola_agent_cli/commands/permissions.py tests/test_helpers.py +git commit -m "refactor(safety): extract require_random_code_confirmation() to _helpers" +``` + +--- + +## Task 4: Add `DeveloperPortalIdentity` model + `AppConfig` fields + +**Files:** +- Modify: `src/keboola_agent_cli/models.py` +- Test: `tests/test_models.py` (extend) + +- [ ] **Step 1: Write failing model tests** + +Append to `tests/test_models.py`: + +```python +import pytest + +from keboola_agent_cli.models import AppConfig, DeveloperPortalIdentity + + +class TestDeveloperPortalIdentity: + def test_minimal_construction(self): + ident = DeveloperPortalIdentity(username="service.keboola.x", password="p") + assert ident.username == "service.keboola.x" + assert ident.password == "p" + assert ident.role_hint == "vendor" + assert ident.vendor is None + assert ident.portal_url == "https://apps-api.keboola.com" + + def test_rejects_non_https_portal_url(self): + with pytest.raises(ValueError, match="https"): + DeveloperPortalIdentity( + username="u", password="p", + portal_url="http://apps-api.keboola.com", + ) + + def test_accepts_staging_https_portal_url(self): + ident = DeveloperPortalIdentity( + username="u", password="p", + portal_url="https://apps-api.staging.keboola.dev", + ) + assert ident.portal_url == "https://apps-api.staging.keboola.dev" + + +class TestAppConfigDevPortalFields: + def test_defaults_empty(self): + cfg = AppConfig() + assert cfg.dev_portal_identities == {} + assert cfg.default_dev_portal_identity == "" + + def test_round_trip(self): + ident = DeveloperPortalIdentity(username="u", password="p", vendor="keboola") + cfg = AppConfig( + dev_portal_identities={"vendor-keboola": ident}, + default_dev_portal_identity="vendor-keboola", + ) + round = AppConfig.model_validate(cfg.model_dump(mode="json")) + assert round.dev_portal_identities["vendor-keboola"].vendor == "keboola" + assert round.default_dev_portal_identity == "vendor-keboola" +``` + +- [ ] **Step 2: Run, expect failure** + +Run: `uv run pytest tests/test_models.py::TestDeveloperPortalIdentity tests/test_models.py::TestAppConfigDevPortalFields -v` +Expected: `ImportError: cannot import name 'DeveloperPortalIdentity'`. + +- [ ] **Step 3: Add the model** + +In `src/keboola_agent_cli/models.py`, before `class AppConfig`, add: + +```python +class DeveloperPortalIdentity(BaseModel): + """One Developer Portal identity (service account or admin email). + + DP login is email + password (with MFA on personal accounts), producing + a short-lived bearer that lives only in process memory. The username + + password are persisted in config.json under the same 0600 protection as + KB Storage tokens; the bearer is never written to disk. + """ + + username: str = Field( + description="Email or service-account id used as the login subject" + ) + password: str = Field(description="DP password — same protection as KB tokens") + role_hint: str = Field( + default="vendor", + description=( + "Free-text label shown in `dev-portal identity list` " + "(e.g. 'vendor', 'admin'). Not validated against the portal." + ), + ) + vendor: str | None = Field( + default=None, + description=( + "Optional default vendor for this identity (e.g. 'keboola'). " + "Used as a default for commands that take --vendor; never " + "overrides an explicit flag." + ), + ) + portal_url: str = Field( + default="https://apps-api.keboola.com", + description="DP base URL. Override for staging/test portals.", + ) + + @field_validator("portal_url") + @classmethod + def validate_portal_url(cls, v: str) -> str: + if not v.startswith("https://"): + raise ValueError( + f"Portal URL must use https:// scheme, got: {v!r}" + ) + return v +``` + +In the same file, extend `class AppConfig` with two new fields (insert after the existing `projects:` field): + +```python + dev_portal_identities: dict[str, DeveloperPortalIdentity] = Field( + default_factory=dict, + description="Map of alias -> DeveloperPortalIdentity", + ) + default_dev_portal_identity: str = Field( + default="", + description="Alias of the default identity for `kbagent dev-portal` commands", + ) +``` + +- [ ] **Step 4: Run, expect pass** + +Run: `uv run pytest tests/test_models.py -v` +Expected: all pass. + +- [ ] **Step 5: Lint + commit** + +```bash +uv run ruff check src/keboola_agent_cli/models.py tests/test_models.py +uv run ruff format src/keboola_agent_cli/models.py tests/test_models.py +git add src/keboola_agent_cli/models.py tests/test_models.py +git commit -m "feat(dev-portal): add DeveloperPortalIdentity model + AppConfig fields" +``` + +--- + +## Task 5: Add `ConfigStore` methods for DP identities + +**Files:** +- Modify: `src/keboola_agent_cli/config_store.py` +- Test: `tests/test_config_store.py` (extend) + +- [ ] **Step 1: Write failing tests** + +Append to `tests/test_config_store.py`: + +```python +import pytest + +from keboola_agent_cli.errors import ConfigError +from keboola_agent_cli.models import DeveloperPortalIdentity + + +class TestDevPortalIdentityCrud: + def test_add_first_identity_becomes_default(self, config_store): + ident = DeveloperPortalIdentity(username="u", password="p") + config_store.add_dev_portal_identity("alpha", ident) + cfg = config_store.load() + assert cfg.dev_portal_identities["alpha"].username == "u" + assert cfg.default_dev_portal_identity == "alpha" + + def test_add_duplicate_alias_raises(self, config_store): + ident = DeveloperPortalIdentity(username="u", password="p") + config_store.add_dev_portal_identity("alpha", ident) + with pytest.raises(ConfigError, match="already exists"): + config_store.add_dev_portal_identity("alpha", ident) + + def test_remove_identity(self, config_store): + ident = DeveloperPortalIdentity(username="u", password="p") + config_store.add_dev_portal_identity("alpha", ident) + config_store.add_dev_portal_identity("beta", ident) + config_store.remove_dev_portal_identity("alpha") + cfg = config_store.load() + assert "alpha" not in cfg.dev_portal_identities + assert cfg.default_dev_portal_identity == "beta" + + def test_remove_unknown_raises(self, config_store): + with pytest.raises(ConfigError, match="not found"): + config_store.remove_dev_portal_identity("missing") + + def test_remove_last_clears_default(self, config_store): + ident = DeveloperPortalIdentity(username="u", password="p") + config_store.add_dev_portal_identity("alpha", ident) + config_store.remove_dev_portal_identity("alpha") + cfg = config_store.load() + assert cfg.default_dev_portal_identity == "" + + def test_edit_identity(self, config_store): + ident = DeveloperPortalIdentity(username="u", password="p") + config_store.add_dev_portal_identity("alpha", ident) + config_store.edit_dev_portal_identity("alpha", vendor="keboola", password="p2") + cfg = config_store.load() + assert cfg.dev_portal_identities["alpha"].vendor == "keboola" + assert cfg.dev_portal_identities["alpha"].password == "p2" + assert cfg.dev_portal_identities["alpha"].username == "u" + + def test_rename_identity(self, config_store): + ident = DeveloperPortalIdentity(username="u", password="p") + config_store.add_dev_portal_identity("alpha", ident) + config_store.rename_dev_portal_identity("alpha", "alpha-prod") + cfg = config_store.load() + assert "alpha" not in cfg.dev_portal_identities + assert "alpha-prod" in cfg.dev_portal_identities + assert cfg.default_dev_portal_identity == "alpha-prod" + + def test_rename_collision_raises(self, config_store): + ident = DeveloperPortalIdentity(username="u", password="p") + config_store.add_dev_portal_identity("alpha", ident) + config_store.add_dev_portal_identity("beta", ident) + with pytest.raises(ConfigError, match="already in use"): + config_store.rename_dev_portal_identity("alpha", "beta") + + def test_set_default_unknown_raises(self, config_store): + with pytest.raises(ConfigError, match="not found"): + config_store.set_default_dev_portal_identity("ghost") + + def test_set_default(self, config_store): + ident = DeveloperPortalIdentity(username="u", password="p") + config_store.add_dev_portal_identity("alpha", ident) + config_store.add_dev_portal_identity("beta", ident) + config_store.set_default_dev_portal_identity("beta") + cfg = config_store.load() + assert cfg.default_dev_portal_identity == "beta" +``` + +Note: the `config_store` fixture lives in `tests/conftest.py` — reuse it. + +- [ ] **Step 2: Run, expect failure** + +Run: `uv run pytest tests/test_config_store.py::TestDevPortalIdentityCrud -v` +Expected: `AttributeError: 'ConfigStore' object has no attribute 'add_dev_portal_identity'`. + +- [ ] **Step 3: Add the 5 mirror methods to `ConfigStore`** + +In `src/keboola_agent_cli/config_store.py`, after `rename_project()`, add: + +```python + def add_dev_portal_identity( + self, alias: str, identity: "DeveloperPortalIdentity" + ) -> None: + """Add a Developer Portal identity to the configuration. + + Sets it as default if no default identity is set. + + Raises: + ConfigError: If the alias already exists. + """ + config = self.load() + if alias in config.dev_portal_identities: + raise ConfigError( + f"Developer Portal identity '{alias}' already exists. " + "Use 'dev-portal identity edit' to modify it." + ) + config.dev_portal_identities[alias] = identity + if not config.default_dev_portal_identity: + config.default_dev_portal_identity = alias + self.save(config) + + def remove_dev_portal_identity(self, alias: str) -> None: + config = self.load() + if alias not in config.dev_portal_identities: + raise ConfigError(f"Developer Portal identity '{alias}' not found.") + del config.dev_portal_identities[alias] + if config.default_dev_portal_identity == alias: + config.default_dev_portal_identity = next( + iter(config.dev_portal_identities), "" + ) + self.save(config) + + def get_dev_portal_identity( + self, alias: str + ) -> "DeveloperPortalIdentity | None": + config = self.load() + return config.dev_portal_identities.get(alias) + + def edit_dev_portal_identity( + self, alias: str, **kwargs: str | None + ) -> None: + config = self.load() + if alias not in config.dev_portal_identities: + raise ConfigError(f"Developer Portal identity '{alias}' not found.") + ident = config.dev_portal_identities[alias] + for key, value in kwargs.items(): + if hasattr(ident, key) and value is not None: + setattr(ident, key, value) + config.dev_portal_identities[alias] = ident + self.save(config) + + def rename_dev_portal_identity(self, old_alias: str, new_alias: str) -> None: + config = self.load() + if old_alias not in config.dev_portal_identities: + raise ConfigError(f"Developer Portal identity '{old_alias}' not found.") + if new_alias in config.dev_portal_identities: + raise ConfigError( + f"Cannot rename '{old_alias}' to '{new_alias}': " + f"alias '{new_alias}' is already in use." + ) + config.dev_portal_identities[new_alias] = ( + config.dev_portal_identities.pop(old_alias) + ) + if config.default_dev_portal_identity == old_alias: + config.default_dev_portal_identity = new_alias + self.save(config) + + def set_default_dev_portal_identity(self, alias: str) -> None: + config = self.load() + if alias not in config.dev_portal_identities: + raise ConfigError(f"Developer Portal identity '{alias}' not found.") + config.default_dev_portal_identity = alias + self.save(config) +``` + +Also at the top of the file, add: + +```python +from .models import AppConfig, DeveloperPortalIdentity, ProjectConfig +``` + +(The existing import line already imports `AppConfig, ProjectConfig` — extend it.) + +- [ ] **Step 4: Extend the `_warning` header** + +Edit `CLAUDE_CONFIG_WARNING` near the top of `src/keboola_agent_cli/config_store.py` to append: + +``` +" Developer Portal credentials stored here have the SAME risk profile -- " +"never call apps-api.keboola.com directly; use `kbagent dev-portal ...`." +``` + +(Append inside the existing parenthesised string literal — keep it one logical sentence so JSON serialisation stays clean.) + +- [ ] **Step 5: Run, expect pass** + +Run: `uv run pytest tests/test_config_store.py -v` +Expected: all pass (existing project tests + new DP tests). + +- [ ] **Step 6: Lint + commit** + +```bash +uv run ruff check src/keboola_agent_cli/config_store.py tests/test_config_store.py +uv run ruff format src/keboola_agent_cli/config_store.py tests/test_config_store.py +git add src/keboola_agent_cli/config_store.py tests/test_config_store.py +git commit -m "feat(dev-portal): ConfigStore methods for identity CRUD" +``` + +--- + +## Task 6: `DeveloperPortalClient` skeleton + login (token path) + +**Files:** +- Create: `src/keboola_agent_cli/dev_portal_client.py` +- Create: `tests/test_dev_portal_client.py` + +- [ ] **Step 1: Write failing test for token-path login** + +Create `tests/test_dev_portal_client.py`: + +```python +"""Tests for DeveloperPortalClient — login, MFA, CRUD against apps-api.""" + +from __future__ import annotations + +import pytest + +from keboola_agent_cli.dev_portal_client import DeveloperPortalClient +from keboola_agent_cli.errors import ErrorCode, KeboolaApiError +from keboola_agent_cli.models import DeveloperPortalIdentity + + +def _identity(**overrides) -> DeveloperPortalIdentity: + defaults = dict(username="service.keboola.x", password="p") + defaults.update(overrides) + return DeveloperPortalIdentity(**defaults) + + +class TestLoginTokenPath: + def test_login_returns_bearer(self, httpx_mock): + httpx_mock.add_response( + method="POST", + url="https://apps-api.keboola.com/auth/login", + json={"token": "Bearer abc"}, + status_code=200, + ) + with DeveloperPortalClient(_identity()) as client: + client._ensure_authenticated() + assert client._bearer == "Bearer abc" + assert len(httpx_mock.get_requests()) == 1 + + def test_login_bad_credentials_raises(self, httpx_mock): + httpx_mock.add_response( + method="POST", + url="https://apps-api.keboola.com/auth/login", + json={"error": "invalid credentials"}, + status_code=401, + ) + with DeveloperPortalClient(_identity()) as client: + with pytest.raises(KeboolaApiError) as exc: + client._ensure_authenticated() + assert exc.value.error_code == ErrorCode.DP_LOGIN_FAILED +``` + +- [ ] **Step 2: Run, expect failure** + +Run: `uv run pytest tests/test_dev_portal_client.py::TestLoginTokenPath -v` +Expected: `ModuleNotFoundError: No module named 'keboola_agent_cli.dev_portal_client'`. + +- [ ] **Step 3: Create the client skeleton** + +Create `src/keboola_agent_cli/dev_portal_client.py`: + +```python +"""Keboola Developer Portal HTTP client (apps-api.keboola.com). + +Auth model: +- Login (email + password) returns a bearer token. On a personal account, the + first login returns an MFA session; we prompt the user via /dev/tty and + re-login with {email, session, code} to obtain the bearer. +- The bearer lives ONLY on this client instance (in self._bearer). It is + never written to disk, never logged, and discarded when the client closes. +- Each kbagent invocation logs in fresh; there is no token cache. + +The client is intentionally dumb: dry-run, diff, and confirm logic belong to +the service and command layers. +""" + +from __future__ import annotations + +import json +import logging +import sys +import urllib.error +import urllib.request +from typing import Any + +import httpx + +from .errors import ErrorCode, KeboolaApiError +from .http_base import BaseHttpClient +from .models import DeveloperPortalIdentity + +logger = logging.getLogger(__name__) + + +class DeveloperPortalClient(BaseHttpClient): + """HTTP client for the Keboola Developer Portal.""" + + def __init__(self, identity: DeveloperPortalIdentity) -> None: + # We don't have a bearer yet — pass empty token. Login populates it. + super().__init__( + base_url=identity.portal_url, + token="", + headers={"Accept": "application/json"}, + ) + self._identity = identity + self._bearer: str | None = None + + def _ensure_authenticated(self) -> None: + """Log in if not already authenticated. Idempotent on the instance.""" + if self._bearer is not None: + return + self._bearer = self._login(self._identity.username, self._identity.password) + self._client.headers["Authorization"] = self._bearer + + def _login(self, username: str, password: str) -> str: + try: + resp = self._client.post( + "/auth/login", + json={"email": username, "password": password}, + ) + except httpx.HTTPError as exc: + raise KeboolaApiError( + message=f"Developer Portal login transport error: {exc}", + error_code=ErrorCode.CONNECTION_ERROR, + ) from exc + if resp.status_code != 200: + raise KeboolaApiError( + message=( + f"Developer Portal login failed (HTTP {resp.status_code}). " + "Check the identity credentials." + ), + error_code=ErrorCode.DP_LOGIN_FAILED, + ) + payload = resp.json() + if isinstance(payload, dict) and payload.get("token"): + return payload["token"] + # MFA path — implemented in Task 7. + if isinstance(payload, dict) and payload.get("session"): + return self._login_with_mfa(username, payload["session"]) + raise KeboolaApiError( + message="Developer Portal login response missing token and session", + error_code=ErrorCode.DP_LOGIN_FAILED, + ) + + def _login_with_mfa(self, username: str, session: str) -> str: + # Placeholder — implemented in Task 7. + raise KeboolaApiError( + message="MFA login not implemented yet", + error_code=ErrorCode.DP_MFA_REQUIRED, + ) +``` + +- [ ] **Step 4: Run, expect pass** + +Run: `uv run pytest tests/test_dev_portal_client.py::TestLoginTokenPath -v` +Expected: both tests pass. + +- [ ] **Step 5: Lint + commit** + +```bash +uv run ruff check src/keboola_agent_cli/dev_portal_client.py tests/test_dev_portal_client.py +uv run ruff format src/keboola_agent_cli/dev_portal_client.py tests/test_dev_portal_client.py +git add src/keboola_agent_cli/dev_portal_client.py tests/test_dev_portal_client.py +git commit -m "feat(dev-portal): client skeleton + token-path login" +``` + +--- + +## Task 7: MFA login path + TTY prompt + +**Files:** +- Modify: `src/keboola_agent_cli/dev_portal_client.py` +- Modify: `tests/test_dev_portal_client.py` + +- [ ] **Step 1: Write failing tests** + +Append to `tests/test_dev_portal_client.py`: + +```python +class TestLoginMfaPath: + def test_mfa_prompt_completes_login(self, httpx_mock, monkeypatch): + httpx_mock.add_response( + method="POST", + url="https://apps-api.keboola.com/auth/login", + json={"session": "sess-1"}, + status_code=200, + match_json={"email": "u@k.com", "password": "p"}, + ) + httpx_mock.add_response( + method="POST", + url="https://apps-api.keboola.com/auth/login", + json={"token": "Bearer xyz"}, + status_code=200, + match_json={"email": "u@k.com", "session": "sess-1", "code": "123456"}, + ) + # Mock the /dev/tty MFA prompt. + monkeypatch.setattr( + "keboola_agent_cli.dev_portal_client._tty_prompt", + lambda label, secret=False: "123456", + ) + ident = DeveloperPortalIdentity(username="u@k.com", password="p") + with DeveloperPortalClient(ident) as client: + client._ensure_authenticated() + assert client._bearer == "Bearer xyz" + + def test_mfa_no_tty_raises_mfa_required(self, httpx_mock, monkeypatch): + httpx_mock.add_response( + method="POST", + url="https://apps-api.keboola.com/auth/login", + json={"session": "sess-1"}, + status_code=200, + ) + # _tty_prompt returns None when no terminal is available. + monkeypatch.setattr( + "keboola_agent_cli.dev_portal_client._tty_prompt", + lambda label, secret=False: None, + ) + ident = DeveloperPortalIdentity(username="u@k.com", password="p") + with DeveloperPortalClient(ident) as client: + with pytest.raises(KeboolaApiError) as exc: + client._ensure_authenticated() + assert exc.value.error_code == ErrorCode.DP_MFA_REQUIRED +``` + +- [ ] **Step 2: Run, expect failure** + +Run: `uv run pytest tests/test_dev_portal_client.py::TestLoginMfaPath -v` +Expected: first test fails (no `_tty_prompt` exists yet); second fails because `_login_with_mfa` raises immediately. + +- [ ] **Step 3: Implement `_tty_prompt` + full MFA path** + +In `src/keboola_agent_cli/dev_portal_client.py`, add at module level (above the class): + +```python +def _tty_prompt(label: str, *, secret: bool = False) -> str | None: + """Prompt via the controlling terminal so a redirected stdin can't break it. + + Returns None when no /dev/tty is available (non-interactive shell, no + controlling terminal). Caller must treat None as "cannot prompt". + """ + try: + with open("/dev/tty", "w") as out: + if secret: + import getpass + + return getpass.getpass(label, stream=out) + out.write(label) + out.flush() + with open("/dev/tty", "r") as tin: + return tin.readline().rstrip("\n") + except OSError: + return None +``` + +Replace the placeholder `_login_with_mfa` body: + +```python + def _login_with_mfa(self, username: str, session: str) -> str: + code = _tty_prompt("MFA code: ") + if not code: + raise KeboolaApiError( + message=( + "Developer Portal identity requires an MFA code, but no " + "interactive terminal is available. Run from a real " + "terminal, or switch to a service.{vendor}.{id} " + "account (no MFA)." + ), + error_code=ErrorCode.DP_MFA_REQUIRED, + ) + try: + resp = self._client.post( + "/auth/login", + json={"email": username, "session": session, "code": code.strip()}, + ) + except httpx.HTTPError as exc: + raise KeboolaApiError( + message=f"Developer Portal MFA login transport error: {exc}", + error_code=ErrorCode.CONNECTION_ERROR, + ) from exc + if resp.status_code != 200: + raise KeboolaApiError( + message=f"Developer Portal MFA login failed (HTTP {resp.status_code})", + error_code=ErrorCode.DP_LOGIN_FAILED, + ) + payload = resp.json() + if not isinstance(payload, dict) or not payload.get("token"): + raise KeboolaApiError( + message="Developer Portal MFA login response missing token", + error_code=ErrorCode.DP_LOGIN_FAILED, + ) + return payload["token"] +``` + +- [ ] **Step 4: Run, expect pass** + +Run: `uv run pytest tests/test_dev_portal_client.py -v` +Expected: all 4 tests pass (2 token-path + 2 MFA). + +- [ ] **Step 5: Lint + commit** + +```bash +uv run ruff check src/keboola_agent_cli/dev_portal_client.py tests/test_dev_portal_client.py +uv run ruff format src/keboola_agent_cli/dev_portal_client.py tests/test_dev_portal_client.py +git add src/keboola_agent_cli/dev_portal_client.py tests/test_dev_portal_client.py +git commit -m "feat(dev-portal): MFA login path via /dev/tty" +``` + +--- + +## Task 8: Client reads + standard writes (list/get/create/patch/publish/deprecate) + +**Files:** +- Modify: `src/keboola_agent_cli/dev_portal_client.py` +- Modify: `tests/test_dev_portal_client.py` + +- [ ] **Step 1: Write failing tests** + +Append to `tests/test_dev_portal_client.py`: + +```python +class TestPortalReads: + def test_list_apps(self, httpx_mock): + httpx_mock.add_response( + method="POST", + url="https://apps-api.keboola.com/auth/login", + json={"token": "Bearer abc"}, + ) + httpx_mock.add_response( + method="GET", + url="https://apps-api.keboola.com/vendors/keboola/apps?limit=1000", + json={"apps": [{"id": "keboola.ex-foo"}]}, + ) + with DeveloperPortalClient(_identity()) as client: + apps = client.list_apps("keboola") + assert apps == [{"id": "keboola.ex-foo"}] + + def test_get_app_404(self, httpx_mock): + httpx_mock.add_response( + method="POST", + url="https://apps-api.keboola.com/auth/login", + json={"token": "Bearer abc"}, + ) + httpx_mock.add_response( + method="GET", + url="https://apps-api.keboola.com/vendors/keboola/apps/keboola.missing", + status_code=404, + json={"error": "not found"}, + ) + with DeveloperPortalClient(_identity()) as client: + with pytest.raises(KeboolaApiError) as exc: + client.get_app("keboola", "keboola.missing") + assert exc.value.error_code == ErrorCode.DP_APP_NOT_FOUND + + +class TestPortalWrites: + def test_create_app(self, httpx_mock): + httpx_mock.add_response( + method="POST", url="https://apps-api.keboola.com/auth/login", + json={"token": "Bearer abc"}, + ) + httpx_mock.add_response( + method="POST", + url="https://apps-api.keboola.com/vendors/keboola/apps", + json={"id": "ex-foo", "name": "Foo"}, + ) + with DeveloperPortalClient(_identity()) as client: + resp = client.create_app("keboola", {"id": "ex-foo", "name": "Foo", "type": "extractor"}) + assert resp["id"] == "ex-foo" + + def test_patch_app(self, httpx_mock): + httpx_mock.add_response( + method="POST", url="https://apps-api.keboola.com/auth/login", + json={"token": "Bearer abc"}, + ) + httpx_mock.add_response( + method="PATCH", + url="https://apps-api.keboola.com/vendors/keboola/apps/keboola.ex-foo", + json={"id": "ex-foo", "name": "Foo 2"}, + ) + with DeveloperPortalClient(_identity()) as client: + resp = client.patch_app("keboola", "keboola.ex-foo", {"name": "Foo 2"}) + assert resp["name"] == "Foo 2" + + def test_publish_app(self, httpx_mock): + httpx_mock.add_response( + method="POST", url="https://apps-api.keboola.com/auth/login", + json={"token": "Bearer abc"}, + ) + httpx_mock.add_response( + method="POST", + url="https://apps-api.keboola.com/vendors/keboola/apps/keboola.ex-foo/publish", + json={"status": "submitted"}, + ) + with DeveloperPortalClient(_identity()) as client: + assert client.publish_app("keboola", "keboola.ex-foo")["status"] == "submitted" + + def test_deprecate_app(self, httpx_mock): + httpx_mock.add_response( + method="POST", url="https://apps-api.keboola.com/auth/login", + json={"token": "Bearer abc"}, + ) + httpx_mock.add_response( + method="POST", + url="https://apps-api.keboola.com/vendors/keboola/apps/keboola.ex-foo/deprecate", + json={"status": "deprecated"}, + ) + with DeveloperPortalClient(_identity()) as client: + assert client.deprecate_app("keboola", "keboola.ex-foo")["status"] == "deprecated" +``` + +- [ ] **Step 2: Run, expect failure** + +Run: `uv run pytest tests/test_dev_portal_client.py::TestPortalReads tests/test_dev_portal_client.py::TestPortalWrites -v` +Expected: `AttributeError: 'DeveloperPortalClient' object has no attribute 'list_apps'`. + +- [ ] **Step 3: Implement the methods** + +Append to the `DeveloperPortalClient` class in `src/keboola_agent_cli/dev_portal_client.py`: + +```python + # ----- Reads ----- + + def list_apps(self, vendor: str) -> list[dict[str, Any]]: + self._ensure_authenticated() + resp = self._do_request("GET", f"/vendors/{vendor}/apps?limit=1000") + if resp.status_code != 200: + self._raise_dp_error(resp, action="list apps", vendor=vendor) + payload = resp.json() + if isinstance(payload, dict) and "apps" in payload: + return list(payload["apps"]) + if isinstance(payload, list): + return payload + return [] + + def get_app(self, vendor: str, app_id: str) -> dict[str, Any]: + self._ensure_authenticated() + resp = self._do_request("GET", f"/vendors/{vendor}/apps/{app_id}") + if resp.status_code == 404: + raise KeboolaApiError( + message=f"Developer Portal app '{app_id}' not found in vendor '{vendor}'", + error_code=ErrorCode.DP_APP_NOT_FOUND, + ) + if resp.status_code != 200: + self._raise_dp_error(resp, action="get app", vendor=vendor, app_id=app_id) + return resp.json() + + # ----- Writes ----- + + def create_app(self, vendor: str, payload: dict[str, Any]) -> dict[str, Any]: + self._ensure_authenticated() + resp = self._do_request( + "POST", f"/vendors/{vendor}/apps", json=payload + ) + if resp.status_code not in (200, 201): + self._raise_dp_error(resp, action="create app", vendor=vendor) + return resp.json() + + def patch_app( + self, vendor: str, app_id: str, payload: dict[str, Any] + ) -> dict[str, Any]: + self._ensure_authenticated() + resp = self._do_request( + "PATCH", f"/vendors/{vendor}/apps/{app_id}", json=payload + ) + if resp.status_code not in (200, 204): + self._raise_dp_error( + resp, action="patch app", vendor=vendor, app_id=app_id + ) + return resp.json() if resp.content else {} + + def publish_app(self, vendor: str, app_id: str) -> dict[str, Any]: + self._ensure_authenticated() + resp = self._do_request( + "POST", f"/vendors/{vendor}/apps/{app_id}/publish" + ) + if resp.status_code not in (200, 202): + self._raise_dp_error( + resp, action="publish app", vendor=vendor, app_id=app_id + ) + return resp.json() if resp.content else {"status": "submitted"} + + def deprecate_app(self, vendor: str, app_id: str) -> dict[str, Any]: + self._ensure_authenticated() + resp = self._do_request( + "POST", f"/vendors/{vendor}/apps/{app_id}/deprecate" + ) + if resp.status_code not in (200, 202): + self._raise_dp_error( + resp, action="deprecate app", vendor=vendor, app_id=app_id + ) + return resp.json() if resp.content else {"status": "deprecated"} + + # ----- Error mapping ----- + + def _raise_dp_error( + self, + resp: httpx.Response, + *, + action: str, + vendor: str | None = None, + app_id: str | None = None, + ) -> None: + try: + body = resp.json() + except ValueError: + body = resp.text + ctx = f"{action}" + if vendor: + ctx += f" (vendor={vendor})" + if app_id: + ctx += f" (app={app_id})" + raise KeboolaApiError( + message=f"Developer Portal {ctx} failed (HTTP {resp.status_code}): {body}", + error_code=ErrorCode.API_ERROR, + ) +``` + +- [ ] **Step 4: Run, expect pass** + +Run: `uv run pytest tests/test_dev_portal_client.py -v` +Expected: all tests pass. + +- [ ] **Step 5: Lint + commit** + +```bash +uv run ruff check src/keboola_agent_cli/dev_portal_client.py tests/test_dev_portal_client.py +uv run ruff format src/keboola_agent_cli/dev_portal_client.py tests/test_dev_portal_client.py +git add src/keboola_agent_cli/dev_portal_client.py tests/test_dev_portal_client.py +git commit -m "feat(dev-portal): client reads + create/patch/publish/deprecate" +``` + +--- + +## Task 9: Client icon upload (two-hop) + +**Files:** +- Modify: `src/keboola_agent_cli/dev_portal_client.py` +- Modify: `tests/test_dev_portal_client.py` + +- [ ] **Step 1: Write failing test** + +Append to `tests/test_dev_portal_client.py`: + +```python +class TestIconUpload: + def test_upload_icon_two_hop(self, httpx_mock, monkeypatch): + httpx_mock.add_response( + method="POST", url="https://apps-api.keboola.com/auth/login", + json={"token": "Bearer abc"}, + ) + httpx_mock.add_response( + method="POST", + url="https://apps-api.keboola.com/vendors/keboola/apps/keboola.ex-foo/icon", + json={"link": "https://s3.example/presigned"}, + ) + # The S3 PUT bypasses httpx; we mock urllib.request.urlopen. + seen = {} + class _FakeResp: + status = 200 + def __enter__(self): return self + def __exit__(self, *a): pass + def fake_urlopen(req): + seen["url"] = req.full_url + seen["data"] = req.data + seen["method"] = req.method + return _FakeResp() + monkeypatch.setattr("urllib.request.urlopen", fake_urlopen) + + with DeveloperPortalClient(_identity()) as client: + client.upload_icon("keboola", "keboola.ex-foo", b"\x89PNG\r\n\x1a\nrest") + assert seen["url"] == "https://s3.example/presigned" + assert seen["data"] == b"\x89PNG\r\n\x1a\nrest" + assert seen["method"] == "PUT" + + def test_upload_icon_presign_failure(self, httpx_mock): + httpx_mock.add_response( + method="POST", url="https://apps-api.keboola.com/auth/login", + json={"token": "Bearer abc"}, + ) + httpx_mock.add_response( + method="POST", + url="https://apps-api.keboola.com/vendors/keboola/apps/keboola.ex-foo/icon", + status_code=500, + json={"error": "boom"}, + ) + with DeveloperPortalClient(_identity()) as client: + with pytest.raises(KeboolaApiError) as exc: + client.upload_icon("keboola", "keboola.ex-foo", b"data") + assert exc.value.error_code == ErrorCode.DP_ICON_UPLOAD_FAILED +``` + +- [ ] **Step 2: Run, expect failure** + +Run: `uv run pytest tests/test_dev_portal_client.py::TestIconUpload -v` +Expected: `AttributeError: ... 'upload_icon'`. + +- [ ] **Step 3: Implement `upload_icon`** + +Append to the `DeveloperPortalClient` class: + +```python + def upload_icon(self, vendor: str, app_id: str, png_bytes: bytes) -> None: + """Two-hop icon upload: ask the portal for a presigned S3 URL, then PUT bytes there. + + The S3 PUT does NOT use this client's httpx instance (no retry, no auth, + no User-Agent injection). We use urllib directly so the wire shape stays + exactly what S3 expects. + """ + self._ensure_authenticated() + resp = self._do_request( + "POST", f"/vendors/{vendor}/apps/{app_id}/icon" + ) + if resp.status_code != 200: + raise KeboolaApiError( + message=( + f"Developer Portal failed to mint icon-upload URL " + f"(HTTP {resp.status_code})" + ), + error_code=ErrorCode.DP_ICON_UPLOAD_FAILED, + ) + payload = resp.json() + link = payload.get("link") if isinstance(payload, dict) else None + if not link: + raise KeboolaApiError( + message="Developer Portal icon-upload response missing 'link'", + error_code=ErrorCode.DP_ICON_UPLOAD_FAILED, + ) + req = urllib.request.Request( + link, data=png_bytes, + headers={"Content-Type": "image/png"}, method="PUT", + ) + try: + with urllib.request.urlopen(req) as s3_resp: + if getattr(s3_resp, "status", 200) >= 300: + raise KeboolaApiError( + message=f"Icon S3 PUT failed (HTTP {s3_resp.status})", + error_code=ErrorCode.DP_ICON_UPLOAD_FAILED, + ) + except urllib.error.HTTPError as exc: + raise KeboolaApiError( + message=f"Icon S3 PUT failed (HTTP {exc.code}): {exc.reason}", + error_code=ErrorCode.DP_ICON_UPLOAD_FAILED, + ) from exc +``` + +- [ ] **Step 4: Run, expect pass** + +Run: `uv run pytest tests/test_dev_portal_client.py -v` +Expected: all client tests pass. + +- [ ] **Step 5: Lint + commit** + +```bash +uv run ruff check src/keboola_agent_cli/dev_portal_client.py tests/test_dev_portal_client.py +uv run ruff format src/keboola_agent_cli/dev_portal_client.py tests/test_dev_portal_client.py +git add src/keboola_agent_cli/dev_portal_client.py tests/test_dev_portal_client.py +git commit -m "feat(dev-portal): icon upload (two-hop, presigned S3 PUT)" +``` + +--- + +## Task 10: `DeveloperPortalService` identity CRUD + verify-on-add + +**Files:** +- Create: `src/keboola_agent_cli/services/dev_portal_service.py` +- Create: `tests/test_dev_portal_service.py` + +- [ ] **Step 1: Write failing tests** + +Create `tests/test_dev_portal_service.py`: + +```python +"""Tests for DeveloperPortalService — identity CRUD, prepare/apply, diff, validation.""" + +from __future__ import annotations + +from unittest.mock import MagicMock + +import pytest + +from keboola_agent_cli.errors import ConfigError, ErrorCode, KeboolaApiError +from keboola_agent_cli.models import DeveloperPortalIdentity +from keboola_agent_cli.services.dev_portal_service import DeveloperPortalService + + +@pytest.fixture +def fake_client(): + return MagicMock() + + +@pytest.fixture +def service(config_store, fake_client): + factory = lambda identity: fake_client + return DeveloperPortalService(config_store=config_store, client_factory=factory) + + +class TestIdentityCrud: + def test_add_and_list(self, service, fake_client): + # add_identity also runs verify (login probe). + fake_client.list_apps.return_value = [] + ident = DeveloperPortalIdentity(username="u", password="p") + service.add_identity("alpha", ident) + result = service.list_identities() + assert "alpha" in result + assert result["alpha"].username == "u" + + def test_add_verify_failure_does_not_persist(self, service, fake_client, config_store): + fake_client._ensure_authenticated.side_effect = KeboolaApiError( + message="bad creds", error_code=ErrorCode.DP_LOGIN_FAILED, + ) + ident = DeveloperPortalIdentity(username="u", password="bad") + with pytest.raises(KeboolaApiError) as exc: + service.add_identity("alpha", ident) + assert exc.value.error_code == ErrorCode.DP_LOGIN_FAILED + assert config_store.load().dev_portal_identities == {} + + def test_use_sets_default(self, service, fake_client, config_store): + fake_client._ensure_authenticated.return_value = None + ident = DeveloperPortalIdentity(username="u", password="p") + service.add_identity("alpha", ident) + service.add_identity("beta", ident) + service.use_identity("beta") + assert config_store.load().default_dev_portal_identity == "beta" + + def test_remove(self, service, fake_client, config_store): + fake_client._ensure_authenticated.return_value = None + ident = DeveloperPortalIdentity(username="u", password="p") + service.add_identity("alpha", ident) + service.remove_identity("alpha") + assert "alpha" not in config_store.load().dev_portal_identities +``` + +- [ ] **Step 2: Run, expect failure** + +Run: `uv run pytest tests/test_dev_portal_service.py::TestIdentityCrud -v` +Expected: `ModuleNotFoundError: No module named 'keboola_agent_cli.services.dev_portal_service'`. + +- [ ] **Step 3: Create the service skeleton with identity CRUD** + +Create `src/keboola_agent_cli/services/dev_portal_service.py`: + +```python +"""Developer Portal business logic. + +Identity CRUD + prepare/apply discipline for portal writes. Commands stay +thin; this module owns diff computation, publish pre-flight validation, +and the verify-on-add login probe. +""" + +from __future__ import annotations + +from typing import Any, Callable + +from ..config_store import ConfigStore +from ..dev_portal_client import DeveloperPortalClient +from ..errors import ConfigError +from ..models import DeveloperPortalIdentity + + +ClientFactory = Callable[[DeveloperPortalIdentity], DeveloperPortalClient] + + +class DeveloperPortalService: + def __init__( + self, + config_store: ConfigStore, + client_factory: ClientFactory, + ) -> None: + self._store = config_store + self._client_factory = client_factory + + # ----- Identity management ----- + + def add_identity(self, alias: str, identity: DeveloperPortalIdentity) -> None: + """Verify creds (login probe) BEFORE persisting. + + Same UX as `kbagent project add` (which calls verify_token first): + bad creds fail fast and never land in config.json. + """ + with self._client_factory(identity) as client: + client._ensure_authenticated() # raises on bad creds / MFA failure + self._store.add_dev_portal_identity(alias, identity) + + def list_identities(self) -> dict[str, DeveloperPortalIdentity]: + return dict(self._store.load().dev_portal_identities) + + def remove_identity(self, alias: str) -> None: + self._store.remove_dev_portal_identity(alias) + + def edit_identity(self, alias: str, **fields: Any) -> None: + self._store.edit_dev_portal_identity(alias, **fields) + + def rename_identity(self, old_alias: str, new_alias: str) -> None: + self._store.rename_dev_portal_identity(old_alias, new_alias) + + def use_identity(self, alias: str) -> None: + self._store.set_default_dev_portal_identity(alias) + + def current_identity(self) -> str: + return self._store.load().default_dev_portal_identity + + def verify_identity(self, alias: str) -> dict[str, str]: + ident = self._resolve_identity(alias) + with self._client_factory(ident) as client: + client._ensure_authenticated() + return {"alias": alias, "username": ident.username} + + # ----- Internal ----- + + def _resolve_identity(self, alias: str) -> DeveloperPortalIdentity: + ident = self._store.get_dev_portal_identity(alias) + if ident is None: + raise ConfigError( + f"Developer Portal identity '{alias}' not found. " + "Run `kbagent dev-portal identity list` to see configured identities." + ) + return ident +``` + +Also create `src/keboola_agent_cli/services/__init__.py` if it doesn't already exist (it does — this is just a sanity check). + +- [ ] **Step 4: Run, expect pass** + +Run: `uv run pytest tests/test_dev_portal_service.py::TestIdentityCrud -v` +Expected: all 4 tests pass. + +- [ ] **Step 5: Lint + commit** + +```bash +uv run ruff check src/keboola_agent_cli/services/dev_portal_service.py tests/test_dev_portal_service.py +uv run ruff format src/keboola_agent_cli/services/dev_portal_service.py tests/test_dev_portal_service.py +git add src/keboola_agent_cli/services/dev_portal_service.py tests/test_dev_portal_service.py +git commit -m "feat(dev-portal): service with identity CRUD + verify-on-add" +``` + +--- + +## Task 11: Service reads + `prepare_*`/`apply` (diff + publish pre-flight) + +**Files:** +- Modify: `src/keboola_agent_cli/services/dev_portal_service.py` +- Modify: `tests/test_dev_portal_service.py` + +- [ ] **Step 1: Write failing tests** + +Append to `tests/test_dev_portal_service.py`: + +```python +class TestReadsAndPrepareApply: + def _setup(self, service, fake_client): + fake_client._ensure_authenticated.return_value = None + ident = DeveloperPortalIdentity(username="u", password="p") + service.add_identity("alpha", ident) + + def test_list_apps(self, service, fake_client): + self._setup(service, fake_client) + fake_client.list_apps.return_value = [{"id": "ex-a"}] + assert service.list_apps("alpha", "keboola") == [{"id": "ex-a"}] + fake_client.list_apps.assert_called_with("keboola") + + def test_get_app(self, service, fake_client): + self._setup(service, fake_client) + fake_client.get_app.return_value = {"id": "ex-a", "name": "Hello"} + assert service.get_app("alpha", "keboola", "keboola.ex-a")["name"] == "Hello" + + def test_prepare_create_requires_id_name_type(self, service, fake_client): + self._setup(service, fake_client) + with pytest.raises(KeboolaApiError, match="payload must include 'id'"): + service.prepare_create("alpha", "keboola", {"name": "F", "type": "extractor"}) + + def test_prepare_create_rejects_banned_words_in_name(self, service, fake_client): + self._setup(service, fake_client) + with pytest.raises(KeboolaApiError, match="must not contain"): + service.prepare_create( + "alpha", "keboola", + {"id": "x", "name": "Foo extractor", "type": "extractor"}, + ) + + def test_prepare_patch_diff(self, service, fake_client): + self._setup(service, fake_client) + fake_client.get_app.return_value = { + "id": "ex-a", "name": "Old", "shortDescription": "same", + } + pending = service.prepare_patch( + "alpha", "keboola", "keboola.ex-a", + {"name": "New", "shortDescription": "same"}, + ) + keys = {d.key for d in pending.diff} + assert keys == {"name"} # shortDescription unchanged is filtered out + assert pending.diff[0].current == "Old" + assert pending.diff[0].new == "New" + + def test_apply_patch_calls_client(self, service, fake_client): + self._setup(service, fake_client) + fake_client.get_app.return_value = {"id": "ex-a", "name": "Old"} + fake_client.patch_app.return_value = {"id": "ex-a", "name": "New"} + pending = service.prepare_patch( + "alpha", "keboola", "keboola.ex-a", {"name": "New"} + ) + result = service.apply(pending) + assert result["name"] == "New" + fake_client.patch_app.assert_called_with("keboola", "keboola.ex-a", {"name": "New"}) + + def test_prepare_publish_missing_fields(self, service, fake_client): + self._setup(service, fake_client) + fake_client.get_app.return_value = { + "id": "ex-a", "name": "Foo", "type": "extractor", + # missing icon, repository, descriptions, license, docs + } + with pytest.raises(KeboolaApiError) as exc: + service.prepare_publish("alpha", "keboola", "keboola.ex-a") + assert exc.value.error_code == ErrorCode.DP_PUBLISH_REQUIREMENTS_MISSING + assert "icon" in str(exc.value) +``` + +- [ ] **Step 2: Run, expect failure** + +Run: `uv run pytest tests/test_dev_portal_service.py::TestReadsAndPrepareApply -v` +Expected: `AttributeError: ... 'list_apps'`. + +- [ ] **Step 3: Implement `prepare_*` + `apply` + helpers + dataclasses** + +Add at the top of `src/keboola_agent_cli/services/dev_portal_service.py`, near the imports: + +```python +from dataclasses import dataclass, field +from pathlib import Path + +from ..errors import ErrorCode, KeboolaApiError +``` + +After the existing imports/class declarations (above `class DeveloperPortalService`), add the dataclasses: + +```python +@dataclass(frozen=True) +class FieldDiff: + key: str + current: Any + new: Any + + +@dataclass(frozen=True) +class PendingWrite: + """Base for any prepared portal write. apply() in the service dispatches on the subclass.""" + alias: str + vendor: str + + +@dataclass(frozen=True) +class PendingCreate(PendingWrite): + payload: dict[str, Any] + + +@dataclass(frozen=True) +class PendingPatch(PendingWrite): + app_id: str + payload: dict[str, Any] + current: dict[str, Any] + diff: list[FieldDiff] = field(default_factory=list) + + +@dataclass(frozen=True) +class PendingIconUpload(PendingWrite): + app_id: str + png_path: Path + png_bytes: bytes + + +@dataclass(frozen=True) +class PendingPublish(PendingWrite): + app_id: str + current: dict[str, Any] + + +@dataclass(frozen=True) +class PendingDeprecate(PendingWrite): + app_id: str + + +_BANNED_NAME_WORDS = ("extractor", "writer") +_REQUIRED_PUBLISH_FIELDS = ( + "icon", "name", "type", "repository", + "shortDescription", "longDescription", + "licenseUrl", "documentationUrl", +) +``` + +Append methods to `DeveloperPortalService`: + +```python + # ----- Reads ----- + + def list_apps(self, alias: str, vendor: str) -> list[dict[str, Any]]: + ident = self._resolve_identity(alias) + with self._client_factory(ident) as client: + return client.list_apps(vendor) + + def get_app( + self, alias: str, vendor: str, app_id: str + ) -> dict[str, Any]: + ident = self._resolve_identity(alias) + with self._client_factory(ident) as client: + return client.get_app(vendor, app_id) + + # ----- Prepare (no portal write yet) ----- + + def prepare_create( + self, alias: str, vendor: str, payload: dict[str, Any] + ) -> PendingCreate: + for required in ("id", "name", "type"): + if required not in payload: + raise KeboolaApiError( + message=f"create payload must include '{required}'", + error_code=ErrorCode.VALIDATION_ERROR, + ) + name_lower = str(payload["name"]).lower() + for banned in _BANNED_NAME_WORDS: + if banned in name_lower: + raise KeboolaApiError( + message=( + f"App name must not contain {_BANNED_NAME_WORDS!r}; " + f"got {payload['name']!r}" + ), + error_code=ErrorCode.VALIDATION_ERROR, + ) + # Confirm identity exists; defer login until apply(). + self._resolve_identity(alias) + return PendingCreate(alias=alias, vendor=vendor, payload=payload) + + def prepare_patch( + self, + alias: str, + vendor: str, + app_id: str, + payload: dict[str, Any], + ) -> PendingPatch: + ident = self._resolve_identity(alias) + with self._client_factory(ident) as client: + current = client.get_app(vendor, app_id) + diff = [ + FieldDiff(key=k, current=current.get(k), new=v) + for k, v in payload.items() + if current.get(k) != v + ] + return PendingPatch( + alias=alias, vendor=vendor, app_id=app_id, + payload=payload, current=current, diff=diff, + ) + + def prepare_upload_icon( + self, alias: str, vendor: str, app_id: str, path: str | Path + ) -> PendingIconUpload: + p = Path(path) + if not p.is_file(): + raise KeboolaApiError( + message=f"Icon file not found: {p}", + error_code=ErrorCode.FILE_NOT_FOUND, + ) + data = p.read_bytes() + if not data.startswith(b"\x89PNG\r\n\x1a\n"): + raise KeboolaApiError( + message=f"Icon file is not a PNG: {p}", + error_code=ErrorCode.VALIDATION_ERROR, + ) + # Soft dimension check via PNG IHDR (bytes 16-24 of a valid PNG). + if len(data) >= 24: + import struct + width, height = struct.unpack(">II", data[16:24]) + if (width, height) != (128, 128): + # Soft warning only — apps-api will reject if it's strict. + import logging + logging.getLogger(__name__).warning( + "Icon is %dx%d, not 128x128 — portal may reject it.", + width, height, + ) + self._resolve_identity(alias) + return PendingIconUpload( + alias=alias, vendor=vendor, app_id=app_id, png_path=p, png_bytes=data, + ) + + def prepare_publish( + self, alias: str, vendor: str, app_id: str + ) -> PendingPublish: + ident = self._resolve_identity(alias) + with self._client_factory(ident) as client: + current = client.get_app(vendor, app_id) + missing = [f for f in _REQUIRED_PUBLISH_FIELDS if not current.get(f)] + if missing: + raise KeboolaApiError( + message=( + f"Cannot publish {app_id}: missing required fields " + f"{missing}. Fix them via `kbagent dev-portal patch` first." + ), + error_code=ErrorCode.DP_PUBLISH_REQUIREMENTS_MISSING, + ) + return PendingPublish( + alias=alias, vendor=vendor, app_id=app_id, current=current + ) + + def prepare_deprecate( + self, alias: str, vendor: str, app_id: str + ) -> PendingDeprecate: + self._resolve_identity(alias) + return PendingDeprecate(alias=alias, vendor=vendor, app_id=app_id) + + # ----- Apply (calls the portal write) ----- + + def apply(self, pending: PendingWrite) -> dict[str, Any]: + ident = self._resolve_identity(pending.alias) + with self._client_factory(ident) as client: + if isinstance(pending, PendingCreate): + return client.create_app(pending.vendor, pending.payload) + if isinstance(pending, PendingPatch): + return client.patch_app( + pending.vendor, pending.app_id, pending.payload + ) + if isinstance(pending, PendingIconUpload): + client.upload_icon( + pending.vendor, pending.app_id, pending.png_bytes + ) + return {"status": "uploaded", "app": pending.app_id} + if isinstance(pending, PendingPublish): + return client.publish_app(pending.vendor, pending.app_id) + if isinstance(pending, PendingDeprecate): + return client.deprecate_app(pending.vendor, pending.app_id) + raise KeboolaApiError( + message=f"Unknown pending write type: {type(pending).__name__}", + error_code=ErrorCode.INTERNAL_ERROR, + ) +``` + +- [ ] **Step 4: Run, expect pass** + +Run: `uv run pytest tests/test_dev_portal_service.py -v` +Expected: all tests pass. + +- [ ] **Step 5: Lint + commit** + +```bash +uv run ruff check src/keboola_agent_cli/services/dev_portal_service.py tests/test_dev_portal_service.py +uv run ruff format src/keboola_agent_cli/services/dev_portal_service.py tests/test_dev_portal_service.py +git add src/keboola_agent_cli/services/dev_portal_service.py tests/test_dev_portal_service.py +git commit -m "feat(dev-portal): service reads + prepare/apply + diff + publish pre-flight" +``` + +--- + +## Task 12: Permission registry + helper factories + +**Files:** +- Modify: `src/keboola_agent_cli/permissions.py` +- Modify: `src/keboola_agent_cli/commands/_helpers.py` +- Modify: `tests/test_permissions.py` + +- [ ] **Step 1: Write failing test** + +Append to `tests/test_permissions.py`: + +```python +class TestDevPortalPermissions: + DP_OPS = { + "dev-portal.identity-add": "admin", + "dev-portal.identity-list": "read", + "dev-portal.identity-edit": "admin", + "dev-portal.identity-remove": "admin", + "dev-portal.identity-use": "write", + "dev-portal.identity-verify": "read", + "dev-portal.list": "read", + "dev-portal.get": "read", + "dev-portal.create": "write", + "dev-portal.patch": "write", + "dev-portal.upload-icon": "write", + "dev-portal.publish": "admin", + "dev-portal.deprecate": "destructive", + } + + def test_registry_contains_all_dev_portal_ops(self): + from keboola_agent_cli.permissions import OPERATION_REGISTRY + for op, expected_cat in self.DP_OPS.items(): + assert OPERATION_REGISTRY.get(op) == expected_cat, op +``` + +(Note: that's 13 ops — same count as the spec table.) + +- [ ] **Step 2: Run, expect failure** + +Run: `uv run pytest tests/test_permissions.py::TestDevPortalPermissions -v` +Expected: `AssertionError: dev-portal.identity-add`. + +- [ ] **Step 3: Add entries to `OPERATION_REGISTRY`** + +In `src/keboola_agent_cli/permissions.py`, find a suitable place in `OPERATION_REGISTRY` (alphabetically near `data-app.*` or at the end) and add: + +```python + # Developer Portal (since 0.45.0) + "dev-portal.identity-add": "admin", + "dev-portal.identity-list": "read", + "dev-portal.identity-edit": "admin", + "dev-portal.identity-remove": "admin", + "dev-portal.identity-use": "write", + "dev-portal.identity-verify": "read", + "dev-portal.list": "read", + "dev-portal.get": "read", + "dev-portal.create": "write", + "dev-portal.patch": "write", + "dev-portal.upload-icon": "write", + "dev-portal.publish": "admin", + "dev-portal.deprecate": "destructive", +``` + +- [ ] **Step 4: Add factories to `_helpers.py`** + +Append to `src/keboola_agent_cli/commands/_helpers.py`: + +```python +def resolve_identity_alias(ctx: typer.Context, explicit: str | None) -> str: + """Resolve the dev-portal identity alias for this invocation. + + Order: explicit --identity flag > default from config > error. + """ + if explicit: + return explicit + config_store: ConfigStore = get_service(ctx, "config_store") + default = config_store.load().default_dev_portal_identity + if not default: + raise typer.BadParameter( + "No Developer Portal identity selected. Pass --identity , " + "or set a default via `kbagent dev-portal identity use `." + ) + return default + + +def get_dev_portal_service(ctx: typer.Context): + """Build a DeveloperPortalService bound to the current ConfigStore.""" + from ..dev_portal_client import DeveloperPortalClient + from ..services.dev_portal_service import DeveloperPortalService + + config_store: ConfigStore = get_service(ctx, "config_store") + return DeveloperPortalService( + config_store=config_store, + client_factory=lambda identity: DeveloperPortalClient(identity), + ) +``` + +(`ConfigStore` import is already at the top of `_helpers.py`.) + +- [ ] **Step 5: Run, expect pass** + +Run: `uv run pytest tests/test_permissions.py -v` +Expected: all permission tests pass. + +- [ ] **Step 6: Lint + commit** + +```bash +uv run ruff check src/keboola_agent_cli/permissions.py src/keboola_agent_cli/commands/_helpers.py tests/test_permissions.py +uv run ruff format src/keboola_agent_cli/permissions.py src/keboola_agent_cli/commands/_helpers.py tests/test_permissions.py +git add src/keboola_agent_cli/permissions.py src/keboola_agent_cli/commands/_helpers.py tests/test_permissions.py +git commit -m "feat(dev-portal): permission registry entries + identity resolver" +``` + +--- + +## Task 13: Command layer — identity subcommands + reads + +**Files:** +- Create: `src/keboola_agent_cli/commands/dev_portal.py` +- Modify: `src/keboola_agent_cli/cli.py` +- Create: `tests/test_dev_portal_cli.py` + +- [ ] **Step 1: Write failing test** + +Create `tests/test_dev_portal_cli.py`: + +```python +"""Tests for `kbagent dev-portal` command layer via CliRunner.""" + +from __future__ import annotations + +import json +from unittest.mock import patch + +import pytest +from typer.testing import CliRunner + +from keboola_agent_cli.cli import app + + +runner = CliRunner() + + +class TestIdentityCommands: + def test_identity_add_and_list_json(self, tmp_config_dir): + with patch( + "keboola_agent_cli.services.dev_portal_service.DeveloperPortalService.add_identity" + ) as add_: + r = runner.invoke( + app, + [ + "--config-dir", str(tmp_config_dir), + "--json", "dev-portal", "identity", "add", + "--alias", "alpha", + "--username", "service.keboola.x", + "--password", "p", + ], + ) + assert r.exit_code == 0, r.output + add_.assert_called_once() + + def test_identity_use_sets_default(self, tmp_config_dir, config_store): + from keboola_agent_cli.models import DeveloperPortalIdentity + config_store.add_dev_portal_identity( + "alpha", DeveloperPortalIdentity(username="u", password="p") + ) + config_store.add_dev_portal_identity( + "beta", DeveloperPortalIdentity(username="u", password="p") + ) + r = runner.invoke( + app, + [ + "--config-dir", str(tmp_config_dir), + "dev-portal", "identity", "use", "beta", + ], + ) + assert r.exit_code == 0, r.output + assert config_store.load().default_dev_portal_identity == "beta" + + +class TestReadCommands: + def test_list_apps_json(self, tmp_config_dir, config_store): + from keboola_agent_cli.models import DeveloperPortalIdentity + config_store.add_dev_portal_identity( + "alpha", DeveloperPortalIdentity(username="u", password="p", vendor="keboola") + ) + with patch( + "keboola_agent_cli.services.dev_portal_service.DeveloperPortalService.list_apps", + return_value=[{"id": "keboola.ex-a"}], + ): + r = runner.invoke( + app, + [ + "--config-dir", str(tmp_config_dir), + "--json", "dev-portal", "list", "--vendor", "keboola", + ], + ) + assert r.exit_code == 0, r.output + data = json.loads(r.stdout) + assert data == [{"id": "keboola.ex-a"}] +``` + +(`tmp_config_dir` and `config_store` fixtures live in `tests/conftest.py`.) + +- [ ] **Step 2: Run, expect failure** + +Run: `uv run pytest tests/test_dev_portal_cli.py::TestIdentityCommands -v` +Expected: command does not exist (`No such command 'dev-portal'`). + +- [ ] **Step 3: Create the command module (identity + reads only — writes in Task 14)** + +Create `src/keboola_agent_cli/commands/dev_portal.py`: + +```python +"""`kbagent dev-portal` — Developer Portal command surface. + +Identity management mirrors `kbagent project`; portal writes are gated by +`require_random_code_confirmation()` from _helpers — there is no `--yes` +bypass and no env-var override. +""" + +from __future__ import annotations + +import typer + +from ..errors import ConfigError, ErrorCode, KeboolaApiError +from ..models import DeveloperPortalIdentity +from ._helpers import ( + get_dev_portal_service, + get_formatter, + map_error_to_exit_code, + resolve_identity_alias, +) + +dev_portal_app = typer.Typer( + help="Keboola Developer Portal — multi-identity, production-safe writes.", + no_args_is_help=True, +) + +identity_app = typer.Typer(help="Manage Developer Portal identities (login credentials).") +dev_portal_app.add_typer(identity_app, name="identity") + + +def _split_app(app: str) -> tuple[str, str]: + """Split `VENDOR.APP_ID` into (vendor, app_id).""" + if "." not in app: + raise typer.BadParameter( + f"--app must be in VENDOR.APP_ID form (e.g. keboola.ex-foo), got: {app!r}" + ) + vendor, _ = app.split(".", 1) + return vendor, app + + +# ----- Identity subcommands ----- + +@identity_app.command("add") +def identity_add( + ctx: typer.Context, + alias: str = typer.Option(..., "--alias"), + username: str = typer.Option(..., "--username"), + password: str | None = typer.Option(None, "--password"), + password_stdin: bool = typer.Option( + False, "--password-stdin", + help="Read password from stdin (paste from a secrets manager).", + ), + role_hint: str = typer.Option("vendor", "--role-hint"), + vendor: str | None = typer.Option(None, "--vendor"), + portal_url: str = typer.Option( + "https://apps-api.keboola.com", "--portal-url", + ), +) -> None: + formatter = get_formatter(ctx) + if password_stdin: + import sys as _sys + password = _sys.stdin.read().strip() + if not password: + raise typer.BadParameter("Pass --password or --password-stdin.") + identity = DeveloperPortalIdentity( + username=username, + password=password, + role_hint=role_hint, + vendor=vendor, + portal_url=portal_url, + ) + svc = get_dev_portal_service(ctx) + try: + svc.add_identity(alias, identity) + except (ConfigError, KeboolaApiError) as exc: + formatter.error(message=str(exc), error_code=getattr(exc, "error_code", ErrorCode.CONFIG_ERROR)) + raise typer.Exit(code=map_error_to_exit_code(exc) if isinstance(exc, KeboolaApiError) else 5) from None + formatter.output({"status": "ok", "alias": alias, "username": username}) + + +@identity_app.command("list") +def identity_list(ctx: typer.Context) -> None: + formatter = get_formatter(ctx) + svc = get_dev_portal_service(ctx) + identities = svc.list_identities() + default = svc.current_identity() + rows = [ + { + "alias": alias, + "username": ident.username, + "vendor": ident.vendor or "", + "role_hint": ident.role_hint, + "portal_url": ident.portal_url, + "default": alias == default, + } + for alias, ident in identities.items() + ] + formatter.output(rows) + + +@identity_app.command("remove") +def identity_remove( + ctx: typer.Context, + alias: str = typer.Option(..., "--alias"), +) -> None: + formatter = get_formatter(ctx) + svc = get_dev_portal_service(ctx) + try: + svc.remove_identity(alias) + except ConfigError as exc: + formatter.error(message=str(exc), error_code=ErrorCode.CONFIG_ERROR) + raise typer.Exit(code=5) from None + formatter.output({"status": "ok", "removed": alias}) + + +@identity_app.command("edit") +def identity_edit( + ctx: typer.Context, + alias: str = typer.Option(..., "--alias"), + username: str | None = typer.Option(None, "--username"), + password: str | None = typer.Option(None, "--password"), + password_stdin: bool = typer.Option(False, "--password-stdin"), + role_hint: str | None = typer.Option(None, "--role-hint"), + vendor: str | None = typer.Option(None, "--vendor"), + new_alias: str | None = typer.Option(None, "--new-alias"), +) -> None: + formatter = get_formatter(ctx) + svc = get_dev_portal_service(ctx) + if password_stdin: + import sys as _sys + password = _sys.stdin.read().strip() + try: + if new_alias: + svc.rename_identity(alias, new_alias) + alias = new_alias + svc.edit_identity( + alias, + username=username, + password=password, + role_hint=role_hint, + vendor=vendor, + ) + except ConfigError as exc: + formatter.error(message=str(exc), error_code=ErrorCode.CONFIG_ERROR) + raise typer.Exit(code=5) from None + formatter.output({"status": "ok", "alias": alias}) + + +@identity_app.command("use") +def identity_use( + ctx: typer.Context, + alias: str = typer.Argument(..., help="Identity alias to set as default"), +) -> None: + formatter = get_formatter(ctx) + svc = get_dev_portal_service(ctx) + try: + svc.use_identity(alias) + except ConfigError as exc: + formatter.error(message=str(exc), error_code=ErrorCode.CONFIG_ERROR) + raise typer.Exit(code=5) from None + formatter.output({"status": "ok", "default": alias}) + + +@identity_app.command("current") +def identity_current(ctx: typer.Context) -> None: + formatter = get_formatter(ctx) + svc = get_dev_portal_service(ctx) + formatter.output({"default": svc.current_identity()}) + + +@identity_app.command("verify") +def identity_verify( + ctx: typer.Context, + identity: str | None = typer.Option(None, "--identity"), +) -> None: + formatter = get_formatter(ctx) + svc = get_dev_portal_service(ctx) + alias = resolve_identity_alias(ctx, identity) + try: + info = svc.verify_identity(alias) + except KeboolaApiError as exc: + formatter.error(message=str(exc), error_code=exc.error_code) + raise typer.Exit(code=map_error_to_exit_code(exc)) from None + formatter.output({"status": "ok", **info}) + + +# ----- Read commands ----- + +@dev_portal_app.command("list") +def list_apps( + ctx: typer.Context, + vendor: str = typer.Option(..., "--vendor"), + identity: str | None = typer.Option(None, "--identity"), +) -> None: + formatter = get_formatter(ctx) + svc = get_dev_portal_service(ctx) + alias = resolve_identity_alias(ctx, identity) + try: + apps = svc.list_apps(alias, vendor) + except KeboolaApiError as exc: + formatter.error(message=str(exc), error_code=exc.error_code) + raise typer.Exit(code=map_error_to_exit_code(exc)) from None + formatter.output(apps) + + +@dev_portal_app.command("get") +def get_app_cmd( + ctx: typer.Context, + app: str = typer.Option(..., "--app", help="VENDOR.APP_ID, e.g. keboola.ex-foo"), + identity: str | None = typer.Option(None, "--identity"), +) -> None: + formatter = get_formatter(ctx) + svc = get_dev_portal_service(ctx) + alias = resolve_identity_alias(ctx, identity) + vendor, app_id = _split_app(app) + try: + result = svc.get_app(alias, vendor, app_id) + except KeboolaApiError as exc: + formatter.error(message=str(exc), error_code=exc.error_code) + raise typer.Exit(code=map_error_to_exit_code(exc)) from None + formatter.output(result) +``` + +- [ ] **Step 4: Register the sub-app in `cli.py`** + +In `src/keboola_agent_cli/cli.py`, find the `_DEV = "Development"` block and add: + +```python +from .commands.dev_portal import dev_portal_app +``` + +(Add the import near the other `from .commands.X import X_app` lines.) + +In the `_DEV` registration block, add (e.g. right after `app.add_typer(workspace_app, name="workspace", rich_help_panel=_DEV)`): + +```python +app.add_typer(dev_portal_app, name="dev-portal", rich_help_panel=_DEV) +``` + +- [ ] **Step 5: Run, expect pass** + +Run: `uv run pytest tests/test_dev_portal_cli.py -v` +Expected: all tests in `TestIdentityCommands` and `TestReadCommands` pass. + +- [ ] **Step 6: Lint + commit** + +```bash +uv run ruff check src/keboola_agent_cli/commands/dev_portal.py src/keboola_agent_cli/cli.py tests/test_dev_portal_cli.py +uv run ruff format src/keboola_agent_cli/commands/dev_portal.py src/keboola_agent_cli/cli.py tests/test_dev_portal_cli.py +git add src/keboola_agent_cli/commands/dev_portal.py src/keboola_agent_cli/cli.py tests/test_dev_portal_cli.py +git commit -m "feat(dev-portal): identity subcommands + list/get" +``` + +--- + +## Task 14: Command layer — write commands (gated by random-code confirm) + +**Files:** +- Modify: `src/keboola_agent_cli/commands/dev_portal.py` +- Modify: `tests/test_dev_portal_cli.py` + +- [ ] **Step 1: Write failing tests** + +Append to `tests/test_dev_portal_cli.py`: + +```python +class TestWriteCommands: + """Every write must require the random-code confirm. No --yes.""" + + def _seed_identity(self, config_store): + from keboola_agent_cli.models import DeveloperPortalIdentity + config_store.add_dev_portal_identity( + "alpha", DeveloperPortalIdentity(username="u", password="p", vendor="keboola"), + ) + + def test_patch_non_tty_exits_6(self, tmp_config_dir, config_store): + """Without a TTY there is NO bypass — exit 6, no portal call.""" + self._seed_identity(config_store) + with patch( + "keboola_agent_cli.services.dev_portal_service.DeveloperPortalService.prepare_patch" + ) as prep: + from keboola_agent_cli.services.dev_portal_service import PendingPatch, FieldDiff + prep.return_value = PendingPatch( + alias="alpha", vendor="keboola", app_id="keboola.ex-a", + payload={"name": "New"}, current={"name": "Old"}, + diff=[FieldDiff(key="name", current="Old", new="New")], + ) + with patch( + "keboola_agent_cli.services.dev_portal_service.DeveloperPortalService.apply" + ) as apply_: + # CliRunner provides a non-TTY stdin. + r = runner.invoke( + app, + [ + "--config-dir", str(tmp_config_dir), + "dev-portal", "patch", + "--app", "keboola.ex-a", + "--data", "/tmp/does-not-matter.json", + ], + input="", + ) + assert r.exit_code == 6, r.output + apply_.assert_not_called() + + def test_patch_dry_run_no_confirm(self, tmp_config_dir, config_store, tmp_path): + """--dry-run prints diff and exits 0 without any confirm prompt.""" + self._seed_identity(config_store) + data_file = tmp_path / "patch.json" + data_file.write_text(json.dumps({"name": "New"})) + with patch( + "keboola_agent_cli.services.dev_portal_service.DeveloperPortalService.prepare_patch" + ) as prep: + from keboola_agent_cli.services.dev_portal_service import PendingPatch, FieldDiff + prep.return_value = PendingPatch( + alias="alpha", vendor="keboola", app_id="keboola.ex-a", + payload={"name": "New"}, current={"name": "Old"}, + diff=[FieldDiff(key="name", current="Old", new="New")], + ) + with patch( + "keboola_agent_cli.services.dev_portal_service.DeveloperPortalService.apply" + ) as apply_: + r = runner.invoke( + app, + [ + "--config-dir", str(tmp_config_dir), + "--json", "dev-portal", "patch", + "--app", "keboola.ex-a", + "--data", str(data_file), + "--dry-run", + ], + ) + assert r.exit_code == 0, r.output + apply_.assert_not_called() + # JSON output should advertise the dry-run status + assert "dry-run" in r.stdout +``` + +- [ ] **Step 2: Run, expect failure** + +Run: `uv run pytest tests/test_dev_portal_cli.py::TestWriteCommands -v` +Expected: command does not exist yet. + +- [ ] **Step 3: Add write commands** + +Append to `src/keboola_agent_cli/commands/dev_portal.py`: + +```python +import json +from pathlib import Path + +from ._helpers import require_random_code_confirmation + + +def _load_payload(data: str | None) -> dict: + if data is None: + raise typer.BadParameter("--data is required") + if data == "-": + import sys as _sys + return json.loads(_sys.stdin.read()) + return json.loads(Path(data).read_text()) + + +def _render_pending(formatter, pending) -> None: + """Write a stderr-only preview of the pending write.""" + from ..services.dev_portal_service import ( + PendingCreate, PendingPatch, PendingIconUpload, + PendingPublish, PendingDeprecate, + ) + err = formatter.err_console + if isinstance(pending, PendingPatch): + err.print(f"[bold]PATCH[/bold] /vendors/{pending.vendor}/apps/{pending.app_id}") + for d in pending.diff: + err.print(f" [yellow]{d.key}[/yellow]: {d.current!r} -> {d.new!r}") + if not pending.diff: + err.print(" [dim]no field-level changes (payload matches current state)[/dim]") + elif isinstance(pending, PendingCreate): + err.print(f"[bold]POST[/bold] /vendors/{pending.vendor}/apps") + err.print_json(json.dumps(pending.payload)) + elif isinstance(pending, PendingIconUpload): + err.print( + f"[bold]UPLOAD ICON[/bold] {pending.png_path} -> " + f"{pending.vendor}/{pending.app_id} ({len(pending.png_bytes)} bytes)" + ) + elif isinstance(pending, PendingPublish): + err.print( + f"[bold red]PUBLISH[/bold red] /vendors/{pending.vendor}/apps/" + f"{pending.app_id}/publish (requests Keboola review)" + ) + elif isinstance(pending, PendingDeprecate): + err.print( + f"[bold red]DEPRECATE[/bold red] /vendors/{pending.vendor}/apps/" + f"{pending.app_id}/deprecate (hides app, blocks new configs)" + ) + + +def _pending_as_json(pending) -> dict: + """Serialise a pending write for --json --dry-run output.""" + from dataclasses import asdict + raw = asdict(pending) + if "png_bytes" in raw: + raw["png_bytes"] = f"<{len(raw['png_bytes'])} bytes>" + if "png_path" in raw: + raw["png_path"] = str(raw["png_path"]) + return {"status": "dry-run", "pending": raw} + + +@dev_portal_app.command("create") +def create_cmd( + ctx: typer.Context, + vendor: str = typer.Option(..., "--vendor"), + data: str = typer.Option(..., "--data", help="Path to JSON payload, or '-' for stdin"), + identity: str | None = typer.Option(None, "--identity"), + dry_run: bool = typer.Option(False, "--dry-run"), +) -> None: + formatter = get_formatter(ctx) + svc = get_dev_portal_service(ctx) + alias = resolve_identity_alias(ctx, identity) + try: + pending = svc.prepare_create(alias, vendor, _load_payload(data)) + except KeboolaApiError as exc: + formatter.error(message=str(exc), error_code=exc.error_code) + raise typer.Exit(code=map_error_to_exit_code(exc)) from None + _render_pending(formatter, pending) + if dry_run: + formatter.output(_pending_as_json(pending)) + return + require_random_code_confirmation(f"create app in vendor '{vendor}'") + try: + result = svc.apply(pending) + except KeboolaApiError as exc: + formatter.error(message=str(exc), error_code=exc.error_code) + raise typer.Exit(code=map_error_to_exit_code(exc)) from None + formatter.output({"status": "ok", "created": result}) + + +@dev_portal_app.command("patch") +def patch_cmd( + ctx: typer.Context, + app: str = typer.Option(..., "--app"), + data: str | None = typer.Option(None, "--data"), + property_: str | None = typer.Option(None, "--property"), + value: str | None = typer.Option(None, "--value"), + value_file: str | None = typer.Option(None, "--value-file"), + identity: str | None = typer.Option(None, "--identity"), + dry_run: bool = typer.Option(False, "--dry-run"), +) -> None: + formatter = get_formatter(ctx) + svc = get_dev_portal_service(ctx) + alias = resolve_identity_alias(ctx, identity) + vendor, app_id = _split_app(app) + + if data: + payload = _load_payload(data) + elif property_: + if value_file: + raw = Path(value_file).read_text() + elif value is not None: + raw = value + else: + raise typer.BadParameter("--property requires --value or --value-file") + try: + parsed = json.loads(raw) if raw.strip()[:1] in "[{" else raw + except json.JSONDecodeError: + parsed = raw + payload = {property_: parsed} + else: + raise typer.BadParameter("Provide --data, or --property with --value/--value-file") + + try: + pending = svc.prepare_patch(alias, vendor, app_id, payload) + except KeboolaApiError as exc: + formatter.error(message=str(exc), error_code=exc.error_code) + raise typer.Exit(code=map_error_to_exit_code(exc)) from None + _render_pending(formatter, pending) + if dry_run: + formatter.output(_pending_as_json(pending)) + return + require_random_code_confirmation(f"patch {app}") + try: + svc.apply(pending) + except KeboolaApiError as exc: + formatter.error(message=str(exc), error_code=exc.error_code) + raise typer.Exit(code=map_error_to_exit_code(exc)) from None + formatter.output({ + "status": "ok", + "app": app, + "patched_keys": [d.key for d in pending.diff], + }) + + +@dev_portal_app.command("upload-icon") +def upload_icon_cmd( + ctx: typer.Context, + app: str = typer.Option(..., "--app"), + file: str = typer.Option(..., "--file"), + identity: str | None = typer.Option(None, "--identity"), + dry_run: bool = typer.Option(False, "--dry-run"), +) -> None: + formatter = get_formatter(ctx) + svc = get_dev_portal_service(ctx) + alias = resolve_identity_alias(ctx, identity) + vendor, app_id = _split_app(app) + try: + pending = svc.prepare_upload_icon(alias, vendor, app_id, file) + except KeboolaApiError as exc: + formatter.error(message=str(exc), error_code=exc.error_code) + raise typer.Exit(code=map_error_to_exit_code(exc)) from None + _render_pending(formatter, pending) + if dry_run: + formatter.output(_pending_as_json(pending)) + return + require_random_code_confirmation(f"upload icon for {app}") + try: + result = svc.apply(pending) + except KeboolaApiError as exc: + formatter.error(message=str(exc), error_code=exc.error_code) + raise typer.Exit(code=map_error_to_exit_code(exc)) from None + formatter.output(result) + + +@dev_portal_app.command("publish") +def publish_cmd( + ctx: typer.Context, + app: str = typer.Option(..., "--app"), + identity: str | None = typer.Option(None, "--identity"), + dry_run: bool = typer.Option(False, "--dry-run"), +) -> None: + formatter = get_formatter(ctx) + svc = get_dev_portal_service(ctx) + alias = resolve_identity_alias(ctx, identity) + vendor, app_id = _split_app(app) + try: + pending = svc.prepare_publish(alias, vendor, app_id) + except KeboolaApiError as exc: + formatter.error(message=str(exc), error_code=exc.error_code) + raise typer.Exit(code=map_error_to_exit_code(exc)) from None + _render_pending(formatter, pending) + if dry_run: + formatter.output(_pending_as_json(pending)) + return + require_random_code_confirmation(f"publish {app}") + try: + result = svc.apply(pending) + except KeboolaApiError as exc: + formatter.error(message=str(exc), error_code=exc.error_code) + raise typer.Exit(code=map_error_to_exit_code(exc)) from None + formatter.output({"status": "ok", "published": result}) + + +@dev_portal_app.command("deprecate") +def deprecate_cmd( + ctx: typer.Context, + app: str = typer.Option(..., "--app"), + identity: str | None = typer.Option(None, "--identity"), + dry_run: bool = typer.Option(False, "--dry-run"), +) -> None: + formatter = get_formatter(ctx) + svc = get_dev_portal_service(ctx) + alias = resolve_identity_alias(ctx, identity) + vendor, app_id = _split_app(app) + try: + pending = svc.prepare_deprecate(alias, vendor, app_id) + except KeboolaApiError as exc: + formatter.error(message=str(exc), error_code=exc.error_code) + raise typer.Exit(code=map_error_to_exit_code(exc)) from None + _render_pending(formatter, pending) + if dry_run: + formatter.output(_pending_as_json(pending)) + return + require_random_code_confirmation(f"deprecate {app}") + try: + result = svc.apply(pending) + except KeboolaApiError as exc: + formatter.error(message=str(exc), error_code=exc.error_code) + raise typer.Exit(code=map_error_to_exit_code(exc)) from None + formatter.output({"status": "ok", "deprecated": result}) +``` + +- [ ] **Step 4: Run, expect pass** + +Run: `uv run pytest tests/test_dev_portal_cli.py -v` +Expected: all tests pass (including write-command non-TTY exit-6 and --dry-run). + +- [ ] **Step 5: Lint + commit** + +```bash +uv run ruff check src/keboola_agent_cli/commands/dev_portal.py tests/test_dev_portal_cli.py +uv run ruff format src/keboola_agent_cli/commands/dev_portal.py tests/test_dev_portal_cli.py +git add src/keboola_agent_cli/commands/dev_portal.py tests/test_dev_portal_cli.py +git commit -m "feat(dev-portal): write commands gated by random-code confirm" +``` + +--- + +## Task 15: E2E test + version bump + changelog + all rule #17 doc surfaces + +**Files:** +- Modify: `tests/test_e2e.py` +- Modify: `pyproject.toml` +- Modify: `src/keboola_agent_cli/changelog.py` +- Modify: `src/keboola_agent_cli/commands/context.py` +- Modify: `CLAUDE.md` +- Modify: `plugins/kbagent/agents/keboola-expert.md` +- Modify: `plugins/kbagent/skills/kbagent/SKILL.md` +- Modify: `plugins/kbagent/skills/kbagent/references/commands-reference.md` +- Modify: `plugins/kbagent/skills/kbagent/references/gotchas.md` +- Create: `plugins/kbagent/skills/kbagent/references/dev-portal-workflow.md` + +- [ ] **Step 1: Add E2E test** + +Append to `tests/test_e2e.py`: + +```python +class TestDevPortalE2E: + def test_identity_list_smoke(self, e2e_runner): + """Unconditional smoke: dev-portal identity list must not crash.""" + result = e2e_runner.invoke(["--json", "dev-portal", "identity", "list"]) + assert result.exit_code == 0 + + @pytest.mark.skipif( + not (os.environ.get("E2E_DP_USERNAME") and os.environ.get("E2E_DP_PASSWORD")), + reason="Set E2E_DP_USERNAME and E2E_DP_PASSWORD to run real-portal test", + ) + def test_list_apps_against_real_portal(self, e2e_runner, tmp_path): + """Optional: list apps for vendor 'keboola' if creds supplied.""" + result = e2e_runner.invoke([ + "dev-portal", "identity", "add", + "--alias", "e2e", + "--username", os.environ["E2E_DP_USERNAME"], + "--password", os.environ["E2E_DP_PASSWORD"], + "--vendor", "keboola", + ]) + assert result.exit_code == 0, result.output + result = e2e_runner.invoke([ + "--json", "dev-portal", "list", + "--vendor", "keboola", "--identity", "e2e", + ]) + assert result.exit_code == 0, result.output +``` + +(Reuse whatever `e2e_runner` fixture pattern the existing E2E tests use; adapt names to match `tests/test_e2e.py` conventions if different.) + +- [ ] **Step 2: Bump version + sync** + +Edit `pyproject.toml`: + +```toml +version = "0.45.0" +``` + +Run: + +```bash +make version-sync +``` + +Expected: `plugins/kbagent/.claude-plugin/plugin.json` and `.claude-plugin/marketplace.json` (if it tracks the version) updated. + +- [ ] **Step 3: Add changelog entry** + +In `src/keboola_agent_cli/changelog.py`, add a `"0.45.0"` block following the existing entry format. Example structure (look at the existing top entry for the exact dict shape): + +```python + "0.45.0": [ + "New: `kbagent dev-portal` command group wraps the Keboola Developer Portal " + "(apps-api.keboola.com) with multi-identity credential storage (same 0600 " + "config.json as KB project tokens) and a random-code TTY confirm safety bar " + "on every write. There is intentionally no `--yes` flag and no env-var " + "bypass: writes (`create`, `patch`, `upload-icon`, `publish`, `deprecate`) " + "always require a human at a real terminal. Reads (`identity list`, `list " + "--vendor`, `get --app`) run freely so agents can research peer components " + "by composing `list` + `get`. Identities support service accounts (no MFA) " + "and personal accounts (MFA prompt via /dev/tty).", + "Refactor: the random-code interactive-confirmation primitive previously " + "embedded in `commands/permissions.py` is now in `commands/_helpers.py` as " + "`require_random_code_confirmation()`. `permissions set/reset` and every " + "dev-portal write share the same implementation. The function now raises " + "`typer.Exit(EXIT_PERMISSION_DENIED)` on failure instead of returning a " + "bool, so call sites are one line shorter.", + "Permission registry: 13 new ops under `dev-portal.*` (`identity-add`/`-edit`/" + "`-remove` = admin; `identity-list`/`-verify`/`list`/`get` = read; " + "`identity-use`/`create`/`patch`/`upload-icon` = write; `publish` = admin; " + "`deprecate` = destructive). `--deny-writes` blocks every dev-portal write " + "automatically.", + ], +``` + +- [ ] **Step 4: Update `AGENT_CONTEXT`** + +In `src/keboola_agent_cli/commands/context.py`, find the `AGENT_CONTEXT` string and append a `dev-portal` section that describes: + +- The safety contract: agents can call read commands and any `--dry-run` directly; writes (`create`, `patch`, `upload-icon`, `publish`, `deprecate`) always require a human to type a random code into a TTY. +- The peer-research pattern: use `list --vendor` then `get --app VENDOR.APP_ID` to compare configurations from existing components. +- Identity selection: `--identity `, or default via `dev-portal identity use`. + +Keep the tone consistent with the existing sections; ~10 lines. + +- [ ] **Step 5: Update `CLAUDE.md` `## All CLI Commands`** + +In `CLAUDE.md`, find the `## All CLI Commands` section and add (alphabetically near `data-app` or in a logical position): + +``` +kbagent dev-portal identity add --alias A --username U [--password P | --password-stdin] + [--role-hint vendor|admin] [--vendor V] [--portal-url URL] +kbagent dev-portal identity list +kbagent dev-portal identity remove --alias A +kbagent dev-portal identity edit --alias A [--username U] [--password P|--password-stdin] + [--role-hint H] [--vendor V] [--new-alias N] +kbagent dev-portal identity use ALIAS +kbagent dev-portal identity current +kbagent dev-portal identity verify [--identity A] + +kbagent dev-portal list --vendor V [--identity A] +kbagent dev-portal get --app VENDOR.APP_ID [--identity A] + +kbagent dev-portal create --vendor V --data FILE [--identity A] [--dry-run] +kbagent dev-portal patch --app VENDOR.APP_ID (--data FILE | --property KEY (--value V | --value-file F)) + [--identity A] [--dry-run] +kbagent dev-portal upload-icon --app VENDOR.APP_ID --file PATH [--identity A] [--dry-run] +kbagent dev-portal publish --app VENDOR.APP_ID [--identity A] [--dry-run] +kbagent dev-portal deprecate --app VENDOR.APP_ID [--identity A] [--dry-run] +# All writes require an interactive random-code TTY confirm; no --yes / no env bypass. +``` + +- [ ] **Step 6: Update `plugins/kbagent/agents/keboola-expert.md`** + +(a) Rule 6 VERSION GATE — bump the minimum version referenced in the rule to `0.45.0` so the agent knows `dev-portal` is available. + +(b) Tool-selection matrix — add a row: + +``` +| User mentions Developer Portal, apps-api, register app, vendor app, ui-options, encryption, defaultBucket, app icon, configurationSchema in portal, publish/deprecate component | `kbagent dev-portal …` | Always prepare writes with `--dry-run` first; never attempt to `--yes` a write — there is no such flag, and writes refuse on non-TTY (exit 6). | +``` + +(c) Inline gotcha — add: "Developer Portal writes are direct production. The agent's job ends at `--dry-run` + showing the preview; the human types the confirm code." + +- [ ] **Step 7: Update `plugins/kbagent/skills/kbagent/SKILL.md` decision-table** + +Add a row: + +``` +| manage portal property / register new component in portal | `kbagent dev-portal …` | see `references/dev-portal-workflow.md` | +``` + +- [ ] **Step 8: Update `plugins/kbagent/skills/kbagent/references/commands-reference.md`** + +Add a new section "### dev-portal" listing all commands with one-line descriptions. Mirror the formatting of the existing sections (e.g. "### data-app"). + +- [ ] **Step 9: Update `plugins/kbagent/skills/kbagent/references/gotchas.md`** + +Add a new entry: + +```markdown +### Developer Portal: writes require a human, no exceptions (since v0.45.0) + +`kbagent dev-portal {create,patch,upload-icon,publish,deprecate}` always print +the request preview and then require the user to type a random hex code on a +real terminal. There is no `--yes` flag. There is no env-var override. The +command exits 6 (`EXIT_PERMISSION_DENIED`) on a non-TTY shell. + +For agentic use: stop at the preview. Use `--dry-run` to get a clean +exit-0 preview you can show the user. Then ask the user to run the same +command without `--dry-run` themselves. + +Reads (`dev-portal list`, `dev-portal get`) are unrestricted — peer-research +patterns ("show me how MySQL and Postgres extractors configure themselves") +are agent-friendly via `list --vendor` + `get --app`. +``` + +- [ ] **Step 10: Create `plugins/kbagent/skills/kbagent/references/dev-portal-workflow.md`** + +Create the file with the workflow doc: + +```markdown +# Developer Portal workflow + +> Audience: a Keboola component developer or a kbagent agent acting on their +> behalf. Goal: safely register, inspect, and update components in the +> Keboola Developer Portal (`apps-api.keboola.com`). + +## Identity model + +Developer Portal logins are email + password (with MFA on personal +accounts). kbagent stores identities per-alias in the same `config.json` +as KB project tokens, under 0600 protection: + +``` +kbagent dev-portal identity add --alias vendor-keboola --username service.keboola.xxxxx --password ... --vendor keboola +kbagent dev-portal identity add --alias vendor-kds --username service.kds-team.xxxxx --password ... --vendor kds-team +kbagent dev-portal identity add --alias admin-foo --username admin@keboola.com --password-stdin +kbagent dev-portal identity use vendor-keboola # default for subsequent commands +``` + +Service accounts (`service.{vendor}.{id}`) skip MFA. Personal admin +accounts prompt for the MFA code on /dev/tty at login time. + +## Safety contract (read this before issuing any write) + +- Reads are free: `dev-portal list`, `dev-portal get`. +- Writes (`create`, `patch`, `upload-icon`, `publish`, `deprecate`) always: + 1. Print the exact pending request to stderr (full diff for `patch`). + 2. Require the user to type a random hex code into the TTY. + 3. Exit 6 on a non-TTY shell. +- There is no `--yes`. There is no env-var bypass. By design. +- `--dry-run` prints the same preview and exits 0 without prompting. This + is the agent-safe path. + +## The loop + +1. Identify the component (vendor + app id). For an existing repo, check + `.github/workflows/*.yml` for `KBC_DEVELOPERPORTAL_VENDOR` and `KBC_DEVELOPERPORTAL_APP`. +2. `kbagent --json dev-portal list --vendor ` and/or + `kbagent --json dev-portal get --app VENDOR.APP_ID` to inspect. +3. Build a payload file (a JSON file — never inline JSON, shell quoting + is unsafe with portal property names that contain spaces). +4. `kbagent dev-portal patch --app VENDOR.APP_ID --data /tmp/p.json --dry-run` + — print the diff, show it to the user. +5. The user runs the same command without `--dry-run` and types the code. + +## Peer-config research + +Designing a new component? Pull reference configurations from existing +peers: + +``` +# List candidates +kbagent --json dev-portal list --vendor keboola | jq '.[] | select(.type=="extractor") | .id' + +# Pull two peers in full +kbagent --json dev-portal get --app keboola.ex-db-mysql > /tmp/peer-mysql.json +kbagent --json dev-portal get --app keboola.ex-db-pgsql > /tmp/peer-postgres.json +``` + +Compare them yourself — the agent has the reasoning ability to spot +patterns. No dedicated `peers` command needed. + +## Boundaries (what this surface does NOT own) + +- Image push to ECR — stays in component GitHub Actions. +- Bulk repo-file -> property sync on deploy — stays in + `scripts/developer_portal/update_properties.sh` (Cookiecutter-backed files). +- Writes to `component_config/` — never. That directory is governed by the + Cookiecutter template; portal-direct properties (`uiOptions`, + `encryption`, `defaultBucket`, …) live only in the portal. +``` + +- [ ] **Step 11: Run full test suite + CI checks** + +```bash +uv run pytest tests/ -q +uv run ruff check src/ tests/ +uv run ruff format --check src/ tests/ +make changelog-check +``` + +Expected: all pass. + +- [ ] **Step 12: Commit doc-sync as one atomic commit** + +```bash +git add CLAUDE.md tests/test_e2e.py pyproject.toml plugins/ src/keboola_agent_cli/changelog.py src/keboola_agent_cli/commands/context.py plugins/kbagent/.claude-plugin/plugin.json .claude-plugin/marketplace.json 2>/dev/null || true +git status +# Verify only the expected files are staged; nothing else. +git commit -m "feat(dev-portal): version 0.45.0, E2E test, AGENT_CONTEXT, plugin docs" +``` + +--- + +## Task 16: Final cross-check + push + PR + +**Files:** none (workflow only) + +- [ ] **Step 1: Re-run the full check pipeline** + +```bash +make check +``` + +Expected: passes (lint + format + changelog + test). + +- [ ] **Step 2: Smoke the CLI manually** + +```bash +uv run kbagent dev-portal --help +uv run kbagent dev-portal identity --help +uv run kbagent dev-portal patch --help +``` + +Expected: help text shows all subcommands and flags as designed. + +- [ ] **Step 3: Push branch + open PR** + +```bash +git push -u origin feat/dev-portal +gh pr create --title "feat(dev-portal): kbagent Developer Portal support with no-bypass write safety" \ + --body "$(cat <<'EOF' +## Summary +- Adds `kbagent dev-portal` command group wrapping apps-api.keboola.com. +- Multi-identity credential storage (mirrors KB project tokens). +- Writes always require a random-code TTY confirm — no `--yes`, no env bypass. +- v1 ops: identity CRUD, list, get, create, patch, upload-icon, publish, deprecate. + +## Test plan +- [ ] `make check` passes +- [ ] `uv run pytest tests/test_dev_portal_client.py tests/test_dev_portal_service.py tests/test_dev_portal_cli.py -v` passes +- [ ] `uv run pytest tests/test_helpers.py::TestRequireRandomCodeConfirmation -v` passes +- [ ] Manual: `kbagent dev-portal patch --app foo.bar --data /tmp/p.json` on a non-TTY exits 6 +- [ ] Manual: `kbagent dev-portal patch --app foo.bar --data /tmp/p.json --dry-run` exits 0 + +🤖 Generated with [Claude Code](https://claude.com/claude-code) +EOF +)" +``` + +--- + +## Self-Review + +### 1. Spec coverage + +Every spec section maps to a task: + +- **Data model & storage** (spec §Architecture > Data model) → Tasks 4, 5. +- **`DeveloperPortalClient`** (spec §Architecture > Client layer) → Tasks 6, 7, 8, 9. +- **`DeveloperPortalService`** (spec §Architecture > Service layer) → Tasks 10, 11. +- **Command layer + write safety** (spec §Architecture > Command layer) → Tasks 13, 14 (Task 14 explicitly tests non-TTY exit-6 and `--dry-run` no-confirm). +- **CLI wiring & permission registry** (spec §CLI wiring) → Tasks 12, 13. +- **Security** (spec §Security) → covered by Tasks 5 (`_warning` extension), 6/7 (bearer-only-in-memory in client construction), 14 (random-code confirm, no env bypass). +- **Testing layout** (spec §Testing) → every layer has its own test file; `tests/test_helpers.py`, `tests/test_config_store.py`, `tests/test_permissions.py` extensions covered. +- **Documentation sync** (spec §Documentation sync) → Task 15 enumerates every silent-drift surface. +- **Out of scope** (spec §Out of scope) → preserved by not having tasks for `dev-portal sync`, `peers`, or audit log. +- **Open questions** (spec §Open questions) → PNG dimension check resolved as soft-warn in Task 11; identity `vendor` field defaults handled in command layer (Task 13 — identity-add requires --vendor explicitly per call). + +### 2. Placeholder scan + +- No "TBD" / "TODO" / "fill in details" / "similar to Task N". +- Every code step shows complete code. +- Every test step shows complete tests. +- Every command step shows the exact command. + +### 3. Type consistency + +- `DeveloperPortalIdentity` fields (`username`, `password`, `role_hint`, `vendor`, `portal_url`) used consistently in Tasks 4, 5, 6, 10, 13. +- `PendingPatch.diff` is `list[FieldDiff]` everywhere (Tasks 11, 14). +- `require_random_code_confirmation(action_description: str) -> None` (raises on failure) — signature consistent across Tasks 3, 14. +- `ErrorCode` entries defined in Task 2 and used by name in Tasks 6, 7, 8, 9, 10, 11. +- `dev_portal_app` Typer instance name consistent in Tasks 13 (creation), 13 (registration in cli.py). +- `_split_app()` helper defined once in Task 13, used in Task 14. + +--- + +**Plan complete and saved to `docs/superpowers/plans/2026-05-28-dev-portal.md`.** + +Two execution options: + +1. **Subagent-Driven (recommended)** — I dispatch a fresh subagent per task, review between tasks, fast iteration. +2. **Inline Execution** — Execute tasks in this session using executing-plans, batch execution with checkpoints. + +Which approach? diff --git a/docs/superpowers/specs/2026-05-28-dev-portal-design.md b/docs/superpowers/specs/2026-05-28-dev-portal-design.md new file mode 100644 index 00000000..3d9c9466 --- /dev/null +++ b/docs/superpowers/specs/2026-05-28-dev-portal-design.md @@ -0,0 +1,397 @@ +# `kbagent dev-portal` — Developer Portal support + +**Status:** Draft, pending implementation +**Date:** 2026-05-28 +**Author:** Matyáš Jirát + +## Summary + +Add a `kbagent dev-portal` command group that wraps the Keboola Developer +Portal API (`https://apps-api.keboola.com`) for component vendors and portal +admins. Every change made through this surface is production-affecting on +every Keboola stack at once, so the safety model is intentionally stricter +than any other kbagent surface: an AI agent can research and *prepare* a +write, but cannot execute one without a human typing a random hex code on a +real terminal. No env-var bypass, no `--yes` flag. + +This work replaces the local-only `component-dev-portal` skill (`dp.py`) by +folding its logic into kbagent so the same primitive is available from the +CLI, the REST `serve` API, the kbagent plugin, and the MCP surface — with +the kbagent firewall, hint-mode codegen, audit-friendly identity model, and +3-layer test discipline applied. + +## Motivation + +The current state of Developer Portal automation in the team: + +- **`dp.py` skill** — useful but local-only, single-user, no integration + with kbagent's permission engine, no JSON output, no REST surface. +- **`scripts/developer_portal/update_properties.sh`** — pushes a fixed list + of Cookiecutter-backed properties from `component_config/*` files on + deploy. Doesn't help with one-off changes, doesn't manage app + registration, doesn't manage `uiOptions`, `encryption`, `network`, + `defaultBucket`, icons, etc. +- **Portal UI** — manual, slow, error-prone for property-level work; no + audit trail of "which identity did what". + +The gaps: + +1. The historically-manual *register a new app* step. +2. Ad-hoc property updates for fields the deploy scripts don't manage. +3. Multi-identity workflow (Keboola Vendor account, KDS Vendor account, + portal admin) — `dp.py` has one set of env vars at a time. +4. Programmatic *read* access for agents that need to see "what does + component X currently look like in the portal?" while designing a peer. +5. A unified, production-safe write path that doesn't depend on each + engineer remembering to use `--yes` carefully. + +## Non-goals + +- **ECR image push.** Stays in component GitHub Actions. +- **Bulk repo-file → property sync on deploy.** Stays in + `scripts/developer_portal/update_properties.sh`. +- **Writes to `component_config/`.** kbagent never materialises portal + state into the repo; that path belongs to the Cookiecutter template. +- **Persistent token caching.** Each kbagent invocation logs in fresh. + +## Design decisions (locked during brainstorming) + +| # | Decision | Rationale | +|---|----------|-----------| +| 1 | **Random-code TTY confirm** is the only write path. No `--yes`, no env override. | Direct-production blast radius. Agent cannot generate the random code. Same primitive as `kbagent permissions set`. | +| 2 | Identities stored as `{username, password, role_hint, vendor, portal_url}` in `AppConfig.dev_portal_identities`, same `config.json`, 0600, same `_warning` header. | Mirrors KB project token storage. One store, one lock, one place to look. | +| 3 | v1 scope = `list`, `get`, `create`, `patch`, `upload-icon`, `publish`, `deprecate`. | Matches `dp.py` plus full lifecycle. `list` + `get` are enough primitives for an agent to compose peer-config research itself — no dedicated helper. | +| 4 | `--identity ` flag per command + persisted default via `dev-portal identity use ALIAS`. | Mirrors `kbagent project` UX exactly. | +| 5 | App addressed as `--app VENDOR.APP_ID` (single arg), parsed to `vendor` + `app_id` internally. | Harder to mis-pair than two separate flags. `create` is the exception — no app id yet. | +| 6 | Uniform safety bar across `create`/`patch`/`upload-icon`/`publish`/`deprecate`. | Permission-engine categories (`write`/`admin`/`destructive`) already give graduated firewall control if persistent policy needs it. | + +## Architecture + +Standard kbagent 3-layer. + +``` +commands/dev_portal.py + │ CLI parsing, formatter, identity resolution, + │ random-code confirm gating, --dry-run, --json + ▼ +services/dev_portal_service.py + │ Identity CRUD, login orchestration, + │ prepare_*/apply pattern, diff computation, + │ publish pre-flight validation + ▼ +dev_portal_client.py (inherits BaseHttpClient) + │ Auth (login + MFA), HTTP verbs against apps-api.keboola.com, + │ icon two-hop (presigned S3 PUT) + ▼ +HTTPS to apps-api.keboola.com +``` + +### Data model + +New Pydantic model: + +```python +class DeveloperPortalIdentity(BaseModel): + username: str + password: str + role_hint: str = "vendor" # free-text label for `identity list` + vendor: str | None = None # default vendor for this identity + portal_url: str = "https://apps-api.keboola.com" + + @field_validator("portal_url") + @classmethod + def validate_portal_url(cls, v: str) -> str: + if not v.startswith("https://"): + raise ValueError("Portal URL must use https://") + return v +``` + +`AppConfig` gains two fields: + +```python +dev_portal_identities: dict[str, DeveloperPortalIdentity] = {} +default_dev_portal_identity: str = "" +``` + +`ConfigStore` gains mirror methods of the project methods: +`add_dev_portal_identity`, `remove_dev_portal_identity`, +`edit_dev_portal_identity`, `rename_dev_portal_identity`, +`set_default_dev_portal_identity`. The existing `_warning` field at the top +of `config.json` is extended to mention DP credentials. + +### Client layer (`dev_portal_client.py`) + +`DeveloperPortalClient(BaseHttpClient)` — gets retry/backoff/timeout for +free. State held only in process memory: + +```python +class DeveloperPortalClient(BaseHttpClient): + def __init__(self, identity: DeveloperPortalIdentity) -> None: ... + + # Auth (lazy on first authenticated call) + def _ensure_authenticated(self) -> None: ... + def _prompt_mfa(self, session: str) -> str: ... # /dev/tty, never stdin + + # Read + def list_apps(self, vendor: str) -> list[dict]: ... + def get_app(self, vendor: str, app_id: str) -> dict: ... + + # Write (dumb — confirm + dry-run belong to the layers above) + def create_app(self, vendor: str, payload: dict) -> dict: ... + def patch_app(self, vendor: str, app_id: str, payload: dict) -> dict: ... + def upload_icon(self, vendor: str, app_id: str, png_bytes: bytes) -> None: ... + def publish_app(self, vendor: str, app_id: str) -> dict: ... + def deprecate_app(self, vendor: str, app_id: str) -> dict: ... +``` + +Auth flow: + +1. `POST /auth/login {email, password}`. +2. `200 {"token": ...}` → bearer cached on the client instance. +3. `200 {"session": ...}` → MFA. If `sys.stdin.isatty()` then prompt via + `/dev/tty` and re-login with `{email, session, code}`. Otherwise raise + `DP_MFA_REQUIRED` with an actionable message naming service accounts. +4. Non-200 → raise `KeboolaApiError("DP_LOGIN_FAILED", …)`. + +Bearer never written to disk, never logged, exists only on the client +instance. Next invocation = fresh login. + +Icon upload is a two-hop: `POST /vendors/{v}/apps/{a}/icon` returns a +presigned URL, then `PUT` bytes to S3 via raw `urllib` (S3 doesn't use our +auth, retry, or timeout). Isolated in one method. + +### Service layer (`services/dev_portal_service.py`) + +Owns business logic. Commands stay thin; client stays dumb. + +```python +class DeveloperPortalService: + def __init__( + self, + config_store: ConfigStore, + client_factory: Callable[[DeveloperPortalIdentity], DeveloperPortalClient], + ) -> None: ... + + # Identity management + def add_identity(self, alias, identity) -> None: ... + def list_identities(self) -> dict[str, DeveloperPortalIdentity]: ... + def remove_identity(self, alias) -> None: ... + def edit_identity(self, alias, **fields) -> None: ... + def use_identity(self, alias) -> None: ... + def current_identity(self) -> str: ... + def verify_identity(self, alias) -> dict: ... # fresh login probe + + # Portal reads + def list_apps(self, alias, vendor) -> list[dict]: ... + def get_app(self, alias, vendor, app_id) -> dict: ... + + # Portal writes — prepare returns a PendingX dataclass with full diff; + # apply executes only after the command layer has run confirm. + def prepare_create(self, alias, vendor, payload) -> PendingCreate: ... + def prepare_patch(self, alias, vendor, app_id, payload) -> PendingPatch: ... + def prepare_upload_icon(self, alias, vendor, app_id, path) -> PendingIconUpload: ... + def prepare_publish(self, alias, vendor, app_id) -> PendingPublish: ... + def prepare_deprecate(self, alias, vendor, app_id) -> PendingWrite: ... + + def apply(self, pending: PendingWrite) -> dict: ... +``` + +`PendingWrite` and friends are small frozen dataclasses. `PendingPatch` +carries the fetched current state plus a list of `FieldDiff(key, current, +new)` for top-level keys that change — diff per top-level key is enough for +the preview and keeps it readable for nested JSON properties. + +Validation pre-flight (no portal call): + +- `prepare_create`: payload must have `id`, `name`, `type`; `name` must not + contain "extractor" or "writer". +- `prepare_publish`: fetch current state; raise + `DP_PUBLISH_REQUIREMENTS_MISSING` listing missing fields (`icon`, `name`, + `type`, `repository`, `shortDescription`, `longDescription`, + `licenseUrl`, `documentationUrl`). +- `prepare_upload_icon`: file exists; reads bytes; soft warning if not 128 + ×128 PNG. + +Peer-config research (e.g. "show me how MySQL and Postgres extractors +configure themselves so I can model a new DB connector after them") is +done by the agent calling `list --vendor V` followed by `get --app +VENDOR.APP` for the specific ids of interest, then comparing the +returned JSON in its own context. No dedicated helper — the agent has +the brains, and `list` + `get` already expose everything it needs. + +### Command layer (`commands/dev_portal.py`) + +``` +kbagent dev-portal identity add --alias A --username U + [--password P | --password-stdin] + [--role-hint vendor|admin] [--vendor V] + [--portal-url URL] +kbagent dev-portal identity list +kbagent dev-portal identity remove --alias A +kbagent dev-portal identity edit --alias A [--username U] + [--password P | --password-stdin] + [--role-hint H] [--vendor V] [--new-alias N] +kbagent dev-portal identity use ALIAS +kbagent dev-portal identity current +kbagent dev-portal identity verify [--identity A] + +kbagent dev-portal list --vendor V [--identity A] +kbagent dev-portal get --app VENDOR.APP [--identity A] + +kbagent dev-portal create --vendor V (--data FILE|@FILE|-) + [--identity A] [--dry-run] +kbagent dev-portal patch --app VENDOR.APP + (--data FILE | --property KEY (--value V | --value-file F)) + [--identity A] [--dry-run] +kbagent dev-portal upload-icon --app VENDOR.APP --file PATH + [--identity A] [--dry-run] +kbagent dev-portal publish --app VENDOR.APP [--identity A] [--dry-run] +kbagent dev-portal deprecate --app VENDOR.APP [--identity A] [--dry-run] +``` + +Write flow (uniform across `create` / `patch` / `upload-icon` / `publish` / +`deprecate`): + +```python +def cmd_patch(ctx, app, data, property_, value, value_file, identity, dry_run): + svc = get_service(ctx, "dev_portal_service") + formatter = get_formatter(ctx) + alias = resolve_identity_alias(ctx, identity) + require_permission(ctx, "dev-portal.patch") + + pending = svc.prepare_patch(alias, vendor, app_id, payload) + render_pending(formatter, pending) # stderr; never stdout + + if dry_run: + formatter.output({"status": "dry-run", "diff": pending.diff_as_json()}) + return + + require_random_code_confirmation(action=f"patch {app}") + result = svc.apply(pending) + formatter.output({"status": "ok", "app": app, + "patched_keys": [d.key for d in pending.diff]}) +``` + +`require_random_code_confirmation()` lives in `commands/_helpers.py`, +extracted from the current implementation in `commands/permissions.py`. +Single primitive used by `permissions set`, `permissions reset`, and every +DP write. Non-TTY = exit 6 with message: + +> This is a production-affecting Developer Portal write. Run from a real +> terminal — there is no `--yes` bypass by design. + +`--dry-run` exits 0 without prompting. This is the safe path agents call +freely. + +### CLI wiring & permission registry + +`cli.py` registers `dev_portal_app` Typer instance under name `dev-portal`. + +`OPERATION_REGISTRY` gains: + +``` +dev-portal.identity-add admin +dev-portal.identity-list read +dev-portal.identity-edit admin +dev-portal.identity-remove admin +dev-portal.identity-use write +dev-portal.identity-verify read +dev-portal.list read +dev-portal.get read +dev-portal.create write +dev-portal.patch write +dev-portal.upload-icon write +dev-portal.publish admin +dev-portal.deprecate destructive +``` + +`--deny-writes` automatically blocks all writes by category. Persistent +policy can pin individual operations the usual way. + +`commands/_helpers.py` gains `resolve_identity_alias(ctx, explicit)` and a +`get_dev_portal_service(ctx)` factory. + +## Security + +- **Bearer never persisted.** In-memory only on the client instance, scoped + to the invocation. +- **Password persisted on disk** with the same protections as KB Storage + tokens: 0600 file, locked dir, `_warning` header, auto-`.gitignore`. +- **No env-var bypass** for the random-code confirm. The + `--allow-env-manage-token` precedent is intentionally *not* mirrored — DP + credentials are persisted (no env needed for creds) and the safety bar is + the confirm, not env-deny. An env-var override of the confirm would + defeat the entire load-bearing safety claim. +- **Subprocess inheritance** already handled by + `mcp_transport._build_minimal_env()` allow-list (strips `KBC_*` and + anything not on the allow-list, so `KBC_DEVELOPERPORTAL_*` env vars + would be stripped by default). +- **`mask_token()`** applied to any error path that might surface the + bearer. +- **MFA prompt** opens `/dev/tty` explicitly so a redirected stdin doesn't + cause it to hang or quietly auto-fail. + +## Testing + +| File | Coverage | +|------|----------| +| `tests/test_dev_portal_client.py` (new) | Mocked HTTP for login (token + MFA-session + bad creds), list/get, create/patch, icon two-hop, publish/deprecate. | +| `tests/test_dev_portal_service.py` (new) | Identity CRUD; diff correctness; publish pre-flight detection; verify-on-add. | +| `tests/test_dev_portal_cli.py` (new) | Identity lifecycle; every write refuses on non-TTY with exit 6; every write succeeds with correct random code; `--dry-run` exits 0 without prompt; `--json` shapes stable. | +| `tests/test_config_store.py` (extend) | Adding/removing/editing identities; default bookkeeping; rename; default fall-through on removal. | +| `tests/test_permissions.py` (extend) | New ops in registry; `--deny-writes` blocks them; `dev-portal.deprecate` is destructive. | +| `tests/test_helpers.py` (extend) | `require_random_code_confirmation()` extracted helper: TTY/non-TTY, correct/wrong code, EOF. | +| `tests/test_e2e.py` (extend) | Smoke `dev-portal identity list`; guarded `dev-portal list --vendor` against the test portal if `E2E_DP_USERNAME`/`E2E_DP_PASSWORD` env vars are set; skip otherwise. | + +## Documentation sync (rule #17 silent-drift surfaces) + +Every PR shipping this work must update: + +- `src/keboola_agent_cli/commands/context.py` (`AGENT_CONTEXT`). +- `CLAUDE.md` `## All CLI Commands` section. +- `plugins/kbagent/agents/keboola-expert.md` — Rule 6 version gate, + tool-selection matrix, inline gotchas. +- `plugins/kbagent/skills/kbagent/SKILL.md` — decision-table row for + "manage portal property / register app". +- `plugins/kbagent/skills/kbagent/references/commands-reference.md`. +- `plugins/kbagent/skills/kbagent/references/dev-portal-workflow.md` + (new file, mirrors the structure of `workspace-workflow.md`). +- `plugins/kbagent/skills/kbagent/references/gotchas.md` — entry tagged + `(since v)` explaining: writes always require a TTY; the agent + must not attempt to invoke a DP write itself; agent's job ends at + `--dry-run` and presenting the preview. +- `src/keboola_agent_cli/changelog.py` — release entry on the version + bump that ships this. + +## Out of scope for v1 + +- **`dev-portal sync`** that mirrors `update_properties.sh` from a + workstation. Possible v2 if the deploy script gap becomes painful. +- **Portal admin operations** beyond `publish`/`deprecate` (token + management, vendor management, etc.). +- **Cross-vendor moves** of existing apps. Not supported by the API in a + clean way; out of scope. +- **Audit log** of which identity wrote what. Possible v2 by tailing + writes into a local SQLite at `${KBAGENT_CONFIG_DIR}/dev-portal-audit.db` + — flagged but not committed to. + +## Migration / rollout + +- No migration required for existing kbagent installs — new fields on + `AppConfig` default to empty. +- The local `component-dev-portal` skill in + `cf-claude-code-kit/plugins/component-developer/` stays in place during + v1 rollout. After kbagent v0.44.x lands, the skill's `SKILL.md` should + be updated to *prefer* the kbagent commands and only fall back to + `dp.py` for behaviour kbagent doesn't yet wrap. Once kbagent parity is + proven in real use, deprecate `dp.py` outright. + +## Open questions to resolve during implementation + +- **PNG dimension check.** Adding `Pillow` as a dependency just for 128×128 + validation is heavy. Soft warning via stdlib `struct` reading the PNG + IHDR chunk is cheaper and dep-free; prefer that. Confirmed at impl time. +- **Identity `vendor` field.** Currently optional, used as a default. Worth + considering whether an identity that has no `vendor` should refuse to + run any operation that needs one (vs. erroring later with a clear + message). Lean toward the latter — explicit per-command `--vendor` wins. diff --git a/plugins/kbagent/.claude-plugin/plugin.json b/plugins/kbagent/.claude-plugin/plugin.json index 159ba29d..bdaf980e 100644 --- a/plugins/kbagent/.claude-plugin/plugin.json +++ b/plugins/kbagent/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "kbagent", - "version": "0.46.1", + "version": "0.53.0", "description": "AI-friendly interface to Keboola Connection projects — explore configs, jobs, lineage, call MCP tools, manage dev branches, and debug SQL in workspaces", "author": { "name": "Keboola", diff --git a/plugins/kbagent/agents/kbagent-pr-reviewer.md b/plugins/kbagent/agents/kbagent-pr-reviewer.md index 4944b6ce..bafbe451 100644 --- a/plugins/kbagent/agents/kbagent-pr-reviewer.md +++ b/plugins/kbagent/agents/kbagent-pr-reviewer.md @@ -135,14 +135,14 @@ verify the file IS updated in the diff. If not, flag it. Specifically: |---|---|---| | `src/keboola_agent_cli/commands/context.py` (`AGENT_CONTEXT`) | Does the new command appear under the right `### ` heading? | BLOCKING | | `CLAUDE.md` `## All CLI Commands` | Does the new command's signature appear in the top-level command list? | BLOCKING | -| `plugins/kbagent/agents/keboola-expert.md` §2 Tool Selection Matrix | If new write/destructive command, is there a `\| ... \| First choice \| Fallback \| NEVER \|` row? | BLOCKING for write/destructive, NON-BLOCKING for read | +| `plugins/kbagent/agents/keboola-expert.md` §2 Tool Selection Matrix | One row **per command GROUP**, not per command. If the PR adds a new write/destructive *group*, is there a `\| ... \| First choice \| Fallback \| NEVER \|` row? A new command inside an existing group needs no new row. The file has a hard 60 KB prompt budget; exhaustive per-command coverage lives in `AGENT_CONTEXT` (loaded dynamically), so a missing matrix row is **never BLOCKING** -- flag it NON-BLOCKING only when the new group has zero rows AND `AGENT_CONTEXT` also omits it. | NON-BLOCKING | | `plugins/kbagent/agents/keboola-expert.md` §1 Rule 6 VERSION GATE | If feature is version-gated, are example version refs (`flow update needs 0.22.0+`) still accurate after this PR? | NON-BLOCKING (informational) | | `plugins/kbagent/agents/keboola-expert.md` §3 Inline Gotchas | If behavior is non-obvious, is there a bullet describing it? | NON-BLOCKING | | `plugins/kbagent/skills/kbagent/references/commands-reference.md` | New command bullet under correct section? | BLOCKING | | `plugins/kbagent/skills/kbagent/references/gotchas.md` | New non-obvious behavior tagged `(since vX.Y.Z)`? | BLOCKING for behavior changes (missing version tag means AI agents recommend behavior on older installs) | | `plugins/kbagent/skills/kbagent/references/-workflow.md` | New workflow file for new topic, or extension of existing file for extended workflow? | NON-BLOCKING | | `src/keboola_agent_cli/permissions.py` `OPERATION_REGISTRY` | Every new CLI command MUST have a `".": ""` entry. | BLOCKING (missing entry = permission engine silently allows the command under restrictive policy = security gap) | -| `src/keboola_agent_cli/hints/definitions/*.py` | Every new CLI command MUST have a `CommandHint` with both `ClientCall` and `ServiceCall`. | BLOCKING | +| ~~`src/keboola_agent_cli/hints/definitions/*.py`~~ | **DEPRECATED -- do NOT check.** `--hint` code generation is superseded by the `kbagent serve` REST API; new commands deliberately omit hint definitions. Never flag a missing `hints/definitions/` entry. | n/a (do not flag) | For each missed surface, your finding cites BOTH the original file/line in the diff that introduced the change AND the file path that should have been @@ -235,7 +235,7 @@ grep -E '^\+' /tmp/kbagent-pr-.diff | grep -E '(token|TOKEN|api_key|p - **BLOCKING**: must fix before merge. Bug, security issue, broken test, silent-drift gap from `CONTRIBUTING.md` Plugin synchronization map, layer violation, missing version tag on `gotchas.md`, missing - `OPERATION_REGISTRY` entry, missing `--hint` definition, broken backward + `OPERATION_REGISTRY` entry, broken backward compat without deprecation, `make check` non-zero. - **NON-BLOCKING**: should fix; not a merge blocker. Test gap on edge case, missing one-liner gotcha, suboptimal naming, unverified behavior diff --git a/plugins/kbagent/agents/keboola-expert.md b/plugins/kbagent/agents/keboola-expert.md index 4f9c0e64..65871f6f 100644 --- a/plugins/kbagent/agents/keboola-expert.md +++ b/plugins/kbagent/agents/keboola-expert.md @@ -1,6 +1,6 @@ --- name: keboola-expert -description: Keboola Connection operations specialist. MUST BE USED proactively for any task touching Keboola projects -- config browsing/updates, jobs, flows, schedules, storage, migrations, dev branches, debugging. Enforces fresh-fetch discipline, --dry-run on writes, CLI over REST, and refuses tasks it cannot safely complete with the installed kbagent version. Delegates write operations through two-step (dry-run -> confirm -> apply) flow without exception. +description: Keboola Connection ops specialist. Enforces fresh-fetch, dry-run, CLI-over-REST, version gate and confirmed apply. tools: Bash, Read, Edit, Write, Grep, Glob, TodoWrite, WebFetch model: sonnet color: blue @@ -67,10 +67,10 @@ a critical failure. needed for the current task (e.g. `flow update` needs 0.22.0+, `schedule find` needs 0.23.0+, `config set-default-bucket` needs 0.26.0+, `data-app create / deploy / start / stop / delete / password` - need 0.27.0+, `config update` script[] string-to-array auto-normalize - against #245 trap needs 0.28.0+, list-element re-split against + need 0.27.0+, `config update` script[] auto-normalize (#245) needs + 0.28.0+, list-element re-split against the #274 ODBC `Actual statement count N != desired 1` crash needs - 0.31.0+, `storage swap-tables` needs 0.28.0+, + 0.31.0+, `storage swap-tables` needs 0.28.0+, `storage clone-table` = 0.52.0+, env-var manage-token auth for `org setup` / `project refresh` / `data-app password` needs 0.29.0+ with `--allow-env-manage-token`, `project invite` / `project member-*` / `project invitation-*` @@ -91,30 +91,30 @@ a critical failure. `edit metric|dataset|constraint|relationship|glossary`, `import`, `promote`, `build`, `token --encrypt` - destructive: `remove metric|dataset|constraint|relationship|glossary` - - alias: `kbagent sl ...` is hidden-equivalent to - `kbagent semantic-layer ...` - - `semantic-layer build` falls back to a deterministic heuristic - (one dataset + one COUNT(*) metric + one glossary entry per - table) until an AI Service JSON-generation endpoint exists; - this is a BEHAVIOR note, not a version gate -- the heuristic - is the only path on 0.41.0, + - alias: `kbagent sl ...` = `kbagent semantic-layer ...` + - `semantic-layer build` is heuristic-only on 0.41.0+ (one dataset + one COUNT(*) metric + one glossary entry per table; not a version gate), `kbagent http get/post/patch/delete ` (self-call against the running serve from a scheduled-agent subprocess; reads `KBAGENT_SERVE_URL` + `KBAGENT_SERVE_TOKEN` env vars) needs 0.40.0+, - `kbagent serve --ui` (mounts the React SPA at `/`, single-process - browser dashboard with auto-injected token) needs 0.40.0+, - AI-agent run timeline persistence (cost / token / per-tool summary - on every persisted `AgentRun` plus `GET /agents/{id}/runs/{run_id}/events` - for replay) needs 0.40.0+, - `POST /ai/chat/stream` (generic Local AI co-pilot chat backed by the - user's local claude / codex / gemini CLI; backs the dashboard - Local AI tile that replaces Kai for non-master-token projects) - needs 0.41.9+, + `kbagent serve --ui` (mounts the React SPA, single-process + dashboard, auto-injected token) needs 0.40.0+, + AI-agent run timeline persistence (cost/token/per-tool summary on each + `AgentRun` + `GET /agents/{id}/runs/{run_id}/events` replay) needs 0.40.0+, + `POST /ai/chat/stream` (Local AI co-pilot chat backed by the user's + local claude / codex / gemini CLI; backs the dashboard Local AI tile + that replaces Kai for non-master-token projects) needs 0.41.9+, data-app CLI sandbox annotation = 0.42.0+ (#304), HTTP `?include_sandbox_annotation=true` = 0.43.1+ #312, `kbagent update --beta` = 0.43.3+, `data-app logs` = 0.43.8+, `kbagent agent ` (CLI parity /agents REST) = 0.44.0+, + `semantic-layer search-context|get-context`, `storage create-table --if-not-exists`, `sync push|pull|diff --branch`, `sync push --no-name-drift-warnings`, fresh-CREATE writeback + KBC.* = 0.47.0+, + Snowflake `workspace create` `private_key` = 0.47.1+, + `sync push` fresh-CREATE variable-link resolution + `--branch ` default-tree promote = 0.47.2+, + `feature` group (stack/project/user feature flags, Manage API) = 0.48.0+, + `dev-portal` = 0.49.0+ (admin-role PATCH = 0.51.1+), + headless `__env__` project (`KBAGENT_PROJECT_FROM_ENV=1` + `KBC_TOKEN` + `KBC_STORAGE_API_URL`) + forgiving stack-URL normalization (bare host / full project deep-link) = 0.50.0+, + `stream` command group (Data Streams / OTLP) = 0.50.0+, `storage retype` is a future composite), you MUST refuse the task and return a handoff message to the parent: `"Cannot proceed safely on kbagent . Missing: . @@ -138,22 +138,24 @@ a critical failure. | Update flow (rename, description, phases) | `kbagent flow update` (partial, no `--file`) | `--file` after fetching current phases, merging locally, passing full YAML | `tool call update_flow` (strips `behavior.onError` pre-MCP v1.60); partial `--file` that drops fields | | Schedule flow | `kbagent flow schedule --cron ... [--timezone]` | `tool call create_flow_schedule` | raw REST to `/storage/configurations/keboola.scheduler` | | Create Snowflake transformation | `kbagent config new --component-id keboola.snowflake-transformation --name N --project P --push --no-files` (0.33.0+; one-shot, no scaffold, body defaults to `{}` and validation auto-skips for empty shell -- then `config update --set ...` to fill in script) **or** `kbagent config new --component-id keboola.snowflake-transformation --project P --output-dir D` + `config update --set ...` (scaffold-then-patch) | `tool call create_sql_transformation` (lower schema, avoids the MCP `create_config` Snowflake refusal) | `tool call create_config` (refuses keboola.snowflake-transformation) -- note: `config new --push` does NOT inherit this refusal because it wraps the raw Storage API directly | -| Update SQL transformation body (script[]) | `kbagent config update --project P --component-id keboola.snowflake-transformation --config-id K --configuration @body.json` (0.28.0+ auto-normalizes string `script` to array; SQL gets statement-level split, Python/R gets `[script]` wrap; envelope's `normalizations: [...]` records every change. 0.31.0+ also re-splits multi-statement LIST elements -- closes the #274 ODBC `statement count 2 vs desired 1` crash that survives the 0.28.0 string fix) | `kbagent --hint client config update ...` if you need to bypass the auto-normalize for some reason | `tool call update_sql_transformation` -- still vulnerable to BOTH the #245 string-vs-array AND #274 list-element runtime crashes because it pushes raw to Storage API; raw `PUT /v2/storage/components/.../configs/...` -- same trap | +| Update SQL transformation body (script[]) | `kbagent config update --project P --component-id keboola.snowflake-transformation --config-id K --configuration @body.json` (0.28.0+ auto-normalizes string `script` to array; SQL gets statement-level split, Python/R gets `[script]` wrap; envelope's `normalizations: [...]` records every change. 0.31.0+ also re-splits multi-statement LIST elements -- closes the #274 ODBC `statement count 2 vs desired 1` crash that survives the 0.28.0 string fix) | -- | `tool call update_sql_transformation` -- still vulnerable to BOTH the #245 string-vs-array AND #274 list-element runtime crashes because it pushes raw to Storage API; raw `PUT /v2/storage/components/.../configs/...` -- same trap | | Run a job (and wait) | `kbagent job run --project P --component-id C --config-id K --wait` | `tool call run_component` | `job run` without `--wait` when user expects the result | +| Provision / read an OTLP Data Streams endpoint | `kbagent stream create-source -p P --name N --type otlp [--if-not-exists]` (auto-creates logs/metrics/traces sinks) then `stream detail N -p P --reveal` for endpoint+secret (0.50.0+) | `stream list`; `--no-sinks` for a bare source | deriving the `stream-in` URL yourself (use `source.otlp.url`); printing the secret unasked (masked by default) | | Search items by name across projects | `kbagent search QUERY [--project P] [--type table\|bucket\|config\|flow] [--limit N]` (0.30.0+) | `tool call search_tables` / `tool call search_configurations` (one resource-type per call) | chaining multiple `tool call` for different types | | Search config JSON bodies | `kbagent search QUERY --search-type config-based [--project P]` (0.30.0+) | `kbagent config search --query Q` (config-body only, no tables/buckets) | repeated `tool call get_config` to grep locally | | Browse configs (exploration) | `kbagent config list` / `kbagent config search --query Q` | `tool call list_configs` | full-project pull via MCP just to grep locally | | Fetch a specific config | `kbagent config detail --project P --component-id C --config-id K --json` | `tool call get_config` | re-using an earlier JSON dump | | Override the auto-derived output bucket on a config | `kbagent config set-default-bucket --bucket in.c-name` (0.26.0+) -- read-modify-write of `storage.output.default_bucket`, preserves siblings; `--clear` removes it | `kbagent config update --set 'storage.output.default_bucket=in.c-name'` (works pre-0.26.0 but not discoverable) | editing the raw JSON in the UI; full-config replace with `--configuration` (wipes other storage keys) | -| Cross-project migration | `kbagent sync pull` + edit files locally + `kbagent sync push --dry-run` | custom script via `kbagent --hint client` | repeated `tool call` loops, one per resource | -| Retype table columns | fetch types via `workspace query`, draft types YAML, write new transformation that produces typed output table, then `kbagent storage swap-tables` (0.28.0+) to flip the typed copy into the original name in a dev branch | `kbagent --hint client create_table_definition` if the future `storage retype` composite (§14.3) is not yet present | `POST /v2/storage/buckets/.../tables-definition` (REST) followed by manual config rewrites | +| Cross-project migration | `kbagent sync pull` + edit files locally + `kbagent sync push --dry-run` | -- | repeated `tool call` loops, one per resource | +| Retype table columns | fetch types via `workspace query`, draft types YAML, write new transformation that produces typed output table, then `kbagent storage swap-tables` (0.28.0+) to flip the typed copy into the original name in any branch | -- | `POST /v2/storage/buckets/.../tables-definition` (REST) followed by manual config rewrites | | Create typed table with native types | `kbagent storage create-table --column pk:VARCHAR(40) --column amount:NUMBER(18,2) --not-null pk --default amount=0` (0.25.0+) | `tool call create_table` (accepts the same `definition.length` shape via MCP) | re-creating via raw REST to `/v2/storage/...tables-definition` | -| Promote typed rebuild back into the original name | `kbagent storage swap-tables --project P --table-id in.c-foo.data --target-table-id in.c-foo.data_change_log --branch --yes` (0.28.0+) -- async storage job (`tableSwap`); client polls to completion before returning. Service refuses without a branch | -- | renaming or deleting + re-uploading (loses history; downstream configs need to be rewritten) | +| Promote typed rebuild back into the original name | `kbagent storage swap-tables --project P --table-id in.c-foo.data --target-table-id in.c-foo.data_change_log --branch --yes` (0.28.0+) -- async storage job (`tableSwap`); client polls to completion. Service refuses without a branch; any branch incl. prod | -- | renaming or deleting + re-uploading (loses history; downstream configs need to be rewritten) | | Re-seed a table without losing its schema / PK / dependents | `kbagent storage truncate-table --project P --table-id in.c-foo.data [--branch ID] [--dry-run] [--yes]` (0.32.0+) -- DELETE `/tables/{id}/rows?allowTruncate=1`; endpoint is uniformly async on every branch (returns a queued `tableRowsDelete` job; client polls via `_wait_for_storage_job`). Do NOT pass `async=true` -- the API rejects it. Batch via repeated `--table-id`. Returns `{truncated[], failed[], dry_run, project_alias}` with `truncated[]` entries carrying `{table_id, rows_before, rows_after, branch_id}`. Permission class: `destructive` | `tool call delete_table_rows` if the upstream MCP exposes it | drop + recreate the table (loses descriptions, PK, sharing edges, and breaks every downstream config reference); deleting rows via raw SQL in a workspace (bypasses the Storage API audit trail) | | Debug a failed job | `kbagent job detail --project P --job-id J --json` + `kbagent job run ... --log-tail-lines 200` | `kbagent workspace from-transformation` for SQL repro | "I think the issue is..." without reading logs | | Ad-hoc SQL / row-count / type audit | `kbagent workspace create` + `kbagent workspace load` + `kbagent workspace query --sql "..."` | `kbagent workspace from-transformation` for existing transform debugging; `workspace list --qs-compatible` (0.42.0+, #304) for data-app reuse | querying Keboola Storage directly via Snowflake credentials outside the workspace abstraction | | Inspect dev branch | `kbagent branch list --project P`, `kbagent branch use --project P --branch ID` | `tool call get_branch` | acting on `main` when a dev branch exists | | Audit project capabilities / features | `kbagent project info --project P` (0.30.0+) -- returns project ID, name, backend, enabled features, quota limits, and metrics | `tool call verify_token` (returns less structured info; no feature list) | inspecting the UI project settings manually | +| Manage feature flags (stack catalogue / project / user) | `kbagent feature list\|project-show\|project-add\|project-remove\|user-show\|user-add\|user-remove --project P [--email E] [--feature NAME] [--dry-run] [--yes]` (0.48.0+) -- Manage API; needs a SUPER-ADMIN manage token (interactive prompt; `--allow-env-manage-token`+`KBC_MANAGE_API_TOKEN` for CI); `--project` resolves the stack URL (+project_id for `project-*`); add=admin, remove=destructive; add body is `{"feature":NAME}` | `kbagent project info` for a project's *enabled* features (read-only, no super-admin) | raw `/manage/...` calls; manage token via a CLI flag | | Create a new config (one-shot remote, no scaffold to disk) | `kbagent config new --project P --component-id C --name N --push --no-files [--configuration @body.json] [--branch ID]` (0.33.0+) -- single CLI call POSTs to `/v2/storage/components/{cid}/configs`; default body is `{}` (FIIA empty-shell pattern, validation auto-skips); explicit `--configuration` body is schema-validated by default (`--no-validate` opts out); works for ALL component types incl. `keboola.snowflake-transformation` | `kbagent config new --output-dir D` then edit + `kbagent sync push` (scaffold-then-push GitOps flow) | `tool call create_config` (refuses keboola.snowflake-transformation; raw MCP envelope, no validation) | | Create a config row | `kbagent config row-create --project P --component-id C --config-id K --name NAME` (0.30.0+) | `tool call create_config_row` | `POST /v2/storage/components/C/configs/K/rows` (raw REST) | | Update a config row | `kbagent config row-update --project P --component-id C --config-id K --row-id R [--name N] [--configuration JSON]` (0.30.0+) | `tool call update_config_row` | `PUT /v2/storage/components/C/configs/K/rows/R` (raw REST) | @@ -161,18 +163,19 @@ a critical failure. | Get OAuth authorization URL | `kbagent config oauth-url --project P --component-id C --config-id K` (0.30.0+) -- returns URL to open in browser to complete OAuth flow | -- | raw `GET /v2/storage/components/C/configs/K/oauth/authorize` | | Inventory data apps | `kbagent data-app list --project P` (0.27.0+; 0.43.9+ skips sandboxes) | `tool call get_configs --component_id keboola.data-apps` (Storage view only -- no state/URL/configVersion) | per-project `tool call` joined to Data Science | | Bring a new data app online from a git repo | `kbagent data-app create --project P --name N --slug S --git-repo URL [--git-pat-env VAR \| --git-public]` (0.27.0+) | `tool call create_config keboola.data-apps` + manual `kbagent encrypt values` + raw `POST /apps` -- only for custom shapes | raw `POST data-science/apps` then `PATCH desiredState=running` without `configVersion + restartIfRunning` (the §9 footgun -- pins to v2 empty shell, errors `dataApp.git.repository is required`) | -| Roll out a new code or config version on a data app | `kbagent data-app deploy --project P --app-id N --wait` (0.27.0+) -- always sends the §9 trio | `kbagent --hint client data-app deploy ...` to inspect the generated `patch_app(desired_state=, config_version=, restart_if_running=True)` call | `tool call update_config` then `tool call run_component` (data apps are not jobs -- the queue runner does not deploy them) | +| Roll out a new code or config version on a data app | `kbagent data-app deploy --project P --app-id N --wait` (0.27.0+) -- always sends the §9 trio | -- | `tool call update_config` then `tool call run_component` (data apps are not jobs -- the queue runner does not deploy them) | | Wake an auto-suspended data app | `kbagent data-app start --project P --app-id N` (0.27.0+) -- does NOT bump configVersion | hitting the app's URL (auto-restart, 30-60s cold boot) | `kbagent data-app deploy` (overkill -- bumps configVersion) | | Pause a running data app | `kbagent data-app stop --project P --app-id N` (0.27.0+) | -- | `kbagent data-app delete` (irreversible; cascades to Storage config) | | Read the simpleAuth password | `kbagent data-app password --project P --app-id N` (0.27.0+) -- needs Manage API token (interactive prompt; `--allow-env-manage-token` + `KBC_MANAGE_API_TOKEN` on 0.29.0+) | -- | trying to "rotate" (not supported -- delete + recreate to mint a new one) | | Tear down a data app | `kbagent data-app delete --project P --app-id N` (0.27.0+) -- cascades to Storage config; URL retired | -- | manually `tool call delete_config keboola.data-apps` -- orphans the deployment record | | Tail a data-app container log (failed deploy / runtime crash) | `kbagent data-app logs --project P --app-id N [--lines N \| --since ISO8601]` (0.43.8+) -- plain-text tail; flags mutex; may echo runtime secrets | `tool call get_data_apps` (20-line cap) | opening the UI "Terminal Log" tab | +| Developer Portal: register / inspect / update a component | reads `kbagent dev-portal list\|get` (agent-safe); writes `create\|patch\|upload-icon\|publish\|deprecate` (0.49.0+) need a human to type a random code on a real TTY; `--dry-run` is the agent-safe preview | `kbagent serve` `GET /dev-portal/apps` (reads) | raw `apps-api.keboola.com`; ANY write from a non-TTY/agent shell (no bypass; exits 6) | | Invite a user to a project (single) | `kbagent project invite --project P --email E --role admin\|guest\|readOnly\|share` (0.29.0+) | raw `requests.post(/manage/projects/{id}/invitations)` only if version-gated out | `kbagent project invite` without `KBC_MANAGE_API_TOKEN` set; passing manage token via CLI flag | -| Invite many users (bulk) | `kbagent project invite --from-csv FILE [--default-role guest] [--workers N] [--dry-run]` (0.29.0+) | `--hint client` to generate a parallel script using `ManageClient` | per-row shell loop calling the CLI -- defeats the parallelism + idempotency the service already does | +| Invite many users (bulk) | `kbagent project invite --from-csv FILE [--default-role guest] [--workers N] [--dry-run]` (0.29.0+) | -- | per-row shell loop calling the CLI -- defeats the parallelism + idempotency the service already does | | List active project members | `kbagent project member-list --project P [--include-pending]` (0.29.0+) | `tool call run_sync_action` against the Manage API | reading `.kbagent/config.json` to infer membership (it only stores the local user's token) | | List pending invitations | `kbagent project invitation-list --project P` (0.29.0+) | -- | -- | | Cancel a pending invitation | `kbagent project invitation-cancel --project P --email E --yes` (0.29.0+) | `--invitation-id ID` if email lookup is ambiguous | DELETE via raw HTTP without going through the service layer | -| Remove an active member | `kbagent project member-remove --project P --email E --yes` (0.29.0+, **destructive**) | `--hint client` for a script that removes by user_id directly | calling `member-remove` without `--yes` in non-interactive contexts (it will prompt and hang) | +| Remove an active member | `kbagent project member-remove --project P --email E --yes` (0.29.0+, **destructive**) | -- | calling `member-remove` without `--yes` in non-interactive contexts (it will prompt and hang) | | Change a member's role | `kbagent project member-set-role --project P --email E --role admin\|guest\|readOnly\|share` (0.29.0+) | -- | `PUT /manage/projects/{id}/users/{userId}` -- the API rejects PUT with 404, the kbagent client correctly uses **PATCH** | | Set / rotate app-runtime secrets | `kbagent data-app secrets-set --project P --app-id N --secret '#KEY=VAL'` (0.29.0+) then `data-app deploy --wait` -- per-project KMS encryption, fail-closed, never auto-deploys | `encrypt values --component-id keboola.data-apps` + `tool call update_config` -- ONLY for a non-standard secrets shape | raw `POST` to encryption + Storage without read-modify-write -- clobbers sibling keys (Storage `merge=True` is shallow) | | Inspect what secrets are set on a data app | `kbagent data-app secrets-list --project P --app-id N` (0.29.0+) -- metadata only, never decrypts | `tool call get_configs --component_id keboola.data-apps` then read `parameters.dataApp.secrets` keys (raw dict, may leak ciphertext) | trying to decrypt -- the Encryption API has no decrypt endpoint | @@ -180,10 +183,11 @@ a critical failure. | Remove a secret / env-var key from a data app | `kbagent data-app secrets-remove --project P --app-id N --key 'KEY' --yes` (0.43.9+: `#` optional; removes encrypted + plain) -- idempotent; missing keys exit 0, `removed: 0` | `tool call update_config` with the secrets sub-dict deleted -- ONLY for batch removes needing a custom change description | `config update --set 'parameters.dataApp.secrets={}'` -- drops EVERY secret, not just the named ones | | Pre-flight a data-app repo before create | `kbagent data-app validate-repo --git-repo URL --type python-js [--git-pat-env VAR]` (0.29.0+) -- BLOCKING / WARN / OK with help-doc citations; ≤5 GitHub API calls regardless of repo size | git-clone the repo locally and inspect by hand | `data-app create --dry-run` (only shows the request bodies; does not validate repo structure) | | Rename a project alias | `kbagent project edit --project OLD --new-alias NEW [--dry-run]` (0.31.0+) -- cascades through `config.json` (`projects` key + `default_project`) and the nested-sync directory `//`. Combined with `--url`/`--token` in one call, those mutations target the new alias post-rename. `--dry-run` previews collision detection, planned disk-rename method, and the lineage-cache warning without mutating state. **Lineage cache (if any) is NOT auto-updated**: rebuild via `kbagent lineage build` after the rename | `kbagent project remove` + `kbagent project add` (re-enters the token; loses any nested sync workspace) | hand-editing `~/.config/keboola-agent-cli/config.json` (no validation, easy to miss `default_project` cascade) | +| Run kbagent headless from a daemon / container / CI with only a token in env (no `project add`, no `config.json`) | Export `KBAGENT_PROJECT_FROM_ENV=1` + `KBC_TOKEN` + `KBC_STORAGE_API_URL`, then `kbagent --json storage file-upload --project __env__ --file X` (0.50.0+). Synthesizes an in-memory `__env__` project; token NEVER persisted (stripped on any save); same env setup also powers `kbagent serve` (POST `project=__env__`) | a one-shot `kbagent project add --project env --token ... --url ...` (works but writes the token to `config.json` on disk -- defeats "no local config") | hand-writing a `config.json` with the token, or passing `--token` per command (no such passthrough on storage/job/config commands) | | Call the running `kbagent serve` from a scheduled-agent subprocess | `kbagent http get/post/patch/delete ` (0.40.0+) -- uses `KBAGENT_SERVE_URL` + `KBAGENT_SERVE_TOKEN` env vars auto-injected by the scheduler. `kbagent http get /openapi.json` to discover endpoints. Treats the live serve as source-of-truth (no stale local config) | forking `kbagent ` (also fine -- `KBAGENT_CONFIG_DIR` is propagated so the spawned CLI sees the SAME config the serve uses; no more "I'm in the wrong directory" surprises) | `curl $KBAGENT_SERVE_URL/...` by hand (works, but `kbagent http` adds auth header automatically, structured error mapping, and JSON-mode formatting) | | Launch the web UI for an end-user (browser dashboard, no Node BFF) | `kbagent serve --ui [--port PORT] [--ui-dist PATH]` (0.40.0+) -- single-process FastAPI mounts the bundled React SPA at `/`, sets an HttpOnly `kbagent_session` cookie on `GET /` so the browser is auto-authenticated. EventSource SSE works via the same cookie -- no token in URL, JS heap, or access log. Requires the bundled wheel (Node 20+ on the install host) OR `make web-build` from a checkout. CORS origins customisable via `--cors-origin` | `kbagent serve` (plain API) + Vite dev server + Node BFF -- the legacy three-process flow with hot reload, see `web/README.md` "Dev mode" section | inventing a `--token-in-url` flag; running uvicorn directly against `web.frontend.dist` -- the path-rewrite middleware + cookie bootstrap only fire from `kbagent serve --ui` | | Schedule / manage Agent Tasks | `kbagent agent ` (0.44.0+) -- CRUD `list/show/create/update/delete`, exec `run [--stream]`, history `runs/run-detail/run-events`, util `test/cron-preview/prompt-improve`. Local-only; cron needs `kbagent serve`. See [agent-tasks-cli-workflow](../skills/kbagent/references/agent-tasks-cli-workflow.md) | `kbagent http /agents...` (0.40.0+) in scheduled subprocesses; Web UI for human authoring | hand-editing `agents.json` | -| List models / metrics / entities in a semantic-layer model | `kbagent --json semantic-layer show --project P [--model M] [--type metric\|dataset\|relationship\|constraint\|glossary]` (0.41.0+); `kbagent --json semantic-layer model list --project P` to enumerate models when --model is ambiguous | `kbagent --json tool call get_semantic_layer_*` if the MCP exposes a read tool (none in the kbagent MCP at v0.41.0) | hand-rolled `urllib`/`httpx` loops against `metastore.*.keboola.com` (the `sl-builder` skill's old approach -- bypasses retry/backoff and the kbagent error envelope) | +| List models / metrics / entities in a semantic-layer model | `kbagent --json semantic-layer show --project P [--model M] [--type metric\|dataset\|relationship\|constraint\|glossary]` (0.41.0+); `kbagent --json semantic-layer model list --project P`; `search-context` / `get-context` (0.47.0+) for glob/id lookup | `kbagent --json tool call get_semantic_layer_*` if the MCP exposes a read tool (none at v0.41.0) | hand-rolled `urllib`/`httpx` loops against `metastore.*.keboola.com` (the `sl-builder` skill's old approach -- bypasses retry/backoff and the kbagent error envelope) | | Validate a semantic-layer model (phantom fields, constraint orphans, AGG-on-STRING) | `kbagent --json semantic-layer validate --project P [--model M] [--deep]` (0.41.0+) -- basic = local structural checks (duplicates, dangling refs, sum-on-pct, constraint orphans, severity-suffix); `--deep` adds parallel Snowflake column-existence probes via the in-process StorageService | hand-coded list+filter Python that re-implements the structural checks (loses the `--deep` Snowflake probe) | running validation by spinning up a workspace and SELECT * FROM every dataset (slow, requires workspace creation, no constraint-orphan detection) | | Snapshot a semantic-layer model to disk (before destructive edits) | `kbagent semantic-layer export --project P [--model M] [--output PATH]` (0.41.0+) -- self-describing JSON, default `./sl_export_{model_name}_{YYYYMMDD_HHMMSS}.json` | `kbagent --json semantic-layer show --project P` and pipe to a file (NOT a clean snapshot -- missing model metadata, no schemaVersion, no round-trip guarantee) | -- | | Diff a dev model against prod / against a snapshot | `kbagent --json semantic-layer diff --project-a dev --project-b prod` (project<->project); swap one side for `--file-a` / `--file-b` to diff against a snapshot (0.41.0+) | export both, run `diff` / `jq` on the JSON manually (no per-type added/removed/changed grouping, no `diff_keys`) | -- | @@ -223,6 +227,11 @@ success, not a failure. lower-level schema. `config new --push` does NOT inherit the MCP refusal because it calls Storage API directly. +- **Snowflake workspace credentials** (0.47.1+): headless + `kbagent workspace create` returns `private_key` for Snowflake; `password` + is empty/unusable. Use the one-time PKCS8 PEM for key-pair auth. BigQuery + keeps the prior shape. + - **`script[]` string-vs-array runtime crash** (0.28.0+ auto-fix; #245): the Storage API silently accepts `parameters.blocks[].codes[].script` as a string, but the runtime validator rejects it (`Expected array, @@ -285,12 +294,24 @@ success, not a failure. plan -- it sets up the user for an impossible step. - **`storage create-table` in a dev branch auto-materializes the bucket** - (0.25.0+): if the target bucket has not been written to in the branch - yet, kbagent creates it there first (mirrors the Go CLI's - `EnsureBucketExists`). The response's `auto_created_bucket: true` is - informational, not an error -- surface it to the user in a write - verification payload but do not treat it as a failure signal. - Production writes never materialize anything. + (0.25.0+): if the target bucket has no branch-local write yet, kbagent + creates it first (mirrors the Go CLI `EnsureBucketExists`). + `auto_created_bucket: true` is informational, not a failure. Production + writes never materialize anything. +- **`storage clone-table` before an in-branch `swap-tables` / column drop** + (0.52.0+): on `storage-branches` projects a dev branch reads prod tables + transparently until first write, so a swap/drop (a write) fails with a + misleading "bucket not found" until the prod table is branch-local. Run + `kbagent storage clone-table --project P --table-id T --branch ` + first (one-way default->branch). See `gotchas.md`. +- **`sync pull --force` is conflict-aware, not a blind overwrite** (0.53.0+): + a locally-modified config whose remote is UNCHANGED is preserved (its pending + delta stays pushable); a true merge conflict (local AND remote both changed + since last pull) ABORTS the pull with exit 1 / `SYNC_CONFLICT` instead of + silently discarding work (pre-0.53.0 it silently corrupted the baseline and + stranded the edits). Safe to force-pull an unrelated config while you have + un-pushed edits elsewhere. To discard a local edit on purpose, delete the + file/dir then pull. See `gotchas.md`. - **`storage truncate-table` is row-only; schema and dependents are preserved** (0.32.0+): the underlying call is @@ -330,8 +351,8 @@ success, not a failure. - **`project member-set-role` uses PATCH, not PUT** (0.29.0+): The Manage API endpoint is `PATCH /manage/projects/{id}/users/{userId}` with `{"role": "..."}`. PUT returns 404 even on real members. kbagent's - `ManageClient.update_project_member_role` emits PATCH; if you write a - `--hint client` script that hits the endpoint directly, do the same. + `ManageClient.update_project_member_role` emits PATCH; if you script + against the endpoint directly, do the same. - **`legacy_branch_storage: true` on `--branch` writes** (0.25.2+): Projects without the `storage-branches` feature flag (legacy fake-branch @@ -474,31 +495,17 @@ success, not a failure. `--allow-env-manage-token` to their invocation, never strip the warning by suppressing stderr. -- **Semantic-layer gotchas (since v0.41.0)** — five behavior contracts - worth committing to memory before touching `semantic-layer add/edit/ - remove`. Full prose lives in - [`gotchas.md` § Semantic-layer](../skills/kbagent/references/gotchas.md); - the short form: - - **Constraint `rule` is a STRING**, never `{bounds: {min, max}}`. The - sl-builder skill docs are wrong on this. kbagent enforces it. - - **Constraint `name` regex `^[a-z][a-z0-9_]*$`** + the 3-vs-4 - severity split: API `severity` is `error | warning | info` (3-level); - the 4-band health (`_critical / _warning / _healthy / _review`) - lives in the NAME SUFFIX, not on the API. - - **`edit metric --new-name` cascades through every constraint** whose - `metrics[]` referenced the old name, and prints the old/new - CODE_METRIC value. Downstream SQL joining on CODE_METRIC will break - silently — surface the change to the operator. - - **`remove metric` orphans constraints** that reference it. The - pre-deletion scan ALWAYS prints the warning (even with `--yes`); - non-TTY without `--yes` exits 2. Recommended: drop/rewrite the - constraints first, then remove the metric. - - **`build` is a HEURISTIC fallback**, not full AI: one dataset + - one COUNT(*) metric + one glossary entry per table. Response carries - `fallback_used: "heuristic"`. Treat the output as a scaffold and - follow up with `add metric`, `add relationship`, `add constraint`. - The full AI wizard lives in the `sl-build` skill under - `04_AI_Kit/ai-kit/`. +- **Semantic-layer gotchas (since v0.41.0)** — full prose in + [`gotchas.md` § Semantic-layer](../skills/kbagent/references/gotchas.md). + Key traps: constraint `rule` is a STRING (not `{bounds: {min, max}}`); + `severity` is 3-level (`error|warning|info`) while the 4-band health + lives in the name suffix (`_critical/_warning/_healthy/_review`), not the + API; `edit metric --new-name` cascades into constraints' `metrics[]` and + changes CODE_METRIC (surface it -- downstream joins break silently); + `remove metric` orphans referencing constraints (drop/rewrite them + first; the scan warns even with `--yes`, non-TTY exits 2); `build` is a + heuristic scaffold (`fallback_used: "heuristic"`), not the full AI wizard + (that lives in the `sl-build` skill). --- @@ -604,8 +611,7 @@ kbagent sync push --project dest - `MCP tool call isError: true, reason: "schema mismatch"`: → DO NOT retry with reformatted inputs. → Look up the native `kbagent ` equivalent in §2. If it exists, - switch to it. If not, `kbagent --hint client ` for a - direct API snippet. (`--hint` is deprecated since 0.45.0; use `kbagent serve` REST API for new integrations.) + switch to it; otherwise use the `kbagent serve` REST API. - `update_flow` returned success but verification shows `behavior.onError = None` on phases that had it before: diff --git a/plugins/kbagent/skills/kbagent/SKILL.md b/plugins/kbagent/skills/kbagent/SKILL.md index 8be14f20..07234b5a 100644 --- a/plugins/kbagent/skills/kbagent/SKILL.md +++ b/plugins/kbagent/skills/kbagent/SKILL.md @@ -41,6 +41,14 @@ description: > list members, remove member, change role, project role, bulk invite, invite from CSV, project access, member management, manage token prompt, --allow-env-manage-token, KBC_MANAGE_API_TOKEN, + feature flag, feature flags, list features, project features, user features, + enable feature, disable feature, set feature flag, add feature, remove feature, + early-adopter-preview, direct-access, pay-as-you-go, /manage/features, + super admin token, super-admin feature, stack feature catalogue, + data stream, data streams, keboola data streams, stream source, OTLP, + OpenTelemetry, otel, OTLP endpoint, OTEL_EXPORTER_OTLP_ENDPOINT, telemetry ingest, + logs metrics traces, stream create-source, stream detail, stream list, + stream delete, otlp source, http source, stream-in, ingest endpoint, semantic-layer, semantic layer, semantic-layer model, metastore, semantic-metric, semantic-dataset, semantic-relationship, semantic-constraint, semantic-glossary, add metric, edit metric, @@ -51,7 +59,11 @@ description: > 4-band health, _critical _warning _healthy _review, CODE_METRIC, DIM_METRIC_THRESHOLD, dangling metric FK, orphaned constraint, phantom field, AGG on STRING, SUM on PCT, deep validate, - sl, kbagent sl, semantic layer wizard, sl-build, sl-add, sl-edit. + sl, kbagent sl, semantic layer wizard, sl-build, sl-add, sl-edit, + developer portal, dev-portal, apps-api, register component, vendor app, + portal property, ui-options, encryption portal, defaultBucket portal, + app icon, configurationSchema portal, publish component, deprecate component, + kbagent dev-portal, portal identity, vendor login, service account portal. --- # kbagent -- Keboola Agent CLI @@ -123,6 +135,13 @@ When working inside a git repository or project directory, run `kbagent init` (o | Remove an active member from a project (destructive) | `kbagent project member-remove --project PROJECT --email EMAIL` | | Change an existing member's role (PATCH) | `kbagent project member-set-role --project PROJECT --email EMAIL --role ROLE` | | Set up projects and register them in the kbagent config | `kbagent org setup --url URL` | +| List all feature flags defined on the stack | `kbagent feature list --project PROJECT` | +| Show feature flags assigned to a project | `kbagent feature project-show --project PROJECT` | +| Enable a feature flag on a project | `kbagent feature project-add --project PROJECT --feature FEATURE` | +| Disable a feature flag on a project (destructive) | `kbagent feature project-remove --project PROJECT --feature FEATURE` | +| Show feature flags assigned to a user | `kbagent feature user-show --project PROJECT --email EMAIL` | +| Enable a feature flag on a user | `kbagent feature user-add --project PROJECT --email EMAIL --feature FEATURE` | +| Disable a feature flag on a user (destructive) | `kbagent feature user-remove --project PROJECT --email EMAIL --feature FEATURE` | | List available components from connected projects | `kbagent component list` | | Show detailed information about a specific component | `kbagent component detail --component-id COMPONENT-ID` | | List configurations from connected projects | `kbagent config list` | @@ -174,7 +193,8 @@ When working inside a git repository or project directory, run `kbagent init` (o | Delete one or more storage tables | `kbagent storage delete-table --project PROJECT --table-id TABLE-ID` | | Truncate (delete all rows from) one or more storage tables | `kbagent storage truncate-table --project PROJECT --table-id TABLE-ID` | | Delete one or more columns from a storage table | `kbagent storage delete-column --project PROJECT --table-id TABLE-ID --column COLUMN` | -| Swap two storage tables in a development branch | `kbagent storage swap-tables --project PROJECT --table-id TABLE-ID --target-table-id TARGET-TABLE-ID` | +| Swap two storage tables (any branch, including the default/production branch) | `kbagent storage swap-tables --project PROJECT --table-id TABLE-ID --target-table-id TARGET-TABLE-ID` | +| Clone (pull) a production table into a development branch | `kbagent storage clone-table --project PROJECT --table-id TABLE-ID` | | Delete one or more storage buckets | `kbagent storage delete-bucket --project PROJECT --bucket-id BUCKET-ID` | | Set the description on a storage bucket | `kbagent storage describe-bucket --project PROJECT --bucket-id BUCKET-ID` | | Set the description on a storage table | `kbagent storage describe-table --project PROJECT --table-id TABLE-ID` | @@ -188,6 +208,10 @@ When working inside a git repository or project directory, run `kbagent init` (o | Delete one or more Storage Files | `kbagent storage file-delete --project PROJECT --file-id FILE-ID` | | Load a Storage File into a table | `kbagent storage load-file --project PROJECT --file-id FILE-ID --table-id TABLE-ID` | | Export a table to a Storage File | `kbagent storage unload-table --project PROJECT --table-id TABLE-ID` | +| List Data Streams sources in a project | `kbagent stream list --project PROJECT` | +| Create an OTLP (or HTTP) source and return its endpoint | `kbagent stream create-source --project PROJECT --name NAME` | +| Show a source's endpoints, protocol, and destination tables | `kbagent stream detail [SOURCE-ID] --project PROJECT` | +| Delete a Data Streams source (destructive) | `kbagent stream delete --project PROJECT` | | List shared buckets available for linking | `kbagent sharing list` | | Enable sharing on a bucket | `kbagent sharing share --project PROJECT --bucket-id BUCKET-ID --type SHARING-TYPE` | | Disable sharing on a bucket | `kbagent sharing unshare --project PROJECT --bucket-id BUCKET-ID` | @@ -253,6 +277,8 @@ When working inside a git repository or project directory, run `kbagent init` (o | Snapshot a semantic-layer model to a self-describing JSON file | `kbagent semantic-layer export --project PROJECT` | | Diff two semantic-layer snapshots (project↔project, project↔file, file↔file) | `kbagent semantic-layer diff` | | Validate a semantic-layer model | `kbagent semantic-layer validate --project PROJECT` | +| Search semantic-layer entities across a project by name pattern | `kbagent semantic-layer search-context --project PROJECT` | +| Fetch a single semantic-layer entity by id, irrespective of its type | `kbagent semantic-layer get-context --project PROJECT --context-id CONTEXT-ID` | | List all semantic-layer models in a project | `kbagent semantic-layer model list --project PROJECT` | | Create a new semantic-layer model | `kbagent semantic-layer model create --project PROJECT --name NAME` | | Delete a semantic-layer model and cascade-delete its children | `kbagent semantic-layer model delete --project PROJECT --model MODEL` | @@ -271,6 +297,10 @@ When working inside a git repository or project directory, run `kbagent init` (o | Remove a constraint | `kbagent semantic-layer remove constraint --project PROJECT --name NAME` | | Remove a relationship. | `kbagent semantic-layer remove relationship --project PROJECT --name NAME` | | Remove a glossary term. | `kbagent semantic-layer remove glossary --project PROJECT --term TERM` | +| List reference-data records (dimension summaries; use ``get`` for members) | `kbagent semantic-layer reference-data list --project PROJECT` | +| Fetch one record (all members) by ``--id`` or by ``--model`` + ``--dimension`` | `kbagent semantic-layer reference-data get --project PROJECT` | +| Create or replace (by model + dimension) a reference-data record | `kbagent semantic-layer reference-data set --project PROJECT --dimension DIMENSION --members-file MEMBERS-FILE` | +| Delete a reference-data record by UUID (server-side soft-delete) | `kbagent semantic-layer reference-data delete --project PROJECT --id ID-` | | Encrypt the project's storage token for transformation `user_properties` | `kbagent sl token --project PROJECT --component-id COMPONENT-ID` | | Build a semantic-layer model from a list of storage tables (non-interactive) | `kbagent sl build --project PROJECT` | | Promote a model from one project to another (NEW + overwrite CHANGED; never deletes) | `kbagent sl promote --from-project FROM-PROJECT --to-project TO-PROJECT` | @@ -279,6 +309,8 @@ When working inside a git repository or project directory, run `kbagent init` (o | Snapshot a semantic-layer model to a self-describing JSON file | `kbagent sl export --project PROJECT` | | Diff two semantic-layer snapshots (project↔project, project↔file, file↔file) | `kbagent sl diff` | | Validate a semantic-layer model | `kbagent sl validate --project PROJECT` | +| Search semantic-layer entities across a project by name pattern | `kbagent sl search-context --project PROJECT` | +| Fetch a single semantic-layer entity by id, irrespective of its type | `kbagent sl get-context --project PROJECT --context-id CONTEXT-ID` | | List all semantic-layer models in a project | `kbagent sl model list --project PROJECT` | | Create a new semantic-layer model | `kbagent sl model create --project PROJECT --name NAME` | | Delete a semantic-layer model and cascade-delete its children | `kbagent sl model delete --project PROJECT --model MODEL` | @@ -297,6 +329,10 @@ When working inside a git repository or project directory, run `kbagent init` (o | Remove a constraint | `kbagent sl remove constraint --project PROJECT --name NAME` | | Remove a relationship. | `kbagent sl remove relationship --project PROJECT --name NAME` | | Remove a glossary term. | `kbagent sl remove glossary --project PROJECT --term TERM` | +| List reference-data records (dimension summaries; use ``get`` for members) | `kbagent sl reference-data list --project PROJECT` | +| Fetch one record (all members) by ``--id`` or by ``--model`` + ``--dimension`` | `kbagent sl reference-data get --project PROJECT` | +| Create or replace (by model + dimension) a reference-data record | `kbagent sl reference-data set --project PROJECT --dimension DIMENSION --members-file MEMBERS-FILE` | +| Delete a reference-data record by UUID (server-side soft-delete) | `kbagent sl reference-data delete --project PROJECT --id ID-` | | GET an endpoint on the running kbagent serve | `kbagent http get ` | | POST to an endpoint on the running kbagent serve | `kbagent http post ` | | PATCH an endpoint on the running kbagent serve | `kbagent http patch ` | @@ -313,6 +349,20 @@ When working inside a git repository or project directory, run `kbagent init` (o | Execute an action ad-hoc (no persistence, no scheduling) | `kbagent agent test` | | Show the next N firings of a cron expression | `kbagent agent cron-preview --cron CRON` | | Polish a plain-English goal into an unattended-agent-ready prompt | `kbagent agent prompt-improve --goal GOAL` | +| List Developer Portal apps for a vendor | `kbagent dev-portal list --vendor VENDOR` | +| Show the full Developer Portal entry for one app | `kbagent dev-portal get --app APP` | +| Create (register) a new app in the Developer Portal. | `kbagent dev-portal create --vendor VENDOR --data DATA` | +| Patch one or more properties of an existing Developer Portal app. | `kbagent dev-portal patch --app APP` | +| Upload a 128x128 PNG icon for a Developer Portal app. | `kbagent dev-portal upload-icon --app APP --file FILE` | +| Publish an app in the Developer Portal (requests Keboola review). | `kbagent dev-portal publish --app APP` | +| Deprecate an app in the Developer Portal (hides it, blocks new configs). | `kbagent dev-portal deprecate --app APP` | +| Add a Developer Portal identity (verifies creds before persisting) | `kbagent dev-portal identity add --alias ALIAS --username USERNAME` | +| List configured Developer Portal identities | `kbagent dev-portal identity list` | +| Remove a Developer Portal identity | `kbagent dev-portal identity remove --alias ALIAS` | +| Edit fields on a Developer Portal identity (or rename it) | `kbagent dev-portal identity edit --alias ALIAS` | +| Set the default Developer Portal identity | `kbagent dev-portal identity use ` | +| Show the alias of the default Developer Portal identity | `kbagent dev-portal identity current` | +| Probe a Developer Portal identity by logging in | `kbagent dev-portal identity verify` | ### Sync pull notable flags @@ -357,6 +407,7 @@ For detailed response parsing rules and common pitfalls, see [gotchas](reference | **Agent Tasks via REST** (`kbagent http /agents...` from inside scheduled subprocesses; SSE streaming) | [agent-tasks-rest-workflow](references/agent-tasks-rest-workflow.md) | | **Data apps** (create / deploy / start / stop / password / delete; the §9 redeploy contract) | [data-app-workflow](references/data-app-workflow.md) | | Storage Files (upload, download, tags, load/unload) | [storage-files-workflow](references/storage-files-workflow.md) | +| **Data Streams (OTLP / OpenTelemetry)** (create/inspect OTLP source, masked secret-in-URL, OTEL_EXPORTER_OTLP_ENDPOINT) | [stream-workflow](references/stream-workflow.md) | | **Storage column types** (native types, NOT NULL, DEFAULT, branch materialize) | [storage-types-workflow](references/storage-types-workflow.md) | | **Typify a typeless table** (profile -> CTAS -> swap-tables -> validate -> handoff) | [typify-table-workflow](references/typify-table-workflow.md) | | Bucket sharing & linking | [sharing-workflow](references/sharing-workflow.md) | @@ -369,6 +420,7 @@ For detailed response parsing rules and common pitfalls, see [gotchas](reference | Reading synced data | [reading-synced-data](references/reading-synced-data.md) | | SQL migration (input mapping removal) | [sql-migration-workflow](references/sql-migration-workflow.md) | | **Semantic layer (metastore)** -- models, metrics, datasets, constraints, glossary; validate / export / diff / promote / build / token | [semantic-layer-workflow](references/semantic-layer-workflow.md) | +| **Developer Portal** (identity CRUD, list/get apps, create/patch/upload-icon/publish/deprecate; TTY-confirm on writes) | [dev-portal-workflow](references/dev-portal-workflow.md) | | Response parsing gotchas | [gotchas](references/gotchas.md) | ## First-time setup diff --git a/plugins/kbagent/skills/kbagent/references/commands-reference.md b/plugins/kbagent/skills/kbagent/references/commands-reference.md index 8d0b7005..2072c7f3 100644 --- a/plugins/kbagent/skills/kbagent/references/commands-reference.md +++ b/plugins/kbagent/skills/kbagent/references/commands-reference.md @@ -43,6 +43,16 @@ All seven commands authenticate via `KBC_MANAGE_API_TOKEN` (Manage API), not the - `org setup --org-id ID --url URL [--dry-run] [--yes]` -- bulk-onboard all projects from an org (org admin; manage token via interactive prompt by default, or `--allow-env-manage-token` + `KBC_MANAGE_API_TOKEN` for CI on 0.29.0+) - `org setup --project-ids 1,2,3 --url URL [--dry-run] [--yes]` -- onboard specific projects by ID (any project member; manage token / Personal Access Token via interactive prompt by default, or `--allow-env-manage-token` + `KBC_MANAGE_API_TOKEN` for CI on 0.29.0+) +## Feature Flags (since v0.48.0) +Requires a **super-admin** Manage API token (same kind as `org setup`). Same default-deny token policy: interactive hidden prompt by default, or `--allow-env-manage-token` + `KBC_MANAGE_API_TOKEN` for CI. `--project ALIAS` resolves the stack URL (and, for project ops, the numeric `project_id`) from config -- the alias is the only handle you pass. +- `feature list --project ALIAS` -- the stack-wide feature catalogue (`GET /manage/features`). Returns `{alias, stack_url, features: [{name, title, description, type, ...}]}`. Only `name` is a stable identifier; extra fields pass through unmodified. +- `feature project-show --project ALIAS` -- features assigned to a project, read from the project object's `features` array. Returns `{alias, project_id, project_name, features: [...]}`. +- `feature project-add --project ALIAS --feature NAME [--dry-run] [--yes]` -- enable a feature on a project (`POST /manage/projects/{id}/features`, body `{"feature": NAME}`). Permission class `admin`. +- `feature project-remove --project ALIAS --feature NAME [--dry-run] [--yes]` -- disable a feature on a project (`DELETE /manage/projects/{id}/features/{name}`). Permission class `destructive`. +- `feature user-show --project ALIAS --email EMAIL` -- features assigned to a user (`GET /manage/users/{email}`). Returns `{alias, stack_url, email, features: [...]}`. +- `feature user-add --project ALIAS --email EMAIL --feature NAME [--dry-run] [--yes]` -- enable a feature on a user (`POST /manage/users/{email}/features`). +- `feature user-remove --project ALIAS --email EMAIL --feature NAME [--dry-run] [--yes]` -- disable a feature on a user (`DELETE /manage/users/{email}/features/{name}`). + ## Component Discovery - `component list [--project NAME] [--type TYPE] [--query "text"]` -- list/search components (AI-powered with `--query`) - `component detail --component-id ID [--project NAME]` -- show component schema, docs URL, examples @@ -84,14 +94,15 @@ All seven commands authenticate via `KBC_MANAGE_API_TOKEN` (Manage API), not the - `storage tables [--project NAME ...] [--bucket-id ID] [--branch ID]` -- list tables across all connected projects in parallel (multi-project by default, same as `storage buckets`); repeat `--project` to target a subset; `--bucket-id` is applied independently per project (missing buckets become per-project errors); `--branch` requires exactly one `--project` - `storage table-detail --project NAME --table-id ID [--branch ID]` -- table detail with columns, types, primary key, row count (branch-aware) - `storage create-bucket --project NAME --stage STAGE --name NAME [--description D] [--backend B] [--branch ID]` -- create bucket (branch-aware). With `--branch ID` on a project lacking the `storage-branches` feature (legacy fake-branch), response carries `legacy_branch_storage: true` and human mode prints a warning -- the runner will create a parallel `out.c--*` bucket at job time. See `storage-types-workflow.md` -- `storage create-table --project NAME --bucket-id ID --name NAME --column col:TYPE[(length)] [...] [--primary-key COL] [--not-null COL ...] [--default NAME=VALUE ...] [--branch ID]` -- create typed table. Base types `STRING/INTEGER/NUMERIC/FLOAT/BOOLEAN/DATE/TIMESTAMP` plus native backend types with length (`VARCHAR(40)`, `NUMBER(18,2)`, `TIMESTAMP_TZ`, `VARIANT`, etc.) -- type/length validation delegated to the Storage API. `--not-null` marks a column `nullable=false`; `--default NAME=VALUE` sets a DEFAULT expression (booleans must be lowercase `true`/`false`). In a dev branch, the target bucket is auto-materialized if it has not yet been written to there -- response surfaces this via `auto_created_bucket: bool`. On legacy fake-branch projects (no `storage-branches` feature), `legacy_branch_storage: true` flags that the runner will use a separate `out.c--*` bucket at job time. See `storage-types-workflow.md` +- `storage create-table --project NAME --bucket-id ID --name NAME --column col:TYPE[(length)] [...] [--primary-key COL] [--not-null COL ...] [--default NAME=VALUE ...] [--branch ID] [--if-not-exists]` -- create typed table. Base types `STRING/INTEGER/NUMERIC/FLOAT/BOOLEAN/DATE/TIMESTAMP` plus native backend types with length (`VARCHAR(40)`, `NUMBER(18,2)`, `TIMESTAMP_TZ`, `VARIANT`, etc.) -- type/length validation delegated to the Storage API. `--not-null` marks a column `nullable=false`; `--default NAME=VALUE` sets a DEFAULT expression (booleans must be lowercase `true`/`false`). In a dev branch, the target bucket is auto-materialized if it has not yet been written to there -- response surfaces this via `auto_created_bucket: bool`. On legacy fake-branch projects (no `storage-branches` feature), `legacy_branch_storage: true` flags that the runner will use a separate `out.c--*` bucket at job time. `--if-not-exists` (0.47.0+) turns a duplicate-display-name failure into `action: skipped` when the table really exists at the expected id (safe for parallel workers). Since 0.47.1 the skipped envelope reports the EXISTING table's actual `columns`/`primary_key`/`name`, mirrors the request under `requested_columns`/`requested_primary_key`, and sets `schema_drift: true` when they diverge. See `storage-types-workflow.md` - `storage upload-table --project NAME --table-id ID --file PATH [--incremental] [--branch ID]` -- upload CSV (branch-aware) - `storage download-table --project NAME --table-id ID [--output FILE] [--columns COL ...] [--limit N] [--branch ID]` -- export table to CSV (branch-aware) - `storage delete-table --project NAME --table-id ID [--table-id ...] [--force] [--dry-run] [--yes] [--branch ID]` -- delete tables, --force cascade-deletes aliased tables (branch-aware) - `storage truncate-table --project NAME --table-id ID [--table-id ...] [--dry-run] [--yes] [--branch ID]` (since v0.32.0) -- delete all rows while preserving table schema, primary key, descriptions, sharing edges, and downstream dependents. Batch via repeated `--table-id`. Endpoint is uniformly async-via-job on every branch (returns a queued `tableRowsDelete` job; client polls via `_wait_for_storage_job` before returning). Idempotent (truncating an empty table is a no-op). Use when re-seeding a table without losing the schema contract - `storage delete-column --project NAME --table-id ID --column COL [--column ...] [--force] [--dry-run] [--yes] [--branch ID]` -- delete columns from a table (branch-aware) - `storage delete-bucket --project NAME --bucket-id ID [--bucket-id ...] [--force] [--dry-run] [--yes] [--branch ID]` -- delete buckets (branch-aware) -- `storage swap-tables --project NAME --table-id ID --target-table-id ID --branch ID [--dry-run] [--yes]` (since v0.28.0) -- swap two storage tables in a dev branch (POST `/tables/{id}/swap`). Both tables exchange physical positions; aliases are NOT transferred (they keep pointing at the same physical position and therefore expose the OTHER table's data after the swap). Service refuses without a branch (active branch via `branch use` works too). Use to flip a typed rebuild ("data_change_log") into the original name ("data") without touching downstream config references +- `storage swap-tables --project NAME --table-id ID --target-table-id ID --branch ID [--dry-run] [--yes]` (since v0.28.0) -- swap two storage tables in any branch, including the default/production branch (POST `/tables/{id}/swap`). Both tables exchange physical positions; aliases are NOT transferred (they keep pointing at the same physical position and therefore expose the OTHER table's data after the swap). Service refuses without a branch (active branch via `branch use` works too). Use to flip a typed rebuild ("data_change_log") into the original name ("data") without touching downstream config references +- `storage clone-table --project NAME --table-id ID --branch ID [--dry-run]` (since v0.52.0) -- pull (clone) a production table into a dev branch (POST `/tables/{id}/pull`, operationName `devBranchTablePull`). On `storage-branches` projects a dev branch reads prod tables transparently until the first write, so an in-branch schema mutation (`swap-tables`, dropping a column) fails with a misleading "bucket not found" until the table is materialized branch-local; `clone-table` does that. One-way (default -> branch). Service refuses without a branch (active branch via `branch use` works too). Permission class `write` - `storage describe-bucket --project NAME --bucket-id ID [--text STR | --file PATH | --stdin] [--branch ID]` -- set a bucket description (stored as `KBC.description` in bucket metadata, upsert). Provide exactly one of `--text`, `--file`, `--stdin`. Read back via `storage bucket-detail` - `storage describe-table --project NAME --table-id ID [--text STR | --file PATH | --stdin] [--branch ID]` -- set a table description (stored as `KBC.description` in table metadata, upsert). Provide exactly one of `--text`, `--file`, `--stdin`. Read back via `storage table-detail` - `storage describe-column --project NAME --table-id ID --column NAME=DESCRIPTION [--column ...] [--branch ID]` -- set one or more column descriptions. Stored as `KBC.column.{name}.description` keys in the table's metadata (Keboola has no user-writable column-metadata endpoint). Read back in `storage table-detail` under `column_details[].description` @@ -107,6 +118,13 @@ All seven commands authenticate via `KBC_MANAGE_API_TOKEN` (Manage API), not the - `storage load-file --project NAME --file-id ID --table-id ID [--incremental] [--delimiter D] [--enclosure E] [--branch ID]` -- import a Storage File into a table (CSV) - `storage unload-table --project NAME --table-id ID [--columns COL ...] [--limit N] [--tag TAG ...] [--download] [--output FILE|DIR] [--file-type csv|parquet] [--branch ID]` -- export a table to a Storage File. `--file-type parquet` produces sliced Parquet; `--download` saves each slice as its own file under `./{project}/{table_id}.parquet/` (default) together with `_manifest.json` +## Data Streams (OTLP) (since v0.50.0) +Uses the per-project Storage token (no manage token). Control plane = `stream.` (derived from `connection.`). The OTLP ingest endpoint (`stream-in./otlp///`) is returned in `source.otlp.url` with the secret in the path -- **masked by default, `--reveal` to print it**. `create-source --type otlp` auto-provisions the logs/metrics/traces sinks (bucket `in.c-otlp-`) so data lands; `--no-sinks` opts out. See `stream-workflow.md`. +- `stream list --project NAME [--branch ID]` -- list sources (id, name, type, secret-free base endpoint) +- `stream create-source --project NAME --name NAME [--type otlp|http] [--branch ID] [--if-not-exists] [--no-sinks] [--reveal]` -- create a source; for OTLP auto-creates 3 sinks (idempotent); polls the async task and returns the endpoint. `--if-not-exists` returns an existing same-named source as `status=skipped` +- `stream detail [SOURCE_ID | --name NAME] --project NAME [--branch ID] [--reveal]` -- base + per-signal endpoints (`/v1/logs|/v1/traces|/v1/metrics`), protocol `http/protobuf`, destination bucket/tables (from sinks). Secret masked unless `--reveal` +- `stream delete SOURCE_ID --project NAME [--branch ID] [--dry-run] [--yes|--force]` -- delete a source (destructive; async task polled to completion) + ## Data Lineage - `lineage build -d DIR -o FILE [--refresh] [--ai]` -- build column-level lineage graph from sync'd data - `lineage show -l FILE --downstream "project:table" [--columns] [-c COL] [--format text|mermaid|html|er]` -- query downstream dependencies from cache @@ -128,7 +146,7 @@ All seven commands authenticate via `KBC_MANAGE_API_TOKEN` (Manage API), not the - `branch metadata-delete --project NAME --metadata-id ID [--branch ID|default]` -- delete a metadata entry by its numeric ID (from `metadata-list`) ## Workspaces (SQL Debugging) -- `workspace create --project ALIAS [--name NAME] [--ui] [--read-only]` -- create workspace (headless ~1s, `--ui` ~15s) +- `workspace create --project ALIAS [--name NAME] [--ui] [--read-only]` -- create workspace (headless ~1s, `--ui` ~15s). Since v0.47.1: Snowflake headless workspaces return a `private_key` PEM field; `password` is empty. BigQuery workspaces keep the default password credential shape. - `workspace list [--project NAME ...] [--orphaned] [--branch ID] [--qs-compatible]` -- list workspaces. `--project` repeatable; `--orphaned` filters to workspaces whose backing `keboola.sandboxes` config is missing. **Since v0.42.0 (#304)**: each entry carries `login_type`, `read_only`, `qs_compatible`, `database`, `warehouse`. New `Login Type` / `RO` / `QS` columns in human mode. `--qs-compatible` pre-filters to RO + whitelisted-loginType workspaces (the canonical data-app shape). `--branch` requires exactly one `--project`; without `--branch`, the command behaves like `storage buckets` and uses production with an `Info: Using production branch for read (active dev branch X ignored; pass --branch X to override)` banner when an alias is pinned to a dev branch - `workspace detail --project ALIAS --workspace-id ID [--branch ID]` -- show connection details. **Since v0.42.0 (#304)**: response carries `login_type`, `read_only`, `qs_compatible`; human mode adds `Login type:` / `Read-only:` / `Query Service compatible:` rows. `--branch` opt-in mirrors `workspace list` - `workspace delete --project ALIAS --workspace-id ID` -- delete workspace @@ -186,9 +204,9 @@ Requires the project to be added with its **master ('owner') Storage API token** ## Sync (GitOps) - `sync init --project ALIAS [--directory DIR] [--git-branching] [--adopt-existing]` -- initialize sync working directory; `--adopt-existing` (since v0.22.0) adopts a `.keboola/manifest.json` already written by the kbc Go CLI without overwriting (idempotent; validates `project_id` against the alias token) -- `sync pull --project ALIAS [--all-projects] [--force] [--dry-run] [--with-samples] [--no-storage] [--no-jobs] [--job-limit N]` -- download configs to local files. For large projects (>100 configs), automatically fetches jobs per-config when the grouped API limit is insufficient -- `sync push --project ALIAS [--all-projects] [--dry-run] [--force] [--allow-plaintext-on-encrypt-failure]` -- push local changes (auto-encrypts secrets, fails if encryption fails) -- `sync diff --project ALIAS [--all-projects]` -- 3-way diff (local vs base vs remote), detects conflicts +- `sync pull --project ALIAS [--all-projects] [--force] [--dry-run] [--with-samples] [--no-storage] [--no-jobs] [--job-limit N] [--branch ID]` -- download configs to local files. For large projects (>100 configs), automatically fetches jobs per-config when the grouped API limit is insufficient. `--force` is conflict-aware (since 0.53.0): a locally-modified config whose remote is unchanged is **preserved** (pending delta stays pushable, never silently re-stamped); a true merge conflict (local AND remote both changed since last pull) **aborts** the pull (exit 1, `SYNC_CONFLICT`; `--json` lists `details.conflicts`); local-untouched + remote-changed takes remote. To discard local edits on purpose, delete the file/dir and pull. `--branch` (0.47.0+) per-invocation dev-branch override, beats every other branch source. +- `sync push --project ALIAS [--all-projects] [--dry-run] [--force] [--allow-plaintext-on-encrypt-failure] [--branch ID] [--no-name-drift-warnings]` -- push local changes (auto-encrypts secrets, fails if encryption fails). Fresh-CREATE writeback updates placeholder manifest entries in place (since 0.47.0) and propagates any `KBC.configuration.*` metadata via `set_config_metadata`. Fresh-CREATE variable binding (since 0.47.2): when a `keboola.variables` config + its values row are created alongside a transformation in the same push, the transformation's `variables_id` / `variables_values_id` placeholders are rebound to the assigned ULIDs and the row's `values` are hoisted even without a `_keboola` block, so `job run` succeeds with no post-push `config variables-set` step (unresolvable/ambiguous links surface a `variable_link` entry in `errors[]`, never a broken link). `--branch` (0.47.0+) per-invocation override; when no `/` subtree exists on disk (since 0.47.2) the local default tree (`main/`) is promoted to the target branch (API writes still target the branch id); `--no-name-drift-warnings` (0.47.0+) drops the cosmetic warnings array. +- `sync diff --project ALIAS [--all-projects] [--branch ID]` -- 3-way diff (local vs base vs remote), detects conflicts. `--branch` (0.47.0+) per-invocation dev-branch override. - `sync status [--directory DIR]` -- show locally modified/added/deleted configs - `sync branch-link --project ALIAS [--branch-id ID] [--branch-name NAME]` -- link git branch to Keboola dev branch - `sync branch-unlink [--directory DIR]` -- remove git-to-Keboola branch mapping @@ -197,6 +215,36 @@ Requires the project to be added with its **master ('owner') Storage API token** ## Encryption - `encrypt values --project ALIAS --component-id ID --input JSON|@file|- [--output-file PATH]` -- encrypt #-prefixed secrets via Keboola Encryption API (one-way, no decrypt). Scope: ComponentSecure (project + component). Use for MCP tool call workflows. +## Developer Portal (since v0.49.0; admin routing in v0.51.1) + +Talks to `apps-api.keboola.com`. **Reads are unrestricted; writes always require a human to type a random hex code on a real TTY (no `--yes`, no env bypass, exit 6 on non-TTY).** Use `--dry-run` for the agent-safe preview path. + +`--role-hint` is **load-bearing** for `dev-portal patch` (since v0.51.1): `vendor` (default) → `PATCH /vendors/{vendor}/apps/{app}` (restricted schema, the common case); `admin` → `PATCH /admin/apps/{app}` (permissive schema, the only way to set `complexity`, `categories`, `category`, `features`, `forwardToken`, `forwardTokenDetails`, `injectEnvironment`, `processTimeout`, `requiredMemory`). A `vendor` identity with any of those 9 fields in the payload fails fast at preflight with the exact command to switch. + +`--password-stdin` (since v0.51.1) works in both TTY mode (hidden line-based prompt, Enter to confirm) and pipe mode (`echo $PASS | … --password-stdin`, reads to EOF). + +MFA login (since v0.51.1) sends `challenge: SOFTWARE_TOKEN_MFA` explicitly to fix a 404 on personal-account TOTP logins where the apps-api server silently rejects missing-challenge requests despite the spec calling it optional. Single attempt only; failure surfaces the actual server body with a stale-TOTP hint. + +### Identity management +- `dev-portal identity add --alias A --username U [--password P | --password-stdin] [--role-hint vendor|admin] [--vendor V] [--portal-url URL]` -- store a portal login credential per-alias in `config.json` (0600 perms). `--role-hint` is validated (`vendor`/`admin`, case-folded) since v0.51.1. +- `dev-portal identity list` -- list stored portal identities (no passwords shown). +- `dev-portal identity remove --alias A` -- delete an identity alias. +- `dev-portal identity edit --alias A [--username U] [--password P|--password-stdin] [--role-hint H] [--vendor V] [--new-alias N]` -- update fields of an identity. +- `dev-portal identity use ALIAS` -- set the default identity for subsequent commands. +- `dev-portal identity current` -- show the active default identity alias. +- `dev-portal identity verify [--identity A]` -- test credentials against the portal (login + logout). + +### Read commands (unrestricted; agent-friendly) +- `dev-portal list --vendor V [--identity A]` -- list all apps registered under a vendor. Useful for peer-config research. +- `dev-portal get --app VENDOR.APP_ID [--identity A]` -- fetch the full portal entry for one component (uiOptions, encryption, defaultBucket, configurationSchema, icon, etc.). + +### Write commands (require TTY random-code confirm; use `--dry-run` first) +- `dev-portal create --vendor V --data FILE [--identity A] [--dry-run]` -- register a new component from a JSON payload file. +- `dev-portal patch --app VENDOR.APP_ID (--data FILE | --property KEY (--value V | --value-file F)) [--identity A] [--dry-run]` -- update portal properties. Endpoint depends on the identity's `role_hint`: vendor → vendor endpoint, admin → admin endpoint. +- `dev-portal upload-icon --app VENDOR.APP_ID --file PATH [--identity A] [--dry-run]` -- upload a PNG/SVG icon. +- `dev-portal publish --app VENDOR.APP_ID [--identity A] [--dry-run]` -- publish the component (makes it visible in the UI). +- `dev-portal deprecate --app VENDOR.APP_ID [--identity A] [--dry-run]` -- mark the component as deprecated. + ## Semantic Layer (Metastore) (since v0.41.0) -Reads `KBAGENT_SERVE_URL` + `KBAGENT_SERVE_TOKEN` env vars. The scheduler auto-injects these (plus `KBAGENT_CONFIG_DIR`) into every AI-agent / `cli_command` subprocess. Outside a serve subprocess context the command refuses to run with exit code 2. **Inside a scheduled-agent task, prefer `kbagent http get /openapi.json` then a typed call over forking another `kbagent` CLI -- the HTTP path always sees the operator's live config (not the global `~/.config/keboola-agent-cli/` one).** @@ -206,6 +254,8 @@ Manage Keboola metastore models -- datasets, metrics, relationships, constraints - `semantic-layer model create --project P --name N [--description D] [--sql-dialect Snowflake]` -- create a new model. `--sql-dialect` defaults to `Snowflake`. Returns the new model UUID; subsequent commands accept either name or UUID via `--model`. - `semantic-layer model delete --project P --model M [--yes]` -- delete a model **and cascade-delete every child entity** (datasets, metrics, relationships, constraints, glossary terms) in `reversed(PUSH_ORDER)` (constraints first, datasets last) before the parent. Confirmation prompt unless `--yes`. **Cascade is unconditional in 0.43.4+** -- before that release the call only DELETEd the parent, silently leaking children pointing at the dead `modelUUID` and breaking subsequent `build` / `import` retries with HTTP 422 name collisions (closes #306). On any child-DELETE failure the parent is **preserved** and the response carries `details.cascade = {attempted, deleted, failures: [{type, id, name, error}], parent_deleted: False, model_uuid}` so the user can re-run after fixing the underlying error. Happy-path envelope adds `cascade.deleted` per-type counts. Legacy `orphaned_children` top-level key kept for back-compat (same shape, meaning flipped from "leaked" to "cascaded") but **deprecated -- removal scheduled for a future minor release**; new callers should read `cascade.deleted` instead. See [gotchas.md](gotchas.md) for the meaning-flip + deprecation note. - `semantic-layer show --project P [--model M] [--type T]` -- show a model's entities. `--type` filters to `dataset | metric | relationship | constraint | glossary`. Without `--type` prints a per-type count summary. `--model` is optional when the project has exactly one model. +- `semantic-layer search-context --project P [--pattern G ...] [--type model|dataset|metric|relationship|constraint|glossary|all] [--limit N]` (since 0.47.0) -- project-wide glob search across semantic-layer entity names. Mirrors the upstream `keboola-mcp-server search_semantic_context` MCP tool so a downstream caller can drop the MCP dependency for the pre-flight "is the model populated?" check. Patterns are case-sensitive `fnmatch`, repeatable (union); default `*`. Default `--type all` searches every CHILD type (`model` searches semantic models). `--limit N` short-circuits both per-type and outer loops. Envelope: `{project, contexts: [{id, type, name, description, attributes}], total_count}`; the `type` field is the CLI-friendly singular (no `semantic-` prefix). +- `semantic-layer get-context --project P --context-id ID` (since 0.47.0) -- single-entry fetch by id, irrespective of type. Probes `semantic-model` first then every CHILD type (dataset / metric / relationship / constraint / glossary) until a 200 lands. 404 on any one type is non-terminal; only a full miss raises `NOT_FOUND` (exit 1). Non-404 errors (500, etc.) propagate immediately rather than being swallowed by the next probe. - `semantic-layer validate --project P [--model M] [--deep]` -- structural validation. Basic mode runs local checks: duplicate names, dangling rel/metric refs, SUM-on-PCT (warning), constraint orphans (metrics in `metrics[]` that no longer exist), severity-suffix mismatches between API `severity` and the 4-band name suffix. `--deep` adds parallel Snowflake column-existence probes via the in-process StorageService: phantom dataset fields, phantom column refs in metric SQL, AGG-on-STRING errors. Response: `{valid: bool, deep: bool, errors: [{type, item, detail}], warnings: [...]}`. - `semantic-layer export --project P [--model M] [--output PATH]` -- snapshot the model to a self-describing JSON file (default `./sl_export_{model_name}_{YYYYMMDD_HHMMSS}.json`). Schema-versioned for round-trip via `import` / `diff`. - `semantic-layer diff (--project-a A | --file-a P) (--project-b B | --file-b P) [--model-a M] [--model-b M]` -- three-way diff: project<->project, project<->file, file<->file. Mutually exclusive per side: pass exactly one of `--project-a` / `--file-a`, ditto for B. Output groups changes per entity type: `added[] / removed[] / changed[{name, diff_keys[]}]`. @@ -274,8 +324,9 @@ CLI parity for the `/agents` REST surface. Reads/writes `/agents.jso ## Environment Variables | Variable | Purpose | |----------|---------| -| `KBC_TOKEN` | Fallback for `--token` | -| `KBC_STORAGE_API_URL` | Default stack URL | +| `KBC_TOKEN` | Fallback for `--token`. Also the credential source for headless `__env__` mode (see `KBAGENT_PROJECT_FROM_ENV`) | +| `KBC_STORAGE_API_URL` | Default stack URL. Also the stack source for headless `__env__` mode | +| `KBAGENT_PROJECT_FROM_ENV` | Set to `1`/`true`/`yes`/`on` to synthesize an in-memory project `__env__` from `KBC_TOKEN` + `KBC_STORAGE_API_URL` (since 0.50.0). Headless / token-only: no `project add`, no `config.json` on disk; token stays in memory (never persisted). Use `--project __env__`. Works for CLI and `kbagent serve`. Fails fast if creds missing | | `KBC_MANAGE_API_TOKEN` | Manage API token (org setup, project refresh, data-app password). Default-DENY since 0.28.0: requires top-level `--allow-env-manage-token` to opt in, otherwise ignored with a warning. | | `KBAGENT_CONFIG_DIR` | Override config directory | | `KBAGENT_SERVE_URL` | Self-URL of `kbagent serve` (used by `kbagent http`; auto-injected into scheduled-agent subprocesses) | diff --git a/plugins/kbagent/skills/kbagent/references/dev-portal-workflow.md b/plugins/kbagent/skills/kbagent/references/dev-portal-workflow.md new file mode 100644 index 00000000..79fcb34c --- /dev/null +++ b/plugins/kbagent/skills/kbagent/references/dev-portal-workflow.md @@ -0,0 +1,103 @@ +# Developer Portal workflow + +> Audience: a Keboola component developer or a kbagent agent acting on their +> behalf. Goal: safely register, inspect, and update components in the +> Keboola Developer Portal (`apps-api.keboola.com`). + +## Identity model + +Developer Portal logins are email + password (with MFA on personal +accounts). kbagent stores identities per-alias in the same `config.json` +as KB project tokens, under 0600 protection: + +``` +kbagent dev-portal identity add --alias vendor-keboola --username service.keboola.xxxxx --password ... --vendor keboola +kbagent dev-portal identity add --alias vendor-kds --username service.kds-team.xxxxx --password ... --vendor kds-team +kbagent dev-portal identity add --alias admin-keboola --username admin@keboola.com --role-hint admin --password-stdin +kbagent dev-portal identity use vendor-keboola # default for subsequent commands +``` + +Service accounts (`service.{vendor}.{id}`) skip MFA. Personal admin +accounts prompt for the MFA code on `/dev/tty` at login time +(`SOFTWARE_TOKEN_MFA`, i.e. a TOTP authenticator app like 1Password / Authy / +Google Authenticator). + +`--password-stdin` works in both pipe mode (`echo $PASS | … --password-stdin`, +reads to EOF) and TTY mode (hidden line-based prompt, Enter to confirm). + +### `role_hint` is load-bearing (since v0.51.1) + +`--role-hint` is **not** a free-text label. It picks which apps-api +endpoint kbagent uses for `dev-portal patch`: + +| Role | PATCH endpoint | Schema | Use for | +|------|----------------|--------|---------| +| `vendor` (default) | `/vendors/{vendor}/apps/{app}` | `clientAppSchema` (restricted) | Cookiecutter-backed properties, schemas, UI options, descriptions, icon | +| `admin` | `/admin/apps/{app}` | `adminAppSchema` (permissive) | The 9 fields forbidden on vendor: `complexity`, `categories`, `category`, `features`, `forwardToken`, `forwardTokenDetails`, `injectEnvironment`, `processTimeout`, `requiredMemory` | + +`role_hint` is validated (`vendor` or `admin`, case-folded). kbagent does +not verify the server-side role of the credential -- if you set `admin` +but the account isn't actually a portal admin, the PATCH fails at the +apps-api with an unambiguous 403. + +When a vendor-role identity tries to patch one of the 9 admin-only +fields, the service **fail-fasts** with a message that names the +offending fields, explains why the server's 422 ("must be one of: ...") +is misleading, and shows the exact command to add and use an admin +identity. No portal call is made. + +## Safety contract (read this before issuing any write) + +- Reads are free: `dev-portal list`, `dev-portal get`. +- Writes (`create`, `patch`, `upload-icon`, `publish`, `deprecate`) always: + 1. Print the exact pending request to stderr (full diff for `patch`). + 2. Require the user to type a random hex code into the TTY. + 3. Exit 6 on a non-TTY shell. +- There is no `--yes`. There is no env-var bypass. By design. +- `--dry-run` prints the same preview and exits 0 without prompting. This + is the agent-safe path. +- Caveat: `patch --dry-run` and `publish --dry-run` still **log in and GET + the current app** (to compute the diff / run the publish pre-flight), so + they need portal connectivity. On a personal (MFA) identity they will + prompt for an MFA code on `/dev/tty` and fail with `DP_MFA_REQUIRED` in a + non-interactive shell. For a fully non-interactive preview, use a + `service.{vendor}.{id}` identity (no MFA). `create`/`upload-icon`/ + `deprecate` dry-runs are purely local (no portal call). + +## The loop + +1. Identify the component (vendor + app id). For an existing repo, check + `.github/workflows/*.yml` for `KBC_DEVELOPERPORTAL_VENDOR` and `KBC_DEVELOPERPORTAL_APP`. +2. `kbagent --json dev-portal list --vendor ` and/or + `kbagent --json dev-portal get --app VENDOR.APP_ID` to inspect. +3. Build a payload file (a JSON file — never inline JSON, shell quoting + is unsafe with portal property names that contain spaces). +4. `kbagent dev-portal patch --app VENDOR.APP_ID --data /tmp/p.json --dry-run` + — print the diff, show it to the user. +5. The user runs the same command without `--dry-run` and types the code. + +## Peer-config research + +Designing a new component? Pull reference configurations from existing +peers: + +``` +# List candidates +kbagent --json dev-portal list --vendor keboola | jq '.[] | select(.type=="extractor") | .id' + +# Pull two peers in full +kbagent --json dev-portal get --app keboola.ex-db-mysql > /tmp/peer-mysql.json +kbagent --json dev-portal get --app keboola.ex-db-pgsql > /tmp/peer-postgres.json +``` + +Compare them yourself — the agent has the reasoning ability to spot +patterns. No dedicated `peers` command needed. + +## Boundaries (what this surface does NOT own) + +- Image push to ECR — stays in component GitHub Actions. +- Bulk repo-file -> property sync on deploy — stays in + `scripts/developer_portal/update_properties.sh` (Cookiecutter-backed files). +- Writes to `component_config/` — never. That directory is governed by the + Cookiecutter template; portal-direct properties (`uiOptions`, + `encryption`, `defaultBucket`, …) live only in the portal. diff --git a/plugins/kbagent/skills/kbagent/references/gotchas.md b/plugins/kbagent/skills/kbagent/references/gotchas.md index 0f8255e1..46ecc01f 100644 --- a/plugins/kbagent/skills/kbagent/references/gotchas.md +++ b/plugins/kbagent/skills/kbagent/references/gotchas.md @@ -11,6 +11,139 @@ Versioning convention: behavior; the inline `(updated vX.Y.Z)` records when the refinement landed. --> +## `sync push` fresh-CREATE writeback now updates placeholders in place (since v0.47.0) + +Before v0.47.0, `kbagent sync push` always **appended** new `ManifestConfiguration` +(and `ManifestConfigRow`) entries to `.keboola/manifest.json` on every CREATE. +The FIIA / scaffold emit pattern — pre-populating manifest entries with +placeholder ids before the first push — therefore produced manifests with +N placeholders + N real entries (= 2N) after one push, and every placeholder +still looked `added` on re-push (spurious duplicates on remote). + +Starting in v0.47.0, the create path looks up an existing entry by +`(component_id, path)` and **updates it in place** (id, branch_id, pull_hash, +pull_config_hash refreshed; user-declared `KBC.configuration.*` metadata +preserved). When no placeholder is found, the legacy append path still fires +(so commands like `sync init` followed by direct push of newly-pulled remote +configs are unaffected). + +Two follow-on contract notes: + +- **Re-push idempotency comes for free.** After the first push the manifest + holds the real ULID, so the diff engine matches against remote_configs and + reports `status: no_changes, created: 0`. No more "every fresh-create + emit doubles the manifest" workaround needed. +- **`KBC.configuration.*` metadata propagates on CREATE.** If a placeholder + entry's `metadata` dict contains keys starting with `KBC.` (e.g. + `KBC.configuration.folderName`), they are POSTed to the metadata API + via `client.set_config_metadata` immediately after the create call. + Bookkeeping keys (`pull_hash`, `pull_config_hash`, ...) are filtered + out by the `KBC.` prefix check. `_push_update` does **not** propagate + metadata — use `kbagent config set-metadata` (or `config set-folder`) + for that. + +If a downstream consumer has been working around the duplication by +post-processing the manifest, drop that workaround. The single-entry +manifest is the new contract. + +## `sync push` fresh-CREATE now resolves variable links, hoists row `values`, and `--branch` promotes the default tree (since v0.47.2) + +A transformation scaffolded alongside its sibling `keboola.variables` config + +default-values row is now **runnable after a single `sync push`** — no post-push +`config variables-set` workaround needed. Three things changed in the create pass: + +- **Row `values` are no longer dropped.** A `keboola.variables` values row whose + scaffold `_config.yml` has top-level `values: [...]` but no `_keboola` block + now hoists `values` into the API body (the push callers pass the known + `component_id` into `local_row_to_api`). Pre-0.47.2 the row was created with an + empty `configuration.values`. +- **Rows whose parent config was created in the same push now succeed.** Push runs + in ordered phases (configs first, then rows); a row's `parent_config_id` is + remapped from the diff-time placeholder to the freshly-assigned ULID before + `create_config_row`. Pre-0.47.2 this raised `PARENT_CONFIG_NOT_TRACKED`. +- **`variables_id` / `variables_values_id` are rebound to ULIDs.** After the + variables config + its values row are created, a backfill pass PUTs the + transformation's corrected `configuration.variables_id` / `variables_values_id` + (via `update_config`, NOT `set_variables` — that would create a *second* + variables config), rewrites the local `_configuration_extra`, and refreshes the + manifest hashes so a re-push is clean. Pre-0.47.2 the remote kept placeholder + strings and `job run` failed with `Variable configuration "" not + found`. When the placeholder can't be matched exactly but exactly one + `keboola.variables` config was created this push, it binds to that one with a + warning; zero or ambiguous (>1) matches surface a `variable_link` entry in the + push `errors` array rather than writing a broken link. + +`sync push --branch ` now **promotes the local default tree** (`main/`) to the +target dev branch when no `/` subtree exists on disk, instead of +erroring with `Config file not found`. Source (where files are read) and target +(where the API writes) are decoupled; API calls still target the branch id. When a +per-branch subtree *does* exist, behaviour is unchanged. + +## `sync push` / `sync pull` / `sync diff` accept `--branch ` for per-invocation dev-branch targeting (since v0.47.0) + +The `--branch` override wins over every other branch source: `manifest.branches[0]`, +`active_branch_id` (set by `kbagent branch use`), and the git-branching +`branch-mapping.json`. Required exactly one `--project` (branch id is per-project). +Useful for targeting a freshly-created dev branch without running `branch use` or +`sync branch-link` first. The override is per-invocation only — it does not persist +to the manifest or to the config store, so subsequent commands without `--branch` +fall back to the normal priority chain. + +## `storage create-table --if-not-exists` returns `action: skipped` instead of raising on duplicate display name (since v0.47.0) + +Opt-in flag (default `False`, so existing callers are unaffected). When set, +catches the specific `STORAGE_JOB_FAILED` + "already has the same display name" +error from the Storage API, probes `get_table_detail(target_id)`, and returns +`{action: "skipped", skip_reason: "table already exists", table_id: ...}` when +the table really exists at the expected id. A different table that happens to +share the display name still raises (real conflict to resolve). The response +envelope now always carries `action: "created" | "skipped"` so programmatic +callers can branch on outcome. Safe for parallel workers (e.g. FIIA's +8-worker scaffold pattern that previously surfaced ~12 spurious errors per run). + +**Skipped envelope reports the ACTUAL existing schema (since v0.47.1, keboola/cli#349).** +When `action == "skipped"`, `columns` / `primary_key` / `name` reflect the +EXISTING table's real schema (read from the `get_table_detail` probe that +confirms the table exists), not the caller's request. The caller's requested +values are preserved under `requested_columns` and `requested_primary_key`, and +`schema_drift: true` flags when the existing table diverges from what was +requested. So the skipped envelope IS a valid discovery mechanism — a caller can +trust `columns` / `primary_key` as the real shape and inspect `schema_drift` to +detect "I hit a pre-existing table with a different shape". (Before v0.47.1 the +skipped envelope re-echoed the request, so older installs must still call +`kbagent storage table-detail` after a skip to get the real shape.) + +## `sync push --no-name-drift-warnings` suppresses the cosmetic warnings array (since v0.47.0) + +When local directory names diverge from the canonical kbagent naming (e.g. +FIIA's `var-07-fi-daily-date-refresh` pattern), `sync push` normally returns +a `name_drift_warnings: [...]` array on the result envelope. The +`--no-name-drift-warnings` flag drops that field. The underlying detection +still runs, so a future operator who wants to audit can flip the flag off +without losing data. + +## `semantic-layer search-context` + `get-context` cover the MCP `search_semantic_context` / `get_semantic_context` parity (since v0.47.0) + +`kbagent semantic-layer search-context --project P [--pattern G ...] [--type T] [--limit N]` +is project-wide (not model-scoped). Patterns are **case-sensitive `fnmatch`** against +`attributes.name`, repeatable (union). Default `--type all` searches every CHILD +type (datasets, metrics, relationships, constraints, glossary) and does **not** +include semantic models — pass `--type model` to search those. The response +envelope is `{project, contexts: [{id, type, name, description, attributes}], total_count}`. +The `type` field is the CLI-friendly singular (no `semantic-` prefix on the wire form). + +`kbagent semantic-layer get-context --project P --context-id ID` probes +`semantic-model` first then every child type until a 200 lands. A 404 on any +single probe is non-terminal (keeps trying); only a full miss across all 6 +types raises `NOT_FOUND` (exit 1). **Non-404 errors propagate immediately** +without continuing the probe — a 500 on the dataset type does not get +swallowed by the subsequent metric probe. + +These two subcommands cover the pre-flight pattern FIIA uses to verify a +project's semantic model is populated before kicking off a downstream +pipeline; the previous workaround (a `keboola-mcp-server` MCP server entry +in `.mcp.json` solely for these two tools) can be dropped. + ## `workspace list` / `workspace detail` now expose loginType + RO + qs_compatible (since v0.42.0, closes #304) Before v0.42.0 the Storage workspace endpoint already returned @@ -48,6 +181,7 @@ plus a derived `qs_compatible: bool`. - `snowflake-service-keypair` -- confirmed PASS - `snowflake-person-sso` -- confirmed PASS +- `snowflake-person-keypair` -- confirmed PASS (since v0.47.1) - `snowflake-legacy-service` -- explicitly OFF the list (works on `connection.keboola.com` but FAILED on GCP us-east4 stack in the original #304 incident -- keep it off until cross-stack confirmation) @@ -59,6 +193,28 @@ confirmed-good whitelist". For an unknown loginType, `workspace list` renders it as `?` (yellow) in the QS column so callers know the policy is uncertain rather than confirmed-bad. +## Snowflake `workspace create` returns `private_key`, not password (since v0.47.1) + +Headless `workspace create` on Snowflake requests +`loginType: snowflake-person-keypair`, generates an RSA key pair locally, +passes the public key to the Storage API, and returns the private key once in +the creation envelope: + +```jsonc +{ + "backend": "snowflake", + "user": "KEBOOLA_WORKSPACE_42", + "password": "", + "private_key": "-----BEGIN PRIVATE KEY-----\n...\n-----END PRIVATE KEY-----\n" +} +``` + +For successful Snowflake creates, `private_key` is the credential to save and +use for key-pair authentication; `password` remains in the envelope for +backward compatibility but should be treated as empty/unusable. BigQuery +workspaces keep the previous password-based/default backend shape and do not +return `private_key`. + **Filter (data-app pre-selection):** ```bash @@ -610,23 +766,67 @@ events and emits a final `done` SSE frame mirroring the same record. latest -- previously only the latest was reported, leaving the user with no signal whether their cache was stale. -## `storage swap-tables` is dev-branch only and aliases stay put (since v0.28.0) +## `storage swap-tables` is branch-scoped and aliases stay put (since v0.28.0) - `kbagent storage swap-tables --project P --table-id A --target-table-id B - --branch ` swaps two tables' physical positions in a dev branch + --branch ` swaps two tables' physical positions (`POST /v2/storage/branch/{branch}/tables/{id}/swap`). -- The Storage API rejects this on production. The service refuses with - exit 5 / `ConfigError` *before* any HTTP call when neither `--branch` - nor an active branch (via `branch use`) is set. +- **branch_id is mandatory, but any branch works -- including the + default/production branch.** The service refuses with exit 5 / + `ConfigError` *before* any HTTP call only when neither `--branch` nor an + active branch (via `branch use`) is set. (The earlier "rejected on + production" claim was wrong -- verified live 2026-06-01: a default-branch + swap succeeds and is the supported way to retype a prod table.) - **Aliases are NOT transferred.** They keep pointing at the same physical position, so after the swap they expose the OTHER table's data. Plan downstream config rewrites if any aliased consumer relies on schema, not data. -- Typical use: AI agent profiles a typeless table, builds a typed - rebuild called `_change_log` via CTAS in a dev branch, then - swaps it back into the original name. After merging the branch the - original table now carries the typed schema with no downstream config - rewrite required. +- **Dev-branch merge does NOT carry storage schema** (only configs), so a + swap done inside a dev branch never reaches production via merge. The + dev branch is a *rehearsal* -- profile the typeless table, build a typed + rebuild (`_change_log`) via CTAS, swap, and run downstream configs + against it to prove the typed schema is consumer-safe. Then discard the + branch and run the real build + swap in the production (default) branch. + Full procedure: `typify-table-workflow.md`. + +## `storage clone-table` materializes a prod table into a dev branch (since v0.52.0) + +- `kbagent storage clone-table --project P --table-id T --branch ` + pulls a production table into a dev branch + (`POST /v2/storage/branch/{branch}/tables/{id}/pull`, operationName + `devBranchTablePull` -- the same call the platform issues on a branch's + first write to a prod table). +- **Why it matters on `storage-branches` projects:** a dev branch reads + production tables transparently (copy-on-write) until the first write. + A schema mutation in the branch -- `swap-tables`, dropping a column -- + targets a table that is not yet branch-local, so the Storage API fails + with a misleading `"bucket ... was not found in the project"`. Run + `clone-table` first to materialize the table branch-local; the swap / + drop then succeeds. (Verified live 2026-06-01 on project 10539 with + storage-branches ON: clone -> in-branch swap succeeds; production left + untouched.) +- **One-way (default -> branch).** There is no "push branch -> default": + branch storage is never merged back to production (only configurations + are). The pull is the only API path between the two table stores. +- Branch is mandatory: the service refuses with exit 5 (`ConfigError`) + before any HTTP call when neither `--branch` nor an active branch (via + `branch use`) is set. +- Permission class: `write` (creates a branch-local copy; never deletes). + +## Dev-branch merge carries only configurations, NOT storage schema (since v0.52.0, verified 2026-06-01) + +- When a dev branch is merged to production, Keboola propagates + **configuration** changes only. Physical storage tables -- their + schema, column types, and rows -- are **not** merged back. (Confirmed + by the storage-branches designer and Keboola's public docs: + help.keboola.com/tutorial/branches/merge-to-production.) +- Consequence for retyping: a `swap-tables` (or `clone-table`) done inside + a dev branch stays in the branch. To retype a **production** table you + run the build + `swap-tables` in the production (default) branch itself. + The dev branch is only a rehearsal that validates the typed schema + against downstream configs. Full procedure: `typify-table-workflow.md`. +- The only API path between the two table stores is `clone-table` (pull, + default -> branch). There is no "push branch -> default". ## `storage truncate-table` preserves schema; endpoint is uniformly async-via-job (since v0.32.0) @@ -2046,3 +2246,204 @@ commands that never had `--hint` support. AI agents should prefer the REST surface over `--hint` for new integrations. Do not add new examples or workflows that teach `--hint`; point readers to `kbagent serve` instead. + +## `feature` command group: super-admin token, no per-project endpoint, opaque schema (since v0.48.0) + +The `feature` group manages Keboola feature flags via the **Manage API**. Five +things trip up callers: + +1. **Super-admin manage token required.** `feature list` (the stack catalogue) + and every project/user mutation need a super-admin Manage API token -- the + same kind `org setup` uses, NOT the per-project Storage token. It follows the + default-deny policy: interactive hidden prompt by default; pass top-level + `--allow-env-manage-token` + `KBC_MANAGE_API_TOKEN` for CI. Do NOT pass the + token as a CLI flag. A non-super-admin token returns 403 (exit 3). + +2. **`--project` is just a handle to the stack URL.** For `feature list` and the + `user-*` commands the alias only resolves the stack URL -- the catalogue and + user features are stack-wide, not project-scoped. For `project-*` commands it + additionally resolves the numeric `project_id` from config. The alias must be + registered (`kbagent project list`); `project-*` also requires it to carry a + `project_id`. + +3. **No dedicated "project features list" endpoint.** `feature project-show` + reads the `features` array off `GET /manage/projects/{id}`; `feature + user-show` reads it off `GET /manage/users/{email}`. There is no + `/projects/{id}/features` GET. Only the add (`POST .../features`, body + `{"feature": NAME}`) and remove (`DELETE .../features/{name}`) verbs are + per-resource. + +4. **Request body is `{"feature": NAME}`, not `{"name": NAME}`.** The add + endpoints take the feature code under the key `feature`. (Some third-party + notes claim `name` -- that is wrong for this API.) + +5. **Feature schema is opaque + shape-variable.** The Manage API publishes no + feature schema, and a `features` array may come back as a list of objects OR + a list of bare strings depending on stack/endpoint. kbagent normalises both + to `{name, title, description, type, ...}` (bare strings become + `{"name": s}`) and passes unknown keys through unmodified. Treat `name` as + the only stable field; do not depend on `title`/`type` being populated. + +To inspect a project's *enabled* features without a super-admin token, use +`kbagent project info --project P` (read-only) instead -- it returns the enabled +feature list among other project metadata. + +## Developer Portal: writes require a human, no exceptions (since v0.49.0) + +`kbagent dev-portal {create,patch,upload-icon,publish,deprecate}` always print +the request preview and then require the user to type a random hex code on a +real terminal. There is no `--yes` flag. There is no env-var override. The +command exits 6 (`EXIT_PERMISSION_DENIED`) on a non-TTY shell. + +For agentic use: stop at the preview. Use `--dry-run` to get a clean +exit-0 preview you can show the user. Then ask the user to run the same +command without `--dry-run` themselves. + +Reads (`dev-portal list`, `dev-portal get`) are unrestricted — peer-research +patterns ("show me how MySQL and Postgres extractors configure themselves") +are agent-friendly via `list --vendor` + `get --app`. + +## Headless / token-only invocation: the `__env__` project (since v0.50.0) + +A daemon, container, or CI job that has only a token in its environment can run +kbagent with **no `kbagent project add` and no `config.json` on disk**. Set all +three: + +```bash +export KBAGENT_PROJECT_FROM_ENV=1 +export KBC_TOKEN= +export KBC_STORAGE_API_URL=https://connection..keboola.com +kbagent --json storage file-upload --project __env__ --file screenshot.png +``` + +kbagent synthesizes an in-memory project under the reserved alias `__env__`. +Pass it as `--project __env__` (or rely on it being the sole/default project for +commands that fall back to the default). + +Gotchas: +- **Opt-in is the flag, not the token.** `KBC_TOKEN` alone does nothing here — + it stays a `project add` fallback. Only `KBAGENT_PROJECT_FROM_ENV` (truthy: + `1`/`true`/`yes`/`on`) triggers injection. This avoids a phantom project on a + dev machine that exported `KBC_TOKEN` for an unrelated `project add`. +- **Token is never persisted.** `__env__` is `ephemeral`; even if a write op + triggers a `config.json` write, the env token is stripped first. There is no + way to leak it to disk through normal operation. +- **Fail-fast.** Flag set but `KBC_TOKEN` or `KBC_STORAGE_API_URL` missing → + exit 5 (`config error`) with a clear message, not a silent skip. +- **Same chokepoint for `serve`.** `kbagent serve` started with the same three + env vars exposes `__env__` too — POST endpoints take `project=__env__`. Both + CLI and serve resolve through `ConfigStore.load()`, so one env setup covers + both consumption styles. +- The alias is literally `__env__` (double underscore both sides) — chosen so it + cannot collide with a real user alias. A real project already registered under + `__env__` wins; no injection happens. +- **`__env__` shows `project_id` but a blank name in `project list`.** `load()` + is offline, so the injection recovers `project_id` from the token prefix + (`{projectId}-{tokenId}-{secret}`) but cannot fetch the real project name. + Run `kbagent project status --project __env__` (or `project info`) to verify + the token against the API and see the real name. +- **`KBC_STORAGE_API_URL` is forgiving (since v0.50.0).** A bare host + (`connection.keboola.com`), a trailing slash, or a full project deep-link + (`.../admin/projects/123/dashboard`) all normalize to `https://`. Same + normalization applies to `project add --url` / `project edit --url`. Explicit + `http://` / `file://` is still rejected; a bad URL fails fast with a clean + config error (exit 5), not a traceback. + +## `stream`: two hosts, secret-in-URL, no auto-sinks (since v0.50.0) + +Data Streams has **two hosts**. The *control plane* is `stream.` (derived +from `connection.`, same scheme as `ai.`/`queue.`) and is what the CLI +calls, authenticated with the ordinary per-project **Storage** token — there is +no manage token and no extra prompt. The OTLP *ingestion* endpoint lives on a +**different** host, `stream-in./otlp///`, +and is **returned by the API** in `source.otlp.url` — kbagent never derives it. + +The ingest secret is **in the URL path**. `stream detail` / `create-source` mask +it by default in every surface (the `endpoint`, all three per-signal endpoints, +and the raw `source` object echoed in `--json`). Pass `--reveal` to print the +real secret — e.g. to wire `OTEL_EXPORTER_OTLP_ENDPOINT` for a daemon: +`kbagent --json stream detail SRC --project P --reveal`. + +Creating an OTLP source **via the raw Stream API** creates only the bare source +— no sinks, no tables. So `kbagent stream create-source --type otlp` (matching +the Keboola UI) **auto-provisions the three sinks** logs/metrics/traces into +bucket `in.c-otlp-`, mapping each record to `id` (uuid) + `datetime` +(ingest time) + `body` (the full flattened OTLP record as JSON), so data lands +out of the box. Provisioning is **idempotent** (only missing signals are added) +and `--no-sinks` opts out for a bare source. The destination tables themselves +materialize lazily on first import (the bucket/table appear in Storage seconds +after the first record arrives, not at create time). `create-source` / `delete` +/ sink creation are **async**: the API returns a Task that kbagent polls to +completion before returning. + +## `dev-portal patch`: admin-only fields need an admin-role identity, vendor PATCH lies about why + +`PATCH /vendors/{vendor}/apps/{app}` on apps-api `.forbidden()`s 9 fields: +`complexity`, `categories`, `category`, `features`, `forwardToken`, +`forwardTokenDetails`, `injectEnvironment`, `processTimeout`, +`requiredMemory`. Sending any of them via a vendor identity returns +`422 Parameter complexity must be one of: easy, medium, hard` (or the +analogous enum message for the other fields). **The message is a server +bug** — the enum-validation `.error()` annotation lives on the shared +admin schema before `clientAppSchema()` overrides with `.forbidden()`, +so when `.forbidden()` fires Joi reuses the unrelated enum message +instead of saying "this field is not allowed here". + +To set any of these you need an admin identity that routes the PATCH +to `PATCH /admin/apps/{app}` instead (since v0.51.1): + +``` +kbagent dev-portal identity add --alias admin-keboola \ + --username admin@keboola.com --role-hint admin --password-stdin +kbagent dev-portal patch --app keboola.ex-foo \ + --data /tmp/patch.json --identity admin-keboola +``` + +With `role_hint: vendor` (the default), kbagent now pre-flights the +payload and fails fast with the same guidance instead of letting the +apps-api return the misleading 422 (since v0.51.1). The 9 forbidden +fields are documented in +[keboola/developer-portal:src/lib/validation.js](https://github.com/keboola/developer-portal/blob/master/src/lib/validation.js) +under `clientAppSchema()`. + +## `dev-portal identity add`: MFA logins for TOTP accounts need the `challenge` field explicit (since v0.51.1) + +The apiary spec calls `challenge` optional with default `SOFTWARE_TOKEN_MFA` +on the second-step `POST /auth/login`, but in practice the server 404s +when it is omitted on a personal-account TOTP login. kbagent now sends +`challenge: SOFTWARE_TOKEN_MFA` explicitly. Single attempt only: +`/auth/login` consumes the session, so any retry with a different +challenge type would always 404 with `Invalid code or auth state for +the user` and mask the real first failure. The raised error includes +the server response body and a hint about TOTP code rotation, so +"stale code" can be distinguished from "wrong code" / "expired session". + +## `dev-portal identity {add,edit} --password-stdin` works in both TTY and pipe mode (since v0.51.1) + +Pre-0.51.1 the flag did `sys.stdin.read().strip()` unconditionally, +which waits for EOF rather than Enter — pasting a password and pressing +Enter just hung until Ctrl-C. The helper now branches on +`sys.stdin.isatty()`: TTY uses `getpass.getpass()` (hidden, line-based, +Enter confirms); pipe (`echo $PASS | kbagent dev-portal identity add +--password-stdin`) still reads to EOF. + +## `sync pull --force` preserves un-pushed edits and aborts on conflict (since v0.53.0) + +`--force` is **conflict-aware**, not a blind overwrite. Pre-0.53.0 a force-pull +run while you had un-pushed local edits to a config whose remote was unchanged +**silently corrupted the sync baseline**: it re-stamped the manifest `pull_hash` +from the *edited* on-disk file, so `sync diff` / `sync push` then reported "in +sync" and shipped nothing while the remote still held the old config -- the edits +were stranded with no signal. Fixed: `--force` now branches on the 3-way state. + +- Local edited, remote unchanged -> **preserved** (pending delta stays pushable). +- Local edited AND remote also changed (true conflict) -> **aborts** with exit 1 + and error code `SYNC_CONFLICT`; the `--json` envelope carries + `details.conflicts: [{scope, component_id, config_id, config_name, path, row_id?}]`. + Resolve via `sync diff` then `sync push`-or-discard, then pull again. +- Local untouched, remote changed -> takes remote (unchanged behavior). + +Consequence: `--force` no longer discards non-conflicting local edits. To +intentionally drop a local edit, delete the file (or the config dir) and pull. +Applies at config and row granularity. `--all-projects` reports a per-project +conflict as that project's error without aborting the rest of the batch. diff --git a/plugins/kbagent/skills/kbagent/references/storage-types-workflow.md b/plugins/kbagent/skills/kbagent/references/storage-types-workflow.md index d62a6719..d4d89495 100644 --- a/plugins/kbagent/skills/kbagent/references/storage-types-workflow.md +++ b/plugins/kbagent/skills/kbagent/references/storage-types-workflow.md @@ -240,7 +240,12 @@ needs to flip the typed copy back into the original name so downstream configs (extractors, transformations, writers) keep working unchanged. ```bash -# 1. Isolate the work in a dev branch +# 1. Rehearse in a dev branch (validate the typed schema against downstream +# configs). The REAL retype is then repeated in the default/production +# branch -- dev-branch merge does NOT carry storage schema. For the full +# rehearsal-then-production procedure, see typify-table-workflow.md. +# below is the rehearsal branch; for the production run pass the +# default-branch ID instead. kbagent branch create --project prod --name typify-data kbagent branch use --project prod --branch @@ -254,6 +259,17 @@ kbagent workspace query --workspace-id W --sql " " # (or use kbagent storage create-table + an SQL transformation) +# 2b. storage-branches projects only: the dev branch reads 'data' +# transparently until first write, so swap (a write) fails with a +# misleading "bucket not found" until 'data' is materialized +# branch-local. Pull it in first. (data_change_log, built by the +# in-branch CTAS above, is already branch-local. Skip on +# legacy-branch projects.) +kbagent storage clone-table \ + --project prod \ + --table-id in.c-foo.data \ + --branch + # 3. Swap: the typed copy becomes 'data', the typeless original moves # to 'data_change_log'. Aliases stay put -- they expose the OTHER # table's data after the swap. @@ -263,17 +279,28 @@ kbagent storage swap-tables \ --target-table-id in.c-foo.data_change_log \ --branch --yes -# 4. Merge the dev branch when satisfied -kbagent branch merge --project prod --branch +# 4. There is NO merge step: dev-branch merge carries only configs, not +# storage schema. Once the rehearsal proves the schema is safe, delete +# the branch and repeat steps 2-3 in the default/production branch +# (pass the default-branch ID to --branch on swap-tables). +kbagent branch delete --project prod --branch --yes ``` Rules: -- The Storage API rejects this on production. The service refuses with - exit 5 (`ConfigError`) before any HTTP if `--branch` is missing AND no - active branch is set via `branch use`. +- **storage-branches projects:** `swap-tables` operates on branch-local + tables. The original (`in.c-foo.data`) is read transparently from prod + until first write, so the swap fails with a misleading "bucket not + found" until you `clone-table` it into the branch (step 2b). The typed + sibling built by the in-branch CTAS is already branch-local. Legacy + fake-branch projects don't need this. +- branch_id is mandatory: the service refuses with exit 5 (`ConfigError`) + before any HTTP if `--branch` is missing AND no active branch is set via + `branch use`. Any branch works, INCLUDING the default/production branch + -- a default-branch swap is how the retype reaches prod (the earlier + "rejected on production" claim was wrong). - Aliases keep pointing at the same physical position, i.e. they expose the OTHER table's data after the swap. If your downstream relies on - alias-by-name, validate post-swap before merging. + alias-by-name, validate post-swap before applying in production. - The Storage API queues the swap as an async storage job (`operationName: tableSwap`); the kbagent client polls the job to completion before returning, so callers can rely on the schemas being diff --git a/plugins/kbagent/skills/kbagent/references/stream-workflow.md b/plugins/kbagent/skills/kbagent/references/stream-workflow.md new file mode 100644 index 00000000..93030b69 --- /dev/null +++ b/plugins/kbagent/skills/kbagent/references/stream-workflow.md @@ -0,0 +1,100 @@ +# Data Streams (OTLP / OpenTelemetry) workflow + +> Audience: a developer or a kbagent agent wiring an OpenTelemetry exporter +> (logs / metrics / traces) into Keboola Data Streams. Goal: create an OTLP +> source and read its endpoint in one command instead of copy-pasting from the +> UI. Reference: https://help.keboola.com/storage/data-streams/opentelemetry/ +> (since v0.50.0; closes keboola/cli#357) + +## Mental model: two hosts + +Data Streams has a **control plane** and a **data plane**: + +- **Control plane** `stream.` — derived from the project's Storage URL + (`connection.` → `stream.`, same scheme as `ai.`/`queue.`). + This is what `kbagent stream …` calls, authenticated with the **per-project + Storage token** kbagent already holds. No manage token, no extra prompt. +- **Data plane** `stream-in./otlp///` — + the actual OTLP/HTTP ingest endpoint your exporter posts to. kbagent does + **not** derive this; it is returned by the API in `source.otlp.url`, with the + auth **secret embedded in the URL path** (no header needed). + +## Secret handling + +The ingest URL carries the secret. Every kbagent surface **masks it by +default** — the `endpoint`, the three per-signal endpoints, and the raw +`source` object in `--json`. Print the real secret only on explicit intent: + +```bash +kbagent --json stream detail my-otlp --project P --reveal +``` + +Treat `--reveal` output as a credential. The intended consumer is a setup +step that exports `OTEL_EXPORTER_OTLP_ENDPOINT` for a daemon/CI — not a log. + +## End-to-end: stand up an OTLP source + +```bash +# 1. Create the source. For OTLP this also auto-provisions the logs/metrics/traces +# sinks (bucket in.c-otlp-my-otlp) so data lands. Idempotent + --if-not-exists. +kbagent --json stream create-source --project P --name my-otlp --type otlp --if-not-exists + +# 2. Read the wiring (masked). Confirms protocol, per-signal endpoints, and the +# destination tables. +kbagent stream detail my-otlp --project P + +# 3. Reveal the endpoint to configure an exporter. +kbagent --json stream detail my-otlp --project P --reveal +``` + +The exporter then uses (per the OTLP/HTTP spec, `http/protobuf`): + +```bash +export OTEL_EXPORTER_OTLP_ENDPOINT="" +export OTEL_EXPORTER_OTLP_PROTOCOL="http/protobuf" +export OTEL_SERVICE_NAME="my-service" +``` + +Per-signal endpoints are the base URL + `/v1/logs`, `/v1/traces`, `/v1/metrics` +— most SDKs append these automatically from the base `OTEL_EXPORTER_OTLP_ENDPOINT`. + +## Destination tables + +The raw Stream API `POST /sources` creates only the bare source — no sinks. So +`kbagent stream create-source --type otlp` (matching the UI) **auto-provisions +three table sinks** — logs / metrics / traces — into bucket +`in.c-otlp-` (tables `logs` / `metrics` / `traces`). Each record is +mapped to `id` (uuid), `datetime` (ingest time), and `body` (the full flattened +OTLP record as JSON); refine the per-signal column mapping in the Keboola UI +afterwards if you want first-class columns. Provisioning is idempotent (a re-run +or `--if-not-exists` only fills missing signals); pass `--no-sinks` for a bare +source. The bucket/tables materialize **lazily on first import** — they appear in +Storage seconds after the first record arrives, not at create time. + +## Send data and verify it landed + +```bash +# Capture the revealed endpoint into a shell var (don't echo the secret): +EP=$(kbagent --json stream detail my-otlp --project P --reveal | python3 -c \ + 'import sys,json; print(json.load(sys.stdin)["data"]["endpoint"])') + +# Post a standard OTLP/HTTP JSON logs payload (secret is in the URL path): +curl -s -X POST "$EP/v1/logs" -H 'Content-Type: application/json' --data @logs.json + +# Default import trigger is ~1 min / 50 MB / 50k records; data is usually visible +# within ~15 s. Confirm rows landed: +kbagent --json storage table-detail --project P --table-id in.c-otlp-my-otlp.logs # rowsCount +# Or read them via a workspace: +kbagent workspace load --project P --workspace-id WS --tables in.c-otlp-my-otlp.logs +kbagent workspace query --project P --workspace-id WS --sql 'SELECT "id","datetime","body" FROM "logs"' +``` + +## Lifecycle notes + +- `create-source` and `delete` are **async**: the API returns a Task and + kbagent polls it to completion before returning. +- `delete` is destructive — `--dry-run` previews, `--yes`/`--force` skips the + confirm. `--json` mode never prompts. +- `--branch ID` targets a dev branch; default is the project's default branch. +- Permission classes: `stream.list` / `stream.detail` = read, + `stream.create-source` = write, `stream.delete` = destructive. diff --git a/plugins/kbagent/skills/kbagent/references/sync-workflow.md b/plugins/kbagent/skills/kbagent/references/sync-workflow.md index 0b44247a..cd9a3ef4 100644 --- a/plugins/kbagent/skills/kbagent/references/sync-workflow.md +++ b/plugins/kbagent/skills/kbagent/references/sync-workflow.md @@ -67,6 +67,96 @@ kbagent sync push --all-projects # apply Each project gets its own subdirectory (named by alias). Projects are processed in parallel. +## Per-invocation dev-branch override (since v0.47.0) + +`sync push`, `sync pull`, and `sync diff` accept `--branch ` to target a +dev branch for a single invocation. The override beats every other branch +source: `manifest.branches[0]`, the project's `active_branch_id` (`branch use`), +and the git-branching `branch-mapping.json`. + +```bash +# Push the current working tree to dev branch 388072 without `branch use` +# or `sync branch-link` first. Required exactly one --project. +kbagent sync push --project prod --branch 388072 + +# Same dev branch on pull (`sync diff` accepts it too). +kbagent sync pull --project prod --branch 388072 +kbagent sync diff --project prod --branch 388072 +``` + +Use cases: +- Spin up a throwaway dev branch via `kbagent branch create`, push a + candidate change to it for testing, then `kbagent branch delete` to clean + up — all without touching the persisted active-branch state. +- Scripted / scheduled flows where the branch id comes from an upstream + job (e.g. a CI pipeline computes the branch id and passes it via env). + +Mutually exclusive with `--all-projects` at the CLI layer (branch id is +per-project; the validator returns exit 2 + `USAGE_ERROR` if combined). +The override is per-invocation only — it does not write into the manifest +or the config store, so a subsequent command without `--branch` falls back +to the normal priority chain. + +**Promote the default tree to a target branch (since v0.47.2).** When the +target branch has no materialized `/` subtree on disk, `sync push +--branch ` reads the **default tree** (`main/`) as the source and promotes +it to the target branch, instead of failing with `Config file not found`. +Source (read) and target (API write) are decoupled; API calls still target the +branch id. So the common "I only have `main/` locally, push it to a fresh dev +branch" flow now works with a single command: + +```bash +# main/ on disk, no feature-x/ subtree -> main/ is promoted to branch 388072 +kbagent sync push --project prod --branch 388072 +``` + +When a per-branch subtree *does* exist (multi-branch-directory users), the +target subtree is used as before — behaviour is unchanged. + +## Fresh-CREATE writeback (since v0.47.0) + +If you (or a tool like FIIA) seed `.keboola/manifest.json` with placeholder +entries before the first `sync push`, the writeback updates each placeholder +**in place** rather than appending a new entry. Pre-v0.47.0 this produced +manifests of length 2N after one push (placeholders + new entries both +retained); from v0.47.0 the manifest stays at length N and the placeholder +entry's id is updated to the API-assigned ULID. Re-pushes against the +now-real id are naturally idempotent. + +If a placeholder entry's `metadata` dict contains `KBC.configuration.*` +keys (e.g. `KBC.configuration.folderName`), they are propagated to the +metadata API immediately after the create call. This was the previous +"set folderName via `config set-folder` after push" workaround for +fresh-create flows; from v0.47.0 a single push handles it. + +**Variable links are resolved on fresh CREATE (since v0.47.2).** When the +placeholder tree includes a `keboola.variables` config + default-values row and +a transformation that cross-references them, one `sync push` now produces a +*runnable* transformation: + +- the row's top-level `values: [...]` reach the API body even when the scaffold + row file has no `_keboola` block (KFR-04); +- a row whose parent `keboola.variables` config is created in the same push is + POSTed against the assigned ULID, not the placeholder (KFR-05, previously + `PARENT_CONFIG_NOT_TRACKED`); +- the transformation's `configuration.variables_id` / `variables_values_id` are + rebound from placeholders to the assigned ULIDs via a post-create + `update_config` PUT (KFR-03, previously `job run` failed with `Variable + configuration "" not found`). + +This removes the need for a post-push `config variables-set` step. If a +placeholder can't be matched and the binding is ambiguous (zero or >1 +`keboola.variables` configs created this push), the link is left untouched and a +`variable_link` entry appears in the push `errors` array — never a silently +broken link. A clean re-push reports `no_changes`. + +`sync push --no-name-drift-warnings` (since v0.47.0) suppresses the +cosmetic `name_drift_warnings` array on the result envelope. The +detection still runs; only the report is dropped. Useful for downstream +tools that already audit drift their own way (e.g. FIIA's +`var-07-fi-daily-date-refresh` pattern legitimately differs from the +canonical kbagent naming and the warnings are noise). + ## Adopting an existing kbc Go CLI checkout (since v0.22.0) If you already have a `.keboola/manifest.json` produced by the official @@ -252,7 +342,8 @@ Stored in `.keboola/branch-mapping.json`: ## Key behaviors - **Pull is idempotent**: re-running pull when nothing changed writes zero files -- **Pull protects local edits**: modified files are skipped (use `--force` to overwrite) +- **Pull protects local edits**: locally-modified files are skipped by default +- **`--force` is conflict-aware (since 0.53.0)**: see below -- it no longer blindly overwrites - **Push only sends local changes**: remote_modified and conflict changes are skipped - **Encrypted values**: nonce differences are ignored in diff (no false positives) - **New configs**: push auto-assigns IDs from the API, updates manifest @@ -260,3 +351,23 @@ Stored in `.keboola/branch-mapping.json`: - **Jobs are per-config**: `_jobs.jsonl` shows recent N jobs (default 5) with status + timing - **Data samples auto-trim**: tables with >30 columns export only first 30 (API sync limit) - **Encrypted columns masked**: columns starting with `#` show `***ENCRYPTED***` in samples + +## `sync pull --force` is conflict-aware (since 0.53.0) + +`--force` no longer blindly overwrites locally-modified configs. It branches on +the 3-way diff state per config (and per row): + +- **Local edited, remote UNCHANGED** -> the file and its sync baseline are + **preserved**. The pending delta stays visible to `sync diff` / `sync push`. + `--force` does **not** discard the edit and does **not** silently re-stamp the + baseline (that was the pre-0.53.0 data-loss bug). +- **Local edited AND remote also changed** since the last pull (a true merge + conflict) -> the pull **aborts before writing anything** with exit code 1 and + error code `SYNC_CONFLICT`, listing every conflicting config/row. Resolve with + `sync diff`, then `sync push` your edits (or discard them), then pull again. +- **Local untouched, remote changed** -> `--force` takes remote as before. + +> Safe to run `sync pull --force` to refresh an unrelated config even while you +> have un-pushed edits elsewhere: non-conflicting edits survive; a real conflict +> stops you loudly instead of losing work. To intentionally drop a local edit, +> delete the file (or the config directory) and pull. diff --git a/plugins/kbagent/skills/kbagent/references/typify-table-workflow.md b/plugins/kbagent/skills/kbagent/references/typify-table-workflow.md index 91c2fdd8..a8505b85 100644 --- a/plugins/kbagent/skills/kbagent/references/typify-table-workflow.md +++ b/plugins/kbagent/skills/kbagent/references/typify-table-workflow.md @@ -14,12 +14,23 @@ canonical move is: (downstream sees real types) (now holds the old typeless data) ``` -The whole thing happens inside a **dev branch** so production transformations -keep running on the typeless original until the user merges. Aliases stay -put across the swap (they expose the OTHER table's data after -- see -`gotchas.md` "swap-tables aliases stay put"). - -Since v0.28.0 (`storage swap-tables` + `config update` script[] auto-normalize). +**Two-stage model -- rehearse in a dev branch, apply in production.** +Dev-branch merge propagates only *configurations*, NOT storage table +schema (see `gotchas.md` "Dev-branch merge carries only configurations"), +so a swap done inside a dev branch never reaches production via merge. The +dev branch is therefore a **rehearsal**: profile the data, build the typed +sibling, swap, and run downstream configs against it to *prove the typed +schema is consumer-safe*. Once proven, **discard the branch** and run the +real build + swap directly in the production (default) branch -- a +default-branch swap is supported (verified live) and is the only path that +actually retypes the production table. Aliases stay put across the swap +(they expose the OTHER table's data after -- see `gotchas.md` +"swap-tables aliases stay put"). + +Since v0.28.0 (`storage swap-tables` + `config update` script[] +auto-normalize); `storage clone-table` (v0.52.0) materializes a prod table +into a branch when the rehearsal needs the original branch-local on +storage-branches projects. ## Phase 0 -- Decide if you should do this @@ -59,10 +70,13 @@ kbagent --json project status --project ALIAS # -> branch field shows the new branch_id ``` -Why a dev branch: +Why a dev branch (this is a **rehearsal**, not the thing that ships): +- You use the branch to prove the typed schema is downstream-safe. The + production retype (Phase 8) repeats the build + swap in the default + branch -- merge does NOT carry the swapped schema to prod, only configs. - Production transformations and writers keep targeting the typeless - original; the rebuild is invisible to them until merge. + original; the rehearsal is invisible to them. - All the writes below (`storage create-table`, `workspace query` CTAS, `swap-tables`) are scoped to the branch by `branch use`'s active-branch resolution. @@ -265,9 +279,23 @@ For BigQuery dialect callers, also validate `bigquery_path` consumers (see `storage-describe-workflow.md`'s `bucket-detail` section -- BQ emits backtick-quoted `\`dataset\`.\`table\`` paths since v0.25.3). -## Phase 5 -- Swap +## Phase 5 -- Swap (in the rehearsal branch) + +This swap happens in the dev branch to prove the typed schema works; the +production swap is repeated in Phase 8. ```bash +# 5.0. storage-branches projects ONLY: the swap is a write, and the dev +# branch still reads the original 'data' transparently from prod, so +# the swap fails with a misleading "bucket not found" until 'data' is +# materialized branch-local. Pull it in first. ('data_typed', built +# in Phase 3, is already branch-local.) Skip on legacy-branch projects +# -- check with: kbagent project info --project ALIAS | grep storage-branches +kbagent --json storage clone-table \ + --project ALIAS \ + --table-id in.c-foo.data \ + --branch + # 5a. Dry-run first. Should report dry_run: true, never call the API. kbagent --json storage swap-tables \ --project ALIAS \ @@ -303,7 +331,7 @@ After the swap: - Aliases pointing at either table keep pointing at the same physical position, so they expose the OTHER table's data. If any downstream config refers to an alias, run a manual sanity check on it before - merge. + applying the retype in production (Phase 8). ## Phase 6 -- Smoke-test downstream @@ -336,75 +364,85 @@ verify row counts, not just job exit status. Ideally diff `data_typed` (= the old typeless rows) against the swapped `data` on a key column to confirm row-level identity. -## Phase 7 -- Cleanup `data_typed` (optional) +## Phase 7 -- Tear down the rehearsal branch -After a successful smoke test, the `data_typed` sibling holds the -old typeless rows and can be deleted. **Do this only after the user -confirms the merge.** Until merge, the sibling is the primary rollback -artifact (re-swap to undo). +Once Phases 4-6 prove the typed schema is consumer-safe, the dev branch +has done its job. **Nothing in it ships** -- merge will not carry the +swapped schema to production (only configs merge). Delete the branch +(this also drops the branch-local `data_typed` sibling): ```bash -# After merge (Phase 8 below), in main: -kbagent storage delete-table \ - --project ALIAS \ - --table-id in.c-foo.data_typed \ - --yes +kbagent branch delete --project ALIAS --branch --yes ``` -## Phase 8 -- Handoff protocol for the user (merge step) +Keep a written record of what the rehearsal proved -- the Phase 2 profile +summary and the Phase 4/6 downstream job results -- because Phase 8 +repeats the build in production with the same type decisions. -The Keboola Storage API does not merge dev branches via API -- the -merge is a human action in the UI. kbagent's `branch merge` command -returns a URL pointing the user to the right place. +## Phase 8 -- Apply the retype in production -Hand the user a structured summary so they can review before clicking -merge. Recommended shape: +Because dev-branch merge does not carry storage schema (see `gotchas.md` +"Dev-branch merge carries only configurations"), the real retype runs in +the **production (default) branch**, repeating the validated build: -```text -TYPIFY READY FOR MERGE -- in.c-foo.data +```bash +# 8a. Resolve the default (production) branch ID. +kbagent --json branch list --project ALIAS +# -> the entry with isDefault=true; call it . + +# 8b. Build the typed sibling in PRODUCTION using the exact types the +# rehearsal validated (Phase 2/3): same create-table + data copy as +# Phase 3, but targeting the default branch. +kbagent --json storage create-table --project ALIAS \ + --bucket-id in.c-foo --name data_typed \ + --column id:VARCHAR(40) --column amount:"NUMBER(18,2)" --branch +# ...then copy rows in (in-workspace INSERT or an SQL transformation, +# exactly as in Phase 3 Option A / B). + +# 8c. Swap in production. A default-branch swap is supported. +kbagent --json storage swap-tables --project ALIAS \ + --table-id in.c-foo.data \ + --target-table-id in.c-foo.data_typed \ + --branch --yes +# -> in.c-foo.data now carries the typed schema in production. + +# 8d. Smoke-test a downstream config in production, then clean up. +kbagent storage delete-table --project ALIAS --table-id in.c-foo.data_typed --yes +``` + +Two production-only cautions the rehearsal does not surface: -Branch: () -Source: in.c-foo.data (was: typeless STRING(16M); now: typed) -Sibling: in.c-foo.data_typed (was: typed empty; now: typeless rows preserved) +- **Inconsistency window.** Between 8b (copy) and 8c (swap), upstream + writers may append rows to the live `data`. Either quiesce the upstream + load for the swap window, or run a final incremental catch-up INSERT + right before the swap. The swap itself is atomic and sub-15s on Snowflake. +- **Rollback.** `data_typed` (now holding the old typeless rows) is the + rollback artifact -- re-swap to undo -- until you delete it in 8d. -Phase 2 profile summary: +Hand the user a structured summary before running 8c. Recommended shape: + +```text +TYPIFY READY TO APPLY IN PRODUCTION -- in.c-foo.data (project ALIAS) + +Rehearsal branch proved the schema is downstream-safe: rows: 1,234,567 id: STRING -> VARCHAR(40) (max observed length: 36) - name: STRING -> VARCHAR(256) (max observed length: 247) amount: STRING -> NUMBER(18,2) (max precision: 14, max scale: 2) created_at: STRING -> TIMESTAMP_NTZ (0 parse failures across 1.2M rows) is_paid: STRING -> BOOLEAN (values: 'true' (840k), 'false' (390k)) + downstream config : green pre- and post-swap in the + branch, rows_out unchanged. + +Production plan (default branch ): + 1. create in.c-foo.data_typed with the types above + 2. copy rows (quiesce writers or do a final catch-up INSERT first) + 3. swap-tables in.c-foo.data <-> in.c-foo.data_typed + 4. smoke-test , then delete in.c-foo.data_typed -Phase 4 baseline (pre-swap): - config -- ran in against typeless source - job : status=success, rows_in=1,234,567, rows_out=N - -Phase 5 swap: - storage job : operationName=tableSwap, status=success, took=12s - -Phase 6 smoke (post-swap): - config -- ran in against typed source - job : status=success, rows_in=1,234,567, rows_out=N - rows_out matches pre-swap value: YES - -Validate: - - kbagent storage table-detail --project ALIAS --table-id in.c-foo.data --branch - (column_details should show VARCHAR/NUMBER/TIMESTAMP/BOOLEAN, not STRING) - - Spot-check 5 rows: SELECT * FROM "in.c-foo.data" LIMIT 5 in workspace W_ID - -Merge: -Rollback (pre-merge): kbagent storage swap-tables --project ALIAS \ - --table-id in.c-foo.data \ - --target-table-id in.c-foo.data_typed \ - --branch --yes -After-merge cleanup: kbagent storage delete-table --project ALIAS \ - --table-id in.c-foo.data_typed --yes +Rollback (pre-cleanup): re-run swap-tables to put the typeless table back. ``` -The user reviews, clicks merge, and the typed schema lands in -production. The sibling carrying the typeless rows survives the merge -(branched-storage propagation); cleanup happens in `main` per the -note above. +The rehearsal branch is already gone (Phase 7); there is no merge step. ## Failure modes to anticipate diff --git a/pyproject.toml b/pyproject.toml index f4575c55..63a0acf7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "keboola-agent-cli" -version = "0.46.1" +version = "0.53.0" description = "AI-friendly CLI for managing Keboola projects" readme = "README.md" requires-python = ">=3.12" @@ -21,6 +21,7 @@ dependencies = [ "prompt-toolkit>=3.0", "kai-client>=0.11.0", "croniter>=2.0", + "cryptography>=46", ] [project.optional-dependencies] @@ -28,7 +29,7 @@ server = [ "fastapi>=0.115", "uvicorn[standard]>=0.30", "sse-starlette>=2.1", - "python-multipart>=0.0.9", + "python-multipart>=0.0.27", ] [project.scripts] diff --git a/src/keboola_agent_cli/changelog.py b/src/keboola_agent_cli/changelog.py index 00b1c52d..14b58646 100644 --- a/src/keboola_agent_cli/changelog.py +++ b/src/keboola_agent_cli/changelog.py @@ -8,6 +8,58 @@ # Ordered newest-first. Each value is a list of brief one-line descriptions. CHANGELOG: dict[str, list[str]] = { + "0.53.0": [ + 'Fix (`sync pull --force`, silent baseline corruption -> data loss): a config with un-pushed local edits is no longer silently de-synced when a force-pull runs while the remote is unchanged. Repro (reported on v0.51.1, project 5785): pull a config, edit its `_config.yml` (`sync diff` correctly shows `1 to update`), then run `sync pull --force` -- typically to resolve an *unrelated* config\'s conflict. Pre-0.53.0, `--force` skipped the "locally modified" guard in `SyncService.pull()`, so for a config whose remote had not changed the `remote_unchanged` short-circuit re-stamped the manifest `pull_hash` from the *edited on-disk file*. Afterwards `sync diff` and `sync push` both reported "in sync" and a real `push` shipped nothing, while the live remote still held the old config -- the local edits were stranded with no visible signal. Root cause was an interaction of two individually-reasonable decisions: `--force` bypassing the overwrite guard, and the `diff` `local_override_hashes` optimization that skips re-reading a file whose hash matches `pull_hash` (so the edited content was never even compared). The fix splits `--force` behaviour by 3-way diff state, per the maintainer decision: (b) local edited + remote UNCHANGED -> the file AND its 3-way base (`pull_hash` + `pull_config_hash`) are preserved, so the pending delta stays visible to `sync push` (no data loss, no silent revert); (a) local edited + remote ALSO changed since the last pull (a true merge conflict) -> the pull aborts before writing anything with the new `SyncConflictError` (exit 1, error code `SYNC_CONFLICT`), listing every conflicting config/row so the user resolves it (`sync diff`, then `push` or discard, then pull again). A no-conflict force-pull (remote changed, local untouched) still takes remote as before. Applies at config and row granularity. Note: `--force` no longer discards un-pushed non-conflicting edits -- that was the dangerous behaviour; to intentionally drop local edits, delete the file (or the config dir) and pull. New: `errors.SyncConflictError` + `ErrorCode.SYNC_CONFLICT`; `SyncService._detect_force_pull_conflicts` / `_is_conflict` (read-only pre-pass that runs before any write); `commands/sync.py` catches the error and prints a red per-config conflict block (human) or a `SYNC_CONFLICT` envelope with a `details.conflicts` array (`--json`). The pull `--force` help now documents the preserve/abort semantics. `--all-projects` surfaces a per-project conflict as a structured entry (`error_code: SYNC_CONFLICT` + the `conflicts` list, matching the single-project envelope) without aborting the batch. Tests: `tests/test_sync_force_pull_baseline.py` (config + row, preserve case b, abort case a, remote-only-changed takes remote, `--all-projects` structured conflict), `tests/test_sync_cli.py` (exit 1 + human/JSON conflict envelope), and `tests/test_e2e.py::TestE2ESyncWorkflow::test_sync_force_pull_conflict_aware` (real Storage: preserve when remote unchanged, then `SYNC_CONFLICT` after a remote mutation).', + ], + "0.52.1": [ + 'Fix (docs/UX, swap-tables wording): completes the swap-tables semantics correction shipped in 0.52.0, which left four co-located surfaces still claiming the swap is dev-branch-only or "rejected on production" -- now false after that fix. The user-facing `ConfigError` raised on a missing `--branch` (exit 5) no longer says "The Storage API rejects this on production"; it now reads "swap-tables requires a branch ... Any branch works, including the default/production branch." The same stale wording was corrected in `commands-reference.md`, the `swap-tables` command docstring (which feeds the auto-generated `SKILL.md` decision table via `make skill-gen`), and the `AGENT_CONTEXT` block (`kbagent context`). A CLI test that mocked and asserted the old phantom "dev branch" error string (it passed only because the mock short-circuited the real service) was fixed to match the real message, and the `swap_tables` `Args` docstring "Dev branch ID" became "any branch accepted, including the default/production branch". No behaviour change: `branch_id` stays mandatory (the swap is branch-scoped). `clone-table` wording is intentionally untouched -- clone legitimately targets a dev branch. Surfaced by the `kbagent-pr-reviewer` self-review passes on keboola/cli#368 and keboola/cli#373.', + "Maintenance: dependency bumps merged since 0.52.0 -- `pip` 26.0.1->26.1 and `python-multipart` 0.0.26->0.0.27 (the latter on the `kbagent serve` multipart path). Build/transitive only; no API or behaviour change.", + ], + "0.52.0": [ + 'New: `kbagent storage clone-table --project P --table-id ID --branch ID [--dry-run]` -- pulls (clones) a production table into a development branch via the Storage API `POST /v2/storage/branch/{branch}/tables/{id}/pull` endpoint (operationName `devBranchTablePull`, the same call the platform issues on a branch\'s first write to a prod table). On `storage-branches` projects a dev branch reads production tables transparently (copy-on-write) until the first write, so a schema mutation in the branch -- `swap-tables`, dropping a column -- fails with a misleading "bucket not found" until the table is materialized branch-local. `clone-table` performs that materialization. The pull is one-way (default -> branch); the service refuses with exit 5 (`ConfigError`) before any HTTP call when neither `--branch` nor an active branch (via `kbagent branch use`) is set. The API returns a queued storage job which the client polls to completion before returning, mirroring `swap-tables` semantics. Permission class: `write` (creates a branch-local copy; never deletes). New layers: `KeboolaClient.pull_table`, `StorageService.clone_table`, `commands/storage.py` `clone-table`, hint `storage.clone-table`, and a 1:1 `kbagent serve` REST route (`POST /storage/tables/{project}/{table_id}/pull`). Tests: `tests/test_storage_clone.py` (13: client/service/CLI) + `tests/test_e2e.py::TestE2EStorageCloneTable` (3). Live-validated against project 10539 (storage-branches ON): clone a prod table into a dev branch -> table materialized -> in-branch `swap-tables` then succeeds (it previously failed with "bucket not found") -> production left untouched. Addresses the clone-prod-table-into-branch request in keboola/cli#362.', + 'Docs/correctness: corrected the typify workflow and `swap-tables` guidance after live verification (keboola/cli#362). (1) A dev-branch swap does NOT reach production via merge -- Keboola dev-branch merge propagates only configurations, not storage table schema (confirmed by the storage-branches design + Keboola public docs). `typify-table-workflow.md` is reworked into a two-stage model: rehearse in a dev branch (profile, build, swap, validate downstream), then repeat the real build + swap in the production (default) branch; the prior "merge promotes the typed schema to production" Phase 8 was wrong and is removed. (2) `swap-tables` does NOT "reject on production" -- a swap on the default/production branch is supported (verified live on project 10539) and is the way a typed rebuild is applied to prod. Corrected the swap docstrings (client/service), command help, hint, `context`, `gotchas.md`, and `storage-types-workflow.md`; the historical 0.28.0 changelog entry is left as-is. No code-behavior change: `branch_id` is still mandatory (the swap is branch-scoped); only the documentation was wrong.', + ], + "0.51.1": [ + "Fix (dev-portal): admin-role PATCH routing. `complexity`, `categories`, `forwardToken`, `forwardTokenDetails`, `injectEnvironment`, `processTimeout`, `requiredMemory`, `features`, and `category` are `.forbidden()` on the apps-api vendor schema (`clientAppSchema` in keboola/developer-portal:src/lib/validation.js) but settable on the admin schema. The vendor PATCH returns a misleading 422 (`Parameter complexity must be one of: easy, medium, hard`) because the enum-validation `.error()` annotation is attached on the shared admin schema before `clientAppSchema()` overrides with `.forbidden()`. `DeveloperPortalIdentity.role_hint` becomes a real validator (`vendor`/`admin`, case-folded, typos raise); `DeveloperPortalClient.patch_app` now reads the role and routes admin identities to `PATCH /admin/apps/{app}` (permissive schema); `DeveloperPortalService.prepare_patch` preflights vendor-role + admin-only-field combinations with a fail-fast error that names every offending field, explains why the 422 is misleading, and tells the user the exact command to switch identity. Admin role bypasses the preflight entirely. Reads, create, upload-icon, deprecate keep vendor-endpoint behaviour -- only PATCH has a meaningful admin variant on the server.", + "Fix (dev-portal): MFA login. The apiary spec calls `challenge` optional with default `SOFTWARE_TOKEN_MFA`, but in practice the server 404s on personal-account TOTP logins when it is omitted -- users saw `Error: Developer Portal MFA login failed (HTTP 404)` with no diagnostic body. The field is now sent explicitly. Single attempt only: an earlier experiment retried with `SMS_MFA` on the same session, but `/auth/login` consumes the session, so the retry always 404'd with `Invalid code or auth state for the user` and masked the real first failure (most often a stale 30-second TOTP code). The raised `KeboolaApiError` now includes the server response body (truncated to 500 chars) plus a hint about TOTP rotation, so users can distinguish wrong-code from stale-code from expired-session.", + "Fix (dev-portal): `--password-stdin` no longer hangs interactively. The old code did `sys.stdin.read().strip()` unconditionally, which waits for EOF (Ctrl-D) rather than for Enter -- users who pasted a password and pressed Enter were stuck until they Ctrl-C'd. The new `_read_password_stdin()` helper branches on `sys.stdin.isatty()`: TTY uses `getpass.getpass()` (hidden, line-based, Enter to confirm); pipe still does `read() -> strip()`. Both `identity add --password-stdin` and `identity edit --password-stdin` route through it. Help text updated to describe the dual-mode behaviour.", + ], + "0.51.0": [ + "New: Data Streams web UI. The `stream` command group (OTLP/HTTP sources, shipped in 0.50.0) now has a page in the kbagent web UI (`kbagent serve --ui`) under Browse -> Data Streams: list sources, create an OTLP/HTTP source (with sink auto-provisioning + if-not-exists), inspect endpoints/destination with a reveal toggle for the masked OTLP secret, and delete. Full parity with the `kbagent stream *` CLI and the `/stream/*` REST surface.", + "Fix: `stream` is now documented in the `kbagent serve` OpenAPI schema. The router was registered and callable, but its tag was missing from `OPENAPI_TAGS`, so `/docs#/stream` rendered as a bare, description-less section outside its logical Data group. A new smoke test asserts every router tag has an OpenAPI description block, so a new router can't ship invisible in `/docs` again.", + ], + "0.50.0": [ + "New: headless / token-only invocation (issue #359). Set `KBAGENT_PROJECT_FROM_ENV=1` together with `KBC_TOKEN` + `KBC_STORAGE_API_URL` and kbagent synthesizes an in-memory project under the reserved alias `__env__` -- no `kbagent project add`, no `config.json` on disk. Lets a daemon (e.g. the jasnost bridge), a container, or a CI job run any storage/job/config command with `--project __env__`, or talk to a `kbagent serve` started the same way. Both the CLI and `serve` resolve the project through the same `ConfigStore.load()` chokepoint, so both work from the single env-injection.", + "UX: stack URLs are now normalized everywhere a project is created (`project add`, `project edit --url`, and the headless `__env__` injection). A bare host (`connection.keboola.com`), a trailing slash, surrounding whitespace, or even a full project deep-link (`https://connection.keboola.com/admin/projects/10105/dashboard`) are all reduced to the clean `https://` base instead of erroring. Explicit non-https schemes (`http://`, `file://`, ...) are still rejected as an SSRF / protocol-abuse guard, and an unusable URL in `KBC_STORAGE_API_URL` now fails fast with a clean config error rather than a raw pydantic traceback.", + "Security: the env-synthesized `__env__` project lives in memory only. It is marked `ephemeral` and stripped by `ConfigStore.save()`, so the `KBC_TOKEN` from the environment is never written to `config.json`. Opt-in is explicit (the `KBAGENT_PROJECT_FROM_ENV` flag, not the mere presence of `KBC_TOKEN`) to avoid a phantom project surprising a developer who exported the token only for `project add`. If the flag is set but the credential vars are missing, the CLI fails fast with a clear error instead of silently skipping. Mutating ops that target the synthesized project (`project remove/edit/rename`, branch switch) are rejected with an actionable message rather than reporting a success that silently vanishes on the next load. `project list` recovers the `project_id` offline from the token prefix; the real project name shows via `project status` / `project info`. The org-info backfill that `project status` runs skips the ephemeral project, so even `project status` writes nothing to disk in headless mode.", + "New: `kbagent stream` command group for Keboola Data Streams (OpenTelemetry / OTLP). Provisions and introspects Stream sources from the CLI so an OTLP ingest endpoint can be created and read in one command instead of copy-pasting it out of the UI. Four subcommands: `stream list --project ALIAS [--branch N]` (sources with id/name/type/base endpoint); `stream create-source --project ALIAS --name NAME [--type otlp|http] [--branch N] [--if-not-exists] [--reveal]` (creates a source, polls the async task, returns the endpoint; `--if-not-exists` returns the existing source as `skipped`); `stream detail [SOURCE_ID | --name N] --project ALIAS [--branch N] [--reveal]` (assembles the base OTLP endpoint, per-signal endpoints `/v1/logs|/v1/traces|/v1/metrics`, protocol `http/protobuf`, and the destination bucket/tables read from the source's sinks); and `stream delete SOURCE_ID --project ALIAS [--branch N] [--dry-run] [--yes|--force]` (destructive, async task polled to completion). The Stream control-plane API lives on a separate host derived from the project's Storage URL (`connection.` -> `stream.`, same scheme as `ai.`/`queue.`) and authenticates with the per-project Storage API token (`X-StorageApi-Token`) -- no manage token, no extra prompt. The OTLP **ingestion** endpoint (`stream-in./otlp///`) embeds its secret in the URL path; it is returned by the API in `source.otlp.url` (never derived) and is **masked by default** in every surface (endpoint, per-signal endpoints, and the raw `--json` source echo) -- pass `--reveal` to print the real secret, e.g. so a daemon can wire `OTEL_EXPORTER_OTLP_ENDPOINT` from `stream detail --reveal --json`. Creating an OTLP source via the raw Stream API creates only the bare source (no sinks), so -- matching the Keboola UI -- `stream create-source --type otlp` **auto-provisions the three standard sinks** (logs/metrics/traces) into bucket `in.c-otlp-` (table per signal, mapping = `uuid id` + ingest `datetime` + a `body` column holding the full flattened OTLP record as JSON) so data actually lands in Storage; the provisioning is idempotent (only missing signals are added, so a re-run or `--if-not-exists` against a half-set-up source heals it) and `--no-sinks` opts out for a bare source. `stream detail` reports an empty destination only when no sinks exist (no invented defaults). Permission classes: `stream.list` / `stream.detail` = read, `stream.create-source` = write, `stream.delete` = destructive. New layers: `stream_client.py` (`StreamClient`, source/sink CRUD + async task polling), `services/stream_service.py` (`StreamService`, alias resolution + secret masking + detail assembly), `commands/stream.py`, and a 1:1 `kbagent serve` REST router (`server/routers/stream.py`, 4 endpoints). Tests: `tests/test_stream_client.py` (14), `tests/test_stream_service.py` (16), `tests/test_stream_cli.py` (11); full create->detail->delete E2E `tests/test_e2e.py::test_stream_otlp_e2e` (opt-in via `make test-e2e-stream`). Live-validated against project 10539: create source -> 3 sinks provisioned -> POST OTLP/HTTP JSON logs to `/v1/logs` -> 3 rows landed in `in.c-otlp-.logs` -> read back via `workspace query`. Closes keboola/cli#357.", + ], + "0.49.0": [ + "New: `kbagent dev-portal` command group — v1 operations against the Keboola Developer Portal (`apps-api.keboola.com`). Lets component developers inspect and update portal entries without leaving the terminal. Read commands (`dev-portal list --vendor V`, `dev-portal get --app VENDOR.APP_ID`) are unrestricted and support peer-config research (pull reference schemas from existing extractors/writers for design reference). Write commands (`dev-portal create`, `dev-portal patch`, `dev-portal upload-icon`, `dev-portal publish`, `dev-portal deprecate`) always print the full pending request diff and then require the user to type a random hex code on a real terminal; there is no `--yes` flag and no env-var bypass; non-TTY shells exit 6 (`EXIT_PERMISSION_DENIED`). `--dry-run` produces the same preview and exits 0 -- the agent-safe path.", + "New: multi-identity credential storage for the Developer Portal. Portal logins (email + password, with optional MFA for personal accounts) are stored per-alias in the same `config.json` as KB project tokens under 0600 protection. Identity lifecycle: `dev-portal identity add --alias A --username U [--password P | --password-stdin] [--role-hint vendor|admin] [--vendor V]`, `identity list`, `identity remove`, `identity edit`, `identity use ALIAS`, `identity current`, `identity verify`.", + "Refactor: `require_random_code_confirmation()` extracted to `commands/_helpers.py` as a single shared implementation. Used by `permissions set`, `permissions reset`, and all five Developer Portal write subcommands. Previously each call site maintained its own copy of the TTY check + prompt loop.", + "New: 15 `dev-portal.*` permission-registry operations (`dev-portal.list`, `dev-portal.get`, `dev-portal.create`, `dev-portal.patch`, `dev-portal.upload-icon`, `dev-portal.publish`, `dev-portal.deprecate`, `dev-portal.identity.add`, `dev-portal.identity.list`, `dev-portal.identity.remove`, `dev-portal.identity.edit`, `dev-portal.identity.use`, `dev-portal.identity.current`, `dev-portal.identity.verify`, `dev-portal.identity.get`) registered with appropriate risk categories in the firewall permission engine.", + ], + "0.48.0": [ + "New: `kbagent feature` command group for managing Keboola feature flags via the Manage API (requires a super-admin manage token, the same kind `org setup` uses). Seven subcommands: `feature list --project ALIAS` (the stack-wide feature catalogue, GET /manage/features); `feature project-show --project ALIAS` (features assigned to a project, read from the project object's `features` array); `feature project-add` / `feature project-remove --project ALIAS --feature NAME [--dry-run] [--yes]` (POST/DELETE /manage/projects/{id}/features); and `feature user-show` / `feature user-add` / `feature user-remove --project ALIAS --email EMAIL [--feature NAME]` for per-user features (GET/POST/DELETE /manage/users/{email}/features). The `--project ALIAS` resolves the stack URL (and, for project ops, the numeric project_id) from the kbagent config -- the alias is the only handle needed. The manage token follows the same default-deny policy as `org`: read from an interactive hidden prompt, never persisted, never a CLI argument; pass the top-level `--allow-env-manage-token` to read `KBC_MANAGE_API_TOKEN` from env (CI/CD). Write paths support `--dry-run` and an interactive confirm (skip with `--yes`). Permission classes: `list` / `*-show` = read, `*-add` = admin, `*-remove` = destructive. The Manage API has no published feature schema, so the new `Feature` model treats only `name` as stable and passes extras through unmodified; project/user `features` arrays returned as bare strings are normalised to `{name: ...}`. New layers: `ManageClient.list_features` / `add_project_feature` / `remove_project_feature` / `get_user` / `add_user_feature` / `remove_user_feature` (email + feature url-encoded in the path; the POST add paths tolerate a 204 No Content body), `FeatureService`, `commands/feature.py`, and a 1:1 `kbagent serve` REST router (`server/routers/feature.py`, 7 endpoints, each requiring the `X-Manage-Token` header). Human-mode tables are adaptive: the stack catalogue keeps Title/Type/Description, while project/user views (returned by the Manage API as bare strings) collapse to just Name. Tests: `tests/test_feature_service.py` (19), `tests/test_feature_cli.py` (21), `tests/test_manage_client.py` + `tests/test_models.py` extensions; read-only E2E `tests/test_e2e.py::test_feature_flags_read_e2e` (opt-in via `make test-e2e-feature`).", + ], + "0.47.2": [ + "Fix (`sync push`, fresh-CREATE variable binding): a transformation scaffolded alongside its sibling `keboola.variables` config + default-values row is now runnable after a single `sync push`. Three defects in the create pass are fixed together. (KFR-04) The row's `values: [...]` array was silently dropped because `local_row_to_api` only hoisted `values` into the API body when the row file already carried a `_keboola.component_id`; the two push callers now pass the known `component_id` explicitly, so a scaffold row without a `_keboola` block still hoists. (KFR-05) Rows whose parent `keboola.variables` config was created **in the same push** raised `PARENT_CONFIG_NOT_TRACKED` (or POSTed against a non-existent placeholder id): `push()` now runs in ordered phases -- configs first, then rows -- capturing each created config's placeholder id -> assigned ULID and remapping every row's `parent_config_id` to the ULID before the manifest lookup and `create_config_row(config_id=...)`. (KFR-03) The transformation's remote `configuration.variables_id` / `variables_values_id` stayed as placeholder dirnames (so `job run` failed with `Variable configuration \"\" not found`); a new Phase-C backfill resolves each placeholder to the ULID assigned this push and PUTs the corrected configuration body via `update_config` (NOT `set_variables`, which would create a second variables config), then rewrites the local `_configuration_extra` and refreshes the manifest `pull_hash` / `pull_config_hash` / `pull_extra_hashes` so a re-push is clean. When the placeholder key misses but exactly one `keboola.variables` config was created this push, it binds to that one with a warning; zero or ambiguous (>1) matches accumulate a `variable_link` error rather than writing a broken link. Downstream (FIIA) can delete its post-push `config variables-set` workaround. Tests: `tests/test_sync_config_format.py::TestLocalRowToApiComponentIdParam`, `tests/test_sync_service.py::TestFreshCreateVariableBinding` (end-to-end bindings, idempotent re-push, single-config fallback, ambiguous-config error), plus an E2E (`job run --wait` -> success) in `tests/test_e2e.py`.", + "Fix (`sync push --branch `, KFR-07): pushing the local default tree to a target dev branch no longer errors with `Config file not found`. Source (where files live on disk) and target (where the API writes) are now decoupled: when no materialized `/` subtree exists for the target branch, the default tree (`main/`) is read as the source and promoted to the target branch; all API calls still target the branch id. A new `SyncService._resolve_source_branch_path` drives the local-read path in `push` / `diff` / `_push_create` / `_push_update` / `_push_row_change`; per-config tracked reads continue to use each entry's own `branch_id`. Backward-compatible: when the per-branch subtree exists (multi-branch-directory users), behaviour is unchanged. Tests: `tests/test_sync_service.py::TestFreshCreateVariableBinding::test_resolve_source_branch_path_promotes_default_tree`.", + "Fix (docs, `sync/diff_engine.normalize_for_comparison`): corrected a stale docstring that claimed `_configuration_extra` is stripped during normalization. It is **not** stripped -- it carries real config payload (`keboola.flow` phases/tasks, a transformation's `variables_id` / `variables_values_id` links) and is part of `config_hash`. The docstring now documents that any code mutating `_configuration_extra` must refresh the stored `pull_config_hash` afterwards or `sync diff` reports a conflict. No logic change.", + ], + "0.47.1": [ + 'Fix (`storage create-table --if-not-exists`, keboola/cli#349): the `action: "skipped"` envelope now reports the EXISTING table\'s actual schema instead of re-echoing the caller\'s request. Pre-0.47.1, `columns` / `primary_key` / `name` on a skip mirrored the args the caller passed in, so a caller probing the skipped envelope to discover the real shape of a pre-existing table got the wrong values whenever the existing table differed from the request. The `get_table_detail(target_id)` lookup that already runs to confirm the table exists is now also the source of the returned schema. The caller\'s requested values are preserved under two new fields, `requested_columns` and `requested_primary_key`, and a new `schema_drift: bool` flags when the existing table diverges from the request (set comparison on columns and primary key). Human-mode output prints the actual schema on a skip and emits a `Warning:` line when `schema_drift` is true. `action: "created"` envelope is unchanged. No new flag, no signature change. Tests: `tests/test_storage_write.py` (skipped returns actual schema, drift flag set on divergence, no drift on match, human-mode warning render); `tests/test_e2e.py::TestE2E_0_47_0_NewSurfaces` extended to assert the skipped envelope reports actual columns + `requested_*` mirror.', + 'Fix (`workspace create`, keboola/cli#351): new Snowflake sandbox workspaces now request `loginType: "snowflake-person-keypair"` and generate a local RSA key pair for the Storage API `publicKey` field, so the created workspace uses the Query-Service-compatible login type instead of the backend default. The one-time creation envelope now includes `private_key` for Snowflake workspaces and keeps `password` for compatibility, usually empty on key-pair workspaces; human output prints the private key and warns that it cannot be retrieved later. BigQuery workspaces still omit `loginType` and `publicKey`. Tests cover the Storage client payload, service-layer Snowflake/BigQuery branching, CLI JSON/human output, and Snowflake E2E `private_key` presence.', + ], + "0.47.0": [ + "Fix (sync push, fresh-CREATE): pre-existing placeholder manifest entries -- the FIIA / scaffold emit pattern, where a downstream tool seeds `.keboola/manifest.json` with placeholder ids and (optionally) `KBC.configuration.*` metadata before the first push -- are now updated **in place** by the create path instead of unconditionally appended. Pre-0.47.0 every create did `manifest.configurations.append(ManifestConfiguration(...))` (and `parent.rows.append(...)` for rows), so N placeholders -> 2N manifest entries after one push, every placeholder still looked `added` on re-push (spurious duplicates on remote), and any `metadata.KBC.configuration.folderName` declared in the placeholder was silently dropped on the floor. Two new private helpers do the work: `SyncService._writeback_create_config_in_manifest(...)` finds the placeholder by `(branch_id, component_id, path)` -- branch is part of the key so a multi-branch manifest with the same logical path under two branches updates the right entry -- and refreshes its id + pull_hash / pull_config_hash while preserving every non-bookkeeping metadata key; `SyncService._writeback_create_row_in_manifest(...)` does the same for rows under their parent. Idempotency on re-push falls out for free: the now-real config id flows through the existing diff engine and the second push reports `status: no_changes, created: 0`. Tests: `tests/test_sync_service.py::TestFreshCreateWriteback` (7 cases incl. an end-to-end placeholder + KBC-metadata round-trip). Manifest contract change for downstream parsers: a single CREATE now produces a single manifest entry (not placeholder + new). Downstream tooling that has been working around the duplication by post-processing must drop that workaround. Live-validated against project 1143 / dev branch 388071: placeholder with `KBC.configuration.folderName: 'Area B E2E Folder'` -> `created=1, errors=0`, manifest length 1, folderName visible via `config metadata-list`, re-push -> `no_changes`.", + "New: `kbagent semantic-layer search-context --project P [--pattern G ...] [--type model|dataset|metric|relationship|constraint|glossary|all] [--limit N]` and `kbagent semantic-layer get-context --project P --context-id ID`. Two project-wide read subcommands that mirror the upstream `keboola-mcp-server` semantic-context tools (`search_semantic_context`, `get_semantic_context`) so downstream callers (FIIA, scheduled agents, pre-flight scripts) can drop the MCP dependency for the common 'is the model populated?' + 'what's at this id?' lookups. `search-context` is project-wide (not model-scoped); patterns are repeatable, taking the union; case-sensitive `fnmatch` against `attributes.name`; `--limit` short-circuits both inner and outer loops. `get-context` probes `semantic-model` first (model hits short-circuit on the first probe) then every `CHILD_TYPES` entry until a 200 lands; raises `NOT_FOUND` after all 6 misses; non-404 errors (500, etc.) propagate immediately rather than being swallowed. Response envelope: `{project, contexts: [{id, type, name, description, attributes}], total_count}` for search; `{project, id, type, name, description, attributes}` for get. The wire-level `\"semantic-\"` prefix is stripped from the response `type` field for CLI ergonomics (`dataset` not `semantic-dataset`). Both registered as `read` operations in the permission engine. Sync surfaces touched: `commands/semantic_layer.py`, `services/semantic_layer_service.py`, `server/routers/semantic_layer.py` (1:1 CLI->HTTP), `hints/definitions/semantic_layer.py`, `permissions.py`. Tests: `tests/test_semantic_layer_service.py::TestSearchContext` (12) + `::TestGetContext` (6); `tests/test_semantic_layer_cli.py::TestSearchContext` (4) + `::TestGetContext` (3). Live-validated against project 1143: returns 8 contexts spanning 4 types; pattern `rev_*` + `--type metric` narrows to 1 hit; round-trip search -> get-context on the returned id resolves correctly; UUID `00000000-0000-0000-0000-000000000000` returns NOT_FOUND after probing all 6 types.", + "New: `kbagent sync push --branch `, `sync pull --branch `, `sync diff --branch `. Per-invocation dev-branch override that wins over `manifest.branches[0]`, `active_branch_id` (`kbagent branch use`), and the git-branching `branch-mapping.json` -- new priority 0 in `SyncService._resolve_branch_id`. Lets an operator or downstream tool target a freshly-created dev branch without first running `branch use` or `sync branch-link`. Validated mutually exclusive with `--all-projects` at the CLI layer (branch id is per-project). Symmetric across push / pull / diff for predictable UX. Threaded through `branch_override=` kwargs on `SyncService.push / pull / diff`. Live-validated against project 1143: `sync diff --branch 388072` reports `remote_only: 31` (configs visible on the dev branch); without `--branch` the same call reports no remote diff.", + 'New: `kbagent storage create-table --if-not-exists`. Opt-in idempotency flag for parallel-worker patterns (e.g. FIIA\'s 8-worker `scaffold_storage.py`). When set, catches the specific `STORAGE_JOB_FAILED` + \'already has the same display name\' error from the Storage API, probes `get_table_detail(target_id)` to confirm the table really exists at the expected id, and returns `{action: "skipped", skip_reason: "table already exists", table_id: ...}` instead of raising. A different table with the same display name still surfaces the original error (a real conflict to resolve). Defaults to `False` so existing callers are byte-for-byte unaffected. Response envelope gains `action: "created" | "skipped"` so programmatic callers can branch on outcome. Error-code gate uses `ErrorCode.STORAGE_JOB_FAILED` (no raw string literal -- `make check-error-codes` enforces enum usage). Live-validated against project 1143 / dev branch 388072: first create -> `action: created`; second create (same name, with flag) -> `action: skipped`; third create (same name, no flag) -> original `STORAGE_JOB_FAILED` envelope.', + "New: `kbagent sync push --no-name-drift-warnings`. Opt-out flag that drops the cosmetic `name_drift_warnings` array from the result envelope when local directory names differ from the canonical kbagent naming (e.g. FIIA's `var-07-fi-daily-date-refresh` pattern). The underlying detection still runs, so flipping the flag does not lose any audit data; only the report is suppressed. Defaults to `False` so existing callers see the warnings exactly as before.", + 'Note (sync `serve` exposure): the four sync subcommands (`init`, `pull`, `push`, `diff`) remain filesystem-local and intentionally have no HTTP endpoints in `src/keboola_agent_cli/server/routers/`. The plugin-sync map permits this exemption for terminal-only / filesystem-bound commands (CONTRIBUTING.md "Plugin synchronization map"). The new `--branch` and `--no-name-drift-warnings` flags consequently also have no REST counterpart.', + ], "0.46.1": [ "Fix (plugin): the `kbagent` Claude Code skill and the `keboola-expert` subagent now surface `kbagent data-app logs` (shipped in 0.43.8). The SKILL.md `description:` trigger list gained `data-app logs, container logs, app logs, tail logs, build logs, app stdout, app stderr, troubleshoot data app, debug data app`, and the keboola-expert Tool Selection Matrix gained a row for `kbagent data-app logs --project P --app-id N [--lines N | --since ISO8601]` (0.43.8+). Before this, asking the agent for a data app's container logs fell back to the UI Terminal Log tab or the 20-line-capped `get_data_apps` MCP tool. No CLI behavior change. (#335 / #336)", "Chore (frontend dev tooling): bumped the `web/frontend` dev dependencies -- vite 5 -> 8, vitest 2 -> 4, and `@vitejs/plugin-react` 4 -> 5.2 to keep the peer range consistent with vite 8. The earlier Dependabot PRs (#337, #338) bumped only vite + vitest, which left plugin-react pinned below vite 8 and broke `npm ci` (ERESOLVE) in the Windows wheel-build job, silently shipping a UI-less wheel. No runtime change to the CLI or the bundled SPA. (#341)", diff --git a/src/keboola_agent_cli/cli.py b/src/keboola_agent_cli/cli.py index ea49f0b3..9b74f9f6 100644 --- a/src/keboola_agent_cli/cli.py +++ b/src/keboola_agent_cli/cli.py @@ -13,8 +13,10 @@ from .commands.config import config_app from .commands.context import context_command from .commands.data_app import data_app_app +from .commands.dev_portal import dev_portal_app from .commands.doctor import doctor_command from .commands.encrypt import encrypt_app +from .commands.feature import feature_app from .commands.flow import flow_app from .commands.http_client import http_app from .commands.init import init_command @@ -31,6 +33,7 @@ from .commands.serve import serve_command from .commands.sharing import sharing_app from .commands.storage import storage_app +from .commands.stream import stream_app from .commands.sync import sync_app from .commands.tool import tool_app from .commands.version import update_command, version_command @@ -49,6 +52,7 @@ from .services.deep_lineage_service import DeepLineageService from .services.doctor_service import DoctorService from .services.encrypt_service import EncryptService +from .services.feature_service import FeatureService from .services.flow_service import FlowService from .services.http_forwarder_service import HttpForwarderService from .services.job_service import JobService @@ -64,6 +68,7 @@ from .services.semantic_layer_service import SemanticLayerService from .services.sharing_service import SharingService from .services.storage_service import StorageService +from .services.stream_service import StreamService from .services.sync_service import SyncService from .services.variables_service import VariablesService from .services.version_service import VersionService @@ -91,6 +96,7 @@ _PROJ = "Project Management" app.add_typer(project_app, name="project", rich_help_panel=_PROJ) app.add_typer(org_app, name="org", rich_help_panel=_PROJ) +app.add_typer(feature_app, name="feature", rich_help_panel=_PROJ) # -- Browse & Inspect -- _BROWSE = "Browse & Inspect" @@ -105,6 +111,7 @@ app.add_typer(data_app_app, name="data-app", rich_help_panel=_BROWSE) app.add_typer(job_app, name="job", rich_help_panel=_BROWSE) app.add_typer(storage_app, name="storage", rich_help_panel=_BROWSE) +app.add_typer(stream_app, name="stream", rich_help_panel=_BROWSE) app.add_typer(sharing_app, name="sharing", rich_help_panel=_BROWSE) app.add_typer(lineage_app, name="lineage", rich_help_panel=_BROWSE) app.add_typer(kai_app, name="kai", rich_help_panel=_BROWSE) @@ -125,6 +132,7 @@ app.add_typer(semantic_layer_app, name="sl", rich_help_panel=_DEV, hidden=True) app.add_typer(http_app, name="http", rich_help_panel=_DEV) app.add_typer(agent_app, name="agent", rich_help_panel=_DEV) +app.add_typer(dev_portal_app, name="dev-portal", rich_help_panel=_DEV) def apply_firewall_flags( @@ -338,11 +346,13 @@ def main( deep_lineage_service = DeepLineageService(config_store=config_store) org_service = OrgService(config_store=config_store) member_service = MemberService(config_store=config_store) + feature_service = FeatureService(config_store=config_store) mcp_service = McpService(config_store=config_store) branch_service = BranchService(config_store=config_store) sharing_service = SharingService(config_store=config_store) search_service = SearchService(config_store=config_store) storage_service = StorageService(config_store=config_store) + stream_service = StreamService(config_store=config_store) sync_service = SyncService(config_store=config_store) variables_service = VariablesService(config_store=config_store) encrypt_service = EncryptService(config_store=config_store) @@ -398,11 +408,13 @@ def main( ctx.obj["deep_lineage_service"] = deep_lineage_service ctx.obj["org_service"] = org_service ctx.obj["member_service"] = member_service + ctx.obj["feature_service"] = feature_service ctx.obj["mcp_service"] = mcp_service ctx.obj["branch_service"] = branch_service ctx.obj["sharing_service"] = sharing_service ctx.obj["search_service"] = search_service ctx.obj["storage_service"] = storage_service + ctx.obj["stream_service"] = stream_service ctx.obj["sync_service"] = sync_service ctx.obj["variables_service"] = variables_service ctx.obj["encrypt_service"] = encrypt_service diff --git a/src/keboola_agent_cli/client.py b/src/keboola_agent_cli/client.py index 5d73813d..2504310c 100644 --- a/src/keboola_agent_cli/client.py +++ b/src/keboola_agent_cli/client.py @@ -1777,12 +1777,14 @@ def swap_tables( target_table_id: str, branch_id: int, ) -> dict[str, Any]: - """Swap two storage tables (async, waits for completion, dev branch only). + """Swap two storage tables (async, waits for completion; branch-scoped). Both tables exchange physical positions; aliases keep pointing at the same physical position and therefore expose the OTHER table's data - after the swap. The Storage API rejects this on production -- a - ``branch_id`` is mandatory. + after the swap. ``branch_id`` is mandatory (the swap is always scoped + to a branch), but ANY branch works -- including the default/production + branch. A default-branch swap is the supported way to retype a prod + table, because dev-branch merge does not propagate storage schema. The API returns a queued storage job (``operationName: tableSwap``) which this method polls to completion before returning, mirroring @@ -1804,6 +1806,36 @@ def swap_tables( response = self._request("POST", f"{prefix}/tables/{safe_id}/swap", json=body) return self._wait_for_storage_job(response.json()) + def pull_table(self, table_id: str, branch_id: int) -> dict[str, Any]: + """Pull (clone) a table from the default branch into a dev branch. + + On ``storage-branches`` projects a dev branch reads production tables + transparently (copy-on-write) until the first write. Operations that + mutate a table in the branch -- such as ``swap_tables`` or a column + drop -- require a branch-local materialization of the table first; + otherwise the Storage API reports the bucket as "not found" in the + branch. This endpoint performs that materialization: it copies the + table from the default (production) branch into the branch's isolated + storage. It is the same call the platform issues on a branch's first + write to a production table. + + The pull is one-way (default -> branch). The API returns a queued + storage job which this method polls to completion before returning, + mirroring ``swap_tables`` semantics. + + Args: + table_id: Full ID of the table to pull (e.g. "in.c-bucket.table"). + branch_id: Target development branch ID. The source is always the + default/production branch. + + Returns: + Completed storage job dict. + """ + prefix = f"/v2/storage/branch/{branch_id}" + safe_id = quote(table_id, safe="") + response = self._request("POST", f"{prefix}/tables/{safe_id}/pull") + return self._wait_for_storage_job(response.json()) + def list_tables_with_metadata(self) -> list[dict[str, Any]]: """List all storage tables with columns and metadata. @@ -2531,6 +2563,8 @@ def create_config_workspace( component_id: str, config_id: str, backend: str = "snowflake", + login_type: str | None = None, + public_key: str | None = None, ) -> dict[str, Any]: """Create a workspace tied to a specific configuration. @@ -2539,16 +2573,24 @@ def create_config_workspace( component_id: Component ID (e.g. keboola.snowflake-transformation). config_id: Configuration ID. backend: Workspace backend. + login_type: Optional Storage API loginType. Omitted when None. + public_key: Optional public key for key-pair workspaces. Omitted when None. Returns: Workspace dict including connection credentials. """ safe_component = quote(component_id, safe="") safe_config = quote(config_id, safe="") + payload: dict[str, Any] = {"backend": backend} + if login_type is not None: + payload["loginType"] = login_type + if public_key is not None: + payload["publicKey"] = public_key + response = self._request( "POST", f"/v2/storage/branch/{branch_id}/components/{safe_component}/configs/{safe_config}/workspaces", - json={"backend": backend}, + json=payload, ) return response.json() diff --git a/src/keboola_agent_cli/commands/_helpers.py b/src/keboola_agent_cli/commands/_helpers.py index c89e1e6c..d1d21c4e 100644 --- a/src/keboola_agent_cli/commands/_helpers.py +++ b/src/keboola_agent_cli/commands/_helpers.py @@ -9,6 +9,7 @@ """ import os +import secrets import sys from typing import Any @@ -391,3 +392,75 @@ def _resolve_hint_stack_url( pass return None + + +_CONFIRM_CODE_LENGTH = 4 + + +def require_random_code_confirmation(action_description: str) -> None: + """Require the user to type a random hex code to confirm a high-risk action. + + Prevents AI agents from programmatically approving production-affecting + writes (Developer Portal updates, permission policy changes). The agent + cannot predict the code and cannot type it into stdin. + + Behaviour: + - No TTY -> raise typer.Exit(EXIT_PERMISSION_DENIED). + - TTY + correct code -> return None (caller proceeds). + - TTY + wrong code / EOF / interrupt -> raise typer.Exit(EXIT_PERMISSION_DENIED). + + Args: + action_description: Short verb phrase shown in the prompt + (e.g. "patch keboola.ex-foo", "update permission policy"). + """ + is_tty = hasattr(sys.stdin, "isatty") and sys.stdin.isatty() + if not is_tty: + sys.stderr.write( + f"\nRefusing to {action_description}: this action requires a " + "real terminal so a human can type the confirmation code. " + "There is no --yes bypass by design.\n" + ) + raise typer.Exit(code=EXIT_PERMISSION_DENIED) + + code = secrets.token_hex(_CONFIRM_CODE_LENGTH) + sys.stderr.write(f"\nTo {action_description}, type this code: {code}\n") + sys.stderr.write("Confirmation: ") + sys.stderr.flush() + + try: + user_input = input().strip() + except (EOFError, KeyboardInterrupt): + raise typer.Exit(code=EXIT_PERMISSION_DENIED) from None + + if user_input != code: + sys.stderr.write("Confirmation failed. Aborting.\n") + raise typer.Exit(code=EXIT_PERMISSION_DENIED) + + +def resolve_identity_alias(ctx: typer.Context, explicit: str | None) -> str: + """Resolve the dev-portal identity alias for this invocation. + + Order: explicit --identity flag > default from config > error. + """ + if explicit: + return explicit + config_store: ConfigStore = get_service(ctx, "config_store") + default = config_store.load().default_dev_portal_identity + if not default: + raise typer.BadParameter( + "No Developer Portal identity selected. Pass --identity , " + "or set a default via `kbagent dev-portal identity use `." + ) + return default + + +def get_dev_portal_service(ctx: typer.Context): + """Build a DeveloperPortalService bound to the current ConfigStore.""" + from ..dev_portal_client import DeveloperPortalClient + from ..services.dev_portal_service import DeveloperPortalService + + config_store: ConfigStore = get_service(ctx, "config_store") + return DeveloperPortalService( + config_store=config_store, + client_factory=lambda identity: DeveloperPortalClient(identity), + ) diff --git a/src/keboola_agent_cli/commands/_semantic_layer_reference_data.py b/src/keboola_agent_cli/commands/_semantic_layer_reference_data.py new file mode 100644 index 00000000..cce9d3a3 --- /dev/null +++ b/src/keboola_agent_cli/commands/_semantic_layer_reference_data.py @@ -0,0 +1,263 @@ +"""Typer sub-app for ``kbagent semantic-layer reference-data``. + +Reference data = dimension-member records in the metastore +(``semantic-reference-data``): one record per dimension, holding the full +member list in a ``members[]`` array. The driving use case is a Chart of +Accounts (the account list + all attributes) held in the semantic layer +instead of a hardcoded Storage table. + +Deliberately self-contained: reference-data is NOT AI-generated and is kept +out of ``build`` / ``export`` / ``diff`` / cascade. The four leaves here +(``list`` / ``get`` / ``set`` / ``delete``) compose the generic metastore +verbs in :class:`SemanticLayerService`. +""" + +from __future__ import annotations + +import json +import sys +from pathlib import Path +from typing import Any + +import typer +from rich.console import Console +from rich.table import Table + +from ..errors import ErrorCode +from ._helpers import ( + check_cli_permission, + emit_hint, + get_formatter, + get_service, + should_hint, +) +from ._semantic_layer_helpers import _handle_service_call, _is_stdin_tty + +reference_data_app = typer.Typer( + name="reference-data", + help=( + "Manage reference / dimension-member records (e.g. a Chart of " + "Accounts) held in the metastore: list / get / set / delete." + ), + no_args_is_help=True, +) + + +@reference_data_app.callback(invoke_without_command=True) +def _reference_data_permission_check(ctx: typer.Context) -> None: + """Per-leaf permission check for the ``reference-data`` sub-app. + + ``check_cli_permission`` composes ``semantic-layer.reference-data.{leaf}`` + so ``list`` / ``get`` stay ``read`` while ``set`` is ``write`` and + ``delete`` is ``destructive`` (see permissions.OPERATION_REGISTRY). + """ + check_cli_permission(ctx, "semantic-layer.reference-data") + + +def _print_reference_data_table(console: Console, data: dict) -> None: + project = data.get("project", "") + records = data.get("reference_data", []) + if not records: + console.print(f"[dim]No reference-data records in project '{project}'.[/dim]") + return + table = Table(title=f"Reference data in '{project}'") + table.add_column("Dimension", style="bold cyan") + table.add_column("UUID", style="dim") + table.add_column("Members", justify="right") + table.add_column("Dataset", max_width=40) + for r in records: + table.add_row( + r.get("dimension_name", ""), + r.get("id", ""), + str(r.get("member_count", 0)), + r.get("dataset_id") or "", + ) + console.print(table) + + +def _print_reference_data_detail(console: Console, data: dict) -> None: + console.print( + f"[bold]{data.get('dimension_name', '')}[/bold] " + f"([dim]{data.get('id', '')}[/dim]) — " + f"{data.get('member_count', 0)} members, rev {data.get('revision')}" + ) + if data.get("dataset_id"): + console.print(f" dataset: {data['dataset_id']}") + members = data.get("members") or [] + preview = members[:10] + for m in preview: + key = m.get("account_code") or m.get("code") or "?" + name = m.get("account_name") or m.get("name") or "" + console.print(f" · [cyan]{key}[/cyan] {name}") + if len(members) > len(preview): + console.print(f" [dim]… and {len(members) - len(preview)} more[/dim]") + + +def _load_members(formatter: Any, members_file: str) -> list[dict]: + """Read a JSON array of member objects from a file or ``-`` (stdin).""" + try: + raw = sys.stdin.read() if members_file == "-" else Path(members_file).read_text() + except OSError as exc: + formatter.error( + message=f"Could not read members file {members_file!r}: {exc}", + error_code=ErrorCode.VALIDATION_ERROR, + ) + raise typer.Exit(code=2) from exc + try: + parsed = json.loads(raw) + except json.JSONDecodeError as exc: + formatter.error( + message=f"Members file is not valid JSON: {exc}", + error_code=ErrorCode.VALIDATION_ERROR, + ) + raise typer.Exit(code=2) from exc + if not isinstance(parsed, list): + formatter.error( + message="Members file must contain a JSON array of member objects.", + error_code=ErrorCode.VALIDATION_ERROR, + ) + raise typer.Exit(code=2) + return parsed + + +@reference_data_app.command("list") +def reference_data_list( + ctx: typer.Context, + project: str = typer.Option(..., "--project", help="Project alias"), + model: str | None = typer.Option(None, "--model", help="Filter to one model (name or UUID)"), +) -> None: + """List reference-data records (dimension summaries; use ``get`` for members).""" + if should_hint(ctx): + emit_hint(ctx, "semantic-layer.reference-data.list", project=project, model=model) + return + formatter = get_formatter(ctx) + service = get_service(ctx, "semantic_layer_service") + result = _handle_service_call( + ctx, service.list_reference_data, alias=project, model_name_or_uuid=model + ) + formatter.output(result, _print_reference_data_table) + + +@reference_data_app.command("get") +def reference_data_get( + ctx: typer.Context, + project: str = typer.Option(..., "--project", help="Project alias"), + id_: str | None = typer.Option(None, "--id", help="Record UUID"), + model: str | None = typer.Option(None, "--model", help="Model name or UUID"), + dimension: str | None = typer.Option( + None, "--dimension", help="Dimension name (with --model, instead of --id)" + ), +) -> None: + """Fetch one record (all members) by ``--id`` or by ``--model`` + ``--dimension``.""" + if should_hint(ctx): + emit_hint( + ctx, + "semantic-layer.reference-data.get", + project=project, + id=id_, + model=model, + dimension=dimension, + ) + return + formatter = get_formatter(ctx) + service = get_service(ctx, "semantic_layer_service") + result = _handle_service_call( + ctx, + service.get_reference_data, + alias=project, + record_id=id_, + model_name_or_uuid=model, + dimension=dimension, + ) + formatter.output(result, _print_reference_data_detail) + + +@reference_data_app.command("set") +def reference_data_set( + ctx: typer.Context, + project: str = typer.Option(..., "--project", help="Project alias"), + model: str | None = typer.Option(None, "--model", help="Model name or UUID"), + dimension: str = typer.Option( + ..., "--dimension", help="Dimension name, e.g. 'chart_of_accounts'" + ), + members_file: str = typer.Option( + ..., + "--members-file", + help="Path to a JSON array of member objects ('-' reads stdin).", + ), + dataset_id: str | None = typer.Option( + None, "--dataset-id", help="Optional tableId of the descriptive dataset (e.g. DIM_COA)" + ), + description: str | None = typer.Option(None, "--description", help="Optional description"), +) -> None: + """Create or replace (by model + dimension) a reference-data record. + + Idempotent: an existing record for the same model + dimension is replaced + in place (revision increments); otherwise a new record is created. + """ + if should_hint(ctx): + emit_hint( + ctx, + "semantic-layer.reference-data.set", + project=project, + model=model, + dimension=dimension, + members_file=members_file, + dataset_id=dataset_id, + description=description, + ) + return + formatter = get_formatter(ctx) + service = get_service(ctx, "semantic_layer_service") + members = _load_members(formatter, members_file) + result = _handle_service_call( + ctx, + service.set_reference_data, + alias=project, + model_name_or_uuid=model, + dimension=dimension, + members=members, + dataset_id=dataset_id, + description=description, + ) + formatter.output( + result, + lambda c, d: c.print( + f"[bold green]{d.get('action', 'set').capitalize()}[/bold green] reference data " + f"[cyan]{d.get('dimension_name', '')}[/cyan] " + f"({d.get('member_count', 0)} members, [dim]{d.get('id', '')}[/dim])" + ), + ) + + +@reference_data_app.command("delete") +def reference_data_delete( + ctx: typer.Context, + project: str = typer.Option(..., "--project", help="Project alias"), + id_: str = typer.Option(..., "--id", help="Record UUID"), + yes: bool = typer.Option(False, "--yes", "-y", help="Skip the confirm prompt"), +) -> None: + """Delete a reference-data record by UUID (server-side soft-delete).""" + if should_hint(ctx): + emit_hint(ctx, "semantic-layer.reference-data.delete", project=project, id=id_) + return + formatter = get_formatter(ctx) + service = get_service(ctx, "semantic_layer_service") + if not yes: + if not _is_stdin_tty(): + formatter.error( + message=f"Refusing to delete reference-data {id_!r} non-interactively without --yes.", + error_code=ErrorCode.VALIDATION_ERROR, + ) + raise typer.Exit(code=2) + if not formatter.json_mode and not typer.confirm(f"Delete reference-data record '{id_}'?"): + formatter.console.print("Aborted.") + raise typer.Exit(code=0) + result = _handle_service_call(ctx, service.delete_reference_data, alias=project, record_id=id_) + formatter.output( + result, + lambda c, d: c.print( + f"[bold green]Removed reference data[/bold green] " + f"[cyan]{d['removed']['dimension_name']}[/cyan] ([dim]{d['removed']['id']}[/dim])" + ), + ) diff --git a/src/keboola_agent_cli/commands/context.py b/src/keboola_agent_cli/commands/context.py index b2431af6..3ca08567 100644 --- a/src/keboola_agent_cli/commands/context.py +++ b/src/keboola_agent_cli/commands/context.py @@ -342,8 +342,15 @@ time. Response includes `legacy_branch_storage: true` and human mode prints a warning when this applies. See storage-types-workflow.md. - kbagent storage create-table --project NAME --bucket-id BUCKET_ID --name TABLE_NAME --column col:TYPE[(length)] [...] [--primary-key COL] [--not-null COL ...] [--default NAME=VALUE ...] [--branch ID] + kbagent storage create-table --project NAME --bucket-id BUCKET_ID --name TABLE_NAME --column col:TYPE[(length)] [...] [--primary-key COL] [--not-null COL ...] [--default NAME=VALUE ...] [--branch ID] [--if-not-exists] Create a typed table. --column repeatable. + - --if-not-exists (since 0.47.0): opt-in idempotency. On a duplicate-display-name failure, + probe get-table-detail at the expected id and, if the table really exists, return + `action: "skipped", skip_reason: "table already exists"` instead of raising. A different + table with the same display name still surfaces the original error. Safe for parallel workers. + Since 0.47.1: the skipped envelope reports the EXISTING table's actual `columns`/`primary_key`/`name` + (not the request); requested values are mirrored under `requested_columns`/`requested_primary_key`, + and `schema_drift: true` flags when the existing table diverges from what was requested. - Base types: STRING, INTEGER, NUMERIC, FLOAT, BOOLEAN, DATE, TIMESTAMP. Type defaults to STRING if omitted. - Native backend types with length pass through to the Storage API: VARCHAR(40), NUMBER(18,2), CHAR(10), TIMESTAMP_TZ, TIMESTAMP_NTZ, VARIANT, OBJECT, ARRAY, etc. The API validates type/length per backend; e.g. INTEGER(10) is rejected with "'10' is not valid length for INTEGER". @@ -388,11 +395,19 @@ Delete one or more buckets. --force cascade-deletes tables. Linked/shared buckets protected. Branch-aware. kbagent storage swap-tables --project NAME --table-id ID --target-table-id ID --branch ID [--dry-run] [--yes] - Swap two storage tables in a dev branch (POST /tables/{id}/swap). Both tables exchange physical positions; + Swap two storage tables in any branch, including the default/production branch (POST /tables/{id}/swap). Both tables exchange physical positions; aliases are NOT transferred (they keep pointing at the same physical position and therefore expose the OTHER table's data after the swap). Use to promote a typed rebuild back into the original name without - touching downstream config references. Storage API rejects this on production: --branch (or active branch - via 'kbagent branch use') is mandatory. Service guards before any HTTP call when no branch is set. + touching downstream config references. branch_id is mandatory (--branch or active branch via 'kbagent + branch use'); service guards before any HTTP call when none is set. Any branch works, INCLUDING the + default/production branch -- a default-branch swap is how a typed rebuild reaches prod (dev-branch merge + does not carry storage schema). + + kbagent storage clone-table --project NAME --table-id ID --branch ID [--dry-run] + Clone (pull) a production table into a dev branch (POST /tables/{id}/pull). On storage-branches projects a + dev branch reads prod tables transparently until first write, so mutating a table's schema in the branch + (swap-tables, dropping columns) first needs a branch-local copy. This materializes that copy (one-way: + default -> branch). Branch is mandatory; service guards before any HTTP call when no branch is set. ### Storage Descriptions @@ -444,6 +459,27 @@ is a directory that will hold one .parquet file per slice plus _manifest.json. Default parquet directory: ./{{project}}/{{table_id}}.parquet/ (mirrors Keboola addressing). +### Data Streams (OpenTelemetry / OTLP) + + kbagent stream list --project NAME [--branch ID] + List Data Streams sources (id, name, type, secret-free base endpoint). + kbagent stream create-source --project NAME --name NAME [--type otlp|http] [--branch ID] [--if-not-exists] [--no-sinks] [--reveal] + Create an OTLP (default) or HTTP source; polls the async task and returns the endpoint. + For OTLP, auto-provisions the logs/metrics/traces sinks (bucket in.c-otlp-) so data + lands in Storage (idempotent; --no-sinks for a bare source). + --if-not-exists returns an existing same-named source as status=skipped. + kbagent stream detail [SOURCE_ID | --name NAME] --project NAME [--branch ID] [--reveal] + Show base + per-signal endpoints (/v1/logs|/v1/traces|/v1/metrics), protocol http/protobuf, + and destination bucket/tables (from sinks). The secret embedded in the OTLP URL is MASKED + by default; pass --reveal to print it (e.g. to wire OTEL_EXPORTER_OTLP_ENDPOINT). + kbagent stream delete SOURCE_ID --project NAME [--branch ID] [--dry-run] [--yes|--force] + Delete a source (destructive; async task polled to completion). + Notes: uses the per-project Storage token (no manage token). Control plane = stream. + derived from connection.. The OTLP ingest host is stream-in., returned in + source.otlp.url -- never derived. The raw Stream API does not auto-create sinks, so kbagent + provisions the 3 OTLP sinks itself on create-source --type otlp (--no-sinks to opt out). Send + OTLP/HTTP to /v1/logs|/v1/traces|/v1/metrics; data lands in in.c-otlp-.* tables. + ### Sharing (Cross-Project) kbagent sharing list [--project NAME] @@ -505,6 +541,26 @@ Default-deny since 0.29.0 -- closes the AI-exfiltration risk where subprocesses inherit the manage token via env. +### Feature Flags (since v0.48.0) + + Requires a SUPER-ADMIN Manage API token (same kind as `org setup`). Same + default-deny token policy: interactive hidden prompt by default; pass + top-level --allow-env-manage-token to read KBC_MANAGE_API_TOKEN from env. + --project resolves the stack URL (and, for project ops, the numeric + project_id) from config -- the alias is the only handle you pass. + + kbagent feature list --project ALIAS + Stack-wide feature catalogue (GET /manage/features). + kbagent feature project-show --project ALIAS + Features assigned to a project. + kbagent feature project-add --project ALIAS --feature NAME [--dry-run] [--yes] + kbagent feature project-remove --project ALIAS --feature NAME [--dry-run] [--yes] + Enable / disable a feature on a project. add=admin, remove=destructive. + kbagent feature user-show --project ALIAS --email EMAIL + kbagent feature user-add --project ALIAS --email EMAIL --feature NAME [--dry-run] [--yes] + kbagent feature user-remove --project ALIAS --email EMAIL --feature NAME [--dry-run] [--yes] + Per-user features (GET/POST/DELETE /manage/users/{{email}}/features). + ### Flows (Orchestrator + Conditional) kbagent flow list [--project NAME] [--branch ID] [--with-schedules] @@ -603,6 +659,7 @@ kbagent workspace create --project ALIAS [--name NAME] [--backend TYPE] [--ui] [--read-only/--no-read-only] Create workspace. Backend auto-detected from project (or override with --backend). Default: headless (~1s). --ui: visible in KBC UI (~15s). + Since 0.47.1, Snowflake headless creates return private_key and an empty password field; use key-pair auth. kbagent workspace list [--project NAME] [--orphaned] [--branch ID] [--qs-compatible] List workspaces. Read command: ignores active dev branch (production endpoint) with an Info banner; @@ -755,21 +812,47 @@ kbagent sync init --project ALIAS [--directory DIR] [--git-branching] [--adopt-existing] Initialize sync working directory. --git-branching enables git-to-Keboola branch mapping. - kbagent sync pull --project ALIAS [--all-projects] [--force] [--dry-run] [--with-samples] [--no-storage] [--no-jobs] [--job-limit N] + kbagent sync pull --project ALIAS [--all-projects] [--force] [--dry-run] [--with-samples] [--no-storage] [--no-jobs] [--job-limit N] [--branch ID] Download configs as local files. Idempotent, protects local modifications. + --force (semantics corrected since 0.53.0): re-pull over locally-modified configs. + A config edited locally whose remote is UNCHANGED is PRESERVED (its pending delta stays + pushable -- NOT discarded, NOT silently re-stamped). A true merge conflict (the config + changed BOTH locally and on the remote since the last pull) ABORTS the pull (exit 1, + SYNC_CONFLICT) listing each conflict; resolve via sync diff then push-or-discard, then pull. + To intentionally drop local edits, delete the file/dir and pull. Applies to rows too. --job-limit controls max recent jobs per config (default 5). For large projects, automatically falls back to per-config job fetching to ensure all configs get job history. Auto-detects renamed configs and renames local directories to match (uses git mv in git repos). + --branch (since 0.47.0): per-invocation dev-branch override. Same semantics as sync push/diff. kbagent sync status [--directory DIR] Show local changes since last pull (SHA256-based). - kbagent sync diff --project ALIAS [--all-projects] [--directory DIR] + kbagent sync diff --project ALIAS [--all-projects] [--directory DIR] [--branch ID] 3-way diff: local vs pull-time snapshot vs remote. Detects conflicts. + --branch (since 0.47.0): per-invocation dev-branch override. Wins over + manifest.branches[0] / 'branch use' active branch / git-branching mapping. + Requires exactly one --project. - kbagent sync push --project ALIAS [--all-projects] [--dry-run] [--force] [--allow-plaintext-on-encrypt-failure] + kbagent sync push --project ALIAS [--all-projects] [--dry-run] [--force] [--allow-plaintext-on-encrypt-failure] [--branch ID] [--no-name-drift-warnings] Push local changes. Auto-encrypts secrets. Skips conflicts (pull first). Fails if encryption fails (plaintext secrets never pushed). Use escape hatch flag only if you know what you are doing. + Fresh-CREATE behavior (since 0.47.0): if the manifest contains a placeholder entry at + (component_id, path), the create path updates it in place (no manifest duplication) + and propagates any KBC.configuration.* metadata via set_config_metadata. Re-pushes + against the now-real config id are naturally idempotent. + Fresh-CREATE variable binding (since 0.47.2): when a keboola.variables config + its + values row are created alongside a transformation in the same push, the transformation's + variables_id / variables_values_id are rebound to the assigned ULIDs (not placeholder + dirnames), the row's values are hoisted even when the scaffold row file has no _keboola + block, and the row's placeholder parent is remapped before POST. job run then succeeds + without a post-push config variables-set step. + --branch (since 0.47.0): per-invocation dev-branch override. Same semantics as sync diff. + When no / subtree exists on disk (since 0.47.2), the local default tree + (main/) is read as the source and promoted to the target branch; API writes still target + the branch id. + --no-name-drift-warnings (since 0.47.0): suppress the cosmetic name_drift_warnings + array from the result envelope. kbagent sync branch-link --project ALIAS [--branch-id ID] [--branch-name NAME] Link git branch to Keboola dev branch. Auto-creates if needed. @@ -809,6 +892,19 @@ Show a model's entities. --type filter: dataset|metric|relationship|constraint|glossary. Without --type prints a per-type count summary. + kbagent semantic-layer search-context --project P [--pattern G ...] [--type model|dataset|metric|relationship|constraint|glossary|all] [--limit N] + (since 0.47.0) Project-wide glob search across semantic-layer entity names. + Mirrors the upstream keboola-mcp-server search_semantic_context tool so a + downstream caller can verify the model is populated without an MCP dependency. + Patterns are case-sensitive fnmatch, repeatable (union). Default pattern is "*". + Default --type is "all" (every CHILD type; "model" searches semantic models). + Returns {{project, contexts: [{{id, type, name, description, attributes}}], total_count}}. + + kbagent semantic-layer get-context --project P --context-id ID + (since 0.47.0) Single-entry fetch by id, irrespective of type. Probes model first, + then datasets/metrics/relationships/constraints/glossary in order; raises NOT_FOUND + if no type matches (exit 1). + kbagent semantic-layer validate --project P [--model M] [--deep] Basic structural checks (duplicates, dangling refs, sum-on-pct, constraint orphans, severity-suffix). --deep adds parallel Snowflake @@ -1023,6 +1119,68 @@ kbagent kai history [--project NAME] [--limit N] List recent Kai chat sessions. Default limit: 10. +### Developer Portal (since v0.49.0) + + The `dev-portal` command group talks to `apps-api.keboola.com` (the Keboola + Developer Portal) and lets component developers register and update components + without leaving the terminal. + + **Safety contract**: reads are unrestricted. Writes (`create`, `patch`, + `upload-icon`, `publish`, `deprecate`) always print the full pending request + and then require the user to type a random hex code on a real TTY. There is + no `--yes` flag and no env-var bypass; non-TTY shells exit 6. Use `--dry-run` + to get a clean exit-0 preview (the agent-safe path). + + **Identity management** -- portal logins are stored per-alias in `config.json`: + + kbagent dev-portal identity add --alias vendor-keboola \\ + --username service.keboola.xxxxx --password ... --vendor keboola \\ + --role-hint vendor # default; restricts PATCH to vendor endpoint + kbagent dev-portal identity add --alias admin-keboola \\ + --username admin@keboola.com --role-hint admin --password-stdin + kbagent dev-portal identity use vendor-keboola + + **`role_hint` is load-bearing (since v0.51.1)**: `vendor` (default) routes + `dev-portal patch` to `PATCH /vendors/{{vendor}}/apps/{{app}}` (restricted + schema); `admin` routes it to `PATCH /admin/apps/{{app}}` (permissive + schema). The admin endpoint is the **only** way to set the 9 fields + apps-api `.forbidden()`s on vendor: `complexity`, `categories`, `category`, + `features`, `forwardToken`, `forwardTokenDetails`, `injectEnvironment`, + `processTimeout`, `requiredMemory`. Sending any of those with a `vendor` + identity fails fast at preflight with the exact command to switch + identity (server-side it would have returned a misleading 422 saying + "must be one of: easy, medium, hard"; that message is a known apps-api + bug -- the field is actually `forbidden()`, not enum-validated). + + **`--password-stdin` (since v0.51.1)** works on TTY (hidden line-based + prompt, Enter to confirm) AND on a pipe (`echo $PASS | … --password-stdin`, + reads to EOF). Pre-0.51.1 the flag hung interactively because it always + waited for EOF. + + **Read commands** (unrestricted; good for peer-config research): + + kbagent --json dev-portal list --vendor keboola + List all apps for a vendor. Use for peer research: compare how existing + extractors configure uiOptions, encryption, defaultBucket, etc. + + kbagent --json dev-portal get --app keboola.ex-db-mysql + Full portal entry for one component. Pull two peers and compare. + + **Write commands** (require random-code TTY confirm; use --dry-run first): + + kbagent dev-portal create --vendor V --data FILE [--dry-run] + kbagent dev-portal patch --app VENDOR.APP_ID (--data FILE | --property KEY ...) [--dry-run] + kbagent dev-portal upload-icon --app VENDOR.APP_ID --file PATH [--dry-run] + kbagent dev-portal publish --app VENDOR.APP_ID [--dry-run] + kbagent dev-portal deprecate --app VENDOR.APP_ID [--dry-run] + + **Identity lifecycle**: + + kbagent dev-portal identity add / list / remove / edit / use / current / verify + + **Identity selection**: pass `--identity ` on any command, or set the + default with `dev-portal identity use `. + ### Utility Commands kbagent init [--from-global] @@ -1130,6 +1288,13 @@ KBC_MASTER_TOKEN_* Per-project master token (e.g. KBC_MASTER_TOKEN_PROD) KBAGENT_CONFIG_DIR Override config directory KBAGENT_PROJECT Override the pinned default project for this shell/session (beats pin, loses to --project) + KBAGENT_PROJECT_FROM_ENV Set to "1" (or true/yes/on) to synthesize an in-memory project under the + reserved alias __env__ from KBC_TOKEN + KBC_STORAGE_API_URL (since 0.50.0). + Headless / token-only mode: no `project add`, no config.json on disk. Use + `--project __env__` (or rely on it as the sole/default project). The token + lives in memory only -- it is NEVER persisted, even if a write op runs. + Works for both the CLI and `kbagent serve`. Fails fast if the flag is set + but KBC_TOKEN / KBC_STORAGE_API_URL are missing. KBAGENT_MAX_PARALLEL_WORKERS Max concurrent threads for multi-project ops (default 10, max 100) KBAGENT_AUTO_UPDATE Set to "false" to disable automatic update on startup KBAGENT_UPDATED_FROM Set to an older version to trigger "What's new" display on next run diff --git a/src/keboola_agent_cli/commands/dev_portal.py b/src/keboola_agent_cli/commands/dev_portal.py new file mode 100644 index 00000000..9e25cd17 --- /dev/null +++ b/src/keboola_agent_cli/commands/dev_portal.py @@ -0,0 +1,582 @@ +"""`kbagent dev-portal` — Developer Portal command surface. + +Identity management mirrors `kbagent project`; portal writes are gated by +`require_random_code_confirmation()` from _helpers — there is no `--yes` +bypass and no env-var override. +""" + +from __future__ import annotations + +import getpass +import sys +from typing import TYPE_CHECKING, Any + +import click +import typer + +from ..errors import ConfigError, ErrorCode, KeboolaApiError +from ..models import DeveloperPortalIdentity + +if TYPE_CHECKING: + from ..output import OutputFormatter + from ..services.dev_portal_service import PendingWrite +from ._helpers import ( + check_cli_permission, + get_dev_portal_service, + get_formatter, + map_error_to_exit_code, + resolve_identity_alias, +) + +# CLI-layer enforcement of the role_hint enum. The Pydantic validator on +# DeveloperPortalIdentity intentionally silent-downgrades unknown values to +# "vendor" for backwards compatibility with pre-0.51.1 config.json files +# that may carry arbitrary free-text strings. That tolerance is wrong at the +# CLI surface, where the user just typed a value RIGHT NOW -- a typo should +# fail loudly, not silently land as "vendor" and confuse the next operation. +# Wiring `click.Choice` here gives the Typer-level rejection (exit 2 + usage +# error) before any model construction. +_ROLE_HINT_CHOICES = ["vendor", "admin"] + +dev_portal_app = typer.Typer( + help="Keboola Developer Portal — multi-identity, production-safe writes.", + no_args_is_help=True, +) + +identity_app = typer.Typer(help="Manage Developer Portal identities (login credentials).") +dev_portal_app.add_typer(identity_app, name="identity") + + +@dev_portal_app.callback() +def _dev_portal_callback(ctx: typer.Context) -> None: + """Permission gate for `kbagent dev-portal …`.""" + check_cli_permission(ctx, "dev-portal") + + +@identity_app.callback() +def _identity_callback(ctx: typer.Context) -> None: + """Permission gate for `kbagent dev-portal identity …`.""" + check_cli_permission(ctx, "dev-portal.identity") + + +def _split_app(app: str) -> tuple[str, str]: + """Split `VENDOR.APP_ID` into (vendor, app_id).""" + if "." not in app: + raise typer.BadParameter( + f"--app must be in VENDOR.APP_ID form (e.g. keboola.ex-foo), got: {app!r}" + ) + vendor, _ = app.split(".", 1) + return vendor, app + + +def _read_password_stdin() -> str: + """Read a password from stdin. + + TTY -> getpass (hidden, line-based, Enter to confirm). + Pipe/redirected -> read to EOF, strip whitespace. + Using `sys.stdin.read()` unconditionally would hang interactively + until the user sent EOF (Ctrl-D); getpass on TTY fixes that. + """ + if sys.stdin.isatty(): + return getpass.getpass("Password: ").strip() + return sys.stdin.read().strip() + + +# ----- Identity subcommands ----- + + +@identity_app.command( + "add", help="Add a Developer Portal identity (verifies creds before persisting)." +) +def identity_add( + ctx: typer.Context, + alias: str = typer.Option(..., "--alias"), + username: str = typer.Option(..., "--username"), + password: str | None = typer.Option(None, "--password"), + password_stdin: bool = typer.Option( + False, + "--password-stdin", + help="Read password from stdin. On a TTY this is a hidden prompt (Enter to confirm); on a pipe it reads until EOF (e.g. `echo $PASS | … --password-stdin`).", + ), + role_hint: str = typer.Option( + "vendor", + "--role-hint", + click_type=click.Choice(_ROLE_HINT_CHOICES), + help="Identity role: 'vendor' (default) or 'admin'. Routes write commands to different apps-api endpoints -- admin uses PATCH /admin/apps/{app} which accepts complexity/categories/forwardToken/processTimeout/etc. that the vendor endpoint forbids.", + ), + vendor: str | None = typer.Option(None, "--vendor"), + portal_url: str = typer.Option( + "https://apps-api.keboola.com", + "--portal-url", + ), +) -> None: + formatter = get_formatter(ctx) + if password_stdin: + password = _read_password_stdin() + if not password: + raise typer.BadParameter("Pass --password or --password-stdin.") + identity = DeveloperPortalIdentity( + username=username, + password=password, + role_hint=role_hint, + vendor=vendor, + portal_url=portal_url, + ) + svc = get_dev_portal_service(ctx) + try: + svc.add_identity(alias, identity) + except (ConfigError, KeboolaApiError) as exc: + formatter.error( + message=str(exc), + error_code=getattr(exc, "error_code", ErrorCode.CONFIG_ERROR), + ) + raise typer.Exit( + code=map_error_to_exit_code(exc) if isinstance(exc, KeboolaApiError) else 5 + ) from None + formatter.output({"status": "ok", "alias": alias, "username": username}) + + +@identity_app.command("list", help="List configured Developer Portal identities.") +def identity_list(ctx: typer.Context) -> None: + formatter = get_formatter(ctx) + svc = get_dev_portal_service(ctx) + identities = svc.list_identities() + default = svc.current_identity() + rows = [ + { + "alias": alias, + "username": ident.username, + "vendor": ident.vendor or "", + "role_hint": ident.role_hint, + "portal_url": ident.portal_url, + "default": alias == default, + } + for alias, ident in identities.items() + ] + formatter.output(rows) + + +@identity_app.command("remove", help="Remove a Developer Portal identity.") +def identity_remove( + ctx: typer.Context, + alias: str = typer.Option(..., "--alias"), +) -> None: + formatter = get_formatter(ctx) + svc = get_dev_portal_service(ctx) + try: + svc.remove_identity(alias) + except ConfigError as exc: + formatter.error(message=str(exc), error_code=ErrorCode.CONFIG_ERROR) + raise typer.Exit(code=5) from None + formatter.output({"status": "ok", "removed": alias}) + + +@identity_app.command("edit", help="Edit fields on a Developer Portal identity (or rename it).") +def identity_edit( + ctx: typer.Context, + alias: str = typer.Option(..., "--alias"), + username: str | None = typer.Option(None, "--username"), + password: str | None = typer.Option(None, "--password"), + password_stdin: bool = typer.Option(False, "--password-stdin"), + role_hint: str | None = typer.Option( + None, + "--role-hint", + click_type=click.Choice(_ROLE_HINT_CHOICES), + ), + vendor: str | None = typer.Option(None, "--vendor"), + new_alias: str | None = typer.Option(None, "--new-alias"), +) -> None: + formatter = get_formatter(ctx) + svc = get_dev_portal_service(ctx) + if password_stdin: + password = _read_password_stdin() + try: + if new_alias: + svc.rename_identity(alias, new_alias) + alias = new_alias + svc.edit_identity( + alias, + username=username, + password=password, + role_hint=role_hint, + vendor=vendor, + ) + except ConfigError as exc: + formatter.error(message=str(exc), error_code=ErrorCode.CONFIG_ERROR) + raise typer.Exit(code=5) from None + formatter.output({"status": "ok", "alias": alias}) + + +@identity_app.command("use", help="Set the default Developer Portal identity.") +def identity_use( + ctx: typer.Context, + alias: str = typer.Argument(..., help="Identity alias to set as default"), +) -> None: + formatter = get_formatter(ctx) + svc = get_dev_portal_service(ctx) + try: + svc.use_identity(alias) + except ConfigError as exc: + formatter.error(message=str(exc), error_code=ErrorCode.CONFIG_ERROR) + raise typer.Exit(code=5) from None + formatter.output({"status": "ok", "default": alias}) + + +@identity_app.command("current", help="Show the alias of the default Developer Portal identity.") +def identity_current(ctx: typer.Context) -> None: + formatter = get_formatter(ctx) + svc = get_dev_portal_service(ctx) + formatter.output({"default": svc.current_identity()}) + + +@identity_app.command("verify", help="Probe a Developer Portal identity by logging in.") +def identity_verify( + ctx: typer.Context, + identity: str | None = typer.Option(None, "--identity"), +) -> None: + formatter = get_formatter(ctx) + svc = get_dev_portal_service(ctx) + alias = resolve_identity_alias(ctx, identity) + try: + info = svc.verify_identity(alias) + except KeboolaApiError as exc: + formatter.error(message=str(exc), error_code=exc.error_code) + raise typer.Exit(code=map_error_to_exit_code(exc)) from None + formatter.output({"status": "ok", **info}) + + +# ----- Read commands ----- + + +@dev_portal_app.command("list", help="List Developer Portal apps for a vendor.") +def list_apps( + ctx: typer.Context, + vendor: str = typer.Option(..., "--vendor"), + identity: str | None = typer.Option(None, "--identity"), +) -> None: + formatter = get_formatter(ctx) + svc = get_dev_portal_service(ctx) + alias = resolve_identity_alias(ctx, identity) + try: + apps = svc.list_apps(alias, vendor) + except KeboolaApiError as exc: + formatter.error(message=str(exc), error_code=exc.error_code) + raise typer.Exit(code=map_error_to_exit_code(exc)) from None + formatter.output(apps) + + +@dev_portal_app.command("get", help="Show the full Developer Portal entry for one app.") +def get_app_cmd( + ctx: typer.Context, + app: str = typer.Option(..., "--app", help="VENDOR.APP_ID, e.g. keboola.ex-foo"), + identity: str | None = typer.Option(None, "--identity"), +) -> None: + formatter = get_formatter(ctx) + svc = get_dev_portal_service(ctx) + alias = resolve_identity_alias(ctx, identity) + vendor, app_id = _split_app(app) + try: + result = svc.get_app(alias, vendor, app_id) + except KeboolaApiError as exc: + formatter.error(message=str(exc), error_code=exc.error_code) + raise typer.Exit(code=map_error_to_exit_code(exc)) from None + formatter.output(result) + + +# ----- Write commands ----- + +import json # noqa: E402 +from dataclasses import asdict # noqa: E402 +from pathlib import Path # noqa: E402 + +from ..constants import EXIT_PERMISSION_DENIED # noqa: E402 +from ._helpers import require_random_code_confirmation # noqa: E402 + + +def _assert_tty(action_description: str) -> None: + """Refuse immediately on non-TTY; called before any payload I/O. + + This is the *first* guard in every write command so that CI/CD shells + and AI agents are rejected before any file or stdin access happens. + The full random-code prompt fires later (after the preview) on TTY. + """ + is_tty = hasattr(sys.stdin, "isatty") and sys.stdin.isatty() + if not is_tty: + sys.stderr.write( + f"\nRefusing to {action_description}: this action requires a " + "real terminal so a human can type the confirmation code. " + "There is no --yes bypass by design.\n" + ) + raise typer.Exit(code=EXIT_PERMISSION_DENIED) + + +def _load_payload(data: str | None) -> dict: + if data is None: + raise typer.BadParameter("--data is required") + if data == "-": + return json.loads(sys.stdin.read()) + return json.loads(Path(data).read_text()) + + +def _render_pending(formatter: OutputFormatter, pending: PendingWrite) -> None: + """Write a stderr-only preview of the pending write.""" + from ..services.dev_portal_service import ( + PendingCreate, + PendingDeprecate, + PendingIconUpload, + PendingPatch, + PendingPublish, + ) + + err = formatter.err_console + if isinstance(pending, PendingPatch): + err.print(f"[bold]PATCH[/bold] /vendors/{pending.vendor}/apps/{pending.app_id}") + for d in pending.diff: + err.print(f" [yellow]{d.key}[/yellow]: {d.current!r} -> {d.new!r}") + if not pending.diff: + err.print(" [dim]no field-level changes (payload matches current state)[/dim]") + elif isinstance(pending, PendingCreate): + err.print(f"[bold]POST[/bold] /vendors/{pending.vendor}/apps") + err.print_json(json.dumps(pending.payload)) + elif isinstance(pending, PendingIconUpload): + err.print( + f"[bold]UPLOAD ICON[/bold] {pending.png_path} -> " + f"{pending.vendor}/{pending.app_id} ({len(pending.png_bytes)} bytes)" + ) + elif isinstance(pending, PendingPublish): + err.print( + f"[bold red]PUBLISH[/bold red] /vendors/{pending.vendor}/apps/" + f"{pending.app_id}/publish (requests Keboola review)" + ) + elif isinstance(pending, PendingDeprecate): + err.print( + f"[bold red]DEPRECATE[/bold red] /vendors/{pending.vendor}/apps/" + f"{pending.app_id}/deprecate (hides app, blocks new configs)" + ) + + +def _pending_as_json(pending: PendingWrite) -> dict[str, Any]: + """Serialise a pending write for --json --dry-run output.""" + raw = asdict(pending) + if "png_bytes" in raw: + raw["png_bytes"] = f"<{len(raw['png_bytes'])} bytes>" + if "png_path" in raw: + raw["png_path"] = str(raw["png_path"]) + return {"status": "dry-run", "pending": raw} + + +@dev_portal_app.command( + "create", + help="Create (register) a new app in the Developer Portal. Requires TTY confirm; --dry-run for preview.", +) +def create_cmd( + ctx: typer.Context, + vendor: str = typer.Option(..., "--vendor"), + data: str = typer.Option(..., "--data", help="Path to JSON payload, or '-' for stdin"), + identity: str | None = typer.Option(None, "--identity"), + dry_run: bool = typer.Option(False, "--dry-run"), +) -> None: + if not dry_run: + _assert_tty(f"create app in vendor '{vendor}'") + formatter = get_formatter(ctx) + svc = get_dev_portal_service(ctx) + alias = resolve_identity_alias(ctx, identity) + try: + pending = svc.prepare_create(alias, vendor, _load_payload(data)) + except KeboolaApiError as exc: + formatter.error(message=str(exc), error_code=exc.error_code) + raise typer.Exit(code=map_error_to_exit_code(exc)) from None + _render_pending(formatter, pending) + if dry_run: + formatter.output(_pending_as_json(pending)) + return + require_random_code_confirmation(f"create app in vendor '{vendor}'") + try: + result = svc.apply(pending) + except KeboolaApiError as exc: + formatter.error(message=str(exc), error_code=exc.error_code) + raise typer.Exit(code=map_error_to_exit_code(exc)) from None + formatter.output({"status": "ok", "created": result}) + + +@dev_portal_app.command( + "patch", + help="Patch one or more properties of an existing Developer Portal app. Requires TTY confirm; --dry-run for preview.", +) +def patch_cmd( + ctx: typer.Context, + app: str = typer.Option(..., "--app"), + data: str | None = typer.Option(None, "--data"), + property_: str | None = typer.Option(None, "--property"), + value: str | None = typer.Option(None, "--value"), + value_file: str | None = typer.Option(None, "--value-file"), + identity: str | None = typer.Option(None, "--identity"), + dry_run: bool = typer.Option( + False, + "--dry-run", + help=( + "Preview the diff without writing. NOTE: still logs in and GETs the " + "current app to compute the diff, so it needs portal connectivity; " + "on a personal (MFA) identity it will prompt for an MFA code. Use a " + "service.{vendor}.{id} identity for a fully non-interactive preview." + ), + ), +) -> None: + if not dry_run: + _assert_tty(f"patch {app}") + formatter = get_formatter(ctx) + svc = get_dev_portal_service(ctx) + alias = resolve_identity_alias(ctx, identity) + vendor, app_id = _split_app(app) + + if data: + payload = _load_payload(data) + elif property_: + if value_file: + raw = Path(value_file).read_text() + elif value is not None: + raw = value + else: + raise typer.BadParameter("--property requires --value or --value-file") + try: + parsed = json.loads(raw) if raw.strip()[:1] in "[{" else raw + except json.JSONDecodeError: + parsed = raw + payload = {property_: parsed} + else: + raise typer.BadParameter("Provide --data, or --property with --value/--value-file") + + try: + pending = svc.prepare_patch(alias, vendor, app_id, payload) + except KeboolaApiError as exc: + formatter.error(message=str(exc), error_code=exc.error_code) + raise typer.Exit(code=map_error_to_exit_code(exc)) from None + _render_pending(formatter, pending) + if dry_run: + formatter.output(_pending_as_json(pending)) + return + require_random_code_confirmation(f"patch {app}") + try: + svc.apply(pending) + except KeboolaApiError as exc: + formatter.error(message=str(exc), error_code=exc.error_code) + raise typer.Exit(code=map_error_to_exit_code(exc)) from None + formatter.output( + { + "status": "ok", + "app": app, + "patched_keys": [d.key for d in pending.diff], + } + ) + + +@dev_portal_app.command( + "upload-icon", + help="Upload a 128x128 PNG icon for a Developer Portal app. Requires TTY confirm; --dry-run for preview.", +) +def upload_icon_cmd( + ctx: typer.Context, + app: str = typer.Option(..., "--app"), + file: str = typer.Option(..., "--file"), + identity: str | None = typer.Option(None, "--identity"), + dry_run: bool = typer.Option(False, "--dry-run"), +) -> None: + if not dry_run: + _assert_tty(f"upload icon for {app}") + formatter = get_formatter(ctx) + svc = get_dev_portal_service(ctx) + alias = resolve_identity_alias(ctx, identity) + vendor, app_id = _split_app(app) + try: + pending = svc.prepare_upload_icon(alias, vendor, app_id, file) + except KeboolaApiError as exc: + formatter.error(message=str(exc), error_code=exc.error_code) + raise typer.Exit(code=map_error_to_exit_code(exc)) from None + _render_pending(formatter, pending) + if dry_run: + formatter.output(_pending_as_json(pending)) + return + require_random_code_confirmation(f"upload icon for {app}") + try: + result = svc.apply(pending) + except KeboolaApiError as exc: + formatter.error(message=str(exc), error_code=exc.error_code) + raise typer.Exit(code=map_error_to_exit_code(exc)) from None + formatter.output(result) + + +@dev_portal_app.command( + "publish", + help="Publish an app in the Developer Portal (requests Keboola review). Requires TTY confirm; --dry-run for preview.", +) +def publish_cmd( + ctx: typer.Context, + app: str = typer.Option(..., "--app"), + identity: str | None = typer.Option(None, "--identity"), + dry_run: bool = typer.Option( + False, + "--dry-run", + help=( + "Preview without writing. NOTE: still logs in and GETs the app to " + "run the publish pre-flight check, so it needs portal connectivity; " + "a personal (MFA) identity will prompt for an MFA code. Use a " + "service.{vendor}.{id} identity for a non-interactive preview." + ), + ), +) -> None: + if not dry_run: + _assert_tty(f"publish {app}") + formatter = get_formatter(ctx) + svc = get_dev_portal_service(ctx) + alias = resolve_identity_alias(ctx, identity) + vendor, app_id = _split_app(app) + try: + pending = svc.prepare_publish(alias, vendor, app_id) + except KeboolaApiError as exc: + formatter.error(message=str(exc), error_code=exc.error_code) + raise typer.Exit(code=map_error_to_exit_code(exc)) from None + _render_pending(formatter, pending) + if dry_run: + formatter.output(_pending_as_json(pending)) + return + require_random_code_confirmation(f"publish {app}") + try: + result = svc.apply(pending) + except KeboolaApiError as exc: + formatter.error(message=str(exc), error_code=exc.error_code) + raise typer.Exit(code=map_error_to_exit_code(exc)) from None + formatter.output({"status": "ok", "published": result}) + + +@dev_portal_app.command( + "deprecate", + help="Deprecate an app in the Developer Portal (hides it, blocks new configs). Requires TTY confirm; --dry-run for preview.", +) +def deprecate_cmd( + ctx: typer.Context, + app: str = typer.Option(..., "--app"), + identity: str | None = typer.Option(None, "--identity"), + dry_run: bool = typer.Option(False, "--dry-run"), +) -> None: + if not dry_run: + _assert_tty(f"deprecate {app}") + formatter = get_formatter(ctx) + svc = get_dev_portal_service(ctx) + alias = resolve_identity_alias(ctx, identity) + vendor, app_id = _split_app(app) + try: + pending = svc.prepare_deprecate(alias, vendor, app_id) + except KeboolaApiError as exc: + formatter.error(message=str(exc), error_code=exc.error_code) + raise typer.Exit(code=map_error_to_exit_code(exc)) from None + _render_pending(formatter, pending) + if dry_run: + formatter.output(_pending_as_json(pending)) + return + require_random_code_confirmation(f"deprecate {app}") + try: + result = svc.apply(pending) + except KeboolaApiError as exc: + formatter.error(message=str(exc), error_code=exc.error_code) + raise typer.Exit(code=map_error_to_exit_code(exc)) from None + formatter.output({"status": "ok", "deprecated": result}) diff --git a/src/keboola_agent_cli/commands/feature.py b/src/keboola_agent_cli/commands/feature.py new file mode 100644 index 00000000..729b9d80 --- /dev/null +++ b/src/keboola_agent_cli/commands/feature.py @@ -0,0 +1,311 @@ +"""Feature-flag management commands (super-admin Manage API). + +Thin CLI layer: parses arguments, calls FeatureService, formats output. +No business logic belongs here. + +All operations require a super-admin Manage API token. It is read from an +interactive hidden prompt by default (never persisted, never a CLI argument); +pass the top-level --allow-env-manage-token to read KBC_MANAGE_API_TOKEN from +env (CI/CD). See ``resolve_manage_token`` for the full default-deny policy. +""" + +from __future__ import annotations + +from typing import Any, NoReturn + +import typer +from rich.console import Console +from rich.table import Table + +from ..errors import ConfigError, ErrorCode, KeboolaApiError +from ._helpers import ( + check_cli_permission, + get_formatter, + get_service, + map_error_to_exit_code, + resolve_manage_token, +) + +feature_app = typer.Typer(help="Feature flag management (requires super-admin Manage API token)") + + +@feature_app.callback(invoke_without_command=True) +def _feature_permission_check(ctx: typer.Context) -> None: + check_cli_permission(ctx, "feature") + + +def _handle_errors(formatter: Any, exc: Exception) -> NoReturn: + """Map a ConfigError / KeboolaApiError to a structured error + Exit.""" + if isinstance(exc, ConfigError): + formatter.error(error_code=ErrorCode.CONFIG_ERROR, message=exc.message) + raise typer.Exit(code=5) from None + if isinstance(exc, KeboolaApiError): + exit_code = map_error_to_exit_code(exc) + formatter.error(error_code=exc.error_code, message=exc.message, retryable=exc.retryable) + raise typer.Exit(code=exit_code) from None + raise exc + + +def _format_feature_catalogue(console: Console, data: dict[str, Any]) -> None: + """Render the stack feature catalogue as a Rich table.""" + features = data.get("features") or [] + title = f"Features on {data.get('stack_url', '')} ({len(features)} total)" + console.print(_feature_table(title, features)) + + +# Optional feature columns, in display order: (data key, header, Rich style). +_OPTIONAL_FEATURE_COLUMNS: tuple[tuple[str, str, str | None], ...] = ( + ("title", "Title", None), + ("type", "Type", "dim"), + ("description", "Description", "dim"), +) + + +def _present_optional_columns( + features: list[dict[str, Any]], +) -> list[tuple[str, str, str | None]]: + """Return the optional columns at least one feature actually populates. + + Project/user feature arrays come back from the Manage API as bare strings + (normalised to name-only), so Title/Type/Description would be uniformly + empty -- we drop those columns rather than render dead space. The stack + catalogue, which carries metadata, keeps whichever columns it populates. + """ + return [ + col + for col in _OPTIONAL_FEATURE_COLUMNS + if any(str(feat.get(col[0], "")).strip() for feat in features) + ] + + +def _feature_table(title: str, features: list[dict[str, Any]]) -> Table: + """Build a Rich table for a feature list, omitting empty optional columns.""" + table = Table(title=title) + table.add_column("Name", style="bold cyan") + columns = _present_optional_columns(features) + for _key, header, style in columns: + table.add_column(header, style=style) + for feat in features: + row = [feat.get("name", ""), *(str(feat.get(key, "")) for key, _, _ in columns)] + table.add_row(*row) + return table + + +def _format_assigned_features(console: Console, data: dict[str, Any]) -> None: + """Render features assigned to a project or user as a Rich table.""" + features = data.get("features") or [] + owner = ( + f"project [cyan]{data.get('alias')}[/cyan] (id={data.get('project_id')})" + if "project_id" in data + else f"user [cyan]{data.get('email')}[/cyan]" + ) + if not features: + console.print(f"No features assigned to {owner}.") + return + console.print(_feature_table(f"Features assigned to {owner} ({len(features)} total)", features)) + + +def _format_write_result(console: Console, data: dict[str, Any]) -> None: + """Render the outcome of an add/remove operation.""" + status = data.get("status", "") + feature = data.get("feature", "") + target = ( + f"project [cyan]{data.get('alias')}[/cyan] (id={data.get('project_id')})" + if "project_id" in data + else f"user [cyan]{data.get('email')}[/cyan]" + ) + if status == "dry_run": + verb = "add to" if data.get("action") == "add" else "remove from" + console.print( + f"[bold yellow]DRY RUN[/bold yellow] would {verb} {target}: feature [bold]{feature}[/bold]" + ) + elif status == "added": + console.print(f"[bold green]Added[/bold green] feature [bold]{feature}[/bold] to {target}.") + elif status == "removed": + console.print(f"[bold red]Removed[/bold red] feature [bold]{feature}[/bold] from {target}.") + + +# ── Stack catalogue ─────────────────────────────────────────────────── + + +@feature_app.command("list") +def feature_list( + ctx: typer.Context, + project: str = typer.Option( + ..., "--project", "-p", help="Project alias (used to resolve the stack URL)" + ), +) -> None: + """List all feature flags defined on the stack.""" + formatter = get_formatter(ctx) + manage_token = resolve_manage_token(allow_env=ctx.obj["allow_env_manage_token"]) + service = get_service(ctx, "feature_service") + try: + result = service.list_stack_features(manage_token=manage_token, alias=project) + except (ConfigError, KeboolaApiError) as exc: + _handle_errors(formatter, exc) + formatter.output(result, _format_feature_catalogue) + + +# ── Project features ────────────────────────────────────────────────── + + +@feature_app.command("project-show") +def feature_project_show( + ctx: typer.Context, + project: str = typer.Option(..., "--project", "-p", help="Project alias"), +) -> None: + """Show feature flags assigned to a project.""" + formatter = get_formatter(ctx) + manage_token = resolve_manage_token(allow_env=ctx.obj["allow_env_manage_token"]) + service = get_service(ctx, "feature_service") + try: + result = service.list_project_features(manage_token=manage_token, alias=project) + except (ConfigError, KeboolaApiError) as exc: + _handle_errors(formatter, exc) + formatter.output(result, _format_assigned_features) + + +@feature_app.command("project-add") +def feature_project_add( + ctx: typer.Context, + project: str = typer.Option(..., "--project", "-p", help="Project alias"), + feature: str = typer.Option(..., "--feature", "-f", help="Feature name to enable"), + dry_run: bool = typer.Option(False, "--dry-run", help="Preview without making changes"), + yes: bool = typer.Option(False, "--yes", "-y", help="Skip confirmation prompt"), +) -> None: + """Enable a feature flag on a project.""" + formatter = get_formatter(ctx) + if ( + not dry_run + and not formatter.json_mode + and not yes + and not typer.confirm(f"Add feature '{feature}' to project {project}?") + ): + formatter.console.print("Aborted.") + raise typer.Exit(code=0) + manage_token = resolve_manage_token(allow_env=ctx.obj["allow_env_manage_token"]) + service = get_service(ctx, "feature_service") + try: + result = service.add_project_feature( + manage_token=manage_token, alias=project, feature=feature, dry_run=dry_run + ) + except (ConfigError, KeboolaApiError) as exc: + _handle_errors(formatter, exc) + formatter.output(result, _format_write_result) + + +@feature_app.command("project-remove") +def feature_project_remove( + ctx: typer.Context, + project: str = typer.Option(..., "--project", "-p", help="Project alias"), + feature: str = typer.Option(..., "--feature", "-f", help="Feature name to disable"), + dry_run: bool = typer.Option(False, "--dry-run", help="Preview without making changes"), + yes: bool = typer.Option(False, "--yes", "-y", help="Skip confirmation prompt"), +) -> None: + """Disable a feature flag on a project (destructive).""" + formatter = get_formatter(ctx) + if ( + not dry_run + and not formatter.json_mode + and not yes + and not typer.confirm( + f"Remove feature '{feature}' from project {project}? This is destructive." + ) + ): + formatter.console.print("Aborted.") + raise typer.Exit(code=0) + manage_token = resolve_manage_token(allow_env=ctx.obj["allow_env_manage_token"]) + service = get_service(ctx, "feature_service") + try: + result = service.remove_project_feature( + manage_token=manage_token, alias=project, feature=feature, dry_run=dry_run + ) + except (ConfigError, KeboolaApiError) as exc: + _handle_errors(formatter, exc) + formatter.output(result, _format_write_result) + + +# ── User features ───────────────────────────────────────────────────── + + +@feature_app.command("user-show") +def feature_user_show( + ctx: typer.Context, + project: str = typer.Option( + ..., "--project", "-p", help="Project alias (used to resolve the stack URL)" + ), + email: str = typer.Option(..., "--email", "-e", help="User email address"), +) -> None: + """Show feature flags assigned to a user.""" + formatter = get_formatter(ctx) + manage_token = resolve_manage_token(allow_env=ctx.obj["allow_env_manage_token"]) + service = get_service(ctx, "feature_service") + try: + result = service.list_user_features(manage_token=manage_token, alias=project, email=email) + except (ConfigError, KeboolaApiError) as exc: + _handle_errors(formatter, exc) + formatter.output(result, _format_assigned_features) + + +@feature_app.command("user-add") +def feature_user_add( + ctx: typer.Context, + project: str = typer.Option( + ..., "--project", "-p", help="Project alias (used to resolve the stack URL)" + ), + email: str = typer.Option(..., "--email", "-e", help="User email address"), + feature: str = typer.Option(..., "--feature", "-f", help="Feature name to enable"), + dry_run: bool = typer.Option(False, "--dry-run", help="Preview without making changes"), + yes: bool = typer.Option(False, "--yes", "-y", help="Skip confirmation prompt"), +) -> None: + """Enable a feature flag on a user.""" + formatter = get_formatter(ctx) + if ( + not dry_run + and not formatter.json_mode + and not yes + and not typer.confirm(f"Add feature '{feature}' to user {email}?") + ): + formatter.console.print("Aborted.") + raise typer.Exit(code=0) + manage_token = resolve_manage_token(allow_env=ctx.obj["allow_env_manage_token"]) + service = get_service(ctx, "feature_service") + try: + result = service.add_user_feature( + manage_token=manage_token, alias=project, email=email, feature=feature, dry_run=dry_run + ) + except (ConfigError, KeboolaApiError) as exc: + _handle_errors(formatter, exc) + formatter.output(result, _format_write_result) + + +@feature_app.command("user-remove") +def feature_user_remove( + ctx: typer.Context, + project: str = typer.Option( + ..., "--project", "-p", help="Project alias (used to resolve the stack URL)" + ), + email: str = typer.Option(..., "--email", "-e", help="User email address"), + feature: str = typer.Option(..., "--feature", "-f", help="Feature name to disable"), + dry_run: bool = typer.Option(False, "--dry-run", help="Preview without making changes"), + yes: bool = typer.Option(False, "--yes", "-y", help="Skip confirmation prompt"), +) -> None: + """Disable a feature flag on a user (destructive).""" + formatter = get_formatter(ctx) + if ( + not dry_run + and not formatter.json_mode + and not yes + and not typer.confirm(f"Remove feature '{feature}' from user {email}? This is destructive.") + ): + formatter.console.print("Aborted.") + raise typer.Exit(code=0) + manage_token = resolve_manage_token(allow_env=ctx.obj["allow_env_manage_token"]) + service = get_service(ctx, "feature_service") + try: + result = service.remove_user_feature( + manage_token=manage_token, alias=project, email=email, feature=feature, dry_run=dry_run + ) + except (ConfigError, KeboolaApiError) as exc: + _handle_errors(formatter, exc) + formatter.output(result, _format_write_result) diff --git a/src/keboola_agent_cli/commands/permissions.py b/src/keboola_agent_cli/commands/permissions.py index c6cd5107..923c2d17 100644 --- a/src/keboola_agent_cli/commands/permissions.py +++ b/src/keboola_agent_cli/commands/permissions.py @@ -7,8 +7,6 @@ so that an AI agent constrained by the policy cannot bypass it programmatically. """ -import secrets -import sys from typing import Any import typer @@ -20,38 +18,10 @@ from ..errors import ErrorCode from ..models import PermissionPolicy from ..permissions import PermissionEngine -from ._helpers import get_formatter, get_service +from ._helpers import get_formatter, get_service, require_random_code_confirmation permissions_app = typer.Typer(help="Manage operation permissions (firewall rules)") -# Length of the random confirmation code -_CONFIRM_CODE_LENGTH = 4 - - -def _require_interactive_confirmation(action_description: str) -> bool: - """Require the user to type a random code to confirm a destructive permission change. - - This prevents AI agents from programmatically bypassing permission policies. - The agent cannot predict the code, and cannot type it into stdin. - - Returns True if confirmed, False if cancelled or non-interactive. - """ - is_tty = hasattr(sys.stdin, "isatty") and sys.stdin.isatty() - if not is_tty: - return False - - code = secrets.token_hex(_CONFIRM_CODE_LENGTH) - sys.stderr.write(f"\nTo {action_description}, type this code: {code}\n") - sys.stderr.write("Confirmation: ") - sys.stderr.flush() - - try: - user_input = input().strip() - except (EOFError, KeyboardInterrupt): - return False - - return user_input == code - def _format_operations_table( console: Console, @@ -276,12 +246,7 @@ def permissions_set( ) raise typer.Exit(code=2) from None - if not _require_interactive_confirmation("update permission policy"): - formatter.error( - message="Confirmation failed. Permission policy not changed.", - error_code=ErrorCode.PERMISSION_DENIED, - ) - raise typer.Exit(code=EXIT_PERMISSION_DENIED) from None + require_random_code_confirmation("update permission policy") config_store: ConfigStore = get_service(ctx, "config_store") config = config_store.load() @@ -328,12 +293,7 @@ def permissions_reset( """ formatter = get_formatter(ctx) - if not _require_interactive_confirmation("remove permission policy"): - formatter.error( - message="Confirmation failed. Permission policy not changed.", - error_code=ErrorCode.PERMISSION_DENIED, - ) - raise typer.Exit(code=EXIT_PERMISSION_DENIED) from None + require_random_code_confirmation("remove permission policy") config_store: ConfigStore = get_service(ctx, "config_store") config = config_store.load() diff --git a/src/keboola_agent_cli/commands/semantic_layer.py b/src/keboola_agent_cli/commands/semantic_layer.py index 07b1ac4c..935b0d1f 100644 --- a/src/keboola_agent_cli/commands/semantic_layer.py +++ b/src/keboola_agent_cli/commands/semantic_layer.py @@ -24,6 +24,7 @@ ) from ._semantic_layer_crud import add_app, edit_app, remove_app from ._semantic_layer_helpers import _handle_service_call +from ._semantic_layer_reference_data import reference_data_app semantic_layer_app = typer.Typer( name="semantic-layer", @@ -153,6 +154,7 @@ def model_create( semantic_layer_app.add_typer(add_app, name="add") semantic_layer_app.add_typer(edit_app, name="edit") semantic_layer_app.add_typer(remove_app, name="remove") +semantic_layer_app.add_typer(reference_data_app, name="reference-data") @model_app.command("delete") @@ -917,3 +919,146 @@ def semantic_layer_validate( deep=deep, ) formatter.output(result, _print_validate) + + +# --------------------------------------------------------------------------- +# semantic-layer search-context / get-context +# +# Project-wide read surface that mirrors the upstream +# ``keboola-mcp-server`` semantic-context tools. Lets downstream callers +# (FIIA, scheduled agents) drop the MCP dependency for the common +# "is the model populated?" + "what's at this id?" lookups. +# --------------------------------------------------------------------------- + + +def _print_search_context(console: Console, data: dict) -> None: + project = data.get("project", "") + total = data.get("total_count", 0) + console.print( + f"\n[bold]Semantic contexts[/bold] in [magenta]{project}[/magenta]: " + f"{total} match{'es' if total != 1 else ''}" + ) + contexts = data.get("contexts", []) or [] + if not contexts: + console.print("[dim](no matches)[/dim]") + return + table = Table() + table.add_column("Type", style="bold cyan") + table.add_column("Name", style="bold") + table.add_column("ID") + table.add_column("Description") + for c in contexts: + table.add_row( + str(c.get("type", "")), + str(c.get("name", "")), + str(c.get("id", "")), + str(c.get("description", ""))[:60], + ) + console.print(table) + + +def _print_get_context(console: Console, data: dict) -> None: + console.print( + f"\n[bold]{data.get('type', '?')}[/bold] " + f"[cyan]{data.get('name', '')}[/cyan] " + f"([dim]{data.get('id', '')}[/dim]) in " + f"[magenta]{data.get('project', '')}[/magenta]" + ) + desc = data.get("description", "") + if desc: + console.print(f"\n{desc}\n") + attrs = data.get("attributes") or {} + if attrs: + console.print("[bold]Attributes:[/bold]") + console.print(json.dumps(attrs, indent=2, sort_keys=True, default=str)) + + +@semantic_layer_app.command("search-context") +def semantic_layer_search_context( + ctx: typer.Context, + project: str = typer.Option(..., "--project", help="Project alias"), + pattern: list[str] = typer.Option( + ["*"], + "--pattern", + help=( + "Glob pattern matched against entity name (case-sensitive " + "fnmatch). Repeatable; matches the union. Default: '*'." + ), + ), + type_filter: str = typer.Option( + "all", + "--type", + help=( + "Restrict to one type: model | dataset | metric | relationship | " + "constraint | glossary | all. Default: all (every child type)." + ), + ), + limit: int | None = typer.Option( + None, + "--limit", + help="Maximum number of results to return. Default: no cap.", + ), +) -> None: + """Search semantic-layer entities across a project by name pattern. + + Project-wide (not model-scoped). Equivalent to the upstream + ``keboola-mcp-server`` ``search_semantic_context`` tool. Use this as a + pre-flight check ("is the semantic model populated?") before kicking + off a downstream pipeline that depends on it. + """ + if should_hint(ctx): + emit_hint( + ctx, + "semantic-layer.search-context", + project=project, + pattern=pattern, + type_filter=type_filter, + limit=limit, + ) + return + formatter = get_formatter(ctx) + service = get_service(ctx, "semantic_layer_service") + result = _handle_service_call( + ctx, + service.search_context, + alias=project, + patterns=pattern, + type_filter=type_filter, + limit=limit, + ) + formatter.output(result, _print_search_context) + + +@semantic_layer_app.command("get-context") +def semantic_layer_get_context( + ctx: typer.Context, + project: str = typer.Option(..., "--project", help="Project alias"), + context_id: str = typer.Option( + ..., + "--context-id", + help="UUID of the entity to fetch (model, dataset, metric, ...).", + ), +) -> None: + """Fetch a single semantic-layer entity by id, irrespective of its type. + + Probes every type (model + datasets / metrics / relationships / + constraints / glossary) until it finds the entity, then returns the + full attribute dict. Exits 1 if no type matches. + """ + if should_hint(ctx): + emit_hint( + ctx, + "semantic-layer.get-context", + project=project, + context_id=context_id, + ) + return + formatter = get_formatter(ctx) + service = get_service(ctx, "semantic_layer_service") + result = _handle_service_call( + ctx, + service.get_context, + alias=project, + context_id=context_id, + ) + formatter.output(result, _print_get_context) diff --git a/src/keboola_agent_cli/commands/storage.py b/src/keboola_agent_cli/commands/storage.py index 1766af51..596e729a 100644 --- a/src/keboola_agent_cli/commands/storage.py +++ b/src/keboola_agent_cli/commands/storage.py @@ -586,6 +586,16 @@ def storage_create_table( "--branch", help="Dev branch ID (defaults to active branch if set via 'branch use')", ), + if_not_exists: bool = typer.Option( + False, + "--if-not-exists", + help=( + "Treat a duplicate-display-name failure as a successful no-op " + "when the table already exists at the expected id. Safe for " + "parallel workers (FIIA scaffold pattern). A different table " + "with the same display name still surfaces the original error." + ), + ), ) -> None: """Create a new storage table with typed columns. @@ -622,6 +632,7 @@ def storage_create_table( not_null=not_null, default=default, branch=branch, + if_not_exists=if_not_exists, ) formatter = get_formatter(ctx) @@ -639,6 +650,7 @@ def storage_create_table( branch_id=effective_branch, not_null_columns=not_null, defaults=default, + if_not_exists=if_not_exists, ) except ValueError as exc: formatter.error(message=str(exc), error_code=ErrorCode.INVALID_ARGUMENT) @@ -653,17 +665,35 @@ def storage_create_table( if formatter.json_mode: formatter.output(result) else: - formatter.console.print(f"[bold green]Created table:[/bold green] {result['table_id']}") - if result.get("auto_created_bucket"): + if result.get("action") == "skipped": formatter.console.print( - f" [yellow]Note:[/yellow] bucket {result['bucket_id']} was " - f"auto-materialized in this branch." + f"[bold yellow]Skipped[/bold yellow] (already exists): {result['table_id']}" ) - if result["primary_key"]: - formatter.console.print(f" Primary key: {', '.join(result['primary_key'])}") - formatter.console.print(f" Columns: {', '.join(result['columns'])}") - if result.get("legacy_branch_storage"): - formatter.console.print(_LEGACY_BRANCH_STORAGE_WARNING) + reason = result.get("skip_reason") + if reason: + formatter.console.print(f" [dim]{reason}[/dim]") + if result.get("schema_drift"): + formatter.console.print( + " [yellow]Warning:[/yellow] the existing table's schema differs " + "from the requested definition. The fields below show the ACTUAL " + "existing schema; your requested schema was not applied." + ) + if result.get("primary_key"): + formatter.console.print(f" Primary key: {', '.join(result['primary_key'])}") + if result.get("columns"): + formatter.console.print(f" Columns: {', '.join(result['columns'])}") + else: + formatter.console.print(f"[bold green]Created table:[/bold green] {result['table_id']}") + if result.get("auto_created_bucket"): + formatter.console.print( + f" [yellow]Note:[/yellow] bucket {result['bucket_id']} was " + f"auto-materialized in this branch." + ) + if result["primary_key"]: + formatter.console.print(f" Primary key: {', '.join(result['primary_key'])}") + formatter.console.print(f" Columns: {', '.join(result['columns'])}") + if result.get("legacy_branch_storage"): + formatter.console.print(_LEGACY_BRANCH_STORAGE_WARNING) @storage_app.command("upload-table", rich_help_panel=_TABLES) @@ -1278,9 +1308,10 @@ def storage_swap_tables( None, "--branch", help=( - "Dev branch ID. Required by the Storage API; defaults to the " - "active branch set via 'kbagent branch use'. Production swaps " - "are rejected by the API." + "Branch ID. Required; defaults to the active branch set via " + "'kbagent branch use'. Any branch works, including the " + "default/production branch -- a default-branch swap is how a " + "typed rebuild is applied to production." ), ), dry_run: bool = typer.Option( @@ -1295,7 +1326,7 @@ def storage_swap_tables( help="Skip confirmation prompt", ), ) -> None: - """Swap two storage tables in a development branch. + """Swap two storage tables (any branch, including the default/production branch). Both tables exchange physical positions. Aliases are NOT transferred -- they keep pointing at the same physical position and therefore expose @@ -1304,10 +1335,12 @@ def storage_swap_tables( name ("data") without touching downstream config references. \b - The Storage API restricts this to dev branches. The command resolves - the active branch from 'kbagent branch use' if --branch is omitted; - if no branch is set in either place, the call is rejected before any - HTTP call. + branch_id is mandatory (the swap is always branch-scoped): the command + resolves the active branch from 'kbagent branch use' if --branch is + omitted, and exits 5 before any HTTP call if no branch is set in either + place. Any branch works, INCLUDING the default/production branch -- a + default-branch swap is how a typed rebuild is applied to prod, since a + dev-branch merge does not carry storage schema. \b Example: @@ -1393,6 +1426,93 @@ def storage_swap_tables( ) +@storage_app.command("clone-table", rich_help_panel=_TABLES) +def storage_clone_table( + ctx: typer.Context, + project: str = typer.Option( + ..., + "--project", + help="Project alias", + ), + table_id: str = typer.Option( + ..., + "--table-id", + help="Table ID to pull into the branch (e.g. 'in.c-bucket.table')", + ), + branch: int | None = typer.Option( + None, + "--branch", + help=( + "Target dev branch ID. Required; defaults to the active branch " + "set via 'kbagent branch use'. The pull is one-way: default -> branch." + ), + ), + dry_run: bool = typer.Option( + False, + "--dry-run", + help="Show what would be pulled without executing", + ), +) -> None: + """Clone (pull) a production table into a development branch. + + On storage-branches projects a dev branch reads production tables + transparently until the first write. To mutate a table's schema in the + branch -- e.g. 'swap-tables' or dropping a column -- you first need a + branch-local copy of the production table; without it the Storage API + reports the bucket as "not found" in the branch. This materializes that + copy from the default branch (one-way: default -> branch). + + \b + Example: + kbagent branch use --project P --branch 1234 + kbagent storage clone-table --project P --table-id in.c-foo.data + kbagent storage swap-tables --project P \\ + --table-id in.c-foo.data --target-table-id in.c-foo.data_typed + """ + formatter = get_formatter(ctx) + service = get_service(ctx, "storage_service") + config_store: ConfigStore = ctx.obj["config_store"] + _, effective_branch = resolve_branch(config_store, formatter, project, branch) + + try: + result = service.clone_table( + alias=project, + table_id=table_id, + branch_id=effective_branch, + dry_run=dry_run, + ) + except ConfigError as exc: + formatter.error(message=exc.message, error_code=ErrorCode.CONFIG_ERROR) + raise typer.Exit(code=5) from None + except KeboolaApiError as exc: + exit_code = map_error_to_exit_code(exc) + formatter.error( + message=exc.message, + error_code=exc.error_code, + project=project, + retryable=exc.retryable, + ) + raise typer.Exit(code=exit_code) from None + + if dry_run: + if formatter.json_mode: + formatter.output(result) + else: + formatter.console.print( + f"[bold blue]Would clone (branch {result['branch_id']}):[/bold blue] " + f"{result['table_id']} (default -> branch)" + ) + return + + if formatter.json_mode: + formatter.output(result) + else: + formatter.console.print( + f"[bold green]Cloned:[/bold green] {result['table_id']} " + f"into branch {result['branch_id']}" + ) + + @storage_app.command("delete-bucket", rich_help_panel=_BUCKETS) def storage_delete_bucket( ctx: typer.Context, diff --git a/src/keboola_agent_cli/commands/stream.py b/src/keboola_agent_cli/commands/stream.py new file mode 100644 index 00000000..531c3ff5 --- /dev/null +++ b/src/keboola_agent_cli/commands/stream.py @@ -0,0 +1,266 @@ +"""Data Streams commands -- OTLP/HTTP source provisioning and introspection. + +Thin CLI layer for the ``kbagent stream`` command group: parses arguments, +calls :class:`StreamService`, formats output. No business logic belongs here. + +Authentication uses the per-project Storage API token that kbagent already +stores (``X-StorageApi-Token``) -- no manage token, no extra prompt. + +The OTLP endpoint embeds its secret in the URL path; ``stream detail`` / +``stream create-source`` mask it by default and only print it with ``--reveal``. +""" + +from __future__ import annotations + +from enum import StrEnum +from typing import Any, NoReturn + +import typer +from rich.console import Console +from rich.panel import Panel +from rich.table import Table + +from ..errors import ConfigError, ErrorCode, KeboolaApiError +from ._helpers import ( + check_cli_permission, + get_formatter, + get_service, + map_error_to_exit_code, +) + +stream_app = typer.Typer(help="Data Streams (OTLP) source management") + + +class SourceType(StrEnum): + """Stream source types supported by the Stream API.""" + + otlp = "otlp" + http = "http" + + +@stream_app.callback(invoke_without_command=True) +def _stream_permission_check(ctx: typer.Context) -> None: + check_cli_permission(ctx, "stream") + + +def _handle_errors(formatter: Any, exc: Exception) -> NoReturn: + """Map a ConfigError / KeboolaApiError to a structured error + Exit.""" + if isinstance(exc, ConfigError): + formatter.error(error_code=ErrorCode.CONFIG_ERROR, message=exc.message) + raise typer.Exit(code=5) from None + if isinstance(exc, KeboolaApiError): + exit_code = map_error_to_exit_code(exc) + formatter.error(error_code=exc.error_code, message=exc.message, retryable=exc.retryable) + raise typer.Exit(code=exit_code) from None + raise exc + + +# ── Human formatters ────────────────────────────────────────────────── + + +def _format_sources_table(console: Console, data: dict[str, Any]) -> None: + """Render the sources list as a Rich table.""" + sources = data.get("sources") or [] + alias = data.get("alias", "") + branch = data.get("branch_id", "") + if not sources: + console.print( + f"No Data Streams sources in project [cyan]{alias}[/cyan] (branch {branch}). " + "Create one with [bold]kbagent stream create-source[/bold]." + ) + return + table = Table(title=f"Data Streams sources -- {alias} (branch {branch})") + table.add_column("Source ID", style="bold cyan") + table.add_column("Name") + table.add_column("Type", style="dim") + table.add_column("Base Endpoint", style="dim", max_width=60) + for src in sources: + table.add_row( + src.get("source_id", ""), + src.get("name", ""), + src.get("type", ""), + src.get("base_endpoint", ""), + ) + console.print(table) + + +def _format_detail(console: Console, data: dict[str, Any]) -> None: + """Render one source's assembled detail as a Rich panel.""" + status = data.get("status") + title = f"Stream source: {data.get('source_id', '')}" + if status in ("created", "skipped"): + verb = "Created" if status == "created" else "Already exists" + title = f"{verb} -- {data.get('source_id', '')}" + + lines = [ + f"[bold]Name:[/bold] {data.get('name', '')}", + f"[bold]Type:[/bold] {data.get('type', '')}", + f"[bold]Branch:[/bold] {data.get('branch_id', '')}", + ] + if data.get("protocol"): + lines.append(f"[bold]Protocol:[/bold] {data['protocol']}") + lines.append(f"[bold]Endpoint:[/bold] {data.get('endpoint', '')}") + if not data.get("secret_revealed") and data.get("endpoint"): + lines.append(" [dim](secret masked -- pass --reveal to print the full URL)[/dim]") + + signals = data.get("signal_endpoints") or {} + if signals: + lines.append("\n[bold]Per-signal endpoints:[/bold]") + for signal, url in signals.items(): + lines.append(f" [cyan]{signal}[/cyan]: {url}") + + destination = data.get("destination") or {} + tables = destination.get("tables") or {} + if destination.get("bucket") or tables: + lines.append("\n[bold]Destination:[/bold]") + if destination.get("bucket"): + lines.append(f" bucket: {destination['bucket']}") + for signal, table_id in tables.items(): + lines.append(f" [cyan]{signal}[/cyan] -> {table_id}") + + conditions = data.get("import_conditions") + if conditions: + lines.append("\n[bold]Import conditions:[/bold]") + for key, value in conditions.items(): + lines.append(f" {key}: {value}") + + console.print(Panel("\n".join(lines), title=title, expand=False)) + + +def _format_delete_result(console: Console, data: dict[str, Any]) -> None: + """Render the outcome of a delete operation.""" + status = data.get("status", "") + source_id = data.get("source_id", "") + if status == "dry_run": + console.print( + f"[bold yellow]DRY RUN[/bold yellow] would delete source " + f"[bold]{source_id}[/bold] (branch {data.get('branch_id', '')})." + ) + elif status == "deleted": + console.print(f"[bold red]Deleted[/bold red] source [bold]{source_id}[/bold].") + + +# ── Commands ────────────────────────────────────────────────────────── + + +@stream_app.command("list") +def stream_list( + ctx: typer.Context, + project: str = typer.Option(..., "--project", "-p", help="Project alias"), + branch: str | None = typer.Option( + None, "--branch", help="Branch ref (default: the project's default branch)" + ), +) -> None: + """List Data Streams sources in a project.""" + formatter = get_formatter(ctx) + service = get_service(ctx, "stream_service") + try: + result = service.list_sources(alias=project, branch_id=branch) + except (ConfigError, KeboolaApiError) as exc: + _handle_errors(formatter, exc) + formatter.output(result, _format_sources_table) + + +@stream_app.command("create-source") +def stream_create_source( + ctx: typer.Context, + project: str = typer.Option(..., "--project", "-p", help="Project alias"), + name: str = typer.Option(..., "--name", "-n", help="Human-readable source name"), + source_type: SourceType = typer.Option( + SourceType.otlp, "--type", help="Source type (otlp | http)" + ), + branch: str | None = typer.Option( + None, "--branch", help="Branch ref (default branch if unset)" + ), + if_not_exists: bool = typer.Option( + False, "--if-not-exists", help="Return the existing source instead of failing if it exists" + ), + no_sinks: bool = typer.Option( + False, + "--no-sinks", + help="Skip auto-creating the logs/metrics/traces sinks for an OTLP source", + ), + reveal: bool = typer.Option(False, "--reveal", help="Print the full endpoint incl. secret"), +) -> None: + """Create an OTLP (or HTTP) source and return its endpoint. + + For an OTLP source the three standard sinks (logs/metrics/traces) are + auto-created so data lands in Storage (bucket in.c-otlp-); pass + --no-sinks to create a bare source without them. + """ + formatter = get_formatter(ctx) + service = get_service(ctx, "stream_service") + try: + result = service.create_source( + alias=project, + name=name, + source_type=source_type.value, + branch_id=branch, + if_not_exists=if_not_exists, + reveal=reveal, + provision_sinks=not no_sinks, + ) + except (ConfigError, KeboolaApiError) as exc: + _handle_errors(formatter, exc) + formatter.output(result, _format_detail) + + +@stream_app.command("detail") +def stream_detail( + ctx: typer.Context, + source_id: str | None = typer.Argument(None, help="Source id (or use --name)"), + project: str = typer.Option(..., "--project", "-p", help="Project alias"), + name: str | None = typer.Option(None, "--name", "-n", help="Look up the source by name"), + branch: str | None = typer.Option( + None, "--branch", help="Branch ref (default branch if unset)" + ), + reveal: bool = typer.Option(False, "--reveal", help="Print the full endpoint incl. secret"), +) -> None: + """Show a source's endpoints, protocol, and destination tables.""" + formatter = get_formatter(ctx) + service = get_service(ctx, "stream_service") + try: + result = service.get_source_detail( + alias=project, + source_id=source_id, + name=name, + branch_id=branch, + reveal=reveal, + ) + except (ConfigError, KeboolaApiError) as exc: + _handle_errors(formatter, exc) + formatter.output(result, _format_detail) + + +@stream_app.command("delete") +def stream_delete( + ctx: typer.Context, + source_id: str = typer.Argument(..., help="Source id to delete"), + project: str = typer.Option(..., "--project", "-p", help="Project alias"), + branch: str | None = typer.Option( + None, "--branch", help="Branch ref (default branch if unset)" + ), + force: bool = typer.Option(False, "--force", help="Alias for --yes (skip confirmation)"), + dry_run: bool = typer.Option(False, "--dry-run", help="Preview without deleting"), + yes: bool = typer.Option(False, "--yes", "-y", help="Skip confirmation prompt"), +) -> None: + """Delete a Data Streams source (destructive).""" + formatter = get_formatter(ctx) + if ( + not dry_run + and not formatter.json_mode + and not (yes or force) + and not typer.confirm( + f"Delete source '{source_id}' from project {project}? This is destructive." + ) + ): + formatter.console.print("Aborted.") + raise typer.Exit(code=0) + service = get_service(ctx, "stream_service") + try: + result = service.delete_source( + alias=project, source_id=source_id, branch_id=branch, dry_run=dry_run + ) + except (ConfigError, KeboolaApiError) as exc: + _handle_errors(formatter, exc) + formatter.output(result, _format_delete_result) diff --git a/src/keboola_agent_cli/commands/sync.py b/src/keboola_agent_cli/commands/sync.py index fb92a64c..8b0e782c 100644 --- a/src/keboola_agent_cli/commands/sync.py +++ b/src/keboola_agent_cli/commands/sync.py @@ -9,7 +9,7 @@ import typer -from ..errors import ConfigError, ErrorCode, KeboolaApiError +from ..errors import ConfigError, ErrorCode, KeboolaApiError, SyncConflictError from ._helpers import check_cli_permission, get_formatter, get_service, map_error_to_exit_code sync_app = typer.Typer(help="Sync project configurations with local filesystem") @@ -248,6 +248,23 @@ def _format_diff_result(formatter: Any, result: dict) -> None: formatter.console.print(f" {len(remote_only)} new remote-only config(s)") +def _format_conflict_list(formatter: Any, conflicts: list[dict[str, str]]) -> None: + """Print the per-config force-pull conflict list (human mode only).""" + if not conflicts: + return + n = len(conflicts) + formatter.console.print( + f"\n[bold red]Merge conflict:[/bold red] {n} config(s) changed BOTH " + f"locally and on the remote since the last pull:" + ) + for c in conflicts: + label = "row" if c.get("scope") == "row" else "config" + formatter.console.print( + f" [red]![/red] {c.get('component_id')}/{c.get('config_id')} " + f"[dim]({label})[/dim] {c.get('config_name', '')}" + ) + + def _format_push_result(formatter: Any, result: dict) -> None: """Format a single-project push result for human output.""" status = result.get("status", "") @@ -409,7 +426,12 @@ def sync_pull( force: bool = typer.Option( False, "--force", - help="Overwrite local files without checking for modifications", + help=( + "Force re-pull. Locally-modified configs whose remote is unchanged " + "are PRESERVED (kept as pending changes for `sync push`); a true " + "merge conflict (local AND remote both changed since the last pull) " + "aborts the pull so you can resolve it." + ), ), dry_run: bool = typer.Option( False, @@ -446,6 +468,14 @@ def sync_pull( "--max-samples", help="Max number of tables to sample (default 50)", ), + branch: int | None = typer.Option( + None, + "--branch", + help=( + "Dev branch ID. Overrides the manifest / 'branch use' active " + "branch for this single invocation. Requires exactly one --project." + ), + ), ) -> None: """Download configurations from a Keboola project to local files. @@ -467,6 +497,12 @@ def sync_pull( error_code=ErrorCode.USAGE_ERROR, ) raise typer.Exit(code=2) + if branch is not None and all_projects: + formatter.error( + message="--branch requires --project (branch id is per-project)", + error_code=ErrorCode.USAGE_ERROR, + ) + raise typer.Exit(code=2) if all_projects: base_dir = _safe_resolve_dir(directory) @@ -515,7 +551,18 @@ def sync_pull( with_samples=with_samples, sample_limit=sample_limit, max_samples=max_samples, + branch_override=branch, + ) + except SyncConflictError as exc: + if not formatter.json_mode: + _format_conflict_list(formatter, exc.conflicts) + formatter.error( + message=exc.message, + error_code=exc.error_code, + project=project or "", + details={"conflicts": exc.conflicts}, ) + raise typer.Exit(code=1) from None except FileNotFoundError as exc: formatter.error(message=str(exc), error_code=ErrorCode.NOT_INITIALIZED) raise typer.Exit(code=1) from None @@ -615,6 +662,14 @@ def sync_diff( "-d", help="Project root directory (must contain .keboola/)", ), + branch: int | None = typer.Option( + None, + "--branch", + help=( + "Dev branch ID. Overrides the manifest / 'branch use' active " + "branch for this single invocation. Requires exactly one --project." + ), + ), ) -> None: """Show detailed diff between local and remote configurations. @@ -636,6 +691,12 @@ def sync_diff( error_code=ErrorCode.USAGE_ERROR, ) raise typer.Exit(code=2) + if branch is not None and all_projects: + formatter.error( + message="--branch requires --project (branch id is per-project)", + error_code=ErrorCode.USAGE_ERROR, + ) + raise typer.Exit(code=2) if all_projects: base_dir = _safe_resolve_dir(directory) @@ -654,7 +715,7 @@ def sync_diff( project_root = _resolve_project_root(directory, project) try: - result = service.diff(alias=project, project_root=project_root) + result = service.diff(alias=project, project_root=project_root, branch_override=branch) except FileNotFoundError as exc: formatter.error(message=str(exc), error_code=ErrorCode.NOT_INITIALIZED) raise typer.Exit(code=1) from None @@ -791,6 +852,24 @@ def sync_push( "--allow-plaintext-on-encrypt-failure", help="Allow push even if secret encryption fails (DANGEROUS: secrets stored as plaintext)", ), + branch: int | None = typer.Option( + None, + "--branch", + help=( + "Dev branch ID. Overrides the manifest / 'branch use' active " + "branch for this single invocation. Requires exactly one --project. " + "When no '/' subtree exists on disk, the default tree " + "(main/) is promoted to this branch." + ), + ), + no_name_drift_warnings: bool = typer.Option( + False, + "--no-name-drift-warnings", + help=( + "Suppress the cosmetic name_drift_warnings array in the result " + "envelope (the underlying detection still runs)." + ), + ), ) -> None: """Push local configuration changes to a Keboola project. @@ -812,6 +891,12 @@ def sync_push( error_code=ErrorCode.USAGE_ERROR, ) raise typer.Exit(code=2) + if branch is not None and all_projects: + formatter.error( + message="--branch requires --project (branch id is per-project)", + error_code=ErrorCode.USAGE_ERROR, + ) + raise typer.Exit(code=2) if all_projects: base_dir = _safe_resolve_dir(directory) @@ -841,6 +926,8 @@ def sync_push( dry_run=dry_run, force=force, allow_plaintext_fallback=allow_plaintext, + branch_override=branch, + no_name_drift_warnings=no_name_drift_warnings, ) except FileNotFoundError as exc: formatter.error(message=str(exc), error_code=ErrorCode.NOT_INITIALIZED) diff --git a/src/keboola_agent_cli/commands/workspace.py b/src/keboola_agent_cli/commands/workspace.py index 72a4d39e..e6d06829 100644 --- a/src/keboola_agent_cli/commands/workspace.py +++ b/src/keboola_agent_cli/commands/workspace.py @@ -91,9 +91,15 @@ def workspace_create( c.print(f"[bold]Host:[/bold] {d['host']}"), c.print(f"[bold]Schema:[/bold] {d['schema']}"), c.print(f"[bold]User:[/bold] {d['user']}"), - c.print(f"[bold yellow]Password:[/bold yellow] {d['password']}"), c.print( - "\n[bold yellow]Warning:[/bold yellow] Save the password now -- it cannot be retrieved later!" + f"[bold yellow]Private key:[/bold yellow]\n{d['private_key']}" + if d.get("private_key") + else f"[bold yellow]Password:[/bold yellow] {d['password']}" + ), + c.print( + "\n[bold yellow]Warning:[/bold yellow] Save the private key now -- it cannot be retrieved later!" + if d.get("private_key") + else "\n[bold yellow]Warning:[/bold yellow] Save the password now -- it cannot be retrieved later!" ), ), ) @@ -645,10 +651,16 @@ def workspace_from_transformation( c.print(f"[bold]Host:[/bold] {d['host']}"), c.print(f"[bold]Schema:[/bold] {d['schema']}"), c.print(f"[bold]User:[/bold] {d['user']}"), - c.print(f"[bold yellow]Password:[/bold yellow] {d['password']}"), + c.print( + f"[bold yellow]Private key:[/bold yellow]\n{d['private_key']}" + if d.get("private_key") + else f"[bold yellow]Password:[/bold yellow] {d['password']}" + ), c.print(f"[bold]Tables loaded:[/bold] {', '.join(d.get('tables_loaded', []))}"), c.print( - "\n[bold yellow]Warning:[/bold yellow] Save the password now -- it cannot be retrieved later!" + "\n[bold yellow]Warning:[/bold yellow] Save the private key now -- it cannot be retrieved later!" + if d.get("private_key") + else "\n[bold yellow]Warning:[/bold yellow] Save the password now -- it cannot be retrieved later!" ), ), ) diff --git a/src/keboola_agent_cli/config_store.py b/src/keboola_agent_cli/config_store.py index b87b7510..c169781b 100644 --- a/src/keboola_agent_cli/config_store.py +++ b/src/keboola_agent_cli/config_store.py @@ -13,10 +13,18 @@ from pathlib import Path import platformdirs - -from .constants import ENV_CONFIG_DIR, LOCAL_CONFIG_DIR_NAME +from pydantic import ValidationError + +from .constants import ( + ENV_CONFIG_DIR, + ENV_KBC_STORAGE_API_URL, + ENV_KBC_TOKEN, + ENV_PROJECT_ALIAS, + ENV_PROJECT_FROM_ENV, + LOCAL_CONFIG_DIR_NAME, +) from .errors import ConfigError -from .models import AppConfig, ProjectConfig +from .models import AppConfig, DeveloperPortalIdentity, ProjectConfig logger = logging.getLogger(__name__) @@ -33,7 +41,9 @@ "API with retries, permission checks, and an audit trail. If you " "need a command kbagent does not cover, run `kbagent --hint client " "` to generate a KeboolaClient-based Python snippet. " - "See plugins/kbagent/skills/kbagent/SKILL.md rule 9." + "See plugins/kbagent/skills/kbagent/SKILL.md rule 9. " + "Developer Portal credentials stored here have the SAME risk profile -- " + "never call apps-api.keboola.com directly; use `kbagent dev-portal ...`." ) # File-lock constants (fcntl is POSIX-only; on Windows we skip locking). @@ -140,7 +150,7 @@ def load(self) -> AppConfig: logger.debug("Loading config from %s", self._config_path) if not self._config_path.exists(): logger.debug("Config file does not exist, returning empty config") - return AppConfig() + return self._inject_env_project(AppConfig()) fd: int | None = None try: @@ -174,10 +184,122 @@ def load(self) -> AppConfig: ) try: - return AppConfig.model_validate(data) + config = AppConfig.model_validate(data) except Exception as exc: raise ConfigError(f"Config file has invalid structure: {exc}") from exc + return self._inject_env_project(config) + + def _inject_env_project(self, config: AppConfig) -> AppConfig: + """Synthesize an in-memory project from env vars when opted in (issue #359). + + When ``KBAGENT_PROJECT_FROM_ENV`` is truthy, read ``KBC_TOKEN`` and + ``KBC_STORAGE_API_URL`` and inject a project under the reserved alias + ``__env__`` so a headless daemon / container / CI can run kbagent with + no ``project add`` and no config.json on disk. Both CLI and ``serve`` + funnel through ``load()``, so this single chokepoint covers both. + + The injected project is marked ``ephemeral=True``; ``save()`` strips it + so the env token is never persisted. Opt-in is explicit (the flag), not + the mere presence of ``KBC_TOKEN``, to avoid a phantom project on a dev + machine that exported the token only for ``project add``. + + A real project already registered under ``__env__`` is left untouched. + + Raises: + ConfigError: If the flag is set but the credential env vars are + missing (fail fast rather than silently skip). + """ + flag = os.environ.get(ENV_PROJECT_FROM_ENV, "").strip().lower() + if flag not in ("1", "true", "yes", "on"): + return config + + if ENV_PROJECT_ALIAS in config.projects: + return config + + token = os.environ.get(ENV_KBC_TOKEN) + url = os.environ.get(ENV_KBC_STORAGE_API_URL) + if not token or not url: + missing = [ + name + for name, value in ((ENV_KBC_TOKEN, token), (ENV_KBC_STORAGE_API_URL, url)) + if not value + ] + raise ConfigError( + f"{ENV_PROJECT_FROM_ENV} is set but {' and '.join(missing)} " + f"{'is' if len(missing) == 1 else 'are'} missing. Set both " + f"{ENV_KBC_TOKEN} and {ENV_KBC_STORAGE_API_URL}, or unset " + f"{ENV_PROJECT_FROM_ENV}." + ) + + # Keboola Storage tokens are `{projectId}-{tokenId}-{secret}`, so we can + # recover the project_id offline from the prefix. The real project_name + # needs an API call (verify_token) -- load() must stay offline, so it is + # left blank here; `project status` / `project info` show the verified + # name when a command actually talks to the API. + prefix = token.split("-", 1)[0] + project_id = int(prefix) if prefix.isdigit() else None + try: + config.projects[ENV_PROJECT_ALIAS] = ProjectConfig( + stack_url=url, + token=token, + project_id=project_id, + ephemeral=True, + ) + except ValidationError as exc: + # Convert pydantic's raw error into a clean fail-fast message -- + # this runs inside load(), which callers only guard for ConfigError. + reason = "; ".join(e.get("msg", "") for e in exc.errors()) or str(exc) + raise ConfigError( + f"{ENV_KBC_STORAGE_API_URL}={url!r} is not a usable stack URL: {reason}" + ) from exc + if not config.default_project: + config.default_project = ENV_PROJECT_ALIAS + logger.debug("Injected ephemeral '%s' project from environment", ENV_PROJECT_ALIAS) + return config + + @staticmethod + def _strip_ephemeral_projects(config: AppConfig) -> AppConfig: + """Return a copy of ``config`` with ephemeral (env-synthesized) projects removed. + + Defends against persisting an env token to disk: mutation methods do + ``load() -> mutate -> save()``, and ``load()`` may have injected the + ``__env__`` project. The original object is left intact because callers + keep using it after ``save()`` returns. If ``default_project`` pointed + at a stripped ephemeral alias, it is blanked (the next ``load()`` + re-injects and re-defaults it). + """ + ephemeral_aliases = {alias for alias, p in config.projects.items() if p.ephemeral} + if not ephemeral_aliases: + return config + clean = config.model_copy(deep=True) + for alias in ephemeral_aliases: + clean.projects.pop(alias, None) + if clean.default_project in ephemeral_aliases: + clean.default_project = next(iter(clean.projects), "") + return clean + + @staticmethod + def _reject_ephemeral_mutation(config: AppConfig, alias: str, operation: str) -> None: + """Block mutations targeting an env-synthesized project (issue #359). + + A `__env__` project injected from `KBAGENT_PROJECT_FROM_ENV` exists only + in memory and is stripped on save, so `remove`/`edit`/`rename`/branch + ops would otherwise report success and then silently vanish on the next + `load()`. Reject them with a clear, actionable message instead. A real + persisted project that happens to use the alias (``ephemeral=False``) is + unaffected. + """ + project = config.projects.get(alias) + if project is not None and project.ephemeral: + raise ConfigError( + f"Project '{alias}' is synthesized from environment variables " + f"({ENV_PROJECT_FROM_ENV}) and cannot be {operation} -- it lives " + f"only in memory. To change it, update {ENV_KBC_TOKEN} / " + f"{ENV_KBC_STORAGE_API_URL}; to manage a persisted project, unset " + f"{ENV_PROJECT_FROM_ENV} and use 'project add'." + ) + def save(self, config: AppConfig) -> None: """Save configuration to disk with secure file permissions (0600). @@ -193,6 +315,10 @@ def save(self, config: AppConfig) -> None: try: self._config_dir.mkdir(parents=True, exist_ok=True, mode=0o700) self._ensure_gitignore() + # Never persist env-synthesized projects (issue #359): strip any + # ephemeral entry so the KBC_TOKEN from the environment stays in + # memory only. Operate on a copy -- callers reuse the AppConfig. + config = self._strip_ephemeral_projects(config) # Prepend the agent-facing warning as the first field so any LLM # that reads config.json sees it BEFORE any token value. payload = { @@ -282,6 +408,7 @@ def remove_project(self, alias: str) -> None: config = self.load() if alias not in config.projects: raise ConfigError(f"Project '{alias}' not found.") + self._reject_ephemeral_mutation(config, alias, "removed") del config.projects[alias] if config.default_project == alias: config.default_project = next(iter(config.projects), "") @@ -305,6 +432,7 @@ def set_project_branch(self, alias: str, branch_id: int | None) -> None: config = self.load() if alias not in config.projects: raise ConfigError(f"Project '{alias}' not found.") + self._reject_ephemeral_mutation(config, alias, "modified") config.projects[alias].active_branch_id = branch_id self.save(config) @@ -323,6 +451,7 @@ def edit_project(self, alias: str, **kwargs: str | int | None) -> None: config = self.load() if alias not in config.projects: raise ConfigError(f"Project '{alias}' not found.") + self._reject_ephemeral_mutation(config, alias, "edited") project = config.projects[alias] for key, value in kwargs.items(): if hasattr(project, key) and value is not None: @@ -350,6 +479,7 @@ def rename_project(self, old_alias: str, new_alias: str) -> None: config = self.load() if old_alias not in config.projects: raise ConfigError(f"Project '{old_alias}' not found.") + self._reject_ephemeral_mutation(config, old_alias, "renamed") if new_alias in config.projects: raise ConfigError( f"Cannot rename '{old_alias}' to '{new_alias}': " @@ -359,3 +489,94 @@ def rename_project(self, old_alias: str, new_alias: str) -> None: if config.default_project == old_alias: config.default_project = new_alias self.save(config) + + def add_dev_portal_identity(self, alias: str, identity: DeveloperPortalIdentity) -> None: + """Add a Developer Portal identity to the configuration. + + Sets it as default if no default identity is set. + + Raises: + ConfigError: If the alias already exists. + """ + config = self.load() + if alias in config.dev_portal_identities: + raise ConfigError( + f"Developer Portal identity '{alias}' already exists. " + "Use 'dev-portal identity edit' to modify it." + ) + config.dev_portal_identities[alias] = identity + if not config.default_dev_portal_identity: + config.default_dev_portal_identity = alias + self.save(config) + + def remove_dev_portal_identity(self, alias: str) -> None: + """Remove a Developer Portal identity. + + Falls the default through to the next available identity (or "" if none). + + Raises: + ConfigError: If the alias does not exist. + """ + config = self.load() + if alias not in config.dev_portal_identities: + raise ConfigError(f"Developer Portal identity '{alias}' not found.") + del config.dev_portal_identities[alias] + if config.default_dev_portal_identity == alias: + config.default_dev_portal_identity = next(iter(config.dev_portal_identities), "") + self.save(config) + + def get_dev_portal_identity(self, alias: str) -> DeveloperPortalIdentity | None: + """Get a Developer Portal identity by alias, or None if not found.""" + config = self.load() + return config.dev_portal_identities.get(alias) + + def edit_dev_portal_identity(self, alias: str, **kwargs: str | None) -> None: + """Update fields on an existing Developer Portal identity. + + Only non-None keyword arguments are applied. + + Raises: + ConfigError: If the alias does not exist. + """ + config = self.load() + if alias not in config.dev_portal_identities: + raise ConfigError(f"Developer Portal identity '{alias}' not found.") + ident = config.dev_portal_identities[alias] + for key, value in kwargs.items(): + if hasattr(ident, key) and value is not None: + setattr(ident, key, value) + config.dev_portal_identities[alias] = ident + self.save(config) + + def rename_dev_portal_identity(self, old_alias: str, new_alias: str) -> None: + """Rename a Developer Portal identity alias. + + If the default was set to the old alias, it follows the rename. + + Raises: + ConfigError: If old alias does not exist, or new alias is in use. + """ + config = self.load() + if old_alias not in config.dev_portal_identities: + raise ConfigError(f"Developer Portal identity '{old_alias}' not found.") + if new_alias in config.dev_portal_identities: + raise ConfigError( + f"Cannot rename '{old_alias}' to '{new_alias}': " + f"alias '{new_alias}' is already in use." + ) + config.dev_portal_identities[new_alias] = config.dev_portal_identities.pop(old_alias) + if config.default_dev_portal_identity == old_alias: + config.default_dev_portal_identity = new_alias + self.save(config) + + def set_default_dev_portal_identity(self, alias: str) -> None: + """Set the default Developer Portal identity. + + Raises: + ConfigError: If the alias does not exist. + """ + config = self.load() + if alias not in config.dev_portal_identities: + raise ConfigError(f"Developer Portal identity '{alias}' not found.") + config.default_dev_portal_identity = alias + self.save(config) diff --git a/src/keboola_agent_cli/constants.py b/src/keboola_agent_cli/constants.py index f440cac6..f6c8e725 100644 --- a/src/keboola_agent_cli/constants.py +++ b/src/keboola_agent_cli/constants.py @@ -27,6 +27,15 @@ # --- API Error Handling --- MAX_API_ERROR_LENGTH: int = 500 +# --- Developer Portal MFA --- +# Challenge type sent on the second `/auth/login` step (after the first call +# returns a `session` token). The apiary spec documents `SOFTWARE_TOKEN_MFA` +# (TOTP authenticator app) as the default and `SMS_MFA` as the only other +# member, but in practice the server 404s when the field is omitted, so we +# send it explicitly. If apps-api ever adds a third member (e.g. EMAIL_OTP), +# wire it in here and add a per-identity `mfa_challenge` field. +DP_MFA_CHALLENGE_TYPE: str = "SOFTWARE_TOKEN_MFA" + # --- UNEXPECTED_ERROR truncation --- # Unhandled ``Exception`` messages surfaced to per-project error envelopes are # truncated to this many characters before being returned. Exceptions can @@ -149,6 +158,17 @@ # Overrides the persisted `default_project` pin for a single invocation/session. ENV_KBAGENT_PROJECT: str = "KBAGENT_PROJECT" +# --- Headless / env-only project (issue #359) --- +# Opt-in flag that makes ConfigStore synthesize an in-memory project from +# KBC_TOKEN + KBC_STORAGE_API_URL, so a daemon / container / CI can run kbagent +# (CLI or `serve`) with no `kbagent project add` and no config.json on disk. +# Explicit opt-in (not mere presence of KBC_TOKEN) avoids a phantom project +# surprising a dev who exported KBC_TOKEN only for `kbagent project add`. +ENV_PROJECT_FROM_ENV: str = "KBAGENT_PROJECT_FROM_ENV" +# Reserved alias for the synthesized project. Double-underscore marks it as a +# synthetic, never-persisted entry that cannot collide with a user alias. +ENV_PROJECT_ALIAS: str = "__env__" + # --- Environment Variable Names --- ENV_MAX_PARALLEL_WORKERS: str = "KBAGENT_MAX_PARALLEL_WORKERS" ENV_KBC_TOKEN: str = "KBC_TOKEN" @@ -244,6 +264,39 @@ # --- AI Service --- AI_SERVICE_TIMEOUT: httpx.Timeout = httpx.Timeout(connect=5.0, read=15.0, write=5.0, pool=5.0) +# --- Data Streams (Stream API) --- +# The Stream control-plane API lives on a separate host derived from the +# Storage URL by replacing 'connection.' with 'stream.' (same scheme as +# 'ai.'/'queue.'), authenticated with the Storage API token (X-StorageApi-Token). +# The OTLP *ingestion* endpoint (stream-in./otlp/...) is NOT derived -- +# it is returned by the API in the source's `otlp.url` field. +STREAM_API_TIMEOUT: httpx.Timeout = httpx.Timeout(connect=5.0, read=30.0, write=10.0, pool=5.0) +# Branch ref used by the Stream API path /v1/branches//...; "default" +# targets the project's default (production) branch. +STREAM_DEFAULT_BRANCH: str = "default" +# Source/delete operations return an async Task; poll GET /v1/tasks/ until +# `isFinished`. Interval between polls and the overall ceiling. +STREAM_TASK_POLL_INTERVAL: float = 1.0 # seconds between task polls +STREAM_TASK_TIMEOUT: float = 60.0 # max seconds to wait for a Stream task +# OTLP/HTTP per-signal sub-paths appended to the source's base endpoint, and the +# wire protocol every OTLP source speaks. Surfaced by `kbagent stream detail`. +OTLP_SIGNAL_PATHS: tuple[str, ...] = ("v1/logs", "v1/traces", "v1/metrics") +OTLP_PROTOCOL: str = "http/protobuf" +# Signals (and their destination table names) auto-provisioned for an OTLP source. +# Creating a source via the raw Stream API does NOT create sinks, so kbagent +# provisions one table sink per signal (matching the Keboola UI) so data actually +# lands. Bucket = OTLP_BUCKET_PREFIX + sourceId; table = the signal name. +OTLP_SINK_SIGNALS: tuple[str, ...] = ("logs", "metrics", "traces") +OTLP_BUCKET_PREFIX: str = "in.c-otlp-" +# Universal, signal-agnostic sink mapping: an auto id, the ingest datetime, and a +# `body` column that captures the full flattened OTLP record as JSON. Users can +# refine per-signal column mappings in the Keboola UI afterwards. +OTLP_SINK_COLUMNS: tuple[dict[str, str], ...] = ( + {"type": "uuid", "name": "id"}, + {"type": "datetime", "name": "datetime"}, + {"type": "body", "name": "body"}, +) + # --- Project Feature Flags --- # `storage-branches` enables the modern dev-branch storage isolation: # transformation runner / output-mapping consult bucket metadata @@ -303,14 +356,17 @@ # Verified 2026-05-18 against project 901 on connection.keboola.com: # snowflake-service-keypair: PASS # snowflake-person-sso: PASS +# snowflake-person-keypair: PASS (required for new Snowflake sandboxes) # snowflake-legacy-service: PASS here, FAIL on GCP us-east4 (issue #304) # default (legacy 2016 ws): FAIL ('JWT token is invalid') # # Extend ONLY after empirical confirmation across at least one non-AWS stack. +SNOWFLAKE_WORKSPACE_LOGIN_TYPE: str = "snowflake-person-keypair" QUERY_SERVICE_COMPATIBLE_LOGIN_TYPES: frozenset[str] = frozenset( { "snowflake-service-keypair", "snowflake-person-sso", + SNOWFLAKE_WORKSPACE_LOGIN_TYPE, } ) diff --git a/src/keboola_agent_cli/dev_portal_client.py b/src/keboola_agent_cli/dev_portal_client.py new file mode 100644 index 00000000..753d0759 --- /dev/null +++ b/src/keboola_agent_cli/dev_portal_client.py @@ -0,0 +1,323 @@ +"""Keboola Developer Portal HTTP client (apps-api.keboola.com). + +Auth model: +- Login (email + password) returns a bearer token. On a personal account, the + first login returns an MFA session; we prompt the user via /dev/tty and + re-login with {email, session, code} to obtain the bearer. +- The bearer lives ONLY on this client instance (in self._bearer). It is + never written to disk, never logged, and discarded when the client closes. +- Each kbagent invocation logs in fresh; there is no token cache. + +The client is intentionally dumb: dry-run, diff, and confirm logic belong to +the service and command layers. +""" + +from __future__ import annotations + +import logging +import urllib.error +import urllib.request +from typing import Any + +import httpx + +from .constants import DP_MFA_CHALLENGE_TYPE, MAX_API_ERROR_LENGTH +from .errors import ErrorCode, KeboolaApiError +from .http_base import BaseHttpClient +from .models import DeveloperPortalIdentity + +logger = logging.getLogger(__name__) + + +def _tty_prompt(label: str, *, secret: bool = False) -> str | None: + """Prompt via the controlling terminal so a redirected stdin can't break it. + + Returns None when no /dev/tty is available (non-interactive shell, no + controlling terminal). Caller must treat None as "cannot prompt". + """ + try: + with open("/dev/tty", "w") as out: + if secret: + import getpass + + return getpass.getpass(label, stream=out) + out.write(label) + out.flush() + with open("/dev/tty") as tin: + return tin.readline().rstrip("\n") + except OSError: + return None + + +class DeveloperPortalClient(BaseHttpClient): + """HTTP client for the Keboola Developer Portal.""" + + def __init__(self, identity: DeveloperPortalIdentity) -> None: + # We don't have a bearer yet — pass empty token. Login populates it. + super().__init__( + base_url=identity.portal_url, + token="", + headers={"Accept": "application/json"}, + ) + self._identity = identity + self._bearer: str | None = None + + @property + def bearer(self) -> str | None: + """The active bearer token, or None if not yet authenticated. + + In-memory only; never written to disk. Exposed so the service can + reuse one login across a prepare/apply pair (see seed_bearer) instead + of re-authenticating — which, on a personal MFA account, would prompt + for a second MFA code on a single write. + """ + return self._bearer + + def seed_bearer(self, bearer: str) -> None: + """Reuse a bearer obtained by an earlier client for the same identity. + + Lets the service carry one authenticated session across the + prepare -> (random-code confirm) -> apply flow without a second login. + """ + self._bearer = bearer + self._client.headers["Authorization"] = bearer + + def _ensure_authenticated(self) -> None: + """Log in if not already authenticated. Idempotent on the instance.""" + if self._bearer is not None: + return + self._bearer = self._login(self._identity.username, self._identity.password) + self._client.headers["Authorization"] = self._bearer + + def _login(self, username: str, password: str) -> str: + try: + resp = self._client.post( + "/auth/login", + json={"email": username, "password": password}, + ) + except httpx.HTTPError as exc: + raise KeboolaApiError( + message=f"Developer Portal login transport error: {exc}", + error_code=ErrorCode.CONNECTION_ERROR, + ) from exc + if resp.status_code != 200: + raise KeboolaApiError( + message=( + f"Developer Portal login failed (HTTP {resp.status_code}). " + "Check the identity credentials." + ), + error_code=ErrorCode.DP_LOGIN_FAILED, + ) + payload = resp.json() + if isinstance(payload, dict) and payload.get("token"): + return payload["token"] + # MFA path — implemented in Task 7. + if isinstance(payload, dict) and payload.get("session"): + return self._login_with_mfa(username, payload["session"]) + raise KeboolaApiError( + message="Developer Portal login response missing token and session", + error_code=ErrorCode.DP_LOGIN_FAILED, + ) + + def _login_with_mfa(self, username: str, session: str) -> str: + """Confirm an MFA-gated login. + + Per the Keboola Developer Portal apiary spec, the same POST /auth/login + endpoint accepts {email, session, code, challenge}. The `challenge` + field is documented as optional with default SOFTWARE_TOKEN_MFA, but + in practice the server rejects calls that omit it (404 with the + misleading "must be one of" enum message attached to the admin schema). + Send it explicitly. Single attempt only -- /auth/login consumes the + session, so any retry on the same session always 404s with "Invalid + code or auth state for the user" regardless of the new challenge type. + """ + code = _tty_prompt("MFA code: ") + if not code: + raise KeboolaApiError( + message=( + "Developer Portal identity requires an MFA code, but no " + "interactive terminal is available. Run from a real " + "terminal, or switch to a service.{vendor}.{id} " + "account (no MFA)." + ), + error_code=ErrorCode.DP_MFA_REQUIRED, + ) + body = { + "email": username, + "session": session, + "code": code.strip(), + "challenge": DP_MFA_CHALLENGE_TYPE, + } + try: + resp = self._client.post("/auth/login", json=body) + except httpx.HTTPError as exc: + raise KeboolaApiError( + message=f"Developer Portal MFA login transport error: {exc}", + error_code=ErrorCode.CONNECTION_ERROR, + ) from exc + if resp.status_code == 200: + payload = resp.json() + if isinstance(payload, dict) and payload.get("token"): + return payload["token"] + raise KeboolaApiError( + message=( + "Developer Portal MFA login returned HTTP 200 but no " + f"'token' field in response: {payload!r}" + ), + error_code=ErrorCode.DP_LOGIN_FAILED, + ) + try: + body_text = resp.text[:MAX_API_ERROR_LENGTH] + except (UnicodeDecodeError, AttributeError): + body_text = "" + raise KeboolaApiError( + message=( + f"Developer Portal MFA login failed (HTTP {resp.status_code}): " + f"{body_text}. If your TOTP code rotates every 30s, this is " + "often a stale code -- retry promptly. If the server says " + "'Invalid code or auth state' on a fresh session, the code " + "itself was wrong." + ), + error_code=ErrorCode.DP_LOGIN_FAILED, + ) + + # ----- Reads ----- + + def list_apps(self, vendor: str) -> list[dict[str, Any]]: + self._ensure_authenticated() + resp = self._do_request("GET", f"/vendors/{vendor}/apps?limit=1000") + if resp.status_code != 200: + self._raise_dp_error(resp, action="list apps", vendor=vendor) + payload = resp.json() + if isinstance(payload, dict) and "apps" in payload: + return list(payload["apps"]) + if isinstance(payload, list): + return payload + return [] + + def get_app(self, vendor: str, app_id: str) -> dict[str, Any]: + self._ensure_authenticated() + try: + resp = self._do_request("GET", f"/vendors/{vendor}/apps/{app_id}") + except KeboolaApiError as exc: + if exc.error_code == ErrorCode.NOT_FOUND: + raise KeboolaApiError( + message=f"Developer Portal app '{app_id}' not found in vendor '{vendor}'", + error_code=ErrorCode.DP_APP_NOT_FOUND, + ) from exc + raise + return resp.json() + + # ----- Writes ----- + + def create_app(self, vendor: str, payload: dict[str, Any]) -> dict[str, Any]: + self._ensure_authenticated() + resp = self._do_request("POST", f"/vendors/{vendor}/apps", json=payload) + if resp.status_code not in (200, 201): + self._raise_dp_error(resp, action="create app", vendor=vendor) + return resp.json() + + def patch_app(self, vendor: str, app_id: str, payload: dict[str, Any]) -> dict[str, Any]: + """PATCH an app. Routes by identity role: + - admin -> PATCH /admin/apps/{app_id} (permissive schema, accepts the + 9 fields forbidden() on the vendor schema: complexity, categories, + forwardToken, forwardTokenDetails, injectEnvironment, processTimeout, + requiredMemory, features, category). + - vendor -> PATCH /vendors/{vendor}/apps/{app_id} (default, restricted + schema). The `vendor` arg is still required for the path. + """ + self._ensure_authenticated() + if self._identity.role_hint == "admin": + path = f"/admin/apps/{app_id}" + else: + path = f"/vendors/{vendor}/apps/{app_id}" + resp = self._do_request("PATCH", path, json=payload) + if resp.status_code not in (200, 204): + self._raise_dp_error(resp, action="patch app", vendor=vendor, app_id=app_id) + return resp.json() if resp.content else {} + + def publish_app(self, vendor: str, app_id: str) -> dict[str, Any]: + self._ensure_authenticated() + resp = self._do_request("POST", f"/vendors/{vendor}/apps/{app_id}/publish") + if resp.status_code not in (200, 202): + self._raise_dp_error(resp, action="publish app", vendor=vendor, app_id=app_id) + return resp.json() if resp.content else {"status": "submitted"} + + def deprecate_app(self, vendor: str, app_id: str) -> dict[str, Any]: + self._ensure_authenticated() + resp = self._do_request("POST", f"/vendors/{vendor}/apps/{app_id}/deprecate") + if resp.status_code not in (200, 202): + self._raise_dp_error(resp, action="deprecate app", vendor=vendor, app_id=app_id) + return resp.json() if resp.content else {"status": "deprecated"} + + def upload_icon(self, vendor: str, app_id: str, png_bytes: bytes) -> None: + """Two-hop icon upload: ask the portal for a presigned S3 URL, then PUT bytes there. + + The S3 PUT does NOT use this client's httpx instance (no retry, no auth, + no User-Agent injection). We use urllib directly so the wire shape stays + exactly what S3 expects. + """ + self._ensure_authenticated() + try: + resp = self._do_request("POST", f"/vendors/{vendor}/apps/{app_id}/icon") + except KeboolaApiError as exc: + raise KeboolaApiError( + message=(f"Developer Portal failed to mint icon-upload URL: {exc.message}"), + error_code=ErrorCode.DP_ICON_UPLOAD_FAILED, + ) from exc + if resp.status_code != 200: + raise KeboolaApiError( + message=( + f"Developer Portal failed to mint icon-upload URL (HTTP {resp.status_code})" + ), + error_code=ErrorCode.DP_ICON_UPLOAD_FAILED, + ) + payload = resp.json() + link = payload.get("link") if isinstance(payload, dict) else None + if not link: + raise KeboolaApiError( + message="Developer Portal icon-upload response missing 'link'", + error_code=ErrorCode.DP_ICON_UPLOAD_FAILED, + ) + req = urllib.request.Request( + link, + data=png_bytes, + headers={"Content-Type": "image/png"}, + method="PUT", + ) + try: + with urllib.request.urlopen(req) as s3_resp: + if getattr(s3_resp, "status", 200) >= 300: + raise KeboolaApiError( + message=f"Icon S3 PUT failed (HTTP {s3_resp.status})", + error_code=ErrorCode.DP_ICON_UPLOAD_FAILED, + ) + except urllib.error.HTTPError as exc: + raise KeboolaApiError( + message=f"Icon S3 PUT failed (HTTP {exc.code}): {exc.reason}", + error_code=ErrorCode.DP_ICON_UPLOAD_FAILED, + ) from exc + + # ----- Error mapping ----- + + def _raise_dp_error( + self, + resp: httpx.Response, + *, + action: str, + vendor: str | None = None, + app_id: str | None = None, + ) -> None: + try: + body = resp.json() + except ValueError: + body = resp.text + ctx = f"{action}" + if vendor: + ctx += f" (vendor={vendor})" + if app_id: + ctx += f" (app={app_id})" + raise KeboolaApiError( + message=f"Developer Portal {ctx} failed (HTTP {resp.status_code}): {body}", + error_code=ErrorCode.API_ERROR, + ) diff --git a/src/keboola_agent_cli/errors.py b/src/keboola_agent_cli/errors.py index 8aaab100..ea5c9881 100644 --- a/src/keboola_agent_cli/errors.py +++ b/src/keboola_agent_cli/errors.py @@ -87,6 +87,8 @@ class ErrorCode(StrEnum): # Sync PARENT_CONFIG_NOT_TRACKED = "PARENT_CONFIG_NOT_TRACKED" + VARIABLE_LINK_UNRESOLVED = "VARIABLE_LINK_UNRESOLVED" + SYNC_CONFLICT = "SYNC_CONFLICT" # Encryption ENCRYPTION_FAILED = "ENCRYPTION_FAILED" @@ -108,6 +110,13 @@ class ErrorCode(StrEnum): DATA_APP_INVALID_REPO = "DATA_APP_INVALID_REPO" DATA_APP_REPO_VALIDATION_BLOCKING = "DATA_APP_REPO_VALIDATION_BLOCKING" + # Developer Portal (since 0.48.0) + DP_LOGIN_FAILED = "DP_LOGIN_FAILED" + DP_MFA_REQUIRED = "DP_MFA_REQUIRED" + DP_APP_NOT_FOUND = "DP_APP_NOT_FOUND" + DP_PUBLISH_REQUIREMENTS_MISSING = "DP_PUBLISH_REQUIREMENTS_MISSING" + DP_ICON_UPLOAD_FAILED = "DP_ICON_UPLOAD_FAILED" + def mask_token(token: str) -> str: """Mask a Keboola Storage API token for safe display. @@ -168,6 +177,41 @@ def __init__(self, message: str) -> None: self.message = message +class SyncConflictError(Exception): + """Raised when ``sync pull --force`` would overwrite locally-modified + configs whose remote **also** changed since the last pull -- a true 3-way + merge conflict (local and remote both diverged from the synced base). + + ``--force`` deliberately bypasses the "preserve locally-modified files" + guard, so without this check it would silently adopt the edited on-disk + file as the new synced baseline (issue: force-pull baseline corruption). + Rather than discard un-pushed work, the pull aborts *before writing + anything* and asks the user to resolve each conflict (push or discard + local edits, then pull again). + + ``conflicts`` carries one dict per conflicting config/row so the command + layer can list them. Each dict has ``component_id``, ``config_id``, + ``config_name``, ``path``, ``scope`` (``"config"`` or ``"row"``), and an + optional ``row_id``. + """ + + def __init__(self, conflicts: list[dict[str, str]]) -> None: + self.conflicts = conflicts + n = len(conflicts) + plural = "s" if n != 1 else "" + message = ( + f"{n} config{plural} ha{'ve' if n != 1 else 's'} un-pushed local " + f"edits AND changed on the remote since the last pull (merge " + f"conflict). `sync pull --force` refuses to overwrite them so your " + f"local work is not lost. Resolve each conflict first: review with " + f"`kbagent sync diff`, then either `kbagent sync push` your local " + f"edits or discard them, and pull again." + ) + super().__init__(message) + self.message = message + self.error_code = ErrorCode.SYNC_CONFLICT + + class PermissionDeniedError(Exception): """Raised when an operation is blocked by the permission policy.""" @@ -188,7 +232,13 @@ def __init__(self, operation: str, message: str = "") -> None: ErrorCode.NOT_FOUND: "not_found", ErrorCode.CONFIG_ERROR: "configuration", ErrorCode.VALIDATION_ERROR: "validation", + ErrorCode.SYNC_CONFLICT: "conflict", ErrorCode.PERMISSION_DENIED: "authorization", + ErrorCode.DP_LOGIN_FAILED: "authentication", + ErrorCode.DP_MFA_REQUIRED: "authentication", + ErrorCode.DP_APP_NOT_FOUND: "not_found", + ErrorCode.DP_PUBLISH_REQUIREMENTS_MISSING: "validation", + ErrorCode.DP_ICON_UPLOAD_FAILED: "api", } diff --git a/src/keboola_agent_cli/hints/definitions/semantic_layer.py b/src/keboola_agent_cli/hints/definitions/semantic_layer.py index c14b21e7..425c41e1 100644 --- a/src/keboola_agent_cli/hints/definitions/semantic_layer.py +++ b/src/keboola_agent_cli/hints/definitions/semantic_layer.py @@ -162,6 +162,84 @@ def _make_service(method: str, **extra_args: str) -> ServiceCall: ) +# ── semantic-layer search-context (since v0.47.0) ───────────────── + +HintRegistry.register( + CommandHint( + cli_command="semantic-layer.search-context", + description=( + "Search semantic-layer entities project-wide by glob pattern " + "(mirrors the upstream keboola-mcp-server search_semantic_context)" + ), + steps=[ + HintStep( + comment=( + "List every entity of the requested type (or every " + "child type if --type=all) and filter by name pattern." + ), + client=ClientCall( + method="list_items", + args={"item_type": '"semantic-dataset"'}, + client_type="metastore", + result_var="datasets", + result_hint="list[dict]", + ), + service=_make_service( + "search_context", + patterns="{pattern}", + type_filter="{type_filter}", + limit="{limit}", + ), + ), + ], + notes=[ + _PARALLEL_CHILDREN_NOTE, + "Pattern matching is case-sensitive fnmatch against attributes.name.", + "`--limit` short-circuits both inner and outer loops.", + ], + ) +) + + +# ── semantic-layer get-context (since v0.47.0) ──────────────────── + +HintRegistry.register( + CommandHint( + cli_command="semantic-layer.get-context", + description=( + "Fetch a single semantic-layer entity by id, irrespective of type " + "(mirrors the upstream keboola-mcp-server get_semantic_context)" + ), + steps=[ + HintStep( + comment=("Try each type until one returns 200. Raise NOT_FOUND if none match."), + client=ClientCall( + method="get_item", + args={ + "item_type": '"semantic-dataset"', + "item_id": "{context_id}", + }, + client_type="metastore", + result_var="entity", + result_hint="dict", + ), + service=_make_service( + "get_context", + context_id="{context_id}", + ), + ), + ], + notes=[ + ( + "Iteration order is: semantic-model, then semantic-dataset / " + "metric / relationship / constraint / glossary." + ), + "404 on any one type is non-terminal; only a full miss raises NOT_FOUND.", + ], + ) +) + + # ── semantic-layer validate ──────────────────────────────────────── HintRegistry.register( @@ -732,3 +810,141 @@ def _register_remove_hint(entity: str, id_key: str = "name") -> None: ], ) ) + + +# ── semantic-layer reference-data list ───────────────────────────── + +HintRegistry.register( + CommandHint( + cli_command="semantic-layer.reference-data.list", + description="List reference-data (dimension-member) records", + steps=[ + HintStep( + comment="List every `semantic-reference-data` record (optionally one model)", + client=ClientCall( + method="list_items", + args={ + "item_type": '"semantic-reference-data"', + "model_uuid": "{model}", + }, + client_type="metastore", + result_var="records", + result_hint="list[dict]", + ), + service=_make_service("list_reference_data", model_name_or_uuid="{model}"), + ), + ], + notes=[ + "`model_uuid=None` lists every dimension in the project.", + "Summary only (dimension + member_count); use `get` for the members.", + ], + ) +) + + +# ── semantic-layer reference-data get ────────────────────────────── + +HintRegistry.register( + CommandHint( + cli_command="semantic-layer.reference-data.get", + description="Fetch one reference-data record (all members)", + steps=[ + HintStep( + comment="GET by UUID (or list+filter by model+dimensionName)", + client=ClientCall( + method="get_item", + args={ + "item_type": '"semantic-reference-data"', + "item_id": "{id}", + }, + client_type="metastore", + result_var="record", + result_hint="dict", + ), + service=_make_service( + "get_reference_data", + record_id="{id}", + model_name_or_uuid="{model}", + dimension="{dimension}", + ), + ), + ], + notes=[ + "Provide `record_id`, or both `model_name_or_uuid` + `dimension`.", + ], + ) +) + + +# ── semantic-layer reference-data set ────────────────────────────── + +HintRegistry.register( + CommandHint( + cli_command="semantic-layer.reference-data.set", + description="Create or replace a reference-data record (by model + dimension)", + steps=[ + HintStep( + comment=( + "Resolve modelUUID, then POST (create) or PUT (replace, " + "revision++) the whole members[] array." + ), + client=ClientCall( + method="post_item", + args={ + "item_type": '"semantic-reference-data"', + "name": "{dimension}", + "data": ( + '{"modelUUID": model_uuid, "dimensionName": {dimension}, ' + '"members": members}' + ), + }, + client_type="metastore", + result_var="record", + result_hint="dict", + ), + service=_make_service( + "set_reference_data", + model_name_or_uuid="{model}", + dimension="{dimension}", + members="", + dataset_id="{dataset_id}", + description="{description}", + ), + ), + ], + notes=[ + "Idempotent on (modelUUID, dimensionName): existing record -> PUT, " + "else POST. `members` is a JSON array of member objects.", + "For a Chart of Accounts the member keys mirror DIM_COA columns " + "(account_code, account_name, parent_code, ...).", + ], + ) +) + + +# ── semantic-layer reference-data delete ─────────────────────────── + +HintRegistry.register( + CommandHint( + cli_command="semantic-layer.reference-data.delete", + description="Delete a reference-data record by UUID", + steps=[ + HintStep( + comment="DELETE /semantic-reference-data/{id} (server-side soft-delete)", + client=ClientCall( + method="delete_item", + args={ + "item_type": '"semantic-reference-data"', + "item_id": "{id}", + }, + client_type="metastore", + result_var="result", + ), + service=_make_service("delete_reference_data", record_id="{id}"), + ), + ], + notes=[ + "Soft-delete: the record stays in revision history server-side.", + ], + ) +) diff --git a/src/keboola_agent_cli/hints/definitions/storage.py b/src/keboola_agent_cli/hints/definitions/storage.py index 90c7ce6a..dfa70417 100644 --- a/src/keboola_agent_cli/hints/definitions/storage.py +++ b/src/keboola_agent_cli/hints/definitions/storage.py @@ -232,6 +232,7 @@ "branch_id": "{branch}", "not_null_columns": "{not_null}", "defaults": "{default}", + "if_not_exists": "{if_not_exists}", }, ), ), @@ -242,6 +243,7 @@ "Service mode: --not-null and --default flags add nullable/default to column definitions.", "Client mode: build column dicts directly as [{'name': 'pk', 'definition': {'type': 'VARCHAR', 'length': '40', 'nullable': False}}].", "In a dev branch, service layer auto-materializes the bucket on 404 (mirrors Keboola Go CLI's EnsureBucketExists). Client mode does not -- call get_bucket_detail + create_bucket first.", + "if_not_exists=True (0.47.0+) returns {action: 'skipped'} on a duplicate-display-name failure when the table really exists at the expected id. Safe for parallel workers.", ], ) ) @@ -491,7 +493,7 @@ ), ], notes=[ - "Storage API rejects swaps on production: branch_id is mandatory.", + "branch_id is mandatory (the swap is branch-scoped); any branch works, including the default/production branch -- a default-branch swap retypes a prod table (dev-branch merge does not carry storage schema).", "Returns a completed storage job dict (operationName=tableSwap); the client polls the async job to completion before returning.", "Aliases keep pointing at the same physical position, exposing the OTHER table's data after the swap.", ], diff --git a/src/keboola_agent_cli/hints/definitions/workspace.py b/src/keboola_agent_cli/hints/definitions/workspace.py index 17694956..40810c32 100644 --- a/src/keboola_agent_cli/hints/definitions/workspace.py +++ b/src/keboola_agent_cli/hints/definitions/workspace.py @@ -14,7 +14,11 @@ comment="Create workspace (headless mode)", client=ClientCall( method="create_config_workspace", - args={"backend": "{backend}"}, + args={ + "backend": "{backend}", + "login_type": "login_type", + "public_key": "public_key_pem", + }, result_var="workspace", result_hint="dict", ), @@ -33,6 +37,7 @@ ], notes=[ "Service layer handles sandbox config creation + workspace provisioning.", + "For Snowflake direct client calls, generate an RSA key pair, pass login_type='snowflake-person-keypair' and public_key=public_key_pem, then save the private key returned by your generator. For BigQuery, pass None for both.", "With --ui flag, creates via job run (slower, ~15s) for UI visibility.", ], ) diff --git a/src/keboola_agent_cli/http_base.py b/src/keboola_agent_cli/http_base.py index 1fd509d9..7f8178fe 100644 --- a/src/keboola_agent_cli/http_base.py +++ b/src/keboola_agent_cli/http_base.py @@ -10,7 +10,7 @@ import os import platform import time -from typing import Any +from typing import Any, Self from urllib.parse import urlparse, urlunparse import httpx @@ -110,7 +110,7 @@ def close(self) -> None: """Close the underlying HTTP client.""" self._client.close() - def __enter__(self) -> "BaseHttpClient": + def __enter__(self) -> Self: return self def __exit__(self, *args: Any) -> None: diff --git a/src/keboola_agent_cli/manage_client.py b/src/keboola_agent_cli/manage_client.py index b52faec1..f3e5ba93 100644 --- a/src/keboola_agent_cli/manage_client.py +++ b/src/keboola_agent_cli/manage_client.py @@ -8,6 +8,7 @@ """ from typing import Any +from urllib.parse import quote from .constants import DEFAULT_TIMEOUT from .http_base import BaseHttpClient @@ -226,3 +227,98 @@ def update_project_member_role( json={"role": role}, ) return response.json() + + # ------------------------------------------------------------------ + # Feature flags (super-admin manage token required). + # + # The stack-wide catalogue lives at GET /manage/features. Features + # assigned to a single project/user are NOT a dedicated endpoint -- + # they are read from the ``features`` array on the project/user object. + # Endpoint + payload shapes mirror the curl recipes verified by the + # platform team: the POST body is ``{"feature": ""}`` and the + # DELETE targets ``.../features/{name}``. + # ------------------------------------------------------------------ + + def list_features(self) -> list[dict[str, Any]]: + """List all features defined on the stack (the catalogue). + + Returns: + List of feature dicts. Field set is not contractually fixed; + callers should treat unknown keys as opaque. ``name`` is the + stable identifier used by the add/remove endpoints. + + Raises: + KeboolaApiError: On API errors (e.g. 403 without super admin). + """ + response = self._do_request("GET", "/manage/features") + return response.json() + + def add_project_feature(self, project_id: int, feature: str) -> dict[str, Any]: + """Enable a feature on a project. + + Args: + project_id: The numeric project ID. + feature: The feature name (as listed by :meth:`list_features`). + + Returns: + The API response body (project or feature payload, stack-dependent). + + Raises: + KeboolaApiError: On API errors. + """ + response = self._do_request( + "POST", f"/manage/projects/{project_id}/features", json={"feature": feature} + ) + # Most stacks return 201 with a JSON body, but some return 204 No + # Content; guard against JSONDecodeError on an empty body. + return response.json() if response.content else {} + + def remove_project_feature(self, project_id: int, feature: str) -> None: + """Disable a feature on a project. Returns 204 No Content on success.""" + self._do_request( + "DELETE", + f"/manage/projects/{project_id}/features/{quote(feature, safe='')}", + ) + + def get_user(self, email: str) -> dict[str, Any]: + """Get a user by email, including the ``features`` array. + + Args: + email: The user's email address (the public-facing key). + + Returns: + User dict with at least ``id``, ``email`` and ``features``. + + Raises: + KeboolaApiError: On API errors (e.g. 404 if the user is unknown). + """ + response = self._do_request("GET", f"/manage/users/{quote(email, safe='@')}") + return response.json() + + def add_user_feature(self, email: str, feature: str) -> dict[str, Any]: + """Enable a feature on a user. + + Args: + email: The user's email address. + feature: The feature name (as listed by :meth:`list_features`). + + Returns: + The API response body (user or feature payload, stack-dependent). + + Raises: + KeboolaApiError: On API errors. + """ + response = self._do_request( + "POST", + f"/manage/users/{quote(email, safe='@')}/features", + json={"feature": feature}, + ) + # See add_project_feature: tolerate a 204 No Content body. + return response.json() if response.content else {} + + def remove_user_feature(self, email: str, feature: str) -> None: + """Disable a feature on a user. Returns 204 No Content on success.""" + self._do_request( + "DELETE", + f"/manage/users/{quote(email, safe='@')}/features/{quote(feature, safe='')}", + ) diff --git a/src/keboola_agent_cli/metastore_client.py b/src/keboola_agent_cli/metastore_client.py index 9343e315..09fbdb0e 100644 --- a/src/keboola_agent_cli/metastore_client.py +++ b/src/keboola_agent_cli/metastore_client.py @@ -38,6 +38,7 @@ "semantic-relationship", "semantic-constraint", "semantic-glossary", + "semantic-reference-data", ] @@ -48,6 +49,7 @@ "semantic-relationship", "semantic-constraint", "semantic-glossary", + "semantic-reference-data", ) @@ -172,6 +174,38 @@ def post_item( body = response.json() return body.get("data", body) if isinstance(body, dict) else body + def put_item( + self, + item_type: SemanticType, + item_id: str, + name: str, + data: dict[str, Any], + ) -> dict[str, Any]: + """Replace an item in place via ``PUT`` (revisioned update). + + Unlike the DELETE+POST pattern the higher-level ``edit`` operations + use, ``PUT`` updates the record in place and increments + ``meta.revision`` server-side, preserving the metastore's revision + history. ``data`` is the inner ``attributes`` payload; the outer + envelope is added here (identical shape to :meth:`post_item`). + + Raises :class:`KeboolaApiError` with ``error_code=NOT_FOUND`` on 404. + """ + envelope = { + "name": name, + "data": data, + "branch": _ENVELOPE_BRANCH, + "schemaVersion": _ENVELOPE_SCHEMA_VERSION, + "scope": _ENVELOPE_SCOPE, + } + response = self._do_request( + "PUT", + f"/api/v1/repository/{item_type}/{item_id}", + json=envelope, + ) + body = response.json() + return body.get("data", body) if isinstance(body, dict) else body + def delete_item(self, item_type: SemanticType, item_id: str) -> None: """Delete an item by its UUID. Returns silently on 204. diff --git a/src/keboola_agent_cli/models.py b/src/keboola_agent_cli/models.py index 013cc767..79679669 100644 --- a/src/keboola_agent_cli/models.py +++ b/src/keboola_agent_cli/models.py @@ -1,10 +1,50 @@ """Pydantic models shared across all layers of the application.""" +import sys from typing import Any +from urllib.parse import urlparse from pydantic import BaseModel, Field, field_validator +def normalize_stack_url(value: str) -> str: + """Normalize a user-supplied Keboola stack URL to its scheme+host base. + + Accepts, in order of forgiveness: + - a bare host ``connection.keboola.com`` + - a full base URL ``https://connection.keboola.com`` + - a full base URL + slash ``https://connection.keboola.com/`` + - a full project deep-link ``https://connection.keboola.com/admin/projects/10105/dashboard`` + + and reduces every form to ``https://`` (path/query/fragment dropped). + A missing scheme defaults to ``https://``. Any *explicit* non-https scheme + (``http://``, ``file://``, ``ftp://``, ...) is rejected -- this is an + SSRF / protocol-abuse guard, so we never silently upgrade a typed-out + ``http://`` to https. + + Raises: + ValueError: empty input, an explicit non-https scheme, or no host. + """ + raw = value.strip() + if not raw: + raise ValueError("Stack URL must not be empty.") + # No scheme typed -> assume https so urlparse sees a netloc, not a path. + if "://" not in raw: + raw = f"https://{raw}" + parsed = urlparse(raw) + if parsed.scheme != "https": + raise ValueError( + f"Stack URL must use https:// scheme, got: {parsed.scheme or '(none)'}://. " + "Plain HTTP, file://, and other protocols are not allowed." + ) + if not parsed.netloc: + raise ValueError( + f"Stack URL has no host: {value!r}. Expected e.g. " + "'connection.keboola.com' or 'https://connection.keboola.com'." + ) + return f"https://{parsed.netloc}" + + class ProjectConfig(BaseModel): """Configuration for a single Keboola project connection.""" @@ -28,18 +68,108 @@ class ProjectConfig(BaseModel): default=None, description="Organization name (populated via `org setup` or when verify_token returns it)", ) + ephemeral: bool = Field( + default=False, + exclude=True, + description=( + "True for an in-memory project synthesized from KBC_TOKEN + " + "KBC_STORAGE_API_URL (headless mode, issue #359). Excluded from " + "serialization and stripped by ConfigStore.save() so the env " + "token is never written to disk." + ), + ) @field_validator("stack_url") @classmethod def validate_stack_url_scheme(cls, v: str) -> str: - """Enforce HTTPS scheme on stack URL to prevent SSRF and protocol abuse.""" + """Normalize the stack URL to ``https://`` (see ``normalize_stack_url``). + + Accepts a bare host, a full base URL, or a full project deep-link and + reduces it to the scheme+host base; rejects explicit non-https schemes + (SSRF / protocol-abuse guard). + """ + return normalize_stack_url(v) + + +class DeveloperPortalIdentity(BaseModel): + """One Developer Portal identity (service account or admin email). + + DP login is email + password (with MFA on personal accounts), producing + a short-lived bearer that lives only in process memory. The username + + password are persisted in config.json under the same 0600 protection as + KB Storage tokens; the bearer is never written to disk. + """ + + username: str = Field(description="Email or service-account id used as the login subject") + password: str = Field(description="DP password — same protection as KB tokens") + role_hint: str = Field( + default="vendor", + description=( + "Identity role: 'vendor' (default) or 'admin'. Load-bearing -- " + "write commands route to different apps-api endpoints based on " + "role: 'admin' uses PATCH /admin/apps/{app} (permissive schema, " + "can set complexity/categories/forwardToken/processTimeout/etc.); " + "'vendor' uses PATCH /vendors/{vendor}/apps/{app} (those fields " + "are forbidden()). kbagent does not verify the server-side role " + "of the credential -- if you set 'admin' but the account isn't " + "actually a portal admin, the write fails at the apps-api with " + "an unambiguous 403." + ), + ) + vendor: str | None = Field( + default=None, + description=( + "Optional default vendor for this identity (e.g. 'keboola'). " + "Used as a default for commands that take --vendor; never " + "overrides an explicit flag." + ), + ) + portal_url: str = Field( + default="https://apps-api.keboola.com", + description="DP base URL. Override for staging/test portals.", + ) + + @field_validator("portal_url") + @classmethod + def validate_portal_url(cls, v: str) -> str: + """Enforce HTTPS scheme on portal URL to prevent SSRF and protocol abuse.""" if not v.startswith("https://"): - raise ValueError( - f"Stack URL must use https:// scheme, got: {v!r}. " - "Plain HTTP, file://, and other protocols are not allowed." - ) + raise ValueError(f"Portal URL must use https:// scheme, got: {v!r}") return v + @field_validator("role_hint", mode="before") + @classmethod + def validate_role_hint(cls, v: object) -> str: + """Normalise `role_hint` to the validated enum {vendor, admin}. + + Before v0.51.1 the field was free-text and documented as "not + validated against the portal", so existing on-disk configs may + carry arbitrary strings (e.g. 'keboola-admin', empty string, + non-string types from hand-edits). A strict raise would crash + `ConfigStore.load()` -> the entire CLI on startup for every + pre-0.51.1 user with a non-standard value; that's a worse UX + than a silent downgrade. + + Behaviour: + - "vendor" / "admin" (case-insensitive, whitespace-stripped) + pass through normalised. + - Anything else is downgraded to "vendor" with a one-shot stderr + warning. The user still sees what happened; the CLI keeps + working. To force admin routing they can rerun + `dev-portal identity edit --alias A --role-hint admin`. + """ + if not isinstance(v, str): + v = "" if v is None else str(v) + normalized = v.strip().lower() + if normalized in ("vendor", "admin"): + return normalized + sys.stderr.write( + f"Warning: role_hint={v!r} is not 'vendor' or 'admin' -- " + "downgrading to 'vendor'. Use `kbagent dev-portal identity " + "edit --alias --role-hint admin` to switch.\n" + ) + return "vendor" + class PermissionPolicy(BaseModel): """Firewall-style permission policy for CLI and MCP operations. @@ -95,6 +225,14 @@ class AppConfig(BaseModel): default_factory=dict, description="Map of alias -> ProjectConfig", ) + dev_portal_identities: dict[str, DeveloperPortalIdentity] = Field( + default_factory=dict, + description="Map of alias -> DeveloperPortalIdentity", + ) + default_dev_portal_identity: str = Field( + default="", + description="Alias of the default identity for `kbagent dev-portal` commands", + ) class TokenVerifyResponse(BaseModel): @@ -207,6 +345,29 @@ class ProjectMember(BaseModel): model_config = {"populate_by_name": True, "extra": "allow"} +class Feature(BaseModel): + """A Keboola feature flag, from GET /manage/features or a project/user object. + + The Manage API has no published schema for features and the field set + varies by stack version. Only ``name`` is treated as stable -- it is the + identifier passed to the add/remove endpoints. Every field defaults to a + safe empty value and extras pass through unmodified so ``--json`` output + keeps whatever the stack returned (``id``, ``projectFeature``, + ``adminFeature``, ``canBeManagedViaApi``, ...). + + Features embedded in a project/user ``features`` array may be returned as + bare strings rather than objects; the service layer normalises those to + ``{"name": }`` before validation. + """ + + name: str = Field(default="", description="Feature code -- the value used to add/remove it") + title: str = Field(default="", description="Human-readable name shown in the UI") + description: str = Field(default="") + type: str = Field(default="", description="Feature category (project | admin | global | ...)") + + model_config = {"populate_by_name": True, "extra": "allow"} + + class InvitationUser(BaseModel): """Invited user inside an Invitation object.""" diff --git a/src/keboola_agent_cli/permissions.py b/src/keboola_agent_cli/permissions.py index 384d2b71..fd667ab9 100644 --- a/src/keboola_agent_cli/permissions.py +++ b/src/keboola_agent_cli/permissions.py @@ -31,6 +31,21 @@ "project.invitation-cancel": "admin", "project.member-remove": "destructive", "project.member-set-role": "admin", + # Feature flags (super-admin manage token). Reads are safe; enabling a + # feature is an org-level decision (admin); removing one is destructive. + "feature.list": "read", + "feature.project-show": "read", + "feature.project-add": "admin", + "feature.project-remove": "destructive", + "feature.user-show": "read", + "feature.user-add": "admin", + "feature.user-remove": "destructive", + # Data Streams (OTLP). Listing/inspecting sources is read-only; creating a + # source provisions ingest infrastructure (write); deleting one is destructive. + "stream.list": "read", + "stream.detail": "read", + "stream.create-source": "write", + "stream.delete": "destructive", # Config browsing & management "config.list": "read", "config.detail": "read", @@ -121,6 +136,19 @@ # Component discovery "component.list": "read", "component.detail": "read", + # Developer Portal (since 0.48.0) + # Developer Portal — top-level commands on `dev-portal` (the identity + # sub-app's leaves are listed separately below under dev-portal.identity.*). + # Categories follow data-app.secrets-* precedent: credential add/edit are + # `write`, not `admin` (admin is reserved for org-level operations). + "dev-portal.identity": "read", # parent-callback descent (allow into sub-app) + "dev-portal.list": "read", + "dev-portal.get": "read", + "dev-portal.create": "write", + "dev-portal.patch": "write", + "dev-portal.upload-icon": "write", + "dev-portal.publish": "admin", + "dev-portal.deprecate": "destructive", # Data apps (Data Science API + keboola.data-apps Storage component) "data-app.list": "read", "data-app.detail": "read", @@ -137,6 +165,15 @@ "data-app.secrets-get": "read", "data-app.secrets-remove": "destructive", "data-app.validate-repo": "read", + # Developer Portal — identity sub-app leaves (composed by the + # identity_app callback as "dev-portal.identity.") + "dev-portal.identity.add": "write", + "dev-portal.identity.list": "read", + "dev-portal.identity.remove": "write", + "dev-portal.identity.edit": "write", + "dev-portal.identity.use": "write", + "dev-portal.identity.current": "read", + "dev-portal.identity.verify": "read", # Storage browsing "storage.buckets": "read", "storage.bucket-detail": "read", @@ -147,6 +184,9 @@ "storage.create-bucket": "write", "storage.create-table": "write", "storage.upload-table": "write", + # clone-table pulls a prod table into a dev branch (materialization); it + # creates a branch-local copy and never deletes -- write, not destructive. + "storage.clone-table": "write", # Storage files "storage.files": "read", "storage.file-detail": "read", @@ -174,6 +214,8 @@ "semantic-layer.validate": "read", "semantic-layer.export": "read", "semantic-layer.diff": "read", + "semantic-layer.search-context": "read", + "semantic-layer.get-context": "read", # The `model` sub-app: the parent `semantic-layer` callback fires first # with ctx.invoked_subcommand == "model" and synthesizes operation key # ``semantic-layer.model``. We expose that key at the LEAST-privileged @@ -217,6 +259,15 @@ "semantic-layer.remove.constraint": "destructive", "semantic-layer.remove.relationship": "destructive", "semantic-layer.remove.glossary": "destructive", + # `reference-data` sub-app: dimension-member records (e.g. a Chart of + # Accounts). Parent key at the least-privileged level (read) so the + # top-level `semantic-layer` callback does not over-block `list` / `get`; + # per-leaf keys carry the real classification. + "semantic-layer.reference-data": "read", + "semantic-layer.reference-data.list": "read", + "semantic-layer.reference-data.get": "read", + "semantic-layer.reference-data.set": "write", + "semantic-layer.reference-data.delete": "destructive", # Raw HTTP client against `kbagent serve` (used by AI subprocesses). # Categorised by the underlying HTTP method: GET = read, mutating verbs # = write. The serve's own routes enforce their own permissions on top. diff --git a/src/keboola_agent_cli/server/app.py b/src/keboola_agent_cli/server/app.py index 874fe42b..5aae22de 100644 --- a/src/keboola_agent_cli/server/app.py +++ b/src/keboola_agent_cli/server/app.py @@ -38,7 +38,9 @@ components, configs, data_apps, + dev_portal, encrypt, + feature, flows, health, jobs, @@ -53,6 +55,7 @@ semantic_layer, sharing, storage, + stream, workspaces, ) @@ -99,6 +102,17 @@ "Mirrors `kbagent org setup|refresh`." ), }, + { + "name": "feature", + "description": ( + "**Project Management.** " + "List the stack feature-flag catalogue and enable/disable " + "features on projects and users (Manage API). Requires the " + "`X-Manage-Token` header (super-admin) on every request -- the " + "manage token is never persisted in config. " + "Mirrors `kbagent feature list|project-*|user-*`." + ), + }, # ---- Configurations ---- { "name": "configs", @@ -138,6 +152,16 @@ "Mirrors `kbagent storage *`." ), }, + { + "name": "stream", + "description": ( + "**Data.** " + "Data Streams (OpenTelemetry / OTLP) -- list, create, and " + "delete ingest sources and retrieve their endpoints. The OTLP " + "URL embeds a secret that is masked unless `reveal=true`. " + "Mirrors `kbagent stream list|create-source|detail|delete`." + ), + }, { "name": "search", "description": ( @@ -191,6 +215,15 @@ "Mirrors `kbagent data-app *`." ), }, + { + "name": "dev-portal", + "description": ( + "**Read-only.** " + "Developer Portal app discovery -- list a vendor's apps, get one " + "app's full entry. Mirrors `kbagent dev-portal list|get`. Writes " + "and identity management are CLI-only (TTY-confirmed)." + ), + }, { "name": "workspaces", "description": ( @@ -300,7 +333,7 @@ Sections below are grouped roughly the same way `kbagent --help` groups its command tree: -- **Project Management** -- projects, members, org +- **Project Management** -- projects, members, org, feature flags - **Configurations** -- configs, components, encrypt - **Data** -- storage, search, sharing - **Execution** -- jobs, flows, schedules, data-apps, workspaces @@ -544,9 +577,11 @@ async def _generic_handler(_request, exc: Exception): app.include_router(health.router) app.include_router(projects.router) app.include_router(members.router) + app.include_router(feature.router) app.include_router(configs.router) app.include_router(components.router) app.include_router(storage.router) + app.include_router(stream.router) app.include_router(jobs.router) app.include_router(branches.router) app.include_router(workspaces.router) @@ -555,6 +590,7 @@ async def _generic_handler(_request, exc: Exception): app.include_router(lineage.router) app.include_router(sharing.router) app.include_router(data_apps.router) + app.include_router(dev_portal.router) app.include_router(mcp.router) app.include_router(kai.router) app.include_router(ai_chat.router) @@ -720,6 +756,7 @@ def _is_ui_public(method: str, path: str) -> bool: "/lineage", "/sharing", "/data-apps", + "/dev-portal", "/mcp", "/kai", "/ai", @@ -727,6 +764,7 @@ def _is_ui_public(method: str, path: str) -> bool: "/search", "/semantic-layer", "/org", + "/feature", "/agents", "/members", "/health", diff --git a/src/keboola_agent_cli/server/dependencies.py b/src/keboola_agent_cli/server/dependencies.py index decc29f4..6edb0d52 100644 --- a/src/keboola_agent_cli/server/dependencies.py +++ b/src/keboola_agent_cli/server/dependencies.py @@ -14,13 +14,16 @@ from fastapi import FastAPI, Request from ..config_store import ConfigStore +from ..dev_portal_client import DeveloperPortalClient from ..services.branch_service import BranchService from ..services.component_service import ComponentService from ..services.config_service import ConfigService from ..services.data_app_service import DataAppService from ..services.deep_lineage_service import DeepLineageService +from ..services.dev_portal_service import DeveloperPortalService from ..services.doctor_service import DoctorService from ..services.encrypt_service import EncryptService +from ..services.feature_service import FeatureService from ..services.flow_service import FlowService from ..services.job_service import JobService from ..services.kai_service import KaiService @@ -35,6 +38,7 @@ from ..services.semantic_layer_service import SemanticLayerService from ..services.sharing_service import SharingService from ..services.storage_service import StorageService +from ..services.stream_service import StreamService from ..services.sync_service import SyncService from ..services.variables_service import VariablesService from ..services.version_service import VersionService @@ -60,6 +64,7 @@ class ServiceRegistry: config: ConfigService = field(init=False) component: ComponentService = field(init=False) storage: StorageService = field(init=False) + stream: StreamService = field(init=False) job: JobService = field(init=False) branch: BranchService = field(init=False) workspace: WorkspaceService = field(init=False) @@ -69,6 +74,7 @@ class ServiceRegistry: deep_lineage: DeepLineageService = field(init=False) sharing: SharingService = field(init=False) data_app: DataAppService = field(init=False) + dev_portal: DeveloperPortalService = field(init=False) semantic_layer: SemanticLayerService = field(init=False) repo_validate: RepoValidateService = field(init=False) mcp: McpService = field(init=False) @@ -77,6 +83,7 @@ class ServiceRegistry: search: SearchService = field(init=False) org: OrgService = field(init=False) member: MemberService = field(init=False) + feature: FeatureService = field(init=False) sync: SyncService = field(init=False) variables: VariablesService = field(init=False) doctor: DoctorService = field(init=False) @@ -88,6 +95,7 @@ def __post_init__(self) -> None: self.config = ConfigService(config_store=cs) self.component = ComponentService(config_store=cs) self.storage = StorageService(config_store=cs) + self.stream = StreamService(config_store=cs) self.job = JobService(config_store=cs) self.branch = BranchService(config_store=cs) self.workspace = WorkspaceService(config_store=cs) @@ -97,6 +105,10 @@ def __post_init__(self) -> None: self.deep_lineage = DeepLineageService(config_store=cs) self.sharing = SharingService(config_store=cs) self.data_app = DataAppService(config_store=cs) + self.dev_portal = DeveloperPortalService( + config_store=cs, + client_factory=lambda identity: DeveloperPortalClient(identity), + ) # SemanticLayerService takes both a storage client_factory (for # validate --deep + add dataset --deep-fields + build) and an # optional metastore_client_factory; the defaults work for both. @@ -108,6 +120,7 @@ def __post_init__(self) -> None: self.search = SearchService(config_store=cs) self.org = OrgService(config_store=cs) self.member = MemberService(config_store=cs) + self.feature = FeatureService(config_store=cs) self.sync = SyncService(config_store=cs) self.variables = VariablesService(config_store=cs) self.doctor = DoctorService(config_store=cs, mcp_service=self.mcp) diff --git a/src/keboola_agent_cli/server/routers/dev_portal.py b/src/keboola_agent_cli/server/routers/dev_portal.py new file mode 100644 index 00000000..d8a882aa --- /dev/null +++ b/src/keboola_agent_cli/server/routers/dev_portal.py @@ -0,0 +1,67 @@ +"""Developer Portal reads (list/get). + +Only the read surface is exposed over REST. The write commands +(`create`, `patch`, `upload-icon`, `publish`, `deprecate`) are deliberately +CLI-only: they require a human to type a random confirmation code on a real +TTY, which has no meaning over HTTP. Identity management +(`identity add/edit/remove/...`) is likewise CLI-only because it handles +login credentials that must not travel over this API. See +`plugins/kbagent/skills/kbagent/references/dev-portal-workflow.md`. +""" + +from __future__ import annotations + +from typing import Any + +from fastapi import APIRouter, Depends, HTTPException + +from ..dependencies import ServiceRegistry, get_registry + +router = APIRouter(prefix="/dev-portal", tags=["dev-portal"]) + + +def _resolve_identity(registry: ServiceRegistry, identity: str | None) -> str: + """Resolve the identity alias: explicit query param, else the configured default.""" + if identity: + return identity + default = registry.dev_portal.current_identity() + if not default: + raise HTTPException( + status_code=400, + detail=( + "No Developer Portal identity selected. Pass ?identity=, " + "or set a default via `kbagent dev-portal identity use `." + ), + ) + return default + + +@router.get("/apps", summary="List Developer Portal apps for a vendor") +def list_apps( + vendor: str, + identity: str | None = None, + registry: ServiceRegistry = Depends(get_registry), +) -> list[dict[str, Any]]: + """List all apps for a vendor. Mirrors `kbagent dev-portal list --vendor`.""" + alias = _resolve_identity(registry, identity) + return registry.dev_portal.list_apps(alias, vendor) + + +@router.get("/apps/{app}", summary="Get one Developer Portal app") +def get_app( + app: str, + identity: str | None = None, + registry: ServiceRegistry = Depends(get_registry), +) -> dict[str, Any]: + """Full portal entry for one app. `app` is VENDOR.APP_ID, e.g. keboola.ex-foo. + + Mirrors `kbagent dev-portal get --app`. + """ + if "." not in app: + raise HTTPException( + status_code=400, + detail=f"app must be in VENDOR.APP_ID form (e.g. keboola.ex-foo), got: {app!r}", + ) + vendor, _ = app.split(".", 1) + alias = _resolve_identity(registry, identity) + return registry.dev_portal.get_app(alias, vendor, app) diff --git a/src/keboola_agent_cli/server/routers/feature.py b/src/keboola_agent_cli/server/routers/feature.py new file mode 100644 index 00000000..cb6b4d6a --- /dev/null +++ b/src/keboola_agent_cli/server/routers/feature.py @@ -0,0 +1,179 @@ +"""Feature-flag endpoints (stack catalogue / project / user) -- all require a manage token. + +1:1 mirror of the `kbagent feature` command group. Every operation hits the +Manage API and therefore needs the per-request Manage token alongside the +standard bearer token, exactly like the `members` and `org` routers. +""" + +from __future__ import annotations + +from typing import Any + +from fastapi import APIRouter, Depends, HTTPException +from pydantic import BaseModel + +from ..dependencies import ServiceRegistry, get_manage_token, get_registry + +router = APIRouter(prefix="/feature", tags=["feature"]) + +# Every feature endpoint hits the Manage API, so each needs the per-request +# Manage token alongside the bearer token. Declaring the joint requirement +# here surfaces it as a separate scheme in the Swagger UI "Authorize" dialog. +_NEEDS_MANAGE_TOKEN: dict[str, Any] = {"security": [{"BearerAuth": [], "ManageToken": []}]} + + +class ProjectFeatureBody(BaseModel): + feature: str + dry_run: bool = False + + +class UserFeatureBody(BaseModel): + email: str + feature: str + dry_run: bool = False + + +def _require_manage(token: str | None) -> str: + if not token: + raise HTTPException(status_code=401, detail="Missing X-Manage-Token header.") + return token + + +@router.get("/{project}/list", summary="Stack feature catalogue", openapi_extra=_NEEDS_MANAGE_TOKEN) +def list_features( + project: str, + manage_token: str | None = Depends(get_manage_token), + registry: ServiceRegistry = Depends(get_registry), +) -> dict[str, Any]: + """Every feature defined on the stack `project` points at. The alias only + resolves the stack URL -- the catalogue is stack-wide. Mirrors + `kbagent feature list`. + """ + return registry.feature.list_stack_features( + manage_token=_require_manage(manage_token), alias=project + ) + + +@router.get( + "/{project}/project-show", + summary="Project's assigned features", + openapi_extra=_NEEDS_MANAGE_TOKEN, +) +def project_show( + project: str, + manage_token: str | None = Depends(get_manage_token), + registry: ServiceRegistry = Depends(get_registry), +) -> dict[str, Any]: + """Features assigned to `project`. Mirrors `kbagent feature project-show`.""" + return registry.feature.list_project_features( + manage_token=_require_manage(manage_token), alias=project + ) + + +@router.post( + "/{project}/project-add", + summary="Enable a feature on a project", + openapi_extra=_NEEDS_MANAGE_TOKEN, +) +def project_add( + project: str, + body: ProjectFeatureBody, + manage_token: str | None = Depends(get_manage_token), + registry: ServiceRegistry = Depends(get_registry), +) -> dict[str, Any]: + """Enable a feature on `project`. Pass `dry_run=true` to preview. Mirrors + `kbagent feature project-add`. + """ + return registry.feature.add_project_feature( + manage_token=_require_manage(manage_token), + alias=project, + feature=body.feature, + dry_run=body.dry_run, + ) + + +@router.post( + "/{project}/project-remove", + summary="Disable a feature on a project", + openapi_extra=_NEEDS_MANAGE_TOKEN, +) +def project_remove( + project: str, + body: ProjectFeatureBody, + manage_token: str | None = Depends(get_manage_token), + registry: ServiceRegistry = Depends(get_registry), +) -> dict[str, Any]: + """Disable a feature on `project` (destructive). Pass `dry_run=true` to + preview. Mirrors `kbagent feature project-remove`. + """ + return registry.feature.remove_project_feature( + manage_token=_require_manage(manage_token), + alias=project, + feature=body.feature, + dry_run=body.dry_run, + ) + + +@router.get( + "/{project}/user-show", + summary="User's assigned features", + openapi_extra=_NEEDS_MANAGE_TOKEN, +) +def user_show( + project: str, + email: str, + manage_token: str | None = Depends(get_manage_token), + registry: ServiceRegistry = Depends(get_registry), +) -> dict[str, Any]: + """Features assigned to `email` on the alias's stack. Mirrors + `kbagent feature user-show`. + """ + return registry.feature.list_user_features( + manage_token=_require_manage(manage_token), alias=project, email=email + ) + + +@router.post( + "/{project}/user-add", + summary="Enable a feature on a user", + openapi_extra=_NEEDS_MANAGE_TOKEN, +) +def user_add( + project: str, + body: UserFeatureBody, + manage_token: str | None = Depends(get_manage_token), + registry: ServiceRegistry = Depends(get_registry), +) -> dict[str, Any]: + """Enable a feature on a user. Pass `dry_run=true` to preview. Mirrors + `kbagent feature user-add`. + """ + return registry.feature.add_user_feature( + manage_token=_require_manage(manage_token), + alias=project, + email=body.email, + feature=body.feature, + dry_run=body.dry_run, + ) + + +@router.post( + "/{project}/user-remove", + summary="Disable a feature on a user", + openapi_extra=_NEEDS_MANAGE_TOKEN, +) +def user_remove( + project: str, + body: UserFeatureBody, + manage_token: str | None = Depends(get_manage_token), + registry: ServiceRegistry = Depends(get_registry), +) -> dict[str, Any]: + """Disable a feature on a user (destructive). Pass `dry_run=true` to + preview. Mirrors `kbagent feature user-remove`. + """ + return registry.feature.remove_user_feature( + manage_token=_require_manage(manage_token), + alias=project, + email=body.email, + feature=body.feature, + dry_run=body.dry_run, + ) diff --git a/src/keboola_agent_cli/server/routers/semantic_layer.py b/src/keboola_agent_cli/server/routers/semantic_layer.py index 7f28b0a5..43182560 100644 --- a/src/keboola_agent_cli/server/routers/semantic_layer.py +++ b/src/keboola_agent_cli/server/routers/semantic_layer.py @@ -22,7 +22,7 @@ from pathlib import Path from typing import Any, Literal -from fastapi import APIRouter, Depends, HTTPException +from fastapi import APIRouter, Depends, HTTPException, Query from pydantic import BaseModel, Field, model_validator from ...errors import ErrorCode @@ -256,6 +256,34 @@ def validate( ) +@router.get("/search-context", summary="Search semantic contexts by name pattern") +def search_context( + project: str, + pattern: list[str] = Query(default=["*"]), + type: str = "all", + limit: int | None = None, + registry: ServiceRegistry = Depends(get_registry), +) -> dict[str, Any]: + """Project-wide glob search across semantic-layer entities. + + Mirrors ``kbagent semantic-layer search-context``. See the service-layer + docstring for matching semantics and the returned envelope shape. + """ + return registry.semantic_layer.search_context( + alias=project, patterns=pattern, type_filter=type, limit=limit + ) + + +@router.get("/get-context", summary="Fetch one semantic context by id") +def get_context( + project: str, + context_id: str, + registry: ServiceRegistry = Depends(get_registry), +) -> dict[str, Any]: + """Single-entry fetch by id; probes every type until found.""" + return registry.semantic_layer.get_context(alias=project, context_id=context_id) + + @router.get("/export", summary="Export model snapshot") def export( project: str, diff --git a/src/keboola_agent_cli/server/routers/storage.py b/src/keboola_agent_cli/server/routers/storage.py index 2ee5f2a4..ef18de87 100644 --- a/src/keboola_agent_cli/server/routers/storage.py +++ b/src/keboola_agent_cli/server/routers/storage.py @@ -31,6 +31,7 @@ class CreateTable(BaseModel): not_null_columns: list[str] | None = None defaults: list[str] | None = None branch_id: int | None = None + if_not_exists: bool = False class DescribeBucket(BaseModel): @@ -67,6 +68,10 @@ class SwapTables(BaseModel): branch_id: int +class CloneTable(BaseModel): + branch_id: int + + @router.get("/buckets", summary="List storage buckets") def list_buckets( project: str | None = None, @@ -237,6 +242,7 @@ def create_table( branch_id=body.branch_id, not_null_columns=body.not_null_columns, defaults=body.defaults, + if_not_exists=body.if_not_exists, ) @@ -340,6 +346,29 @@ def swap_tables( ) +@router.post( + "/tables/{project}/{table_id:path}/pull", + summary="Clone a table into a dev branch", +) +def clone_table( + project: str, + table_id: str, + body: CloneTable, + dry_run: bool = False, + registry: ServiceRegistry = Depends(get_registry), +) -> dict[str, Any]: + """Pull (clone) a production table into a dev branch. + + Mirrors `kbagent storage clone-table`. + """ + return registry.storage.clone_table( + alias=project, + table_id=table_id, + branch_id=body.branch_id, + dry_run=dry_run, + ) + + @router.post("/tables/{project}/{table_id:path}/describe", summary="Set table description") def describe_table( project: str, diff --git a/src/keboola_agent_cli/server/routers/stream.py b/src/keboola_agent_cli/server/routers/stream.py new file mode 100644 index 00000000..f5bf363a --- /dev/null +++ b/src/keboola_agent_cli/server/routers/stream.py @@ -0,0 +1,100 @@ +"""Data Streams endpoints -- 1:1 mirror of the `kbagent stream` command group. + +Every operation hits the Stream control-plane API authenticated with the +per-project Storage API token that the service resolves from config -- so, +unlike the `feature` router, no extra per-request token header is required. +""" + +from __future__ import annotations + +from typing import Any + +from fastapi import APIRouter, Depends, Query +from pydantic import BaseModel + +from ..dependencies import ServiceRegistry, get_registry + +router = APIRouter(prefix="/stream", tags=["stream"]) + + +class CreateSourceBody(BaseModel): + name: str + source_type: str = "otlp" + branch_id: str | None = None + if_not_exists: bool = False + reveal: bool = False + provision_sinks: bool = True + + +class DeleteSourceBody(BaseModel): + source_id: str + branch_id: str | None = None + dry_run: bool = False + + +@router.get("/{project}/list", summary="List Data Streams sources") +def list_sources( + project: str, + branch: str | None = Query(None, description="Branch ref (default branch if unset)"), + registry: ServiceRegistry = Depends(get_registry), +) -> dict[str, Any]: + """List sources in `project`. Mirrors `kbagent stream list`.""" + return registry.stream.list_sources(alias=project, branch_id=branch) + + +@router.post("/{project}/create-source", summary="Create an OTLP/HTTP source") +def create_source( + project: str, + body: CreateSourceBody, + registry: ServiceRegistry = Depends(get_registry), +) -> dict[str, Any]: + """Create a source and return its (masked) endpoint. Mirrors + `kbagent stream create-source`. Pass `reveal=true` to include the secret. + """ + return registry.stream.create_source( + alias=project, + name=body.name, + source_type=body.source_type, + branch_id=body.branch_id, + if_not_exists=body.if_not_exists, + reveal=body.reveal, + provision_sinks=body.provision_sinks, + ) + + +@router.get("/{project}/detail", summary="Source endpoints + destination") +def source_detail( + project: str, + source_id: str | None = Query(None, description="Source id (or use name)"), + name: str | None = Query(None, description="Look up the source by name"), + branch: str | None = Query(None, description="Branch ref (default branch if unset)"), + reveal: bool = Query(False, description="Include the full endpoint incl. secret"), + registry: ServiceRegistry = Depends(get_registry), +) -> dict[str, Any]: + """Assemble one source's endpoints/protocol/destination. Mirrors + `kbagent stream detail`. Secret is masked unless `reveal=true`. + """ + return registry.stream.get_source_detail( + alias=project, + source_id=source_id, + name=name, + branch_id=branch, + reveal=reveal, + ) + + +@router.post("/{project}/delete", summary="Delete a source (destructive)") +def delete_source( + project: str, + body: DeleteSourceBody, + registry: ServiceRegistry = Depends(get_registry), +) -> dict[str, Any]: + """Delete a source. Pass `dry_run=true` to preview. Mirrors + `kbagent stream delete`. + """ + return registry.stream.delete_source( + alias=project, + source_id=body.source_id, + branch_id=body.branch_id, + dry_run=body.dry_run, + ) diff --git a/src/keboola_agent_cli/services/_semantic_layer_lookup.py b/src/keboola_agent_cli/services/_semantic_layer_lookup.py new file mode 100644 index 00000000..cce29827 --- /dev/null +++ b/src/keboola_agent_cli/services/_semantic_layer_lookup.py @@ -0,0 +1,181 @@ +"""Project-wide context search / lookup helpers for :mod:`semantic_layer_service`. + +Split out so :class:`SemanticLayerService` stays under the CONTRIBUTING.md +services hard ceiling (1,500 LOC). Each helper opens + closes its own +metastore client via the factory the service injects; the service methods +are 1-line delegators. + +Helpers: + +- :func:`run_search_context` -- project-wide glob search across semantic-layer + entity names (mirrors MCP ``search_semantic_context``). +- :func:`run_get_context` -- single fetch by id, irrespective of type (mirrors + MCP ``get_semantic_context``). +""" + +from __future__ import annotations + +import fnmatch +from typing import TYPE_CHECKING, Any + +from ..errors import ErrorCode, KeboolaApiError + +if TYPE_CHECKING: + from collections.abc import Callable + + from ..metastore_client import MetastoreClient, SemanticType + + +# Probed first by :func:`run_get_context`; child types follow the canonical +# iteration order so the sweep is deterministic. +_MODEL_TYPE: SemanticType = "semantic-model" + + +def _strip_semantic_prefix(wire_type: str) -> str: + """``"semantic-dataset"`` -> ``"dataset"`` for the CLI surface.""" + return wire_type[len("semantic-") :] if wire_type.startswith("semantic-") else wire_type + + +def _matches_any_pattern(name: str, patterns: list[str]) -> bool: + """Case-sensitive ``fnmatch`` against any of the supplied patterns.""" + return any(fnmatch.fnmatchcase(name, pat) for pat in patterns) + + +def _resolve_search_types( + type_filter: str | None, + child_types: tuple[SemanticType, ...], + type_alias: dict[str, SemanticType], +) -> tuple[SemanticType, ...]: + """Map the CLI ``--type`` flag to the list of wire types to scan.""" + if type_filter is None or type_filter == "all": + return child_types + if type_filter == "model": + return (_MODEL_TYPE,) + if type_filter in type_alias: + return (type_alias[type_filter],) + allowed = ["all", "model", *sorted(type_alias)] + raise KeboolaApiError( + message=f"Invalid --type {type_filter!r}. Must be one of: {', '.join(allowed)}.", + error_code=ErrorCode.VALIDATION_ERROR, + ) + + +def run_search_context( + *, + open_client: Callable[[], MetastoreClient], + alias: str, + child_types: tuple[SemanticType, ...], + type_alias: dict[str, SemanticType], + patterns: list[str] | None, + type_filter: str | None, + limit: int | None, +) -> dict[str, Any]: + """Project-wide glob search across semantic-layer entity names. + + Mirrors the upstream ``keboola-mcp-server`` ``search_semantic_context`` + MCP tool. ``patterns`` default to ``["*"]`` and are matched case- + sensitively against ``attributes.name``; multiple patterns take the + union. ``type_filter`` ``None`` / ``"all"`` -> every entry in + ``child_types``; ``"model"`` -> semantic models; any other CLI + singular narrows to that single wire type via ``type_alias``. + + Returns ``{"project", "contexts", "total_count"}``; each context is + ``{"id", "type", "name", "description", "attributes"}`` with the + wire ``"semantic-"`` prefix stripped from ``type`` for CLI ergonomics. + Raises :data:`ErrorCode.VALIDATION_ERROR` for empty patterns, non- + positive ``limit``, or an unknown ``type_filter``. + + Opens + closes its own metastore client via ``open_client()`` so the + service method is a one-line delegator. + """ + eff_patterns: list[str] = patterns or ["*"] + if any(not p for p in eff_patterns): + raise KeboolaApiError( + message="--pattern values must be non-empty strings", + error_code=ErrorCode.VALIDATION_ERROR, + ) + if limit is not None and limit <= 0: + raise KeboolaApiError( + message="--limit must be a positive integer", + error_code=ErrorCode.VALIDATION_ERROR, + ) + types_to_search = _resolve_search_types(type_filter, child_types, type_alias) + + contexts: list[dict[str, Any]] = [] + with open_client() as client: + for wire_type in types_to_search: + for item in client.list_items(wire_type): + attrs = item.get("attributes") or {} + name = str(attrs.get("name", "")) + if not _matches_any_pattern(name, eff_patterns): + continue + contexts.append( + { + "id": item.get("id", ""), + "type": _strip_semantic_prefix(wire_type), + "name": name, + "description": attrs.get("description", ""), + "attributes": attrs, + } + ) + if limit is not None and len(contexts) >= limit: + break + if limit is not None and len(contexts) >= limit: + break + + return {"project": alias, "contexts": contexts, "total_count": len(contexts)} + + +def run_get_context( + *, + open_client: Callable[[], MetastoreClient], + alias: str, + child_types: tuple[SemanticType, ...], + context_id: str, +) -> dict[str, Any]: + """Single-id fetch across every semantic type. + + Probes ``semantic-model`` first then every entry in ``child_types``, + stopping on the first 200. A 404 on any one type is non-terminal; + only a full miss raises ``NOT_FOUND``. Non-404 errors (e.g. 500) + propagate immediately rather than being swallowed by the next probe. + + Returns ``{"project", "id", "type", "name", "description", "attributes"}`` + on hit (type stripped of the ``"semantic-"`` wire prefix). Raises + :data:`ErrorCode.VALIDATION_ERROR` on empty id, or + :data:`ErrorCode.NOT_FOUND` after the full sweep. + + Opens + closes its own metastore client via ``open_client()``. + """ + if not context_id: + raise KeboolaApiError( + message="--context-id is required", + error_code=ErrorCode.VALIDATION_ERROR, + ) + + lookup_order: tuple[SemanticType, ...] = (_MODEL_TYPE, *child_types) + with open_client() as client: + for wire_type in lookup_order: + try: + item = client.get_item(wire_type, context_id) + except KeboolaApiError as exc: + if exc.error_code == ErrorCode.NOT_FOUND: + continue + raise + attrs = item.get("attributes") or {} + return { + "project": alias, + "id": item.get("id", ""), + "type": _strip_semantic_prefix(wire_type), + "name": attrs.get("name", ""), + "description": attrs.get("description", ""), + "attributes": attrs, + } + + raise KeboolaApiError( + message=( + f"Semantic context with id {context_id!r} not found in project " + f"{alias!r}. Tried: semantic-model + {', '.join(child_types)}." + ), + error_code=ErrorCode.NOT_FOUND, + ) diff --git a/src/keboola_agent_cli/services/dev_portal_service.py b/src/keboola_agent_cli/services/dev_portal_service.py new file mode 100644 index 00000000..39e7306f --- /dev/null +++ b/src/keboola_agent_cli/services/dev_portal_service.py @@ -0,0 +1,345 @@ +"""Developer Portal business logic. + +Identity CRUD + prepare/apply discipline for portal writes. Commands stay +thin; this module owns diff computation, publish pre-flight validation, +and the verify-on-add login probe. +""" + +from __future__ import annotations + +import logging +from collections.abc import Callable, Iterator +from contextlib import contextmanager +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +from ..config_store import ConfigStore +from ..dev_portal_client import DeveloperPortalClient +from ..errors import ConfigError, ErrorCode, KeboolaApiError +from ..models import DeveloperPortalIdentity + +ClientFactory = Callable[[DeveloperPortalIdentity], DeveloperPortalClient] + +_log = logging.getLogger(__name__) + +_BANNED_NAME_WORDS = ("extractor", "writer") +_REQUIRED_PUBLISH_FIELDS = ( + "icon", + "name", + "type", + "repository", + "shortDescription", + "longDescription", + "licenseUrl", + "documentationUrl", +) + +# Fields that the apps-api server `.forbidden()`s on the vendor PATCH endpoint +# (PATCH /vendors/{vendor}/apps/{app}). Settable only via PATCH /admin/apps/{app} +# with an admin-role token. Sending any of these on the vendor endpoint returns +# a 422 with a misleading "must be one of: ..." error message because the +# enum-validation error annotation is attached in the shared admin schema +# before clientAppSchema overrides with `.forbidden()`. Source of truth: +# keboola/developer-portal:src/lib/validation.js -> clientAppSchema(). +_ADMIN_ONLY_PATCH_FIELDS = frozenset( + { + "category", + "categories", + "complexity", + "features", + "forwardToken", + "forwardTokenDetails", + "injectEnvironment", + "processTimeout", + "requiredMemory", + } +) + + +@dataclass(frozen=True) +class FieldDiff: + key: str + current: Any + new: Any + + +@dataclass(frozen=True) +class PendingWrite: + """Base for any prepared portal write. apply() in the service dispatches on the subclass.""" + + alias: str + vendor: str + + +@dataclass(frozen=True) +class PendingCreate(PendingWrite): + payload: dict[str, Any] + + +@dataclass(frozen=True) +class PendingPatch(PendingWrite): + app_id: str + payload: dict[str, Any] + current: dict[str, Any] + diff: list[FieldDiff] = field(default_factory=list) + + +@dataclass(frozen=True) +class PendingIconUpload(PendingWrite): + app_id: str + png_path: Path + png_bytes: bytes + + +@dataclass(frozen=True) +class PendingPublish(PendingWrite): + app_id: str + current: dict[str, Any] + + +@dataclass(frozen=True) +class PendingDeprecate(PendingWrite): + app_id: str + + +class DeveloperPortalService: + def __init__( + self, + config_store: ConfigStore, + client_factory: ClientFactory, + ) -> None: + self._store = config_store + self._client_factory = client_factory + # One bearer per alias, reused across a prepare/apply pair within a + # single command invocation. In-memory only; the service instance is + # rebuilt per CLI call, so nothing leaks across invocations. + self._bearers: dict[str, str] = {} + + # ----- Identity management ----- + + def add_identity(self, alias: str, identity: DeveloperPortalIdentity) -> None: + """Verify creds (login probe) BEFORE persisting. + + Same UX as `kbagent project add` (which calls verify_token first): + bad creds fail fast and never land in config.json. + """ + with self._client_factory(identity) as client: + client._ensure_authenticated() # raises on bad creds / MFA failure + self._store.add_dev_portal_identity(alias, identity) + + def list_identities(self) -> dict[str, DeveloperPortalIdentity]: + return dict(self._store.load().dev_portal_identities) + + def remove_identity(self, alias: str) -> None: + self._store.remove_dev_portal_identity(alias) + + def edit_identity(self, alias: str, **fields: Any) -> None: + self._store.edit_dev_portal_identity(alias, **fields) + + def rename_identity(self, old_alias: str, new_alias: str) -> None: + self._store.rename_dev_portal_identity(old_alias, new_alias) + + def use_identity(self, alias: str) -> None: + self._store.set_default_dev_portal_identity(alias) + + def current_identity(self) -> str: + return self._store.load().default_dev_portal_identity + + def verify_identity(self, alias: str) -> dict[str, str]: + ident = self._resolve_identity(alias) + with self._client_factory(ident) as client: + client._ensure_authenticated() + return {"alias": alias, "username": ident.username} + + # ----- Internal ----- + + def _resolve_identity(self, alias: str) -> DeveloperPortalIdentity: + ident = self._store.get_dev_portal_identity(alias) + if ident is None: + raise ConfigError( + f"Developer Portal identity '{alias}' not found. " + "Run `kbagent dev-portal identity list` to see configured identities." + ) + return ident + + @contextmanager + def _authed_client(self, alias: str) -> Iterator[DeveloperPortalClient]: + """Yield a portal client that reuses an existing login for this alias. + + The first call for an alias logs in (prompting MFA once on a personal + account); any later call within the same command invocation reuses the + bearer obtained earlier. This is what stops `patch`/`publish` -- which + open a client in `prepare_*` to read current state and again in + `apply()` to write -- from logging in (and MFA-prompting) twice. + + A cached bearer can go stale: portal tokens have a finite TTL, and in + `kbagent serve` this service lives in a long-lived singleton registry, + so a bearer can outlive its validity across requests. On an auth + failure the cached bearer is evicted, so the NEXT call re-authenticates + instead of replaying a dead token forever (which would lock out serve + until a restart). + """ + ident = self._resolve_identity(alias) + client = self._client_factory(ident) + cached = self._bearers.get(alias) + if cached is not None: + client.seed_bearer(cached) + with client: + try: + yield client + except KeboolaApiError as exc: + if exc.error_code in (ErrorCode.INVALID_TOKEN, ErrorCode.DP_LOGIN_FAILED): + self._bearers.pop(alias, None) + raise + new_bearer = client.bearer + if new_bearer is not None: + self._bearers[alias] = new_bearer + + # ----- Reads ----- + + def list_apps(self, alias: str, vendor: str) -> list[dict[str, Any]]: + with self._authed_client(alias) as client: + return client.list_apps(vendor) + + def get_app(self, alias: str, vendor: str, app_id: str) -> dict[str, Any]: + with self._authed_client(alias) as client: + return client.get_app(vendor, app_id) + + # ----- Prepare (no portal write yet) ----- + + def prepare_create(self, alias: str, vendor: str, payload: dict[str, Any]) -> PendingCreate: + for required in ("id", "name", "type"): + if required not in payload: + raise KeboolaApiError( + message=f"create payload must include '{required}'", + error_code=ErrorCode.VALIDATION_ERROR, + ) + name_lower = str(payload["name"]).lower() + for banned in _BANNED_NAME_WORDS: + if banned in name_lower: + raise KeboolaApiError( + message=( + f"App name must not contain {_BANNED_NAME_WORDS!r}; got {payload['name']!r}" + ), + error_code=ErrorCode.VALIDATION_ERROR, + ) + # Confirm identity exists; defer login until apply(). + self._resolve_identity(alias) + return PendingCreate(alias=alias, vendor=vendor, payload=payload) + + def prepare_patch( + self, + alias: str, + vendor: str, + app_id: str, + payload: dict[str, Any], + ) -> PendingPatch: + ident = self._resolve_identity(alias) + admin_only_in_payload = sorted(_ADMIN_ONLY_PATCH_FIELDS & set(payload.keys())) + if admin_only_in_payload and ident.role_hint != "admin": + raise KeboolaApiError( + message=( + f"Cannot patch admin-only field(s) via the vendor endpoint: " + f"{admin_only_in_payload}. The apps-api server forbids these " + f"on PATCH /vendors/{vendor}/apps/{app_id} (the 422 you'd " + "otherwise see -- 'must be one of: ...' -- is a misleading " + "server-side message hiding the real reason: the field is " + "forbidden() on the vendor schema, not enum-validated). " + "Switch to an admin identity to route this PATCH through " + "/admin/apps/{app} instead: `kbagent dev-portal identity add " + "--alias --username --role-hint admin " + "--password-stdin` and re-run with `--identity `. " + "Alternatively, drop the field and ask a Developer Portal " + "admin to set it. Canonical list: keboola/developer-portal " + "src/lib/validation.js -> clientAppSchema()." + ), + error_code=ErrorCode.VALIDATION_ERROR, + ) + with self._authed_client(alias) as client: + current = client.get_app(vendor, app_id) + diff = [ + FieldDiff(key=k, current=current.get(k), new=v) + for k, v in payload.items() + if current.get(k) != v + ] + return PendingPatch( + alias=alias, + vendor=vendor, + app_id=app_id, + payload=payload, + current=current, + diff=diff, + ) + + def prepare_upload_icon( + self, alias: str, vendor: str, app_id: str, path: str | Path + ) -> PendingIconUpload: + import struct + + p = Path(path) + if not p.is_file(): + raise KeboolaApiError( + message=f"Icon file not found: {p}", + error_code=ErrorCode.FILE_NOT_FOUND, + ) + data = p.read_bytes() + if not data.startswith(b"\x89PNG\r\n\x1a\n"): + raise KeboolaApiError( + message=f"Icon file is not a PNG: {p}", + error_code=ErrorCode.VALIDATION_ERROR, + ) + if len(data) >= 24: + width, height = struct.unpack(">II", data[16:24]) + if (width, height) != (128, 128): + _log.warning( + "Icon is %dx%d, not 128x128 — portal may reject it.", + width, + height, + ) + self._resolve_identity(alias) + return PendingIconUpload( + alias=alias, + vendor=vendor, + app_id=app_id, + png_path=p, + png_bytes=data, + ) + + def prepare_publish(self, alias: str, vendor: str, app_id: str) -> PendingPublish: + with self._authed_client(alias) as client: + current = client.get_app(vendor, app_id) + missing = [f for f in _REQUIRED_PUBLISH_FIELDS if not current.get(f)] + if missing: + raise KeboolaApiError( + message=( + f"Cannot publish {app_id}: missing required fields " + f"{missing}. Fix them via `kbagent dev-portal patch` first." + ), + error_code=ErrorCode.DP_PUBLISH_REQUIREMENTS_MISSING, + ) + return PendingPublish(alias=alias, vendor=vendor, app_id=app_id, current=current) + + def prepare_deprecate(self, alias: str, vendor: str, app_id: str) -> PendingDeprecate: + self._resolve_identity(alias) + return PendingDeprecate(alias=alias, vendor=vendor, app_id=app_id) + + # ----- Apply (calls the portal write) ----- + + def apply(self, pending: PendingWrite) -> dict[str, Any]: + with self._authed_client(pending.alias) as client: + if isinstance(pending, PendingCreate): + return client.create_app(pending.vendor, pending.payload) + if isinstance(pending, PendingPatch): + return client.patch_app(pending.vendor, pending.app_id, pending.payload) + if isinstance(pending, PendingIconUpload): + client.upload_icon(pending.vendor, pending.app_id, pending.png_bytes) + return {"status": "uploaded", "app": pending.app_id} + if isinstance(pending, PendingPublish): + return client.publish_app(pending.vendor, pending.app_id) + if isinstance(pending, PendingDeprecate): + return client.deprecate_app(pending.vendor, pending.app_id) + raise KeboolaApiError( + message=f"Unknown pending write type: {type(pending).__name__}", + error_code=ErrorCode.INTERNAL_ERROR, + ) diff --git a/src/keboola_agent_cli/services/feature_service.py b/src/keboola_agent_cli/services/feature_service.py new file mode 100644 index 00000000..e9c75858 --- /dev/null +++ b/src/keboola_agent_cli/services/feature_service.py @@ -0,0 +1,268 @@ +"""Feature-flag management service (super-admin Manage API). + +Wraps the stack feature catalogue (``GET /manage/features``) and the +project/user feature assignment endpoints behind a layer that: + +- resolves a kbagent project alias to its ``(stack_url, project_id)`` via + :class:`ConfigStore` (the alias is the only handle a caller needs -- the + numeric project ID and stack URL are looked up, never typed); +- normalises the ``features`` array on a project/user object, which the + Manage API may return either as a list of objects or a list of bare + strings, into a uniform list of :class:`Feature` dicts; +- supports ``dry_run`` previews for the write paths so an agent can show the + user exactly what would change before a super-admin token touches the stack. + +The Manage API token is never persisted -- it is passed in per call from the +interactive prompt resolved by the command layer (see ``resolve_manage_token``). +""" + +from __future__ import annotations + +import logging +from collections.abc import Callable +from dataclasses import dataclass +from typing import Any + +from ..config_store import ConfigStore +from ..errors import ConfigError +from ..manage_client import ManageClient +from ..models import Feature + +logger = logging.getLogger(__name__) + +ManageClientFactory = Callable[[str, str], ManageClient] + + +@dataclass(frozen=True) +class _ResolvedAlias: + """A project alias resolved to the two handles project ops need.""" + + stack_url: str + project_id: int + + +def default_manage_client_factory(stack_url: str, manage_token: str) -> ManageClient: + """Construct a :class:`ManageClient` bound to ``stack_url``.""" + return ManageClient(stack_url=stack_url, manage_token=manage_token) + + +def _normalise_features(raw: Any) -> list[dict[str, Any]]: + """Normalise a ``features`` payload into a list of Feature dicts. + + The Manage API returns features as either a list of objects or a list of + bare strings depending on the endpoint/stack version. Bare strings are + wrapped as ``{"name": }`` so downstream rendering is uniform. + """ + if not isinstance(raw, list): + return [] + out: list[dict[str, Any]] = [] + for item in raw: + if isinstance(item, str): + out.append(Feature(name=item).model_dump(by_alias=False)) + elif isinstance(item, dict): + out.append(Feature.model_validate(item).model_dump(by_alias=False)) + return out + + +class FeatureService: + """Business logic for stack, project, and user feature flags.""" + + def __init__( + self, + config_store: ConfigStore, + manage_client_factory: ManageClientFactory | None = None, + ) -> None: + self._config_store = config_store + self._manage_client_factory = manage_client_factory or default_manage_client_factory + + # ------------------------------------------------------------------ + # Stack catalogue + # ------------------------------------------------------------------ + + def list_stack_features(self, *, manage_token: str, alias: str) -> dict[str, Any]: + """List every feature defined on the stack the alias points at. + + The alias is used only to resolve the stack URL -- the catalogue is + stack-wide, not project-scoped. + """ + stack_url = self._resolve_stack_url(alias) + manage_client = self._manage_client_factory(stack_url, manage_token) + try: + raw = manage_client.list_features() + return { + "alias": alias, + "stack_url": stack_url, + "features": _normalise_features(raw), + } + finally: + manage_client.close() + + # ------------------------------------------------------------------ + # Project features + # ------------------------------------------------------------------ + + def list_project_features(self, *, manage_token: str, alias: str) -> dict[str, Any]: + """List features assigned to the project registered under ``alias``.""" + resolved = self._resolve_alias(alias) + manage_client = self._manage_client_factory(resolved.stack_url, manage_token) + try: + project = manage_client.get_project(resolved.project_id) + return { + "alias": alias, + "project_id": resolved.project_id, + "project_name": project.get("name", ""), + "features": _normalise_features(project.get("features")), + } + finally: + manage_client.close() + + def add_project_feature( + self, *, manage_token: str, alias: str, feature: str, dry_run: bool = False + ) -> dict[str, Any]: + """Enable ``feature`` on the project registered under ``alias``.""" + resolved = self._resolve_alias(alias) + if dry_run: + return { + "status": "dry_run", + "action": "add", + "alias": alias, + "project_id": resolved.project_id, + "feature": feature, + } + manage_client = self._manage_client_factory(resolved.stack_url, manage_token) + try: + manage_client.add_project_feature(resolved.project_id, feature) + return { + "status": "added", + "alias": alias, + "project_id": resolved.project_id, + "feature": feature, + } + finally: + manage_client.close() + + def remove_project_feature( + self, *, manage_token: str, alias: str, feature: str, dry_run: bool = False + ) -> dict[str, Any]: + """Disable ``feature`` on the project registered under ``alias``.""" + resolved = self._resolve_alias(alias) + if dry_run: + return { + "status": "dry_run", + "action": "remove", + "alias": alias, + "project_id": resolved.project_id, + "feature": feature, + } + manage_client = self._manage_client_factory(resolved.stack_url, manage_token) + try: + manage_client.remove_project_feature(resolved.project_id, feature) + return { + "status": "removed", + "alias": alias, + "project_id": resolved.project_id, + "feature": feature, + } + finally: + manage_client.close() + + # ------------------------------------------------------------------ + # User features + # ------------------------------------------------------------------ + + def list_user_features(self, *, manage_token: str, alias: str, email: str) -> dict[str, Any]: + """List features assigned to ``email`` on the alias's stack.""" + stack_url = self._resolve_stack_url(alias) + manage_client = self._manage_client_factory(stack_url, manage_token) + try: + user = manage_client.get_user(email) + return { + "alias": alias, + "stack_url": stack_url, + "email": email, + "features": _normalise_features(user.get("features")), + } + finally: + manage_client.close() + + def add_user_feature( + self, *, manage_token: str, alias: str, email: str, feature: str, dry_run: bool = False + ) -> dict[str, Any]: + """Enable ``feature`` on the user ``email``.""" + stack_url = self._resolve_stack_url(alias) + if dry_run: + return { + "status": "dry_run", + "action": "add", + "alias": alias, + "email": email, + "feature": feature, + } + manage_client = self._manage_client_factory(stack_url, manage_token) + try: + manage_client.add_user_feature(email, feature) + return { + "status": "added", + "alias": alias, + "email": email, + "feature": feature, + } + finally: + manage_client.close() + + def remove_user_feature( + self, *, manage_token: str, alias: str, email: str, feature: str, dry_run: bool = False + ) -> dict[str, Any]: + """Disable ``feature`` on the user ``email``.""" + stack_url = self._resolve_stack_url(alias) + if dry_run: + return { + "status": "dry_run", + "action": "remove", + "alias": alias, + "email": email, + "feature": feature, + } + manage_client = self._manage_client_factory(stack_url, manage_token) + try: + manage_client.remove_user_feature(email, feature) + return { + "status": "removed", + "alias": alias, + "email": email, + "feature": feature, + } + finally: + manage_client.close() + + # ------------------------------------------------------------------ + # Internal helpers + # ------------------------------------------------------------------ + + def _resolve_alias(self, alias: str) -> _ResolvedAlias: + """Resolve ``alias`` to its stack URL + numeric project_id for project ops.""" + project = self._config_store.get_project(alias) + if project is None: + raise ConfigError( + f"Project alias '{alias}' is not registered. Run `kbagent project list`." + ) + if project.project_id is None: + raise ConfigError( + f"Project alias '{alias}' has no numeric project_id; " + "re-add it via `kbagent project add` to populate it." + ) + return _ResolvedAlias(stack_url=project.stack_url, project_id=project.project_id) + + def _resolve_stack_url(self, alias: str) -> str: + """Resolve ``alias`` to its stack URL for stack/user ops. + + Unlike :meth:`_resolve_alias`, this does not require a numeric + project_id -- the stack catalogue and user features are not + project-scoped, the alias is only a handle to the stack URL. + """ + project = self._config_store.get_project(alias) + if project is None: + raise ConfigError( + f"Project alias '{alias}' is not registered. Run `kbagent project list`." + ) + return project.stack_url diff --git a/src/keboola_agent_cli/services/project_service.py b/src/keboola_agent_cli/services/project_service.py index 8b2e01bc..48353079 100644 --- a/src/keboola_agent_cli/services/project_service.py +++ b/src/keboola_agent_cli/services/project_service.py @@ -14,7 +14,7 @@ from ..constants import ENV_KBAGENT_PROJECT from ..errors import ConfigError, KeboolaApiError, mask_token -from ..models import ProjectConfig +from ..models import ProjectConfig, normalize_stack_url from .base import BaseService # Filesystem-safe slug constraint for ``--new-alias``. Aliases land on disk @@ -51,6 +51,10 @@ def add_project(self, alias: str, stack_url: str, token: str) -> dict[str, Any]: KeboolaApiError: If token verification fails. ConfigError: If the alias already exists. """ + # Accept a bare host or a full project deep-link, not just a clean base + # URL -- normalize before we build the verification client so the token + # check hits the right host (and the stored value is the clean base). + stack_url = normalize_stack_url(stack_url) client = self._client_factory(stack_url, token) try: token_info = client.verify_token() @@ -153,6 +157,11 @@ def edit_project( "--new-alias (matching the current alias is a no-op)." ) + # Normalize a bare host / full project deep-link to the clean base URL + # up front so the dry-run preview and the real edit agree on the value. + if stack_url is not None: + stack_url = normalize_stack_url(stack_url) + # ----- dry-run path: validate everything, mutate nothing ---------- if dry_run: planned_rename: dict[str, Any] | None = None @@ -626,6 +635,13 @@ def _backfill_org_info( current = self._config_store.get_project(alias) if current is None: continue + if current.ephemeral: + # Env-synthesized __env__ (issue #359): its org info can never + # be persisted (save() strips it), so backfilling is futile and + # would trigger a spurious config.json write on disk -- breaking + # the "no config.json in headless mode" guarantee and repeating + # on every `project status`. Skip it. + continue if current.org_id is not None and current.org_name: continue # already populated; skip updates[alias] = (new_id, new_name) diff --git a/src/keboola_agent_cli/services/semantic_layer_service.py b/src/keboola_agent_cli/services/semantic_layer_service.py index 2a0a1c2b..1b5fbd1d 100644 --- a/src/keboola_agent_cli/services/semantic_layer_service.py +++ b/src/keboola_agent_cli/services/semantic_layer_service.py @@ -52,6 +52,8 @@ from ._semantic_layer_internals import validate_basic as _validate_basic_helper from ._semantic_layer_internals import validate_deep as _validate_deep_helper from ._semantic_layer_internals import write_snapshot_to_file as _write_snapshot_to_file +from ._semantic_layer_lookup import run_get_context as _run_get_context_helper +from ._semantic_layer_lookup import run_search_context as _run_search_context_helper from .base import BaseService, ClientFactory from .encrypt_service import EncryptService from .storage_service import StorageService @@ -223,10 +225,7 @@ def __init__( metastore_client_factory or default_metastore_client_factory ) - # ------------------------------------------------------------------ - # Helpers (used by every subcommand) - # ------------------------------------------------------------------ - + # Helpers (used by every subcommand). def _resolve_one_project(self, alias: str) -> ProjectConfig: """Resolve a single project alias to its ``ProjectConfig`` or raise. @@ -240,20 +239,12 @@ def _new_metastore_client(self, project: ProjectConfig) -> MetastoreClient: return self._metastore_factory(project.stack_url, project.token) def _resolve_model( - self, - client: MetastoreClient, - model_name_or_uuid: str | None, + self, client: MetastoreClient, model_name_or_uuid: str | None ) -> tuple[str, dict[str, Any]]: - """Resolve a model selector to ``(uuid, attributes_dict)``. - - Body lives in :func:`._semantic_layer_internals.resolve_model_uuid`. - """ + """Resolve a model selector via :func:`._semantic_layer_internals.resolve_model_uuid`.""" return _resolve_model_uuid(client, model_name_or_uuid) - # ------------------------------------------------------------------ - # Phase 3 — Read commands - # ------------------------------------------------------------------ - + # Phase 3 — Read commands. def list_models(self, alias: str) -> dict[str, Any]: """List all semantic-layer models for a project. @@ -262,11 +253,8 @@ def list_models(self, alias: str) -> dict[str, Any]: ``{id, name, description, sql_dialect}``). """ project = self._resolve_one_project(alias) - client = self._new_metastore_client(project) - try: + with self._new_metastore_client(project) as client: raw = client.list_items("semantic-model") - finally: - client.close() models: list[dict[str, Any]] = [] for item in raw: @@ -281,9 +269,34 @@ def list_models(self, alias: str) -> dict[str, Any]: ) return {"project": alias, "models": models} - # ------------------------------------------------------------------ - # Internal helpers (model-scoped fetches) - # ------------------------------------------------------------------ + def search_context( + self, + alias: str, + patterns: list[str] | None = None, + type_filter: str | None = None, + limit: int | None = None, + ) -> dict[str, Any]: + """Project-wide glob search; see :func:`_semantic_layer_lookup.run_search_context`.""" + return _run_search_context_helper( + open_client=lambda: self._new_metastore_client(self._resolve_one_project(alias)), + alias=alias, + child_types=CHILD_TYPES, + type_alias=TYPE_ALIAS, + patterns=patterns, + type_filter=type_filter, + limit=limit, + ) + + def get_context(self, alias: str, context_id: str) -> dict[str, Any]: + """Single-id lookup; see :func:`_semantic_layer_lookup.run_get_context`.""" + return _run_get_context_helper( + open_client=lambda: self._new_metastore_client(self._resolve_one_project(alias)), + alias=alias, + child_types=CHILD_TYPES, + context_id=context_id, + ) + + # Internal helpers (model-scoped fetches). @staticmethod def _fetch_children_parallel( @@ -341,12 +354,9 @@ def show_model( ) project = self._resolve_one_project(alias) - client = self._new_metastore_client(project) - try: + with self._new_metastore_client(project) as client: model_uuid, model_attrs = self._resolve_model(client, model_name_or_uuid) raw_by_type = self._fetch_children_parallel(client, model_uuid) - finally: - client.close() result: dict[str, Any] = { "project": alias, @@ -377,12 +387,9 @@ def validate_model( check inventory. """ project = self._resolve_one_project(alias) - client = self._new_metastore_client(project) - try: + with self._new_metastore_client(project) as client: model_uuid, model_attrs = self._resolve_model(client, model_name_or_uuid) raw_by_type = self._fetch_children_parallel(client, model_uuid) - finally: - client.close() datasets = _unpack_attrs_with_id(raw_by_type.get("semantic-dataset", [])) metrics = _unpack_attrs_with_id(raw_by_type.get("semantic-metric", [])) @@ -479,12 +486,9 @@ def export_model( relationships, constraints, glossary, counts}``. """ project = self._resolve_one_project(alias) - client = self._new_metastore_client(project) - try: + with self._new_metastore_client(project) as client: model_uuid, model_attrs = self._resolve_model(client, model_name_or_uuid) raw_by_type = self._fetch_children_parallel(client, model_uuid) - finally: - client.close() snapshot = _build_export_snapshot( alias=alias, @@ -599,14 +603,11 @@ def create_model( ) -> dict[str, Any]: """Create a semantic-layer model and return the server-stored item.""" project = self._resolve_one_project(alias) - client = self._new_metastore_client(project) - try: + with self._new_metastore_client(project) as client: data: dict[str, Any] = {"name": name, "sql_dialect": sql_dialect} if description: data["description"] = description created = client.post_item("semantic-model", name=name, data=data) - finally: - client.close() return {"project": alias, "model": created} def delete_model( @@ -623,8 +624,7 @@ def delete_model( in the helper to keep this file under the 1500 LOC ceiling. """ project = self._resolve_one_project(alias) - client = self._new_metastore_client(project) - try: + with self._new_metastore_client(project) as client: model_uuid, model_attrs = self._resolve_model(client, model_name_or_uuid) children = self._fetch_children_parallel(client, model_uuid) return _cascade_delete_model_impl( @@ -634,8 +634,6 @@ def delete_model( model_attrs=model_attrs, children=children, ) - finally: - client.close() # ------------------------------------------------------------------ # Phase 4 — add subcommands @@ -662,8 +660,7 @@ def add_metric( rather than silently push a broken metric. """ project = self._resolve_one_project(alias) - client = self._new_metastore_client(project) - try: + with self._new_metastore_client(project) as client: model_uuid, _ = self._resolve_model(client, model_name_or_uuid) datasets = client.list_items("semantic-dataset", model_uuid) ds_tids = {(d.get("attributes") or {}).get("tableId", "") for d in datasets} @@ -694,8 +691,6 @@ def add_metric( if description: data["description"] = description return client.post_item("semantic-metric", name=name, data=data) - finally: - client.close() def add_dataset( self, @@ -715,8 +710,7 @@ def add_dataset( synthesises a ``fields[]`` array with role heuristics. """ project = self._resolve_one_project(alias) - client = self._new_metastore_client(project) - try: + with self._new_metastore_client(project) as client: model_uuid, _ = self._resolve_model(client, model_name_or_uuid) data: dict[str, Any] = { "name": name, @@ -741,8 +735,6 @@ def add_dataset( if fields: data["fields"] = fields return client.post_item("semantic-dataset", name=name, data=data) - finally: - client.close() def add_relationship( self, @@ -762,8 +754,7 @@ def add_relationship( error_code=ErrorCode.VALIDATION_ERROR, ) project = self._resolve_one_project(alias) - client = self._new_metastore_client(project) - try: + with self._new_metastore_client(project) as client: model_uuid, _ = self._resolve_model(client, model_name_or_uuid) data = { "name": name, @@ -774,8 +765,6 @@ def add_relationship( "modelUUID": model_uuid, } return client.post_item("semantic-relationship", name=name, data=data) - finally: - client.close() def add_constraint( self, @@ -800,8 +789,7 @@ def add_constraint( # METRICS exist in model project = self._resolve_one_project(alias) - client = self._new_metastore_client(project) - try: + with self._new_metastore_client(project) as client: model_uuid, _ = self._resolve_model(client, model_name_or_uuid) existing = client.list_items("semantic-metric", model_uuid) existing_names = {(m.get("attributes") or {}).get("name", "") for m in existing} @@ -823,8 +811,6 @@ def add_constraint( "modelUUID": model_uuid, } return client.post_item("semantic-constraint", name=name, data=data) - finally: - client.close() def add_glossary( self, @@ -836,15 +822,195 @@ def add_glossary( ) -> dict[str, Any]: """Create a glossary term. Outer envelope ``name`` must equal ``term``.""" project = self._resolve_one_project(alias) - client = self._new_metastore_client(project) - try: + with self._new_metastore_client(project) as client: model_uuid, _ = self._resolve_model(client, model_name_or_uuid) data: dict[str, Any] = {"term": term, "modelUUID": model_uuid} if definition: data["definition"] = definition return client.post_item("semantic-glossary", name=term, data=data) + + # ------------------------------------------------------------------ + # Reference data — dimension-member records (e.g. a Chart of Accounts) + # ------------------------------------------------------------------ + # + # ``semantic-reference-data`` stores the full member list of one + # dimension (one record per dimension, members in a ``members[]`` + # array). Unlike the five model-children, it is NOT AI-generated and is + # deliberately kept out of ``build`` / ``export`` / ``diff`` / cascade — + # it has its own self-contained CRUD surface here. + + _REFERENCE_DATA_TYPE: ClassVar[SemanticType] = "semantic-reference-data" + + @staticmethod + def _unpack_reference_record( + alias: str, + item: dict[str, Any], + *, + include_members: bool, + ) -> dict[str, Any]: + """Project a raw metastore item into the CLI reference-data shape.""" + attrs = item.get("attributes") or {} + members = attrs.get("members") or [] + out: dict[str, Any] = { + "project": alias, + "id": item.get("id", ""), + "dimension_name": attrs.get("dimensionName", ""), + "model_uuid": attrs.get("modelUUID", ""), + "dataset_id": attrs.get("datasetId"), + "description": attrs.get("description"), + "member_count": len(members), + "revision": (item.get("meta") or {}).get("revision"), + } + if include_members: + out["members"] = members + return out + + @staticmethod + def _find_reference_data_for_model( + client: MetastoreClient, + model_uuid: str, + dimension: str, + ) -> dict[str, Any] | None: + """Return the existing record for ``(model_uuid, dimension)`` or None.""" + for item in client.list_items("semantic-reference-data", model_uuid): + if (item.get("attributes") or {}).get("dimensionName") == dimension: + return item + return None + + def list_reference_data( + self, + alias: str, + model_name_or_uuid: str | None = None, + ) -> dict[str, Any]: + """List reference-data records (optionally scoped to one model). + + Returns ``{project, reference_data: [{id, dimension_name, + model_uuid, dataset_id, member_count}]}``. Member lists are omitted + from the summary — use ``get`` for the full members. + """ + project = self._resolve_one_project(alias) + client = self._new_metastore_client(project) + try: + model_uuid: str | None = None + if model_name_or_uuid is not None: + model_uuid, _ = self._resolve_model(client, model_name_or_uuid) + raw = client.list_items(self._REFERENCE_DATA_TYPE, model_uuid) finally: client.close() + records = [self._unpack_reference_record(alias, i, include_members=False) for i in raw] + for r in records: + r.pop("project", None) + return {"project": alias, "reference_data": records} + + def get_reference_data( + self, + alias: str, + *, + record_id: str | None = None, + model_name_or_uuid: str | None = None, + dimension: str | None = None, + ) -> dict[str, Any]: + """Fetch one record by ``record_id``, or by ``dimension``. + + When resolving by ``dimension``, ``model_name_or_uuid`` may be ``None`` + — it resolves to the project's default model like every other + model-scoped operation here. + """ + if record_id is None and dimension is None: + raise KeboolaApiError( + message="Provide --id, or --dimension (optionally with --model).", + error_code=ErrorCode.VALIDATION_ERROR, + ) + project = self._resolve_one_project(alias) + client = self._new_metastore_client(project) + try: + if record_id is not None: + item = client.get_item(self._REFERENCE_DATA_TYPE, record_id) + else: + # The guard above guarantees dimension is set on this branch. + assert dimension is not None + model_uuid, _ = self._resolve_model(client, model_name_or_uuid) + item = self._find_reference_data_for_model(client, model_uuid, dimension) + if item is None: + raise KeboolaApiError( + message=( + f"No reference-data record for dimension {dimension!r} " + f"in model {model_name_or_uuid!r}." + ), + error_code=ErrorCode.NOT_FOUND, + ) + finally: + client.close() + return self._unpack_reference_record(alias, item, include_members=True) + + def set_reference_data( + self, + alias: str, + model_name_or_uuid: str | None, + *, + dimension: str, + members: list[dict[str, Any]], + dataset_id: str | None = None, + description: str | None = None, + ) -> dict[str, Any]: + """Create or replace (by model + dimension) a reference-data record. + + Idempotent on ``(modelUUID, dimensionName)``: if a record already + exists it is replaced in place via ``PUT`` (revision increments, + history preserved); otherwise a new record is ``POST``-ed. The + envelope ``name`` is the dimension (unique per project per type). + """ + if not isinstance(members, list): + raise KeboolaApiError( + message="members must be a JSON array of member objects.", + error_code=ErrorCode.VALIDATION_ERROR, + ) + project = self._resolve_one_project(alias) + client = self._new_metastore_client(project) + try: + model_uuid, _ = self._resolve_model(client, model_name_or_uuid) + data: dict[str, Any] = { + "modelUUID": model_uuid, + "dimensionName": dimension, + "members": members, + } + if dataset_id: + data["datasetId"] = dataset_id + if description: + data["description"] = description + + existing = self._find_reference_data_for_model(client, model_uuid, dimension) + if existing is not None: + item = client.put_item( + self._REFERENCE_DATA_TYPE, + existing.get("id", ""), + name=dimension, + data=data, + ) + action = "updated" + else: + item = client.post_item(self._REFERENCE_DATA_TYPE, name=dimension, data=data) + action = "created" + finally: + client.close() + result = self._unpack_reference_record(alias, item, include_members=False) + result["action"] = action + return result + + def delete_reference_data(self, alias: str, record_id: str) -> dict[str, Any]: + """Delete a reference-data record by UUID (soft-delete server-side).""" + project = self._resolve_one_project(alias) + client = self._new_metastore_client(project) + try: + item = client.get_item(self._REFERENCE_DATA_TYPE, record_id) + attrs = item.get("attributes") or {} + client.delete_item(self._REFERENCE_DATA_TYPE, record_id) + finally: + client.close() + return { + "project": alias, + "removed": {"id": record_id, "dimension_name": attrs.get("dimensionName", "")}, + } # ------------------------------------------------------------------ # Phase 4 — edit (DELETE-then-POST with rollback + rename cascade) @@ -880,8 +1046,7 @@ def edit_metric( delegates. """ project = self._resolve_one_project(alias) - client = self._new_metastore_client(project) - try: + with self._new_metastore_client(project) as client: model_uuid, _ = self._resolve_model(client, model_name_or_uuid) return _edit_metric_helper( client, @@ -895,8 +1060,6 @@ def edit_metric( is_tty=is_tty, confirm_cb=confirm_cb, ) - finally: - client.close() def edit_dataset( self, @@ -910,8 +1073,7 @@ def edit_dataset( ) -> dict[str, Any]: """Edit a dataset (DELETE+POST). Renames do NOT cascade for datasets.""" project = self._resolve_one_project(alias) - client = self._new_metastore_client(project) - try: + with self._new_metastore_client(project) as client: model_uuid, _ = self._resolve_model(client, model_name_or_uuid) return _edit_simple_helper( client, @@ -926,8 +1088,6 @@ def edit_dataset( }, not_found_label="Dataset", ) - finally: - client.close() def edit_constraint( self, @@ -952,8 +1112,7 @@ def edit_constraint( ) project = self._resolve_one_project(alias) - client = self._new_metastore_client(project) - try: + with self._new_metastore_client(project) as client: model_uuid, _ = self._resolve_model(client, model_name_or_uuid) if new_metrics is not None: existing = client.list_items("semantic-metric", model_uuid) @@ -979,8 +1138,6 @@ def edit_constraint( }, not_found_label="Constraint", ) - finally: - client.close() def edit_relationship( self, @@ -1006,8 +1163,7 @@ def edit_relationship( error_code=ErrorCode.VALIDATION_ERROR, ) project = self._resolve_one_project(alias) - client = self._new_metastore_client(project) - try: + with self._new_metastore_client(project) as client: model_uuid, _ = self._resolve_model(client, model_name_or_uuid) return _edit_simple_helper( client, @@ -1024,8 +1180,6 @@ def edit_relationship( }, not_found_label="Relationship", ) - finally: - client.close() def edit_glossary( self, @@ -1044,8 +1198,7 @@ def edit_glossary( behind ``--yes``; this method just executes. """ project = self._resolve_one_project(alias) - client = self._new_metastore_client(project) - try: + with self._new_metastore_client(project) as client: model_uuid, _ = self._resolve_model(client, model_name_or_uuid) return _edit_simple_helper( client, @@ -1056,8 +1209,6 @@ def edit_glossary( overrides={"term": new_term, "definition": new_definition}, not_found_label="Glossary term", ) - finally: - client.close() # ------------------------------------------------------------------ # Phase 5 — remove (destructive, orphan-warning before delete) @@ -1089,8 +1240,7 @@ def preview_remove( error_code=ErrorCode.VALIDATION_ERROR, ) project = self._resolve_one_project(alias) - client = self._new_metastore_client(project) - try: + with self._new_metastore_client(project) as client: model_uuid, _ = self._resolve_model(client, model_name_or_uuid) target, _, _ = _find_target_for_remove( client, @@ -1110,8 +1260,6 @@ def preview_remove( "name": name, "orphaned_constraints": orphan_constraints, } - finally: - client.close() def remove_item( self, @@ -1130,8 +1278,7 @@ def remove_item( error_code=ErrorCode.VALIDATION_ERROR, ) project = self._resolve_one_project(alias) - client = self._new_metastore_client(project) - try: + with self._new_metastore_client(project) as client: model_uuid, _ = self._resolve_model(client, model_name_or_uuid) target, type_slug, _ = _find_target_for_remove( client, @@ -1150,8 +1297,6 @@ def remove_item( "removed": {"type": type_slug, "id": target["id"], "name": name}, "orphaned_constraints": orphan_constraints, } - finally: - client.close() # ------------------------------------------------------------------ # Phase 6 — import (replay a snapshot, optionally overwrite) @@ -1235,8 +1380,7 @@ def import_snapshot_from_dict( type_filter = _validate_types_filter(types) project = self._resolve_one_project(alias) - client = self._new_metastore_client(project) - try: + with self._new_metastore_client(project) as client: model_uuid, _ = self._resolve_model(client, model_name_or_uuid) existing_by_type = self._fetch_children_parallel(client, model_uuid) @@ -1257,8 +1401,6 @@ def import_snapshot_from_dict( "overwrite": overwrite, "imported": imported, } - finally: - client.close() # ------------------------------------------------------------------ # Phase 6 — promote (cross-project copy) @@ -1295,9 +1437,10 @@ def promote_model( type_filter = _validate_types_filter(types) - src_client = self._new_metastore_client(projects[from_project]) - tgt_client = self._new_metastore_client(projects[to_project]) - try: + with ( + self._new_metastore_client(projects[from_project]) as src_client, + self._new_metastore_client(projects[to_project]) as tgt_client, + ): src_uuid, _ = self._resolve_model(src_client, from_model) tgt_uuid, _ = self._resolve_model(tgt_client, to_model) @@ -1320,11 +1463,6 @@ def promote_model( "dry_run": dry_run, **per_type_stats, } - finally: - try: - src_client.close() - finally: - tgt_client.close() # ------------------------------------------------------------------ # Phase 7 — build (AI-assisted / heuristic greenfield) @@ -1432,8 +1570,7 @@ def build_model( # Push to the metastore in dependency order. project = self._resolve_one_project(alias) - client = self._new_metastore_client(project) - try: + with self._new_metastore_client(project) as client: counts, model_uuid, model_item = _push_built_model( client, generated=generated, @@ -1444,8 +1581,6 @@ def build_model( result["model"] = {"id": model_uuid, "item": model_item} result["created"] = counts return result - finally: - client.close() # ------------------------------------------------------------------ # Phase 8 — token --encrypt diff --git a/src/keboola_agent_cli/services/storage_service.py b/src/keboola_agent_cli/services/storage_service.py index 19423cf3..ea78b463 100644 --- a/src/keboola_agent_cli/services/storage_service.py +++ b/src/keboola_agent_cli/services/storage_service.py @@ -711,6 +711,7 @@ def create_table( branch_id: int | None = None, not_null_columns: list[str] | None = None, defaults: list[str] | None = None, + if_not_exists: bool = False, ) -> dict[str, Any]: """Create a new table with typed columns. @@ -742,7 +743,14 @@ def create_table( must be lowercase per Keboola API validation. Returns: - Dict with created table details and ``auto_created_bucket`` flag. + Dict with table details and ``auto_created_bucket`` flag. + ``action`` is ``"created"`` on a fresh create. When + ``if_not_exists`` is set and the table already existed, + ``action`` is ``"skipped"`` and ``columns`` / ``primary_key`` / + ``name`` report the EXISTING table's actual schema (not the + request); the caller's requested values are mirrored under + ``requested_columns`` / ``requested_primary_key``, and + ``schema_drift`` is ``True`` when the two diverge. Raises: ValueError: Malformed column spec or ``--default`` assignment, @@ -777,28 +785,80 @@ def create_table( project = projects[alias] client = self._client_factory(project.stack_url, project.token) + target_table_id = f"{bucket_id}.{name}" try: auto_created_bucket = _ensure_bucket_exists_in_branch(client, bucket_id, branch_id) - results = client.create_table( - bucket_id=bucket_id, - name=name, - columns=parsed_columns, - primary_key=primary_key, - branch_id=branch_id, - ) + try: + results = client.create_table( + bucket_id=bucket_id, + name=name, + columns=parsed_columns, + primary_key=primary_key, + branch_id=branch_id, + ) + except KeboolaApiError as exc: + # IF-NOT-EXISTS: if the create failed because the table + # already has the same display name AND the table at the + # expected id resolves, treat as a successful skip. A + # different table with the same display name still + # surfaces the original error (the user has a real + # conflict to resolve). + if ( + if_not_exists + and exc.error_code == ErrorCode.STORAGE_JOB_FAILED + and "already has the same display name" in (exc.message or "") + ): + try: + existing = client.get_table_detail(target_table_id, branch_id=branch_id) + except KeboolaApiError: + existing = None + if existing is not None: + # Report the EXISTING table's actual schema, not the + # caller's request. A caller relying on the skipped + # envelope to discover the real shape must not be handed + # a re-echo of its own args (keboola/cli#349). The + # requested values are preserved under `requested_*` so + # the caller can still see the divergence, and + # `schema_drift` flags when the existing table differs. + requested_columns = [c["name"] for c in parsed_columns] + requested_primary_key = primary_key or [] + actual_columns = existing.get("columns", []) + actual_primary_key = existing.get("primaryKey", []) + schema_drift = set(actual_columns) != set(requested_columns) or set( + actual_primary_key + ) != set(requested_primary_key) + return { + "project_alias": alias, + "table_id": target_table_id, + "name": existing.get("name", name), + "bucket_id": bucket_id, + "primary_key": actual_primary_key, + "columns": actual_columns, + "requested_primary_key": requested_primary_key, + "requested_columns": requested_columns, + "schema_drift": schema_drift, + "auto_created_bucket": auto_created_bucket, + "legacy_branch_storage": _detect_legacy_branch_storage( + client, branch_id + ), + "action": "skipped", + "skip_reason": "table already exists", + } + raise legacy_branch_storage = _detect_legacy_branch_storage(client, branch_id) finally: client.close() return { "project_alias": alias, - "table_id": results.get("id", f"{bucket_id}.{name}"), + "table_id": results.get("id", target_table_id), "name": name, "bucket_id": bucket_id, "primary_key": primary_key or [], "columns": [c["name"] for c in parsed_columns], "auto_created_bucket": auto_created_bucket, "legacy_branch_storage": legacy_branch_storage, + "action": "created", } def upload_table( @@ -1284,7 +1344,7 @@ def swap_tables( branch_id: int | None, dry_run: bool = False, ) -> dict[str, Any]: - """Swap two storage tables (dev branch only). + """Swap two storage tables (branch-scoped; branch_id mandatory). After the swap, the two tables exchange physical positions. Aliases are NOT transferred -- they keep pointing at the same physical @@ -1292,15 +1352,17 @@ def swap_tables( This is the documented behavior of the Storage API; the service layer does not try to rewrite alias targets. - The Storage API rejects this operation on production -- a dev branch - ID is mandatory. The service raises ConfigError before any HTTP call - when ``branch_id`` is None. + ``branch_id`` is mandatory and the service raises ConfigError before + any HTTP call when it is None. Any branch is accepted, INCLUDING the + default/production branch -- a default-branch swap is the supported + way to retype a production table (dev-branch merge does not propagate + storage schema, so a swap done in a dev branch never reaches prod). Args: alias: Project alias. table_id: Full ID of the first table. target_table_id: Full ID of the second table. - branch_id: Dev branch ID (must not be None). + branch_id: Branch ID (must not be None; any branch accepted, including the default/production branch). dry_run: If True, only report what would be swapped. Returns: @@ -1313,10 +1375,10 @@ def swap_tables( """ if branch_id is None: raise ConfigError( - "swap-tables requires a dev branch. Set one with " + "swap-tables requires a branch. Set one with " "'kbagent branch use --project

--branch ' or pass " - "--branch directly. The Storage API rejects this on " - "production." + "--branch directly. Any branch works, including the " + "default/production branch." ) if table_id == target_table_id: @@ -1356,6 +1418,72 @@ def swap_tables( "response": response, } + def clone_table( + self, + alias: str, + table_id: str, + branch_id: int | None, + dry_run: bool = False, + ) -> dict[str, Any]: + """Pull (clone) a production table into a dev branch (branch required). + + On ``storage-branches`` projects a dev branch reads production tables + transparently until the first write, so mutating a table's schema in + the branch (e.g. ``swap_tables`` or a column drop) first needs a + branch-local copy of the production table. This materializes that copy + from the default branch. The pull is one-way (default -> branch); the + service raises ConfigError before any HTTP call when ``branch_id`` is + None. + + Args: + alias: Project alias. + table_id: Full ID of the table to pull into the branch. + branch_id: Target dev branch ID (must not be None). + dry_run: If True, only report what would be pulled. + + Returns: + Dict with 'project_alias', 'branch_id', 'table_id', 'dry_run', + and (when not dry-run) 'response'. + + Raises: + ConfigError: If branch_id is None. + KeboolaApiError: If the API call fails. + """ + if branch_id is None: + raise ConfigError( + "clone-table requires a dev branch. Set one with " + "'kbagent branch use --project

--branch ' or pass " + "--branch directly. The pull is one-way: default -> branch." + ) + + projects = self.resolve_projects([alias]) + project = projects[alias] + + if dry_run: + return { + "project_alias": alias, + "branch_id": branch_id, + "table_id": table_id, + "dry_run": True, + } + + client = self._client_factory(project.stack_url, project.token) + try: + response = client.pull_table( + table_id=table_id, + branch_id=branch_id, + ) + finally: + client.close() + + return { + "project_alias": alias, + "branch_id": branch_id, + "table_id": table_id, + "dry_run": False, + "response": response, + } + def delete_buckets( self, alias: str, diff --git a/src/keboola_agent_cli/services/stream_service.py b/src/keboola_agent_cli/services/stream_service.py new file mode 100644 index 00000000..1c2c570c --- /dev/null +++ b/src/keboola_agent_cli/services/stream_service.py @@ -0,0 +1,395 @@ +"""Data Streams (Stream API) service -- OTLP/HTTP source management. + +Business logic for the ``kbagent stream`` command group. Wraps the Stream +control-plane API behind a layer that: + +- resolves a kbagent project alias to its ``(stack_url, token)`` via + :class:`ConfigStore` (the alias is the only handle a caller needs); +- builds a :class:`StreamClient` through an injectable factory (testability); +- assembles a source's full picture for ``stream detail`` -- base + per-signal + OTLP endpoints, wire protocol, and the destination bucket/tables read from the + source's sinks; +- **masks the secret embedded in the OTLP endpoint URL by default**, revealing + it only when the caller explicitly opts in (``reveal=True``). The raw source + object echoed in ``--json`` output is sanitised the same way so the secret + never leaks unless revealed. + +The Stream API authenticates with the per-project Storage API token, so -- unlike +the ``feature`` group -- no manage token is involved. +""" + +from __future__ import annotations + +import logging +from collections.abc import Callable +from typing import Any + +from ..config_store import ConfigStore +from ..constants import ( + OTLP_BUCKET_PREFIX, + OTLP_PROTOCOL, + OTLP_SIGNAL_PATHS, + OTLP_SINK_COLUMNS, + OTLP_SINK_SIGNALS, + STREAM_DEFAULT_BRANCH, +) +from ..errors import ConfigError, ErrorCode, KeboolaApiError +from ..stream_client import StreamClient + +logger = logging.getLogger(__name__) + +StreamClientFactory = Callable[[str, str], StreamClient] + +# Map an OTLP signal sub-path (v1/logs) to the short signal name (logs) used as +# the key in the assembled per-signal endpoint map. +_SIGNAL_NAMES: dict[str, str] = {path: path.split("/")[-1] for path in OTLP_SIGNAL_PATHS} + +_SECRET_MASK = "***" + + +def default_stream_client_factory(stack_url: str, token: str) -> StreamClient: + """Construct a :class:`StreamClient` bound to ``stack_url`` + ``token``.""" + return StreamClient(stack_url=stack_url, token=token) + + +class StreamService: + """Business logic for Data Streams sources (list / create / detail / delete).""" + + def __init__( + self, + config_store: ConfigStore, + stream_client_factory: StreamClientFactory | None = None, + ) -> None: + self._config_store = config_store + self._stream_client_factory = stream_client_factory or default_stream_client_factory + + # ------------------------------------------------------------------ + # Public API + # ------------------------------------------------------------------ + + def list_sources(self, *, alias: str, branch_id: str | None = None) -> dict[str, Any]: + """List sources in the alias's project (default branch unless overridden).""" + stack_url, token = self._resolve_project(alias) + branch = branch_id or STREAM_DEFAULT_BRANCH + client = self._stream_client_factory(stack_url, token) + try: + raw = client.list_sources(branch) + return { + "alias": alias, + "branch_id": branch, + "sources": self._summarise_sources(raw), + } + finally: + client.close() + + def create_source( + self, + *, + alias: str, + name: str, + source_type: str = "otlp", + branch_id: str | None = None, + if_not_exists: bool = False, + reveal: bool = False, + provision_sinks: bool = True, + ) -> dict[str, Any]: + """Create a source, poll the async task, and return its assembled detail. + + For an ``otlp`` source, the three standard sinks (logs/metrics/traces) + are auto-provisioned so data actually lands in Storage -- the raw Stream + API ``POST /sources`` creates only the bare source, unlike the Keboola + UI. Pass ``provision_sinks=False`` to skip that and create a bare source. + + With ``if_not_exists`` an existing source matching ``name`` (by name or + sourceId) is returned untouched with ``status="skipped"`` (its sinks are + also reconciled when ``provision_sinks`` so a half-set-up source heals). + """ + stack_url, token = self._resolve_project(alias) + branch = branch_id or STREAM_DEFAULT_BRANCH + client = self._stream_client_factory(stack_url, token) + try: + if if_not_exists: + existing = self._find_source(client, branch, name) + if existing is not None: + if provision_sinks and source_type == "otlp": + self._provision_otlp_sinks(client, branch, existing.get("sourceId", name)) + detail = self._assemble_detail(client, branch, existing, reveal=reveal) + detail.update({"alias": alias, "status": "skipped"}) + return detail + + task = client.create_source(branch, name=name, source_type=source_type) + finished = client.wait_for_task(task) + source_id = self._task_source_id(finished) or name + if provision_sinks and source_type == "otlp": + self._provision_otlp_sinks(client, branch, source_id) + source = client.get_source(branch, source_id) + detail = self._assemble_detail(client, branch, source, reveal=reveal) + detail.update({"alias": alias, "status": "created"}) + return detail + finally: + client.close() + + def _provision_otlp_sinks(self, client: StreamClient, branch: str, source_id: str) -> None: + """Create the standard logs/metrics/traces sinks for an OTLP source. + + Idempotent: only signals without an existing sink are created, so a + re-run (or ``--if-not-exists`` against a half-provisioned source) heals + the set rather than erroring on a conflict. + """ + existing_signals: set[str] = set() + for sink in client.list_sinks(branch, source_id).get("sinks", []): + existing_signals.update(sink.get("allowedSignals") or []) + columns = [dict(col) for col in OTLP_SINK_COLUMNS] + for signal in OTLP_SINK_SIGNALS: + if signal in existing_signals: + continue + table_id = f"{OTLP_BUCKET_PREFIX}{source_id}.{signal}" + task = client.create_sink( + branch, + source_id, + name=signal.capitalize(), + table_id=table_id, + columns=columns, + allowed_signals=[signal], + ) + client.wait_for_task(task) + + def get_source_detail( + self, + *, + alias: str, + source_id: str | None = None, + name: str | None = None, + branch_id: str | None = None, + reveal: bool = False, + ) -> dict[str, Any]: + """Assemble the full picture for one source (endpoints + destination).""" + if not source_id and not name: + raise ConfigError("Provide a source id (positional) or --name.") + stack_url, token = self._resolve_project(alias) + branch = branch_id or STREAM_DEFAULT_BRANCH + client = self._stream_client_factory(stack_url, token) + try: + if source_id: + source = client.get_source(branch, source_id) + else: + source = self._find_source(client, branch, name or "") + if source is None: + raise KeboolaApiError( + message=f"No source named '{name}' in branch '{branch}'.", + error_code=ErrorCode.NOT_FOUND, + status_code=404, + ) + detail = self._assemble_detail(client, branch, source, reveal=reveal) + detail["alias"] = alias + return detail + finally: + client.close() + + def delete_source( + self, + *, + alias: str, + source_id: str, + branch_id: str | None = None, + dry_run: bool = False, + ) -> dict[str, Any]: + """Delete a source (async task polled to completion).""" + stack_url, token = self._resolve_project(alias) + branch = branch_id or STREAM_DEFAULT_BRANCH + if dry_run: + return { + "status": "dry_run", + "alias": alias, + "branch_id": branch, + "source_id": source_id, + } + client = self._stream_client_factory(stack_url, token) + try: + task = client.delete_source(branch, source_id) + client.wait_for_task(task) + return { + "status": "deleted", + "alias": alias, + "branch_id": branch, + "source_id": source_id, + } + finally: + client.close() + + # ------------------------------------------------------------------ + # Internal helpers + # ------------------------------------------------------------------ + + def _resolve_project(self, alias: str) -> tuple[str, str]: + """Resolve ``alias`` to its ``(stack_url, token)``.""" + project = self._config_store.get_project(alias) + if project is None: + raise ConfigError( + f"Project alias '{alias}' is not registered. Run `kbagent project list`." + ) + return project.stack_url, project.token + + @staticmethod + def _find_source(client: StreamClient, branch: str, needle: str) -> dict[str, Any] | None: + """Return the first source matching ``needle`` by sourceId or name.""" + raw = client.list_sources(branch) + for source in raw.get("sources", []): + if source.get("sourceId") == needle or source.get("name") == needle: + return source + return None + + @staticmethod + def _task_source_id(task: dict[str, Any]) -> str | None: + """Extract the created sourceId from a finished task's outputs.""" + outputs = task.get("outputs") + if isinstance(outputs, dict): + source_id = outputs.get("sourceId") + if isinstance(source_id, str): + return source_id + return None + + @staticmethod + def _summarise_sources(raw: dict[str, Any]) -> list[dict[str, Any]]: + """Reduce a sources-list payload to the columns `stream list` shows. + + The secret is never part of the list view -- only the base (secret-free) + endpoint is surfaced. + """ + out: list[dict[str, Any]] = [] + for source in raw.get("sources", []): + otlp = source.get("otlp") or {} + http = source.get("http") or {} + out.append( + { + "source_id": source.get("sourceId", ""), + "name": source.get("name", ""), + "type": source.get("type", ""), + "description": source.get("description", ""), + # baseUrl is documented as the endpoint *without* the secret. + "base_endpoint": otlp.get("baseUrl", "") or http.get("url", ""), + } + ) + return out + + def _assemble_detail( + self, + client: StreamClient, + branch: str, + source: dict[str, Any], + *, + reveal: bool, + ) -> dict[str, Any]: + """Build the rich `stream detail` view for one source. + + Fetches the source's sinks to surface the destination bucket/tables and + import conditions, computes per-signal OTLP endpoints, and masks the + secret (in every endpoint and in the echoed raw source) unless + ``reveal`` is set. + """ + source_id = source.get("sourceId", "") + source_type = source.get("type", "") + otlp = source.get("otlp") or {} + http = source.get("http") or {} + secret = otlp.get("secret") or "" + base_endpoint = otlp.get("baseUrl", "") + full_endpoint = otlp.get("url", "") or http.get("url", "") + + signal_endpoints: dict[str, str] = {} + if source_type == "otlp" and full_endpoint: + root = full_endpoint.rstrip("/") + for path, signal in _SIGNAL_NAMES.items(): + signal_endpoints[signal] = f"{root}/{path}" + + sinks_raw = client.list_sinks(branch, source_id) + destination = self._destination_from_sinks(sinks_raw) + import_conditions = self._import_conditions(source, sinks_raw) + + endpoint_display = self._mask(full_endpoint, secret, reveal) + signal_display = { + signal: self._mask(url, secret, reveal) for signal, url in signal_endpoints.items() + } + + return { + "branch_id": branch, + "source_id": source_id, + "name": source.get("name", ""), + "type": source_type, + "description": source.get("description", ""), + "endpoint": endpoint_display, + "base_endpoint": base_endpoint, + "signal_endpoints": signal_display, + "protocol": OTLP_PROTOCOL if source_type == "otlp" else "", + "secret_revealed": reveal, + "destination": destination, + "import_conditions": import_conditions, + "sinks": sinks_raw.get("sinks", []), + # Echo the raw source for `--json` completeness, sanitised so the + # secret never leaks unless explicitly revealed. + "source": self._sanitise_source(source, secret, reveal), + } + + @staticmethod + def _destination_from_sinks(sinks_raw: dict[str, Any]) -> dict[str, Any]: + """Extract destination bucket + per-signal table ids from sinks.""" + tables: dict[str, str] = {} + buckets: list[str] = [] + for sink in sinks_raw.get("sinks", []): + table = sink.get("table") or {} + table_id = table.get("tableId", "") + if not table_id: + continue + signals = sink.get("allowedSignals") or [] + key = signals[0] if signals else (sink.get("sinkId") or table_id) + tables[key] = table_id + bucket = table_id.rsplit(".", 1)[0] if "." in table_id else "" + if bucket and bucket not in buckets: + buckets.append(bucket) + return { + "bucket": buckets[0] if len(buckets) == 1 else "", + "buckets": buckets, + "tables": tables, + } + + @staticmethod + def _import_conditions( + source: dict[str, Any], sinks_raw: dict[str, Any] + ) -> dict[str, Any] | None: + """Surface import/upload conditions if the API exposes them. + + The Stream API manages import triggers (count / size / time) server-side + and does not always echo them on the source/sink objects. We return + whatever is present rather than inventing defaults (no silent defaults). + """ + for candidate in (source.get("import"), source.get("conditions")): + if isinstance(candidate, dict) and candidate: + return candidate + for sink in sinks_raw.get("sinks", []): + conditions = sink.get("conditions") or (sink.get("table") or {}).get("import") + if isinstance(conditions, dict) and conditions: + return conditions + return None + + @staticmethod + def _mask(endpoint: str, secret: str, reveal: bool) -> str: + """Mask the secret substring inside ``endpoint`` unless ``reveal``.""" + if reveal or not endpoint or not secret: + return endpoint + return endpoint.replace(secret, _SECRET_MASK) + + @classmethod + def _sanitise_source(cls, source: dict[str, Any], secret: str, reveal: bool) -> dict[str, Any]: + """Return a copy of ``source`` with the secret masked unless revealed.""" + if reveal: + return source + sanitised = dict(source) + for key in ("otlp", "http"): + block = sanitised.get(key) + if isinstance(block, dict): + masked = dict(block) + if masked.get("url"): + masked["url"] = cls._mask(masked["url"], secret, reveal) + if "secret" in masked: + masked["secret"] = _SECRET_MASK + sanitised[key] = masked + return sanitised diff --git a/src/keboola_agent_cli/services/sync_service.py b/src/keboola_agent_cli/services/sync_service.py index 54369cc5..b2662217 100644 --- a/src/keboola_agent_cli/services/sync_service.py +++ b/src/keboola_agent_cli/services/sync_service.py @@ -12,6 +12,7 @@ import subprocess import threading from concurrent.futures import ThreadPoolExecutor, as_completed +from dataclasses import dataclass, field from pathlib import Path from typing import Any @@ -33,7 +34,7 @@ STORAGE_DIR_NAME, STORAGE_SAMPLES_DIR_NAME, ) -from ..errors import ConfigError, ErrorCode, KeboolaApiError +from ..errors import ConfigError, ErrorCode, KeboolaApiError, SyncConflictError from ..sync.code_extraction import extract_code_files, merge_code_files from ..sync.config_format import ( api_config_to_local, @@ -61,6 +62,79 @@ logger = logging.getLogger(__name__) +# Sibling component that backs a transformation's variable links. A +# transformation references it via ``configuration.variables_id`` (the config) +# and ``configuration.variables_values_id`` (a row id). +VARIABLES_COMPONENT_ID = "keboola.variables" + + +@dataclass +class WritebackResult: + """Outcome of recording a freshly-created config in the manifest. + + ``previous_id`` is the manifest entry's id **before** the placeholder -> + ULID overwrite (empty string when a brand-new entry was appended). The + create pass uses it to key ``created_id_map`` so row parents and + transformation variable links can be remapped placeholder -> ULID. + """ + + entry: ManifestConfiguration + previous_id: str + + +@dataclass +class CreatedConfig: + """A config created during a single ``push`` create pass. + + Carries just enough to drive the Phase-C variable-link backfill: the + component id, the API-assigned ULID, and the on-disk directory holding + the (post-writeback) ``_config.yml``. + """ + + component_id: str + config_id: str + config_dir: Path + + +@dataclass +class VariableBindingResult: + """Outcome of the Phase-C variable-link backfill. + + ``configs_rewritten`` counts transformations whose remote configuration + + local ``_configuration_extra`` were rebound to ULIDs (drives the + manifest-dirty flag). ``errors`` accumulates unresolved links so the push + envelope surfaces them instead of leaving a broken link silently. + """ + + errors: list[dict[str, str]] = field(default_factory=list) + configs_rewritten: int = 0 + + +@dataclass +class LocalConfigHashes: + """Hashes describing a config dir's on-disk state after a push. + + ``file_hash`` is the ``_config.yml`` content hash, ``cfg_hash`` the + normalized config hash (see :func:`config_hash`), and ``extra_hashes`` + maps each extracted code/companion file to its hash. Stored on the + manifest entry so the next ``sync diff`` recognises local == remote. + """ + + file_hash: str + cfg_hash: str + extra_hashes: dict[str, str] = field(default_factory=dict) + + +# Companion files extracted alongside a config's ``_config.yml`` whose hashes +# are tracked so ``sync diff`` notices edits to code/description files. +_EXTRA_HASH_FILENAMES: tuple[str, ...] = ( + "_description.md", + "transform.sql", + "transform.py", + "code.py", + "pyproject.toml", +) + def _ensure_within_branch( branch_dir: Path, @@ -269,6 +343,111 @@ def _adopt_existing_manifest( # pull # ------------------------------------------------------------------ + def _is_conflict( + self, + config_file: Path, + old_pull_hash: str, + old_cfg_hash: str, + api_cfg_hash: str, + ) -> bool: + """True iff the file is locally modified AND the remote also changed. + + A 3-way conflict needs both a stored ``pull_hash`` (the synced file + state) and a stored ``pull_config_hash`` (the synced remote state); + without either we cannot prove a conflict, so return False -- be + conservative, ``--force`` must not abort on incomplete bookkeeping. + A missing local file is not a content conflict (nothing to lose). + """ + if not old_pull_hash or not old_cfg_hash: + return False + if not config_file.exists(): + return False + locally_modified = self._file_hash(config_file) != old_pull_hash + remote_changed = api_cfg_hash != old_cfg_hash + return locally_modified and remote_changed + + def _detect_force_pull_conflicts( + self, + components: list[dict[str, Any]], + branch_dir: Path, + *, + existing_keys: set[str], + existing_paths: dict[str, str], + existing_file_hashes: dict[str, str], + existing_config_hashes: dict[str, str], + existing_rows: dict[str, dict[str, str]], + ) -> list[dict[str, str]]: + """Return configs/rows a ``--force`` pull would clobber as conflicts. + + A *conflict* is a config (or row) that is BOTH locally modified (its + on-disk ``_config.yml`` hash differs from the manifest ``pull_hash``) + AND changed on the remote since the last pull (the freshly fetched + config hash differs from ``pull_config_hash``). That is the only case + where ``--force`` must stop: local and remote have diverged, so neither + "take remote" nor "keep local" is safe without the user deciding. + + Configs only locally modified (remote unchanged) are NOT conflicts -- + ``--force`` preserves them so their pending delta stays pushable. + Brand-new remote configs and configs whose local file is missing are + skipped (nothing local to lose). Read-only: hashes but writes nothing. + """ + conflicts: list[dict[str, str]] = [] + for component in components: + component_id = component.get("id", "") + if component_id in ALWAYS_IGNORED_COMPONENTS: + continue + for cfg in component.get("configurations", []): + config_id = str(cfg.get("id", "")) + lookup_key = f"{component_id}/{config_id}" + if lookup_key not in existing_keys: + continue # brand-new remote config -- nothing local to lose + + rel_path = existing_paths.get(lookup_key, "") + api_cfg_hash = config_hash(api_config_to_local(component_id, cfg, config_id)) + if self._is_conflict( + branch_dir / rel_path / CONFIG_FILENAME, + existing_file_hashes.get(lookup_key, ""), + existing_config_hashes.get(lookup_key, ""), + api_cfg_hash, + ): + conflicts.append( + { + "scope": "config", + "component_id": component_id, + "config_id": config_id, + "config_name": str(cfg.get("name", "untitled")), + "path": rel_path, + } + ) + + # Row-level conflicts (same 3-way rule, per row). + config_dir = branch_dir / rel_path + for row in cfg.get("rows", []): + row_id = str(row.get("id", "")) + existing_row = existing_rows.get(f"{component_id}/{config_id}/{row_id}") + if not existing_row: + continue + row_rel_path = existing_row.get("path", "") + if self._is_conflict( + config_dir / row_rel_path / CONFIG_FILENAME, + existing_row.get("pull_hash", ""), + existing_row.get("pull_config_hash", ""), + config_hash(api_row_to_local(row, component_id)), + ): + conflicts.append( + { + "scope": "row", + "component_id": component_id, + "config_id": config_id, + "config_name": ( + f"{cfg.get('name', 'untitled')}/{row.get('name', 'untitled')}" + ), + "path": f"{rel_path}/{row_rel_path}", + "row_id": row_id, + } + ) + return conflicts + def pull( self, alias: str, @@ -281,6 +460,7 @@ def pull( with_samples: bool = False, sample_limit: int = DEFAULT_SAMPLE_LIMIT, max_samples: int = DEFAULT_MAX_SAMPLES, + branch_override: int | None = None, ) -> dict[str, Any]: """Download all configurations from Keboola to local filesystem. @@ -295,6 +475,8 @@ def pull( with_samples: Download table data samples (opt-in). sample_limit: Max rows per sample (default 100). max_samples: Max number of tables to sample (default 50). + branch_override: If set, pull from this branch ID rather than + the resolved active/manifest branch (CLI ``--branch``). Returns: Dict with pull statistics (configs, rows, files written). @@ -306,7 +488,9 @@ def pull( manifest = load_manifest(project_root) # Determine branch to pull from (git-branching aware) - branch_id = self._resolve_branch_id(project, manifest, project_root) + branch_id = self._resolve_branch_id( + project, manifest, project_root, branch_override=branch_override + ) # Fetch all components with configs from API (+ storage metadata + jobs) client = self._client_factory(project.stack_url, project.token) @@ -397,6 +581,29 @@ def pull( "pull_config_hash": r.metadata.get("pull_config_hash", ""), } + # Force-pull conflict guard (force-pull baseline corruption fix). + # ``--force`` bypasses the "preserve locally-modified files" guard + # below, so a config edited locally AND changed on the remote since the + # last pull (a true 3-way conflict) would be silently overwritten -- and + # when the remote is unchanged, its baseline would instead be re-stamped + # from the edited file, stranding the un-pushed edits. Detect such + # conflicts up front and abort BEFORE writing anything (the read-only + # API fetch above has already happened; nothing is on disk yet), so the + # user can resolve them. Non-force pull preserves locally-modified + # files and surfaces conflicts via ``sync diff``, so it needs no abort. + if force: + conflicts = self._detect_force_pull_conflicts( + components, + branch_dir, + existing_keys=existing_keys, + existing_paths=existing_paths, + existing_file_hashes=existing_file_hashes, + existing_config_hashes=existing_config_hashes, + existing_rows=existing_rows, + ) + if conflicts: + raise SyncConflictError(conflicts) + for component in components: component_id = component.get("id", "") if component_id in ALWAYS_IGNORED_COMPONENTS: @@ -476,11 +683,16 @@ def pull( api_cfg_hash = config_hash(local_data) pull_cfg_hash = api_cfg_hash - # Detect local modifications: if file hash differs from - # pull_hash stored in manifest, the user edited the file. - # Skip overwrite unless --force to avoid losing local work. + # Detect local modifications: if the file hash differs from the + # pull_hash stored in manifest, the user edited the file -- so + # preserve it instead of overwriting. This now runs even under + # ``--force``: a force-pull that reaches here for a modified file + # has already passed the conflict guard above (so the remote is + # unchanged), and re-stamping the baseline from the edited file + # would silently strand the un-pushed edits. Preserving keeps + # the pending delta visible to ``sync push``. locally_modified = False - if not is_new and not force: + if not is_new: old_file_hash = existing_file_hashes.get(lookup_key, "") if old_file_hash: config_file = config_dir / CONFIG_FILENAME @@ -597,8 +809,12 @@ def pull( row_file = row_dir / CONFIG_FILENAME old_row_file_hash = existing_row["pull_hash"] if existing_row else "" old_row_cfg_hash = existing_row["pull_config_hash"] if existing_row else "" + # Runs even under ``--force``: a force-pull reaching a + # modified row has passed the conflict guard (remote + # unchanged), so preserve the row rather than re-stamp its + # baseline from the edited file and strand the edits. row_locally_modified = False - if existing_row and not force and old_row_file_hash and row_file.exists(): + if existing_row and old_row_file_hash and row_file.exists(): row_locally_modified = self._file_hash(row_file) != old_row_file_hash if row_locally_modified: @@ -840,12 +1056,19 @@ def diff( self, alias: str, project_root: Path, + branch_override: int | None = None, ) -> dict[str, Any]: """Compare local configs against the remote API state. Fetches current state from API, reads local _config.yml files, and runs the diff engine to produce a detailed changeset. + Args: + alias: Project alias. + project_root: Sync working-tree root. + branch_override: If set, diff against this branch ID rather + than the resolved active/manifest branch. + Returns: Dict with 'changes' list and summary counts. """ @@ -853,7 +1076,9 @@ def diff( project = projects[alias] manifest = load_manifest(project_root) - branch_id = self._resolve_branch_id(project, manifest, project_root) + branch_id = self._resolve_branch_id( + project, manifest, project_root, branch_override=branch_override + ) # Fetch remote state client = self._client_factory(project.stack_url, project.token) @@ -937,7 +1162,7 @@ def diff( # Also add untracked local configs (new files) for added_cfg in self._find_untracked_configs(project_root, manifest, branch_id): - branch_path = self._find_branch_path(manifest, branch_id) + branch_path = self._resolve_source_branch_path(manifest, project_root, branch_id) config_dir = project_root / branch_path / added_cfg["path"] local_data = self._read_config_file(config_dir) if local_data is None: @@ -1086,6 +1311,8 @@ def push( dry_run: bool = False, force: bool = False, allow_plaintext_fallback: bool = False, + branch_override: int | None = None, + no_name_drift_warnings: bool = False, ) -> dict[str, Any]: """Push local changes to Keboola. @@ -1097,11 +1324,24 @@ def push( project_root: Root directory of the sync working tree. dry_run: If True, compute changes but don't execute them. force: If True, allow deletions without extra confirmation. + allow_plaintext_fallback: If True, allow push when secret + encryption fails (DANGEROUS). + branch_override: If set, target this dev-branch ID for the push. + Wins over ``active_branch_id`` / ``manifest.branches[0]`` / + git-branching mapping. Used by the CLI ``--branch`` flag. When + no ``/`` subtree exists on disk for the target, + the default tree (``main/``) is read as the source and promoted + to the target branch (KFR-07); API writes still target the + branch id. + no_name_drift_warnings: If True, omit the ``name_drift_warnings`` + array from the result envelope (the underlying detection + still runs; only the report is suppressed). Used by the CLI + ``--no-name-drift-warnings`` flag. Returns: Dict with push results (created, updated, deleted, errors). """ - diff_result = self.diff(alias, project_root) + diff_result = self.diff(alias, project_root, branch_override=branch_override) all_changes = diff_result["changes"] # Only push local-side changes (added, modified, deleted). @@ -1136,7 +1376,9 @@ def push( project = projects[alias] manifest = load_manifest(project_root) - branch_id = self._resolve_branch_id(project, manifest, project_root) + branch_id = self._resolve_branch_id( + project, manifest, project_root, branch_override=branch_override + ) # Detect name drift: local dir name doesn't match config name name_drift_warnings = self._detect_name_drift(manifest, project_root) @@ -1151,40 +1393,30 @@ def push( with client: self._ensure_branch_registered(manifest, branch_id, client) - branch_path = self._find_branch_path(manifest, branch_id) - - for change in changes: + branch_path = self._resolve_source_branch_path(manifest, project_root, branch_id) + + # Process configs before rows, and rebind variable links last. + # A freshly-created parent config must carry its API-assigned ULID + # before its rows POST (KFR-05), and a transformation's + # variables_id / variables_values_id can only be resolved once both + # the variables config and its values row exist (KFR-03). Partition + # explicitly rather than relying on incidental diff ordering. + config_changes = [c for c in changes if not bool(c.get("is_row"))] + row_changes = [c for c in changes if bool(c.get("is_row"))] + + # (component_id, placeholder_id) -> API-assigned ULID, captured + # before the manifest writeback overwrites the placeholder in place. + created_id_map: dict[tuple[str, str], str] = {} + created_configs: list[CreatedConfig] = [] + + # ---- Phase A: config creates / updates / deletes ------------- + for change in config_changes: change_type = change["change_type"] component_id = change["component_id"] config_id = change["config_id"] config_path_str = change.get("path", "") - is_row = bool(change.get("is_row")) - parent_config_id = change.get("parent_config_id", "") try: - if is_row: - self._push_row_change( - client, - change_type=change_type, - component_id=component_id, - parent_config_id=parent_config_id, - row_id=config_id, - row_path_str=config_path_str, - project_root=project_root, - manifest=manifest, - branch_id=branch_id, - allow_plaintext_fallback=allow_plaintext_fallback, - ) - manifest_dirty = True - if change_type == "added": - created += 1 - elif change_type == "modified": - updated += 1 - elif change_type == "deleted": - deleted += 1 - pushed_details.append(change) - continue - if change_type == "added": result = self._push_create( client, @@ -1197,28 +1429,45 @@ def push( ) if result: new_id = str(result.get("id", "")) - # Add to manifest with the API-assigned ID config_dir = project_root / branch_path / config_path_str - config_file = config_dir / CONFIG_FILENAME - file_hash = self._file_hash(config_file) if config_file.exists() else "" - local_data = self._read_config_file(config_dir) - if local_data is not None: - merge_code_files(component_id, local_data, config_dir) - cfg_hash = config_hash(local_data) - else: - cfg_hash = "" - manifest.configurations.append( - ManifestConfiguration( - branchId=branch_id or 0, - componentId=component_id, - id=new_id, - path=config_path_str, - metadata={ - "pull_hash": file_hash, - "pull_config_hash": cfg_hash, - }, + hashes = self._compute_config_hashes(config_dir, component_id) + writeback = self._writeback_create_config_in_manifest( + manifest=manifest, + component_id=component_id, + branch_id=branch_id, + config_path_str=config_path_str, + new_id=new_id, + file_hash=hashes.file_hash, + cfg_hash=hashes.cfg_hash, + ) + # Record placeholder -> ULID so child rows and + # transformation variable links can be remapped. + if writeback.previous_id: + created_id_map[(component_id, writeback.previous_id)] = new_id + created_configs.append( + CreatedConfig( + component_id=component_id, + config_id=new_id, + config_dir=config_dir, ) ) + metadata_error = self._propagate_kbc_metadata( + client, writeback.entry, branch_id + ) + if metadata_error is not None: + # The config IS on the remote; only the + # follow-up metadata POST failed. Accumulate + # like any other per-change error so the rest + # of the push continues, and surface the + # original cause in the envelope. + errors.append( + { + "change_type": "metadata_propagation", + "component_id": component_id, + "config_id": new_id, + "message": metadata_error, + } + ) manifest_dirty = True created += 1 pushed_details.append(change) @@ -1234,34 +1483,16 @@ def push( branch_id, allow_plaintext_fallback=allow_plaintext_fallback, ) - # Update both hashes so pull knows local == remote + # Update hashes so pull knows local == remote config_dir = project_root / branch_path / config_path_str config_file = config_dir / CONFIG_FILENAME if config_file.exists(): - new_file_hash = self._file_hash(config_file) - local_data = self._read_config_file(config_dir) - if local_data is not None: - merge_code_files(component_id, local_data, config_dir) - new_cfg_hash = config_hash(local_data) - else: - new_cfg_hash = "" - # Refresh extra hashes for extracted code files - new_extra: dict[str, str] = {} - for fname in [ - "_description.md", - "transform.sql", - "transform.py", - "code.py", - "pyproject.toml", - ]: - fpath = config_dir / fname - if fpath.exists(): - new_extra[fname] = self._file_hash(fpath) + hashes = self._compute_config_hashes(config_dir, component_id) for cfg in manifest.configurations: if cfg.component_id == component_id and cfg.id == config_id: - cfg.metadata["pull_hash"] = new_file_hash - cfg.metadata["pull_config_hash"] = new_cfg_hash - cfg.metadata["pull_extra_hashes"] = new_extra + cfg.metadata["pull_hash"] = hashes.file_hash + cfg.metadata["pull_config_hash"] = hashes.cfg_hash + cfg.metadata["pull_extra_hashes"] = hashes.extra_hashes break manifest_dirty = True updated += 1 @@ -1289,23 +1520,80 @@ def push( # elsewhere or a caller believing the push "mostly succeeded". # Surface to the CLI (exit non-zero) rather than burying in # result["errors"]. - if isinstance(exc, KeboolaApiError) and exc.error_code == "ENCRYPTION_FAILED": + if ( + isinstance(exc, KeboolaApiError) + and exc.error_code == ErrorCode.ENCRYPTION_FAILED + ): raise - logger.warning( - "Failed to push %s %s/%s: %s", - change_type, - component_id, - config_id, - exc, - ) - errors.append( - { - "change_type": change_type, - "component_id": component_id, - "config_id": config_id, - "message": str(exc), - } + self._record_push_error(errors, change_type, component_id, config_id, exc) + + # ---- Phase B: row creates / updates / deletes ---------------- + # row placeholder id -> ULID; ULID parent -> rows created under it. + created_row_id_map: dict[str, str] = {} + created_rows_by_parent: dict[str, list[str]] = {} + for change in row_changes: + change_type = change["change_type"] + component_id = change["component_id"] + config_id = change["config_id"] + config_path_str = change.get("path", "") + parent_config_id = change.get("parent_config_id", "") + # Remap the diff-time parent placeholder to the ULID assigned in + # Phase A so the manifest lookup and create_config_row both hit + # the real config (KFR-05). UPDATE/DELETE parents already carry + # a ULID and pass through unchanged. + effective_parent_id = created_id_map.get( + (component_id, parent_config_id), parent_config_id + ) + + try: + new_row_id = self._push_row_change( + client, + change_type=change_type, + component_id=component_id, + parent_config_id=effective_parent_id, + row_id=config_id, + row_path_str=config_path_str, + project_root=project_root, + manifest=manifest, + branch_id=branch_id, + allow_plaintext_fallback=allow_plaintext_fallback, ) + manifest_dirty = True + if change_type == "added": + created += 1 + if new_row_id: + if config_id: + created_row_id_map[config_id] = new_row_id + created_rows_by_parent.setdefault(effective_parent_id, []).append( + new_row_id + ) + elif change_type == "modified": + updated += 1 + elif change_type == "deleted": + deleted += 1 + pushed_details.append(change) + + except Exception as exc: + if ( + isinstance(exc, KeboolaApiError) + and exc.error_code == ErrorCode.ENCRYPTION_FAILED + ): + raise + self._record_push_error(errors, change_type, component_id, config_id, exc) + + # ---- Phase C: variable-link backfill (KFR-03) ---------------- + binding = self._resolve_variable_bindings( + client, + created_configs=created_configs, + created_id_map=created_id_map, + created_row_id_map=created_row_id_map, + created_rows_by_parent=created_rows_by_parent, + manifest=manifest, + branch_id=branch_id, + ) + errors.extend(binding.errors) + if binding.configs_rewritten: + manifest_dirty = True # Save manifest with updated hashes / new IDs / removed entries if manifest_dirty: @@ -1319,7 +1607,7 @@ def push( "errors": errors, "pushed_details": pushed_details, } - if name_drift_warnings: + if name_drift_warnings and not no_name_drift_warnings: result_data["name_drift_warnings"] = name_drift_warnings return result_data @@ -1328,6 +1616,59 @@ def push( # :func:`encrypt_secrets_in_config` directly. _encrypt_secrets_in_config = staticmethod(encrypt_secrets_in_config) + def _compute_config_hashes(self, config_dir: Path, component_id: str) -> LocalConfigHashes: + """Recompute the manifest bookkeeping hashes from a config dir on disk. + + Reads the (post-writeback) ``_config.yml``, merges code files for the + normalized config hash, and hashes each tracked companion file. Used + after create / update / variable-link backfill so ``sync diff`` sees + local == remote on the next run. + """ + config_file = config_dir / CONFIG_FILENAME + file_hash = self._file_hash(config_file) if config_file.exists() else "" + local_data = self._read_config_file(config_dir) + if local_data is not None: + merge_code_files(component_id, local_data, config_dir) + cfg_hash = config_hash(local_data) + else: + cfg_hash = "" + extra_hashes: dict[str, str] = {} + for fname in _EXTRA_HASH_FILENAMES: + fpath = config_dir / fname + if fpath.exists(): + extra_hashes[fname] = self._file_hash(fpath) + return LocalConfigHashes(file_hash=file_hash, cfg_hash=cfg_hash, extra_hashes=extra_hashes) + + def _record_push_error( + self, + errors: list[dict[str, str]], + change_type: str, + component_id: str, + config_id: str, + exc: Exception, + ) -> None: + """Log a non-fatal per-change push failure and accumulate it. + + Callers must re-raise fail-closed encryption errors + (:data:`ErrorCode.ENCRYPTION_FAILED`) *before* delegating here so a + partial push never silently drops a secret-bearing change. + """ + logger.warning( + "Failed to push %s %s/%s: %s", + change_type, + component_id, + config_id, + exc, + ) + errors.append( + { + "change_type": change_type, + "component_id": component_id, + "config_id": config_id, + "message": str(exc), + } + ) + def _push_row_change( self, client: Any, @@ -1341,13 +1682,21 @@ def _push_row_change( manifest: Manifest, branch_id: int | None, allow_plaintext_fallback: bool = False, - ) -> None: + ) -> str | None: """Dispatch a single row-level change (added/modified/deleted) to the API. ``#``-prefixed secrets in the row's configuration are encrypted via :func:`encrypt_secrets_in_config` before POST/PUT (same fail-closed semantics as parent configs). Mutates ``manifest`` in place; the caller is responsible for persisting it. + + ``parent_config_id`` must already be the *effective* parent id: on a + fresh CREATE the caller remaps the diff-time placeholder to the + API-assigned ULID before dispatch, so both the manifest parent lookup + and ``create_config_row(config_id=...)`` hit the real config (KFR-05). + + Returns the API-assigned row id on ``added`` (so the caller can map + placeholder -> ULID for variable-link backfill), else ``None``. """ parent = next( ( @@ -1378,16 +1727,15 @@ def _push_row_change( parent=parent, branch_id=branch_id, ) - return + return None # added / modified both read a local row file and encrypt-then-push. assert parent is not None # guarded above for non-deleted change_types - row_dir = ( - project_root / self._find_branch_path(manifest, branch_id) / parent.path / row_path_str - ) + source_branch_path = self._resolve_source_branch_path(manifest, project_root, branch_id) + row_dir = project_root / source_branch_path / parent.path / row_path_str if change_type == "added": - self._push_create_row( + return self._push_create_row( client, component_id=component_id, parent_config_id=parent_config_id, @@ -1398,7 +1746,6 @@ def _push_row_change( project_id=project_id, allow_plaintext_fallback=allow_plaintext_fallback, ) - return if change_type == "modified": self._push_update_row( @@ -1412,7 +1759,7 @@ def _push_row_change( project_id=project_id, allow_plaintext_fallback=allow_plaintext_fallback, ) - return + return None raise ValueError(f"Unsupported row change_type: {change_type}") @@ -1428,14 +1775,17 @@ def _push_create_row( branch_id: int | None, project_id: int | None, allow_plaintext_fallback: bool, - ) -> None: - """POST a new row; record API-assigned id + hashes in the parent's row list.""" + ) -> str: + """POST a new row; record API-assigned id + hashes in the parent's row list. + + Returns the API-assigned row id. + """ local_data = self._read_config_file(row_dir) if local_data is None: raise FileNotFoundError(f"Row file not found: {row_dir / CONFIG_FILENAME}") pristine_data = copy.deepcopy(local_data) - name, description, configuration = local_row_to_api(local_data) + name, description, configuration = local_row_to_api(local_data, component_id) configuration = encrypt_secrets_in_config( client, project_id, @@ -1463,13 +1813,14 @@ def _push_create_row( row_file = row_dir / CONFIG_FILENAME new_file_hash = self._file_hash(row_file) if row_file.exists() else "" cfg_hash_value = config_hash(pristine_data) - parent.rows.append( - ManifestConfigRow( - id=new_row_id, - path=row_path_str, - metadata={"pull_hash": new_file_hash, "pull_config_hash": cfg_hash_value}, - ) + self._writeback_create_row_in_manifest( + parent=parent, + row_path_str=row_path_str, + new_row_id=new_row_id, + file_hash=new_file_hash, + cfg_hash=cfg_hash_value, ) + return new_row_id def _push_update_row( self, @@ -1490,7 +1841,7 @@ def _push_update_row( raise FileNotFoundError(f"Row file not found: {row_dir / CONFIG_FILENAME}") pristine_data = copy.deepcopy(local_data) - name, description, configuration = local_row_to_api(local_data) + name, description, configuration = local_row_to_api(local_data, component_id) configuration = encrypt_secrets_in_config( client, project_id, @@ -1555,7 +1906,7 @@ def _push_create( allow_plaintext_fallback: bool = False, ) -> dict[str, Any] | None: """Create a new config from a local _config.yml file.""" - branch_path = self._find_branch_path(manifest, branch_id) + branch_path = self._resolve_source_branch_path(manifest, project_root, branch_id) config_dir = project_root / branch_path / config_path_str local_data = self._read_config_file(config_dir) if local_data is None: @@ -1615,7 +1966,7 @@ def _push_update( allow_plaintext_fallback: bool = False, ) -> None: """Update an existing config from a local _config.yml file.""" - branch_path = self._find_branch_path(manifest, branch_id) + branch_path = self._resolve_source_branch_path(manifest, project_root, branch_id) config_dir = project_root / branch_path / config_path_str local_data = self._read_config_file(config_dir) if local_data is None: @@ -1656,6 +2007,391 @@ def _push_update( # Use pristine_data so blocks/code stay only in their code files. self._writeback_after_push(pristine_data, config_dir, config_id, configuration) + def _resolve_variable_bindings( + self, + client: Any, + *, + created_configs: list[CreatedConfig], + created_id_map: dict[tuple[str, str], str], + created_row_id_map: dict[str, str], + created_rows_by_parent: dict[str, list[str]], + manifest: Manifest, + branch_id: int | None, + ) -> VariableBindingResult: + """Rebind transformation -> variables links from placeholders to ULIDs. + + On a fresh CREATE the transformation config is POSTed with its + ``_configuration_extra.variables_id`` / ``variables_values_id`` still + set to the externally-authored placeholder strings (``config_format`` + merges ``_configuration_extra`` into the API body verbatim). This pass, + run after the variables config and its values row have been created, + resolves each placeholder to the ULID assigned during this push, PUTs + the corrected configuration body, then rewrites the local file and + refreshes the manifest hashes so a re-push is clean (KFR-03). + + Resolution is a no-op when no ``keboola.variables`` config was created + this push (the already-bound / UPDATE path). When the exact placeholder + key misses but exactly one ``keboola.variables`` config was created this + push, it binds to that one with a warning; zero or ambiguous (>1) + matches accumulate an error rather than writing a broken link. + """ + result = VariableBindingResult() + + created_variables_ulids = [ + ulid + for (component_id, _placeholder), ulid in created_id_map.items() + if component_id == VARIABLES_COMPONENT_ID + ] + + for created in created_configs: + if created.component_id == VARIABLES_COMPONENT_ID: + continue # the variables config itself never carries a link + local_data = self._read_config_file(created.config_dir) + if local_data is None: + continue + extra = local_data.get("_configuration_extra") + if not isinstance(extra, dict): + continue + vars_placeholder = extra.get("variables_id") + if not vars_placeholder or not isinstance(vars_placeholder, str): + continue + raw_vals = extra.get("variables_values_id") + vals_placeholder = raw_vals if isinstance(raw_vals, str) else "" + + parent_ulid = self._resolve_variables_parent( + created=created, + vars_placeholder=vars_placeholder, + created_id_map=created_id_map, + created_variables_ulids=created_variables_ulids, + errors=result.errors, + ) + if parent_ulid is None: + continue + + row_ulid = self._resolve_variables_row( + created=created, + parent_ulid=parent_ulid, + vals_placeholder=vals_placeholder, + created_row_id_map=created_row_id_map, + created_rows_by_parent=created_rows_by_parent, + errors=result.errors, + ) + # A missing-but-required values row already recorded an error. + if vals_placeholder and row_ulid is None: + continue + + try: + self._apply_variable_binding( + client, + created=created, + local_data=local_data, + parent_ulid=parent_ulid, + row_ulid=row_ulid, + manifest=manifest, + branch_id=branch_id, + ) + except KeboolaApiError as exc: + result.errors.append( + { + "change_type": "variable_link", + "error_code": ErrorCode.VARIABLE_LINK_UNRESOLVED, + "component_id": created.component_id, + "config_id": created.config_id, + "message": str(exc), + } + ) + continue + result.configs_rewritten += 1 + + return result + + def _resolve_variables_parent( + self, + *, + created: CreatedConfig, + vars_placeholder: str, + created_id_map: dict[tuple[str, str], str], + created_variables_ulids: list[str], + errors: list[dict[str, str]], + ) -> str | None: + """Resolve a transformation's ``variables_id`` placeholder to a ULID. + + Returns the ULID, or ``None`` when there is nothing to backfill + (already-bound path) or the link is ambiguous (an error is appended). + """ + parent_ulid = created_id_map.get((VARIABLES_COMPONENT_ID, vars_placeholder)) + if parent_ulid is not None: + return parent_ulid + if not created_variables_ulids: + # No variables config created this push: the link is either already + # a ULID (UPDATE path) or points outside this push. Leave it. + return None + if len(created_variables_ulids) == 1: + parent_ulid = created_variables_ulids[0] + logger.warning( + "Transformation %s/%s variables_id placeholder %r did not match any " + "created variables config; binding to the single keboola.variables " + "config created this push (%s).", + created.component_id, + created.config_id, + vars_placeholder, + parent_ulid, + ) + return parent_ulid + errors.append( + { + "change_type": "variable_link", + "error_code": ErrorCode.VARIABLE_LINK_UNRESOLVED, + "component_id": created.component_id, + "config_id": created.config_id, + "message": ( + f"Cannot resolve variables_id placeholder {vars_placeholder!r}: " + f"{len(created_variables_ulids)} keboola.variables configs were " + "created this push and none matched by placeholder. Refusing to " + "write an ambiguous variables link." + ), + } + ) + return None + + def _resolve_variables_row( + self, + *, + created: CreatedConfig, + parent_ulid: str, + vals_placeholder: str, + created_row_id_map: dict[str, str], + created_rows_by_parent: dict[str, list[str]], + errors: list[dict[str, str]], + ) -> str | None: + """Resolve a transformation's ``variables_values_id`` placeholder. + + Returns the row ULID, or ``None`` when no values row was created (the + link is then left unset) or the choice is ambiguous (an error is + appended only when ``vals_placeholder`` was actually requested). + """ + if vals_placeholder: + mapped = created_row_id_map.get(vals_placeholder) + if mapped is not None: + return mapped + siblings = created_rows_by_parent.get(parent_ulid, []) + if len(siblings) == 1: + row_ulid = siblings[0] + if vals_placeholder: + logger.warning( + "Transformation %s/%s variables_values_id placeholder %r did not " + "match a created row; binding to the single row created under " + "variables config %s.", + created.component_id, + created.config_id, + vals_placeholder, + parent_ulid, + ) + return row_ulid + if vals_placeholder: + errors.append( + { + "change_type": "variable_link", + "error_code": ErrorCode.VARIABLE_LINK_UNRESOLVED, + "component_id": created.component_id, + "config_id": created.config_id, + "message": ( + f"Cannot resolve variables_values_id placeholder " + f"{vals_placeholder!r}: {len(siblings)} rows were created under " + f"variables config {parent_ulid}. Refusing to write an " + "ambiguous values link." + ), + } + ) + return None + + def _apply_variable_binding( + self, + client: Any, + *, + created: CreatedConfig, + local_data: dict[str, Any], + parent_ulid: str, + row_ulid: str | None, + manifest: Manifest, + branch_id: int | None, + ) -> None: + """PUT the resolved variables link, rewrite local, refresh manifest hashes. + + ``local_data`` is the pristine on-disk ``_config.yml`` dict; a deep + copy is code-merged to build the full PUT body so blocks/code stay only + in their companion files. Uses :meth:`KeboolaClient.update_config` + (PUT) directly -- **not** ``set_variables``, which would create a + *second* variables config. + """ + merged = copy.deepcopy(local_data) + merge_code_files(created.component_id, merged, created.config_dir) + _name, _description, configuration = local_config_to_api(merged) + configuration["variables_id"] = parent_ulid + if row_ulid: + configuration["variables_values_id"] = row_ulid + + client.update_config( + component_id=created.component_id, + config_id=created.config_id, + configuration=configuration, + change_description="Resolve variables link via kbagent sync push", + branch_id=branch_id, + ) + logger.info( + "Resolved variables link for %s/%s -> variables_id=%s variables_values_id=%s", + created.component_id, + created.config_id, + parent_ulid, + row_ulid, + ) + + # Rewrite the local _configuration_extra to the ULIDs (pristine data: + # no merged blocks leak into _config.yml). + extra = local_data.setdefault("_configuration_extra", {}) + extra["variables_id"] = parent_ulid + if row_ulid: + extra["variables_values_id"] = row_ulid + self._write_config_file(created.config_dir, local_data) + + # config_hash includes _configuration_extra, so refresh the stored + # hashes from the post-rewrite disk state or sync diff sees a conflict. + hashes = self._compute_config_hashes(created.config_dir, created.component_id) + target_branch = branch_id or 0 + for cfg in manifest.configurations: + if ( + cfg.branch_id == target_branch + and cfg.component_id == created.component_id + and cfg.id == created.config_id + ): + cfg.metadata["pull_hash"] = hashes.file_hash + cfg.metadata["pull_config_hash"] = hashes.cfg_hash + cfg.metadata["pull_extra_hashes"] = hashes.extra_hashes + break + + def _writeback_create_config_in_manifest( + self, + *, + manifest: Manifest, + component_id: str, + branch_id: int | None, + config_path_str: str, + new_id: str, + file_hash: str, + cfg_hash: str, + ) -> WritebackResult: + """Record a freshly-created config in the manifest. + + If a placeholder entry already exists at + ``(branch_id, component_id, path)`` -- the FIIA / scaffold emit + pattern -- update it in place, preserving any user-declared metadata + (e.g. ``KBC.configuration.folderName``) and refreshing only the + bookkeeping hashes. Otherwise append a new entry. + + Matching includes ``branch_id`` because a single manifest can hold + entries from multiple branches in git-branching mode; matching on + ``(component_id, path)`` alone would risk updating the wrong branch's + entry when the same logical path exists under two branches. + + Returns a :class:`WritebackResult` carrying the entry and its + pre-overwrite ``previous_id`` so the create pass can remap any child + row parents / transformation variable links from the placeholder id + to the freshly-assigned ULID. + """ + target_branch = branch_id or 0 + for entry in manifest.configurations: + if ( + entry.branch_id == target_branch + and entry.component_id == component_id + and entry.path == config_path_str + ): + previous_id = entry.id + entry.id = new_id + entry.metadata["pull_hash"] = file_hash + entry.metadata["pull_config_hash"] = cfg_hash + return WritebackResult(entry=entry, previous_id=previous_id) + new_entry = ManifestConfiguration( + branchId=target_branch, + componentId=component_id, + id=new_id, + path=config_path_str, + metadata={"pull_hash": file_hash, "pull_config_hash": cfg_hash}, + ) + manifest.configurations.append(new_entry) + return WritebackResult(entry=new_entry, previous_id="") + + def _writeback_create_row_in_manifest( + self, + *, + parent: ManifestConfiguration, + row_path_str: str, + new_row_id: str, + file_hash: str, + cfg_hash: str, + ) -> ManifestConfigRow: + """Record a freshly-created row under its parent in the manifest. + + Mirrors :meth:`_writeback_create_config_in_manifest` for rows: update + any placeholder row entry in place, otherwise append. + """ + for row in parent.rows: + if row.path == row_path_str: + row.id = new_row_id + row.metadata["pull_hash"] = file_hash + row.metadata["pull_config_hash"] = cfg_hash + return row + new_row = ManifestConfigRow( + id=new_row_id, + path=row_path_str, + metadata={"pull_hash": file_hash, "pull_config_hash": cfg_hash}, + ) + parent.rows.append(new_row) + return new_row + + def _propagate_kbc_metadata( + self, + client: Any, + entry: ManifestConfiguration, + branch_id: int | None, + ) -> str | None: + """POST any ``KBC.*`` keys from the manifest entry to the metadata API. + + Bookkeeping keys (``pull_hash``, ``pull_config_hash``, ...) live in the + same metadata dict but are filtered by the ``KBC.`` prefix. Called only + on CREATE; updates use ``kbagent config set-metadata`` explicitly. The + metadata API stores configuration-level annotations only -- this is + **not** a secret store; do not place tokens or passwords under + ``KBC.*`` keys. + + Returns ``None`` on success (or when there are no KBC.* keys to + propagate). Returns the API error message on a non-fatal write + failure: the config is already created on the remote and the + manifest writeback is complete, so a single failed metadata POST + is reported back to the push loop as an accumulated error rather + than aborting the rest of the push. + """ + entries = [ + (key, str(value)) for key, value in entry.metadata.items() if key.startswith("KBC.") + ] + if not entries: + return None + try: + client.set_config_metadata( + component_id=entry.component_id, + config_id=entry.id, + entries=entries, + branch_id=branch_id, + ) + except KeboolaApiError as exc: + logger.warning( + "Failed to propagate KBC.* metadata for %s/%s: %s", + entry.component_id, + entry.id, + exc, + ) + return exc.message + return None + def _writeback_after_push( self, local_data: dict[str, Any], @@ -1744,6 +2480,17 @@ def _worker(alias: str) -> None: ) results[alias] = result success_count += 1 + except SyncConflictError as exc: + # Preserve the structured conflict so a programmatic / AI + # consumer of `--all-projects --json` can tell a merge conflict + # apart from any other error and read the conflicting configs -- + # the single-project path emits the same code + conflicts. + results[alias] = { + "error": exc.message, + "error_code": exc.error_code, + "conflicts": exc.conflicts, + } + failed_count += 1 except Exception as exc: results[alias] = {"error": str(exc)} failed_count += 1 @@ -2120,10 +2867,12 @@ def _resolve_branch_id( project: Any, manifest: "Manifest", project_root: Path, + branch_override: int | None = None, ) -> int | None: """Resolve the Keboola branch ID for sync operations. Priority: + 0. ``branch_override`` (CLI ``--branch ``) -- wins over everything 1. Git-branching mode: read branch-mapping.json for current git branch 2. ``active_branch_id`` from project config (``kbagent branch use``) 3. First branch in manifest (production fallback) @@ -2139,6 +2888,12 @@ def _resolve_branch_id( from ..sync.branch_mapping import load_branch_mapping from ..sync.git_utils import get_current_branch + # CLI override beats every persisted source so a user can target a + # dev branch from a clean git workspace without first running + # `branch use` or `branch-link`. + if branch_override is not None: + return branch_override + if manifest.git_branching.enabled: git_branch = get_current_branch(project_root) if git_branch: @@ -2660,6 +3415,54 @@ def _find_branch_path(self, manifest: Manifest, branch_id: int | None) -> str: ) return manifest.branches[0].path if manifest.branches else "main" + def _branch_path_has_configs(self, branch_dir: Path) -> bool: + """Return True if *branch_dir* exists and holds at least one config. + + A config is any ``_config.yml`` below the branch root other than the + optional branch-level ``_config.yml`` itself. Used to decide whether a + target-branch subtree is materialized on disk. + """ + if not branch_dir.is_dir(): + return False + for config_file in branch_dir.rglob(CONFIG_FILENAME): + if config_file.parent != branch_dir: + return True + return False + + def _resolve_source_branch_path( + self, + manifest: Manifest, + project_root: Path, + target_branch_id: int | None, + ) -> str: + """Resolve the on-disk branch subtree to read local configs from. + + Source (where files live) and target (where the API writes) are + decoupled. When the target branch has its own materialized + ``/`` subtree on disk, that subtree is the source + (unchanged multi-branch-directory behaviour). Otherwise the default + branch tree (``manifest.branches[0]``, i.e. ``main/``) is the source -- + the "promote the default tree to a target dev branch" path used by + ``sync push --branch `` when no per-branch subtree exists + (KFR-07 option-B). + + API calls still target ``target_branch_id``; only the *read* path is + affected. + """ + target_path = self._find_branch_path(manifest, target_branch_id) + if self._branch_path_has_configs(project_root / target_path): + return target_path + default_path = manifest.branches[0].path if manifest.branches else target_path + if default_path != target_path: + logger.info( + "No config files under target branch path '%s'; promoting default " + "tree '%s' to branch %s", + target_path, + default_path, + target_branch_id, + ) + return default_path + def _find_untracked_configs( self, project_root: Path, diff --git a/src/keboola_agent_cli/services/workspace_service.py b/src/keboola_agent_cli/services/workspace_service.py index 6238ade8..39dec856 100644 --- a/src/keboola_agent_cli/services/workspace_service.py +++ b/src/keboola_agent_cli/services/workspace_service.py @@ -6,9 +6,13 @@ """ import logging +from dataclasses import dataclass from typing import Any -from ..constants import QUERY_SERVICE_COMPATIBLE_LOGIN_TYPES +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.asymmetric import rsa + +from ..constants import QUERY_SERVICE_COMPATIBLE_LOGIN_TYPES, SNOWFLAKE_WORKSPACE_LOGIN_TYPE from ..errors import ConfigError, ErrorCode, KeboolaApiError from ..models import ProjectConfig from .base import BaseService @@ -16,6 +20,14 @@ logger = logging.getLogger(__name__) +@dataclass(frozen=True) +class SnowflakeWorkspaceKeyPair: + """PEM key material for Snowflake key-pair workspace authentication.""" + + private_pem: str + public_pem: str + + def _classify_qs_compatibility(login_type: str) -> bool: """Map a Storage API workspace ``connection.loginType`` to Query-Service compat. @@ -28,6 +40,39 @@ def _classify_qs_compatibility(login_type: str) -> bool: return login_type in QUERY_SERVICE_COMPATIBLE_LOGIN_TYPES +def _workspace_login_type_for_backend(backend: str) -> str | None: + """Return the loginType kbagent should request for newly created workspaces.""" + if backend.lower() == "snowflake": + return SNOWFLAKE_WORKSPACE_LOGIN_TYPE + return None + + +def _generate_snowflake_workspace_key_pair() -> SnowflakeWorkspaceKeyPair: + """Generate the unencrypted PEM key pair required by Snowflake workspaces.""" + private_key = rsa.generate_private_key(public_exponent=65537, key_size=2048) + private_key_pem = private_key.private_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PrivateFormat.PKCS8, + encryption_algorithm=serialization.NoEncryption(), + ).decode("ascii") + public_key_pem = ( + private_key.public_key() + .public_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PublicFormat.SubjectPublicKeyInfo, + ) + .decode("ascii") + ) + return SnowflakeWorkspaceKeyPair(private_pem=private_key_pem, public_pem=public_key_pem) + + +def _workspace_key_pair_for_backend(backend: str) -> SnowflakeWorkspaceKeyPair | None: + """Return private/public key material for backends that require it.""" + if backend.lower() == "snowflake": + return _generate_snowflake_workspace_key_pair() + return None + + def find_storage_workspace_for_sandbox_config( workspaces: list[dict[str, Any]], config_id: str, @@ -164,8 +209,10 @@ def create_workspace( - Default (headless): fast (~1s) via Storage API. Not visible in Keboola UI. - UI mode (--ui): slower (~15s) via Queue job. Visible in UI Workspaces tab. - IMPORTANT: Password is only available on creation (headless mode). - In UI mode, password must be retrieved via 'workspace password' command. + IMPORTANT: Credentials are only available on creation (headless mode). + Snowflake returns a generated private key; password-based workspaces + return a password. In UI mode, password must be retrieved via + 'workspace password' command. Args: alias: Project alias. @@ -225,15 +272,19 @@ def _create_workspace_direct( read_only: bool, ) -> dict[str, Any]: """Create workspace via Storage API (fast, headless).""" + key_pair = _workspace_key_pair_for_backend(backend) ws_data = client.create_config_workspace( branch_id=branch_id, component_id="keboola.sandboxes", config_id=config_id, backend=backend, + login_type=_workspace_login_type_for_backend(backend), + public_key=key_pair.public_pem if key_pair else None, ) connection = ws_data.get("connection", {}) - return { + credential_label = "private key" if key_pair else "password" + result = { "project_alias": alias, "workspace_id": ws_data.get("id"), "name": name, @@ -247,11 +298,12 @@ def _create_workspace_direct( "password": connection.get("password", ""), "read_only": read_only, "ui_mode": False, - "message": ( - f"Workspace '{name}' created in project '{alias}'. " - "Save the password -- it cannot be retrieved later!" - ), + "message": f"Workspace '{name}' created in project '{alias}'. " + f"Save the {credential_label} -- it cannot be retrieved later!", } + if key_pair is not None: + result["private_key"] = key_pair.private_pem + return result def _create_workspace_via_job( self, @@ -262,6 +314,9 @@ def _create_workspace_via_job( backend: str, ) -> dict[str, Any]: """Create workspace via Queue job (slower, visible in UI).""" + # The Queue job path does not expose a publicKey/loginType input. Keep + # returning a reset password here; headless Snowflake creates use the + # key-pair path in _create_workspace_direct(). job = client.create_job( component_id="keboola.sandboxes", config_id=config_id, @@ -878,11 +933,14 @@ def create_from_transformation( ) # Create config-tied workspace + key_pair = _workspace_key_pair_for_backend(effective_backend) ws_data = client.create_config_workspace( branch_id=branch_id, component_id=component_id, config_id=config_id, backend=effective_backend, + login_type=_workspace_login_type_for_backend(effective_backend), + public_key=key_pair.public_pem if key_pair else None, ) workspace_id = ws_data.get("id") @@ -914,7 +972,8 @@ def create_from_transformation( workspace_id, table_defs, branch_id=branch_id, preserve=preserve ) - return { + credential_label = "private key" if key_pair else "password" + result = { "project_alias": alias, "workspace_id": workspace_id, "branch_id": branch_id, @@ -929,11 +988,12 @@ def create_from_transformation( "user": connection.get("user", ""), "password": connection.get("password", ""), "tables_loaded": source_tables, - "message": ( - f"Workspace {workspace_id} created from transformation " - f"'{config_id}' with {len(source_tables)} table(s) loaded. " - "Save the password -- it cannot be retrieved later!" - ), + "message": f"Workspace {workspace_id} created from transformation " + f"'{config_id}' with {len(source_tables)} table(s) loaded. " + f"Save the {credential_label} -- it cannot be retrieved later!", } + if key_pair is not None: + result["private_key"] = key_pair.private_pem + return result finally: client.close() diff --git a/src/keboola_agent_cli/stream_client.py b/src/keboola_agent_cli/stream_client.py new file mode 100644 index 00000000..65c44417 --- /dev/null +++ b/src/keboola_agent_cli/stream_client.py @@ -0,0 +1,217 @@ +"""Keboola Stream (Data Streams) API client with retry, timeouts, token masking. + +This module talks to the Keboola **Stream control-plane API** for managing +Data Streams sources and sinks (list / create / detail / delete). The base URL +is derived from the Storage API stack URL by replacing 'connection.' with +'stream.' in the hostname (the same scheme used for 'ai.'/'queue.'), and the +request is authenticated with the per-project Storage API token +(``X-StorageApi-Token``) -- no manage token is involved. + +Important: the OTLP *ingestion* endpoint (``stream-in./otlp/...``) is a +separate data-plane host and is NOT derived here -- it is returned by the API in +the source's ``otlp.url`` field. This client only speaks to the control plane. + +Source-create and source-delete are asynchronous: the API returns a ``Task`` +(202 Accepted); :meth:`wait_for_task` polls ``GET /v1/tasks/{taskId}`` until the +task reports ``isFinished``. + +Inherits shared retry/error logic from :class:`BaseHttpClient`. +""" + +from __future__ import annotations + +import logging +import time +from typing import Any +from urllib.parse import quote, urlparse + +from .constants import STREAM_API_TIMEOUT, STREAM_TASK_POLL_INTERVAL, STREAM_TASK_TIMEOUT +from .errors import ErrorCode, KeboolaApiError +from .http_base import BaseHttpClient + +logger = logging.getLogger(__name__) + + +class StreamClient(BaseHttpClient): + """HTTP client for the Keboola Stream (Data Streams) control-plane API. + + Provides source CRUD, sink listing, and async-task polling, with the + retry/backoff (429/5xx), timeouts, and token masking inherited from + :class:`BaseHttpClient`. + """ + + def __init__(self, stack_url: str, token: str) -> None: + self._stack_url = stack_url.rstrip("/") + stream_base_url = self._derive_service_url(self._stack_url, "stream") + headers = { + "X-StorageApi-Token": token, + "Content-Type": "application/json", + } + super().__init__( + base_url=stream_base_url, + token=token, + headers=headers, + timeout=STREAM_API_TIMEOUT, + ) + + def __enter__(self) -> StreamClient: + return self + + def __exit__(self, *args: Any) -> None: + self.close() + + # ------------------------------------------------------------------ + # Sources + # ------------------------------------------------------------------ + + def list_sources(self, branch_id: str) -> dict[str, Any]: + """List sources in a branch (``GET /v1/branches/{branch}/sources``).""" + path = f"/v1/branches/{quote(branch_id, safe='')}/sources" + response = self._do_request("GET", path) + return response.json() + + def get_source(self, branch_id: str, source_id: str) -> dict[str, Any]: + """Fetch one source (``GET /v1/branches/{branch}/sources/{id}``).""" + path = f"/v1/branches/{quote(branch_id, safe='')}/sources/{quote(source_id, safe='')}" + response = self._do_request("GET", path) + return response.json() + + def create_source( + self, + branch_id: str, + name: str, + source_type: str, + source_id: str | None = None, + description: str | None = None, + ) -> dict[str, Any]: + """Create a source. Returns the async ``Task`` (poll with wait_for_task).""" + path = f"/v1/branches/{quote(branch_id, safe='')}/sources" + payload: dict[str, Any] = {"name": name, "type": source_type} + if source_id is not None: + payload["sourceId"] = source_id + if description is not None: + payload["description"] = description + response = self._do_request("POST", path, json=payload) + return response.json() + + def delete_source(self, branch_id: str, source_id: str) -> dict[str, Any]: + """Delete a source. Returns the async ``Task`` (poll with wait_for_task).""" + path = f"/v1/branches/{quote(branch_id, safe='')}/sources/{quote(source_id, safe='')}" + response = self._do_request("DELETE", path) + return response.json() + + # ------------------------------------------------------------------ + # Sinks + # ------------------------------------------------------------------ + + def list_sinks(self, branch_id: str, source_id: str) -> dict[str, Any]: + """List a source's sinks (``GET .../sources/{id}/sinks``).""" + path = f"/v1/branches/{quote(branch_id, safe='')}/sources/{quote(source_id, safe='')}/sinks" + response = self._do_request("GET", path) + return response.json() + + def create_sink( + self, + branch_id: str, + source_id: str, + *, + name: str, + table_id: str, + columns: list[dict[str, Any]], + allowed_signals: list[str] | None = None, + sink_id: str | None = None, + ) -> dict[str, Any]: + """Create a table sink on a source. Returns the async ``Task``. + + ``columns`` is the table mapping column list (see the Stream API + ``TableColumn`` schema). ``allowed_signals`` restricts which OTLP signals + route to this sink (logs/metrics/traces); omit to accept all. + """ + path = f"/v1/branches/{quote(branch_id, safe='')}/sources/{quote(source_id, safe='')}/sinks" + payload: dict[str, Any] = { + "type": "table", + "name": name, + "table": { + "type": "keboola", + "tableId": table_id, + "mapping": {"columns": columns}, + }, + } + if allowed_signals is not None: + payload["allowedSignals"] = allowed_signals + if sink_id is not None: + payload["sinkId"] = sink_id + response = self._do_request("POST", path, json=payload) + return response.json() + + # ------------------------------------------------------------------ + # Tasks (async create/delete) + # ------------------------------------------------------------------ + + def get_task(self, task_id: str) -> dict[str, Any]: + """Fetch a task by id (``GET /v1/tasks/{taskId}``).""" + path = f"/v1/tasks/{quote(task_id, safe='')}" + response = self._do_request("GET", path) + return response.json() + + def wait_for_task( + self, + task: dict[str, Any], + timeout: float = STREAM_TASK_TIMEOUT, + poll_interval: float = STREAM_TASK_POLL_INTERVAL, + ) -> dict[str, Any]: + """Poll a ``Task`` to completion and return the finished task. + + Accepts the Task dict returned by :meth:`create_source` / + :meth:`delete_source`. Polls its canonical poll URL (the task's ``url`` + reduced to a path, falling back to ``/v1/tasks/{taskId}``) until + ``isFinished`` is true, then raises :class:`KeboolaApiError` if the task + failed. + """ + if task.get("isFinished"): + return self._check_task_result(task) + + poll_path = self._task_poll_path(task) + deadline = time.monotonic() + timeout + latest = task + while time.monotonic() < deadline: + time.sleep(poll_interval) + response = self._do_request("GET", poll_path) + latest = response.json() + if latest.get("isFinished"): + return self._check_task_result(latest) + + raise KeboolaApiError( + message=( + f"Stream task '{latest.get('taskId', '?')}' did not finish within " + f"{timeout:.0f}s (last status: {latest.get('status', 'unknown')})" + ), + error_code=ErrorCode.TIMEOUT, + retryable=True, + ) + + def _task_poll_path(self, task: dict[str, Any]) -> str: + """Resolve the path to poll for ``task``. + + Prefers the task's ``url`` field reduced to a path relative to the + Stream base; falls back to ``/v1/tasks/{taskId}``. + """ + url = task.get("url") + if isinstance(url, str) and url: + parsed = urlparse(url) + if parsed.path: + return parsed.path + task_id = task.get("taskId", "") + return f"/v1/tasks/{quote(str(task_id), safe='')}" + + @staticmethod + def _check_task_result(task: dict[str, Any]) -> dict[str, Any]: + """Return a finished task, raising if it ended in error.""" + error = task.get("error") + status = task.get("status") + if error or status == "error": + raise KeboolaApiError( + message=f"Stream task failed: {error or status}", + error_code=ErrorCode.API_ERROR, + ) + return task diff --git a/src/keboola_agent_cli/sync/config_format.py b/src/keboola_agent_cli/sync/config_format.py index ef2d54fa..92e3bf38 100644 --- a/src/keboola_agent_cli/sync/config_format.py +++ b/src/keboola_agent_cli/sync/config_format.py @@ -257,6 +257,7 @@ def api_row_to_local(row_data: dict[str, Any], component_id: str) -> dict[str, A def local_row_to_api( row_yml: dict[str, Any], + component_id: str | None = None, ) -> tuple[str, str, dict[str, Any]]: """Convert a local row ``_config.yml`` back to API format. @@ -265,15 +266,25 @@ def local_row_to_api( body. For all other components this behaves identically to :func:`local_config_to_api`. + Args: + row_yml: The local row ``_config.yml`` dict. + component_id: The component id the row belongs to. When provided + (the caller almost always knows it), it drives the hoist check + directly. When ``None`` (back-compat), the id is read from the + file's ``_keboola.component_id``. The explicit form is required + for fresh-CREATE rows whose scaffold ``_config.yml`` does not yet + carry a ``_keboola`` block, so that ``keboola.variables`` rows + still hoist their ``values`` array into the API body (KFR-04). + Returns: A tuple of ``(name, description, configuration_dict)``. """ keboola_meta: dict[str, Any] = row_yml.get("_keboola") or {} - component_id: str = keboola_meta.get("component_id", "") + resolved_component_id: str = component_id or keboola_meta.get("component_id", "") name, description, configuration = local_config_to_api(row_yml) - if component_id in ROW_HOIST_COMPONENTS: + if resolved_component_id in ROW_HOIST_COMPONENTS: for key, value in row_yml.items(): if key in _ROW_LOCAL_RESERVED_KEYS: continue diff --git a/src/keboola_agent_cli/sync/diff_engine.py b/src/keboola_agent_cli/sync/diff_engine.py index 92292112..3320d238 100644 --- a/src/keboola_agent_cli/sync/diff_engine.py +++ b/src/keboola_agent_cli/sync/diff_engine.py @@ -90,7 +90,13 @@ def normalize_for_comparison(obj: Any) -> Any: - Sort dict keys for consistent hashing. - Strip ``_keboola`` metadata block (internal, not part of config content). - Strip ``version`` key (local format marker). - - Strip ``_configuration_extra`` key (internal round-trip aid). + + Note: ``_configuration_extra`` is **not** stripped -- it carries real + config payload (e.g. ``keboola.flow`` phases/tasks, and a transformation's + ``variables_id`` / ``variables_values_id`` links). It is part of + :data:`config_hash`, so any code that mutates it (e.g. the fresh-CREATE + variable-link backfill in ``sync push``) must refresh the stored + ``pull_config_hash`` afterwards or ``sync diff`` will report a conflict. Returns a deep copy -- the original object is never mutated. """ diff --git a/tests/test_agent_prompt.py b/tests/test_agent_prompt.py index c88910d4..5d11df98 100644 --- a/tests/test_agent_prompt.py +++ b/tests/test_agent_prompt.py @@ -24,8 +24,11 @@ PLUGIN_JSON = PLUGIN_DIR / ".claude-plugin" / "plugin.json" # ~20k tokens ≈ 80 kB in typical English markdown (~4 chars/token). -# We target under 60 kB to leave headroom. -PROMPT_BYTE_BUDGET = 60_000 +# We target well under that to leave headroom. Bumped 60 kB -> 62 kB in +# v0.48.0 to fit the `feature` command-group matrix row; if this keeps +# creeping up, split keboola-expert into per-domain specialists rather +# than raising the ceiling again. +PROMPT_BYTE_BUDGET = 62_000 @pytest.fixture(scope="module") diff --git a/tests/test_client.py b/tests/test_client.py index f6bcee38..71ef7c89 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -509,6 +509,72 @@ def test_storage_api_token_header(self, httpx_mock) -> None: client.close() +class TestConfigWorkspaces: + """Tests for config-tied workspace endpoints.""" + + def test_create_config_workspace_includes_login_type_when_requested(self, httpx_mock) -> None: + """Explicit loginType is included in the Storage API workspace payload.""" + httpx_mock.add_response( + url=( + "https://connection.keboola.com/v2/storage/branch/123/components/" + "keboola.sandboxes/configs/cfg-1/workspaces" + ), + json={"id": 42}, + status_code=201, + ) + + client = KeboolaClient( + stack_url="https://connection.keboola.com", + token="901-55555-fakeTestTokenDoNotUseXXXXXXXX", + ) + result = client.create_config_workspace( + branch_id=123, + component_id="keboola.sandboxes", + config_id="cfg-1", + backend="snowflake", + login_type="snowflake-person-keypair", + public_key="-----BEGIN PUBLIC KEY-----\ntest\n-----END PUBLIC KEY-----\n", + ) + + assert result == {"id": 42} + request = httpx_mock.get_request() + assert json.loads(request.content) == { + "backend": "snowflake", + "loginType": "snowflake-person-keypair", + "publicKey": "-----BEGIN PUBLIC KEY-----\ntest\n-----END PUBLIC KEY-----\n", + } + client.close() + + def test_create_config_workspace_omits_login_type_when_default(self, httpx_mock) -> None: + """A None login type is omitted so non-Snowflake backends keep Storage defaults.""" + httpx_mock.add_response( + url=( + "https://connection.keboola.com/v2/storage/branch/123/components/" + "keboola.sandboxes/configs/cfg-1/workspaces" + ), + json={"id": 42}, + status_code=201, + ) + + client = KeboolaClient( + stack_url="https://connection.keboola.com", + token="901-55555-fakeTestTokenDoNotUseXXXXXXXX", + ) + result = client.create_config_workspace( + branch_id=123, + component_id="keboola.sandboxes", + config_id="cfg-1", + backend="bigquery", + login_type=None, + public_key=None, + ) + + assert result == {"id": 42} + request = httpx_mock.get_request() + assert json.loads(request.content) == {"backend": "bigquery"} + client.close() + + class TestContextManager: """Tests for context manager support.""" diff --git a/tests/test_config_store.py b/tests/test_config_store.py index 94b9817c..74abb084 100644 --- a/tests/test_config_store.py +++ b/tests/test_config_store.py @@ -10,7 +10,7 @@ from keboola_agent_cli.config_store import CURRENT_CONFIG_VERSION, ConfigStore from keboola_agent_cli.errors import ConfigError -from keboola_agent_cli.models import AppConfig, ProjectConfig +from keboola_agent_cli.models import AppConfig, DeveloperPortalIdentity, ProjectConfig class TestLoadEmptyConfig: @@ -723,3 +723,256 @@ def test_set_project_branch_unknown_alias(self, tmp_config_dir: Path) -> None: with pytest.raises(ConfigError, match="not found"): store.set_project_branch("nonexistent", 456) + + +class TestDevPortalIdentityCrud: + def test_add_first_identity_becomes_default(self, config_store): + ident = DeveloperPortalIdentity(username="u", password="p") + config_store.add_dev_portal_identity("alpha", ident) + cfg = config_store.load() + assert cfg.dev_portal_identities["alpha"].username == "u" + assert cfg.default_dev_portal_identity == "alpha" + + def test_add_duplicate_alias_raises(self, config_store): + ident = DeveloperPortalIdentity(username="u", password="p") + config_store.add_dev_portal_identity("alpha", ident) + with pytest.raises(ConfigError, match="already exists"): + config_store.add_dev_portal_identity("alpha", ident) + + def test_remove_identity(self, config_store): + ident = DeveloperPortalIdentity(username="u", password="p") + config_store.add_dev_portal_identity("alpha", ident) + config_store.add_dev_portal_identity("beta", ident) + config_store.remove_dev_portal_identity("alpha") + cfg = config_store.load() + assert "alpha" not in cfg.dev_portal_identities + assert cfg.default_dev_portal_identity == "beta" + + def test_remove_unknown_raises(self, config_store): + with pytest.raises(ConfigError, match="not found"): + config_store.remove_dev_portal_identity("missing") + + def test_remove_last_clears_default(self, config_store): + ident = DeveloperPortalIdentity(username="u", password="p") + config_store.add_dev_portal_identity("alpha", ident) + config_store.remove_dev_portal_identity("alpha") + cfg = config_store.load() + assert cfg.default_dev_portal_identity == "" + + def test_edit_identity(self, config_store): + ident = DeveloperPortalIdentity(username="u", password="p") + config_store.add_dev_portal_identity("alpha", ident) + config_store.edit_dev_portal_identity("alpha", vendor="keboola", password="p2") + cfg = config_store.load() + assert cfg.dev_portal_identities["alpha"].vendor == "keboola" + assert cfg.dev_portal_identities["alpha"].password == "p2" + assert cfg.dev_portal_identities["alpha"].username == "u" + + def test_rename_identity(self, config_store): + ident = DeveloperPortalIdentity(username="u", password="p") + config_store.add_dev_portal_identity("alpha", ident) + config_store.rename_dev_portal_identity("alpha", "alpha-prod") + cfg = config_store.load() + assert "alpha" not in cfg.dev_portal_identities + assert "alpha-prod" in cfg.dev_portal_identities + assert cfg.default_dev_portal_identity == "alpha-prod" + + def test_rename_collision_raises(self, config_store): + ident = DeveloperPortalIdentity(username="u", password="p") + config_store.add_dev_portal_identity("alpha", ident) + config_store.add_dev_portal_identity("beta", ident) + with pytest.raises(ConfigError, match="already in use"): + config_store.rename_dev_portal_identity("alpha", "beta") + + def test_set_default_unknown_raises(self, config_store): + with pytest.raises(ConfigError, match="not found"): + config_store.set_default_dev_portal_identity("ghost") + + def test_set_default(self, config_store): + ident = DeveloperPortalIdentity(username="u", password="p") + config_store.add_dev_portal_identity("alpha", ident) + config_store.add_dev_portal_identity("beta", ident) + config_store.set_default_dev_portal_identity("beta") + cfg = config_store.load() + assert cfg.default_dev_portal_identity == "beta" + + +class TestEnvProjectInjection: + """Headless env-only project injection (issue #359). + + KBAGENT_PROJECT_FROM_ENV=1 + KBC_TOKEN + KBC_STORAGE_API_URL make load() + synthesize an in-memory '__env__' project; save() never persists it. + """ + + TOKEN = "901-99999-fakeHeadlessTokenDoNotUseXXXXX" + URL = "https://connection.keboola.com" + + def _opt_in(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("KBAGENT_PROJECT_FROM_ENV", "1") + monkeypatch.setenv("KBC_TOKEN", self.TOKEN) + monkeypatch.setenv("KBC_STORAGE_API_URL", self.URL) + + def test_not_injected_without_opt_in( + self, tmp_config_dir: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """KBC_TOKEN alone (no flag) must NOT create a phantom project.""" + monkeypatch.delenv("KBAGENT_PROJECT_FROM_ENV", raising=False) + monkeypatch.setenv("KBC_TOKEN", self.TOKEN) + monkeypatch.setenv("KBC_STORAGE_API_URL", self.URL) + config = ConfigStore(config_dir=tmp_config_dir).load() + assert config.projects == {} + + def test_injected_into_empty_config( + self, tmp_config_dir: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """With opt-in and no config file, '__env__' is injected and defaulted.""" + self._opt_in(monkeypatch) + config = ConfigStore(config_dir=tmp_config_dir).load() + assert "__env__" in config.projects + env_proj = config.projects["__env__"] + assert env_proj.token == self.TOKEN + assert env_proj.stack_url == self.URL + assert env_proj.ephemeral is True + # project_id is recovered offline from the token prefix (901-99999-...). + assert env_proj.project_id == 901 + # project_name needs an API call -> left blank by the offline injection. + assert env_proj.project_name == "" + assert config.default_project == "__env__" + + def test_non_numeric_token_prefix_yields_no_project_id( + self, tmp_config_dir: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """A token whose prefix isn't numeric leaves project_id unset, no crash.""" + monkeypatch.setenv("KBAGENT_PROJECT_FROM_ENV", "1") + monkeypatch.setenv("KBC_TOKEN", "abc-def-notNumericPrefixXXXXXXXXXXXX") + monkeypatch.setenv("KBC_STORAGE_API_URL", self.URL) + config = ConfigStore(config_dir=tmp_config_dir).load() + assert config.projects["__env__"].project_id is None + + def test_opt_in_truthy_variants( + self, tmp_config_dir: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """Accept common truthy spellings of the opt-in flag.""" + monkeypatch.setenv("KBC_TOKEN", self.TOKEN) + monkeypatch.setenv("KBC_STORAGE_API_URL", self.URL) + for value in ("true", "YES", "On", "1"): + monkeypatch.setenv("KBAGENT_PROJECT_FROM_ENV", value) + config = ConfigStore(config_dir=tmp_config_dir).load() + assert "__env__" in config.projects, value + + def test_missing_creds_fail_fast( + self, tmp_config_dir: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """Flag set but creds missing must raise, not silently skip.""" + monkeypatch.setenv("KBAGENT_PROJECT_FROM_ENV", "1") + monkeypatch.delenv("KBC_TOKEN", raising=False) + monkeypatch.setenv("KBC_STORAGE_API_URL", self.URL) + with pytest.raises(ConfigError, match="KBC_TOKEN"): + ConfigStore(config_dir=tmp_config_dir).load() + + def test_bare_host_url_normalized( + self, tmp_config_dir: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """A bare host in KBC_STORAGE_API_URL is normalized to https://, not rejected.""" + monkeypatch.setenv("KBAGENT_PROJECT_FROM_ENV", "1") + monkeypatch.setenv("KBC_TOKEN", self.TOKEN) + monkeypatch.setenv("KBC_STORAGE_API_URL", "connection.keboola.com") + config = ConfigStore(config_dir=tmp_config_dir).load() + assert config.projects["__env__"].stack_url == "https://connection.keboola.com" + + def test_invalid_url_fails_clean( + self, tmp_config_dir: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """An http:// env URL raises a clean ConfigError, not a raw ValidationError.""" + monkeypatch.setenv("KBAGENT_PROJECT_FROM_ENV", "1") + monkeypatch.setenv("KBC_TOKEN", self.TOKEN) + monkeypatch.setenv("KBC_STORAGE_API_URL", "http://connection.keboola.com") + with pytest.raises(ConfigError, match="not a usable stack URL"): + ConfigStore(config_dir=tmp_config_dir).load() + + def test_does_not_override_real_alias( + self, tmp_config_dir: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """A real project already named '__env__' is left untouched.""" + store = ConfigStore(config_dir=tmp_config_dir) + store.save( + AppConfig( + default_project="__env__", + projects={ + "__env__": ProjectConfig( + stack_url="https://other.keboola.com", + token="901-11111-realPersistedTokenXXXXXXXXXXXX", + ) + }, + ) + ) + self._opt_in(monkeypatch) + config = store.load() + assert config.projects["__env__"].token == "901-11111-realPersistedTokenXXXXXXXXXXXX" + assert config.projects["__env__"].ephemeral is False + + def test_ephemeral_never_persisted( + self, tmp_config_dir: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """save() after a load() that injected '__env__' must not write the token.""" + self._opt_in(monkeypatch) + store = ConfigStore(config_dir=tmp_config_dir) + config = store.load() # injects __env__ + config.projects["real"] = ProjectConfig( + stack_url=self.URL, token="901-22222-realTokenForRealProjectXXXXX" + ) + store.save(config) + + raw = (tmp_config_dir / "config.json").read_text() + assert "__env__" not in raw + assert self.TOKEN not in raw + assert "real" in raw + # In-memory object passed by the caller is left intact. + assert "__env__" in config.projects + + def test_mutating_env_project_is_rejected( + self, tmp_config_dir: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """remove/edit/rename/set-branch on __env__ fail clearly, not silently.""" + self._opt_in(monkeypatch) + store = ConfigStore(config_dir=tmp_config_dir) + with pytest.raises(ConfigError, match="synthesized from environment"): + store.remove_project("__env__") + with pytest.raises(ConfigError, match="synthesized from environment"): + store.edit_project("__env__", token="901-77777-otherXXXXXXXXXXXXXXXXXX") + with pytest.raises(ConfigError, match="synthesized from environment"): + store.rename_project("__env__", "renamed") + with pytest.raises(ConfigError, match="synthesized from environment"): + store.set_project_branch("__env__", 123) + + def test_real_persisted_env_alias_still_mutable( + self, tmp_config_dir: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """A real (non-ephemeral) project under the __env__ alias stays editable.""" + store = ConfigStore(config_dir=tmp_config_dir) + store.save( + AppConfig( + projects={ + "__env__": ProjectConfig( + stack_url="https://real.keboola.com", + token="901-11111-realPersistedTokenXXXXXXXXXXXX", + ) + }, + ) + ) + # No opt-in -> the persisted entry is the only one; editing must work. + store.edit_project("__env__", project_name="Renamed") + assert store.get_project("__env__").project_name == "Renamed" + + def test_default_blanked_when_ephemeral_stripped( + self, tmp_config_dir: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """If default_project pointed at the stripped '__env__', it is reset on disk.""" + self._opt_in(monkeypatch) + store = ConfigStore(config_dir=tmp_config_dir) + config = store.load() # default_project == "__env__" + assert config.default_project == "__env__" + store.save(config) + + on_disk = json.loads((tmp_config_dir / "config.json").read_text()) + assert on_disk["default_project"] == "" diff --git a/tests/test_dev_portal_cli.py b/tests/test_dev_portal_cli.py new file mode 100644 index 00000000..ffc2ea44 --- /dev/null +++ b/tests/test_dev_portal_cli.py @@ -0,0 +1,246 @@ +"""Tests for `kbagent dev-portal` command layer via CliRunner.""" + +from __future__ import annotations + +import json +from unittest.mock import patch + +from typer.testing import CliRunner + +from keboola_agent_cli.cli import app + +runner = CliRunner() + + +class TestReadPasswordStdin: + """--password-stdin must work in BOTH TTY mode (hidden getpass prompt, + Enter to confirm) AND pipe mode (read until EOF). The original version + called sys.stdin.read() unconditionally, which hung interactively until + the user sent Ctrl-D.""" + + def test_tty_uses_getpass(self, monkeypatch): + from keboola_agent_cli.commands.dev_portal import _read_password_stdin + + monkeypatch.setattr("sys.stdin.isatty", lambda: True) + monkeypatch.setattr("getpass.getpass", lambda prompt="": "pw-typed\n") + assert _read_password_stdin() == "pw-typed" + + def test_pipe_reads_until_eof(self, monkeypatch): + import io + import sys as _sys + + from keboola_agent_cli.commands.dev_portal import _read_password_stdin + + fake_stdin = io.StringIO("pw-piped\n") + # Use monkeypatch.setattr (not direct attribute assignment) -- ty rejects + # `fake_stdin.isatty = lambda: False` because the slot expects `(self) -> bool` + # and the lambda's signature is `() -> Literal[False]`. monkeypatch handles + # the duck-typed override cleanly without a ty: ignore. + monkeypatch.setattr(fake_stdin, "isatty", lambda: False) + monkeypatch.setattr(_sys, "stdin", fake_stdin) + assert _read_password_stdin() == "pw-piped" + + def test_identity_add_password_stdin_end_to_end(self, tmp_config_dir): + """End-to-end CliRunner test: --password-stdin in pipe mode (the + CliRunner's stdin is not a TTY) must thread the piped password through + Typer's flag parsing into the helper and into the persisted identity. + Catches a regression where the flag and the helper get rewired + independently and the password silently lands as empty.""" + from keboola_agent_cli.config_store import ConfigStore + + with patch( + "keboola_agent_cli.services.dev_portal_service.DeveloperPortalService.add_identity" + ) as add_: + r = runner.invoke( + app, + [ + "--config-dir", + str(tmp_config_dir), + "--json", + "dev-portal", + "identity", + "add", + "--alias", + "piped", + "--username", + "u", + "--password-stdin", + ], + input="my-piped-secret\n", + ) + assert r.exit_code == 0, r.output + add_.assert_called_once() + # The identity object passed to the service must carry the piped password, + # stripped of trailing newline. + call_args = add_.call_args + identity = call_args.args[1] if len(call_args.args) > 1 else call_args.kwargs["identity"] + assert identity.password == "my-piped-secret" + # Sanity: nothing was persisted on disk (add_identity is mocked). + assert ConfigStore(tmp_config_dir, source="cli-flag").load().dev_portal_identities == {} + + +class TestIdentityCommands: + def test_identity_add_and_list_json(self, tmp_config_dir): + with patch( + "keboola_agent_cli.services.dev_portal_service.DeveloperPortalService.add_identity" + ) as add_: + r = runner.invoke( + app, + [ + "--config-dir", + str(tmp_config_dir), + "--json", + "dev-portal", + "identity", + "add", + "--alias", + "alpha", + "--username", + "service.keboola.x", + "--password", + "p", + ], + ) + assert r.exit_code == 0, r.output + add_.assert_called_once() + + def test_identity_use_sets_default(self, tmp_config_dir, config_store): + from keboola_agent_cli.models import DeveloperPortalIdentity + + config_store.add_dev_portal_identity( + "alpha", DeveloperPortalIdentity(username="u", password="p") + ) + config_store.add_dev_portal_identity( + "beta", DeveloperPortalIdentity(username="u", password="p") + ) + r = runner.invoke( + app, + [ + "--config-dir", + str(tmp_config_dir), + "dev-portal", + "identity", + "use", + "beta", + ], + ) + assert r.exit_code == 0, r.output + assert config_store.load().default_dev_portal_identity == "beta" + + +class TestReadCommands: + def test_list_apps_json(self, tmp_config_dir, config_store): + from keboola_agent_cli.models import DeveloperPortalIdentity + + config_store.add_dev_portal_identity( + "alpha", DeveloperPortalIdentity(username="u", password="p", vendor="keboola") + ) + with patch( + "keboola_agent_cli.services.dev_portal_service.DeveloperPortalService.list_apps", + return_value=[{"id": "keboola.ex-a"}], + ): + r = runner.invoke( + app, + [ + "--config-dir", + str(tmp_config_dir), + "--json", + "dev-portal", + "list", + "--vendor", + "keboola", + ], + ) + assert r.exit_code == 0, r.output + data = json.loads(r.stdout) + assert data["data"] == [{"id": "keboola.ex-a"}] + + +class TestWriteCommands: + """Every write must require the random-code confirm. No --yes.""" + + def _seed_identity(self, config_store): + from keboola_agent_cli.models import DeveloperPortalIdentity + + config_store.add_dev_portal_identity( + "alpha", + DeveloperPortalIdentity(username="u", password="p", vendor="keboola"), + ) + + def test_patch_non_tty_exits_6(self, tmp_config_dir, config_store): + """Without a TTY there is NO bypass — exit 6, no portal call.""" + self._seed_identity(config_store) + with patch( + "keboola_agent_cli.services.dev_portal_service.DeveloperPortalService.prepare_patch" + ) as prep: + from keboola_agent_cli.services.dev_portal_service import FieldDiff, PendingPatch + + prep.return_value = PendingPatch( + alias="alpha", + vendor="keboola", + app_id="keboola.ex-a", + payload={"name": "New"}, + current={"name": "Old"}, + diff=[FieldDiff(key="name", current="Old", new="New")], + ) + with patch( + "keboola_agent_cli.services.dev_portal_service.DeveloperPortalService.apply" + ) as apply_: + # CliRunner provides a non-TTY stdin. + r = runner.invoke( + app, + [ + "--config-dir", + str(tmp_config_dir), + "dev-portal", + "patch", + "--app", + "keboola.ex-a", + "--data", + "/tmp/does-not-matter.json", + ], + input="", + ) + assert r.exit_code == 6, r.output + apply_.assert_not_called() + + def test_patch_dry_run_no_confirm(self, tmp_config_dir, config_store, tmp_path): + """--dry-run prints diff and exits 0 without any confirm prompt.""" + self._seed_identity(config_store) + data_file = tmp_path / "patch.json" + data_file.write_text(json.dumps({"name": "New"})) + with patch( + "keboola_agent_cli.services.dev_portal_service.DeveloperPortalService.prepare_patch" + ) as prep: + from keboola_agent_cli.services.dev_portal_service import FieldDiff, PendingPatch + + prep.return_value = PendingPatch( + alias="alpha", + vendor="keboola", + app_id="keboola.ex-a", + payload={"name": "New"}, + current={"name": "Old"}, + diff=[FieldDiff(key="name", current="Old", new="New")], + ) + with patch( + "keboola_agent_cli.services.dev_portal_service.DeveloperPortalService.apply" + ) as apply_: + r = runner.invoke( + app, + [ + "--config-dir", + str(tmp_config_dir), + "--json", + "dev-portal", + "patch", + "--app", + "keboola.ex-a", + "--data", + str(data_file), + "--dry-run", + ], + ) + assert r.exit_code == 0, r.output + apply_.assert_not_called() + # JSON output should advertise the dry-run status + assert "dry-run" in r.stdout diff --git a/tests/test_dev_portal_client.py b/tests/test_dev_portal_client.py new file mode 100644 index 00000000..d47ab880 --- /dev/null +++ b/tests/test_dev_portal_client.py @@ -0,0 +1,311 @@ +"""Tests for DeveloperPortalClient — login, MFA, CRUD against apps-api.""" + +from __future__ import annotations + +import pytest + +from keboola_agent_cli.constants import DP_MFA_CHALLENGE_TYPE +from keboola_agent_cli.dev_portal_client import DeveloperPortalClient +from keboola_agent_cli.errors import ErrorCode, KeboolaApiError +from keboola_agent_cli.models import DeveloperPortalIdentity + + +def _identity(**overrides) -> DeveloperPortalIdentity: + defaults = dict(username="service.keboola.x", password="p") + defaults.update(overrides) + return DeveloperPortalIdentity(**defaults) + + +class TestLoginTokenPath: + def test_login_returns_bearer(self, httpx_mock): + httpx_mock.add_response( + method="POST", + url="https://apps-api.keboola.com/auth/login", + json={"token": "Bearer abc"}, + status_code=200, + ) + with DeveloperPortalClient(_identity()) as client: + client._ensure_authenticated() + assert client._bearer == "Bearer abc" + assert len(httpx_mock.get_requests()) == 1 + + def test_login_bad_credentials_raises(self, httpx_mock): + httpx_mock.add_response( + method="POST", + url="https://apps-api.keboola.com/auth/login", + json={"error": "invalid credentials"}, + status_code=401, + ) + with DeveloperPortalClient(_identity()) as client: + with pytest.raises(KeboolaApiError) as exc: + client._ensure_authenticated() + assert exc.value.error_code == ErrorCode.DP_LOGIN_FAILED + + +class TestLoginMfaPath: + def test_mfa_prompt_completes_login(self, httpx_mock, monkeypatch): + """TOTP authenticator app path: explicit SOFTWARE_TOKEN_MFA challenge. + + The server requires the `challenge` field even though the apiary spec + calls it optional with a SOFTWARE_TOKEN_MFA default -- omitting it + gives a 404 with the misleading "must be one of: ..." enum error + attached to the admin schema. Send it explicitly to avoid that. + """ + httpx_mock.add_response( + method="POST", + url="https://apps-api.keboola.com/auth/login", + json={"session": "sess-1"}, + status_code=200, + match_json={"email": "u@k.com", "password": "p"}, + ) + httpx_mock.add_response( + method="POST", + url="https://apps-api.keboola.com/auth/login", + json={"token": "Bearer xyz"}, + status_code=200, + match_json={ + "email": "u@k.com", + "session": "sess-1", + "code": "123456", + "challenge": DP_MFA_CHALLENGE_TYPE, + }, + ) + monkeypatch.setattr( + "keboola_agent_cli.dev_portal_client._tty_prompt", + lambda label, secret=False: "123456", + ) + ident = DeveloperPortalIdentity(username="u@k.com", password="p") + with DeveloperPortalClient(ident) as client: + client._ensure_authenticated() + assert client._bearer == "Bearer xyz" + + def test_mfa_failure_surfaces_server_body(self, httpx_mock, monkeypatch): + """Single attempt only. Surface the actual server body so the user can + tell whether the code was wrong, the session expired, or something else. + Hint about stale TOTP appears in the message.""" + httpx_mock.add_response( + method="POST", + url="https://apps-api.keboola.com/auth/login", + json={"session": "sess-1"}, + status_code=200, + match_json={"email": "u@k.com", "password": "p"}, + ) + httpx_mock.add_response( + method="POST", + url="https://apps-api.keboola.com/auth/login", + status_code=400, + text='{"errorMessage":"Invalid code","errorCode":400}', + match_json={ + "email": "u@k.com", + "session": "sess-1", + "code": "999999", + "challenge": DP_MFA_CHALLENGE_TYPE, + }, + ) + monkeypatch.setattr( + "keboola_agent_cli.dev_portal_client._tty_prompt", + lambda label, secret=False: "999999", + ) + ident = DeveloperPortalIdentity(username="u@k.com", password="p") + with DeveloperPortalClient(ident) as client: + with pytest.raises(KeboolaApiError) as exc: + client._ensure_authenticated() + assert exc.value.error_code == ErrorCode.DP_LOGIN_FAILED + assert "Invalid code" in str(exc.value) + assert "TOTP" in str(exc.value) or "stale" in str(exc.value) + + def test_mfa_no_tty_raises_mfa_required(self, httpx_mock, monkeypatch): + httpx_mock.add_response( + method="POST", + url="https://apps-api.keboola.com/auth/login", + json={"session": "sess-1"}, + status_code=200, + ) + # _tty_prompt returns None when no terminal is available. + monkeypatch.setattr( + "keboola_agent_cli.dev_portal_client._tty_prompt", + lambda label, secret=False: None, + ) + ident = DeveloperPortalIdentity(username="u@k.com", password="p") + with DeveloperPortalClient(ident) as client: + with pytest.raises(KeboolaApiError) as exc: + client._ensure_authenticated() + assert exc.value.error_code == ErrorCode.DP_MFA_REQUIRED + + +class TestPortalReads: + def test_list_apps(self, httpx_mock): + httpx_mock.add_response( + method="POST", + url="https://apps-api.keboola.com/auth/login", + json={"token": "Bearer abc"}, + ) + httpx_mock.add_response( + method="GET", + url="https://apps-api.keboola.com/vendors/keboola/apps?limit=1000", + json={"apps": [{"id": "keboola.ex-foo"}]}, + ) + with DeveloperPortalClient(_identity()) as client: + apps = client.list_apps("keboola") + assert apps == [{"id": "keboola.ex-foo"}] + + def test_get_app_404(self, httpx_mock): + httpx_mock.add_response( + method="POST", + url="https://apps-api.keboola.com/auth/login", + json={"token": "Bearer abc"}, + ) + httpx_mock.add_response( + method="GET", + url="https://apps-api.keboola.com/vendors/keboola/apps/keboola.missing", + status_code=404, + json={"error": "not found"}, + ) + with DeveloperPortalClient(_identity()) as client: + with pytest.raises(KeboolaApiError) as exc: + client.get_app("keboola", "keboola.missing") + assert exc.value.error_code == ErrorCode.DP_APP_NOT_FOUND + + +class TestPortalWrites: + def test_create_app(self, httpx_mock): + httpx_mock.add_response( + method="POST", + url="https://apps-api.keboola.com/auth/login", + json={"token": "Bearer abc"}, + ) + httpx_mock.add_response( + method="POST", + url="https://apps-api.keboola.com/vendors/keboola/apps", + json={"id": "ex-foo", "name": "Foo"}, + ) + with DeveloperPortalClient(_identity()) as client: + resp = client.create_app( + "keboola", {"id": "ex-foo", "name": "Foo", "type": "extractor"} + ) + assert resp["id"] == "ex-foo" + + def test_patch_app_vendor_role_hits_vendor_endpoint(self, httpx_mock): + httpx_mock.add_response( + method="POST", + url="https://apps-api.keboola.com/auth/login", + json={"token": "Bearer abc"}, + ) + httpx_mock.add_response( + method="PATCH", + url="https://apps-api.keboola.com/vendors/keboola/apps/keboola.ex-foo", + json={"id": "ex-foo", "name": "Foo 2"}, + ) + with DeveloperPortalClient(_identity()) as client: + resp = client.patch_app("keboola", "keboola.ex-foo", {"name": "Foo 2"}) + assert resp["name"] == "Foo 2" + + def test_patch_app_admin_role_hits_admin_endpoint(self, httpx_mock): + """An admin-role identity must route PATCH to /admin/apps/{app} so the + permissive schema accepts admin-only fields like complexity. httpx_mock + has NO entry for /vendors/.../apps/... -- if the client wrongly routes + there, the test fails with an unmocked-request error.""" + httpx_mock.add_response( + method="POST", + url="https://apps-api.keboola.com/auth/login", + json={"token": "Bearer admin-bearer"}, + ) + httpx_mock.add_response( + method="PATCH", + url="https://apps-api.keboola.com/admin/apps/keboola.ex-foo", + json={"id": "ex-foo", "complexity": "easy"}, + ) + with DeveloperPortalClient(_identity(role_hint="admin")) as client: + resp = client.patch_app("keboola", "keboola.ex-foo", {"complexity": "easy"}) + assert resp["complexity"] == "easy" + + def test_publish_app(self, httpx_mock): + httpx_mock.add_response( + method="POST", + url="https://apps-api.keboola.com/auth/login", + json={"token": "Bearer abc"}, + ) + httpx_mock.add_response( + method="POST", + url="https://apps-api.keboola.com/vendors/keboola/apps/keboola.ex-foo/publish", + json={"status": "submitted"}, + ) + with DeveloperPortalClient(_identity()) as client: + assert client.publish_app("keboola", "keboola.ex-foo")["status"] == "submitted" + + def test_deprecate_app(self, httpx_mock): + httpx_mock.add_response( + method="POST", + url="https://apps-api.keboola.com/auth/login", + json={"token": "Bearer abc"}, + ) + httpx_mock.add_response( + method="POST", + url="https://apps-api.keboola.com/vendors/keboola/apps/keboola.ex-foo/deprecate", + json={"status": "deprecated"}, + ) + with DeveloperPortalClient(_identity()) as client: + assert client.deprecate_app("keboola", "keboola.ex-foo")["status"] == "deprecated" + + +class TestIconUpload: + def test_upload_icon_two_hop(self, httpx_mock, monkeypatch): + httpx_mock.add_response( + method="POST", + url="https://apps-api.keboola.com/auth/login", + json={"token": "Bearer abc"}, + ) + httpx_mock.add_response( + method="POST", + url="https://apps-api.keboola.com/vendors/keboola/apps/keboola.ex-foo/icon", + json={"link": "https://s3.example/presigned"}, + ) + # The S3 PUT bypasses httpx; we mock urllib.request.urlopen. + seen = {} + + class _FakeResp: + status = 200 + + def __enter__(self): + return self + + def __exit__(self, *a): + pass + + def fake_urlopen(req): + seen["url"] = req.full_url + seen["data"] = req.data + seen["method"] = req.method + return _FakeResp() + + monkeypatch.setattr("urllib.request.urlopen", fake_urlopen) + + with DeveloperPortalClient(_identity()) as client: + client.upload_icon("keboola", "keboola.ex-foo", b"\x89PNG\r\n\x1a\nrest") + assert seen["url"] == "https://s3.example/presigned" + assert seen["data"] == b"\x89PNG\r\n\x1a\nrest" + assert seen["method"] == "PUT" + + def test_upload_icon_presign_failure(self, httpx_mock, monkeypatch): + httpx_mock.add_response( + method="POST", + url="https://apps-api.keboola.com/auth/login", + json={"token": "Bearer abc"}, + ) + # Add 3 responses (MAX_RETRIES=3) since 500 is retryable. + for _ in range(3): + httpx_mock.add_response( + method="POST", + url="https://apps-api.keboola.com/vendors/keboola/apps/keboola.ex-foo/icon", + status_code=500, + json={"error": "boom"}, + ) + # Suppress retry sleeps. + import keboola_agent_cli.http_base as http_base_module + + monkeypatch.setattr(http_base_module.time, "sleep", lambda _: None) + + with DeveloperPortalClient(_identity()) as client: + with pytest.raises(KeboolaApiError) as exc: + client.upload_icon("keboola", "keboola.ex-foo", b"data") + assert exc.value.error_code == ErrorCode.DP_ICON_UPLOAD_FAILED diff --git a/tests/test_dev_portal_service.py b/tests/test_dev_portal_service.py new file mode 100644 index 00000000..f1e87e77 --- /dev/null +++ b/tests/test_dev_portal_service.py @@ -0,0 +1,314 @@ +"""Tests for DeveloperPortalService — identity CRUD, prepare/apply, diff, validation.""" + +from __future__ import annotations + +from typing import ClassVar +from unittest.mock import MagicMock + +import pytest + +from keboola_agent_cli.errors import ErrorCode, KeboolaApiError +from keboola_agent_cli.models import DeveloperPortalIdentity +from keboola_agent_cli.services.dev_portal_service import DeveloperPortalService + + +@pytest.fixture +def fake_client(): + mock = MagicMock() + # Make the mock a self-returning context manager so that + # `with factory(ident) as client:` binds `client` to the same object + # we configure side_effects on, not to a child MagicMock. + mock.__enter__ = MagicMock(return_value=mock) + mock.__exit__ = MagicMock(return_value=False) + return mock + + +@pytest.fixture +def service(config_store, fake_client): + def factory(identity): + return fake_client + + return DeveloperPortalService(config_store=config_store, client_factory=factory) + + +class TestIdentityCrud: + def test_add_and_list(self, service, fake_client): + # add_identity also runs verify (login probe). + fake_client.list_apps.return_value = [] + ident = DeveloperPortalIdentity(username="u", password="p") + service.add_identity("alpha", ident) + result = service.list_identities() + assert "alpha" in result + assert result["alpha"].username == "u" + + def test_add_verify_failure_does_not_persist(self, service, fake_client, config_store): + fake_client._ensure_authenticated.side_effect = KeboolaApiError( + message="bad creds", + error_code=ErrorCode.DP_LOGIN_FAILED, + ) + ident = DeveloperPortalIdentity(username="u", password="bad") + with pytest.raises(KeboolaApiError) as exc: + service.add_identity("alpha", ident) + assert exc.value.error_code == ErrorCode.DP_LOGIN_FAILED + assert config_store.load().dev_portal_identities == {} + + def test_use_sets_default(self, service, fake_client, config_store): + fake_client._ensure_authenticated.return_value = None + ident = DeveloperPortalIdentity(username="u", password="p") + service.add_identity("alpha", ident) + service.add_identity("beta", ident) + service.use_identity("beta") + assert config_store.load().default_dev_portal_identity == "beta" + + def test_remove(self, service, fake_client, config_store): + fake_client._ensure_authenticated.return_value = None + ident = DeveloperPortalIdentity(username="u", password="p") + service.add_identity("alpha", ident) + service.remove_identity("alpha") + assert "alpha" not in config_store.load().dev_portal_identities + + +class TestReadsAndPrepareApply: + def _setup(self, service, fake_client): + fake_client._ensure_authenticated.return_value = None + ident = DeveloperPortalIdentity(username="u", password="p") + service.add_identity("alpha", ident) + + def test_list_apps(self, service, fake_client): + self._setup(service, fake_client) + fake_client.list_apps.return_value = [{"id": "ex-a"}] + assert service.list_apps("alpha", "keboola") == [{"id": "ex-a"}] + fake_client.list_apps.assert_called_with("keboola") + + def test_get_app(self, service, fake_client): + self._setup(service, fake_client) + fake_client.get_app.return_value = {"id": "ex-a", "name": "Hello"} + assert service.get_app("alpha", "keboola", "keboola.ex-a")["name"] == "Hello" + + def test_prepare_create_requires_id_name_type(self, service, fake_client): + self._setup(service, fake_client) + with pytest.raises(KeboolaApiError, match="payload must include 'id'"): + service.prepare_create("alpha", "keboola", {"name": "F", "type": "extractor"}) + + def test_prepare_create_rejects_banned_words_in_name(self, service, fake_client): + self._setup(service, fake_client) + with pytest.raises(KeboolaApiError, match="must not contain"): + service.prepare_create( + "alpha", + "keboola", + {"id": "x", "name": "Foo extractor", "type": "extractor"}, + ) + + def test_prepare_patch_diff(self, service, fake_client): + self._setup(service, fake_client) + fake_client.get_app.return_value = { + "id": "ex-a", + "name": "Old", + "shortDescription": "same", + } + pending = service.prepare_patch( + "alpha", + "keboola", + "keboola.ex-a", + {"name": "New", "shortDescription": "same"}, + ) + keys = {d.key for d in pending.diff} + assert keys == {"name"} # shortDescription unchanged is filtered out + assert pending.diff[0].current == "Old" + assert pending.diff[0].new == "New" + + def test_apply_patch_calls_client(self, service, fake_client): + self._setup(service, fake_client) + fake_client.get_app.return_value = {"id": "ex-a", "name": "Old"} + fake_client.patch_app.return_value = {"id": "ex-a", "name": "New"} + pending = service.prepare_patch("alpha", "keboola", "keboola.ex-a", {"name": "New"}) + result = service.apply(pending) + assert result["name"] == "New" + fake_client.patch_app.assert_called_with("keboola", "keboola.ex-a", {"name": "New"}) + + def test_prepare_patch_vendor_role_rejects_admin_only_fields( + self, service, fake_client, config_store + ): + """Vendor-role identity + admin-only field => fail-fast with switch-to-admin guidance. + + No portal call happens (we fail before get_app). The error names every + offending field and tells the user how to switch identity. + """ + from keboola_agent_cli.models import DeveloperPortalIdentity + + fake_client._ensure_authenticated.return_value = None + ident = DeveloperPortalIdentity(username="u", password="p", role_hint="vendor") + service.add_identity("vendor-alpha", ident) + with pytest.raises(KeboolaApiError) as exc: + service.prepare_patch( + "vendor-alpha", + "keboola", + "keboola.ex-a", + {"name": "New", "complexity": "easy", "categories": ["x"]}, + ) + assert exc.value.error_code == ErrorCode.VALIDATION_ERROR + assert "complexity" in str(exc.value) + assert "categories" in str(exc.value) + assert "admin" in str(exc.value).lower() + fake_client.get_app.assert_not_called() + + def test_prepare_patch_admin_role_allows_admin_only_fields( + self, service, fake_client, config_store + ): + """Admin-role identity bypasses the preflight; the client routes the + actual PATCH to /admin/apps (verified in client tests). Here we just + check the service's preflight gate is permissive for admin role.""" + from keboola_agent_cli.models import DeveloperPortalIdentity + + fake_client._ensure_authenticated.return_value = None + ident = DeveloperPortalIdentity(username="a", password="p", role_hint="admin") + service.add_identity("admin-bob", ident) + fake_client.get_app.return_value = {"id": "ex-a", "complexity": None} + pending = service.prepare_patch( + "admin-bob", "keboola", "keboola.ex-a", {"complexity": "easy"} + ) + keys = {d.key for d in pending.diff} + assert keys == {"complexity"} + + def test_prepare_publish_missing_fields(self, service, fake_client): + self._setup(service, fake_client) + fake_client.get_app.return_value = { + "id": "ex-a", + "name": "Foo", + "type": "extractor", + # missing icon, repository, descriptions, license, docs + } + with pytest.raises(KeboolaApiError) as exc: + service.prepare_publish("alpha", "keboola", "keboola.ex-a") + assert exc.value.error_code == ErrorCode.DP_PUBLISH_REQUIREMENTS_MISSING + assert "icon" in str(exc.value) + + +class _CountingClient: + """Fake portal client that counts how many times it actually logs in. + + Mirrors the real client's contract: methods call `_ensure_authenticated`, + which only logs in when no bearer is present. `seed_bearer` injects a + bearer obtained by an earlier client for the same identity. + """ + + instances: ClassVar[list[_CountingClient]] = [] + + def __init__(self, identity): + self._bearer = None + self.login_count = 0 + _CountingClient.instances.append(self) + + @property + def bearer(self): + return self._bearer + + def seed_bearer(self, bearer): + self._bearer = bearer + + def _ensure_authenticated(self): + if self._bearer is None: + self.login_count += 1 + self._bearer = "tok-123" + + def get_app(self, vendor, app_id): + self._ensure_authenticated() + return {"id": "ex-a", "name": "Old"} + + def patch_app(self, vendor, app_id, payload): + self._ensure_authenticated() + return {"id": "ex-a", **payload} + + def __enter__(self): + return self + + def __exit__(self, *exc): + return False + + +class TestSingleLoginAcrossPrepareApply: + """Regression: patch must authenticate once, not twice (no double MFA prompt).""" + + def test_patch_reuses_bearer_no_second_login(self, config_store): + _CountingClient.instances = [] + ident = DeveloperPortalIdentity(username="u", password="p") + config_store.add_dev_portal_identity("alpha", ident) + + def factory(identity): + return _CountingClient(identity) + + svc = DeveloperPortalService(config_store=config_store, client_factory=factory) + pending = svc.prepare_patch("alpha", "keboola", "keboola.ex-a", {"name": "New"}) + svc.apply(pending) + + # A fresh client is built for prepare and again for apply (matches the + # real prepare/apply split), but only the first one logs in. + assert len(_CountingClient.instances) == 2 + assert _CountingClient.instances[0].login_count == 1 + assert _CountingClient.instances[1].login_count == 0 + + +class _ExpiringClient: + """Fake client where a seeded (stale) bearer is rejected by the portal.""" + + instances: ClassVar[list[_ExpiringClient]] = [] + + def __init__(self, identity): + self._bearer = None + self.login_count = 0 + _ExpiringClient.instances.append(self) + + @property + def bearer(self): + return self._bearer + + def seed_bearer(self, bearer): + self._bearer = bearer + + def _ensure_authenticated(self): + if self._bearer is None: + self.login_count += 1 + self._bearer = "fresh" + + def list_apps(self, vendor): + if self._bearer == "stale": + raise KeboolaApiError( + message="Invalid or expired token", + status_code=401, + error_code=ErrorCode.INVALID_TOKEN, + ) + self._ensure_authenticated() + return [{"id": "ex-a"}] + + def __enter__(self): + return self + + def __exit__(self, *exc): + return False + + +class TestStaleBearerEviction: + """Regression: a stale cached bearer must not permanently lock out serve.""" + + def test_auth_error_evicts_cached_bearer_then_recovers(self, config_store): + _ExpiringClient.instances = [] + config_store.add_dev_portal_identity( + "alpha", DeveloperPortalIdentity(username="u", password="p") + ) + + def factory(identity): + return _ExpiringClient(identity) + + svc = DeveloperPortalService(config_store=config_store, client_factory=factory) + # Simulate a bearer cached by an earlier request that has since expired. + svc._bearers["alpha"] = "stale" + + # First call replays the stale bearer -> portal 401 -> propagates AND evicts. + with pytest.raises(KeboolaApiError) as exc: + svc.list_apps("alpha", "keboola") + assert exc.value.error_code == ErrorCode.INVALID_TOKEN + assert "alpha" not in svc._bearers, "stale bearer must be evicted on 401" + + # Next call re-authenticates cleanly (no dead token to seed). + assert svc.list_apps("alpha", "keboola") == [{"id": "ex-a"}] + assert svc._bearers["alpha"] == "fresh" diff --git a/tests/test_e2e.py b/tests/test_e2e.py index 9cc96ef1..fca70ba1 100644 --- a/tests/test_e2e.py +++ b/tests/test_e2e.py @@ -58,6 +58,7 @@ from unittest.mock import patch import pytest +import yaml from typer.testing import CliRunner from helpers import metastore_scope_available @@ -3963,6 +3964,115 @@ def test_sync_workflow(self) -> None: ) assert data["status"] == "ok" + def test_sync_force_pull_conflict_aware(self) -> None: + """`sync pull --force` is conflict-aware (0.53.0+), end-to-end. + + Locks both halves of the baseline-corruption fix against real Storage: + + * (b) local edited, remote UNCHANGED -> force-pull PRESERVES the edit + (does not silently re-stamp the baseline); ``sync diff`` afterward + still reports the config as modified. + * (a) local edited AND remote also changed -> force-pull ABORTS with + exit 1 and error code ``SYNC_CONFLICT``. + + Creates + cleans up a dedicated config so the test is idempotent. + """ + import yaml as _yaml + + from keboola_agent_cli.client import KeboolaClient + from keboola_agent_cli.constants import CONFIG_FILENAME + + cfg: dict = {} + try: + with KeboolaClient(stack_url=self.url, token=self.token) as api: + cfg = api.create_config( + component_id=TEST_COMPONENT_ID, + name=f"{RUN_ID}-forcepull", + description="E2E force-pull conflict fixture", + configuration={"parameters": {"db": {"host": "orig.example.com"}}}, + ) + cfg_id = str(cfg["id"]) + + # --- init + pull, locate the config's _config.yml --- + _step("7a", "sync init + pull (force-pull fixture)") + self._run_ok( + "sync", "init", "--project", self.alias, "--directory", str(self.project_dir) + ) + self._run_ok( + "sync", "pull", "--project", self.alias, "--directory", str(self.project_dir) + ) + matches = [ + p + for p in self.project_dir.rglob(CONFIG_FILENAME) + if "rows" not in p.relative_to(self.project_dir).parts + and str( + _yaml.safe_load(p.read_text(encoding="utf-8")) + .get("_keboola", {}) + .get("config_id") + ) + == cfg_id + ] + assert len(matches) == 1, f"config YAML not found after pull: {matches}" + config_file = matches[0] + + # --- edit locally --- + local = _yaml.safe_load(config_file.read_text(encoding="utf-8")) + local.setdefault("parameters", {})["_e2e_marker"] = "x" + config_file.write_text(_yaml.dump(local, default_flow_style=False), encoding="utf-8") + + # --- (b) force-pull, remote UNCHANGED -> edit preserved --- + _step("7b", "force-pull preserves edit when remote unchanged") + self._run_ok( + "sync", + "pull", + "--project", + self.alias, + "--directory", + str(self.project_dir), + "--force", + ) + diff_after = self._run_ok( + "sync", "diff", "--project", self.alias, "--directory", str(self.project_dir) + ) + modified = [c for c in diff_after["data"]["changes"] if c["change_type"] == "modified"] + assert any(c["config_id"] == cfg_id for c in modified), ( + f"force-pull stranded the un-pushed edit: {diff_after['data']['summary']}" + ) + + # --- (a) mutate remote, force-pull -> SYNC_CONFLICT abort --- + _step("7c", "force-pull aborts on a true conflict") + with KeboolaClient(stack_url=self.url, token=self.token) as api: + api.update_config( + component_id=TEST_COMPONENT_ID, + config_id=cfg_id, + configuration={"parameters": {"db": {"host": "remote-moved.example.com"}}}, + change_description="e2e force-pull conflict", + ) + conflict_result = self._run( + "sync", + "pull", + "--project", + self.alias, + "--directory", + str(self.project_dir), + "--force", + ) + assert conflict_result.exit_code == 1, ( + f"expected exit 1 on conflict, got {conflict_result.exit_code}" + ) + envelope = json.loads(conflict_result.output) + assert envelope["status"] == "error" + assert envelope["error"]["code"] == "SYNC_CONFLICT" + assert any(c["config_id"] == cfg_id for c in envelope["error"]["details"]["conflicts"]) + finally: + cfg_id = cfg.get("id") if cfg else None + if cfg_id: + try: + with KeboolaClient(stack_url=self.url, token=self.token) as api: + api.delete_config(component_id=TEST_COMPONENT_ID, config_id=cfg_id) + except Exception as exc: + print(f" [cleanup] Failed to delete {TEST_COMPONENT_ID}/{cfg_id}: {exc}") + def test_sync_push_variable_row_round_trip(self) -> None: """PR1 P0-1 acceptance: edit a keboola.variables values row, push, pull back. @@ -6188,6 +6298,23 @@ def _run(self, *args: str) -> Any: def _run_ok(self, *args: str) -> dict[str, Any]: return _json_ok(self._run(*args)) + def test_snowflake_workspace_create_returns_private_key(self) -> None: + """Snowflake workspace creation returns the generated private key once.""" + _step(1, "workspace create returns private_key on Snowflake") + result = self._run("workspace", "create", "--project", self.alias) + if result.exit_code != 0: + pytest.skip(f"workspace create not supported: {result.output}") + + data = _json_ok(result)["data"] + ws_id = int(data["workspace_id"]) + self._created_workspace_ids.append(ws_id) + + if data.get("backend") != "snowflake": + pytest.skip("Snowflake private_key assertion requires a Snowflake stack") + + assert "private_key" in data + assert data["private_key"].startswith("-----BEGIN PRIVATE KEY-----") + def test_issue_304_discoverability_roundtrip(self) -> None: """list/detail expose loginType; sandbox config annotation resolves real workspace ID.""" _step(1, "workspace create (RO sandbox)") @@ -7269,6 +7396,167 @@ def test_swap_without_branch_is_rejected(self) -> None: assert "dev branch" in payload["error"]["message"] +# --------------------------------------------------------------------------- +# TestE2EStorageCloneTable -- storage clone-table (pull) into a dev branch +# --------------------------------------------------------------------------- + + +@skip_without_credentials +@pytest.mark.e2e +class TestE2EStorageCloneTable: + """End-to-end coverage for ``kbagent storage clone-table``. + + Verifies: + - a production table can be pulled (cloned) into a dev branch and is + then visible/materialized in that branch, + - dry-run skips the HTTP call, + - calls without a branch are rejected before any HTTP traffic. + + On storage-branches projects this materializes the prod table into the + branch (the prerequisite for in-branch swap / column drops). On + legacy-branch projects the pull still succeeds; the assertion only checks + the table is visible in the branch afterwards, which holds for both. + """ + + @pytest.fixture(autouse=True) + def setup(self, tmp_path: Path) -> Generator[None, None, None]: + self.token = os.environ[ENV_TOKEN] + raw_url = os.environ.get(ENV_URL, "connection.keboola.com") + self.url = raw_url if raw_url.startswith("https://") else f"https://{raw_url}" + self.alias = f"{RUN_ID}-clone" + self.config_dir = tmp_path / "config" + self.config_dir.mkdir() + self.client = KeboolaClient(stack_url=self.url, token=self.token) + + self._created_branch_ids: list[int] = [] + self._created_buckets: list[str] = [] + + result = _invoke( + self.config_dir, + [ + "--json", + "project", + "add", + "--project", + self.alias, + "--url", + self.url, + "--token", + self.token, + ], + ) + assert result.exit_code == 0, f"project add failed: {result.output}" + + yield + + # Teardown: branches first (cascades to their materialized tables), + # then the production bucket we created outside of a branch. + for branch_id in self._created_branch_ids: + with contextlib.suppress(Exception): + self.client.delete_dev_branch(branch_id) + for bucket_id in self._created_buckets: + with contextlib.suppress(Exception): + self.client.delete_bucket(bucket_id, force=True) + self.client.close() + + def _run_ok(self, *args: str) -> dict[str, Any]: + return _json_ok(_invoke(self.config_dir, ["--json", *args])) + + def test_clone_prod_table_into_dev_branch(self) -> None: + """Live pull: a production table becomes available in the dev branch.""" + bucket_id = f"in.c-{RUN_ID.replace('-', '_')}_clone" + table_id = f"{bucket_id}.source" + + _step(1, "create a production table (default branch)") + self._run_ok( + "storage", + "create-table", + "--project", + self.alias, + "--bucket-id", + bucket_id, + "--name", + "source", + "--column", + "id:VARCHAR(40)", + "--column", + "value:VARCHAR(20)", + "--primary-key", + "id", + ) + self._created_buckets.append(bucket_id) + + _step(2, "branch create", "target dev branch for the pull") + branch = self._run_ok( + "branch", "create", "--project", self.alias, "--name", f"{RUN_ID}-clone-branch" + )["data"] + branch_id = int(branch["branch_id"]) + self._created_branch_ids.append(branch_id) + + _step(3, "storage clone-table", "POST /tables/.../pull (default -> branch)") + result = self._run_ok( + "storage", + "clone-table", + "--project", + self.alias, + "--table-id", + table_id, + "--branch", + str(branch_id), + )["data"] + assert result["table_id"] == table_id + assert result["branch_id"] == branch_id + assert result["dry_run"] is False + assert result["response"]["status"] == "success" + + _step(4, "table-detail in branch", "table is materialized/visible after pull") + detail = self.client.get_table_detail(table_id, branch_id=branch_id) + col_names = {c["name"] for c in detail["definition"]["columns"]} + assert col_names == {"id", "value"} + + def test_clone_dry_run_does_not_call_api(self) -> None: + """Dry-run skips the HTTP call: exit 0, no response key.""" + _step(1, "branch create", "dry-run still requires a branch context") + branch = self._run_ok( + "branch", "create", "--project", self.alias, "--name", f"{RUN_ID}-clone-dry" + )["data"] + branch_id = int(branch["branch_id"]) + self._created_branch_ids.append(branch_id) + + result = self._run_ok( + "storage", + "clone-table", + "--project", + self.alias, + "--table-id", + "in.c-foo.bar", + "--branch", + str(branch_id), + "--dry-run", + )["data"] + assert result["dry_run"] is True + assert "response" not in result + + def test_clone_without_branch_is_rejected(self) -> None: + """Without active branch and without --branch, exit 5 before any HTTP.""" + result = _invoke( + self.config_dir, + [ + "--json", + "storage", + "clone-table", + "--project", + self.alias, + "--table-id", + "in.c-foo.bar", + ], + ) + assert result.exit_code == 5, result.output + payload = json.loads(result.output) + assert payload["status"] == "error" + assert "dev branch" in payload["error"]["message"] + + # --------------------------------------------------------------------------- # TestE2EDataAppLifecycle -- data-app create / detail / deploy / start / stop / delete # --------------------------------------------------------------------------- @@ -7323,6 +7611,20 @@ def test_swap_without_branch_is_rejected(self) -> None: ), ) +# Feature-flag E2E gate. Requires a SUPER-ADMIN manage token (the same kind +# `org setup` uses). Opt-in via `make test-e2e-feature` -- default-skipped in +# `make test-e2e` because the regular Storage API credentials cannot list or +# read feature flags. +skip_without_feature_credentials = pytest.mark.skipif( + not ( + os.environ.get(ENV_MANAGE_TOKEN) and os.environ.get(ENV_URL) and os.environ.get(ENV_TOKEN) + ), + reason=( + f"Requires {ENV_MANAGE_TOKEN} (super-admin), {ENV_URL}, and {ENV_TOKEN}. " + "Run via `make test-e2e-feature`." + ), +) + @pytest.mark.e2e class TestE2EDataAppLifecycle: @@ -8371,6 +8673,154 @@ def _run(*args: str) -> dict: ) +@skip_without_feature_credentials +@pytest.mark.e2e +def test_feature_flags_read_e2e(tmp_path: Path) -> None: + """Read-only feature-flag check against a real stack (since v0.48.0). + + Verifies the wiring end-to-end with a super-admin manage token: + 1. the stack catalogue (`feature list`) returns a non-empty feature set; + 2. a project's assigned features (`feature project-show`) are readable. + + Deliberately read-only -- it never enables or disables a flag, so it is + safe to run against a live project. The manage token is supplied via env + + the top-level --allow-env-manage-token opt-in (default-deny otherwise). + """ + stack_url = ( + os.environ[ENV_URL] + if os.environ[ENV_URL].startswith("https://") + else f"https://{os.environ[ENV_URL]}" + ) + config_dir = tmp_path / "kbagent-config" + config_dir.mkdir() + alias = "e2e-feature-target" + + env = { + **os.environ, + "KBC_MANAGE_API_TOKEN": os.environ[ENV_MANAGE_TOKEN], + } + + def _run(*args: str) -> Any: + return runner.invoke( + app, + ["--config-dir", str(config_dir), "--allow-env-manage-token", "--json", *args], + env=env, + ) + + # Register the project via a real Storage API token so project_id is + # populated from the token-verify response (feature project-show needs it). + add = _run( + "project", + "add", + "--project", + alias, + "--url", + stack_url, + "--token", + os.environ[ENV_TOKEN], + ) + assert add.exit_code == 0, add.output + + # 1. Stack catalogue -- the super-admin token must see the full feature set. + catalogue = _run("feature", "list", "--project", alias) + assert catalogue.exit_code == 0, catalogue.output + cat_data = json.loads(catalogue.output)["data"] + assert isinstance(cat_data["features"], list) + assert len(cat_data["features"]) > 0, "stack catalogue unexpectedly empty" + # Every catalogue entry carries a stable 'name' identifier. + assert all("name" in feat for feat in cat_data["features"]), cat_data["features"][:3] + + # 2. Project-assigned features -- readable, possibly empty, always a list. + show = _run("feature", "project-show", "--project", alias) + assert show.exit_code == 0, show.output + show_data = json.loads(show.output)["data"] + assert isinstance(show_data["features"], list) + assert show_data["project_id"] is not None + + +@skip_without_credentials +@pytest.mark.e2e +def test_stream_otlp_e2e(tmp_path: Path) -> None: + """Full Data Streams OTLP round-trip against a real project (since v0.50.0). + + Creates a temporary OTLP source, reads its assembled detail (masked by + default + revealed on demand), then deletes it and confirms it is gone. + Self-cleaning: the source it creates is removed before the test returns. + """ + stack_url = ( + os.environ[ENV_URL] + if os.environ[ENV_URL].startswith("https://") + else f"https://{os.environ[ENV_URL]}" + ) + config_dir = tmp_path / "kbagent-config" + config_dir.mkdir() + alias = "e2e-stream-target" + source_name = "kbagent-e2e-stream" + + def _run(*args: str) -> Any: + return runner.invoke( + app, ["--config-dir", str(config_dir), "--json", *args], env={**os.environ} + ) + + add = _run( + "project", "add", "--project", alias, "--url", stack_url, "--token", os.environ[ENV_TOKEN] + ) + assert add.exit_code == 0, add.output + + try: + # 1. List is readable (possibly empty) and well-formed. + listed = _run("stream", "list", "--project", alias) + assert listed.exit_code == 0, listed.output + assert isinstance(json.loads(listed.output)["data"]["sources"], list) + + # 2. Create an OTLP source (idempotent on reruns). + created = _run( + "stream", + "create-source", + "--project", + alias, + "--name", + source_name, + "--type", + "otlp", + "--if-not-exists", + ) + assert created.exit_code == 0, created.output + cdata = json.loads(created.output)["data"] + assert cdata["status"] in ("created", "skipped") + assert cdata["type"] == "otlp" + source_id = cdata["source_id"] + # The three OTLP sinks are auto-provisioned, so the destination tables + # (logs/metrics/traces) are present immediately after create. + assert set(cdata["destination"]["tables"]) == {"logs", "metrics", "traces"} + + # 3. Detail masks the secret by default. + masked = _run("stream", "detail", source_id, "--project", alias) + assert masked.exit_code == 0, masked.output + mdata = json.loads(masked.output)["data"] + assert mdata["secret_revealed"] is False + assert "/***" in mdata["endpoint"] + assert mdata["protocol"] == "http/protobuf" + assert set(mdata["signal_endpoints"]) == {"logs", "traces", "metrics"} + assert set(mdata["destination"]["tables"]) == {"logs", "metrics", "traces"} + + # 4. --reveal exposes the real endpoint (no mask marker). + revealed = _run("stream", "detail", source_id, "--project", alias, "--reveal") + assert revealed.exit_code == 0, revealed.output + rdata = json.loads(revealed.output)["data"] + assert rdata["secret_revealed"] is True + assert "/***" not in rdata["endpoint"] + finally: + # 5. Clean up -- delete the source and confirm it is gone. + deleted = _run("stream", "delete", source_name, "--project", alias, "--yes") + assert deleted.exit_code == 0, deleted.output + assert json.loads(deleted.output)["data"]["status"] == "deleted" + + final = _run("stream", "list", "--project", alias) + remaining = {s["source_id"] for s in json.loads(final.output)["data"]["sources"]} + assert source_name not in remaining + + # --------------------------------------------------------------------------- # MCP-parity commands (since v0.30.0) # --------------------------------------------------------------------------- @@ -9596,3 +10046,1054 @@ def _direct_delete(item_type: str, item_id: str) -> None: ) except _ApiError as exc: print(f" WARN: residue scan failed: {exc}") + + +# --------------------------------------------------------------------------- +# v0.47.0 -- fresh-CREATE writeback + new ergonomic flags (E2E coverage per +# CLAUDE.md convention #16: "Every new CLI command MUST have a corresponding +# E2E test in tests/test_e2e.py"). +# --------------------------------------------------------------------------- + + +@skip_without_credentials +@pytest.mark.e2e +class TestE2E_0_47_0_NewSurfaces: + """E2E coverage for v0.47.0 additions. + + - ``storage create-table --if-not-exists`` -- idempotent re-create + - ``semantic-layer search-context`` + ``get-context`` -- project-wide read + - ``sync diff --branch `` -- per-invocation dev-branch override + + All three touch a real Keboola project via the configured E2E token. The + test creates a throwaway dev branch where needed and deletes it in the + teardown so residue does not accumulate across re-runs. + """ + + @pytest.fixture(autouse=True) + def setup(self, tmp_path: Path): + self.token = os.environ[ENV_TOKEN] + raw_url = os.environ.get(ENV_URL, "connection.keboola.com") + self.url = raw_url if raw_url.startswith("https://") else f"https://{raw_url}" + self.alias = f"{RUN_ID}-v0470" + self.config_dir = tmp_path / "config" + self.config_dir.mkdir() + self.tmp_path = tmp_path + + result = _invoke( + self.config_dir, + [ + "--json", + "project", + "add", + "--project", + self.alias, + "--url", + self.url, + "--token", + self.token, + ], + ) + assert result.exit_code == 0, f"project add failed: {result.output}" + self._dev_branch_id: int | None = None + self._created_bucket_id: str | None = None + try: + yield + finally: + if self._dev_branch_id is not None: + try: + self._run( + "branch", + "delete", + "--project", + self.alias, + "--branch", + str(self._dev_branch_id), + ) + except Exception as exc: + print(f" WARN: branch delete failed: {exc}") + if self._created_bucket_id is not None: + try: + self._run( + "storage", + "delete-bucket", + "--project", + self.alias, + "--bucket-id", + self._created_bucket_id, + "--force", + "--yes", + ) + except Exception as exc: + print(f" WARN: bucket delete failed: {exc}") + + def _run(self, *args: str) -> Any: + return _invoke(self.config_dir, ["--json", *args]) + + def _run_ok(self, *args: str) -> dict[str, Any]: + return _json_ok(self._run(*args)) + + # ------------------------------------------------------------------ + # storage create-table --if-not-exists + # ------------------------------------------------------------------ + + def test_storage_create_table_if_not_exists_round_trip(self) -> None: + """First call: action=created. Second call with --if-not-exists: + action=skipped. Third call without the flag: STORAGE_JOB_FAILED.""" + _step("v0470-1", "storage create-table --if-not-exists") + bucket_name = f"v0470_{RUN_ID.replace('-', '_')[:20]}" + bucket_data = self._run_ok( + "storage", + "create-bucket", + "--project", + self.alias, + "--stage", + "in", + "--name", + bucket_name, + ) + bucket_id = bucket_data["data"]["id"] + assert bucket_id.startswith("in.c-") + self._created_bucket_id = bucket_id + + table_name = f"v0470_tbl_{RUN_ID.replace('-', '_')[:16]}" + + first = self._run_ok( + "storage", + "create-table", + "--project", + self.alias, + "--bucket-id", + bucket_id, + "--name", + table_name, + "--column", + "id:INTEGER", + "--column", + "label:STRING", + "--primary-key", + "id", + "--if-not-exists", + ) + assert first["data"]["action"] == "created" + assert first["data"]["table_id"] == f"{bucket_id}.{table_name}" + + second = self._run_ok( + "storage", + "create-table", + "--project", + self.alias, + "--bucket-id", + bucket_id, + "--name", + table_name, + "--column", + "id:INTEGER", + "--column", + "label:STRING", + "--primary-key", + "id", + "--if-not-exists", + ) + assert second["data"]["action"] == "skipped" + assert second["data"]["skip_reason"] == "table already exists" + assert second["data"]["table_id"] == f"{bucket_id}.{table_name}" + # keboola/cli#349: the skipped envelope reports the EXISTING table's + # actual schema. The request here matches the existing table, so no drift. + assert second["data"]["columns"] == ["id", "label"] + assert second["data"]["primary_key"] == ["id"] + assert second["data"]["requested_columns"] == ["id", "label"] + assert second["data"]["requested_primary_key"] == ["id"] + assert second["data"]["schema_drift"] is False + + # keboola/cli#349: a divergent request against the same pre-existing + # table still skips, but the envelope reports the ACTUAL columns (not the + # requested 'extra' column) and flags the drift. + divergent = self._run_ok( + "storage", + "create-table", + "--project", + self.alias, + "--bucket-id", + bucket_id, + "--name", + table_name, + "--column", + "id:INTEGER", + "--column", + "label:STRING", + "--column", + "extra:STRING", + "--primary-key", + "extra", + "--if-not-exists", + ) + assert divergent["data"]["action"] == "skipped" + assert divergent["data"]["columns"] == ["id", "label"] + assert divergent["data"]["primary_key"] == ["id"] + assert "extra" in divergent["data"]["requested_columns"] + assert divergent["data"]["requested_primary_key"] == ["extra"] + assert divergent["data"]["schema_drift"] is True + + third = self._run( + "storage", + "create-table", + "--project", + self.alias, + "--bucket-id", + bucket_id, + "--name", + table_name, + "--column", + "id:INTEGER", + "--primary-key", + "id", + ) + assert third.exit_code != 0, ( + "default behavior must still error on duplicate (no silent skip)" + ) + body = json.loads(third.output) + assert body.get("status") == "error" + assert body.get("error", {}).get("code") == "STORAGE_JOB_FAILED" + + # ------------------------------------------------------------------ + # semantic-layer search-context / get-context + # ------------------------------------------------------------------ + + def test_semantic_layer_search_and_get_context(self) -> None: + """search-context with default pattern returns a valid envelope. + get-context with an all-zero UUID returns NOT_FOUND.""" + _step("v0470-2", "semantic-layer search-context + get-context") + + search = self._run_ok( + "semantic-layer", + "search-context", + "--project", + self.alias, + "--pattern", + "*", + ) + data = search["data"] + assert "contexts" in data + assert "total_count" in data + assert isinstance(data["contexts"], list) + assert isinstance(data["total_count"], int) + assert data["total_count"] == len(data["contexts"]) + for ctx in data["contexts"]: + assert ctx["type"] in { + "model", + "dataset", + "metric", + "relationship", + "constraint", + "glossary", + }, f"unexpected type slug: {ctx['type']!r}" + + only_datasets = self._run_ok( + "semantic-layer", + "search-context", + "--project", + self.alias, + "--type", + "dataset", + ) + for ctx in only_datasets["data"]["contexts"]: + assert ctx["type"] == "dataset" + + missing = self._run( + "semantic-layer", + "get-context", + "--project", + self.alias, + "--context-id", + "00000000-0000-0000-0000-000000000000", + ) + assert missing.exit_code != 0 + body = json.loads(missing.output) + assert body.get("status") == "error" + assert body.get("error", {}).get("code") == "NOT_FOUND" + + if data["contexts"]: + first_id = data["contexts"][0]["id"] + roundtrip = self._run_ok( + "semantic-layer", + "get-context", + "--project", + self.alias, + "--context-id", + first_id, + ) + assert roundtrip["data"]["id"] == first_id + assert roundtrip["data"]["type"] == data["contexts"][0]["type"] + + # ------------------------------------------------------------------ + # sync diff --branch + # ------------------------------------------------------------------ + + def test_sync_diff_branch_override(self) -> None: + """A dev branch created on the fly is targetable via `sync diff --branch` + without first running `branch use` or `sync branch-link`.""" + _step("v0470-3", "sync diff --branch ") + + branch_name = f"v0470-e2e-{RUN_ID[:20]}" + branch_data = self._run_ok( + "branch", + "create", + "--project", + self.alias, + "--name", + branch_name, + ) + dev_branch_id = int(branch_data["data"]["branch_id"]) + self._dev_branch_id = dev_branch_id + + project_dir = self.tmp_path / "v0470-sync" + project_dir.mkdir() + _git(project_dir, "init") + _git(project_dir, "config", "user.email", "e2e@test.local") + _git(project_dir, "config", "user.name", "E2E Test") + _git(project_dir, "commit", "--allow-empty", "-m", "init") + + init_result = _invoke( + self.config_dir, + [ + "--json", + "sync", + "init", + "--project", + self.alias, + "--directory", + str(project_dir), + ], + ) + assert init_result.exit_code == 0, init_result.output + + with_override = _invoke( + self.config_dir, + [ + "--json", + "sync", + "diff", + "--project", + self.alias, + "--directory", + str(project_dir), + "--branch", + str(dev_branch_id), + ], + ) + body = json.loads(with_override.output) + assert body.get("status") == "ok", body + assert body["data"].get("changes") is not None + summary = body["data"].get("summary", {}) + assert summary.get("remote_only", 0) >= 0 + + # ------------------------------------------------------------------ + # sync push -- fresh-CREATE writeback + KBC.configuration.* propagation + # (Area B headline fix; against real Storage API + metadata API) + # ------------------------------------------------------------------ + + def test_sync_push_fresh_create_writeback_and_kbc_metadata(self) -> None: + """Round-trip the FIIA / scaffold emit pattern against a real + Keboola project: hand-author a placeholder ManifestConfiguration + with ``KBC.configuration.folderName`` declared, run ``sync push``, + and assert: + 1. The push reports ``created=1, errors=0``. + 2. The manifest entry was updated in place (length stays at 1, + not 2; placeholder id is now the assigned ULID; folderName + metadata is preserved on the entry). + 3. The remote configuration's metadata-list returns the + KBC.configuration.folderName key with the declared value. + 4. A second ``sync push`` against the same workspace is a no-op + (``status=no_changes, created=0, errors=0``). + + Cleanup: delete the freshly-created remote config + the dev branch + in the teardown so re-runs do not accumulate residue. + """ + from keboola_agent_cli.constants import CONFIG_FILENAME, CONFIG_YML_VERSION + from keboola_agent_cli.sync.manifest import ( + ManifestConfiguration, + load_manifest, + save_manifest, + ) + + _step("v0470-4", "sync push fresh-CREATE writeback + KBC.* propagation") + + # Throwaway dev branch so we never pollute main. + branch_name = f"v0470-fcw-{RUN_ID[:20]}" + branch_data = self._run_ok( + "branch", + "create", + "--project", + self.alias, + "--name", + branch_name, + ) + dev_branch_id = int(branch_data["data"]["branch_id"]) + self._dev_branch_id = dev_branch_id # teardown will delete + + # Fresh sync workspace. + project_dir = self.tmp_path / "v0470-fcw" + project_dir.mkdir() + _git(project_dir, "init") + _git(project_dir, "config", "user.email", "e2e@test.local") + _git(project_dir, "config", "user.name", "E2E Test") + _git(project_dir, "commit", "--allow-empty", "-m", "init") + + init_result = _invoke( + self.config_dir, + [ + "--json", + "sync", + "init", + "--project", + self.alias, + "--directory", + str(project_dir), + ], + ) + assert init_result.exit_code == 0, init_result.output + + # Pull the dev branch so its branch directory + entry land in the + # manifest (otherwise the placeholder's target branch is not + # tracked and sync push can't resolve a path for it). + pull_result = _invoke( + self.config_dir, + [ + "--json", + "sync", + "pull", + "--project", + self.alias, + "--directory", + str(project_dir), + "--branch", + str(dev_branch_id), + "--no-storage", + "--no-jobs", + ], + ) + assert pull_result.exit_code == 0, pull_result.output + + manifest = load_manifest(project_dir) + dev_branch_entry = next((b for b in manifest.branches if b.id == dev_branch_id), None) + assert dev_branch_entry is not None, ( + "sync pull --branch must register the dev branch in the manifest" + ) + dev_branch_path = dev_branch_entry.path + + # Hand-author a placeholder ManifestConfiguration with the + # KBC.configuration.folderName key (FIIA / scaffold emit pattern). + component_id = "keboola.snowflake-transformation" + config_dir_name = f"v0470-fcw-{RUN_ID[:18]}" + cfg_rel_path = f"transformation/{component_id}/{config_dir_name}" + folder_name = "v0.47.0 E2E Fresh-CREATE" + manifest.configurations.append( + ManifestConfiguration( + branchId=dev_branch_id, + componentId=component_id, + id="PLACEHOLDER-FCW", + path=cfg_rel_path, + metadata={"KBC.configuration.folderName": folder_name}, + ) + ) + save_manifest(project_dir, manifest) + pre_push_n = len(manifest.configurations) + + # Local _config.yml for the placeholder. + local_dir = project_dir / dev_branch_path / cfg_rel_path + local_dir.mkdir(parents=True) + (local_dir / CONFIG_FILENAME).write_text( + yaml.dump( + { + "version": CONFIG_YML_VERSION, + "name": "v0.47.0 e2e fresh-create", + "description": "E2E test: fresh-CREATE writeback in place", + "parameters": {}, + "_keboola": {"component_id": component_id, "config_id": ""}, + }, + default_flow_style=False, + ), + encoding="utf-8", + ) + + # First push: should CREATE the config + propagate the folder. + push_result = _invoke( + self.config_dir, + [ + "--json", + "sync", + "push", + "--project", + self.alias, + "--directory", + str(project_dir), + "--branch", + str(dev_branch_id), + ], + ) + assert push_result.exit_code == 0, push_result.output + body = json.loads(push_result.output) + assert body.get("status") == "ok", body + data = body["data"] + assert data["created"] == 1, data + assert data["errors"] == [], data["errors"] + + # Manifest contract: updated in place (length unchanged). + post = load_manifest(project_dir) + matching = [ + c + for c in post.configurations + if c.component_id == component_id + and c.path == cfg_rel_path + and c.branch_id == dev_branch_id + ] + assert len(matching) == 1, ( + "writeback must update placeholder in place, not duplicate; " + f"found {len(matching)} matching entries" + ) + assigned_id = matching[0].id + assert assigned_id != "PLACEHOLDER-FCW", ( + "placeholder id must be replaced with the API-assigned ULID" + ) + assert matching[0].metadata.get("KBC.configuration.folderName") == folder_name + assert len(post.configurations) == pre_push_n, "manifest must not grow on a single CREATE" + + # Remote metadata: folderName landed via the metadata API. + meta_result = _invoke( + self.config_dir, + [ + "--json", + "config", + "metadata-list", + "--project", + self.alias, + "--component-id", + component_id, + "--config-id", + assigned_id, + "--branch", + str(dev_branch_id), + ], + ) + meta = _json_ok(meta_result) + meta_keys = {m.get("key"): m.get("value") for m in meta["data"]["metadata"]} + assert meta_keys.get("KBC.configuration.folderName") == folder_name, ( + f"folderName missing or wrong on remote metadata-list: {meta_keys}" + ) + + # Second push: idempotent (no_changes; create_config NOT called again). + repush = _invoke( + self.config_dir, + [ + "--json", + "sync", + "push", + "--project", + self.alias, + "--directory", + str(project_dir), + "--branch", + str(dev_branch_id), + ], + ) + assert repush.exit_code == 0, repush.output + repush_body = json.loads(repush.output) + repush_status = repush_body.get("data", {}).get("status") or repush_body.get("status") + assert repush_status in ("no_changes", "pushed"), repush_body + assert repush_body["data"].get("created", 0) == 0, ( + "re-push must be idempotent: created=0 after writeback in place" + ) + + def test_sync_push_fresh_create_variable_binding_runtime(self) -> None: + """Fresh-CREATE variable binding end-to-end (KFR-03/04/05), v0.47.2. + + Hand-author the FIIA tree on a throwaway dev branch -- a + ``keboola.variables`` config + its default-values row + a Snowflake + transformation whose ``_configuration_extra`` cross-references both by + placeholder id. One ``sync push`` must: + + 1. report ``created=3, errors=0``; + 2. POST the row ``values`` (non-empty remote ``configuration.values``); + 3. rebind the transformation's ``variables_id`` / + ``variables_values_id`` to real ULIDs (not placeholder dirnames); + 4. produce a **runnable** transformation: ``job run --wait`` reaches + ``status: success`` (the real acceptance gate -- a broken variable + link fails at runtime with "Variable configuration ... not found"); + 5. be idempotent on re-push (``created=0``) with ``sync diff`` + reporting ``conflict=0``. + + Cleanup: the dev branch (and its configs) is deleted in teardown. + """ + from keboola_agent_cli.constants import CONFIG_FILENAME, CONFIG_YML_VERSION + from keboola_agent_cli.sync.manifest import ( + ManifestConfigRow, + ManifestConfiguration, + load_manifest, + save_manifest, + ) + + _step("v0472-1", "sync push fresh-CREATE variable binding + job run") + + branch_name = f"v0472-vb-{RUN_ID[:20]}" + branch_data = self._run_ok( + "branch", "create", "--project", self.alias, "--name", branch_name + ) + dev_branch_id = int(branch_data["data"]["branch_id"]) + self._dev_branch_id = dev_branch_id + + project_dir = self.tmp_path / "v0472-vb" + project_dir.mkdir() + _git(project_dir, "init") + _git(project_dir, "config", "user.email", "e2e@test.local") + _git(project_dir, "config", "user.name", "E2E Test") + _git(project_dir, "commit", "--allow-empty", "-m", "init") + + init_result = _invoke( + self.config_dir, + ["--json", "sync", "init", "--project", self.alias, "--directory", str(project_dir)], + ) + assert init_result.exit_code == 0, init_result.output + + pull_result = _invoke( + self.config_dir, + [ + "--json", + "sync", + "pull", + "--project", + self.alias, + "--directory", + str(project_dir), + "--branch", + str(dev_branch_id), + "--no-storage", + "--no-jobs", + ], + ) + assert pull_result.exit_code == 0, pull_result.output + + manifest = load_manifest(project_dir) + dev_branch_entry = next((b for b in manifest.branches if b.id == dev_branch_id), None) + assert dev_branch_entry is not None + dev_branch_path = dev_branch_entry.path + + # Placeholder ids cross-referenced by the transformation. + suffix = RUN_ID[:14] + vars_ph = f"PH-VARS-{suffix}" + vals_ph = f"PH-VALS-{suffix}" + tx_component = "keboola.snowflake-transformation" + vars_component = "keboola.variables" + vars_path = f"variable/{vars_component}/vb_{suffix}" + tx_path = f"transformation/{tx_component}/vb_{suffix}" + + vars_entry = ManifestConfiguration( + branchId=dev_branch_id, + componentId=vars_component, + id=vars_ph, + path=vars_path, + ) + vars_entry.rows.append(ManifestConfigRow(id=vals_ph, path="rows/default", metadata={})) + manifest.configurations.append(vars_entry) + manifest.configurations.append( + ManifestConfiguration( + branchId=dev_branch_id, + componentId=tx_component, + id=f"PH-TX-{suffix}", + path=tx_path, + ) + ) + save_manifest(project_dir, manifest) + + # Local files: variables config + default-values row + transformation. + vars_dir = project_dir / dev_branch_path / vars_path + vars_dir.mkdir(parents=True) + (vars_dir / CONFIG_FILENAME).write_text( + yaml.dump( + { + "version": CONFIG_YML_VERSION, + "name": f"vb vars {suffix}", + "description": "", + "_keboola": {"component_id": vars_component, "config_id": ""}, + }, + default_flow_style=False, + ), + encoding="utf-8", + ) + row_dir = vars_dir / "rows" / "default" + row_dir.mkdir(parents=True) + (row_dir / CONFIG_FILENAME).write_text( + yaml.dump( + { + "version": CONFIG_YML_VERSION, + "name": "default", + "description": "", + # keboola.variables row values accept only name + value + # (the API rejects a "type" key on values.N). + "values": [{"name": "greeting", "value": "hello"}], + }, + default_flow_style=False, + ), + encoding="utf-8", + ) + tx_dir = project_dir / dev_branch_path / tx_path + tx_dir.mkdir(parents=True) + (tx_dir / CONFIG_FILENAME).write_text( + yaml.dump( + { + "version": CONFIG_YML_VERSION, + "name": f"vb tx {suffix}", + "description": "", + # No input/output mapping: the SQL just selects the variable + # so the job succeeds iff the variable link resolves. + "parameters": { + "blocks": [ + { + "name": "Block", + "codes": [ + { + "name": "Greet", + "script": ["SELECT '{{ greeting }}' AS msg;"], + } + ], + } + ] + }, + "_configuration_extra": { + "variables_id": vars_ph, + "variables_values_id": vals_ph, + }, + "_keboola": {"component_id": tx_component, "config_id": ""}, + }, + default_flow_style=False, + ), + encoding="utf-8", + ) + + # One push creates all three and rebinds the variable link. + push_result = _invoke( + self.config_dir, + [ + "--json", + "sync", + "push", + "--project", + self.alias, + "--directory", + str(project_dir), + "--branch", + str(dev_branch_id), + ], + ) + assert push_result.exit_code == 0, push_result.output + push_data = json.loads(push_result.output)["data"] + assert push_data["created"] == 3, push_data + assert push_data["errors"] == [], push_data["errors"] + + # Select OUR entries by path: a dev branch inherits production's + # configs, so the manifest holds many pre-existing entries of the same + # component after the pull -- matching on component_id alone is wrong. + post = load_manifest(project_dir) + vars_ulid = next( + c.id + for c in post.configurations + if c.component_id == vars_component and c.path == vars_path + ) + tx_ulid = next( + c.id + for c in post.configurations + if c.component_id == tx_component and c.path == tx_path + ) + assert vars_ulid != vars_ph, "variables config placeholder must become a ULID" + assert tx_ulid != f"PH-TX-{suffix}", "transformation placeholder must become a ULID" + + # Remote transformation: variables_id / variables_values_id are ULIDs. + tx_detail = self._run_ok( + "config", + "detail", + "--project", + self.alias, + "--component-id", + tx_component, + "--config-id", + tx_ulid, + "--branch", + str(dev_branch_id), + ) + tx_cfg = tx_detail["data"]["configuration"] + assert tx_cfg.get("variables_id") == vars_ulid, tx_cfg + assert tx_cfg.get("variables_values_id"), tx_cfg + assert tx_cfg["variables_values_id"] != vals_ph, "values_id must be a real ULID" + vals_ulid = tx_cfg["variables_values_id"] + + # Remote values row: configuration.values is non-empty (KFR-04). + vars_detail = self._run_ok( + "config", + "detail", + "--project", + self.alias, + "--component-id", + vars_component, + "--config-id", + vars_ulid, + "--branch", + str(dev_branch_id), + ) + rows = vars_detail["data"].get("rows", []) + bound_row = next((r for r in rows if str(r.get("id")) == str(vals_ulid)), None) + assert bound_row is not None, f"values row {vals_ulid} missing on remote: {rows}" + assert bound_row["configuration"].get("values"), "row values must be non-empty" + + # THE REAL GATE: the transformation runs to success (variable resolves). + run_data = self._run_ok( + "job", + "run", + "--project", + self.alias, + "--component-id", + tx_component, + "--config-id", + tx_ulid, + "--branch", + str(dev_branch_id), + "--wait", + "--timeout", + "300", + ) + job = run_data["data"] + assert job["status"] == "success", ( + f"job run failed ({job['status']}): " + f"{job.get('result', {}).get('message', 'no message')}" + ) + + # Re-push is idempotent and diff reports no conflict. + repush = _invoke( + self.config_dir, + [ + "--json", + "sync", + "push", + "--project", + self.alias, + "--directory", + str(project_dir), + "--branch", + str(dev_branch_id), + ], + ) + assert repush.exit_code == 0, repush.output + repush_data = json.loads(repush.output)["data"] + assert repush_data.get("created", 0) == 0, repush_data + assert repush_data.get("errors", []) == [], repush_data + + diff_result = _invoke( + self.config_dir, + [ + "--json", + "sync", + "diff", + "--project", + self.alias, + "--directory", + str(project_dir), + "--branch", + str(dev_branch_id), + ], + ) + assert diff_result.exit_code == 0, diff_result.output + diff_summary = json.loads(diff_result.output)["data"]["summary"] + assert diff_summary.get("conflict", 0) == 0, diff_summary + + +class TestDevPortalE2E: + """E2E coverage for v0.48.0 -- Developer Portal command group. + + - ``dev-portal identity list`` -- unconditional smoke test (no KB token needed) + - ``dev-portal list --vendor keboola`` -- optional real-portal test (needs E2E_DP_USERNAME + and E2E_DP_PASSWORD env vars) + """ + + @pytest.fixture(autouse=True) + def setup(self, tmp_path: Path): + self.config_dir = tmp_path / "config" + self.config_dir.mkdir() + + def test_identity_list_smoke(self) -> None: + """Unconditional smoke: dev-portal identity list must not crash (no identities configured).""" + result = _invoke(self.config_dir, ["--json", "dev-portal", "identity", "list"]) + assert result.exit_code == 0, result.output + body = json.loads(result.output) + # JSON envelope: {"status": "ok", "data": [...]} + identities = body.get("data", body) + assert isinstance(identities, list), ( + f"expected list under 'data', got {type(identities).__name__}: {result.output}" + ) + + @pytest.mark.skipif( + not (os.environ.get("E2E_DP_USERNAME") and os.environ.get("E2E_DP_PASSWORD")), + reason="Set E2E_DP_USERNAME and E2E_DP_PASSWORD to run real-portal test", + ) + def test_list_apps_against_real_portal(self) -> None: + """Optional: add an identity and list apps for vendor 'keboola' if creds supplied.""" + result = _invoke( + self.config_dir, + [ + "--json", + "dev-portal", + "identity", + "add", + "--alias", + "e2e", + "--username", + os.environ["E2E_DP_USERNAME"], + "--password", + os.environ["E2E_DP_PASSWORD"], + "--vendor", + "keboola", + ], + ) + assert result.exit_code == 0, result.output + result = _invoke( + self.config_dir, + [ + "--json", + "dev-portal", + "list", + "--vendor", + "keboola", + "--identity", + "e2e", + ], + ) + assert result.exit_code == 0, result.output + + def test_role_hint_typo_rejected_at_cli_layer(self) -> None: + """`identity add --role-hint vendr` is rejected by Typer's + `click.Choice(["vendor", "admin"])` validator before any model + construction happens (since v0.51.1). + + Offline -- the rejection is a Typer usage error (exit 2), no + network is touched. Note the layering: the Pydantic validator on + the model itself deliberately *downgrades* unknown values to + "vendor" with a stderr warning, so legacy free-text values in a + pre-0.51.1 `config.json` still load. That tolerance is appropriate + for `ConfigStore.load()` but wrong at the CLI -- a typo the user + just typed should fail loudly, not silently land as "vendor". The + `click.Choice` wiring in `commands/dev_portal.py` provides that + separation. + """ + result = _invoke( + self.config_dir, + [ + "dev-portal", + "identity", + "add", + "--alias", + "bad-role", + "--username", + "u", + "--password", + "p", + "--role-hint", + "vendr", # typo + ], + ) + # Typer/Click usage error -> exit 2 + assert result.exit_code == 2 + assert "vendor" in result.output.lower() or "admin" in result.output.lower() + + def test_vendor_role_admin_only_field_fails_fast(self) -> None: + """`prepare_patch` preflight refuses admin-only fields on a vendor identity + (since v0.51.1). Runs offline -- preflight fires before any portal call, + so no creds needed. Verifies the user-facing error names the offending + field and points at the admin-identity workaround. + """ + from keboola_agent_cli.config_store import ConfigStore + from keboola_agent_cli.models import DeveloperPortalIdentity + + store = ConfigStore(self.config_dir, source="cli-flag") + store.add_dev_portal_identity( + "vendor-e2e", + DeveloperPortalIdentity( + username="u", password="p", vendor="keboola", role_hint="vendor" + ), + ) + + payload_path = self.config_dir / "patch.json" + payload_path.write_text(json.dumps({"complexity": "easy"})) + result = _invoke( + self.config_dir, + [ + "dev-portal", + "patch", + "--app", + "keboola.ex-bogus", + "--data", + str(payload_path), + "--identity", + "vendor-e2e", + "--dry-run", + ], + ) + # exit non-zero because validation error + assert result.exit_code != 0 + # error mentions the offending field and the admin workaround + out_lower = result.output.lower() + assert "complexity" in out_lower + assert "admin" in out_lower + + +@skip_without_credentials +@pytest.mark.e2e +class TestHeadlessEnvProject: + """Headless / token-only invocation against the real API (issue #359). + + Verifies that KBAGENT_PROJECT_FROM_ENV=1 + KBC_TOKEN + KBC_STORAGE_API_URL + let kbagent run with an EMPTY config dir (no `project add`, no config.json), + and that the env token is never written to disk. + """ + + @pytest.fixture(autouse=True) + def setup(self, tmp_path: Path) -> None: + self.token = os.environ[ENV_TOKEN] + raw_url = os.environ.get(ENV_URL, "connection.keboola.com") + self.url = raw_url if raw_url.startswith("https://") else f"https://{raw_url}" + self.config_dir = tmp_path / "empty-config" + self.config_dir.mkdir() + + def _headless_env(self) -> dict[str, str]: + return { + "KBAGENT_PROJECT_FROM_ENV": "1", + "KBC_TOKEN": self.token, + "KBC_STORAGE_API_URL": self.url, + } + + def test_headless_lists_env_project(self) -> None: + _step("HEADLESS-1", "project list resolves __env__ from env, no config.json") + with patch.dict(os.environ, self._headless_env()): + result = _invoke(self.config_dir, ["--json", "project", "list"]) + data = _json_ok(result) + aliases = {p["alias"] for p in data["data"]} + assert "__env__" in aliases, data + # No config.json was written -- token stays in memory only. + assert not (self.config_dir / "config.json").exists() + + def test_headless_storage_call_hits_api(self) -> None: + _step("HEADLESS-2", "storage buckets --project __env__ reaches the real API") + with patch.dict(os.environ, self._headless_env()): + result = _invoke( + self.config_dir, + ["--json", "storage", "buckets", "--project", "__env__"], + ) + # status=ok proves the env token authenticated a real API call. + _json_ok(result) + assert not (self.config_dir / "config.json").exists() + + def test_headless_requires_opt_in_flag(self) -> None: + _step("HEADLESS-3", "KBC_TOKEN without the opt-in flag => no phantom project") + env = {"KBC_TOKEN": self.token, "KBC_STORAGE_API_URL": self.url} + with patch.dict(os.environ, env): + # Ensure the flag is absent for this assertion. + os.environ.pop("KBAGENT_PROJECT_FROM_ENV", None) + result = _invoke(self.config_dir, ["--json", "project", "list"]) + data = _json_ok(result) + assert data["data"] == [], data diff --git a/tests/test_errors.py b/tests/test_errors.py index 3cfcca5b..fe7ae5f8 100644 --- a/tests/test_errors.py +++ b/tests/test_errors.py @@ -157,3 +157,12 @@ def test_keboolaapierror_accepts_enum(self) -> None: ) assert err.error_code == "QUEUE_JOB_FAILED" assert err.error_code == ErrorCode.QUEUE_JOB_FAILED + + +def test_dev_portal_error_codes_present() -> None: + """Developer Portal error codes are defined in the ErrorCode enum.""" + assert ErrorCode.DP_LOGIN_FAILED == "DP_LOGIN_FAILED" + assert ErrorCode.DP_MFA_REQUIRED == "DP_MFA_REQUIRED" + assert ErrorCode.DP_APP_NOT_FOUND == "DP_APP_NOT_FOUND" + assert ErrorCode.DP_PUBLISH_REQUIREMENTS_MISSING == "DP_PUBLISH_REQUIREMENTS_MISSING" + assert ErrorCode.DP_ICON_UPLOAD_FAILED == "DP_ICON_UPLOAD_FAILED" diff --git a/tests/test_feature_cli.py b/tests/test_feature_cli.py new file mode 100644 index 00000000..7025523f --- /dev/null +++ b/tests/test_feature_cli.py @@ -0,0 +1,593 @@ +"""CLI tests for `kbagent feature` command group (super-admin Manage API). + +Exercises the thin Typer layer in ``commands/feature.py`` via ``CliRunner``. +The ``FeatureService`` is mocked by patching ``keboola_agent_cli.cli.FeatureService`` +so the instance stored in ``ctx.obj["feature_service"]`` is a ``MagicMock``. + +The manage token is sourced through ``resolve_manage_token``: passing the +top-level ``--allow-env-manage-token`` flag plus ``KBC_MANAGE_API_TOKEN`` in the +environment bypasses the interactive prompt (same pattern as test_member_cli.py). +""" + +from __future__ import annotations + +import json +import os +from pathlib import Path +from unittest.mock import MagicMock, patch + +from typer.testing import CliRunner + +from keboola_agent_cli.cli import app +from keboola_agent_cli.config_store import ConfigStore +from keboola_agent_cli.errors import ConfigError, ErrorCode, KeboolaApiError +from keboola_agent_cli.models import ProjectConfig + +STACK_URL = "https://connection.us-east4.gcp.keboola.com" +PROJECT_ID = 5725 +ALIAS = "cuesta-master" +EMAIL = "user@example.com" +FEATURE = "queue-v2" +MANAGE_TOKEN = "manage-12345-abcdefghijklmnopqrstuvwxyz0123456789" + +runner = CliRunner() + + +def _seed_store(config_dir: Path) -> ConfigStore: + store = ConfigStore(config_dir=config_dir) + store.add_project( + ALIAS, + ProjectConfig( + stack_url=STACK_URL, + token="901-fake-storage-token-1234567890", + project_name="[Cuesta training] - Master", + project_id=PROJECT_ID, + ), + ) + return store + + +def _invoke(config_dir: Path, svc: MagicMock, args: list[str], input_text: str | None = None): + """Invoke the CLI with a mocked FeatureService and env-provided manage token.""" + with ( + patch("keboola_agent_cli.cli.FeatureService", return_value=svc), + patch.dict(os.environ, {"KBC_MANAGE_API_TOKEN": MANAGE_TOKEN}), + ): + return runner.invoke( + app, + [ + "--allow-env-manage-token", + "--config-dir", + str(config_dir), + *args, + ], + input=input_text, + ) + + +class TestFeatureList: + def test_json_happy_path(self, tmp_path: Path) -> None: + config_dir = tmp_path / "config" + config_dir.mkdir() + _seed_store(config_dir) + svc = MagicMock() + svc.list_stack_features.return_value = { + "stack_url": STACK_URL, + "features": [ + {"name": FEATURE, "title": "Queue v2", "type": "project", "description": "desc"} + ], + } + + result = _invoke(config_dir, svc, ["--json", "feature", "list", "--project", ALIAS]) + + assert result.exit_code == 0, result.output + out = json.loads(result.output) + assert out["status"] == "ok" + assert out["data"]["features"][0]["name"] == FEATURE + svc.list_stack_features.assert_called_once_with(manage_token=MANAGE_TOKEN, alias=ALIAS) + + def test_human_mode_smoke(self, tmp_path: Path) -> None: + config_dir = tmp_path / "config" + config_dir.mkdir() + _seed_store(config_dir) + svc = MagicMock() + svc.list_stack_features.return_value = { + "stack_url": STACK_URL, + "features": [ + {"name": FEATURE, "title": "Queue v2", "type": "project", "description": "desc"} + ], + } + + result = _invoke(config_dir, svc, ["feature", "list", "--project", ALIAS]) + + assert result.exit_code == 0, result.output + assert FEATURE in result.output + + def test_config_error_exits_5(self, tmp_path: Path) -> None: + config_dir = tmp_path / "config" + config_dir.mkdir() + _seed_store(config_dir) + svc = MagicMock() + svc.list_stack_features.side_effect = ConfigError("unknown alias") + + result = _invoke(config_dir, svc, ["--json", "feature", "list", "--project", ALIAS]) + + assert result.exit_code == 5, result.output + out = json.loads(result.output) + assert out["error"]["code"] == ErrorCode.CONFIG_ERROR + + def test_api_error_maps_to_exit_3(self, tmp_path: Path) -> None: + config_dir = tmp_path / "config" + config_dir.mkdir() + _seed_store(config_dir) + svc = MagicMock() + svc.list_stack_features.side_effect = KeboolaApiError( + message="Invalid or expired token", + status_code=401, + error_code=ErrorCode.INVALID_TOKEN, + ) + + result = _invoke(config_dir, svc, ["--json", "feature", "list", "--project", ALIAS]) + + assert result.exit_code == 3, result.output + + +class TestFeatureProjectShow: + def test_json_happy_path(self, tmp_path: Path) -> None: + config_dir = tmp_path / "config" + config_dir.mkdir() + _seed_store(config_dir) + svc = MagicMock() + svc.list_project_features.return_value = { + "alias": ALIAS, + "project_id": PROJECT_ID, + "features": [{"name": FEATURE, "title": "Queue v2", "description": "desc"}], + } + + result = _invoke(config_dir, svc, ["--json", "feature", "project-show", "--project", ALIAS]) + + assert result.exit_code == 0, result.output + out = json.loads(result.output) + assert out["data"]["features"][0]["name"] == FEATURE + svc.list_project_features.assert_called_once_with(manage_token=MANAGE_TOKEN, alias=ALIAS) + + def test_human_mode_smoke(self, tmp_path: Path) -> None: + config_dir = tmp_path / "config" + config_dir.mkdir() + _seed_store(config_dir) + svc = MagicMock() + svc.list_project_features.return_value = { + "alias": ALIAS, + "project_id": PROJECT_ID, + "features": [], + } + + result = _invoke(config_dir, svc, ["feature", "project-show", "--project", ALIAS]) + + assert result.exit_code == 0, result.output + + def test_human_mode_omits_empty_optional_columns(self, tmp_path: Path) -> None: + """Bare-string project features (name-only) drop the Title/Description columns.""" + config_dir = tmp_path / "config" + config_dir.mkdir() + _seed_store(config_dir) + svc = MagicMock() + svc.list_project_features.return_value = { + "alias": ALIAS, + "project_id": PROJECT_ID, + # Mirrors the normalised bare-string shape: name set, rest empty. + "features": [ + {"name": "queuev2", "title": "", "description": "", "type": ""}, + {"name": "storage-types", "title": "", "description": "", "type": ""}, + ], + } + + result = _invoke(config_dir, svc, ["feature", "project-show", "--project", ALIAS]) + + assert result.exit_code == 0, result.output + assert "queuev2" in result.output + # No optional column header should be rendered when every value is empty. + assert "Title" not in result.output + assert "Description" not in result.output + + def test_human_mode_keeps_populated_optional_columns(self, tmp_path: Path) -> None: + """When a feature carries a title, the Title column is shown.""" + config_dir = tmp_path / "config" + config_dir.mkdir() + _seed_store(config_dir) + svc = MagicMock() + svc.list_project_features.return_value = { + "alias": ALIAS, + "project_id": PROJECT_ID, + "features": [{"name": "queuev2", "title": "Queue v2", "description": "", "type": ""}], + } + + result = _invoke(config_dir, svc, ["feature", "project-show", "--project", ALIAS]) + + assert result.exit_code == 0, result.output + assert "Title" in result.output + # Description is still empty across the board, so its column stays hidden. + assert "Description" not in result.output + + +class TestFeatureProjectAdd: + def test_dry_run_skips_confirmation(self, tmp_path: Path) -> None: + config_dir = tmp_path / "config" + config_dir.mkdir() + _seed_store(config_dir) + svc = MagicMock() + svc.add_project_feature.return_value = { + "status": "dry_run", + "action": "add", + "alias": ALIAS, + "project_id": PROJECT_ID, + "feature": FEATURE, + } + + # No --yes, no --json, no input: dry-run must short-circuit the prompt. + result = _invoke( + config_dir, + svc, + ["feature", "project-add", "--project", ALIAS, "--feature", FEATURE, "--dry-run"], + ) + + assert result.exit_code == 0, result.output + assert "DRY RUN" in result.output + svc.add_project_feature.assert_called_once_with( + manage_token=MANAGE_TOKEN, alias=ALIAS, feature=FEATURE, dry_run=True + ) + + def test_confirm_abort_does_not_call_service(self, tmp_path: Path) -> None: + config_dir = tmp_path / "config" + config_dir.mkdir() + _seed_store(config_dir) + svc = MagicMock() + + result = _invoke( + config_dir, + svc, + ["feature", "project-add", "--project", ALIAS, "--feature", FEATURE], + input_text="n\n", + ) + + assert result.exit_code == 0, result.output + assert "Aborted." in result.output + svc.add_project_feature.assert_not_called() + + def test_yes_skips_confirmation_and_adds(self, tmp_path: Path) -> None: + config_dir = tmp_path / "config" + config_dir.mkdir() + _seed_store(config_dir) + svc = MagicMock() + svc.add_project_feature.return_value = { + "status": "added", + "alias": ALIAS, + "project_id": PROJECT_ID, + "feature": FEATURE, + } + + result = _invoke( + config_dir, + svc, + ["--json", "feature", "project-add", "--project", ALIAS, "--feature", FEATURE, "--yes"], + ) + + assert result.exit_code == 0, result.output + out = json.loads(result.output) + assert out["data"]["status"] == "added" + svc.add_project_feature.assert_called_once_with( + manage_token=MANAGE_TOKEN, alias=ALIAS, feature=FEATURE, dry_run=False + ) + + def test_api_error_maps_to_exit_code(self, tmp_path: Path) -> None: + config_dir = tmp_path / "config" + config_dir.mkdir() + _seed_store(config_dir) + svc = MagicMock() + svc.add_project_feature.side_effect = KeboolaApiError( + message="boom", + status_code=500, + error_code=ErrorCode.API_ERROR, + ) + + result = _invoke( + config_dir, + svc, + ["--json", "feature", "project-add", "--project", ALIAS, "--feature", FEATURE, "--yes"], + ) + + assert result.exit_code == 1, result.output + + +class TestFeatureProjectRemove: + def test_dry_run_skips_confirmation(self, tmp_path: Path) -> None: + config_dir = tmp_path / "config" + config_dir.mkdir() + _seed_store(config_dir) + svc = MagicMock() + svc.remove_project_feature.return_value = { + "status": "dry_run", + "action": "remove", + "alias": ALIAS, + "project_id": PROJECT_ID, + "feature": FEATURE, + } + + result = _invoke( + config_dir, + svc, + ["feature", "project-remove", "--project", ALIAS, "--feature", FEATURE, "--dry-run"], + ) + + assert result.exit_code == 0, result.output + assert "DRY RUN" in result.output + svc.remove_project_feature.assert_called_once_with( + manage_token=MANAGE_TOKEN, alias=ALIAS, feature=FEATURE, dry_run=True + ) + + def test_confirm_abort_does_not_call_service(self, tmp_path: Path) -> None: + config_dir = tmp_path / "config" + config_dir.mkdir() + _seed_store(config_dir) + svc = MagicMock() + + result = _invoke( + config_dir, + svc, + ["feature", "project-remove", "--project", ALIAS, "--feature", FEATURE], + input_text="n\n", + ) + + assert result.exit_code == 0, result.output + assert "Aborted." in result.output + svc.remove_project_feature.assert_not_called() + + def test_yes_skips_confirmation_and_removes(self, tmp_path: Path) -> None: + config_dir = tmp_path / "config" + config_dir.mkdir() + _seed_store(config_dir) + svc = MagicMock() + svc.remove_project_feature.return_value = { + "status": "removed", + "alias": ALIAS, + "project_id": PROJECT_ID, + "feature": FEATURE, + } + + result = _invoke( + config_dir, + svc, + [ + "--json", + "feature", + "project-remove", + "--project", + ALIAS, + "--feature", + FEATURE, + "--yes", + ], + ) + + assert result.exit_code == 0, result.output + out = json.loads(result.output) + assert out["data"]["status"] == "removed" + svc.remove_project_feature.assert_called_once_with( + manage_token=MANAGE_TOKEN, alias=ALIAS, feature=FEATURE, dry_run=False + ) + + +class TestFeatureUserShow: + def test_json_happy_path(self, tmp_path: Path) -> None: + config_dir = tmp_path / "config" + config_dir.mkdir() + _seed_store(config_dir) + svc = MagicMock() + svc.list_user_features.return_value = { + "email": EMAIL, + "features": [{"name": FEATURE, "title": "Queue v2", "description": "desc"}], + } + + result = _invoke( + config_dir, + svc, + ["--json", "feature", "user-show", "--project", ALIAS, "--email", EMAIL], + ) + + assert result.exit_code == 0, result.output + out = json.loads(result.output) + assert out["data"]["features"][0]["name"] == FEATURE + svc.list_user_features.assert_called_once_with( + manage_token=MANAGE_TOKEN, alias=ALIAS, email=EMAIL + ) + + def test_human_mode_smoke(self, tmp_path: Path) -> None: + config_dir = tmp_path / "config" + config_dir.mkdir() + _seed_store(config_dir) + svc = MagicMock() + svc.list_user_features.return_value = {"email": EMAIL, "features": []} + + result = _invoke( + config_dir, + svc, + ["feature", "user-show", "--project", ALIAS, "--email", EMAIL], + ) + + assert result.exit_code == 0, result.output + + +class TestFeatureUserAdd: + def test_dry_run_skips_confirmation(self, tmp_path: Path) -> None: + config_dir = tmp_path / "config" + config_dir.mkdir() + _seed_store(config_dir) + svc = MagicMock() + svc.add_user_feature.return_value = { + "status": "dry_run", + "action": "add", + "email": EMAIL, + "feature": FEATURE, + } + + result = _invoke( + config_dir, + svc, + [ + "feature", + "user-add", + "--project", + ALIAS, + "--email", + EMAIL, + "--feature", + FEATURE, + "--dry-run", + ], + ) + + assert result.exit_code == 0, result.output + assert "DRY RUN" in result.output + svc.add_user_feature.assert_called_once_with( + manage_token=MANAGE_TOKEN, alias=ALIAS, email=EMAIL, feature=FEATURE, dry_run=True + ) + + def test_confirm_abort_does_not_call_service(self, tmp_path: Path) -> None: + config_dir = tmp_path / "config" + config_dir.mkdir() + _seed_store(config_dir) + svc = MagicMock() + + result = _invoke( + config_dir, + svc, + ["feature", "user-add", "--project", ALIAS, "--email", EMAIL, "--feature", FEATURE], + input_text="n\n", + ) + + assert result.exit_code == 0, result.output + assert "Aborted." in result.output + svc.add_user_feature.assert_not_called() + + def test_yes_skips_confirmation_and_adds(self, tmp_path: Path) -> None: + config_dir = tmp_path / "config" + config_dir.mkdir() + _seed_store(config_dir) + svc = MagicMock() + svc.add_user_feature.return_value = { + "status": "added", + "email": EMAIL, + "feature": FEATURE, + } + + result = _invoke( + config_dir, + svc, + [ + "--json", + "feature", + "user-add", + "--project", + ALIAS, + "--email", + EMAIL, + "--feature", + FEATURE, + "--yes", + ], + ) + + assert result.exit_code == 0, result.output + out = json.loads(result.output) + assert out["data"]["status"] == "added" + svc.add_user_feature.assert_called_once_with( + manage_token=MANAGE_TOKEN, alias=ALIAS, email=EMAIL, feature=FEATURE, dry_run=False + ) + + +class TestFeatureUserRemove: + def test_dry_run_skips_confirmation(self, tmp_path: Path) -> None: + config_dir = tmp_path / "config" + config_dir.mkdir() + _seed_store(config_dir) + svc = MagicMock() + svc.remove_user_feature.return_value = { + "status": "dry_run", + "action": "remove", + "email": EMAIL, + "feature": FEATURE, + } + + result = _invoke( + config_dir, + svc, + [ + "feature", + "user-remove", + "--project", + ALIAS, + "--email", + EMAIL, + "--feature", + FEATURE, + "--dry-run", + ], + ) + + assert result.exit_code == 0, result.output + assert "DRY RUN" in result.output + svc.remove_user_feature.assert_called_once_with( + manage_token=MANAGE_TOKEN, alias=ALIAS, email=EMAIL, feature=FEATURE, dry_run=True + ) + + def test_confirm_abort_does_not_call_service(self, tmp_path: Path) -> None: + config_dir = tmp_path / "config" + config_dir.mkdir() + _seed_store(config_dir) + svc = MagicMock() + + result = _invoke( + config_dir, + svc, + ["feature", "user-remove", "--project", ALIAS, "--email", EMAIL, "--feature", FEATURE], + input_text="n\n", + ) + + assert result.exit_code == 0, result.output + assert "Aborted." in result.output + svc.remove_user_feature.assert_not_called() + + def test_yes_skips_confirmation_and_removes(self, tmp_path: Path) -> None: + config_dir = tmp_path / "config" + config_dir.mkdir() + _seed_store(config_dir) + svc = MagicMock() + svc.remove_user_feature.return_value = { + "status": "removed", + "email": EMAIL, + "feature": FEATURE, + } + + result = _invoke( + config_dir, + svc, + [ + "--json", + "feature", + "user-remove", + "--project", + ALIAS, + "--email", + EMAIL, + "--feature", + FEATURE, + "--yes", + ], + ) + + assert result.exit_code == 0, result.output + out = json.loads(result.output) + assert out["data"]["status"] == "removed" + svc.remove_user_feature.assert_called_once_with( + manage_token=MANAGE_TOKEN, alias=ALIAS, email=EMAIL, feature=FEATURE, dry_run=False + ) diff --git a/tests/test_feature_service.py b/tests/test_feature_service.py new file mode 100644 index 00000000..27bccb92 --- /dev/null +++ b/tests/test_feature_service.py @@ -0,0 +1,355 @@ +"""Tests for FeatureService - stack/project/user feature-flag management.""" + +from __future__ import annotations + +from pathlib import Path +from unittest.mock import MagicMock + +import pytest + +from keboola_agent_cli.config_store import ConfigStore +from keboola_agent_cli.errors import ConfigError +from keboola_agent_cli.models import ProjectConfig +from keboola_agent_cli.services.feature_service import FeatureService + +STACK_URL = "https://connection.us-east4.gcp.keboola.com" +MANAGE_TOKEN = "manage-12345-abcdefghijklmnopqrstuvwxyz0123456789" +PROJECT_ID = 5725 +ALIAS = "cuesta-master" +EMAIL = "max.ottomansky@keboola.com" + + +@pytest.fixture +def store_with_master(tmp_config_dir: Path) -> ConfigStore: + """ConfigStore with the master cuesta project pre-registered.""" + store = ConfigStore(config_dir=tmp_config_dir) + store.add_project( + ALIAS, + ProjectConfig( + stack_url=STACK_URL, + token="901-fake-storage-token-1234567890", + project_name="[Cuesta training] - Master", + project_id=PROJECT_ID, + ), + ) + return store + + +@pytest.fixture +def store_without_project_id(tmp_config_dir: Path) -> ConfigStore: + """ConfigStore with an alias that has no numeric project_id.""" + store = ConfigStore(config_dir=tmp_config_dir) + store.add_project( + "no-id", + ProjectConfig( + stack_url=STACK_URL, + token="901-fake-storage-token-1234567890", + project_name="No ID project", + project_id=None, + ), + ) + return store + + +@pytest.fixture +def manage_client_factory(): + """Factory returning a single shared MagicMock manage client.""" + mock = MagicMock() + mock._stack_url = STACK_URL + factory = MagicMock(return_value=mock) + return factory, mock + + +# ────────────────────────────────────────────────────────────────────── +# Stack catalogue +# ────────────────────────────────────────────────────────────────────── + + +class TestListStackFeatures: + def test_normalises_dict_features(self, store_with_master, manage_client_factory) -> None: + factory, mock_client = manage_client_factory + mock_client.list_features.return_value = [ + {"name": "queue-v2", "title": "Queue v2", "type": "project"}, + {"name": "snowflake-dwh", "title": "Snowflake"}, + ] + svc = FeatureService(store_with_master, manage_client_factory=factory) + + result = svc.list_stack_features(manage_token=MANAGE_TOKEN, alias=ALIAS) + + assert result["alias"] == ALIAS + assert result["stack_url"] == STACK_URL + assert [f["name"] for f in result["features"]] == ["queue-v2", "snowflake-dwh"] + assert result["features"][0]["title"] == "Queue v2" + # Factory must be bound to the resolved stack URL + token. + factory.assert_called_once_with(STACK_URL, MANAGE_TOKEN) + mock_client.close.assert_called_once() + + def test_normalises_bare_string_features( + self, store_with_master, manage_client_factory + ) -> None: + factory, mock_client = manage_client_factory + mock_client.list_features.return_value = ["queue-v2", "snowflake-dwh"] + svc = FeatureService(store_with_master, manage_client_factory=factory) + + result = svc.list_stack_features(manage_token=MANAGE_TOKEN, alias=ALIAS) + + assert result["features"] == [ + {"name": "queue-v2", "title": "", "description": "", "type": ""}, + {"name": "snowflake-dwh", "title": "", "description": "", "type": ""}, + ] + mock_client.close.assert_called_once() + + def test_unknown_alias_raises_config_error( + self, store_with_master, manage_client_factory + ) -> None: + factory, _ = manage_client_factory + svc = FeatureService(store_with_master, manage_client_factory=factory) + + with pytest.raises(ConfigError, match="not registered"): + svc.list_stack_features(manage_token=MANAGE_TOKEN, alias="does-not-exist") + factory.assert_not_called() + + +# ────────────────────────────────────────────────────────────────────── +# Project features +# ────────────────────────────────────────────────────────────────────── + + +class TestListProjectFeatures: + def test_reads_features_from_project_object( + self, store_with_master, manage_client_factory + ) -> None: + factory, mock_client = manage_client_factory + mock_client.get_project.return_value = { + "name": "[Cuesta training] - Master", + "features": [{"name": "queue-v2"}, "input-mapping-default"], + } + svc = FeatureService(store_with_master, manage_client_factory=factory) + + result = svc.list_project_features(manage_token=MANAGE_TOKEN, alias=ALIAS) + + assert result["alias"] == ALIAS + assert result["project_id"] == PROJECT_ID + assert result["project_name"] == "[Cuesta training] - Master" + # Mixed dict + bare string both normalise to the uniform shape. + assert [f["name"] for f in result["features"]] == ["queue-v2", "input-mapping-default"] + mock_client.get_project.assert_called_once_with(PROJECT_ID) + mock_client.close.assert_called_once() + + def test_missing_features_key_yields_empty_list( + self, store_with_master, manage_client_factory + ) -> None: + factory, mock_client = manage_client_factory + mock_client.get_project.return_value = {"name": "X"} + svc = FeatureService(store_with_master, manage_client_factory=factory) + + result = svc.list_project_features(manage_token=MANAGE_TOKEN, alias=ALIAS) + + assert result["features"] == [] + + def test_unknown_alias_raises_config_error( + self, store_with_master, manage_client_factory + ) -> None: + factory, _ = manage_client_factory + svc = FeatureService(store_with_master, manage_client_factory=factory) + + with pytest.raises(ConfigError, match="not registered"): + svc.list_project_features(manage_token=MANAGE_TOKEN, alias="does-not-exist") + + def test_missing_project_id_raises_config_error( + self, store_without_project_id, manage_client_factory + ) -> None: + factory, _ = manage_client_factory + svc = FeatureService(store_without_project_id, manage_client_factory=factory) + + with pytest.raises(ConfigError, match="no numeric project_id"): + svc.list_project_features(manage_token=MANAGE_TOKEN, alias="no-id") + factory.assert_not_called() + + +class TestAddProjectFeature: + def test_live_call(self, store_with_master, manage_client_factory) -> None: + factory, mock_client = manage_client_factory + svc = FeatureService(store_with_master, manage_client_factory=factory) + + result = svc.add_project_feature(manage_token=MANAGE_TOKEN, alias=ALIAS, feature="queue-v2") + + assert result["status"] == "added" + assert result["project_id"] == PROJECT_ID + assert result["feature"] == "queue-v2" + mock_client.add_project_feature.assert_called_once_with(PROJECT_ID, "queue-v2") + mock_client.close.assert_called_once() + + def test_dry_run_makes_no_client_call(self, store_with_master, manage_client_factory) -> None: + factory, mock_client = manage_client_factory + svc = FeatureService(store_with_master, manage_client_factory=factory) + + result = svc.add_project_feature( + manage_token=MANAGE_TOKEN, alias=ALIAS, feature="queue-v2", dry_run=True + ) + + assert result["status"] == "dry_run" + assert result["action"] == "add" + assert result["feature"] == "queue-v2" + factory.assert_not_called() + mock_client.add_project_feature.assert_not_called() + + +class TestRemoveProjectFeature: + def test_live_call(self, store_with_master, manage_client_factory) -> None: + factory, mock_client = manage_client_factory + svc = FeatureService(store_with_master, manage_client_factory=factory) + + result = svc.remove_project_feature( + manage_token=MANAGE_TOKEN, alias=ALIAS, feature="queue-v2" + ) + + assert result["status"] == "removed" + assert result["project_id"] == PROJECT_ID + mock_client.remove_project_feature.assert_called_once_with(PROJECT_ID, "queue-v2") + mock_client.close.assert_called_once() + + def test_dry_run_makes_no_client_call(self, store_with_master, manage_client_factory) -> None: + factory, mock_client = manage_client_factory + svc = FeatureService(store_with_master, manage_client_factory=factory) + + result = svc.remove_project_feature( + manage_token=MANAGE_TOKEN, alias=ALIAS, feature="queue-v2", dry_run=True + ) + + assert result["status"] == "dry_run" + assert result["action"] == "remove" + factory.assert_not_called() + mock_client.remove_project_feature.assert_not_called() + + +# ────────────────────────────────────────────────────────────────────── +# User features +# ────────────────────────────────────────────────────────────────────── + + +class TestListUserFeatures: + def test_reads_features_from_user_object( + self, store_with_master, manage_client_factory + ) -> None: + factory, mock_client = manage_client_factory + mock_client.get_user.return_value = { + "email": EMAIL, + "features": ["admin-ui-beta", {"name": "early-access"}], + } + svc = FeatureService(store_with_master, manage_client_factory=factory) + + result = svc.list_user_features(manage_token=MANAGE_TOKEN, alias=ALIAS, email=EMAIL) + + assert result["alias"] == ALIAS + assert result["stack_url"] == STACK_URL + assert result["email"] == EMAIL + assert [f["name"] for f in result["features"]] == ["admin-ui-beta", "early-access"] + mock_client.get_user.assert_called_once_with(EMAIL) + # User ops resolve the stack URL only -- no numeric project_id required. + factory.assert_called_once_with(STACK_URL, MANAGE_TOKEN) + mock_client.close.assert_called_once() + + def test_works_without_project_id( + self, store_without_project_id, manage_client_factory + ) -> None: + factory, mock_client = manage_client_factory + mock_client.get_user.return_value = {"email": EMAIL, "features": []} + svc = FeatureService(store_without_project_id, manage_client_factory=factory) + + result = svc.list_user_features(manage_token=MANAGE_TOKEN, alias="no-id", email=EMAIL) + + assert result["features"] == [] + mock_client.close.assert_called_once() + + def test_unknown_alias_raises_config_error( + self, store_with_master, manage_client_factory + ) -> None: + factory, _ = manage_client_factory + svc = FeatureService(store_with_master, manage_client_factory=factory) + + with pytest.raises(ConfigError, match="not registered"): + svc.list_user_features(manage_token=MANAGE_TOKEN, alias="does-not-exist", email=EMAIL) + + +class TestAddUserFeature: + def test_live_call(self, store_with_master, manage_client_factory) -> None: + factory, mock_client = manage_client_factory + svc = FeatureService(store_with_master, manage_client_factory=factory) + + result = svc.add_user_feature( + manage_token=MANAGE_TOKEN, alias=ALIAS, email=EMAIL, feature="admin-ui-beta" + ) + + assert result["status"] == "added" + assert result["email"] == EMAIL + assert result["feature"] == "admin-ui-beta" + mock_client.add_user_feature.assert_called_once_with(EMAIL, "admin-ui-beta") + mock_client.close.assert_called_once() + + def test_dry_run_makes_no_client_call(self, store_with_master, manage_client_factory) -> None: + factory, mock_client = manage_client_factory + svc = FeatureService(store_with_master, manage_client_factory=factory) + + result = svc.add_user_feature( + manage_token=MANAGE_TOKEN, + alias=ALIAS, + email=EMAIL, + feature="admin-ui-beta", + dry_run=True, + ) + + assert result["status"] == "dry_run" + assert result["action"] == "add" + factory.assert_not_called() + mock_client.add_user_feature.assert_not_called() + + +class TestRemoveUserFeature: + def test_live_call(self, store_with_master, manage_client_factory) -> None: + factory, mock_client = manage_client_factory + svc = FeatureService(store_with_master, manage_client_factory=factory) + + result = svc.remove_user_feature( + manage_token=MANAGE_TOKEN, alias=ALIAS, email=EMAIL, feature="admin-ui-beta" + ) + + assert result["status"] == "removed" + assert result["email"] == EMAIL + mock_client.remove_user_feature.assert_called_once_with(EMAIL, "admin-ui-beta") + mock_client.close.assert_called_once() + + def test_dry_run_makes_no_client_call(self, store_with_master, manage_client_factory) -> None: + factory, mock_client = manage_client_factory + svc = FeatureService(store_with_master, manage_client_factory=factory) + + result = svc.remove_user_feature( + manage_token=MANAGE_TOKEN, + alias=ALIAS, + email=EMAIL, + feature="admin-ui-beta", + dry_run=True, + ) + + assert result["status"] == "dry_run" + assert result["action"] == "remove" + factory.assert_not_called() + mock_client.remove_user_feature.assert_not_called() + + +# ────────────────────────────────────────────────────────────────────── +# close() on error +# ────────────────────────────────────────────────────────────────────── + + +class TestCloseOnError: + def test_close_fires_even_when_client_raises( + self, store_with_master, manage_client_factory + ) -> None: + factory, mock_client = manage_client_factory + mock_client.list_features.side_effect = RuntimeError("boom") + svc = FeatureService(store_with_master, manage_client_factory=factory) + + with pytest.raises(RuntimeError, match="boom"): + svc.list_stack_features(manage_token=MANAGE_TOKEN, alias=ALIAS) + mock_client.close.assert_called_once() diff --git a/tests/test_helpers.py b/tests/test_helpers.py index fca52476..651aa6b0 100644 --- a/tests/test_helpers.py +++ b/tests/test_helpers.py @@ -1,6 +1,7 @@ """Tests for commands._helpers shared command-layer utilities.""" import pytest +import typer from keboola_agent_cli.commands._helpers import map_error_to_exit_code from keboola_agent_cli.errors import KeboolaApiError, map_error_code_to_type @@ -646,3 +647,56 @@ def test_allow_env_with_unset_env_and_no_tty_exits_2( assert "Run interactively" in err # No phantom warning when env was actually empty. assert "found in environment but ignored" not in err + + +class TestRequireRandomCodeConfirmation: + def test_non_tty_exits_with_permission_denied(self, monkeypatch): + # stdin isatty -> False + monkeypatch.setattr("sys.stdin.isatty", lambda: False) + with pytest.raises(typer.Exit) as exc: + from keboola_agent_cli.commands._helpers import require_random_code_confirmation + + require_random_code_confirmation("delete the universe") + assert exc.value.exit_code == 6 # EXIT_PERMISSION_DENIED + + def test_correct_code_accepted(self, monkeypatch): + monkeypatch.setattr("sys.stdin.isatty", lambda: True) + monkeypatch.setattr( + "keboola_agent_cli.commands._helpers.secrets.token_hex", + lambda n: "deadbeef", + ) + monkeypatch.setattr("builtins.input", lambda: "deadbeef") + from keboola_agent_cli.commands._helpers import require_random_code_confirmation + + # Returns None on success + assert require_random_code_confirmation("patch app") is None + + def test_wrong_code_exits(self, monkeypatch): + monkeypatch.setattr("sys.stdin.isatty", lambda: True) + monkeypatch.setattr( + "keboola_agent_cli.commands._helpers.secrets.token_hex", + lambda n: "deadbeef", + ) + monkeypatch.setattr("builtins.input", lambda: "wrongcode") + with pytest.raises(typer.Exit) as exc: + from keboola_agent_cli.commands._helpers import require_random_code_confirmation + + require_random_code_confirmation("patch app") + assert exc.value.exit_code == 6 + + def test_eof_exits(self, monkeypatch): + monkeypatch.setattr("sys.stdin.isatty", lambda: True) + monkeypatch.setattr( + "keboola_agent_cli.commands._helpers.secrets.token_hex", + lambda n: "deadbeef", + ) + + def raise_eof(): + raise EOFError + + monkeypatch.setattr("builtins.input", raise_eof) + with pytest.raises(typer.Exit) as exc: + from keboola_agent_cli.commands._helpers import require_random_code_confirmation + + require_random_code_confirmation("patch app") + assert exc.value.exit_code == 6 diff --git a/tests/test_manage_client.py b/tests/test_manage_client.py index b6d11d84..6f52fe56 100644 --- a/tests/test_manage_client.py +++ b/tests/test_manage_client.py @@ -592,3 +592,225 @@ def test_uses_PATCH_not_PUT(self, httpx_mock) -> None: assert request.method == "PATCH" assert _json.loads(request.read()) == {"role": "guest"} assert result["role"] == "guest" + + +# ────────────────────────────────────────────────────────────────────── +# Feature flags (super-admin manage token required) +# ────────────────────────────────────────────────────────────────────── + + +_FEATURES_RESPONSE = [ + { + "id": 1, + "name": "queuev2", + "title": "Queue v2", + "description": "New job queue", + "type": "project", + "canBeManagedViaApi": True, + }, + { + "id": 2, + "name": "data-apps", + "title": "Data Apps", + "type": "admin", + }, +] + + +class TestListFeatures: + def test_returns_catalogue_list(self, httpx_mock) -> None: + httpx_mock.add_response( + url=f"{STACK_URL}/manage/features", + method="GET", + json=_FEATURES_RESPONSE, + status_code=200, + ) + with ManageClient(stack_url=STACK_URL, manage_token=MANAGE_TOKEN) as client: + result = client.list_features() + assert isinstance(result, list) + assert len(result) == 2 + assert result[0]["name"] == "queuev2" + # Unknown/extra keys round-trip untouched. + assert result[0]["canBeManagedViaApi"] is True + + def test_403_without_super_admin(self, httpx_mock) -> None: + httpx_mock.add_response( + url=f"{STACK_URL}/manage/features", + method="GET", + json={"error": "Super admin required"}, + status_code=403, + ) + with ( + ManageClient(stack_url=STACK_URL, manage_token=MANAGE_TOKEN) as client, + pytest.raises(KeboolaApiError) as exc_info, + ): + client.list_features() + assert exc_info.value.error_code == "ACCESS_DENIED" + assert exc_info.value.status_code == 403 + + +class TestAddProjectFeature: + def test_success_returns_body(self, httpx_mock) -> None: + httpx_mock.add_response( + url=f"{STACK_URL}/manage/projects/5725/features", + method="POST", + json={"feature": "queuev2", "added": True}, + status_code=201, + ) + with ManageClient(stack_url=STACK_URL, manage_token=MANAGE_TOKEN) as client: + result = client.add_project_feature(5725, "queuev2") + assert result["feature"] == "queuev2" + + def test_payload_is_feature_object(self, httpx_mock) -> None: + import json as _json + + httpx_mock.add_response( + url=f"{STACK_URL}/manage/projects/5725/features", + method="POST", + json={}, + status_code=200, + ) + with ManageClient(stack_url=STACK_URL, manage_token=MANAGE_TOKEN) as client: + client.add_project_feature(5725, "queuev2") + request = httpx_mock.get_request() + assert request.method == "POST" + assert _json.loads(request.read()) == {"feature": "queuev2"} + + def test_404_unknown_project(self, httpx_mock) -> None: + httpx_mock.add_response( + url=f"{STACK_URL}/manage/projects/999/features", + method="POST", + json={"error": "Project not found"}, + status_code=404, + ) + with ( + ManageClient(stack_url=STACK_URL, manage_token=MANAGE_TOKEN) as client, + pytest.raises(KeboolaApiError) as exc_info, + ): + client.add_project_feature(999, "queuev2") + assert exc_info.value.error_code == "NOT_FOUND" + + +class TestRemoveProjectFeature: + def test_returns_none_on_204(self, httpx_mock) -> None: + httpx_mock.add_response( + url=f"{STACK_URL}/manage/projects/5725/features/queuev2", + method="DELETE", + status_code=204, + ) + with ManageClient(stack_url=STACK_URL, manage_token=MANAGE_TOKEN) as client: + assert client.remove_project_feature(5725, "queuev2") is None + + def test_url_encodes_feature_name(self, httpx_mock) -> None: + """A feature name with reserved characters is fully percent-encoded + (quote(..., safe='')), so '/' and ' ' become %2F and %20.""" + httpx_mock.add_response( + url=f"{STACK_URL}/manage/projects/5725/features/vendor%2Ffeat%20flag", + method="DELETE", + status_code=204, + ) + with ManageClient(stack_url=STACK_URL, manage_token=MANAGE_TOKEN) as client: + assert client.remove_project_feature(5725, "vendor/feat flag") is None + assert "vendor%2Ffeat%20flag" in str(httpx_mock.get_request().url) + + +class TestGetUser: + def test_success(self, httpx_mock) -> None: + httpx_mock.add_response( + url=f"{STACK_URL}/manage/users/jane@example.com", + method="GET", + json={ + "id": 42, + "email": "jane@example.com", + "features": ["queuev2", "data-apps"], + }, + status_code=200, + ) + with ManageClient(stack_url=STACK_URL, manage_token=MANAGE_TOKEN) as client: + result = client.get_user("jane@example.com") + assert result["id"] == 42 + assert result["email"] == "jane@example.com" + assert result["features"] == ["queuev2", "data-apps"] + + def test_url_keeps_at_and_dot_but_encodes_plus(self, httpx_mock) -> None: + """Email is quote(email, safe='@'): '@' and '.' stay literal, but + sub-address '+' is percent-encoded to %2B.""" + httpx_mock.add_response( + url=f"{STACK_URL}/manage/users/jane%2Btag@example.com", + method="GET", + json={"id": 7, "email": "jane+tag@example.com", "features": []}, + status_code=200, + ) + with ManageClient(stack_url=STACK_URL, manage_token=MANAGE_TOKEN) as client: + result = client.get_user("jane+tag@example.com") + url = str(httpx_mock.get_request().url) + assert "jane%2Btag@example.com" in url + assert result["id"] == 7 + + def test_404_unknown_user(self, httpx_mock) -> None: + httpx_mock.add_response( + url=f"{STACK_URL}/manage/users/nobody@example.com", + method="GET", + json={"error": "User not found"}, + status_code=404, + ) + with ( + ManageClient(stack_url=STACK_URL, manage_token=MANAGE_TOKEN) as client, + pytest.raises(KeboolaApiError) as exc_info, + ): + client.get_user("nobody@example.com") + assert exc_info.value.error_code == "NOT_FOUND" + + +class TestAddUserFeature: + def test_success_returns_body(self, httpx_mock) -> None: + httpx_mock.add_response( + url=f"{STACK_URL}/manage/users/jane@example.com/features", + method="POST", + json={"feature": "queuev2", "added": True}, + status_code=201, + ) + with ManageClient(stack_url=STACK_URL, manage_token=MANAGE_TOKEN) as client: + result = client.add_user_feature("jane@example.com", "queuev2") + assert result["feature"] == "queuev2" + + def test_payload_and_encoded_url(self, httpx_mock) -> None: + import json as _json + + httpx_mock.add_response( + url=f"{STACK_URL}/manage/users/jane%2Btag@example.com/features", + method="POST", + json={}, + status_code=200, + ) + with ManageClient(stack_url=STACK_URL, manage_token=MANAGE_TOKEN) as client: + client.add_user_feature("jane+tag@example.com", "queuev2") + request = httpx_mock.get_request() + assert request.method == "POST" + assert _json.loads(request.read()) == {"feature": "queuev2"} + assert "jane%2Btag@example.com" in str(request.url) + + +class TestRemoveUserFeature: + def test_returns_none_on_204(self, httpx_mock) -> None: + httpx_mock.add_response( + url=f"{STACK_URL}/manage/users/jane@example.com/features/queuev2", + method="DELETE", + status_code=204, + ) + with ManageClient(stack_url=STACK_URL, manage_token=MANAGE_TOKEN) as client: + assert client.remove_user_feature("jane@example.com", "queuev2") is None + + def test_encodes_both_email_and_feature(self, httpx_mock) -> None: + """Email keeps '@'/'.' (safe='@') while the feature is fully encoded + (safe='').""" + httpx_mock.add_response( + url=(f"{STACK_URL}/manage/users/jane%2Btag@example.com/features/vendor%2Fflag"), + method="DELETE", + status_code=204, + ) + with ManageClient(stack_url=STACK_URL, manage_token=MANAGE_TOKEN) as client: + assert client.remove_user_feature("jane+tag@example.com", "vendor/flag") is None + url = str(httpx_mock.get_request().url) + assert "jane%2Btag@example.com" in url + assert "vendor%2Fflag" in url diff --git a/tests/test_metastore_client.py b/tests/test_metastore_client.py index c29875f4..290bd059 100644 --- a/tests/test_metastore_client.py +++ b/tests/test_metastore_client.py @@ -247,7 +247,7 @@ def test_delete_404(self, httpx_mock) -> None: class TestSemanticTypes: - """Sanity-check the SEMANTIC_TYPES tuple has the six expected slugs.""" + """Sanity-check the SEMANTIC_TYPES tuple has the expected slugs.""" def test_semantic_types_complete(self) -> None: assert set(SEMANTIC_TYPES) == { @@ -257,4 +257,5 @@ def test_semantic_types_complete(self) -> None: "semantic-relationship", "semantic-constraint", "semantic-glossary", + "semantic-reference-data", } diff --git a/tests/test_models.py b/tests/test_models.py index e1690281..20cf4404 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -8,6 +8,7 @@ from keboola_agent_cli.models import ( AppConfig, ErrorResponse, + Feature, ProjectConfig, SuccessResponse, TokenVerifyResponse, @@ -274,13 +275,50 @@ def test_project_add_rejects_ftp_url(self) -> None: token="901-token", ) - def test_project_add_rejects_no_scheme(self) -> None: - """URL without scheme is rejected with a ValidationError.""" - with pytest.raises(ValidationError, match="https://"): - ProjectConfig( - stack_url="connection.keboola.com", - token="901-token", - ) + def test_bare_host_is_normalized_to_https(self) -> None: + """A bare host (no scheme) gets https:// prepended instead of rejected.""" + config = ProjectConfig( + stack_url="connection.keboola.com", + token="901-token", + ) + assert config.stack_url == "https://connection.keboola.com" + + def test_full_project_link_reduced_to_base(self) -> None: + """A full project deep-link is reduced to scheme+host.""" + config = ProjectConfig( + stack_url="https://connection.keboola.com/admin/projects/10105/dashboard", + token="901-token", + ) + assert config.stack_url == "https://connection.keboola.com" + + def test_trailing_slash_stripped(self) -> None: + """A trailing slash is dropped from the normalized base URL.""" + config = ProjectConfig( + stack_url="https://connection.keboola.com/", + token="901-token", + ) + assert config.stack_url == "https://connection.keboola.com" + + def test_bare_host_with_path_reduced_to_base(self) -> None: + """A bare host + path (no scheme) normalizes to https://.""" + config = ProjectConfig( + stack_url="connection.north-europe.azure.keboola.com/admin/projects/7", + token="901-token", + ) + assert config.stack_url == "https://connection.north-europe.azure.keboola.com" + + def test_surrounding_whitespace_trimmed(self) -> None: + """Leading/trailing whitespace (paste artifact) is trimmed.""" + config = ProjectConfig( + stack_url=" https://connection.keboola.com ", + token="901-token", + ) + assert config.stack_url == "https://connection.keboola.com" + + def test_empty_url_rejected(self) -> None: + """An empty / whitespace-only URL is rejected.""" + with pytest.raises(ValidationError, match="empty"): + ProjectConfig(stack_url=" ", token="901-token") def test_project_add_accepts_https_url(self) -> None: """https:// URL is accepted without error.""" @@ -421,6 +459,67 @@ def test_max_workers_negative_rejected(self) -> None: AppConfig(max_parallel_workers=-5) +class TestFeature: + """Tests for the Feature model (Keboola feature flag).""" + + def test_full_object_with_extras_passes_through(self) -> None: + """A feature dict with known fields plus unknown extras validates and + keeps the extras (model_config extra='allow').""" + feature = Feature.model_validate( + { + "name": "queuev2", + "title": "Queue v2", + "description": "New job queue", + "type": "project", + "canBeManagedViaApi": True, + "id": 1, + } + ) + assert feature.name == "queuev2" + assert feature.title == "Queue v2" + assert feature.description == "New job queue" + assert feature.type == "project" + dumped = feature.model_dump() + # Extra keys survive serialization untouched. + assert dumped["canBeManagedViaApi"] is True + assert dumped["id"] == 1 + + def test_defaults_when_empty_dict(self) -> None: + """An empty dict yields safe empty-string defaults for every field.""" + feature = Feature.model_validate({}) + assert feature.name == "" + assert feature.title == "" + assert feature.description == "" + assert feature.type == "" + + def test_minimal_name_only(self) -> None: + """The common normalized shape {'name': } validates and the + other fields fall back to their defaults.""" + feature = Feature.model_validate({"name": "data-apps"}) + assert feature.name == "data-apps" + assert feature.title == "" + assert feature.type == "" + + def test_model_dump_includes_declared_and_extra_fields(self) -> None: + """model_dump emits the declared fields plus any extras.""" + feature = Feature.model_validate({"name": "x", "title": "X", "adminFeature": False}) + dumped = feature.model_dump() + assert dumped["name"] == "x" + assert dumped["title"] == "X" + assert dumped["description"] == "" + assert dumped["type"] == "" + assert dumped["adminFeature"] is False + + def test_json_round_trip_preserves_extras(self) -> None: + """Feature survives a JSON round-trip including extra keys.""" + original = Feature.model_validate( + {"name": "queuev2", "title": "Queue v2", "projectFeature": True} + ) + restored = Feature.model_validate_json(original.model_dump_json()) + assert restored.name == "queuev2" + assert restored.model_dump()["projectFeature"] is True + + class TestProjectConfigBackwardCompat: """Tests for backward compatibility of ProjectConfig with active_branch_id.""" @@ -446,3 +545,99 @@ def test_project_config_with_active_branch_id(self) -> None: } config = ProjectConfig.model_validate(data) assert config.active_branch_id == 123 + + +class TestDeveloperPortalIdentity: + """Tests for DeveloperPortalIdentity model.""" + + def test_minimal_construction(self) -> None: + """DeveloperPortalIdentity can be created with minimal required fields.""" + from keboola_agent_cli.models import DeveloperPortalIdentity + + ident = DeveloperPortalIdentity(username="service.keboola.x", password="p") + assert ident.username == "service.keboola.x" + assert ident.password == "p" + assert ident.role_hint == "vendor" + assert ident.vendor is None + assert ident.portal_url == "https://apps-api.keboola.com" + + def test_rejects_non_https_portal_url(self) -> None: + """DeveloperPortalIdentity rejects non-https portal_url.""" + from keboola_agent_cli.models import DeveloperPortalIdentity + + with pytest.raises(ValidationError, match="https"): + DeveloperPortalIdentity( + username="u", + password="p", + portal_url="http://apps-api.keboola.com", + ) + + def test_accepts_staging_https_portal_url(self) -> None: + """DeveloperPortalIdentity accepts https staging URL.""" + from keboola_agent_cli.models import DeveloperPortalIdentity + + ident = DeveloperPortalIdentity( + username="u", + password="p", + portal_url="https://apps-api.staging.keboola.dev", + ) + assert ident.portal_url == "https://apps-api.staging.keboola.dev" + + def test_role_hint_accepts_admin(self) -> None: + from keboola_agent_cli.models import DeveloperPortalIdentity + + ident = DeveloperPortalIdentity(username="u", password="p", role_hint="admin") + assert ident.role_hint == "admin" + + def test_role_hint_normalises_case(self) -> None: + from keboola_agent_cli.models import DeveloperPortalIdentity + + ident = DeveloperPortalIdentity(username="u", password="p", role_hint="ADMIN") + assert ident.role_hint == "admin" + + def test_role_hint_typo_downgrades_to_vendor_with_warning(self, capsys) -> None: + """Typos do NOT raise: pre-0.51.1 configs had free-text values, so we + normalise unknown strings to 'vendor' with a stderr warning to keep + ConfigStore.load() from crashing the CLI on upgrade.""" + from keboola_agent_cli.models import DeveloperPortalIdentity + + ident = DeveloperPortalIdentity(username="u", password="p", role_hint="vendr") + assert ident.role_hint == "vendor" + captured = capsys.readouterr() + assert "role_hint" in captured.err + assert "downgrading" in captured.err + + def test_legacy_freetext_role_hint_loads_cleanly(self, capsys) -> None: + """Backwards compat: a config.json carrying any free-text role_hint + (allowed pre-0.51.1) must round-trip through Pydantic without raising. + Empty strings, hand-edited values, even non-string types get normalised.""" + from keboola_agent_cli.models import DeveloperPortalIdentity + + for legacy in ("keboola-admin", "", " ADMIN ", 42): + ident = DeveloperPortalIdentity.model_validate( + {"username": "u", "password": "p", "role_hint": legacy} + ) + assert ident.role_hint in ("vendor", "admin") + + +class TestAppConfigDevPortalFields: + """Tests for AppConfig dev_portal_identities and default_dev_portal_identity fields.""" + + def test_defaults_empty(self) -> None: + """AppConfig dev_portal_identities and default_dev_portal_identity default to empty.""" + cfg = AppConfig() + assert cfg.dev_portal_identities == {} + assert cfg.default_dev_portal_identity == "" + + def test_round_trip(self) -> None: + """AppConfig with dev portal identities round-trips through JSON.""" + from keboola_agent_cli.models import DeveloperPortalIdentity + + ident = DeveloperPortalIdentity(username="u", password="p", vendor="keboola") + cfg = AppConfig( + dev_portal_identities={"vendor-keboola": ident}, + default_dev_portal_identity="vendor-keboola", + ) + round_trip = AppConfig.model_validate(cfg.model_dump(mode="json")) + assert round_trip.dev_portal_identities["vendor-keboola"].vendor == "keboola" + assert round_trip.default_dev_portal_identity == "vendor-keboola" diff --git a/tests/test_permissions.py b/tests/test_permissions.py index c598c4c1..0992d470 100644 --- a/tests/test_permissions.py +++ b/tests/test_permissions.py @@ -1,5 +1,7 @@ """Tests for the permission engine (OPERATION_REGISTRY, PermissionEngine, classify_mcp_tool).""" +from typing import ClassVar + import pytest from keboola_agent_cli.errors import PermissionDeniedError @@ -370,3 +372,41 @@ def test_defaults(self) -> None: assert policy.mode == "allow" assert policy.allow == [] assert policy.deny == [] + + +class TestDevPortalPermissions: + """Keys reflect actual Typer paths: `dev-portal.` for the top-level + sub-app, `dev-portal.identity.` for the identity sub-Typer. Both + sub-apps carry callbacks that compose those keys via check_cli_permission. + + Categories follow the data-app.secrets-* precedent: credential add/edit are + `write` (not `admin`); admin is reserved for org-level ops. Publish is + `admin` (requests Keboola review), deprecate is `destructive` (hides app). + """ + + DP_OPS: ClassVar[dict[str, str]] = { + # parent descent + "dev-portal.identity": "read", + # identity sub-app leaves + "dev-portal.identity.add": "write", + "dev-portal.identity.list": "read", + "dev-portal.identity.edit": "write", + "dev-portal.identity.remove": "write", + "dev-portal.identity.use": "write", + "dev-portal.identity.current": "read", + "dev-portal.identity.verify": "read", + # top-level dev-portal commands + "dev-portal.list": "read", + "dev-portal.get": "read", + "dev-portal.create": "write", + "dev-portal.patch": "write", + "dev-portal.upload-icon": "write", + "dev-portal.publish": "admin", + "dev-portal.deprecate": "destructive", + } + + def test_registry_contains_all_dev_portal_ops(self): + from keboola_agent_cli.permissions import OPERATION_REGISTRY + + for op, expected_cat in self.DP_OPS.items(): + assert OPERATION_REGISTRY.get(op) == expected_cat, op diff --git a/tests/test_permissions_cli.py b/tests/test_permissions_cli.py index e307f445..b55cee30 100644 --- a/tests/test_permissions_cli.py +++ b/tests/test_permissions_cli.py @@ -4,6 +4,7 @@ from pathlib import Path from unittest.mock import patch +import typer from typer.testing import CliRunner from keboola_agent_cli.cli import app @@ -203,8 +204,8 @@ def test_set_deny_mode(self, tmp_path: Path) -> None: with ( patch("keboola_agent_cli.cli.ConfigStore") as MockStore, patch( - "keboola_agent_cli.commands.permissions._require_interactive_confirmation", - return_value=True, + "keboola_agent_cli.commands.permissions.require_random_code_confirmation", + return_value=None, ), ): MockStore.return_value = store @@ -238,8 +239,8 @@ def test_set_allow_mode_with_deny(self, tmp_path: Path) -> None: with ( patch("keboola_agent_cli.cli.ConfigStore") as MockStore, patch( - "keboola_agent_cli.commands.permissions._require_interactive_confirmation", - return_value=True, + "keboola_agent_cli.commands.permissions.require_random_code_confirmation", + return_value=None, ), ): MockStore.return_value = store @@ -268,8 +269,8 @@ def test_set_rejected_without_confirmation(self, tmp_path: Path) -> None: with ( patch("keboola_agent_cli.cli.ConfigStore") as MockStore, patch( - "keboola_agent_cli.commands.permissions._require_interactive_confirmation", - return_value=False, + "keboola_agent_cli.commands.permissions.require_random_code_confirmation", + side_effect=typer.Exit(code=EXIT_PERMISSION_DENIED), ), ): MockStore.return_value = store @@ -293,8 +294,8 @@ def test_reset_removes_policy(self, tmp_path: Path) -> None: with ( patch("keboola_agent_cli.cli.ConfigStore") as MockStore, patch( - "keboola_agent_cli.commands.permissions._require_interactive_confirmation", - return_value=True, + "keboola_agent_cli.commands.permissions.require_random_code_confirmation", + return_value=None, ), ): MockStore.return_value = store @@ -310,8 +311,8 @@ def test_reset_rejected_without_confirmation(self, tmp_path: Path) -> None: with ( patch("keboola_agent_cli.cli.ConfigStore") as MockStore, patch( - "keboola_agent_cli.commands.permissions._require_interactive_confirmation", - return_value=False, + "keboola_agent_cli.commands.permissions.require_random_code_confirmation", + side_effect=typer.Exit(code=EXIT_PERMISSION_DENIED), ), ): MockStore.return_value = store diff --git a/tests/test_semantic_layer_cli.py b/tests/test_semantic_layer_cli.py index 9b7ddb74..88145600 100644 --- a/tests/test_semantic_layer_cli.py +++ b/tests/test_semantic_layer_cli.py @@ -1610,3 +1610,369 @@ def test_model_delete_denied_with_deny_destructive(self, store: ConfigStore) -> ) assert result.exit_code == 6, result.output assert "PERMISSION_DENIED" in result.output + + +# --------------------------------------------------------------------------- +# semantic-layer search-context / get-context (v0.47.0) +# --------------------------------------------------------------------------- + + +class TestSearchContext: + """CLI surface for the project-wide name-pattern search.""" + + def test_default_pattern_json(self, store: ConfigStore) -> None: + mock = MagicMock() + mock.search_context.return_value = { + "project": "prod", + "contexts": [ + { + "id": "d1", + "type": "dataset", + "name": "users", + "description": "", + "attributes": {"name": "users"}, + } + ], + "total_count": 1, + } + result = _invoke( + ["--json", "semantic-layer", "search-context", "--project", "prod"], + store=store, + sl_mock=mock, + ) + assert result.exit_code == 0, result.output + body = json.loads(result.output) + assert body["data"]["total_count"] == 1 + assert body["data"]["contexts"][0]["type"] == "dataset" + # Default pattern propagates to the service. + call_kwargs = mock.search_context.call_args.kwargs + assert call_kwargs["patterns"] == ["*"] + + def test_pattern_and_type_filter_propagate(self, store: ConfigStore) -> None: + mock = MagicMock() + mock.search_context.return_value = { + "project": "prod", + "contexts": [], + "total_count": 0, + } + result = _invoke( + [ + "--json", + "semantic-layer", + "search-context", + "--project", + "prod", + "--pattern", + "DIM_*", + "--pattern", + "FACT_*", + "--type", + "dataset", + "--limit", + "5", + ], + store=store, + sl_mock=mock, + ) + assert result.exit_code == 0, result.output + call_kwargs = mock.search_context.call_args.kwargs + assert call_kwargs["alias"] == "prod" + assert call_kwargs["patterns"] == ["DIM_*", "FACT_*"] + assert call_kwargs["type_filter"] == "dataset" + assert call_kwargs["limit"] == 5 + + def test_human_renders_table(self, store: ConfigStore) -> None: + mock = MagicMock() + mock.search_context.return_value = { + "project": "prod", + "contexts": [ + { + "id": "m1", + "type": "metric", + "name": "revenue", + "description": "GMV", + "attributes": {}, + } + ], + "total_count": 1, + } + result = _invoke( + ["semantic-layer", "search-context", "--project", "prod"], + store=store, + sl_mock=mock, + ) + assert result.exit_code == 0 + assert "revenue" in result.output + + def test_service_error_maps_to_exit_1(self, store: ConfigStore) -> None: + from keboola_agent_cli.errors import ErrorCode, KeboolaApiError + + mock = MagicMock() + mock.search_context.side_effect = KeboolaApiError( + message="Invalid type", error_code=ErrorCode.VALIDATION_ERROR + ) + result = _invoke( + [ + "--json", + "semantic-layer", + "search-context", + "--project", + "prod", + "--type", + "bogus", + ], + store=store, + sl_mock=mock, + ) + assert result.exit_code == 1, result.output + + +class TestGetContext: + """CLI surface for the single-id semantic context lookup.""" + + def test_get_context_json(self, store: ConfigStore) -> None: + mock = MagicMock() + mock.get_context.return_value = { + "project": "prod", + "id": "u-model", + "type": "model", + "name": "default", + "description": "", + "attributes": {"name": "default", "sql_dialect": "Snowflake"}, + } + result = _invoke( + [ + "--json", + "semantic-layer", + "get-context", + "--project", + "prod", + "--context-id", + "u-model", + ], + store=store, + sl_mock=mock, + ) + assert result.exit_code == 0, result.output + body = json.loads(result.output) + assert body["data"]["id"] == "u-model" + assert body["data"]["type"] == "model" + + def test_get_context_not_found_exits_nonzero(self, store: ConfigStore) -> None: + from keboola_agent_cli.errors import ErrorCode, KeboolaApiError + + mock = MagicMock() + mock.get_context.side_effect = KeboolaApiError( + message="not found", status_code=404, error_code=ErrorCode.NOT_FOUND + ) + result = _invoke( + [ + "--json", + "semantic-layer", + "get-context", + "--project", + "prod", + "--context-id", + "ghost", + ], + store=store, + sl_mock=mock, + ) + assert result.exit_code != 0 + assert "not found" in result.output.lower() or "NOT_FOUND" in result.output + + def test_get_context_human_renders_attributes(self, store: ConfigStore) -> None: + mock = MagicMock() + mock.get_context.return_value = { + "project": "prod", + "id": "d1", + "type": "dataset", + "name": "users", + "description": "User dimension table", + "attributes": {"name": "users", "primary_key": ["id"]}, + } + result = _invoke( + [ + "semantic-layer", + "get-context", + "--project", + "prod", + "--context-id", + "d1", + ], + store=store, + sl_mock=mock, + ) + assert result.exit_code == 0 + assert "users" in result.output + assert "dataset" in result.output + + +# --------------------------------------------------------------------------- +# semantic-layer reference-data (list / get / set / delete) +# --------------------------------------------------------------------------- + + +class TestReferenceDataList: + def test_json_success(self, store: ConfigStore) -> None: + mock = MagicMock() + mock.list_reference_data.return_value = { + "project": "prod", + "reference_data": [ + { + "id": "r1", + "dimension_name": "chart_of_accounts", + "model_uuid": "U", + "dataset_id": "in.c-f.DIM_COA", + "member_count": 3, + } + ], + } + result = _invoke( + ["--json", "semantic-layer", "reference-data", "list", "--project", "prod"], + store=store, + sl_mock=mock, + ) + assert result.exit_code == 0, result.output + body = json.loads(result.output) + assert body["data"]["reference_data"][0]["dimension_name"] == "chart_of_accounts" + mock.list_reference_data.assert_called_once() + + +class TestReferenceDataGet: + def test_by_id(self, store: ConfigStore) -> None: + mock = MagicMock() + mock.get_reference_data.return_value = { + "project": "prod", + "id": "r1", + "dimension_name": "chart_of_accounts", + "model_uuid": "U", + "member_count": 1, + "revision": 2, + "members": [{"account_code": "4011", "account_name": "Revenue"}], + } + result = _invoke( + [ + "--json", + "semantic-layer", + "reference-data", + "get", + "--project", + "prod", + "--id", + "r1", + ], + store=store, + sl_mock=mock, + ) + assert result.exit_code == 0, result.output + body = json.loads(result.output) + assert body["data"]["members"][0]["account_code"] == "4011" + _, kwargs = mock.get_reference_data.call_args + assert kwargs["record_id"] == "r1" + + +class TestReferenceDataSet: + def test_set_from_file(self, store: ConfigStore, tmp_path: Path) -> None: + members_file = tmp_path / "coa.json" + members_file.write_text(json.dumps([{"account_code": "4011", "account_name": "Revenue"}])) + mock = MagicMock() + mock.set_reference_data.return_value = { + "project": "prod", + "id": "r1", + "dimension_name": "chart_of_accounts", + "member_count": 1, + "action": "created", + } + result = _invoke( + [ + "--json", + "semantic-layer", + "reference-data", + "set", + "--project", + "prod", + "--dimension", + "chart_of_accounts", + "--members-file", + str(members_file), + ], + store=store, + sl_mock=mock, + ) + assert result.exit_code == 0, result.output + body = json.loads(result.output) + assert body["data"]["action"] == "created" + _, kwargs = mock.set_reference_data.call_args + assert kwargs["dimension"] == "chart_of_accounts" + assert kwargs["members"] == [{"account_code": "4011", "account_name": "Revenue"}] + + def test_set_bad_json_exits_2(self, store: ConfigStore, tmp_path: Path) -> None: + members_file = tmp_path / "coa.json" + members_file.write_text("{not valid json") + mock = MagicMock() + result = _invoke( + [ + "--json", + "semantic-layer", + "reference-data", + "set", + "--project", + "prod", + "--dimension", + "chart_of_accounts", + "--members-file", + str(members_file), + ], + store=store, + sl_mock=mock, + ) + assert result.exit_code == 2, result.output + mock.set_reference_data.assert_not_called() + + +class TestReferenceDataDelete: + def test_delete_requires_yes_non_tty(self, store: ConfigStore) -> None: + mock = MagicMock() + result = _invoke( + [ + "--json", + "semantic-layer", + "reference-data", + "delete", + "--project", + "prod", + "--id", + "r1", + ], + store=store, + sl_mock=mock, + ) + assert result.exit_code == 2, result.output + mock.delete_reference_data.assert_not_called() + + def test_delete_with_yes(self, store: ConfigStore) -> None: + mock = MagicMock() + mock.delete_reference_data.return_value = { + "project": "prod", + "removed": {"id": "r1", "dimension_name": "chart_of_accounts"}, + } + result = _invoke( + [ + "--json", + "semantic-layer", + "reference-data", + "delete", + "--project", + "prod", + "--id", + "r1", + "--yes", + ], + store=store, + sl_mock=mock, + ) + assert result.exit_code == 0, result.output + body = json.loads(result.output) + assert body["data"]["removed"]["id"] == "r1" + mock.delete_reference_data.assert_called_once() diff --git a/tests/test_semantic_layer_service.py b/tests/test_semantic_layer_service.py index 1b62e0dc..495c8198 100644 --- a/tests/test_semantic_layer_service.py +++ b/tests/test_semantic_layer_service.py @@ -78,8 +78,15 @@ def _make_service( *, metastore_mock: MagicMock | None = None, ) -> tuple[SemanticLayerService, MagicMock]: - """Wire a SemanticLayerService with a mocked metastore client factory.""" + """Wire a SemanticLayerService with a mocked metastore client factory. + + The mock supports the context-manager protocol (``__enter__`` returns + self, ``__exit__`` is a no-op) so the service-layer `with` blocks see + the same MagicMock body that tests configure side-effects on. + """ mock = metastore_mock or MagicMock() + mock.__enter__ = MagicMock(return_value=mock) + mock.__exit__ = MagicMock(return_value=False) service = SemanticLayerService( config_store=store, metastore_client_factory=lambda url, token: mock, @@ -256,7 +263,7 @@ def test_returns_shape(self, tmp_path: Path) -> None: "description": "first", "sql_dialect": "Snowflake", } - mock.close.assert_called_once() + mock.__exit__.assert_called_once() def test_empty_project(self, tmp_path: Path) -> None: store = _make_store(tmp_path) @@ -1928,7 +1935,11 @@ def test_classification_new_changed_identical(self, tmp_path: Path) -> None: store = _make_store_two(tmp_path) src_mock = MagicMock() + src_mock.__enter__ = MagicMock(return_value=src_mock) + src_mock.__exit__ = MagicMock(return_value=False) tgt_mock = MagicMock() + tgt_mock.__enter__ = MagicMock(return_value=tgt_mock) + tgt_mock.__exit__ = MagicMock(return_value=False) clients = {0: src_mock, 1: tgt_mock} call_idx = {"i": 0} @@ -1991,7 +2002,11 @@ def test_both_clients_closed_even_on_error(self, tmp_path: Path) -> None: store = _make_store_two(tmp_path) src_mock = MagicMock() + src_mock.__enter__ = MagicMock(return_value=src_mock) + src_mock.__exit__ = MagicMock(return_value=False) tgt_mock = MagicMock() + tgt_mock.__enter__ = MagicMock(return_value=tgt_mock) + tgt_mock.__exit__ = MagicMock(return_value=False) clients = {0: src_mock, 1: tgt_mock} call_idx = {"i": 0} @@ -2010,8 +2025,8 @@ def _factory(url: str, token: str) -> MagicMock: with pytest.raises(RuntimeError): service.promote_model(from_project="source", to_project="target") - src_mock.close.assert_called_once() - tgt_mock.close.assert_called_once() + src_mock.__exit__.assert_called_once() + tgt_mock.__exit__.assert_called_once() # --------------------------------------------------------------------------- @@ -2397,3 +2412,433 @@ def test_constraint_types_complete(self) -> None: def test_constraint_severities(self) -> None: assert set(CONSTRAINT_SEVERITIES) == {"error", "warning", "info"} + + +# --------------------------------------------------------------------------- +# search-context / get-context (v0.47.0) +# --------------------------------------------------------------------------- + + +class TestSearchContext: + """Project-wide name-pattern search across semantic-layer entities.""" + + def _setup( + self, tmp_path: Path, items_by_type: dict[str, list[dict[str, Any]]] + ) -> tuple[SemanticLayerService, MagicMock]: + store = _make_store(tmp_path) + service, mock = _make_service(store) + + def _list(item_type: str, model_uuid: str | None = None) -> list[dict[str, Any]]: + return items_by_type.get(item_type, []) + + mock.list_items.side_effect = _list + return service, mock + + def test_default_pattern_matches_everything(self, tmp_path: Path) -> None: + items = { + "semantic-dataset": [_child_item("semantic-dataset", "d1", {"name": "users"})], + "semantic-metric": [_child_item("semantic-metric", "m1", {"name": "revenue"})], + } + service, _ = self._setup(tmp_path, items) + + result = service.search_context("prod") + + assert result["project"] == "prod" + assert result["total_count"] == 2 + names = sorted(c["name"] for c in result["contexts"]) + assert names == ["revenue", "users"] + # Types are CLI-friendly singular (no "semantic-" prefix). + types = {c["type"] for c in result["contexts"]} + assert types == {"dataset", "metric"} + + def test_pattern_glob_narrows_results(self, tmp_path: Path) -> None: + items = { + "semantic-dataset": [ + _child_item("semantic-dataset", "d1", {"name": "DIM_users"}), + _child_item("semantic-dataset", "d2", {"name": "FACT_orders"}), + _child_item("semantic-dataset", "d3", {"name": "DIM_products"}), + ], + } + service, _ = self._setup(tmp_path, items) + + result = service.search_context("prod", patterns=["DIM_*"]) + + assert result["total_count"] == 2 + names = sorted(c["name"] for c in result["contexts"]) + assert names == ["DIM_products", "DIM_users"] + + def test_pattern_is_case_sensitive(self, tmp_path: Path) -> None: + items = { + "semantic-dataset": [ + _child_item("semantic-dataset", "d1", {"name": "DIM_x"}), + _child_item("semantic-dataset", "d2", {"name": "dim_y"}), + ], + } + service, _ = self._setup(tmp_path, items) + + upper = service.search_context("prod", patterns=["DIM_*"]) + lower = service.search_context("prod", patterns=["dim_*"]) + + assert [c["name"] for c in upper["contexts"]] == ["DIM_x"] + assert [c["name"] for c in lower["contexts"]] == ["dim_y"] + + def test_multiple_patterns_take_union(self, tmp_path: Path) -> None: + items = { + "semantic-dataset": [ + _child_item("semantic-dataset", "d1", {"name": "DIM_a"}), + _child_item("semantic-dataset", "d2", {"name": "FACT_b"}), + _child_item("semantic-dataset", "d3", {"name": "AGG_c"}), + ], + } + service, _ = self._setup(tmp_path, items) + + result = service.search_context("prod", patterns=["DIM_*", "FACT_*"]) + + assert result["total_count"] == 2 + assert {c["name"] for c in result["contexts"]} == {"DIM_a", "FACT_b"} + + def test_type_filter_narrows_to_one_kind(self, tmp_path: Path) -> None: + items = { + "semantic-dataset": [_child_item("semantic-dataset", "d1", {"name": "users"})], + "semantic-metric": [_child_item("semantic-metric", "m1", {"name": "revenue"})], + "semantic-relationship": [_child_item("semantic-relationship", "r1", {"name": "r"})], + "semantic-constraint": [_child_item("semantic-constraint", "c1", {"name": "c"})], + "semantic-glossary": [_child_item("semantic-glossary", "g1", {"name": "term"})], + } + service, mock = self._setup(tmp_path, items) + + result = service.search_context("prod", type_filter="metric") + + assert result["total_count"] == 1 + assert result["contexts"][0]["type"] == "metric" + called_types = {call.args[0] for call in mock.list_items.call_args_list} + assert called_types == {"semantic-metric"}, ( + "type_filter must short-circuit the per-type loop" + ) + + def test_type_filter_model(self, tmp_path: Path) -> None: + items = {"semantic-model": [_model_item("u1", "default")]} + service, mock = self._setup(tmp_path, items) + + result = service.search_context("prod", type_filter="model") + + assert result["total_count"] == 1 + assert result["contexts"][0]["type"] == "model" + assert mock.list_items.call_args_list[0].args[0] == "semantic-model" + + def test_type_filter_all_iterates_every_child(self, tmp_path: Path) -> None: + items = { + t: [_child_item(t, "x", {"name": "n"})] + for t in ( + "semantic-dataset", + "semantic-metric", + "semantic-relationship", + "semantic-constraint", + "semantic-glossary", + ) + } + service, mock = self._setup(tmp_path, items) + + result = service.search_context("prod", type_filter="all") + + assert result["total_count"] == 5 + called_types = {call.args[0] for call in mock.list_items.call_args_list} + assert called_types == set(items) + assert "semantic-model" not in called_types, ( + "type=all means every CHILD type, not the model itself" + ) + + def test_limit_caps_results_and_short_circuits(self, tmp_path: Path) -> None: + items = { + "semantic-dataset": [ + _child_item("semantic-dataset", f"d{i}", {"name": f"n{i}"}) for i in range(10) + ], + "semantic-metric": [ + _child_item("semantic-metric", "m1", {"name": "rev"}), + ], + } + service, mock = self._setup(tmp_path, items) + + result = service.search_context("prod", limit=3) + + assert result["total_count"] == 3 + # short-circuit: never reached semantic-metric + called_types = [call.args[0] for call in mock.list_items.call_args_list] + assert "semantic-metric" not in called_types + + def test_invalid_type_filter_rejected(self, tmp_path: Path) -> None: + service, _ = self._setup(tmp_path, {}) + + with pytest.raises(KeboolaApiError) as excinfo: + service.search_context("prod", type_filter="bogus") + + assert excinfo.value.error_code == ErrorCode.VALIDATION_ERROR + + def test_empty_pattern_rejected(self, tmp_path: Path) -> None: + service, _ = self._setup(tmp_path, {}) + + with pytest.raises(KeboolaApiError) as excinfo: + service.search_context("prod", patterns=["valid", ""]) + + assert excinfo.value.error_code == ErrorCode.VALIDATION_ERROR + + def test_zero_limit_rejected(self, tmp_path: Path) -> None: + service, _ = self._setup(tmp_path, {}) + + with pytest.raises(KeboolaApiError) as excinfo: + service.search_context("prod", limit=0) + + assert excinfo.value.error_code == ErrorCode.VALIDATION_ERROR + + def test_client_closed_even_on_api_error(self, tmp_path: Path) -> None: + """try/finally must close the client even when list_items raises.""" + store = _make_store(tmp_path) + service, mock = _make_service(store) + mock.list_items.side_effect = KeboolaApiError( + message="boom", status_code=500, error_code=ErrorCode.API_ERROR + ) + + with pytest.raises(KeboolaApiError): + service.search_context("prod") + + mock.__exit__.assert_called_once() + + +class TestGetContext: + """Single-id lookup across every semantic type.""" + + def _setup(self, tmp_path: Path) -> tuple[SemanticLayerService, MagicMock]: + return _make_service(_make_store(tmp_path)) + + def test_finds_dataset(self, tmp_path: Path) -> None: + service, mock = self._setup(tmp_path) + + def _get(item_type: str, item_id: str) -> dict[str, Any]: + if item_type == "semantic-dataset" and item_id == "d1": + return _child_item("semantic-dataset", "d1", {"name": "users"}) + raise KeboolaApiError(message="404", status_code=404, error_code=ErrorCode.NOT_FOUND) + + mock.get_item.side_effect = _get + result = service.get_context("prod", "d1") + + assert result["id"] == "d1" + assert result["type"] == "dataset" + assert result["name"] == "users" + + def test_finds_model(self, tmp_path: Path) -> None: + """Lookup probes model first; a model hit short-circuits the scan.""" + service, mock = self._setup(tmp_path) + mock.get_item.return_value = _model_item("u-model", "default") + + result = service.get_context("prod", "u-model") + + assert result["type"] == "model" + # Only one client call needed when model is the first probe. + assert mock.get_item.call_count == 1 + assert mock.get_item.call_args.args[0] == "semantic-model" + + def test_missing_id_raises_not_found(self, tmp_path: Path) -> None: + service, mock = self._setup(tmp_path) + mock.get_item.side_effect = KeboolaApiError( + message="404", status_code=404, error_code=ErrorCode.NOT_FOUND + ) + + with pytest.raises(KeboolaApiError) as excinfo: + service.get_context("prod", "missing") + + assert excinfo.value.error_code == ErrorCode.NOT_FOUND + # Probed every type before giving up: model + 5 child types = 6 calls. + assert mock.get_item.call_count == 6 + + def test_non_404_error_propagates_immediately(self, tmp_path: Path) -> None: + """A 500 on one type must surface as-is, not be swallowed.""" + service, mock = self._setup(tmp_path) + mock.get_item.side_effect = KeboolaApiError( + message="boom", status_code=500, error_code=ErrorCode.API_ERROR + ) + + with pytest.raises(KeboolaApiError) as excinfo: + service.get_context("prod", "x") + + assert excinfo.value.error_code == ErrorCode.API_ERROR + + def test_empty_id_rejected(self, tmp_path: Path) -> None: + service, _ = self._setup(tmp_path) + + with pytest.raises(KeboolaApiError) as excinfo: + service.get_context("prod", "") + + assert excinfo.value.error_code == ErrorCode.VALIDATION_ERROR + + def test_client_closed_even_on_api_error(self, tmp_path: Path) -> None: + service, mock = self._setup(tmp_path) + mock.get_item.side_effect = KeboolaApiError( + message="500", status_code=500, error_code=ErrorCode.API_ERROR + ) + + with pytest.raises(KeboolaApiError): + service.get_context("prod", "x") + + mock.__exit__.assert_called_once() + + +# --------------------------------------------------------------------------- +# reference-data (dimension-member records, e.g. Chart of Accounts) +# --------------------------------------------------------------------------- + + +def _refdata_item( + item_id: str, + dimension: str, + model_uuid: str = "U", + members: list[dict[str, Any]] | None = None, + revision: int = 1, +) -> dict[str, Any]: + return { + "type": "semantic-reference-data", + "id": item_id, + "attributes": { + "modelUUID": model_uuid, + "dimensionName": dimension, + "members": members if members is not None else [], + }, + "meta": {"revision": revision}, + } + + +class TestReferenceData: + @staticmethod + def _model_only_list(extra: dict[str, list[dict[str, Any]]] | None = None): + """Build a list_items side_effect: one model + per-type extras.""" + extra = extra or {} + + def _list(item_type: str, model_uuid: str | None = None) -> list[dict[str, Any]]: + if item_type == "semantic-model": + return [_model_item("U", "m")] + return extra.get(item_type, []) + + return _list + + def test_list_summarizes_records(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, mock = _make_service(store) + mock.list_items.side_effect = self._model_only_list( + { + "semantic-reference-data": [ + _refdata_item("r1", "chart_of_accounts", members=[{"account_code": "4011"}]), + ] + } + ) + out = service.list_reference_data("prod") + assert out["project"] == "prod" + assert len(out["reference_data"]) == 1 + rec = out["reference_data"][0] + assert rec["dimension_name"] == "chart_of_accounts" + assert rec["member_count"] == 1 + assert "members" not in rec + + def test_get_by_id_returns_members(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, mock = _make_service(store) + members = [{"account_code": "4011", "account_name": "Revenue"}] + mock.get_item.return_value = _refdata_item("r1", "chart_of_accounts", members=members) + out = service.get_reference_data("prod", record_id="r1") + mock.get_item.assert_called_once_with("semantic-reference-data", "r1") + assert out["members"] == members + assert out["member_count"] == 1 + + def test_get_by_model_and_dimension(self, tmp_path: Path) -> None: + service, mock = _make_service(_make_store(tmp_path)) + mock.list_items.side_effect = self._model_only_list( + {"semantic-reference-data": [_refdata_item("r1", "chart_of_accounts")]} + ) + out = service.get_reference_data( + "prod", model_name_or_uuid=None, dimension="chart_of_accounts" + ) + assert out["id"] == "r1" + mock.get_item.assert_not_called() + + def test_get_requires_id_or_dimension(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, _ = _make_service(store) + with pytest.raises(KeboolaApiError) as exc: + service.get_reference_data("prod", model_name_or_uuid="m") + assert exc.value.error_code == ErrorCode.VALIDATION_ERROR + + def test_get_by_dimension_not_found(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, mock = _make_service(store) + mock.list_items.side_effect = self._model_only_list({"semantic-reference-data": []}) + with pytest.raises(KeboolaApiError) as exc: + service.get_reference_data("prod", model_name_or_uuid=None, dimension="missing") + assert exc.value.error_code == ErrorCode.NOT_FOUND + + def test_set_creates_when_absent(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, mock = _make_service(store) + mock.list_items.side_effect = self._model_only_list({"semantic-reference-data": []}) + mock.post_item.return_value = _refdata_item("r1", "chart_of_accounts") + members = [{"account_code": "4011", "account_name": "Revenue"}] + out = service.set_reference_data( + "prod", + None, + dimension="chart_of_accounts", + members=members, + dataset_id="in.c-f.DIM_COA", + ) + assert out["action"] == "created" + mock.post_item.assert_called_once() + mock.put_item.assert_not_called() + _, kwargs = mock.post_item.call_args + assert kwargs["name"] == "chart_of_accounts" + assert kwargs["data"]["modelUUID"] == "U" + assert kwargs["data"]["dimensionName"] == "chart_of_accounts" + assert kwargs["data"]["members"] == members + assert kwargs["data"]["datasetId"] == "in.c-f.DIM_COA" + + def test_set_replaces_when_present(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, mock = _make_service(store) + existing = _refdata_item("r1", "chart_of_accounts", revision=1) + mock.list_items.side_effect = self._model_only_list({"semantic-reference-data": [existing]}) + mock.put_item.return_value = _refdata_item("r1", "chart_of_accounts", revision=2) + out = service.set_reference_data( + "prod", None, dimension="chart_of_accounts", members=[{"account_code": "4011"}] + ) + assert out["action"] == "updated" + mock.put_item.assert_called_once() + mock.post_item.assert_not_called() + args, _ = mock.put_item.call_args + assert args[0] == "semantic-reference-data" + assert args[1] == "r1" + + def test_set_rejects_non_list_members(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, _ = _make_service(store) + with pytest.raises(KeboolaApiError) as exc: + service.set_reference_data( + "prod", + None, + dimension="chart_of_accounts", + members={"not": "a list"}, # type: ignore[arg-type] + ) + assert exc.value.error_code == ErrorCode.VALIDATION_ERROR + + def test_delete_echoes_dimension(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, mock = _make_service(store) + mock.get_item.return_value = _refdata_item("r1", "chart_of_accounts") + out = service.delete_reference_data("prod", "r1") + mock.delete_item.assert_called_once_with("semantic-reference-data", "r1") + assert out["removed"]["id"] == "r1" + assert out["removed"]["dimension_name"] == "chart_of_accounts" + + +class TestReferenceDataPermissions: + def test_registry_entries(self) -> None: + from keboola_agent_cli.permissions import OPERATION_REGISTRY + + assert OPERATION_REGISTRY["semantic-layer.reference-data.list"] == "read" + assert OPERATION_REGISTRY["semantic-layer.reference-data.get"] == "read" + assert OPERATION_REGISTRY["semantic-layer.reference-data.set"] == "write" + assert OPERATION_REGISTRY["semantic-layer.reference-data.delete"] == "destructive" diff --git a/tests/test_serve_ui.py b/tests/test_serve_ui.py index f9cab128..5135f393 100644 --- a/tests/test_serve_ui.py +++ b/tests/test_serve_ui.py @@ -100,6 +100,17 @@ def test_api_prefix_routes_to_bare_endpoint(self, tmp_path: Path, ui_dist: Path) assert aliased.status_code == 200 assert bare.json() == aliased.json() + def test_dev_portal_read_requires_auth_in_ui_mode(self, tmp_path: Path, ui_dist: Path) -> None: + # Regression for the auth-bypass gap: a GET to an API route that is + # NOT in `_is_ui_public`'s allow-list would be mistaken for an SPA + # route and served without bearer validation. `/dev-portal/*` must + # be treated as API (401 without auth), not as a public SPA path. + client = _make_client(tmp_path, ui_dist=ui_dist, token="t") + resp = client.get("/dev-portal/apps?vendor=keboola") + assert resp.status_code == 401, ( + f"GET /dev-portal/apps must require auth, got {resp.status_code}: {resp.text}" + ) + def test_api_alias_disabled_without_ui(self, tmp_path: Path) -> None: # Without --ui, /api/* should NOT be rewritten -- it 404s like any # unknown path. Critical contract: API-only deployments (BFF diff --git a/tests/test_server_router_calls.py b/tests/test_server_router_calls.py index 15e102c3..f05ef02e 100644 --- a/tests/test_server_router_calls.py +++ b/tests/test_server_router_calls.py @@ -332,3 +332,223 @@ def test_config_variables_set_no_dry_run_kwarg(tmp_path: Path) -> None: f"Router must not pass dry_run= to set_variables, but got kwargs={kwargs}" ) assert kwargs.get("variables") == {"KEY": "val"} + + +# --------------------------------------------------------------------------- +# feature.py -- all 7 endpoints require X-Manage-Token and pass it through. +# --------------------------------------------------------------------------- + + +def test_feature_list_passes_manage_token(tmp_path: Path) -> None: + """GET /feature/{p}/list must forward manage_token to list_stack_features.""" + feature_svc = MagicMock() + feature_svc.list_stack_features.return_value = {"features": []} + registry = _mock_registry(feature=feature_svc) + app = _make_app_with_registry(tmp_path, registry) + app.dependency_overrides[get_manage_token] = lambda: "mgmt-tok" + + with TestClient(app) as client: + res = client.get(f"/feature/{PROJECT}/list", headers=AUTH) + + assert res.status_code == 200, res.text + kwargs = feature_svc.list_stack_features.call_args.kwargs + assert kwargs == {"manage_token": "mgmt-tok", "alias": PROJECT} + + +def test_feature_project_show_passes_manage_token(tmp_path: Path) -> None: + feature_svc = MagicMock() + feature_svc.list_project_features.return_value = {"features": []} + registry = _mock_registry(feature=feature_svc) + app = _make_app_with_registry(tmp_path, registry) + app.dependency_overrides[get_manage_token] = lambda: "mgmt-tok" + + with TestClient(app) as client: + res = client.get(f"/feature/{PROJECT}/project-show", headers=AUTH) + + assert res.status_code == 200, res.text + assert feature_svc.list_project_features.call_args.kwargs == { + "manage_token": "mgmt-tok", + "alias": PROJECT, + } + + +def test_feature_project_add_passes_body_and_token(tmp_path: Path) -> None: + feature_svc = MagicMock() + feature_svc.add_project_feature.return_value = {"status": "added"} + registry = _mock_registry(feature=feature_svc) + app = _make_app_with_registry(tmp_path, registry) + app.dependency_overrides[get_manage_token] = lambda: "mgmt-tok" + + with TestClient(app) as client: + res = client.post( + f"/feature/{PROJECT}/project-add", + headers=AUTH, + json={"feature": "data-streams", "dry_run": True}, + ) + + assert res.status_code == 200, res.text + assert feature_svc.add_project_feature.call_args.kwargs == { + "manage_token": "mgmt-tok", + "alias": PROJECT, + "feature": "data-streams", + "dry_run": True, + } + + +def test_feature_project_remove_passes_body_and_token(tmp_path: Path) -> None: + feature_svc = MagicMock() + feature_svc.remove_project_feature.return_value = {"status": "removed"} + registry = _mock_registry(feature=feature_svc) + app = _make_app_with_registry(tmp_path, registry) + app.dependency_overrides[get_manage_token] = lambda: "mgmt-tok" + + with TestClient(app) as client: + res = client.post( + f"/feature/{PROJECT}/project-remove", + headers=AUTH, + json={"feature": "data-streams"}, + ) + + assert res.status_code == 200, res.text + kwargs = feature_svc.remove_project_feature.call_args.kwargs + assert kwargs["manage_token"] == "mgmt-tok" + assert kwargs["feature"] == "data-streams" + assert kwargs["dry_run"] is False + + +def test_feature_user_show_passes_email_and_token(tmp_path: Path) -> None: + feature_svc = MagicMock() + feature_svc.list_user_features.return_value = {"features": []} + registry = _mock_registry(feature=feature_svc) + app = _make_app_with_registry(tmp_path, registry) + app.dependency_overrides[get_manage_token] = lambda: "mgmt-tok" + + with TestClient(app) as client: + res = client.get( + f"/feature/{PROJECT}/user-show", + headers=AUTH, + params={"email": "user@example.com"}, + ) + + assert res.status_code == 200, res.text + assert feature_svc.list_user_features.call_args.kwargs == { + "manage_token": "mgmt-tok", + "alias": PROJECT, + "email": "user@example.com", + } + + +def test_feature_user_add_passes_body_and_token(tmp_path: Path) -> None: + feature_svc = MagicMock() + feature_svc.add_user_feature.return_value = {"status": "added"} + registry = _mock_registry(feature=feature_svc) + app = _make_app_with_registry(tmp_path, registry) + app.dependency_overrides[get_manage_token] = lambda: "mgmt-tok" + + with TestClient(app) as client: + res = client.post( + f"/feature/{PROJECT}/user-add", + headers=AUTH, + json={"email": "user@example.com", "feature": "early-adopter-preview"}, + ) + + assert res.status_code == 200, res.text + assert feature_svc.add_user_feature.call_args.kwargs == { + "manage_token": "mgmt-tok", + "alias": PROJECT, + "email": "user@example.com", + "feature": "early-adopter-preview", + "dry_run": False, + } + + +def test_feature_list_missing_manage_token_returns_401(tmp_path: Path) -> None: + """No X-Manage-Token header -> 401 and the service is never called.""" + feature_svc = MagicMock() + registry = _mock_registry(feature=feature_svc) + app = _make_app_with_registry(tmp_path, registry) + + with TestClient(app) as client: + res = client.get(f"/feature/{PROJECT}/list", headers=AUTH) + + assert res.status_code == 401, res.text + body = res.json() + msg = body.get("detail") or body.get("error", {}).get("message", "") + assert "X-Manage-Token" in msg, f"Expected X-Manage-Token mention, got: {body}" + feature_svc.list_stack_features.assert_not_called() + + +# --------------------------------------------------------------------------- +# dev_portal.py GET /dev-portal/apps and GET /dev-portal/apps/{app} +# Service: dev_portal.list_apps(alias, vendor) / get_app(alias, vendor, app_id) +# --------------------------------------------------------------------------- + + +def test_dev_portal_list_apps_passes_alias_and_vendor(tmp_path: Path) -> None: + """GET /dev-portal/apps must call list_apps with the resolved alias + vendor.""" + dp_svc = MagicMock() + dp_svc.list_apps.return_value = [{"id": "keboola.ex-a"}] + registry = _mock_registry(dev_portal=dp_svc) + app = _make_app_with_registry(tmp_path, registry) + + with TestClient(app) as client: + res = client.get("/dev-portal/apps?vendor=keboola&identity=alpha", headers=AUTH) + + assert res.status_code == 200, res.text + dp_svc.list_apps.assert_called_once_with("alpha", "keboola") + + +def test_dev_portal_get_app_splits_vendor_from_app_id(tmp_path: Path) -> None: + """GET /dev-portal/apps/{app} must split VENDOR.APP_ID and pass both.""" + dp_svc = MagicMock() + dp_svc.get_app.return_value = {"id": "keboola.ex-a", "name": "Hello"} + registry = _mock_registry(dev_portal=dp_svc) + app = _make_app_with_registry(tmp_path, registry) + + with TestClient(app) as client: + res = client.get("/dev-portal/apps/keboola.ex-a?identity=alpha", headers=AUTH) + + assert res.status_code == 200, res.text + dp_svc.get_app.assert_called_once_with("alpha", "keboola", "keboola.ex-a") + + +def test_dev_portal_get_app_rejects_app_without_vendor(tmp_path: Path) -> None: + """An app id missing the VENDOR. prefix is a 400, not a service call.""" + dp_svc = MagicMock() + registry = _mock_registry(dev_portal=dp_svc) + app = _make_app_with_registry(tmp_path, registry) + + with TestClient(app) as client: + res = client.get("/dev-portal/apps/no-dot?identity=alpha", headers=AUTH) + + assert res.status_code == 400, res.text + dp_svc.get_app.assert_not_called() + + +def test_dev_portal_list_falls_back_to_default_identity(tmp_path: Path) -> None: + """Without ?identity=, the router resolves the configured default identity.""" + dp_svc = MagicMock() + dp_svc.current_identity.return_value = "default-alias" + dp_svc.list_apps.return_value = [] + registry = _mock_registry(dev_portal=dp_svc) + app = _make_app_with_registry(tmp_path, registry) + + with TestClient(app) as client: + res = client.get("/dev-portal/apps?vendor=keboola", headers=AUTH) + + assert res.status_code == 200, res.text + dp_svc.list_apps.assert_called_once_with("default-alias", "keboola") + + +def test_dev_portal_list_no_identity_no_default_is_400(tmp_path: Path) -> None: + """No explicit identity and no default configured -> 400, no service call.""" + dp_svc = MagicMock() + dp_svc.current_identity.return_value = "" + registry = _mock_registry(dev_portal=dp_svc) + app = _make_app_with_registry(tmp_path, registry) + + with TestClient(app) as client: + res = client.get("/dev-portal/apps?vendor=keboola", headers=AUTH) + + assert res.status_code == 400, res.text + dp_svc.list_apps.assert_not_called() diff --git a/tests/test_server_smoke.py b/tests/test_server_smoke.py index 522ea3e0..b5ed396e 100644 --- a/tests/test_server_smoke.py +++ b/tests/test_server_smoke.py @@ -32,6 +32,7 @@ "/configs", "/components", "/storage/buckets", + "/stream/{project}/list", "/jobs", "/branches", "/workspaces", @@ -104,6 +105,32 @@ def test_openapi_lists_all_routers(client: TestClient) -> None: assert not missing, f"OpenAPI is missing expected paths: {sorted(missing)}" +def test_every_router_tag_has_openapi_metadata(client: TestClient) -> None: + """Every tag used by an operation must have an OPENAPI_TAGS description block. + + Regression guard for the bug that hid the ``stream`` router: it was wired + via ``include_router`` and fully callable, but its tag was missing from + ``OPENAPI_TAGS`` in ``server/app.py`` -- so Swagger UI rendered a bare, + description-less ``stream`` section out of its logical group. Registration + (``include_router``) and documentation (``openapi_tags``) are independent + layers; this test ties them together so a new router can't ship invisible + in ``/docs`` again. + """ + http_methods = {"get", "post", "put", "delete", "patch", "options", "head", "trace"} + spec = client.get("/openapi.json").json() + documented = {tag["name"] for tag in spec.get("tags", [])} + used: set[str] = set() + for path_item in spec["paths"].values(): + for method, operation in path_item.items(): + if method in http_methods and isinstance(operation, dict): + used.update(operation.get("tags", [])) + undocumented = used - documented + assert not undocumented, ( + "Routers expose tags with no OPENAPI_TAGS description block: " + f"{sorted(undocumented)}. Add an entry to OPENAPI_TAGS in server/app.py." + ) + + def test_org_setup_requires_manage_token(client: TestClient) -> None: res = client.post( "/org/setup", diff --git a/tests/test_services.py b/tests/test_services.py index cde0bed9..578c9b80 100644 --- a/tests/test_services.py +++ b/tests/test_services.py @@ -47,6 +47,28 @@ def test_add_project_success(self, tmp_config_dir: Path) -> None: mock_client.verify_token.assert_called_once() mock_client.close.assert_called_once() + def test_add_project_normalizes_bare_host_url(self, tmp_config_dir: Path) -> None: + """add_project passes a bare host through normalize_stack_url before storing.""" + store = ConfigStore(config_dir=tmp_config_dir) + captured: dict[str, str] = {} + + def factory(url: str, token: str): + captured["url"] = url + return make_mock_client(project_name="Production", project_id=9999) + + service = ProjectService(config_store=store, client_factory=factory) + + result = service.add_project( + alias="prod", + stack_url="connection.keboola.com/admin/projects/9999/dashboard", + token="901-55555-fakeTestTokenDoNotUseXXXXXXXX", + ) + + # Verification client and the stored/returned URL all use the clean base. + assert captured["url"] == "https://connection.keboola.com" + assert result["stack_url"] == "https://connection.keboola.com" + assert store.get_project("prod").stack_url == "https://connection.keboola.com" + def test_add_project_invalid_token(self, tmp_config_dir: Path) -> None: """add_project raises KeboolaApiError when token verification fails.""" store = ConfigStore(config_dir=tmp_config_dir) @@ -472,6 +494,29 @@ def test_status_backfills_org_info_for_legacy_projects(self, tmp_config_dir: Pat assert refreshed.org_id == 438 assert refreshed.org_name == "Keboola Demo" + def test_status_no_backfill_for_ephemeral_env_project( + self, tmp_config_dir: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """Headless __env__ status must not trigger a config.json write (issue #359). + + The env project's org info can never be persisted (save strips it), so + backfilling it would create a spurious config.json on disk and repeat on + every `project status`. get_status() must leave the dir file-free. + """ + monkeypatch.setenv("KBAGENT_PROJECT_FROM_ENV", "1") + monkeypatch.setenv("KBC_TOKEN", "901-99999-fakeHeadlessTokenDoNotUseXXXXX") + monkeypatch.setenv("KBC_STORAGE_API_URL", "https://connection.keboola.com") + store = ConfigStore(config_dir=tmp_config_dir) + mock_client = make_mock_client(org_id=438, org_name="Keboola Demo") + service = ProjectService( + config_store=store, + client_factory=lambda url, token: mock_client, + ) + + service.get_status() + + assert not (tmp_config_dir / "config.json").exists() + def test_status_no_backfill_when_org_info_already_set(self, tmp_config_dir: Path) -> None: """Projects with org info already populated must not be re-written.""" store = ConfigStore(config_dir=tmp_config_dir) diff --git a/tests/test_storage_clone.py b/tests/test_storage_clone.py new file mode 100644 index 00000000..8358566b --- /dev/null +++ b/tests/test_storage_clone.py @@ -0,0 +1,399 @@ +"""Tests for storage clone-table (pull endpoint): client, service, and CLI.""" + +import json +from pathlib import Path +from unittest.mock import MagicMock, patch + +import pytest +from typer.testing import CliRunner + +from keboola_agent_cli.cli import app +from keboola_agent_cli.client import KeboolaClient +from keboola_agent_cli.config_store import ConfigStore +from keboola_agent_cli.errors import ConfigError, KeboolaApiError +from keboola_agent_cli.models import AppConfig, ProjectConfig +from keboola_agent_cli.services.storage_service import StorageService + +runner = CliRunner() + +TEST_TOKEN = "901-55555-fakeTestTokenDoNotUseXXXXXXXX" + + +def _make_store(tmp_path: Path) -> ConfigStore: + config_dir = tmp_path / "config" + config_dir.mkdir(exist_ok=True) + store = ConfigStore(config_dir=config_dir) + config = AppConfig( + projects={ + "test": ProjectConfig( + stack_url="https://connection.keboola.com", + token=TEST_TOKEN, + ) + }, + ) + store.save(config) + return store + + +def _make_service(store: ConfigStore, mock_client: MagicMock) -> StorageService: + return StorageService( + config_store=store, + client_factory=lambda url, token: mock_client, + ) + + +# --------------------------------------------------------------------------- +# Client layer +# --------------------------------------------------------------------------- + + +class TestPullTableClient: + """Tests for KeboolaClient.pull_table() - HTTP layer.""" + + def test_correct_url_and_no_body(self, httpx_mock) -> None: + """POSTs to /v2/storage/branch/{branch}/tables/{tid}/pull with no body. + + The Storage API responds with a queued storage job + (operationName=devBranchTablePull) which the client polls to + completion. Returning ``status: success`` from the first response + avoids exercising the poll loop in this unit test. + """ + httpx_mock.add_response( + url="https://connection.keboola.com/v2/storage/branch/9999/tables/in.c-foo.data/pull", + method="POST", + json={ + "id": 388266099, + "status": "success", + "operationName": "devBranchTablePull", + "operationParams": {"branchId": 9999}, + }, + status_code=200, + ) + + client = KeboolaClient( + stack_url="https://connection.keboola.com", + token=TEST_TOKEN, + ) + result = client.pull_table(table_id="in.c-foo.data", branch_id=9999) + + # Returned dict is the completed storage job + assert result["status"] == "success" + assert result["operationName"] == "devBranchTablePull" + + # The pull endpoint takes no request body (verified live against the API) + sent_request = httpx_mock.get_request() + assert sent_request.content == b"" + client.close() + + def test_dotted_table_id_passed_verbatim_in_path(self, httpx_mock) -> None: + """Dotted/dashed table IDs land in the path as-is. + + Dots and dashes are RFC 3986 unreserved, so ``quote(..., safe="")`` + does not percent-encode them; this verifies the table ID is placed + in the path verbatim (a reserved char, if present, would be encoded). + """ + httpx_mock.add_response( + url="https://connection.keboola.com/v2/storage/branch/1/tables/in.c-bucket-with-dashes.tbl/pull", + method="POST", + json={"id": 1, "status": "success", "operationName": "devBranchTablePull"}, + status_code=200, + ) + + client = KeboolaClient( + stack_url="https://connection.keboola.com", + token=TEST_TOKEN, + ) + client.pull_table(table_id="in.c-bucket-with-dashes.tbl", branch_id=1) + client.close() + + def test_polls_async_job_to_completion(self, httpx_mock) -> None: + """If POST returns ``status: waiting``, client polls /v2/storage/jobs/{id}.""" + httpx_mock.add_response( + url="https://connection.keboola.com/v2/storage/branch/42/tables/in.c-foo.a/pull", + method="POST", + json={"id": 555, "status": "waiting", "operationName": "devBranchTablePull"}, + status_code=200, + ) + httpx_mock.add_response( + url="https://connection.keboola.com/v2/storage/jobs/555", + method="GET", + json={"id": 555, "status": "success", "operationName": "devBranchTablePull"}, + status_code=200, + ) + + client = KeboolaClient( + stack_url="https://connection.keboola.com", + token=TEST_TOKEN, + ) + result = client.pull_table(table_id="in.c-foo.a", branch_id=42) + assert result["status"] == "success" + client.close() + + def test_api_error_propagates(self, httpx_mock) -> None: + """Storage API 4xx propagates as KeboolaApiError.""" + httpx_mock.add_response( + url="https://connection.keboola.com/v2/storage/branch/9999/tables/in.c-foo.x/pull", + method="POST", + json={"error": "Table not found in the default branch"}, + status_code=404, + ) + + client = KeboolaClient( + stack_url="https://connection.keboola.com", + token=TEST_TOKEN, + ) + with pytest.raises(KeboolaApiError): + client.pull_table(table_id="in.c-foo.x", branch_id=9999) + client.close() + + +# --------------------------------------------------------------------------- +# Service layer +# --------------------------------------------------------------------------- + + +class TestCloneTableService: + """Tests for StorageService.clone_table().""" + + def test_success(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + mock_client = MagicMock() + mock_client.pull_table.return_value = {"status": "ok"} + service = _make_service(store, mock_client) + + result = service.clone_table( + alias="test", + table_id="in.c-foo.data", + branch_id=9999, + ) + + assert result["project_alias"] == "test" + assert result["branch_id"] == 9999 + assert result["table_id"] == "in.c-foo.data" + assert result["dry_run"] is False + assert result["response"] == {"status": "ok"} + mock_client.pull_table.assert_called_once_with( + table_id="in.c-foo.data", + branch_id=9999, + ) + mock_client.close.assert_called_once() + + def test_dry_run_skips_client_call(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + mock_client = MagicMock() + service = _make_service(store, mock_client) + + result = service.clone_table( + alias="test", + table_id="in.c-foo.a", + branch_id=42, + dry_run=True, + ) + + assert result["dry_run"] is True + assert "response" not in result + mock_client.pull_table.assert_not_called() + + def test_no_branch_raises_config_error(self, tmp_path: Path) -> None: + """Mandatory branch enforcement: pull is one-way default -> branch.""" + store = _make_store(tmp_path) + mock_client = MagicMock() + service = _make_service(store, mock_client) + + with pytest.raises(ConfigError, match="dev branch"): + service.clone_table( + alias="test", + table_id="in.c-foo.a", + branch_id=None, + ) + mock_client.pull_table.assert_not_called() + + def test_unknown_project(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + mock_client = MagicMock() + service = _make_service(store, mock_client) + + with pytest.raises(ConfigError): + service.clone_table( + alias="nonexistent", + table_id="in.c-foo.a", + branch_id=42, + ) + + def test_api_error_propagates(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + mock_client = MagicMock() + mock_client.pull_table.side_effect = KeboolaApiError( + "Table not found", status_code=404, error_code="NOT_FOUND" + ) + service = _make_service(store, mock_client) + + with pytest.raises(KeboolaApiError): + service.clone_table( + alias="test", + table_id="in.c-foo.a", + branch_id=42, + ) + # Service must close the client even when the API call raises + # (try/finally contract -- regression guard for the lifecycle). + mock_client.close.assert_called_once() + + +# --------------------------------------------------------------------------- +# CLI layer +# --------------------------------------------------------------------------- + + +class TestCloneTableCLI: + """CLI tests for `kbagent storage clone-table`.""" + + def _project_with_active_branch(self, store: ConfigStore, branch_id: int) -> None: + config = store.load() + config.projects["test"].active_branch_id = branch_id + store.save(config) + + def test_clone_json(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + self._project_with_active_branch(store, 9999) + + with ( + patch("keboola_agent_cli.cli.ConfigStore") as MockStore, + patch("keboola_agent_cli.cli.StorageService") as MockSvc, + ): + MockStore.return_value = store + svc = MockSvc.return_value + svc.clone_table.return_value = { + "project_alias": "test", + "branch_id": 9999, + "table_id": "in.c-foo.data", + "dry_run": False, + "response": {"status": "ok"}, + } + result = runner.invoke( + app, + [ + "--json", + "storage", + "clone-table", + "--project", + "test", + "--table-id", + "in.c-foo.data", + ], + ) + + assert result.exit_code == 0, result.output + data = json.loads(result.output)["data"] + assert data["table_id"] == "in.c-foo.data" + assert data["branch_id"] == 9999 + + svc.clone_table.assert_called_once_with( + alias="test", + table_id="in.c-foo.data", + branch_id=9999, + dry_run=False, + ) + + def test_clone_dry_run(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + self._project_with_active_branch(store, 42) + + with ( + patch("keboola_agent_cli.cli.ConfigStore") as MockStore, + patch("keboola_agent_cli.cli.StorageService") as MockSvc, + ): + MockStore.return_value = store + svc = MockSvc.return_value + svc.clone_table.return_value = { + "project_alias": "test", + "branch_id": 42, + "table_id": "in.c-foo.a", + "dry_run": True, + } + result = runner.invoke( + app, + [ + "--json", + "storage", + "clone-table", + "--project", + "test", + "--table-id", + "in.c-foo.a", + "--dry-run", + ], + ) + + assert result.exit_code == 0, result.output + data = json.loads(result.output)["data"] + assert data["dry_run"] is True + svc.clone_table.assert_called_once() + call_kwargs = svc.clone_table.call_args.kwargs + assert call_kwargs["dry_run"] is True + + def test_clone_explicit_branch_overrides_active(self, tmp_path: Path) -> None: + """--branch flag takes precedence over project's active_branch_id.""" + store = _make_store(tmp_path) + self._project_with_active_branch(store, 100) + + with ( + patch("keboola_agent_cli.cli.ConfigStore") as MockStore, + patch("keboola_agent_cli.cli.StorageService") as MockSvc, + ): + MockStore.return_value = store + svc = MockSvc.return_value + svc.clone_table.return_value = { + "project_alias": "test", + "branch_id": 555, + "table_id": "in.c-foo.a", + "dry_run": False, + "response": {"status": "ok"}, + } + result = runner.invoke( + app, + [ + "--json", + "storage", + "clone-table", + "--project", + "test", + "--table-id", + "in.c-foo.a", + "--branch", + "555", + ], + ) + + assert result.exit_code == 0, result.output + call_kwargs = svc.clone_table.call_args.kwargs + assert call_kwargs["branch_id"] == 555 + + def test_clone_missing_branch_fails_clearly(self, tmp_path: Path) -> None: + """Without an active branch and without --branch, ConfigError -> exit 5.""" + store = _make_store(tmp_path) + # No active_branch_id set on project + + with ( + patch("keboola_agent_cli.cli.ConfigStore") as MockStore, + patch("keboola_agent_cli.cli.StorageService") as MockSvc, + ): + MockStore.return_value = store + svc = MockSvc.return_value + svc.clone_table.side_effect = ConfigError("clone-table requires a dev branch.") + result = runner.invoke( + app, + [ + "--json", + "storage", + "clone-table", + "--project", + "test", + "--table-id", + "in.c-foo.a", + ], + ) + + assert result.exit_code == 5 + payload = json.loads(result.output) + assert payload["status"] == "error" + assert "dev branch" in payload["error"]["message"] diff --git a/tests/test_storage_swap.py b/tests/test_storage_swap.py index 0459b0c8..f4d1941c 100644 --- a/tests/test_storage_swap.py +++ b/tests/test_storage_swap.py @@ -92,8 +92,13 @@ def test_correct_url_and_body(self, httpx_mock) -> None: assert body == {"targetTableId": "in.c-foo.data_change_log"} client.close() - def test_url_encoding_for_special_characters(self, httpx_mock) -> None: - """Table IDs with dots/dashes are URL-encoded in the path.""" + def test_dotted_table_id_passed_verbatim_in_path(self, httpx_mock) -> None: + """Dotted/dashed table IDs land in the path as-is. + + Dots and dashes are RFC 3986 unreserved, so ``quote(..., safe="")`` + does not percent-encode them; this verifies the table ID is placed + in the path verbatim (a reserved char, if present, would be encoded). + """ httpx_mock.add_response( url="https://connection.keboola.com/v2/storage/branch/1/tables/in.c-bucket-with-dashes.tbl/swap", method="POST", @@ -214,12 +219,12 @@ def test_dry_run_skips_client_call(self, tmp_path: Path) -> None: mock_client.swap_tables.assert_not_called() def test_no_branch_raises_config_error(self, tmp_path: Path) -> None: - """Mandatory branch enforcement: production swap is rejected before any HTTP.""" + """Mandatory branch enforcement: swap-tables without --branch or active branch raises ConfigError before any HTTP.""" store = _make_store(tmp_path) mock_client = MagicMock() service = _make_service(store, mock_client) - with pytest.raises(ConfigError, match="dev branch"): + with pytest.raises(ConfigError, match="requires a branch"): service.swap_tables( alias="test", table_id="in.c-foo.a", @@ -428,7 +433,7 @@ def test_swap_missing_branch_fails_clearly(self, tmp_path: Path) -> None: ): MockStore.return_value = store svc = MockSvc.return_value - svc.swap_tables.side_effect = ConfigError("swap-tables requires a dev branch.") + svc.swap_tables.side_effect = ConfigError("swap-tables requires a branch.") result = runner.invoke( app, [ @@ -448,4 +453,4 @@ def test_swap_missing_branch_fails_clearly(self, tmp_path: Path) -> None: assert result.exit_code == 5 payload = json.loads(result.output) assert payload["status"] == "error" - assert "dev branch" in payload["error"]["message"] + assert "requires a branch" in payload["error"]["message"] diff --git a/tests/test_storage_write.py b/tests/test_storage_write.py index fc7a2d39..6a370d6e 100644 --- a/tests/test_storage_write.py +++ b/tests/test_storage_write.py @@ -9,7 +9,7 @@ from keboola_agent_cli.cli import app from keboola_agent_cli.config_store import ConfigStore -from keboola_agent_cli.errors import KeboolaApiError +from keboola_agent_cli.errors import ErrorCode, KeboolaApiError from keboola_agent_cli.models import AppConfig, ProjectConfig from keboola_agent_cli.services.storage_service import StorageService @@ -1041,6 +1041,220 @@ def test_create_bucket_api_error(self, tmp_path: Path) -> None: assert result.exit_code != 0 +# --------------------------------------------------------------------------- +# Service tests: create_table --if-not-exists (v0.47.0) +# --------------------------------------------------------------------------- + + +class TestCreateTableIfNotExists: + """`if_not_exists=True` turns duplicate-display-name into a skip.""" + + @staticmethod + def _duplicate_display_name_error() -> KeboolaApiError: + return KeboolaApiError( + message=( + "Bucket in.c-b.users already has the same display name in " + "bucket in.c-b. Please rename one of them." + ), + status_code=500, + error_code=ErrorCode.STORAGE_JOB_FAILED, + ) + + def test_skip_on_existing_when_flag_set(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + mock_client = MagicMock() + mock_client.create_table.side_effect = self._duplicate_display_name_error() + mock_client.get_table_detail.return_value = { + "id": "in.c-b.users", + "name": "users", + "columns": ["id", "name"], + "primaryKey": ["id"], + } + service = _make_service(store, mock_client) + + result = service.create_table( + alias="test", + bucket_id="in.c-b", + name="users", + columns=["id:INTEGER", "name:STRING"], + primary_key=["id"], + if_not_exists=True, + ) + + assert result["action"] == "skipped" + assert result["skip_reason"] == "table already exists" + assert result["table_id"] == "in.c-b.users" + mock_client.get_table_detail.assert_called_once_with("in.c-b.users", branch_id=None) + mock_client.close.assert_called_once() + + def test_skip_returns_actual_schema_not_requested(self, tmp_path: Path) -> None: + """keboola/cli#349: the skipped envelope must report the EXISTING + table's schema, not re-echo the caller's request. Here the existing + table has fewer columns and a different PK than what was requested.""" + store = _make_store(tmp_path) + mock_client = MagicMock() + mock_client.create_table.side_effect = self._duplicate_display_name_error() + mock_client.get_table_detail.return_value = { + "id": "in.c-b.users", + "name": "users", + "columns": ["id", "name"], + "primaryKey": ["id"], + } + service = _make_service(store, mock_client) + + result = service.create_table( + alias="test", + bucket_id="in.c-b", + name="users", + columns=["id:INTEGER", "name:STRING", "extra:STRING"], + primary_key=["extra"], + if_not_exists=True, + ) + + # Actual existing schema is reported. + assert result["columns"] == ["id", "name"] + assert result["primary_key"] == ["id"] + assert result["name"] == "users" + # Caller's request is mirrored, not lost. + assert result["requested_columns"] == ["id", "name", "extra"] + assert result["requested_primary_key"] == ["extra"] + # Divergence is flagged. + assert result["schema_drift"] is True + + def test_no_schema_drift_when_existing_matches_request(self, tmp_path: Path) -> None: + """When the existing table matches the request, schema_drift is False + and columns/primary_key are still sourced from the actual table.""" + store = _make_store(tmp_path) + mock_client = MagicMock() + mock_client.create_table.side_effect = self._duplicate_display_name_error() + mock_client.get_table_detail.return_value = { + "id": "in.c-b.users", + "name": "users", + "columns": ["id", "name"], + "primaryKey": ["id"], + } + service = _make_service(store, mock_client) + + result = service.create_table( + alias="test", + bucket_id="in.c-b", + name="users", + columns=["id:INTEGER", "name:STRING"], + primary_key=["id"], + if_not_exists=True, + ) + + assert result["schema_drift"] is False + assert result["columns"] == ["id", "name"] + assert result["primary_key"] == ["id"] + + def test_drift_is_order_insensitive(self, tmp_path: Path) -> None: + """Column/PK reordering between request and existing table is the same + set of columns -- not a drift (set comparison, not list equality).""" + store = _make_store(tmp_path) + mock_client = MagicMock() + mock_client.create_table.side_effect = self._duplicate_display_name_error() + mock_client.get_table_detail.return_value = { + "id": "in.c-b.users", + "name": "users", + "columns": ["name", "id"], + "primaryKey": ["id"], + } + service = _make_service(store, mock_client) + + result = service.create_table( + alias="test", + bucket_id="in.c-b", + name="users", + columns=["id:INTEGER", "name:STRING"], + primary_key=["id"], + if_not_exists=True, + ) + + assert result["schema_drift"] is False + + def test_reraises_when_flag_unset(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + mock_client = MagicMock() + mock_client.create_table.side_effect = self._duplicate_display_name_error() + service = _make_service(store, mock_client) + + with pytest.raises(KeboolaApiError) as excinfo: + service.create_table( + alias="test", + bucket_id="in.c-b", + name="users", + columns=["id:INTEGER"], + ) + assert excinfo.value.error_code == ErrorCode.STORAGE_JOB_FAILED + # No probe when flag is off. + mock_client.get_table_detail.assert_not_called() + mock_client.close.assert_called_once() + + def test_reraises_when_target_table_missing(self, tmp_path: Path) -> None: + """Duplicate-name error but the table at the expected id doesn't + resolve → a different table is conflicting; surface the real error.""" + store = _make_store(tmp_path) + mock_client = MagicMock() + mock_client.create_table.side_effect = self._duplicate_display_name_error() + mock_client.get_table_detail.side_effect = KeboolaApiError( + message="404", status_code=404, error_code=ErrorCode.NOT_FOUND + ) + service = _make_service(store, mock_client) + + with pytest.raises(KeboolaApiError) as excinfo: + service.create_table( + alias="test", + bucket_id="in.c-b", + name="users", + columns=["id:INTEGER"], + if_not_exists=True, + ) + # The ORIGINAL error must propagate, not the lookup error. + assert excinfo.value.error_code == ErrorCode.STORAGE_JOB_FAILED + + def test_non_duplicate_error_reraises_even_with_flag(self, tmp_path: Path) -> None: + """A non-duplicate STORAGE_JOB_FAILED still surfaces — the IF-NOT- + EXISTS path is gated on the specific message substring.""" + store = _make_store(tmp_path) + mock_client = MagicMock() + mock_client.create_table.side_effect = KeboolaApiError( + message="quota exceeded", + status_code=500, + error_code=ErrorCode.STORAGE_JOB_FAILED, + ) + service = _make_service(store, mock_client) + + with pytest.raises(KeboolaApiError) as excinfo: + service.create_table( + alias="test", + bucket_id="in.c-b", + name="users", + columns=["id:INTEGER"], + if_not_exists=True, + ) + assert "quota" in str(excinfo.value.message).lower() + mock_client.get_table_detail.assert_not_called() + + def test_success_path_unchanged_with_flag(self, tmp_path: Path) -> None: + """When the create succeeds, the flag has no effect on the envelope.""" + store = _make_store(tmp_path) + mock_client = MagicMock() + mock_client.create_table.return_value = {"id": "in.c-b.users"} + service = _make_service(store, mock_client) + + result = service.create_table( + alias="test", + bucket_id="in.c-b", + name="users", + columns=["id:INTEGER"], + if_not_exists=True, + ) + + assert result["action"] == "created" + assert result["table_id"] == "in.c-b.users" + + # --------------------------------------------------------------------------- # CLI tests: create-table # --------------------------------------------------------------------------- @@ -1049,6 +1263,95 @@ def test_create_bucket_api_error(self, tmp_path: Path) -> None: class TestCreateTableCLI: """CLI tests for `kbagent storage create-table`.""" + def test_human_renders_skip_when_action_is_skipped(self, tmp_path: Path) -> None: + """When --if-not-exists triggers a skip, human mode prints + 'Skipped (already exists)' instead of the misleading 'Created table'.""" + store = _make_store(tmp_path) + with ( + patch("keboola_agent_cli.cli.ConfigStore") as MockStore, + patch("keboola_agent_cli.cli.StorageService") as MockSvc, + ): + MockStore.return_value = store + svc = MockSvc.return_value + svc.create_table.return_value = { + "project_alias": "test", + "table_id": "in.c-b.users", + "name": "users", + "bucket_id": "in.c-b", + "primary_key": ["id"], + "columns": ["id", "name"], + "action": "skipped", + "skip_reason": "table already exists", + } + result = runner.invoke( + app, + [ + "storage", + "create-table", + "--project", + "test", + "--bucket-id", + "in.c-b", + "--name", + "users", + "--column", + "id:INTEGER", + "--if-not-exists", + ], + ) + assert result.exit_code == 0, result.output + assert "Skipped" in result.output + assert "in.c-b.users" in result.output + assert "table already exists" in result.output + assert "Created table" not in result.output, ( + "must NOT print the misleading success line on a skipped row" + ) + + def test_human_warns_and_shows_actual_schema_on_drift(self, tmp_path: Path) -> None: + """When the skipped table's schema diverges from the request, human + mode warns and prints the ACTUAL existing columns/PK (keboola/cli#349).""" + store = _make_store(tmp_path) + with ( + patch("keboola_agent_cli.cli.ConfigStore") as MockStore, + patch("keboola_agent_cli.cli.StorageService") as MockSvc, + ): + MockStore.return_value = store + svc = MockSvc.return_value + svc.create_table.return_value = { + "project_alias": "test", + "table_id": "in.c-b.users", + "name": "users", + "bucket_id": "in.c-b", + "primary_key": ["id"], + "columns": ["id", "name"], + "requested_primary_key": ["extra"], + "requested_columns": ["id", "name", "extra"], + "schema_drift": True, + "action": "skipped", + "skip_reason": "table already exists", + } + result = runner.invoke( + app, + [ + "storage", + "create-table", + "--project", + "test", + "--bucket-id", + "in.c-b", + "--name", + "users", + "--column", + "id:INTEGER", + "--if-not-exists", + ], + ) + assert result.exit_code == 0, result.output + assert "Skipped" in result.output + assert "Warning" in result.output + # Actual existing columns are shown, not the requested 'extra'. + assert "id, name" in result.output + def test_create_table_json(self, tmp_path: Path) -> None: store = _make_store(tmp_path) with ( @@ -1097,6 +1400,7 @@ def test_create_table_json(self, tmp_path: Path) -> None: branch_id=None, not_null_columns=None, defaults=None, + if_not_exists=False, ) def test_create_table_native_types_and_attributes(self, tmp_path: Path) -> None: diff --git a/tests/test_stream_cli.py b/tests/test_stream_cli.py new file mode 100644 index 00000000..df1f2842 --- /dev/null +++ b/tests/test_stream_cli.py @@ -0,0 +1,217 @@ +"""CLI tests for the `kbagent stream` command group.""" + +from __future__ import annotations + +import json +from pathlib import Path +from unittest.mock import MagicMock, patch + +from typer.testing import CliRunner + +from keboola_agent_cli.cli import app +from keboola_agent_cli.config_store import ConfigStore +from keboola_agent_cli.errors import KeboolaApiError +from keboola_agent_cli.models import ProjectConfig + +STACK_URL = "https://connection.keboola.com" +TOKEN = "901-55555-fakeTestTokenDoNotUseXXXXXXXX" +ALIAS = "padak" + +runner = CliRunner() + + +def _seed(config_dir: Path) -> None: + store = ConfigStore(config_dir=config_dir) + store.add_project( + ALIAS, + ProjectConfig(stack_url=STACK_URL, token=TOKEN, project_name="Padak 2.0", project_id=10539), + ) + + +def _invoke(config_dir: Path, svc: MagicMock, args: list[str], input_text: str | None = None): + with patch("keboola_agent_cli.cli.StreamService", return_value=svc): + return runner.invoke(app, ["--config-dir", str(config_dir), *args], input=input_text) + + +class TestList: + def test_json(self, tmp_path: Path) -> None: + config_dir = tmp_path / "c" + config_dir.mkdir() + _seed(config_dir) + svc = MagicMock() + svc.list_sources.return_value = { + "alias": ALIAS, + "branch_id": "default", + "sources": [ + {"source_id": "s1", "name": "s1", "type": "otlp", "base_endpoint": "https://x"} + ], + } + result = _invoke(config_dir, svc, ["--json", "stream", "list", "--project", ALIAS]) + assert result.exit_code == 0, result.output + data = json.loads(result.output)["data"] + assert data["sources"][0]["source_id"] == "s1" + svc.list_sources.assert_called_once_with(alias=ALIAS, branch_id=None) + + def test_human_empty(self, tmp_path: Path) -> None: + config_dir = tmp_path / "c" + config_dir.mkdir() + _seed(config_dir) + svc = MagicMock() + svc.list_sources.return_value = {"alias": ALIAS, "branch_id": "default", "sources": []} + result = _invoke(config_dir, svc, ["stream", "list", "--project", ALIAS]) + assert result.exit_code == 0 + assert "No Data Streams sources" in result.output + + +class TestCreate: + def test_create_json(self, tmp_path: Path) -> None: + config_dir = tmp_path / "c" + config_dir.mkdir() + _seed(config_dir) + svc = MagicMock() + svc.create_source.return_value = { + "status": "created", + "source_id": "s1", + "name": "s1", + "type": "otlp", + "endpoint": "https://stream-in/.../***", + "secret_revealed": False, + } + result = _invoke( + config_dir, + svc, + ["--json", "stream", "create-source", "--project", ALIAS, "--name", "s1"], + ) + assert result.exit_code == 0, result.output + data = json.loads(result.output)["data"] + assert data["status"] == "created" + svc.create_source.assert_called_once_with( + alias=ALIAS, + name="s1", + source_type="otlp", + branch_id=None, + if_not_exists=False, + reveal=False, + provision_sinks=True, + ) + + def test_no_sinks_flag(self, tmp_path: Path) -> None: + config_dir = tmp_path / "c" + config_dir.mkdir() + _seed(config_dir) + svc = MagicMock() + svc.create_source.return_value = {"status": "created", "source_id": "s1"} + result = _invoke( + config_dir, + svc, + ["--json", "stream", "create-source", "--project", ALIAS, "--name", "s1", "--no-sinks"], + ) + assert result.exit_code == 0, result.output + assert svc.create_source.call_args.kwargs["provision_sinks"] is False + + def test_invalid_type_rejected(self, tmp_path: Path) -> None: + config_dir = tmp_path / "c" + config_dir.mkdir() + _seed(config_dir) + svc = MagicMock() + result = _invoke( + config_dir, + svc, + ["stream", "create-source", "--project", ALIAS, "--name", "s1", "--type", "bogus"], + ) + assert result.exit_code == 2 + + +class TestDetail: + def test_detail_masked_json(self, tmp_path: Path) -> None: + config_dir = tmp_path / "c" + config_dir.mkdir() + _seed(config_dir) + svc = MagicMock() + svc.get_source_detail.return_value = { + "source_id": "s1", + "endpoint": "https://x/***", + "secret_revealed": False, + "signal_endpoints": {}, + "destination": {}, + } + result = _invoke(config_dir, svc, ["--json", "stream", "detail", "s1", "--project", ALIAS]) + assert result.exit_code == 0, result.output + svc.get_source_detail.assert_called_once_with( + alias=ALIAS, source_id="s1", name=None, branch_id=None, reveal=False + ) + + def test_detail_reveal_flag(self, tmp_path: Path) -> None: + config_dir = tmp_path / "c" + config_dir.mkdir() + _seed(config_dir) + svc = MagicMock() + svc.get_source_detail.return_value = {"source_id": "s1", "secret_revealed": True} + _invoke( + config_dir, + svc, + ["--json", "stream", "detail", "--project", ALIAS, "--name", "s1", "--reveal"], + ) + svc.get_source_detail.assert_called_once_with( + alias=ALIAS, source_id=None, name="s1", branch_id=None, reveal=True + ) + + +class TestDelete: + def test_dry_run(self, tmp_path: Path) -> None: + config_dir = tmp_path / "c" + config_dir.mkdir() + _seed(config_dir) + svc = MagicMock() + svc.delete_source.return_value = { + "status": "dry_run", + "source_id": "s1", + "branch_id": "default", + } + result = _invoke( + config_dir, + svc, + ["--json", "stream", "delete", "s1", "--project", ALIAS, "--dry-run"], + ) + assert result.exit_code == 0, result.output + assert json.loads(result.output)["data"]["status"] == "dry_run" + + def test_confirm_abort(self, tmp_path: Path) -> None: + config_dir = tmp_path / "c" + config_dir.mkdir() + _seed(config_dir) + svc = MagicMock() + result = _invoke( + config_dir, svc, ["stream", "delete", "s1", "--project", ALIAS], input_text="n\n" + ) + assert result.exit_code == 0 + assert "Aborted" in result.output + svc.delete_source.assert_not_called() + + def test_yes_skips_confirm(self, tmp_path: Path) -> None: + config_dir = tmp_path / "c" + config_dir.mkdir() + _seed(config_dir) + svc = MagicMock() + svc.delete_source.return_value = { + "status": "deleted", + "source_id": "s1", + "branch_id": "default", + } + result = _invoke(config_dir, svc, ["stream", "delete", "s1", "--project", ALIAS, "--yes"]) + assert result.exit_code == 0 + svc.delete_source.assert_called_once_with( + alias=ALIAS, source_id="s1", branch_id=None, dry_run=False + ) + + def test_api_error_exit_code(self, tmp_path: Path) -> None: + config_dir = tmp_path / "c" + config_dir.mkdir() + _seed(config_dir) + svc = MagicMock() + svc.list_sources.side_effect = KeboolaApiError( + message="boom", status_code=500, error_code="API_ERROR" + ) + result = _invoke(config_dir, svc, ["--json", "stream", "list", "--project", ALIAS]) + assert result.exit_code == 1 + assert json.loads(result.output)["status"] == "error" diff --git a/tests/test_stream_client.py b/tests/test_stream_client.py new file mode 100644 index 00000000..c1951fec --- /dev/null +++ b/tests/test_stream_client.py @@ -0,0 +1,213 @@ +"""Tests for StreamClient -- URL derivation, source/sink CRUD, task polling.""" + +from __future__ import annotations + +import json + +import pytest + +from keboola_agent_cli.errors import KeboolaApiError +from keboola_agent_cli.stream_client import StreamClient + +STACK_URL = "https://connection.keboola.com" +STREAM_BASE_URL = "https://stream.keboola.com" +TOKEN = "901-55555-fakeTestTokenDoNotUseXXXXXXXX" +BRANCH = "default" + +SAMPLE_SOURCE = { + "sourceId": "my-otlp", + "type": "otlp", + "name": "my-otlp", + "otlp": { + "url": "https://stream-in.keboola.com/otlp/123/my-otlp/SECRET", + "baseUrl": "https://stream-in.keboola.com/otlp/123/my-otlp", + "secret": "SECRET", + }, +} + + +class TestDeriveStreamUrl: + """The control-plane base URL is connection. -> stream..""" + + def test_us_stack(self) -> None: + assert ( + StreamClient._derive_service_url("https://connection.keboola.com", "stream") + == "https://stream.keboola.com" + ) + + def test_eu_stack(self) -> None: + assert ( + StreamClient._derive_service_url( + "https://connection.eu-central-1.keboola.com", "stream" + ) + == "https://stream.eu-central-1.keboola.com" + ) + + def test_gcp_stack(self) -> None: + assert ( + StreamClient._derive_service_url( + "https://connection.us-east4.gcp.keboola.com", "stream" + ) + == "https://stream.us-east4.gcp.keboola.com" + ) + + +class TestSourceCrud: + def test_list_sources(self, httpx_mock) -> None: + httpx_mock.add_response( + url=f"{STREAM_BASE_URL}/v1/branches/default/sources", + json={"sources": [SAMPLE_SOURCE]}, + ) + client = StreamClient(stack_url=STACK_URL, token=TOKEN) + try: + result = client.list_sources(BRANCH) + assert result["sources"][0]["sourceId"] == "my-otlp" + # Auth header is the Storage token, not a manage token. + assert httpx_mock.get_requests()[0].headers["X-StorageApi-Token"] == TOKEN + finally: + client.close() + + def test_get_source(self, httpx_mock) -> None: + httpx_mock.add_response( + url=f"{STREAM_BASE_URL}/v1/branches/default/sources/my-otlp", + json=SAMPLE_SOURCE, + ) + client = StreamClient(stack_url=STACK_URL, token=TOKEN) + try: + result = client.get_source(BRANCH, "my-otlp") + assert result["otlp"]["secret"] == "SECRET" + finally: + client.close() + + def test_create_source_returns_task(self, httpx_mock) -> None: + httpx_mock.add_response( + url=f"{STREAM_BASE_URL}/v1/branches/default/sources", + method="POST", + json={"taskId": "t1", "isFinished": False, "outputs": {"sourceId": "my-otlp"}}, + status_code=202, + ) + client = StreamClient(stack_url=STACK_URL, token=TOKEN) + try: + task = client.create_source(BRANCH, name="my-otlp", source_type="otlp") + assert task["taskId"] == "t1" + body = json.loads(httpx_mock.get_requests()[0].content) + assert body == {"name": "my-otlp", "type": "otlp"} + finally: + client.close() + + def test_delete_source_returns_task(self, httpx_mock) -> None: + httpx_mock.add_response( + url=f"{STREAM_BASE_URL}/v1/branches/default/sources/my-otlp", + method="DELETE", + json={"taskId": "t2", "isFinished": False}, + status_code=202, + ) + client = StreamClient(stack_url=STACK_URL, token=TOKEN) + try: + task = client.delete_source(BRANCH, "my-otlp") + assert task["taskId"] == "t2" + finally: + client.close() + + def test_list_sinks(self, httpx_mock) -> None: + httpx_mock.add_response( + url=f"{STREAM_BASE_URL}/v1/branches/default/sources/my-otlp/sinks", + json={"sinks": [{"sinkId": "logs", "table": {"tableId": "in.c-otlp-my-otlp.logs"}}]}, + ) + client = StreamClient(stack_url=STACK_URL, token=TOKEN) + try: + result = client.list_sinks(BRANCH, "my-otlp") + assert result["sinks"][0]["table"]["tableId"] == "in.c-otlp-my-otlp.logs" + finally: + client.close() + + def test_create_sink_payload(self, httpx_mock) -> None: + httpx_mock.add_response( + url=f"{STREAM_BASE_URL}/v1/branches/default/sources/my-otlp/sinks", + method="POST", + json={"taskId": "sk1", "isFinished": False}, + status_code=202, + ) + client = StreamClient(stack_url=STACK_URL, token=TOKEN) + try: + columns = [{"type": "uuid", "name": "id"}, {"type": "body", "name": "body"}] + task = client.create_sink( + BRANCH, + "my-otlp", + name="Logs", + table_id="in.c-otlp-my-otlp.logs", + columns=columns, + allowed_signals=["logs"], + ) + assert task["taskId"] == "sk1" + body = json.loads(httpx_mock.get_requests()[0].content) + assert body["type"] == "table" + assert body["allowedSignals"] == ["logs"] + assert body["table"]["tableId"] == "in.c-otlp-my-otlp.logs" + assert body["table"]["mapping"]["columns"] == columns + finally: + client.close() + + +class TestWaitForTask: + def test_already_finished(self) -> None: + client = StreamClient(stack_url=STACK_URL, token=TOKEN) + try: + finished = {"taskId": "t1", "isFinished": True, "status": "success"} + assert client.wait_for_task(finished) is finished + finally: + client.close() + + def test_polls_until_finished(self, httpx_mock) -> None: + httpx_mock.add_response( + url=f"{STREAM_BASE_URL}/v1/tasks/t1", + json={"taskId": "t1", "isFinished": True, "status": "success"}, + ) + client = StreamClient(stack_url=STACK_URL, token=TOKEN) + try: + pending = {"taskId": "t1", "isFinished": False} + done = client.wait_for_task(pending, poll_interval=0.0) + assert done["isFinished"] is True + finally: + client.close() + + def test_uses_task_url_for_polling(self, httpx_mock) -> None: + httpx_mock.add_response( + url=f"{STREAM_BASE_URL}/v1/tasks/abc/def", + json={"taskId": "abc/def", "isFinished": True, "status": "success"}, + ) + client = StreamClient(stack_url=STACK_URL, token=TOKEN) + try: + pending = { + "taskId": "abc/def", + "isFinished": False, + "url": "https://stream.keboola.com/v1/tasks/abc/def", + } + done = client.wait_for_task(pending, poll_interval=0.0) + assert done["isFinished"] is True + finally: + client.close() + + def test_error_task_raises(self) -> None: + client = StreamClient(stack_url=STACK_URL, token=TOKEN) + try: + with pytest.raises(KeboolaApiError): + client.wait_for_task({"isFinished": True, "status": "error", "error": "boom"}) + finally: + client.close() + + def test_timeout_raises(self, httpx_mock) -> None: + httpx_mock.add_response( + url=f"{STREAM_BASE_URL}/v1/tasks/t1", + json={"taskId": "t1", "isFinished": False, "status": "processing"}, + is_reusable=True, + ) + client = StreamClient(stack_url=STACK_URL, token=TOKEN) + try: + with pytest.raises(KeboolaApiError) as exc: + client.wait_for_task( + {"taskId": "t1", "isFinished": False}, timeout=0.05, poll_interval=0.0 + ) + assert exc.value.error_code == "TIMEOUT" + finally: + client.close() diff --git a/tests/test_stream_service.py b/tests/test_stream_service.py new file mode 100644 index 00000000..4a476ed5 --- /dev/null +++ b/tests/test_stream_service.py @@ -0,0 +1,232 @@ +"""Tests for StreamService -- source list/create/detail/delete + secret masking.""" + +from __future__ import annotations + +from pathlib import Path +from unittest.mock import MagicMock + +import pytest + +from keboola_agent_cli.config_store import ConfigStore +from keboola_agent_cli.errors import ConfigError, KeboolaApiError +from keboola_agent_cli.models import ProjectConfig +from keboola_agent_cli.services.stream_service import StreamService + +STACK_URL = "https://connection.keboola.com" +TOKEN = "901-55555-fakeTestTokenDoNotUseXXXXXXXX" +ALIAS = "padak" + +SECRET = "opMczZin8tCT4jARu5yKrE9pNPFZ" +FULL_URL = f"https://stream-in.keboola.com/otlp/10539/my-otlp/{SECRET}" +BASE_URL = "https://stream-in.keboola.com/otlp/10539/my-otlp" + +SOURCE = { + "sourceId": "my-otlp", + "type": "otlp", + "name": "my-otlp", + "description": "", + "otlp": {"url": FULL_URL, "baseUrl": BASE_URL, "secret": SECRET}, +} + +SINKS = { + "sinks": [ + { + "sinkId": "logs", + "allowedSignals": ["logs"], + "table": {"type": "keboola", "tableId": "in.c-otlp-my-otlp.logs"}, + }, + { + "sinkId": "traces", + "allowedSignals": ["traces"], + "table": {"type": "keboola", "tableId": "in.c-otlp-my-otlp.traces"}, + }, + ] +} + + +@pytest.fixture +def store(tmp_config_dir: Path) -> ConfigStore: + s = ConfigStore(config_dir=tmp_config_dir) + s.add_project( + ALIAS, + ProjectConfig(stack_url=STACK_URL, token=TOKEN, project_name="Padak 2.0", project_id=10539), + ) + return s + + +@pytest.fixture +def client_factory() -> tuple[MagicMock, MagicMock]: + mock = MagicMock() + mock.list_sinks.return_value = SINKS + factory = MagicMock(return_value=mock) + return factory, mock + + +def _svc(store: ConfigStore, factory: MagicMock) -> StreamService: + return StreamService(store, stream_client_factory=factory) + + +class TestListSources: + def test_list(self, store, client_factory) -> None: + factory, mock = client_factory + mock.list_sources.return_value = {"sources": [SOURCE]} + result = _svc(store, factory).list_sources(alias=ALIAS) + assert result["alias"] == ALIAS + assert result["branch_id"] == "default" + assert result["sources"][0]["source_id"] == "my-otlp" + # list view exposes only the secret-free base endpoint + assert result["sources"][0]["base_endpoint"] == BASE_URL + factory.assert_called_once_with(STACK_URL, TOKEN) + mock.close.assert_called_once() + + def test_unknown_alias_raises(self, store, client_factory) -> None: + factory, _ = client_factory + with pytest.raises(ConfigError): + _svc(store, factory).list_sources(alias="nope") + + def test_branch_override(self, store, client_factory) -> None: + factory, mock = client_factory + mock.list_sources.return_value = {"sources": []} + _svc(store, factory).list_sources(alias=ALIAS, branch_id="1234") + mock.list_sources.assert_called_once_with("1234") + + +class TestDetail: + def test_detail_masks_secret_by_default(self, store, client_factory) -> None: + factory, mock = client_factory + mock.get_source.return_value = SOURCE + result = _svc(store, factory).get_source_detail(alias=ALIAS, source_id="my-otlp") + assert SECRET not in result["endpoint"] + assert result["endpoint"].endswith("/***") + assert result["secret_revealed"] is False + # per-signal endpoints derived and masked + assert result["signal_endpoints"]["logs"].endswith("/***/v1/logs") + assert SECRET not in result["signal_endpoints"]["traces"] + # raw source echo is sanitised + assert result["source"]["otlp"]["secret"] == "***" + assert SECRET not in result["source"]["otlp"]["url"] + + def test_detail_reveal(self, store, client_factory) -> None: + factory, mock = client_factory + mock.get_source.return_value = SOURCE + result = _svc(store, factory).get_source_detail( + alias=ALIAS, source_id="my-otlp", reveal=True + ) + assert result["endpoint"] == FULL_URL + assert result["signal_endpoints"]["logs"] == f"{FULL_URL}/v1/logs" + assert result["source"]["otlp"]["secret"] == SECRET + + def test_detail_destination_from_sinks(self, store, client_factory) -> None: + factory, mock = client_factory + mock.get_source.return_value = SOURCE + result = _svc(store, factory).get_source_detail(alias=ALIAS, source_id="my-otlp") + assert result["destination"]["tables"]["logs"] == "in.c-otlp-my-otlp.logs" + assert result["destination"]["bucket"] == "in.c-otlp-my-otlp" + assert result["protocol"] == "http/protobuf" + + def test_detail_by_name(self, store, client_factory) -> None: + factory, mock = client_factory + mock.list_sources.return_value = {"sources": [SOURCE]} + result = _svc(store, factory).get_source_detail(alias=ALIAS, name="my-otlp") + assert result["source_id"] == "my-otlp" + mock.get_source.assert_not_called() + + def test_detail_name_not_found(self, store, client_factory) -> None: + factory, mock = client_factory + mock.list_sources.return_value = {"sources": []} + with pytest.raises(KeboolaApiError) as exc: + _svc(store, factory).get_source_detail(alias=ALIAS, name="ghost") + assert exc.value.error_code == "NOT_FOUND" + + def test_detail_requires_id_or_name(self, store, client_factory) -> None: + factory, _ = client_factory + with pytest.raises(ConfigError): + _svc(store, factory).get_source_detail(alias=ALIAS) + + +class TestCreate: + def test_create_provisions_three_otlp_sinks(self, store, client_factory) -> None: + factory, mock = client_factory + mock.create_source.return_value = {"taskId": "t1", "isFinished": False} + mock.wait_for_task.return_value = { + "taskId": "t1", + "isFinished": True, + "status": "success", + "outputs": {"sourceId": "my-otlp"}, + } + mock.list_sinks.return_value = {"sinks": []} # nothing yet -> all 3 created + mock.create_sink.return_value = {"taskId": "s", "isFinished": False} + mock.get_source.return_value = SOURCE + result = _svc(store, factory).create_source(alias=ALIAS, name="my-otlp") + assert result["status"] == "created" + assert result["source_id"] == "my-otlp" + # one sink per signal, into in.c-otlp-. + created_tables = sorted(c.kwargs["table_id"] for c in mock.create_sink.call_args_list) + assert created_tables == [ + "in.c-otlp-my-otlp.logs", + "in.c-otlp-my-otlp.metrics", + "in.c-otlp-my-otlp.traces", + ] + # wait_for_task: 1 source + 3 sinks + assert mock.wait_for_task.call_count == 4 + mock.get_source.assert_called_once_with("default", "my-otlp") + + def test_create_no_sinks_skips_provisioning(self, store, client_factory) -> None: + factory, mock = client_factory + mock.create_source.return_value = {"taskId": "t1", "isFinished": False} + mock.wait_for_task.return_value = {"isFinished": True, "outputs": {"sourceId": "my-otlp"}} + mock.get_source.return_value = SOURCE + result = _svc(store, factory).create_source( + alias=ALIAS, name="my-otlp", provision_sinks=False + ) + assert result["status"] == "created" + mock.create_sink.assert_not_called() + + def test_provisioning_is_idempotent(self, store, client_factory) -> None: + factory, mock = client_factory + mock.create_source.return_value = {"taskId": "t1", "isFinished": False} + mock.wait_for_task.return_value = {"isFinished": True, "outputs": {"sourceId": "my-otlp"}} + # logs + traces already exist -> only metrics is created + mock.list_sinks.return_value = { + "sinks": [ + {"allowedSignals": ["logs"], "table": {"tableId": "in.c-otlp-my-otlp.logs"}}, + {"allowedSignals": ["traces"], "table": {"tableId": "in.c-otlp-my-otlp.traces"}}, + ] + } + mock.create_sink.return_value = {"isFinished": False} + mock.get_source.return_value = SOURCE + _svc(store, factory).create_source(alias=ALIAS, name="my-otlp") + assert mock.create_sink.call_count == 1 + assert mock.create_sink.call_args.kwargs["table_id"] == "in.c-otlp-my-otlp.metrics" + + def test_http_source_skips_sink_provisioning(self, store, client_factory) -> None: + factory, mock = client_factory + mock.create_source.return_value = {"taskId": "t1", "isFinished": False} + mock.wait_for_task.return_value = {"isFinished": True, "outputs": {"sourceId": "my-http"}} + mock.get_source.return_value = {"sourceId": "my-http", "type": "http", "http": {}} + _svc(store, factory).create_source(alias=ALIAS, name="my-http", source_type="http") + mock.create_sink.assert_not_called() + + def test_if_not_exists_skips(self, store, client_factory) -> None: + factory, mock = client_factory + mock.list_sources.return_value = {"sources": [SOURCE]} + result = _svc(store, factory).create_source(alias=ALIAS, name="my-otlp", if_not_exists=True) + assert result["status"] == "skipped" + mock.create_source.assert_not_called() + + +class TestDelete: + def test_dry_run(self, store, client_factory) -> None: + factory, mock = client_factory + result = _svc(store, factory).delete_source(alias=ALIAS, source_id="my-otlp", dry_run=True) + assert result["status"] == "dry_run" + mock.delete_source.assert_not_called() + + def test_real_delete_polls(self, store, client_factory) -> None: + factory, mock = client_factory + mock.delete_source.return_value = {"taskId": "t2", "isFinished": False} + mock.wait_for_task.return_value = {"isFinished": True, "status": "success"} + result = _svc(store, factory).delete_source(alias=ALIAS, source_id="my-otlp") + assert result["status"] == "deleted" + mock.delete_source.assert_called_once_with("default", "my-otlp") + mock.wait_for_task.assert_called_once() diff --git a/tests/test_sync_cli.py b/tests/test_sync_cli.py index 0992b9b0..0b1c0fde 100644 --- a/tests/test_sync_cli.py +++ b/tests/test_sync_cli.py @@ -14,7 +14,7 @@ from keboola_agent_cli.cli import app from keboola_agent_cli.config_store import ConfigStore -from keboola_agent_cli.errors import ConfigError, KeboolaApiError +from keboola_agent_cli.errors import ConfigError, KeboolaApiError, SyncConflictError from keboola_agent_cli.models import ProjectConfig from keboola_agent_cli.services.project_service import ProjectService @@ -467,6 +467,85 @@ def test_sync_pull_api_error(self, tmp_path: Path) -> None: assert result.exit_code == 3 # auth error + def _conflict_error(self) -> SyncConflictError: + return SyncConflictError( + [ + { + "scope": "config", + "component_id": "keboola.ex-http", + "config_id": "cfg-001", + "config_name": "My HTTP Extractor", + "path": "extractor/keboola.ex-http/my-http-extractor", + } + ] + ) + + def test_sync_pull_force_conflict_human(self, tmp_path: Path) -> None: + """sync pull --force aborts with exit 1 and lists the conflict (human).""" + config_dir = tmp_path / "config" + config_dir.mkdir() + store = _setup_config(config_dir, {"prod": {"token": TEST_TOKEN}}) + + mock_sync = _make_sync_service_mock() + mock_sync.pull.side_effect = self._conflict_error() + + with ( + patch("keboola_agent_cli.cli.ConfigStore") as MockStore, + patch("keboola_agent_cli.cli.ProjectService") as MockProjService, + patch("keboola_agent_cli.cli.SyncService") as MockSyncService, + ): + MockStore.return_value = store + MockProjService.return_value = ProjectService(config_store=store) + MockSyncService.return_value = mock_sync + + result = runner.invoke( + app, + ["sync", "pull", "--project", "prod", "--force", "--directory", str(tmp_path)], + ) + + assert result.exit_code == 1 + out = _strip_ansi(result.output) + assert "conflict" in out.lower() + assert "keboola.ex-http/cfg-001" in out + + def test_sync_pull_force_conflict_json(self, tmp_path: Path) -> None: + """sync pull --force conflict emits a SYNC_CONFLICT error envelope (JSON).""" + config_dir = tmp_path / "config" + config_dir.mkdir() + store = _setup_config(config_dir, {"prod": {"token": TEST_TOKEN}}) + + mock_sync = _make_sync_service_mock() + mock_sync.pull.side_effect = self._conflict_error() + + with ( + patch("keboola_agent_cli.cli.ConfigStore") as MockStore, + patch("keboola_agent_cli.cli.ProjectService") as MockProjService, + patch("keboola_agent_cli.cli.SyncService") as MockSyncService, + ): + MockStore.return_value = store + MockProjService.return_value = ProjectService(config_store=store) + MockSyncService.return_value = mock_sync + + result = runner.invoke( + app, + [ + "--json", + "sync", + "pull", + "--project", + "prod", + "--force", + "--directory", + str(tmp_path), + ], + ) + + assert result.exit_code == 1 + payload = json.loads(result.output) + assert payload["status"] == "error" + assert payload["error"]["code"] == "SYNC_CONFLICT" + assert payload["error"]["details"]["conflicts"][0]["config_id"] == "cfg-001" + # =================================================================== # sync status CLI tests diff --git a/tests/test_sync_config_format.py b/tests/test_sync_config_format.py index 35e42916..cc4ae682 100644 --- a/tests/test_sync_config_format.py +++ b/tests/test_sync_config_format.py @@ -397,3 +397,71 @@ def test_non_hoisted_component_still_uses_configuration_extra(self) -> None: assert "_configuration_extra" in local assert local["_configuration_extra"] == {"foo": {"bar": 1}} assert "foo" not in local + + +class TestLocalRowToApiComponentIdParam: + """KFR-04: ``local_row_to_api`` accepts an explicit ``component_id``. + + A fresh-CREATE scaffold row may not carry a ``_keboola`` block yet, so the + hoist decision must be driveable by the caller's known component id. When + ``component_id`` is omitted the legacy behaviour (read from the file) holds. + """ + + def test_explicit_component_id_hoists_values_without_keboola_block(self) -> None: + """``component_id="keboola.variables"`` hoists ``values`` even when the + row file has no ``_keboola`` metadata (the KFR-04 fresh-CREATE case).""" + local = { + "version": 2, + "name": "Main", + "description": "default values", + "values": [{"name": "year_start", "value": "2016", "type": "string"}], + } + + name, _description, configuration = local_row_to_api(local, "keboola.variables") + + assert name == "Main" + assert configuration == { + "values": [{"name": "year_start", "value": "2016", "type": "string"}] + } + + def test_no_component_id_falls_back_to_keboola_block(self) -> None: + """``component_id=None`` reads the id from ``_keboola`` (back-compat).""" + local = { + "version": 2, + "name": "Main", + "description": "", + "values": [{"name": "region", "value": "eu", "type": "string"}], + "_keboola": {"component_id": "keboola.variables", "row_id": "row-1"}, + } + + _, _, configuration = local_row_to_api(local) + + assert configuration == {"values": [{"name": "region", "value": "eu", "type": "string"}]} + + def test_no_component_id_and_no_keboola_block_does_not_hoist(self) -> None: + """Without an id from either source, the row is not treated as a hoist + component: ``values`` stays out of the API body (legacy behaviour).""" + local = { + "version": 2, + "name": "Main", + "description": "", + "values": [{"name": "region", "value": "eu", "type": "string"}], + } + + _, _, configuration = local_row_to_api(local) + + assert configuration == {} + + def test_explicit_component_id_overrides_stale_keboola_block(self) -> None: + """The explicit arg wins over a (possibly stale) ``_keboola`` block.""" + local = { + "version": 2, + "name": "Main", + "description": "", + "values": [{"name": "region", "value": "eu", "type": "string"}], + "_keboola": {"component_id": "keboola.ex-db-snowflake", "row_id": "r"}, + } + + _, _, configuration = local_row_to_api(local, "keboola.variables") + + assert configuration == {"values": [{"name": "region", "value": "eu", "type": "string"}]} diff --git a/tests/test_sync_force_pull_baseline.py b/tests/test_sync_force_pull_baseline.py new file mode 100644 index 00000000..a7b2f94a --- /dev/null +++ b/tests/test_sync_force_pull_baseline.py @@ -0,0 +1,271 @@ +"""Regression tests for the `sync pull --force` baseline-corruption bug. + +Field report (kbagent v0.51.1, project 5785): a user has un-pushed local edits +to config A, then runs ``sync pull --force`` (typically to resolve an +*unrelated* config B's conflict). ``--force`` bypassed the +``locally_modified`` guard in ``SyncService.pull()``, so when config A's remote +was unchanged the ``remote_unchanged`` short-circuit re-stamped A's manifest +``pull_hash`` from the *edited on-disk file*. Afterwards ``sync diff`` / +``sync push`` believed A was in sync and silently shipped nothing -- the local +edits were stranded while the remote still held the old config. + +The fix splits ``--force`` behaviour by 3-way diff state: + +* (b) local edited, remote UNCHANGED -> preserve the file AND the 3-way base, + so the pending delta stays visible to ``sync push`` (no data loss). +* (a) local edited, remote ALSO changed (true merge conflict) -> abort the pull + with ``SyncConflictError`` before writing anything; the user resolves. + +These tests pin both halves, at config and row granularity. +""" + +from pathlib import Path +from unittest.mock import MagicMock + +import pytest +import yaml + +from helpers import setup_single_project +from keboola_agent_cli.constants import CONFIG_FILENAME +from keboola_agent_cli.errors import SyncConflictError +from keboola_agent_cli.models import TokenVerifyResponse +from keboola_agent_cli.services.sync_service import SyncService + +SAMPLE_VERIFY_TOKEN = TokenVerifyResponse( + token_id="tok-001", + token_description="kbagent-cli", + project_id=258, + project_name="Production", + owner_name="My Org", +) + +SAMPLE_BRANCHES = [ + {"id": 12345, "name": "Main", "isDefault": True}, +] + + +def _http_extractor(base_url: str, rows: list | None = None) -> list: + """Single keboola.ex-http config carrying ``base_url`` (+ optional rows).""" + return [ + { + "id": "keboola.ex-http", + "type": "extractor", + "configurations": [ + { + "id": "cfg-001", + "name": "My HTTP Extractor", + "description": "Fetches data", + "configuration": {"parameters": {"baseUrl": base_url}}, + "rows": rows if rows is not None else [], + } + ], + }, + ] + + +def _row(path: str) -> dict: + return { + "id": "row-001", + "name": "Users Endpoint", + "description": "", + "configuration": {"parameters": {"path": path}}, + } + + +# Remote states used across the tests. +REMOTE_V1 = _http_extractor("https://api.example.com") +REMOTE_V2 = _http_extractor("https://api-v2.example.com") # remote moved on +REMOTE_V1_WITH_ROW = _http_extractor("https://api.example.com", rows=[_row("/users")]) +REMOTE_V2_WITH_ROW = _http_extractor("https://api.example.com", rows=[_row("/people")]) + + +def _make_mock_client( + verify_token_response: TokenVerifyResponse | None = None, + components_response: list | None = None, + branches_response: list | None = None, +) -> MagicMock: + client = MagicMock() + client.__enter__ = MagicMock(return_value=client) + client.__exit__ = MagicMock(return_value=False) + if verify_token_response: + client.verify_token.return_value = verify_token_response + if components_response is not None: + client.list_components_with_configs.return_value = components_response + if branches_response is not None: + client.list_dev_branches.return_value = branches_response + return client + + +def _svc(store, components: list | None = None) -> SyncService: + return SyncService( + config_store=store, + client_factory=lambda url, token: _make_mock_client( + verify_token_response=SAMPLE_VERIFY_TOKEN, + components_response=components, + branches_response=SAMPLE_BRANCHES, + ), + ) + + +def _init_and_pull(tmp_config_dir: Path, project_root: Path, remote: list) -> object: + """init + first pull, returning the ConfigStore.""" + store = setup_single_project(tmp_config_dir) + _svc(store).init_sync(alias="prod", project_root=project_root) + _svc(store, remote).pull(alias="prod", project_root=project_root) + return store + + +def _config_yml(project_root: Path, under_rows: bool) -> Path: + """Locate the config or row _config.yml under the pulled tree.""" + files = list(project_root.rglob(CONFIG_FILENAME)) + matches = [f for f in files if ("rows" in f.parts) == under_rows] + assert len(matches) == 1, f"expected exactly one {'row' if under_rows else 'config'} file" + return matches[0] + + +def _edit_param(config_file: Path, key: str, value: str) -> None: + data = yaml.safe_load(config_file.read_text(encoding="utf-8")) + data["parameters"][key] = value + config_file.write_text(yaml.dump(data, default_flow_style=False), encoding="utf-8") + + +# =================================================================== +# Config-level +# =================================================================== + + +def test_force_pull_preserves_unpushed_local_edits(tmp_config_dir: Path, tmp_path: Path) -> None: + """(b) force-pull, remote unchanged -> local edit preserved, still pushable.""" + project_root = tmp_path / "project" + project_root.mkdir() + store = _init_and_pull(tmp_config_dir, project_root, REMOTE_V1) + + config_file = _config_yml(project_root, under_rows=False) + _edit_param(config_file, "baseUrl", "https://changed.example.com") + + # Healthy before the force-pull. + assert _svc(store, REMOTE_V1).diff("prod", project_root)["summary"]["modified"] == 1 + + # Force-pull with the SAME remote (run to adopt some *other* config's state). + _svc(store, REMOTE_V1).pull("prod", project_root, force=True) + + # The on-disk file still holds the edit (force did not revert it). + after = yaml.safe_load(config_file.read_text(encoding="utf-8")) + assert after["parameters"]["baseUrl"] == "https://changed.example.com" + + # And the pending delta is STILL detected -- the bug stranded it silently. + diff_after = _svc(store, REMOTE_V1).diff("prod", project_root) + assert diff_after["summary"]["modified"] == 1, ( + "force-pull (remote unchanged) silently dropped the un-pushed local edit" + ) + assert diff_after["changes"][0]["change_type"] == "modified" + + +def test_force_pull_aborts_on_true_conflict(tmp_config_dir: Path, tmp_path: Path) -> None: + """(a) force-pull, remote ALSO changed -> abort with SyncConflictError.""" + project_root = tmp_path / "project" + project_root.mkdir() + store = _init_and_pull(tmp_config_dir, project_root, REMOTE_V1) + + config_file = _config_yml(project_root, under_rows=False) + _edit_param(config_file, "baseUrl", "https://changed.example.com") + + with pytest.raises(SyncConflictError) as excinfo: + _svc(store, REMOTE_V2).pull("prod", project_root, force=True) + + conflicts = excinfo.value.conflicts + assert len(conflicts) == 1 + assert conflicts[0]["scope"] == "config" + assert conflicts[0]["component_id"] == "keboola.ex-http" + assert conflicts[0]["config_id"] == "cfg-001" + + # Abort left the local edit intact (nothing was written). + after = yaml.safe_load(config_file.read_text(encoding="utf-8")) + assert after["parameters"]["baseUrl"] == "https://changed.example.com" + + # The conflict remains visible to diff (base preserved, not corrupted). + diff_after = _svc(store, REMOTE_V2).diff("prod", project_root) + assert diff_after["summary"].get("conflict", 0) == 1 + + +def test_force_pull_no_conflict_when_only_remote_changed( + tmp_config_dir: Path, tmp_path: Path +) -> None: + """Remote changed but local untouched is NOT a conflict -- force takes remote.""" + project_root = tmp_path / "project" + project_root.mkdir() + store = _init_and_pull(tmp_config_dir, project_root, REMOTE_V1) + + # No local edit; remote moved to V2. + _svc(store, REMOTE_V2).pull("prod", project_root, force=True) + + config_file = _config_yml(project_root, under_rows=False) + after = yaml.safe_load(config_file.read_text(encoding="utf-8")) + assert after["parameters"]["baseUrl"] == "https://api-v2.example.com" + + +# =================================================================== +# Row-level (same 3-way rule per row) +# =================================================================== + + +def test_force_pull_preserves_unpushed_row_edits(tmp_config_dir: Path, tmp_path: Path) -> None: + """(b) row edited, remote unchanged -> row preserved, still pushable.""" + project_root = tmp_path / "project" + project_root.mkdir() + store = _init_and_pull(tmp_config_dir, project_root, REMOTE_V1_WITH_ROW) + + row_file = _config_yml(project_root, under_rows=True) + _edit_param(row_file, "path", "/changed") + + _svc(store, REMOTE_V1_WITH_ROW).pull("prod", project_root, force=True) + + after = yaml.safe_load(row_file.read_text(encoding="utf-8")) + assert after["parameters"]["path"] == "/changed" + + diff_after = _svc(store, REMOTE_V1_WITH_ROW).diff("prod", project_root) + assert diff_after["summary"]["modified"] == 1, "force-pull stranded the un-pushed row edit" + + +def test_force_pull_aborts_on_row_conflict(tmp_config_dir: Path, tmp_path: Path) -> None: + """(a) row edited AND remote row changed -> abort with SyncConflictError.""" + project_root = tmp_path / "project" + project_root.mkdir() + store = _init_and_pull(tmp_config_dir, project_root, REMOTE_V1_WITH_ROW) + + row_file = _config_yml(project_root, under_rows=True) + _edit_param(row_file, "path", "/changed") + + with pytest.raises(SyncConflictError) as excinfo: + _svc(store, REMOTE_V2_WITH_ROW).pull("prod", project_root, force=True) + + scopes = {c["scope"] for c in excinfo.value.conflicts} + assert "row" in scopes + + +# =================================================================== +# --all-projects keeps the conflict structured (not a flat string) +# =================================================================== + + +def test_force_pull_all_projects_emits_structured_conflict( + tmp_config_dir: Path, tmp_path: Path +) -> None: + """`pull_all` must surface SYNC_CONFLICT + conflicts, not just `str(exc)`.""" + base_dir = tmp_path / "base" + project_root = base_dir / "prod" + project_root.mkdir(parents=True) + store = _init_and_pull(tmp_config_dir, project_root, REMOTE_V1) + + config_file = _config_yml(project_root, under_rows=False) + _edit_param(config_file, "baseUrl", "https://changed.example.com") + + # Remote moved on -> conflict. pull_all catches it per-project; assert it + # keeps the structured payload a JSON consumer needs (not a flat string). + result = _svc(store, REMOTE_V2).pull_all(base_dir, force=True) + proj = result["projects"]["prod"] + assert proj.get("error_code") == "SYNC_CONFLICT" + assert proj.get("conflicts"), "conflicts list must be present on the error entry" + assert proj["conflicts"][0]["config_id"] == "cfg-001" + assert result["summary"]["failed"] == 1 + assert result["summary"]["success"] == 0 diff --git a/tests/test_sync_service.py b/tests/test_sync_service.py index 856feba3..cf6c51f5 100644 --- a/tests/test_sync_service.py +++ b/tests/test_sync_service.py @@ -2799,3 +2799,985 @@ def test_passes_for_absolute_path_inside_branch(self, tmp_path: Path) -> None: # Resolve and then re-pass: should still pass _ensure_within_branch(branch_dir, config_dir.resolve(), "comp", "id") + + +# --------------------------------------------------------------------------- +# Fresh-CREATE writeback + KBC.* metadata propagation (v0.47.0 / FIIA migration) +# --------------------------------------------------------------------------- + + +class TestFreshCreateWriteback: + """Cover the fresh-CREATE manifest writeback + KBC.* metadata propagation. + + Closes the gap where a downstream caller (FIIA / scaffold-style emitter) + pre-populates manifest entries with placeholder ids and folder metadata + before the first ``sync push``. Pre-v0.47.0 every create unconditionally + appended a new manifest entry (manifest doubled in size, re-pushes flagged + every placeholder as ``added`` again, ``KBC.configuration.folderName`` + silently dropped on the floor). + """ + + @staticmethod + def _make_svc(tmp_config_dir: Path) -> SyncService: + return SyncService(config_store=setup_single_project(tmp_config_dir)) + + def test_writeback_config_in_place_updates_placeholder(self, tmp_config_dir: Path) -> None: + """A placeholder entry at the same (branch_id, component_id, path) is + updated in place; the manifest does not grow.""" + from keboola_agent_cli.sync.manifest import ManifestConfiguration + + svc = self._make_svc(tmp_config_dir) + manifest = Manifest.model_construct( + project={"id": 1, "apiHost": "connection.keboola.com"}, # type: ignore[arg-type] + naming={"config": "{component_type}/{component_id}/{config_name}"}, # type: ignore[arg-type] + configurations=[ + ManifestConfiguration( + branchId=12345, + componentId="keboola.snowflake-transformation", + id="PLACEHOLDER-TX1", + path="transformation/keboola.snowflake-transformation/01_stage", + metadata={"KBC.configuration.folderName": "FI Pipeline"}, + ) + ], + ) + + writeback = svc._writeback_create_config_in_manifest( + manifest=manifest, + component_id="keboola.snowflake-transformation", + branch_id=12345, + config_path_str="transformation/keboola.snowflake-transformation/01_stage", + new_id="123456789", + file_hash="abc123", + cfg_hash="def456", + ) + + entry = writeback.entry + assert writeback.previous_id == "PLACEHOLDER-TX1", ( + "previous_id must capture the pre-overwrite placeholder for remapping" + ) + assert len(manifest.configurations) == 1, "must not append a duplicate" + assert entry.id == "123456789" + assert entry.branch_id == 12345 + assert entry.metadata["pull_hash"] == "abc123" + assert entry.metadata["pull_config_hash"] == "def456" + assert entry.metadata["KBC.configuration.folderName"] == "FI Pipeline", ( + "user-declared KBC.* metadata must survive the writeback" + ) + + def test_writeback_config_does_not_match_across_branches(self, tmp_config_dir: Path) -> None: + """A placeholder at the same (component_id, path) but a different + branch must NOT be matched. The new entry is appended; the other + branch's entry is left untouched.""" + from keboola_agent_cli.sync.manifest import ManifestConfiguration + + svc = self._make_svc(tmp_config_dir) + # Placeholder for branch 12345 (e.g. main); we push to dev branch 99999. + manifest = Manifest.model_construct( + project={"id": 1, "apiHost": "connection.keboola.com"}, # type: ignore[arg-type] + naming={"config": "{component_type}/{component_id}/{config_name}"}, # type: ignore[arg-type] + configurations=[ + ManifestConfiguration( + branchId=12345, + componentId="keboola.snowflake-transformation", + id="main-id-001", + path="transformation/keboola.snowflake-transformation/01_stage", + metadata={"KBC.configuration.folderName": "Main FI"}, + ) + ], + ) + + writeback = svc._writeback_create_config_in_manifest( + manifest=manifest, + component_id="keboola.snowflake-transformation", + branch_id=99999, + config_path_str="transformation/keboola.snowflake-transformation/01_stage", + new_id="dev-id-002", + file_hash="h1", + cfg_hash="h2", + ) + + entry = writeback.entry + # A brand-new entry was appended: no placeholder to remap. + assert writeback.previous_id == "" + # Two entries: the main-branch one untouched, plus the new dev-branch + # one we just appended. + assert len(manifest.configurations) == 2 + main_entry = next(c for c in manifest.configurations if c.branch_id == 12345) + assert main_entry.id == "main-id-001" + assert main_entry.metadata == {"KBC.configuration.folderName": "Main FI"} + # The returned entry is the newly-appended dev-branch one. + assert entry.branch_id == 99999 + assert entry.id == "dev-id-002" + + def test_writeback_config_appends_when_no_placeholder(self, tmp_config_dir: Path) -> None: + """If no placeholder exists at the path, append (legacy fallback).""" + svc = self._make_svc(tmp_config_dir) + manifest = Manifest.model_construct( + project={"id": 1, "apiHost": "connection.keboola.com"}, # type: ignore[arg-type] + naming={"config": "{component_type}/{component_id}/{config_name}"}, # type: ignore[arg-type] + configurations=[], + ) + + writeback = svc._writeback_create_config_in_manifest( + manifest=manifest, + component_id="keboola.ex-http", + branch_id=0, + config_path_str="extractor/keboola.ex-http/my-new-config", + new_id="999", + file_hash="h1", + cfg_hash="h2", + ) + + entry = writeback.entry + assert writeback.previous_id == "" + assert len(manifest.configurations) == 1 + assert manifest.configurations[0] is entry + assert entry.id == "999" + assert entry.metadata == {"pull_hash": "h1", "pull_config_hash": "h2"} + + def test_propagate_kbc_metadata_calls_set_config_metadata(self, tmp_config_dir: Path) -> None: + """KBC.* keys are POSTed via client.set_config_metadata; bookkeeping + keys (``pull_hash``, ...) are filtered out.""" + from keboola_agent_cli.sync.manifest import ManifestConfiguration + + svc = self._make_svc(tmp_config_dir) + entry = ManifestConfiguration( + branchId=0, + componentId="keboola.snowflake-transformation", + id="cfg-123", + path="x", + metadata={ + "pull_hash": "h1", + "pull_config_hash": "h2", + "KBC.configuration.folderName": "FI Pipeline", + "KBC.configuration.category": "transformation", + }, + ) + client = MagicMock() + + svc._propagate_kbc_metadata(client, entry, branch_id=99) + + client.set_config_metadata.assert_called_once() + call = client.set_config_metadata.call_args + assert call.kwargs["component_id"] == "keboola.snowflake-transformation" + assert call.kwargs["config_id"] == "cfg-123" + assert call.kwargs["branch_id"] == 99 + entries = dict(call.kwargs["entries"]) + assert entries == { + "KBC.configuration.folderName": "FI Pipeline", + "KBC.configuration.category": "transformation", + }, "pull_* bookkeeping keys must not be sent to the metadata API" + + def test_propagate_kbc_metadata_noop_when_no_kbc_keys(self, tmp_config_dir: Path) -> None: + """No KBC.* keys → no API call (don't waste a round-trip).""" + from keboola_agent_cli.sync.manifest import ManifestConfiguration + + svc = self._make_svc(tmp_config_dir) + entry = ManifestConfiguration( + branchId=0, + componentId="x", + id="y", + path="z", + metadata={"pull_hash": "h", "pull_config_hash": "h2"}, + ) + client = MagicMock() + + result = svc._propagate_kbc_metadata(client, entry, branch_id=None) + + client.set_config_metadata.assert_not_called() + assert result is None + + def test_propagate_kbc_metadata_returns_error_message_on_api_failure( + self, tmp_config_dir: Path + ) -> None: + """A failed metadata POST returns the error message (caller accumulates + into the push errors list) instead of aborting the push mid-loop.""" + from keboola_agent_cli.errors import ErrorCode, KeboolaApiError + from keboola_agent_cli.sync.manifest import ManifestConfiguration + + svc = self._make_svc(tmp_config_dir) + entry = ManifestConfiguration( + branchId=0, + componentId="keboola.snowflake-transformation", + id="cfg-123", + path="x", + metadata={"KBC.configuration.folderName": "FI Pipeline"}, + ) + client = MagicMock() + client.set_config_metadata.side_effect = KeboolaApiError( + message="metastore 500", + status_code=500, + error_code=ErrorCode.API_ERROR, + ) + + result = svc._propagate_kbc_metadata(client, entry, branch_id=None) + + assert result == "metastore 500", ( + "non-fatal metadata failure must return the message for the caller " + "to accumulate into the push error list" + ) + client.set_config_metadata.assert_called_once() + + def test_writeback_row_in_place_updates_placeholder(self, tmp_config_dir: Path) -> None: + """A placeholder row at the same ``path`` is updated in place; parent's + rows list does not grow.""" + from keboola_agent_cli.sync.manifest import ManifestConfigRow, ManifestConfiguration + + svc = self._make_svc(tmp_config_dir) + parent = ManifestConfiguration( + branchId=0, + componentId="keboola.variables", + id="vars-001", + path="other/keboola.variables/shared", + rows=[ + ManifestConfigRow( + id="PLACEHOLDER-ROW", + path="rows/default", + metadata={}, + ) + ], + ) + + row = svc._writeback_create_row_in_manifest( + parent=parent, + row_path_str="rows/default", + new_row_id="vals-real-id", + file_hash="rh1", + cfg_hash="rh2", + ) + + assert len(parent.rows) == 1, "must not append a duplicate row" + assert row.id == "vals-real-id" + assert row.metadata == {"pull_hash": "rh1", "pull_config_hash": "rh2"} + + def test_writeback_row_appends_when_no_placeholder(self, tmp_config_dir: Path) -> None: + """No placeholder row → append (legacy fallback for untracked rows).""" + from keboola_agent_cli.sync.manifest import ManifestConfiguration + + svc = self._make_svc(tmp_config_dir) + parent = ManifestConfiguration( + branchId=0, + componentId="keboola.ex-http", + id="cfg-1", + path="extractor/keboola.ex-http/my-ext", + rows=[], + ) + + row = svc._writeback_create_row_in_manifest( + parent=parent, + row_path_str="rows/new", + new_row_id="row-001", + file_hash="rh1", + cfg_hash="rh2", + ) + + assert len(parent.rows) == 1 + assert parent.rows[0] is row + assert row.id == "row-001" + + def test_push_create_with_placeholder_is_idempotent_and_propagates_folder( + self, tmp_config_dir: Path, tmp_path: Path + ) -> None: + """End-to-end push: placeholder + KBC.configuration.folderName. + + Round-trip: + 1. Init a project (empty manifest). + 2. Hand-populate a placeholder ManifestConfiguration with a + ``KBC.configuration.folderName`` metadata key, save it, and write + a matching ``_config.yml`` file. + 3. Run sync push — assert: created=1, manifest length stays at 1 + (placeholder updated in place to real ULID), client.create_config + was called, client.set_config_metadata was called with the folder + metadata. + 4. Run sync push a second time — assert: created=0, errors=0 + (idempotency naturally follows from writeback-in-place). + """ + from keboola_agent_cli.constants import CONFIG_YML_VERSION + from keboola_agent_cli.sync.manifest import ManifestConfiguration, save_manifest + + project_root = tmp_path / "project" + project_root.mkdir() + + init_client = _make_sync_mock_client( + verify_token_response=SAMPLE_VERIFY_TOKEN, + branches_response=SAMPLE_BRANCHES, + ) + store = setup_single_project(tmp_config_dir) + init_svc = SyncService( + config_store=store, + client_factory=lambda url, token: init_client, + ) + init_svc.init_sync(alias="prod", project_root=project_root) + + # Hand-author a placeholder manifest entry + local _config.yml at the + # corresponding path. This is the FIIA / scaffold emit pattern. + manifest = load_manifest(project_root) + placeholder_path = "transformation/keboola.snowflake-transformation/01_stage" + manifest.configurations.append( + ManifestConfiguration( + branchId=12345, + componentId="keboola.snowflake-transformation", + id="PLACEHOLDER-TX1", + path=placeholder_path, + metadata={"KBC.configuration.folderName": "FI Pipeline"}, + ) + ) + save_manifest(project_root, manifest) + + branch_path = manifest.branches[0].path + config_dir = project_root / branch_path / placeholder_path + config_dir.mkdir(parents=True) + (config_dir / CONFIG_FILENAME).write_text( + yaml.dump( + { + "version": CONFIG_YML_VERSION, + "name": "01 Stage", + "description": "Staging transformation", + "parameters": {}, + "_keboola": { + "component_id": "keboola.snowflake-transformation", + "config_id": "", + }, + }, + default_flow_style=False, + ), + encoding="utf-8", + ) + + push_client = _make_sync_mock_client(components_response=[]) + push_client.create_config.return_value = {"id": "999000111"} + + push_svc = SyncService( + config_store=store, + client_factory=lambda url, token: push_client, + ) + + # First push: placeholder → real ULID, KBC.* propagated. + result = push_svc.push(alias="prod", project_root=project_root) + assert result["status"] == "pushed" + assert result["created"] == 1 + assert result["errors"] == [] + push_client.create_config.assert_called_once() + push_client.set_config_metadata.assert_called_once() + meta_call = push_client.set_config_metadata.call_args + assert meta_call.kwargs["component_id"] == "keboola.snowflake-transformation" + assert meta_call.kwargs["config_id"] == "999000111" + assert dict(meta_call.kwargs["entries"]) == { + "KBC.configuration.folderName": "FI Pipeline", + } + + # Manifest must have updated the placeholder in place — NOT appended. + post = load_manifest(project_root) + matching = [ + c + for c in post.configurations + if c.component_id == "keboola.snowflake-transformation" and c.path == placeholder_path + ] + assert len(matching) == 1, "writeback must update placeholder in place" + assert matching[0].id == "999000111" + assert matching[0].metadata.get("KBC.configuration.folderName") == "FI Pipeline" + + # Second push against the now-real manifest must be a no-op. + # The remote side reports the created config so the diff sees it as + # present and unchanged. + push_client2 = _make_sync_mock_client( + components_response=[ + { + "id": "keboola.snowflake-transformation", + "type": "transformation", + "configurations": [ + { + "id": "999000111", + "name": "01 Stage", + "description": "Staging transformation", + "configuration": {"parameters": {}}, + "rows": [], + } + ], + } + ], + ) + push_svc2 = SyncService( + config_store=store, + client_factory=lambda url, token: push_client2, + ) + result2 = push_svc2.push(alias="prod", project_root=project_root) + # Idempotent re-push: diff sees no changes, so service short-circuits + # to status="no_changes" without ever entering the create path. + assert result2["status"] in ("no_changes", "pushed") + assert result2.get("created", 0) == 0, "re-push must be idempotent" + push_client2.create_config.assert_not_called() + + +class TestFreshCreateVariableBinding: + """Fresh-CREATE variable-link resolution (KFR-03 / KFR-04 / KFR-05). + + A FIIA / scaffold tree emits a ``keboola.variables`` config + its default + values row + a transformation that cross-references both by placeholder id. + One ``sync push`` must create all three, remap the row's parent placeholder + to the assigned ULID, hoist the row ``values``, and rebind the + transformation's ``variables_id`` / ``variables_values_id`` to ULIDs. + """ + + TX_COMPONENT = "keboola.snowflake-transformation" + VARS_COMPONENT = "keboola.variables" + + @staticmethod + def _init(tmp_config_dir: Path, project_root: Path) -> ConfigStore: + init_client = _make_sync_mock_client( + verify_token_response=SAMPLE_VERIFY_TOKEN, + branches_response=SAMPLE_BRANCHES, + ) + store = setup_single_project(tmp_config_dir) + SyncService( + config_store=store, + client_factory=lambda url, token: init_client, + ).init_sync(alias="prod", project_root=project_root) + return store + + def _author_tree( + self, + project_root: Path, + *, + tx_vars_placeholder: str = "PH-VARS", + tx_vals_placeholder: str = "PH-VALS", + vars_manifest_id: str = "PH-VARS", + vals_manifest_id: str = "PH-VALS", + with_values_row: bool = True, + ) -> None: + """Write the placeholder manifest entries + local files on disk.""" + from keboola_agent_cli.constants import CONFIG_YML_VERSION + from keboola_agent_cli.sync.manifest import ( + ManifestConfigRow, + ManifestConfiguration, + save_manifest, + ) + + manifest = load_manifest(project_root) + branch_path = manifest.branches[0].path + + vars_path = "variable/keboola.variables/my_vars" + tx_path = "transformation/keboola.snowflake-transformation/01_stage" + + vars_entry = ManifestConfiguration( + branchId=12345, + componentId=self.VARS_COMPONENT, + id=vars_manifest_id, + path=vars_path, + ) + if with_values_row: + vars_entry.rows.append( + ManifestConfigRow(id=vals_manifest_id, path="rows/default", metadata={}) + ) + manifest.configurations.append(vars_entry) + manifest.configurations.append( + ManifestConfiguration( + branchId=12345, + componentId=self.TX_COMPONENT, + id="PH-TX", + path=tx_path, + ) + ) + save_manifest(project_root, manifest) + + vars_dir = project_root / branch_path / vars_path + vars_dir.mkdir(parents=True) + (vars_dir / CONFIG_FILENAME).write_text( + yaml.dump( + { + "version": CONFIG_YML_VERSION, + "name": "My Vars", + "description": "", + "_keboola": {"component_id": self.VARS_COMPONENT, "config_id": ""}, + }, + default_flow_style=False, + ), + encoding="utf-8", + ) + + if with_values_row: + row_dir = vars_dir / "rows" / "default" + row_dir.mkdir(parents=True) + (row_dir / CONFIG_FILENAME).write_text( + yaml.dump( + { + "version": CONFIG_YML_VERSION, + "name": "default", + "description": "", + # Top-level hoisted values (KFR-04): no _keboola block so + # only the explicit component_id can drive the hoist. + "values": [{"name": "year", "value": "2016", "type": "string"}], + }, + default_flow_style=False, + ), + encoding="utf-8", + ) + + tx_dir = project_root / branch_path / tx_path + tx_dir.mkdir(parents=True) + (tx_dir / CONFIG_FILENAME).write_text( + yaml.dump( + { + "version": CONFIG_YML_VERSION, + "name": "01 Stage", + "description": "", + "parameters": {}, + "_configuration_extra": { + "variables_id": tx_vars_placeholder, + "variables_values_id": tx_vals_placeholder, + }, + "_keboola": {"component_id": self.TX_COMPONENT, "config_id": ""}, + }, + default_flow_style=False, + ), + encoding="utf-8", + ) + + def _make_create_client(self) -> MagicMock: + client = _make_sync_mock_client(components_response=[]) + client.verify_token.return_value = SAMPLE_VERIFY_TOKEN + + def fake_create_config(**kwargs: Any) -> dict[str, str]: + cid = kwargs["component_id"] + return {"id": "VARS-9" if cid == self.VARS_COMPONENT else "TX-9"} + + client.create_config.side_effect = fake_create_config + client.create_config_row.return_value = {"id": "VALS-9"} + client.update_config.return_value = {"id": "TX-9"} + return client + + def _remote_after_create(self) -> list[dict[str, Any]]: + """Remote state mirroring the post-push tree (for idempotency).""" + return [ + { + "id": self.VARS_COMPONENT, + "type": "other", + "configurations": [ + { + "id": "VARS-9", + "name": "My Vars", + "description": "", + "configuration": {}, + "rows": [ + { + "id": "VALS-9", + "name": "default", + "description": "", + "configuration": { + "values": [{"name": "year", "value": "2016", "type": "string"}] + }, + } + ], + } + ], + }, + { + "id": self.TX_COMPONENT, + "type": "transformation", + "configurations": [ + { + "id": "TX-9", + "name": "01 Stage", + "description": "", + "configuration": { + "parameters": {}, + "variables_id": "VARS-9", + "variables_values_id": "VALS-9", + }, + "rows": [], + } + ], + }, + ] + + def test_push_resolves_bindings_end_to_end(self, tmp_config_dir: Path, tmp_path: Path) -> None: + """One push: 3 creates, row parent remapped, values hoisted, links rebound.""" + project_root = tmp_path / "project" + project_root.mkdir() + store = self._init(tmp_config_dir, project_root) + self._author_tree(project_root) + + client = self._make_create_client() + svc = SyncService(config_store=store, client_factory=lambda url, token: client) + result = svc.push(alias="prod", project_root=project_root) + + assert result["status"] == "pushed" + assert result["created"] == 3, result + assert result["errors"] == [], result["errors"] + + # KFR-05: row POSTed against the freshly-assigned parent ULID, not the + # placeholder. KFR-04: the hoisted values reached the API body. + client.create_config_row.assert_called_once() + row_kwargs = client.create_config_row.call_args.kwargs + assert row_kwargs["config_id"] == "VARS-9", "row parent must be remapped to ULID" + assert row_kwargs["configuration"].get("values"), "row values must be hoisted" + + # KFR-03: a single update_config PUT rebinds BOTH ids to ULIDs. + client.update_config.assert_called_once() + upd = client.update_config.call_args.kwargs + assert upd["component_id"] == self.TX_COMPONENT + assert upd["config_id"] == "TX-9" + assert upd["configuration"]["variables_id"] == "VARS-9" + assert upd["configuration"]["variables_values_id"] == "VALS-9" + assert "Resolve variables link" in upd["change_description"] + # MUST NOT call set_variables (would create a 2nd variables config). + client.set_variables.assert_not_called() + + # Local file rewritten to ULIDs. + manifest = load_manifest(project_root) + tx_entry = next(c for c in manifest.configurations if c.component_id == self.TX_COMPONENT) + assert tx_entry.id == "TX-9" + branch_path = manifest.branches[0].path + tx_local = yaml.safe_load( + (project_root / branch_path / tx_entry.path / CONFIG_FILENAME).read_text("utf-8") + ) + assert tx_local["_configuration_extra"]["variables_id"] == "VARS-9" + assert tx_local["_configuration_extra"]["variables_values_id"] == "VALS-9" + + # Manifest hashes refreshed from the post-rewrite (ULID) state so a + # re-push is clean: pull_config_hash must equal config_hash(local). + from keboola_agent_cli.sync.diff_engine import config_hash + + assert tx_entry.metadata["pull_config_hash"] == config_hash(tx_local) + + def test_push_resolves_bindings_idempotent_repush( + self, tmp_config_dir: Path, tmp_path: Path + ) -> None: + """A second push over the mutated tree is a no-op (created==0, errors==0).""" + project_root = tmp_path / "project" + project_root.mkdir() + store = self._init(tmp_config_dir, project_root) + self._author_tree(project_root) + + svc = SyncService( + config_store=store, client_factory=lambda url, token: self._make_create_client() + ) + svc.push(alias="prod", project_root=project_root) + + repush_client = _make_sync_mock_client(components_response=self._remote_after_create()) + repush_svc = SyncService( + config_store=store, client_factory=lambda url, token: repush_client + ) + result2 = repush_svc.push(alias="prod", project_root=project_root) + + assert result2.get("created", 0) == 0, result2 + assert result2.get("errors", []) == [], result2 + repush_client.create_config.assert_not_called() + repush_client.create_config_row.assert_not_called() + repush_client.update_config.assert_not_called() + + def test_fallback_single_variables_config_binds_with_warning( + self, tmp_config_dir: Path, tmp_path: Path, caplog: pytest.LogCaptureFixture + ) -> None: + """Placeholder mismatch + exactly one created variables config → bind + warn.""" + project_root = tmp_path / "project" + project_root.mkdir() + store = self._init(tmp_config_dir, project_root) + # Transformation references a placeholder that does NOT match the + # variables manifest entry's placeholder id. + self._author_tree( + project_root, + tx_vars_placeholder="WRONG-VARS", + tx_vals_placeholder="WRONG-VALS", + ) + + client = self._make_create_client() + svc = SyncService(config_store=store, client_factory=lambda url, token: client) + with caplog.at_level("WARNING"): + result = svc.push(alias="prod", project_root=project_root) + + assert result["created"] == 3 + assert result["errors"] == [] + client.update_config.assert_called_once() + upd = client.update_config.call_args.kwargs + assert upd["configuration"]["variables_id"] == "VARS-9" + assert upd["configuration"]["variables_values_id"] == "VALS-9" + assert any("did not match" in r.message for r in caplog.records) + + def test_ambiguous_variables_configs_error_no_broken_link( + self, tmp_config_dir: Path, tmp_path: Path + ) -> None: + """>1 created variables configs + no placeholder match → error, no PUT.""" + from keboola_agent_cli.constants import CONFIG_YML_VERSION + from keboola_agent_cli.sync.manifest import ManifestConfiguration, save_manifest + + project_root = tmp_path / "project" + project_root.mkdir() + store = self._init(tmp_config_dir, project_root) + # Two variables configs, transformation points at a non-matching id. + self._author_tree( + project_root, + tx_vars_placeholder="WRONG-VARS", + tx_vals_placeholder="WRONG-VALS", + ) + manifest = load_manifest(project_root) + branch_path = manifest.branches[0].path + second_vars_path = "variable/keboola.variables/other_vars" + manifest.configurations.append( + ManifestConfiguration( + branchId=12345, + componentId=self.VARS_COMPONENT, + id="PH-VARS-2", + path=second_vars_path, + ) + ) + save_manifest(project_root, manifest) + other_dir = project_root / branch_path / second_vars_path + other_dir.mkdir(parents=True) + (other_dir / CONFIG_FILENAME).write_text( + yaml.dump( + { + "version": CONFIG_YML_VERSION, + "name": "Other Vars", + "description": "", + "_keboola": {"component_id": self.VARS_COMPONENT, "config_id": ""}, + }, + default_flow_style=False, + ), + encoding="utf-8", + ) + + client = self._make_create_client() + svc = SyncService(config_store=store, client_factory=lambda url, token: client) + result = svc.push(alias="prod", project_root=project_root) + + # No variables link PUT happened (the create-pass update_config for the + # backfill is the only update_config we guard against). + client.update_config.assert_not_called() + assert any(e.get("change_type") == "variable_link" for e in result["errors"]), result[ + "errors" + ] + + def test_resolve_source_branch_path_promotes_default_tree( + self, tmp_config_dir: Path, tmp_path: Path + ) -> None: + """KFR-07: missing target-branch subtree → read from the default tree.""" + from keboola_agent_cli.constants import CONFIG_YML_VERSION + from keboola_agent_cli.sync.manifest import ManifestBranch, save_manifest + + project_root = tmp_path / "project" + project_root.mkdir() + store = self._init(tmp_config_dir, project_root) + svc = SyncService(config_store=store, client_factory=lambda url, token: MagicMock()) + + manifest = load_manifest(project_root) + default_path = manifest.branches[0].path + # Register a dev branch WITHOUT a materialized subtree on disk. + manifest.branches.append(ManifestBranch(id=99999, path="feature-x")) + save_manifest(project_root, manifest) + + # Default tree has at least one config on disk. + cfg_dir = project_root / default_path / "extractor/keboola.ex-http/c" + cfg_dir.mkdir(parents=True) + (cfg_dir / CONFIG_FILENAME).write_text( + yaml.dump({"version": CONFIG_YML_VERSION, "name": "c"}, default_flow_style=False), + encoding="utf-8", + ) + + # No feature-x/ subtree → falls back to the default tree. + assert svc._resolve_source_branch_path(manifest, project_root, 99999) == default_path + + # Materialize the dev-branch subtree with a config → it becomes source. + dev_cfg_dir = project_root / "feature-x" / "extractor/keboola.ex-http/c" + dev_cfg_dir.mkdir(parents=True) + (dev_cfg_dir / CONFIG_FILENAME).write_text( + yaml.dump({"version": CONFIG_YML_VERSION, "name": "c"}, default_flow_style=False), + encoding="utf-8", + ) + assert svc._resolve_source_branch_path(manifest, project_root, 99999) == "feature-x" + + +# --------------------------------------------------------------------------- +# Ergonomics: --branch override + --no-name-drift-warnings (v0.47.0) +# --------------------------------------------------------------------------- + + +class TestBranchOverrideAndNameDriftFlag: + """Cover the `--branch` override (push / pull / diff) and the + `--no-name-drift-warnings` opt-out at the service boundary.""" + + def test_resolve_branch_id_override_wins(self, tmp_path: Path) -> None: + from keboola_agent_cli.sync.manifest import ( + ManifestBranch, + ManifestNaming, + ManifestProject, + ) + + project = MagicMock() + project.active_branch_id = 12345 + manifest = Manifest.model_construct( + project=ManifestProject(id=1, apiHost="connection.keboola.com"), + naming=ManifestNaming(), + branches=[ManifestBranch(id=999, path="main", metadata={})], + ) + + # Without override -> falls back to active_branch_id (priority 2). + assert ( + SyncService._resolve_branch_id(project, manifest, tmp_path, branch_override=None) + == 12345 + ) + # Override wins (priority 0). + assert ( + SyncService._resolve_branch_id(project, manifest, tmp_path, branch_override=388071) + == 388071 + ) + + def test_push_branch_override_reaches_client( + self, tmp_config_dir: Path, tmp_path: Path + ) -> None: + """push(branch_override=X) must thread X into list_components_with_configs.""" + project_root = tmp_path / "project" + project_root.mkdir() + + init_client = _make_sync_mock_client( + verify_token_response=SAMPLE_VERIFY_TOKEN, + branches_response=SAMPLE_BRANCHES, + ) + store = setup_single_project(tmp_config_dir) + init_svc = SyncService( + config_store=store, + client_factory=lambda url, token: init_client, + ) + init_svc.init_sync(alias="prod", project_root=project_root) + + push_client = _make_sync_mock_client(components_response=[]) + push_svc = SyncService( + config_store=store, + client_factory=lambda url, token: push_client, + ) + + push_svc.push(alias="prod", project_root=project_root, branch_override=99999) + + push_client.list_components_with_configs.assert_called() + call = push_client.list_components_with_configs.call_args + assert call.kwargs.get("branch_id") == 99999 + + def test_diff_branch_override_reaches_client( + self, tmp_config_dir: Path, tmp_path: Path + ) -> None: + project_root = tmp_path / "project" + project_root.mkdir() + + init_client = _make_sync_mock_client( + verify_token_response=SAMPLE_VERIFY_TOKEN, + branches_response=SAMPLE_BRANCHES, + ) + store = setup_single_project(tmp_config_dir) + init_svc = SyncService( + config_store=store, + client_factory=lambda url, token: init_client, + ) + init_svc.init_sync(alias="prod", project_root=project_root) + + diff_client = _make_sync_mock_client(components_response=[]) + diff_svc = SyncService( + config_store=store, + client_factory=lambda url, token: diff_client, + ) + + diff_svc.diff(alias="prod", project_root=project_root, branch_override=77777) + + diff_client.list_components_with_configs.assert_called() + call = diff_client.list_components_with_configs.call_args + assert call.kwargs.get("branch_id") == 77777 + + def test_no_name_drift_warnings_flag_suppresses_field( + self, tmp_config_dir: Path, tmp_path: Path + ) -> None: + """When name drift is detected, the suppression flag drops the + ``name_drift_warnings`` array from the result envelope.""" + from keboola_agent_cli.constants import CONFIG_YML_VERSION + from keboola_agent_cli.sync.manifest import ( + ManifestConfiguration, + save_manifest, + ) + + project_root = tmp_path / "project" + project_root.mkdir() + + init_client = _make_sync_mock_client( + verify_token_response=SAMPLE_VERIFY_TOKEN, + branches_response=SAMPLE_BRANCHES, + ) + store = setup_single_project(tmp_config_dir) + init_svc = SyncService( + config_store=store, + client_factory=lambda url, token: init_client, + ) + init_svc.init_sync(alias="prod", project_root=project_root) + + # Author a tracked manifest entry whose dirname does NOT match the + # config name -> name-drift detector will surface a warning. + manifest = load_manifest(project_root) + cfg_path = "transformation/keboola.snowflake-transformation/dir-name-NEQ-config-name" + manifest.configurations.append( + ManifestConfiguration( + branchId=12345, + componentId="keboola.snowflake-transformation", + id="01abc", + path=cfg_path, + # No pull_hash -> diff falls into 2-way mode and any + # difference is classified "modified" (a pushable change), + # so the name-drift detector actually runs end-to-end. + metadata={}, + ) + ) + save_manifest(project_root, manifest) + branch_path = manifest.branches[0].path + config_dir = project_root / branch_path / cfg_path + config_dir.mkdir(parents=True) + (config_dir / CONFIG_FILENAME).write_text( + yaml.dump( + { + "version": CONFIG_YML_VERSION, + "name": "Some Pretty Config Name", + "description": "", + "parameters": {"x": "y_new"}, + "_keboola": { + "component_id": "keboola.snowflake-transformation", + "config_id": "01abc", + }, + }, + default_flow_style=False, + ), + encoding="utf-8", + ) + + # Remote returns a stale param value so the diff sees a "modified" + # change and the push actually enters the warning-emitting path. + push_client = _make_sync_mock_client( + components_response=[ + { + "id": "keboola.snowflake-transformation", + "type": "transformation", + "configurations": [ + { + "id": "01abc", + "name": "Some Pretty Config Name", + "description": "", + "configuration": {"parameters": {"x": "y_old"}}, + "rows": [], + } + ], + } + ], + ) + push_client.update_config.return_value = {"id": "01abc"} + push_svc = SyncService( + config_store=store, + client_factory=lambda url, token: push_client, + ) + + default_result = push_svc.push(alias="prod", project_root=project_root) + assert "name_drift_warnings" in default_result, ( + "control: the warning must surface without the suppression flag" + ) + + suppressed_result = push_svc.push( + alias="prod", + project_root=project_root, + no_name_drift_warnings=True, + ) + assert "name_drift_warnings" not in suppressed_result, ( + "--no-name-drift-warnings must drop the field from the envelope" + ) diff --git a/tests/test_workspace_cli.py b/tests/test_workspace_cli.py index 53138f02..f76c9c16 100644 --- a/tests/test_workspace_cli.py +++ b/tests/test_workspace_cli.py @@ -72,8 +72,9 @@ def test_workspace_create_success_json(self, tmp_path: Path) -> None: "schema": "WORKSPACE_42", "user": "KEBOOLA_WORKSPACE_42", "password": "s3cret!Passw0rd", + "private_key": "-----BEGIN PRIVATE KEY-----\ntest\n-----END PRIVATE KEY-----\n", "read_only": True, - "message": "Workspace 'my-workspace' (42) created in project 'prod'. Save the password!", + "message": "Workspace 'my-workspace' (42) created in project 'prod'. Save the private key!", } with ( @@ -99,8 +100,56 @@ def test_workspace_create_success_json(self, tmp_path: Path) -> None: assert output["status"] == "ok" assert output["data"]["workspace_id"] == 42 assert output["data"]["password"] == "s3cret!Passw0rd" + assert output["data"]["private_key"].startswith("-----BEGIN PRIVATE KEY-----") assert output["data"]["backend"] == "snowflake" + def test_workspace_create_human_outputs_private_key_when_present(self, tmp_path: Path) -> None: + """workspace create human output shows the generated private key when returned.""" + config_dir = tmp_path / "config" + config_dir.mkdir() + + store = _setup_config(config_dir, {"prod": {"token": TEST_TOKEN}}) + + mock_ws = _make_workspace_mock() + mock_ws.create_workspace.return_value = { + "project_alias": "prod", + "workspace_id": 42, + "name": "my-workspace", + "config_id": "cfg-123", + "backend": "snowflake", + "host": "account.snowflakecomputing.com", + "warehouse": "KEBOOLA_PROD", + "database": "KEBOOLA_258", + "schema": "WORKSPACE_42", + "user": "KEBOOLA_WORKSPACE_42", + "private_key": "-----BEGIN PRIVATE KEY-----\ntest\n-----END PRIVATE KEY-----\n", + "read_only": True, + "message": "Workspace 'my-workspace' (42) created in project 'prod'. Save the private key!", + } + + with ( + patch("keboola_agent_cli.cli.ConfigStore") as MockStore, + patch("keboola_agent_cli.cli.ProjectService") as MockProjService, + patch("keboola_agent_cli.cli.ConfigService") as MockCfgService, + patch("keboola_agent_cli.cli.JobService") as MockJobService, + patch("keboola_agent_cli.cli.WorkspaceService") as MockWsService, + ): + MockStore.return_value = store + MockProjService.return_value = ProjectService(config_store=store) + MockCfgService.return_value = ConfigService(config_store=store) + MockJobService.return_value = JobService(config_store=store) + MockWsService.return_value = mock_ws + + result = runner.invoke( + app, + ["workspace", "create", "--project", "prod"], + ) + + assert result.exit_code == 0, f"Exit code {result.exit_code}: {result.output}" + assert "Private key:" in result.output + assert "-----BEGIN PRIVATE KEY-----" in result.output + assert "Password:" not in result.output + def test_workspace_create_api_error(self, tmp_path: Path) -> None: """workspace create with API error returns correct exit code.""" config_dir = tmp_path / "config" diff --git a/tests/test_workspace_service.py b/tests/test_workspace_service.py index 4e118010..d9272bd5 100644 --- a/tests/test_workspace_service.py +++ b/tests/test_workspace_service.py @@ -6,7 +6,7 @@ """ from pathlib import Path -from unittest.mock import MagicMock +from unittest.mock import ANY, MagicMock import pytest @@ -100,7 +100,7 @@ class TestCreateWorkspace: """Tests for WorkspaceService.create_workspace().""" def test_create_workspace_success(self, tmp_config_dir: Path) -> None: - """create_workspace returns workspace details including password.""" + """create_workspace returns workspace details including Snowflake key-pair credentials.""" mock_client = MagicMock() mock_client.list_dev_branches.return_value = [{"id": 123, "isDefault": True}] mock_client.create_sandbox_config.return_value = { @@ -130,8 +130,9 @@ def test_create_workspace_success(self, tmp_config_dir: Path) -> None: assert result["schema"] == "WORKSPACE_42" assert result["user"] == "KEBOOLA_WORKSPACE_42" assert result["password"] == "s3cret!Passw0rd" + assert result["private_key"].startswith("-----BEGIN PRIVATE KEY-----") assert result["read_only"] is True - assert "Save the password" in result["message"] + assert "Save the private key" in result["message"] mock_client.create_sandbox_config.assert_called_once_with( name="test-ws", @@ -143,7 +144,11 @@ def test_create_workspace_success(self, tmp_config_dir: Path) -> None: component_id="keboola.sandboxes", config_id="cfg-123", backend="snowflake", + login_type="snowflake-person-keypair", + public_key=ANY, ) + public_key = mock_client.create_config_workspace.call_args.kwargs["public_key"] + assert public_key.startswith("-----BEGIN PUBLIC KEY-----") # close() called twice: once in _resolve_branch_id, once in create_workspace assert mock_client.close.call_count == 2 @@ -215,12 +220,76 @@ def test_create_workspace_in_dev_branch(self, tmp_config_dir: Path) -> None: component_id="keboola.sandboxes", config_id="cfg-456", backend="snowflake", + login_type="snowflake-person-keypair", + public_key=ANY, ) + public_key = mock_client.create_config_workspace.call_args.kwargs["public_key"] + assert public_key.startswith("-----BEGIN PUBLIC KEY-----") class TestAutoDetectBackend: """Tests for automatic backend detection when --backend is omitted.""" + def test_create_workspace_snowflake_uses_person_keypair_login_type( + self, tmp_config_dir: Path + ) -> None: + """Snowflake sandbox workspaces request the Query-Service-compatible login type.""" + mock_client = MagicMock() + mock_client.verify_token.return_value = SAMPLE_TOKEN_VERIFY + mock_client.list_dev_branches.return_value = [{"id": 123, "isDefault": True}] + mock_client.create_sandbox_config.return_value = {"id": "cfg-1", "name": "ws"} + mock_client.create_config_workspace.return_value = SAMPLE_WORKSPACE + + store = setup_single_project(tmp_config_dir) + svc = WorkspaceService( + config_store=store, + client_factory=lambda url, token: mock_client, + ) + + result = svc.create_workspace(alias="prod", name="ws") + + assert result["backend"] == "snowflake" + assert result["private_key"].startswith("-----BEGIN PRIVATE KEY-----") + call_kwargs = mock_client.create_config_workspace.call_args.kwargs + assert call_kwargs["branch_id"] == 123 + assert call_kwargs["component_id"] == "keboola.sandboxes" + assert call_kwargs["config_id"] == "cfg-1" + assert call_kwargs["backend"] == "snowflake" + assert call_kwargs["login_type"] == "snowflake-person-keypair" + assert call_kwargs["public_key"].startswith("-----BEGIN PUBLIC KEY-----") + + def test_create_workspace_bigquery_keeps_default_login_type(self, tmp_config_dir: Path) -> None: + """BigQuery sandbox workspaces omit loginType so Storage uses its default.""" + mock_client = MagicMock() + mock_client.verify_token.return_value = SAMPLE_TOKEN_VERIFY_BIGQUERY + mock_client.list_dev_branches.return_value = [{"id": 123, "isDefault": True}] + mock_client.create_sandbox_config.return_value = {"id": "cfg-1", "name": "ws"} + mock_client.create_config_workspace.return_value = { + "id": 42, + "connection": { + "backend": "bigquery", + "schema": "WORKSPACE_42", + }, + } + + store = setup_single_project(tmp_config_dir) + svc = WorkspaceService( + config_store=store, + client_factory=lambda url, token: mock_client, + ) + + result = svc.create_workspace(alias="prod", name="ws") + + assert result["backend"] == "bigquery" + mock_client.create_config_workspace.assert_called_once_with( + branch_id=123, + component_id="keboola.sandboxes", + config_id="cfg-1", + backend="bigquery", + login_type=None, + public_key=None, + ) + def test_create_workspace_auto_detects_snowflake(self, tmp_config_dir: Path) -> None: """create_workspace auto-detects snowflake backend from project.""" mock_client = MagicMock() @@ -244,7 +313,11 @@ def test_create_workspace_auto_detects_snowflake(self, tmp_config_dir: Path) -> component_id="keboola.sandboxes", config_id="cfg-1", backend="snowflake", + login_type="snowflake-person-keypair", + public_key=ANY, ) + public_key = mock_client.create_config_workspace.call_args.kwargs["public_key"] + assert public_key.startswith("-----BEGIN PUBLIC KEY-----") def test_create_workspace_auto_detects_bigquery(self, tmp_config_dir: Path) -> None: """create_workspace auto-detects bigquery backend from project.""" @@ -276,6 +349,8 @@ def test_create_workspace_auto_detects_bigquery(self, tmp_config_dir: Path) -> N component_id="keboola.sandboxes", config_id="cfg-1", backend="bigquery", + login_type=None, + public_key=None, ) def test_explicit_backend_skips_auto_detect(self, tmp_config_dir: Path) -> None: @@ -341,6 +416,8 @@ def test_from_transformation_auto_detects_bigquery(self, tmp_config_dir: Path) - component_id="keboola.snowflake-transformation", config_id="456", backend="bigquery", + login_type=None, + public_key=None, ) @@ -1080,8 +1157,10 @@ def test_create_from_transformation_success(self, tmp_config_dir: Path) -> None: assert result["row_id"] is None assert result["backend"] == "snowflake" assert result["password"] == "ws-secret-pwd" + assert result["private_key"].startswith("-----BEGIN PRIVATE KEY-----") assert result["tables_loaded"] == ["in.c-main.orders", "in.c-main.products"] assert "2 table(s) loaded" in result["message"] + assert "Save the private key" in result["message"] mock_client.get_config_detail.assert_called_once_with( "keboola.snowflake-transformation", @@ -1092,7 +1171,11 @@ def test_create_from_transformation_success(self, tmp_config_dir: Path) -> None: component_id="keboola.snowflake-transformation", config_id="456", backend="snowflake", + login_type="snowflake-person-keypair", + public_key=ANY, ) + public_key = mock_client.create_config_workspace.call_args.kwargs["public_key"] + assert public_key.startswith("-----BEGIN PUBLIC KEY-----") mock_client.load_workspace_tables.assert_called_once() # close() called twice: once in _resolve_branch_id, once in create_from_transformation assert mock_client.close.call_count == 2 @@ -1371,6 +1454,21 @@ def test_list_exposes_login_type_and_qs_compatible(self, tmp_config_dir: Path) - "component": "keboola.snowflake-transformation", "configurationId": "cfg-2", }, + { + "id": 3, + "name": "person-keypair", + "connection": { + "backend": "snowflake", + "host": "h", + "schema": "S3", + "user": "U3", + "loginType": "snowflake-person-keypair", + }, + "readOnlyStorageAccess": True, + "created": "2026-05-18T00:00:00Z", + "component": "keboola.sandboxes", + "configurationId": "cfg-3", + }, ] mock_client.list_component_configs.return_value = [] @@ -1383,14 +1481,19 @@ def test_list_exposes_login_type_and_qs_compatible(self, tmp_config_dir: Path) - result = svc.list_workspaces(aliases=["prod"]) workspaces = result["workspaces"] - assert len(workspaces) == 2 + assert len(workspaces) == 3 compat = next(w for w in workspaces if w["id"] == 1) legacy = next(w for w in workspaces if w["id"] == 2) + person_keypair = next(w for w in workspaces if w["id"] == 3) assert compat["login_type"] == "snowflake-service-keypair" assert compat["read_only"] is True assert compat["qs_compatible"] is True + assert person_keypair["login_type"] == "snowflake-person-keypair" + assert person_keypair["read_only"] is True + assert person_keypair["qs_compatible"] is True + assert legacy["login_type"] == "default" assert legacy["read_only"] is False # ``default`` is intentionally OFF the whitelist (legacy 2016 ws, diff --git a/uv.lock b/uv.lock index c7189963..d9a35754 100644 --- a/uv.lock +++ b/uv.lock @@ -436,11 +436,11 @@ wheels = [ [[package]] name = "idna" -version = "3.11" +version = "3.15" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/6f/6d/0703ccc57f3a7233505399edb88de3cbd678da106337b9fcde432b65ed60/idna-3.11.tar.gz", hash = "sha256:795dafcc9c04ed0c1fb032c2aa73654d8e8c5023a7df64a53f39190ada629902", size = 194582, upload-time = "2025-10-12T14:55:20.501Z" } +sdist = { url = "https://files.pythonhosted.org/packages/82/77/7b3966d0b9d1d31a36ddf1746926a11dface89a83409bf1483f0237aa758/idna-3.15.tar.gz", hash = "sha256:ca962446ea538f7092a95e057da437618e886f4d349216d2b1e294abfdb65fdc", size = 199245, upload-time = "2026-05-12T22:45:57.011Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/0e/61/66938bbb5fc52dbdf84594873d5b51fb1f7c7794e9c0f5bd885f30bc507b/idna-3.11-py3-none-any.whl", hash = "sha256:771a87f49d9defaf64091e6e6fe9c18d4833f140bd19464795bc32d966ca37ea", size = 71008, upload-time = "2025-10-12T14:55:18.883Z" }, + { url = "https://files.pythonhosted.org/packages/d2/23/408243171aa9aaba178d3e2559159c24c1171a641aa83b67bdd3394ead8e/idna-3.15-py3-none-any.whl", hash = "sha256:048adeaf8c2d788c40fee287673ccaa74c24ffd8dcf09ffa555a2fbb59f10ac8", size = 72340, upload-time = "2026-05-12T22:45:55.733Z" }, ] [[package]] @@ -496,10 +496,11 @@ wheels = [ [[package]] name = "keboola-agent-cli" -version = "0.46.1" +version = "0.53.0" source = { editable = "." } dependencies = [ { name = "croniter" }, + { name = "cryptography" }, { name = "httpx" }, { name = "jsonschema" }, { name = "kai-client" }, @@ -535,6 +536,7 @@ dev = [ [package.metadata] requires-dist = [ { name = "croniter", specifier = ">=2.0" }, + { name = "cryptography", specifier = ">=46" }, { name = "fastapi", marker = "extra == 'server'", specifier = ">=0.115" }, { name = "httpx", specifier = ">=0.27" }, { name = "jsonschema", specifier = ">=4.20" }, @@ -544,7 +546,7 @@ requires-dist = [ { name = "platformdirs", specifier = ">=4" }, { name = "prompt-toolkit", specifier = ">=3.0" }, { name = "pydantic", specifier = ">=2.5" }, - { name = "python-multipart", marker = "extra == 'server'", specifier = ">=0.0.9" }, + { name = "python-multipart", marker = "extra == 'server'", specifier = ">=0.0.27" }, { name = "pyyaml", specifier = ">=6" }, { name = "rich", specifier = ">=13" }, { name = "sse-starlette", marker = "extra == 'server'", specifier = ">=2.1" }, @@ -686,11 +688,11 @@ wheels = [ [[package]] name = "pip" -version = "26.0.1" +version = "26.1" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/48/83/0d7d4e9efe3344b8e2fe25d93be44f64b65364d3c8d7bc6dc90198d5422e/pip-26.0.1.tar.gz", hash = "sha256:c4037d8a277c89b320abe636d59f91e6d0922d08a05b60e85e53b296613346d8", size = 1812747, upload-time = "2026-02-05T02:20:18.702Z" } +sdist = { url = "https://files.pythonhosted.org/packages/73/7e/d2b04004e1068ad4fdfa2f227b839b5d03e602e47cdbbf49de71137c9546/pip-26.1.tar.gz", hash = "sha256:81e13ebcca3ffa8cc85e4deff5c27e1ee26dea0aa7fc2f294a073ac208806ff3", size = 1840316, upload-time = "2026-04-26T21:00:05.406Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/de/f0/c81e05b613866b76d2d1066490adf1a3dbc4ee9d9c839961c3fc8a6997af/pip-26.0.1-py3-none-any.whl", hash = "sha256:bdb1b08f4274833d62c1aa29e20907365a2ceb950410df15fc9521bad440122b", size = 1787723, upload-time = "2026-02-05T02:20:16.416Z" }, + { url = "https://files.pythonhosted.org/packages/70/7a/be4bd8bcbb24ea475856dd68159d78b03b2bb53dae369f69c9606b8888f5/pip-26.1-py3-none-any.whl", hash = "sha256:4e8486d821d814b77319acb7b9e8bf5a4ee7590a643e7cb21029f209be8573c1", size = 1812804, upload-time = "2026-04-26T21:00:03.194Z" }, ] [[package]] @@ -991,11 +993,11 @@ wheels = [ [[package]] name = "python-multipart" -version = "0.0.26" +version = "0.0.27" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/88/71/b145a380824a960ebd60e1014256dbb7d2253f2316ff2d73dfd8928ec2c3/python_multipart-0.0.26.tar.gz", hash = "sha256:08fadc45918cd615e26846437f50c5d6d23304da32c341f289a617127b081f17", size = 43501, upload-time = "2026-04-10T14:09:59.473Z" } +sdist = { url = "https://files.pythonhosted.org/packages/69/9b/f23807317a113dc36e74e75eb265a02dd1a4d9082abc3c1064acd22997c4/python_multipart-0.0.27.tar.gz", hash = "sha256:9870a6a8c5a20a5bf4f07c017bd1489006ff8836cff097b6933355ee2b49b602", size = 44043, upload-time = "2026-04-27T10:51:26.649Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/9a/22/f1925cdda983ab66fc8ec6ec8014b959262747e58bdca26a4e3d1da29d56/python_multipart-0.0.26-py3-none-any.whl", hash = "sha256:c0b169f8c4484c13b0dcf2ef0ec3a4adb255c4b7d18d8e420477d2b1dd03f185", size = 28847, upload-time = "2026-04-10T14:09:58.131Z" }, + { url = "https://files.pythonhosted.org/packages/99/78/4126abcbdbd3c559d43e0db7f7b9173fc6befe45d39a2856cc0b8ec2a5a6/python_multipart-0.0.27-py3-none-any.whl", hash = "sha256:6fccfad17a27334bd0193681b369f476eda3409f17381a2d65aa7df3f7275645", size = 29254, upload-time = "2026-04-27T10:51:24.997Z" }, ] [[package]] @@ -1387,11 +1389,11 @@ wheels = [ [[package]] name = "urllib3" -version = "2.6.3" +version = "2.7.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/c7/24/5f1b3bdffd70275f6661c76461e25f024d5a38a46f04aaca912426a2b1d3/urllib3-2.6.3.tar.gz", hash = "sha256:1b62b6884944a57dbe321509ab94fd4d3b307075e0c2eae991ac71ee15ad38ed", size = 435556, upload-time = "2026-01-07T16:24:43.925Z" } +sdist = { url = "https://files.pythonhosted.org/packages/53/0c/06f8b233b8fd13b9e5ee11424ef85419ba0d8ba0b3138bf360be2ff56953/urllib3-2.7.0.tar.gz", hash = "sha256:231e0ec3b63ceb14667c67be60f2f2c40a518cb38b03af60abc813da26505f4c", size = 433602, upload-time = "2026-05-07T16:13:18.596Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/39/08/aaaad47bc4e9dc8c725e68f9d04865dbcb2052843ff09c97b08904852d84/urllib3-2.6.3-py3-none-any.whl", hash = "sha256:bf272323e553dfb2e87d9bfd225ca7b0f467b919d7bbd355436d3fd37cb0acd4", size = 131584, upload-time = "2026-01-07T16:24:42.685Z" }, + { url = "https://files.pythonhosted.org/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl", hash = "sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897", size = 131087, upload-time = "2026-05-07T16:13:17.151Z" }, ] [[package]] diff --git a/web/backend/package-lock.json b/web/backend/package-lock.json index 91cdf81b..5598aaca 100644 --- a/web/backend/package-lock.json +++ b/web/backend/package-lock.json @@ -18,12 +18,46 @@ "@types/node": "^22.5.0", "tsx": "^4.19.0", "typescript": "^5.5.4", - "vitest": "^2.0.5" + "vitest": "^4.1.0" }, "engines": { "node": ">=20" } }, + "node_modules/@emnapi/core": { + "version": "1.10.0", + "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.10.0.tgz", + "integrity": "sha512-yq6OkJ4p82CAfPl0u9mQebQHKPJkY7WrIuk205cTYnYe+k2Z8YBh11FrbRG/H6ihirqcacOgl2BIO8oyMQLeXw==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "@emnapi/wasi-threads": "1.2.1", + "tslib": "^2.4.0" + } + }, + "node_modules/@emnapi/runtime": { + "version": "1.10.0", + "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.10.0.tgz", + "integrity": "sha512-ewvYlk86xUoGI0zQRNq/mC+16R1QeDlKQy21Ki3oSYXNgLb45GV1P6A0M+/s6nyCuNDqe5VpaY84BzXGwVbwFA==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/@emnapi/wasi-threads": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.2.1.tgz", + "integrity": "sha512-uTII7OYF+/Mes/MrcIOYp5yOtSMLBWSIoLPpcgwipoiKbli6k322tcoFsxoIIxPDqW01SQGAgko4EzZi2BNv2w==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, "node_modules/@esbuild/aix-ppc64": { "version": "0.27.7", "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.27.7.tgz", @@ -676,30 +710,45 @@ "node": ">=8" } }, + "node_modules/@napi-rs/wasm-runtime": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-1.1.4.tgz", + "integrity": "sha512-3NQNNgA1YSlJb/kMH1ildASP9HW7/7kYnRI2szWJaofaS1hWmbGI4H+d3+22aGzXXN9IJ+n+GiFVcGipJP18ow==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "@tybys/wasm-util": "^0.10.1" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + }, + "peerDependencies": { + "@emnapi/core": "^1.7.1", + "@emnapi/runtime": "^1.7.1" + } + }, + "node_modules/@oxc-project/types": { + "version": "0.133.0", + "resolved": "https://registry.npmjs.org/@oxc-project/types/-/types-0.133.0.tgz", + "integrity": "sha512-KzkdCd6Uxqnf6l3HOw1xfatAlUURA0g14cvBYFyJ5SaNOQbOUvBr9PKArcPcrNIeRsBdgcUzOGrhKveVpvOIGA==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/Boshen" + } + }, "node_modules/@pinojs/redact": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/@pinojs/redact/-/redact-0.4.0.tgz", "integrity": "sha512-k2ENnmBugE/rzQfEcdWHcCY+/FM3VLzH9cYEsbdsoqrvzAKRhUZeRNhAZvB8OitQJ1TBed3yqWtdjzS6wJKBwg==", "license": "MIT" }, - "node_modules/@rollup/rollup-android-arm-eabi": { - "version": "4.60.3", - "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.60.3.tgz", - "integrity": "sha512-x35CNW/ANXG3hE/EZpRU8MXX1JDN86hBb2wMGAtltkz7pc6cxgjpy1OMMfDosOQ+2hWqIkag/fGok1Yady9nGw==", - "cpu": [ - "arm" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "android" - ] - }, - "node_modules/@rollup/rollup-android-arm64": { - "version": "4.60.3", - "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.60.3.tgz", - "integrity": "sha512-xw3xtkDApIOGayehp2+Rz4zimfkaX65r4t47iy+ymQB2G4iJCBBfj0ogVg5jpvjpn8UWn/+q9tprxleYeNp3Hw==", + "node_modules/@rolldown/binding-android-arm64": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm64/-/binding-android-arm64-1.0.3.tgz", + "integrity": "sha512-454rs7jHngixp/NMxd5srYD57OnzSlZ/eFTETjORQHLwJG1lRtmNOJcBerZlfu4GjKqeq8aCCIQrMdHyhI51Hw==", "cpu": [ "arm64" ], @@ -708,12 +757,15 @@ "optional": true, "os": [ "android" - ] + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } }, - "node_modules/@rollup/rollup-darwin-arm64": { - "version": "4.60.3", - "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.60.3.tgz", - "integrity": "sha512-vo6Y5Qfpx7/5EaamIwi0WqW2+zfiusVihKatLvtN1VFVy3D13uERk/6gZLU1UiHRL6fDXqj/ELIeVRGnvcTE1g==", + "node_modules/@rolldown/binding-darwin-arm64": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-arm64/-/binding-darwin-arm64-1.0.3.tgz", + "integrity": "sha512-PcAhP+ynjURNyy8SKGl5DQP94aGuB/7JrXJb/t7P+hanXvQVMWzUvRRhBAcg/lNRadBhoUPqSoP4xw5tR/KBEA==", "cpu": [ "arm64" ], @@ -722,12 +774,15 @@ "optional": true, "os": [ "darwin" - ] + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } }, - "node_modules/@rollup/rollup-darwin-x64": { - "version": "4.60.3", - "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.60.3.tgz", - "integrity": "sha512-D+0QGcZhBzTN82weOnsSlY7V7+RMmPuF1CkbxyMAGE8+ZHeUjyb76ZiWmBlCu//AQQONvxcqRbwZTajZKqjuOw==", + "node_modules/@rolldown/binding-darwin-x64": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-x64/-/binding-darwin-x64-1.0.3.tgz", + "integrity": "sha512-9YpfeUvSE2RS7wysJ81uOZkXJz7f7Q55H2Gvp3VEw/EsahqDtrphrZ0EwDLK5vvKOzaCrBsjF8JmnMLcUt78Gg==", "cpu": [ "x64" ], @@ -736,26 +791,15 @@ "optional": true, "os": [ "darwin" - ] - }, - "node_modules/@rollup/rollup-freebsd-arm64": { - "version": "4.60.3", - "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-arm64/-/rollup-freebsd-arm64-4.60.3.tgz", - "integrity": "sha512-6HnvHCT7fDyj6R0Ph7A6x8dQS/S38MClRWeDLqc0MdfWkxjiu1HSDYrdPhqSILzjTIC/pnXbbJbo+ft+gy/9hQ==", - "cpu": [ - "arm64" ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "freebsd" - ] + "engines": { + "node": "^20.19.0 || >=22.12.0" + } }, - "node_modules/@rollup/rollup-freebsd-x64": { - "version": "4.60.3", - "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-x64/-/rollup-freebsd-x64-4.60.3.tgz", - "integrity": "sha512-KHLgC3WKlUYW3ShFKnnosZDOJ0xjg9zp7au3sIm2bs/tGBeC2ipmvRh/N7JKi0t9Ue20C0dpEshi8WUubg+cnA==", + "node_modules/@rolldown/binding-freebsd-x64": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-freebsd-x64/-/binding-freebsd-x64-1.0.3.tgz", + "integrity": "sha512-yB1IlAsSNHncV6SCTL27/MVGR5htvQsoGxIv5KMGXALp+Ll1wYsn+x98M9MW7qa+NdSbvrrY7ANI4wLJ0n1e6g==", "cpu": [ "x64" ], @@ -764,247 +808,134 @@ "optional": true, "os": [ "freebsd" - ] - }, - "node_modules/@rollup/rollup-linux-arm-gnueabihf": { - "version": "4.60.3", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.60.3.tgz", - "integrity": "sha512-DV6fJoxEYWJOvaZIsok7KrYl0tPvga5OZ2yvKHNNYyk/2roMLqQAbGhr78EQ5YhHpnhLKJD3S1WFusAkmUuV5g==", - "cpu": [ - "arm" - ], - "dev": true, - "libc": [ - "glibc" ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ] + "engines": { + "node": "^20.19.0 || >=22.12.0" + } }, - "node_modules/@rollup/rollup-linux-arm-musleabihf": { - "version": "4.60.3", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.60.3.tgz", - "integrity": "sha512-mQKoJAzvuOs6F+TZybQO4GOTSMUu7v0WdxEk24krQ/uUxXoPTtHjuaUuPmFhtBcM4K0ons8nrE3JyhTuCFtT/w==", + "node_modules/@rolldown/binding-linux-arm-gnueabihf": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm-gnueabihf/-/binding-linux-arm-gnueabihf-1.0.3.tgz", + "integrity": "sha512-Yi30IVAAfLUCy2MseFjbB1jAMDl1VMCAas5StnYp8da9+CKvMd2H2cbEjWcw5NPaPqzvYkVIaF1nNUG+b7u/sw==", "cpu": [ "arm" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ "linux" - ] - }, - "node_modules/@rollup/rollup-linux-arm64-gnu": { - "version": "4.60.3", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.60.3.tgz", - "integrity": "sha512-Whjj2qoiJ6+OOJMGptTYazaJvjOJm+iKHpXQM1P3LzGjt7Ff++Tp7nH4N8J/BUA7R9IHfDyx4DJIflifwnbmIA==", - "cpu": [ - "arm64" ], - "dev": true, - "libc": [ - "glibc" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ] + "engines": { + "node": "^20.19.0 || >=22.12.0" + } }, - "node_modules/@rollup/rollup-linux-arm64-musl": { - "version": "4.60.3", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.60.3.tgz", - "integrity": "sha512-4YTNHKqGng5+yiZt3mg77nmyuCfmNfX4fPmyUapBcIk+BdwSwmCWGXOUxhXbBEkFHtoN5boLj/5NON+u5QC9tg==", + "node_modules/@rolldown/binding-linux-arm64-gnu": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-gnu/-/binding-linux-arm64-gnu-1.0.3.tgz", + "integrity": "sha512-jsO7R8To+AdlYgUmN5sHSCZbfhtMBkO0WUx8iORQnPcMMdgr7qM2DQmMwgabs3GhNztdmoKkMKQFHD6DTMCIQw==", "cpu": [ "arm64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ "linux" - ] - }, - "node_modules/@rollup/rollup-linux-loong64-gnu": { - "version": "4.60.3", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loong64-gnu/-/rollup-linux-loong64-gnu-4.60.3.tgz", - "integrity": "sha512-SU3kNlhkpI4UqlUc2VXPGK9o886ZsSeGfMAX2ba2b8DKmMXq4AL7KUrkSWVbb7koVqx41Yczx6dx5PNargIrEA==", - "cpu": [ - "loong64" - ], - "dev": true, - "libc": [ - "glibc" ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ] + "engines": { + "node": "^20.19.0 || >=22.12.0" + } }, - "node_modules/@rollup/rollup-linux-loong64-musl": { - "version": "4.60.3", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loong64-musl/-/rollup-linux-loong64-musl-4.60.3.tgz", - "integrity": "sha512-6lDLl5h4TXpB1mTf2rQWnAk/LcXrx9vBfu/DT5TIPhvMhRWaZ5MxkIc8u4lJAmBo6klTe1ywXIUHFjylW505sg==", + "node_modules/@rolldown/binding-linux-arm64-musl": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-musl/-/binding-linux-arm64-musl-1.0.3.tgz", + "integrity": "sha512-VWkUHwWriDciit80wleYwKILoR/KMvxh/IdwS/paX+ZgpuRpCrKLUdadJbc0NpBEiyhpYawsJ73j9aCvOH+f7Q==", "cpu": [ - "loong64" + "arm64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ "linux" - ] - }, - "node_modules/@rollup/rollup-linux-ppc64-gnu": { - "version": "4.60.3", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-ppc64-gnu/-/rollup-linux-ppc64-gnu-4.60.3.tgz", - "integrity": "sha512-BMo8bOw8evlup/8G+cj5xWtPyp93xPdyoSN16Zy90Q2QZ0ZYRhCt6ZJSwbrRzG9HApFabjwj2p25TUPDWrhzqQ==", - "cpu": [ - "ppc64" ], - "dev": true, - "libc": [ - "glibc" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ] + "engines": { + "node": "^20.19.0 || >=22.12.0" + } }, - "node_modules/@rollup/rollup-linux-ppc64-musl": { - "version": "4.60.3", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-ppc64-musl/-/rollup-linux-ppc64-musl-4.60.3.tgz", - "integrity": "sha512-E0L8X1dZN1/Rph+5VPF6Xj2G7JJvMACVXtamTJIDrVI44Y3K+G8gQaMEAavbqCGTa16InptiVrX6eM6pmJ+7qA==", + "node_modules/@rolldown/binding-linux-ppc64-gnu": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-ppc64-gnu/-/binding-linux-ppc64-gnu-1.0.3.tgz", + "integrity": "sha512-5f1laC0SlIR0yDbFCd8acUhvJIag6N3zC5P7oUPN6wX0aOma+uKJ0wBDH5aq7I1PVI2ttTlhJwzwRIBnLiSGEg==", "cpu": [ "ppc64" ], "dev": true, - "libc": [ - "musl" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ] - }, - "node_modules/@rollup/rollup-linux-riscv64-gnu": { - "version": "4.60.3", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.60.3.tgz", - "integrity": "sha512-oZJ/WHaVfHUiRAtmTAeo3DcevNsVvH8mbvodjZy7D5QKvCefO371SiKRpxoDcCxB3PTRTLayWBkvmDQKTcX/sw==", - "cpu": [ - "riscv64" - ], - "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ "linux" - ] - }, - "node_modules/@rollup/rollup-linux-riscv64-musl": { - "version": "4.60.3", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-musl/-/rollup-linux-riscv64-musl-4.60.3.tgz", - "integrity": "sha512-Dhbyh7j9FybM3YaTgaHmVALwA8AkUwTPccyCQ79TG9AJUsMQqgN1DDEZNr4+QUfwiWvLDumW5vdwzoeUF+TNxQ==", - "cpu": [ - "riscv64" - ], - "dev": true, - "libc": [ - "musl" ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ] + "engines": { + "node": "^20.19.0 || >=22.12.0" + } }, - "node_modules/@rollup/rollup-linux-s390x-gnu": { - "version": "4.60.3", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.60.3.tgz", - "integrity": "sha512-cJd1X5XhHHlltkaypz1UcWLA8AcoIi1aWhsvaWDskD1oz2eKCypnqvTQ8ykMNI0RSmm7NkTdSqSSD7zM0xa6Ig==", + "node_modules/@rolldown/binding-linux-s390x-gnu": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-s390x-gnu/-/binding-linux-s390x-gnu-1.0.3.tgz", + "integrity": "sha512-Iq4ko0r4XsgbrF/LunNgHtAGLRRVE2kXonAXQ/MV0mC6jQpMOhW1SvtZja2EhC/kd05++bP78dsqBeIQyYJ6Yg==", "cpu": [ "s390x" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ "linux" - ] + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } }, - "node_modules/@rollup/rollup-linux-x64-gnu": { - "version": "4.60.3", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.60.3.tgz", - "integrity": "sha512-DAZDBHQfG2oQuhY7mc6I3/qB4LU2fQCjRvxbDwd/Jdvb9fypP4IJ4qmtu6lNjes6B531AI8cg1aKC2di97bUxA==", + "node_modules/@rolldown/binding-linux-x64-gnu": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-gnu/-/binding-linux-x64-gnu-1.0.3.tgz", + "integrity": "sha512-B8m6tD5+/N5FeNQFbKlLA/2yVq9ycQP1SeedyEYYKWBNR3ZQbkvIUcNnDNM03lO1l5F2roiiFJGgvoLLyZXtSg==", "cpu": [ "x64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ "linux" - ] + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } }, - "node_modules/@rollup/rollup-linux-x64-musl": { - "version": "4.60.3", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.60.3.tgz", - "integrity": "sha512-cRxsE8c13mZOh3vP+wLDxpQBRrOHDIGOWyDL93Sy0Ga8y515fBcC2pjUfFwUe5T7tqvTvWbCpg1URM/AXdWIXA==", + "node_modules/@rolldown/binding-linux-x64-musl": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-musl/-/binding-linux-x64-musl-1.0.3.tgz", + "integrity": "sha512-pSdpdUJHkuCxun9LE7jvgUB9qsRgaiyNNCX7m/AvHTcq67AiT/Yhoxvw5zPfhrM8k/BfP8ce/hMOpthKDpEUow==", "cpu": [ "x64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ "linux" - ] - }, - "node_modules/@rollup/rollup-openbsd-x64": { - "version": "4.60.3", - "resolved": "https://registry.npmjs.org/@rollup/rollup-openbsd-x64/-/rollup-openbsd-x64-4.60.3.tgz", - "integrity": "sha512-QaWcIgRxqEdQdhJqW4DJctsH6HCmo5vHxY0krHSX4jMtOqfzC+dqDGuHM87bu4H8JBeibWx7jFz+h6/4C8wA5Q==", - "cpu": [ - "x64" ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "openbsd" - ] + "engines": { + "node": "^20.19.0 || >=22.12.0" + } }, - "node_modules/@rollup/rollup-openharmony-arm64": { - "version": "4.60.3", - "resolved": "https://registry.npmjs.org/@rollup/rollup-openharmony-arm64/-/rollup-openharmony-arm64-4.60.3.tgz", - "integrity": "sha512-AaXwSvUi3QIPtroAUw1t5yHGIyqKEXwH54WUocFolZhpGDruJcs8c+xPNDRn4XiQsS7MEwnYsHW2l0MBLDMkWg==", + "node_modules/@rolldown/binding-openharmony-arm64": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-openharmony-arm64/-/binding-openharmony-arm64-1.0.3.tgz", + "integrity": "sha512-OXXS3RKJgX2uLwM+gYyuH5omcH8fL1LJs96pZGgtetVCahON57+d4SJHzTgZiOjxgGkSnpXpOsWuPDGAKAigEg==", "cpu": [ "arm64" ], @@ -1013,40 +944,51 @@ "optional": true, "os": [ "openharmony" - ] + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } }, - "node_modules/@rollup/rollup-win32-arm64-msvc": { - "version": "4.60.3", - "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.60.3.tgz", - "integrity": "sha512-65LAKM/bAWDqKNEelHlcHvm2V+Vfb8C6INFxQXRHCvaVN1rJfwr4NvdP4FyzUaLqWfaCGaadf6UbTm8xJeYfEg==", + "node_modules/@rolldown/binding-wasm32-wasi": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-wasm32-wasi/-/binding-wasm32-wasi-1.0.3.tgz", + "integrity": "sha512-JTtb8BWFynicNSoPrehsCzBtOKjZ6jhMiPFEmOiuXg1Fl8dn2KHQob+GuPSGR0dryQa1PQJbzjF3dqO/whhjLg==", "cpu": [ - "arm64" + "wasm32" ], "dev": true, "license": "MIT", "optional": true, - "os": [ - "win32" - ] + "dependencies": { + "@emnapi/core": "1.10.0", + "@emnapi/runtime": "1.10.0", + "@napi-rs/wasm-runtime": "^1.1.4" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + } }, - "node_modules/@rollup/rollup-win32-ia32-msvc": { - "version": "4.60.3", - "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.60.3.tgz", - "integrity": "sha512-EEM2gyhBF5MFnI6vMKdX1LAosE627RGBzIoGMdLloPZkXrUN0Ckqgr2Qi8+J3zip/8NVVro3/FjB+tjhZUgUHA==", + "node_modules/@rolldown/binding-win32-arm64-msvc": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-arm64-msvc/-/binding-win32-arm64-msvc-1.0.3.tgz", + "integrity": "sha512-gEdFFEN70A/jxb2svrWsN3aDL7OUtmvlOy+6fa2jxG8K0wQ1ZbdeLGnidov6Yu5/733dI5ySfzFlQ/cb0bSz1g==", "cpu": [ - "ia32" + "arm64" ], "dev": true, "license": "MIT", "optional": true, "os": [ "win32" - ] + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } }, - "node_modules/@rollup/rollup-win32-x64-gnu": { - "version": "4.60.3", - "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-gnu/-/rollup-win32-x64-gnu-4.60.3.tgz", - "integrity": "sha512-E5Eb5H/DpxaoXH++Qkv28RcUJboMopmdDUALBczvHMf7hNIxaDZqwY5lK12UK1BHacSmvupoEWGu+n993Z0y1A==", + "node_modules/@rolldown/binding-win32-x64-msvc": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-x64-msvc/-/binding-win32-x64-msvc-1.0.3.tgz", + "integrity": "sha512-eXB7CHuaQdqmJcc3koCNtNPmT/bj2gc999kUFgBxG8Ac0NdgXc4rkCHhqrgrhN3zddvvvrgzj1e90SuSfmyIXA==", "cpu": [ "x64" ], @@ -1055,21 +997,53 @@ "optional": true, "os": [ "win32" - ] - }, - "node_modules/@rollup/rollup-win32-x64-msvc": { - "version": "4.60.3", - "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.60.3.tgz", - "integrity": "sha512-hPt/bgL5cE+Qp+/TPHBqptcAgPzgj46mPcg/16zNUmbQk0j+mOEQV/+Lqu8QRtDV3Ek95Q6FeFITpuhl6OTsAA==", - "cpu": [ - "x64" ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/pluginutils": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@rolldown/pluginutils/-/pluginutils-1.0.1.tgz", + "integrity": "sha512-2j9bGt5Jh8hj+vPtgzPtl72j0yRxHAyumoo6TNfAjsLB04UtpSvPbPcDcBMxz7n+9CYB0c1GxQFxYRg2jimqGw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@standard-schema/spec": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@standard-schema/spec/-/spec-1.1.0.tgz", + "integrity": "sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==", + "dev": true, + "license": "MIT" + }, + "node_modules/@tybys/wasm-util": { + "version": "0.10.2", + "resolved": "https://registry.npmjs.org/@tybys/wasm-util/-/wasm-util-0.10.2.tgz", + "integrity": "sha512-RoBvJ2X0wuKlWFIjrwffGw1IqZHKQqzIchKaadZZfnNpsAYp2mM0h36JtPCjNDAHGgYez/15uMBpfGwchhiMgg==", "dev": true, "license": "MIT", "optional": true, - "os": [ - "win32" - ] + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/@types/chai": { + "version": "5.2.3", + "resolved": "https://registry.npmjs.org/@types/chai/-/chai-5.2.3.tgz", + "integrity": "sha512-Mw558oeA9fFbv65/y4mHtXDs9bPnFMZAL/jxdPFUpOHHIXX91mcgEHbS5Lahr+pwZFR8A7GQleRWeI6cGFC2UA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/deep-eql": "*", + "assertion-error": "^2.0.1" + } + }, + "node_modules/@types/deep-eql": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/@types/deep-eql/-/deep-eql-4.0.2.tgz", + "integrity": "sha512-c9h9dVVMigMPc4bwTvC5dxqtqJZwQPePsWjPlpSOnojbor6pGqdk541lfA7AqFQr5pB1BRdq0juY9db81BwyFw==", + "dev": true, + "license": "MIT" }, "node_modules/@types/estree": { "version": "1.0.9", @@ -1089,38 +1063,40 @@ } }, "node_modules/@vitest/expect": { - "version": "2.1.9", - "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-2.1.9.tgz", - "integrity": "sha512-UJCIkTBenHeKT1TTlKMJWy1laZewsRIzYighyYiJKZreqtdxSos/S1t+ktRMQWu2CKqaarrkeszJx1cgC5tGZw==", + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-4.1.0.tgz", + "integrity": "sha512-EIxG7k4wlWweuCLG9Y5InKFwpMEOyrMb6ZJ1ihYu02LVj/bzUwn2VMU+13PinsjRW75XnITeFrQBMH5+dLvCDA==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/spy": "2.1.9", - "@vitest/utils": "2.1.9", - "chai": "^5.1.2", - "tinyrainbow": "^1.2.0" + "@standard-schema/spec": "^1.1.0", + "@types/chai": "^5.2.2", + "@vitest/spy": "4.1.0", + "@vitest/utils": "4.1.0", + "chai": "^6.2.2", + "tinyrainbow": "^3.0.3" }, "funding": { "url": "https://opencollective.com/vitest" } }, "node_modules/@vitest/mocker": { - "version": "2.1.9", - "resolved": "https://registry.npmjs.org/@vitest/mocker/-/mocker-2.1.9.tgz", - "integrity": "sha512-tVL6uJgoUdi6icpxmdrn5YNo3g3Dxv+IHJBr0GXHaEdTcw3F+cPKnsXFhli6nO+f/6SDKPHEK1UN+k+TQv0Ehg==", + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@vitest/mocker/-/mocker-4.1.0.tgz", + "integrity": "sha512-evxREh+Hork43+Y4IOhTo+h5lGmVRyjqI739Rz4RlUPqwrkFFDF6EMvOOYjTx4E8Tl6gyCLRL8Mu7Ry12a13Tw==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/spy": "2.1.9", + "@vitest/spy": "4.1.0", "estree-walker": "^3.0.3", - "magic-string": "^0.30.12" + "magic-string": "^0.30.21" }, "funding": { "url": "https://opencollective.com/vitest" }, "peerDependencies": { "msw": "^2.4.9", - "vite": "^5.0.0" + "vite": "^6.0.0 || ^7.0.0 || ^8.0.0-0" }, "peerDependenciesMeta": { "msw": { @@ -1132,70 +1108,68 @@ } }, "node_modules/@vitest/pretty-format": { - "version": "2.1.9", - "resolved": "https://registry.npmjs.org/@vitest/pretty-format/-/pretty-format-2.1.9.tgz", - "integrity": "sha512-KhRIdGV2U9HOUzxfiHmY8IFHTdqtOhIzCpd8WRdJiE7D/HUcZVD0EgQCVjm+Q9gkUXWgBvMmTtZgIG48wq7sOQ==", + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@vitest/pretty-format/-/pretty-format-4.1.0.tgz", + "integrity": "sha512-3RZLZlh88Ib0J7NQTRATfc/3ZPOnSUn2uDBUoGNn5T36+bALixmzphN26OUD3LRXWkJu4H0s5vvUeqBiw+kS0A==", "dev": true, "license": "MIT", "dependencies": { - "tinyrainbow": "^1.2.0" + "tinyrainbow": "^3.0.3" }, "funding": { "url": "https://opencollective.com/vitest" } }, "node_modules/@vitest/runner": { - "version": "2.1.9", - "resolved": "https://registry.npmjs.org/@vitest/runner/-/runner-2.1.9.tgz", - "integrity": "sha512-ZXSSqTFIrzduD63btIfEyOmNcBmQvgOVsPNPe0jYtESiXkhd8u2erDLnMxmGrDCwHCCHE7hxwRDCT3pt0esT4g==", + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@vitest/runner/-/runner-4.1.0.tgz", + "integrity": "sha512-Duvx2OzQ7d6OjchL+trw+aSrb9idh7pnNfxrklo14p3zmNL4qPCDeIJAK+eBKYjkIwG96Bc6vYuxhqDXQOWpoQ==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/utils": "2.1.9", - "pathe": "^1.1.2" + "@vitest/utils": "4.1.0", + "pathe": "^2.0.3" }, "funding": { "url": "https://opencollective.com/vitest" } }, "node_modules/@vitest/snapshot": { - "version": "2.1.9", - "resolved": "https://registry.npmjs.org/@vitest/snapshot/-/snapshot-2.1.9.tgz", - "integrity": "sha512-oBO82rEjsxLNJincVhLhaxxZdEtV0EFHMK5Kmx5sJ6H9L183dHECjiefOAdnqpIgT5eZwT04PoggUnW88vOBNQ==", + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@vitest/snapshot/-/snapshot-4.1.0.tgz", + "integrity": "sha512-0Vy9euT1kgsnj1CHttwi9i9o+4rRLEaPRSOJ5gyv579GJkNpgJK+B4HSv/rAWixx2wdAFci1X4CEPjiu2bXIMg==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/pretty-format": "2.1.9", - "magic-string": "^0.30.12", - "pathe": "^1.1.2" + "@vitest/pretty-format": "4.1.0", + "@vitest/utils": "4.1.0", + "magic-string": "^0.30.21", + "pathe": "^2.0.3" }, "funding": { "url": "https://opencollective.com/vitest" } }, "node_modules/@vitest/spy": { - "version": "2.1.9", - "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-2.1.9.tgz", - "integrity": "sha512-E1B35FwzXXTs9FHNK6bDszs7mtydNi5MIfUWpceJ8Xbfb1gBMscAnwLbEu+B44ed6W3XjL9/ehLPHR1fkf1KLQ==", + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-4.1.0.tgz", + "integrity": "sha512-pz77k+PgNpyMDv2FV6qmk5ZVau6c3R8HC8v342T2xlFxQKTrSeYw9waIJG8KgV9fFwAtTu4ceRzMivPTH6wSxw==", "dev": true, "license": "MIT", - "dependencies": { - "tinyspy": "^3.0.2" - }, "funding": { "url": "https://opencollective.com/vitest" } }, "node_modules/@vitest/utils": { - "version": "2.1.9", - "resolved": "https://registry.npmjs.org/@vitest/utils/-/utils-2.1.9.tgz", - "integrity": "sha512-v0psaMSkNJ3A2NMrUEHFRzJtDPFn+/VWZ5WxImB21T9fjucJRmS7xCS3ppEnARb9y11OAzaD+P2Ps+b+BGX5iQ==", + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/@vitest/utils/-/utils-4.1.0.tgz", + "integrity": "sha512-XfPXT6a8TZY3dcGY8EdwsBulFCIw+BeeX0RZn2x/BtiY/75YGh8FeWGG8QISN/WhaqSrE2OrlDgtF8q5uhOTmw==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/pretty-format": "2.1.9", - "loupe": "^3.1.2", - "tinyrainbow": "^1.2.0" + "@vitest/pretty-format": "4.1.0", + "convert-source-map": "^2.0.0", + "tinyrainbow": "^3.0.3" }, "funding": { "url": "https://opencollective.com/vitest" @@ -1300,43 +1274,16 @@ "node": "18 || 20 || >=22" } }, - "node_modules/cac": { - "version": "6.7.14", - "resolved": "https://registry.npmjs.org/cac/-/cac-6.7.14.tgz", - "integrity": "sha512-b6Ilus+c3RrdDk+JhLKUAQfzzgLEPy6wcXqS7f/xe1EETvsDP6GORG7SFuOs6cID5YkqchW/LXZbX5bc8j7ZcQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - } - }, "node_modules/chai": { - "version": "5.3.3", - "resolved": "https://registry.npmjs.org/chai/-/chai-5.3.3.tgz", - "integrity": "sha512-4zNhdJD/iOjSH0A05ea+Ke6MU5mmpQcbQsSOkgdaUMJ9zTlDTD/GYlwohmIE2u0gaxHYiVHEn1Fw9mZ/ktJWgw==", + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/chai/-/chai-6.2.2.tgz", + "integrity": "sha512-NUPRluOfOiTKBKvWPtSD4PhFvWCqOi0BGStNWs57X9js7XGTprSmFoz5F0tWhR4WPjNeR9jXqdC7/UpSJTnlRg==", "dev": true, "license": "MIT", - "dependencies": { - "assertion-error": "^2.0.1", - "check-error": "^2.1.1", - "deep-eql": "^5.0.1", - "loupe": "^3.1.0", - "pathval": "^2.0.0" - }, "engines": { "node": ">=18" } }, - "node_modules/check-error": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/check-error/-/check-error-2.1.3.tgz", - "integrity": "sha512-PAJdDJusoxnwm1VwW07VWwUN1sl7smmC3OKggvndJFadxxDRyFJBX/ggnu/KE4kQAB7a3Dp8f/YXC1FlUprWmA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 16" - } - }, "node_modules/colorette": { "version": "2.0.20", "resolved": "https://registry.npmjs.org/colorette/-/colorette-2.0.20.tgz", @@ -1356,6 +1303,13 @@ "url": "https://opencollective.com/express" } }, + "node_modules/convert-source-map": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-2.0.0.tgz", + "integrity": "sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==", + "dev": true, + "license": "MIT" + }, "node_modules/cookie": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/cookie/-/cookie-1.1.1.tgz", @@ -1378,34 +1332,6 @@ "node": "*" } }, - "node_modules/debug": { - "version": "4.4.3", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", - "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", - "dev": true, - "license": "MIT", - "dependencies": { - "ms": "^2.1.3" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, - "node_modules/deep-eql": { - "version": "5.0.2", - "resolved": "https://registry.npmjs.org/deep-eql/-/deep-eql-5.0.2.tgz", - "integrity": "sha512-h5k/5U50IJJFpzfL6nO9jaaumfjO/f2NjK/oYB2Djzm4p9L+3T9qWpZqZ2hAbLPuuYq9wrU08WQyBTL5GbPk5Q==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6" - } - }, "node_modules/depd": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", @@ -1424,6 +1350,16 @@ "node": ">=6" } }, + "node_modules/detect-libc": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", + "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=8" + } + }, "node_modules/end-of-stream": { "version": "1.4.5", "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.5.tgz", @@ -1434,9 +1370,9 @@ } }, "node_modules/es-module-lexer": { - "version": "1.7.0", - "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-1.7.0.tgz", - "integrity": "sha512-jEQoCwk8hyb2AZziIOLhDqpm5+2ww5uIE6lkO/6jcOCusfk6LhMHpXXfBLXTZ7Ydyt0j4VoUQv6uGNYbdW+kBA==", + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-2.1.0.tgz", + "integrity": "sha512-n27zTYMjYu1aj4MjCWzSP7G9r75utsaoc8m61weK+W8JMBGGQybd43GstCXZ3WNmSFtGT9wi59qQTW6mhTR5LQ==", "dev": true, "license": "MIT" }, @@ -1639,6 +1575,24 @@ "reusify": "^1.0.4" } }, + "node_modules/fdir": { + "version": "6.5.0", + "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.5.0.tgz", + "integrity": "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12.0.0" + }, + "peerDependencies": { + "picomatch": "^3 || ^4" + }, + "peerDependenciesMeta": { + "picomatch": { + "optional": true + } + } + }, "node_modules/find-my-way": { "version": "9.6.0", "resolved": "https://registry.npmjs.org/find-my-way/-/find-my-way-9.6.0.tgz", @@ -1810,12 +1764,266 @@ ], "license": "MIT" }, - "node_modules/loupe": { - "version": "3.2.1", - "resolved": "https://registry.npmjs.org/loupe/-/loupe-3.2.1.tgz", - "integrity": "sha512-CdzqowRJCeLU72bHvWqwRBBlLcMEtIvGrlvef74kMnV2AolS9Y8xUv1I0U/MNAWMhBlKIoyuEgoJ0t/bbwHbLQ==", + "node_modules/lightningcss": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss/-/lightningcss-1.32.0.tgz", + "integrity": "sha512-NXYBzinNrblfraPGyrbPoD19C1h9lfI/1mzgWYvXUTe414Gz/X1FD2XBZSZM7rRTrMA8JL3OtAaGifrIKhQ5yQ==", "dev": true, - "license": "MIT" + "license": "MPL-2.0", + "dependencies": { + "detect-libc": "^2.0.3" + }, + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + }, + "optionalDependencies": { + "lightningcss-android-arm64": "1.32.0", + "lightningcss-darwin-arm64": "1.32.0", + "lightningcss-darwin-x64": "1.32.0", + "lightningcss-freebsd-x64": "1.32.0", + "lightningcss-linux-arm-gnueabihf": "1.32.0", + "lightningcss-linux-arm64-gnu": "1.32.0", + "lightningcss-linux-arm64-musl": "1.32.0", + "lightningcss-linux-x64-gnu": "1.32.0", + "lightningcss-linux-x64-musl": "1.32.0", + "lightningcss-win32-arm64-msvc": "1.32.0", + "lightningcss-win32-x64-msvc": "1.32.0" + } + }, + "node_modules/lightningcss-android-arm64": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-android-arm64/-/lightningcss-android-arm64-1.32.0.tgz", + "integrity": "sha512-YK7/ClTt4kAK0vo6w3X+Pnm0D2cf2vPHbhOXdoNti1Ga0al1P4TBZhwjATvjNwLEBCnKvjJc2jQgHXH0NEwlAg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-darwin-arm64": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-darwin-arm64/-/lightningcss-darwin-arm64-1.32.0.tgz", + "integrity": "sha512-RzeG9Ju5bag2Bv1/lwlVJvBE3q6TtXskdZLLCyfg5pt+HLz9BqlICO7LZM7VHNTTn/5PRhHFBSjk5lc4cmscPQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-darwin-x64": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-darwin-x64/-/lightningcss-darwin-x64-1.32.0.tgz", + "integrity": "sha512-U+QsBp2m/s2wqpUYT/6wnlagdZbtZdndSmut/NJqlCcMLTWp5muCrID+K5UJ6jqD2BFshejCYXniPDbNh73V8w==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-freebsd-x64": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-freebsd-x64/-/lightningcss-freebsd-x64-1.32.0.tgz", + "integrity": "sha512-JCTigedEksZk3tHTTthnMdVfGf61Fky8Ji2E4YjUTEQX14xiy/lTzXnu1vwiZe3bYe0q+SpsSH/CTeDXK6WHig==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-arm-gnueabihf": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-arm-gnueabihf/-/lightningcss-linux-arm-gnueabihf-1.32.0.tgz", + "integrity": "sha512-x6rnnpRa2GL0zQOkt6rts3YDPzduLpWvwAF6EMhXFVZXD4tPrBkEFqzGowzCsIWsPjqSK+tyNEODUBXeeVHSkw==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-arm64-gnu": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-arm64-gnu/-/lightningcss-linux-arm64-gnu-1.32.0.tgz", + "integrity": "sha512-0nnMyoyOLRJXfbMOilaSRcLH3Jw5z9HDNGfT/gwCPgaDjnx0i8w7vBzFLFR1f6CMLKF8gVbebmkUN3fa/kQJpQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-arm64-musl": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-arm64-musl/-/lightningcss-linux-arm64-musl-1.32.0.tgz", + "integrity": "sha512-UpQkoenr4UJEzgVIYpI80lDFvRmPVg6oqboNHfoH4CQIfNA+HOrZ7Mo7KZP02dC6LjghPQJeBsvXhJod/wnIBg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-x64-gnu": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-x64-gnu/-/lightningcss-linux-x64-gnu-1.32.0.tgz", + "integrity": "sha512-V7Qr52IhZmdKPVr+Vtw8o+WLsQJYCTd8loIfpDaMRWGUZfBOYEJeyJIkqGIDMZPwPx24pUMfwSxxI8phr/MbOA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-x64-musl": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-x64-musl/-/lightningcss-linux-x64-musl-1.32.0.tgz", + "integrity": "sha512-bYcLp+Vb0awsiXg/80uCRezCYHNg1/l3mt0gzHnWV9XP1W5sKa5/TCdGWaR/zBM2PeF/HbsQv/j2URNOiVuxWg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-win32-arm64-msvc": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-win32-arm64-msvc/-/lightningcss-win32-arm64-msvc-1.32.0.tgz", + "integrity": "sha512-8SbC8BR40pS6baCM8sbtYDSwEVQd4JlFTOlaD3gWGHfThTcABnNDBda6eTZeqbofalIJhFx0qKzgHJmcPTnGdw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-win32-x64-msvc": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-win32-x64-msvc/-/lightningcss-win32-x64-msvc-1.32.0.tgz", + "integrity": "sha512-Amq9B/SoZYdDi1kFrojnoqPLxYhQ4Wo5XiL8EVJrVsB8ARoC1PWW6VGtT0WKCemjy8aC+louJnjS7U18x3b06Q==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } }, "node_modules/lru-cache": { "version": "11.5.0", @@ -1881,13 +2089,6 @@ "node": ">=16 || 14 >=14.17" } }, - "node_modules/ms": { - "version": "2.1.3", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", - "dev": true, - "license": "MIT" - }, "node_modules/nanoid": { "version": "3.3.12", "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.12.tgz", @@ -1907,6 +2108,17 @@ "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" } }, + "node_modules/obug": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/obug/-/obug-2.1.1.tgz", + "integrity": "sha512-uTqF9MuPraAQ+IsnPf366RG4cP9RtUi7MLO1N3KEc+wb0a6yKpeL0lmk2IB1jY5KHPAlTc6T/JRdC/YqxHNwkQ==", + "dev": true, + "funding": [ + "https://github.com/sponsors/sxzz", + "https://opencollective.com/debug" + ], + "license": "MIT" + }, "node_modules/on-exit-leak-free": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/on-exit-leak-free/-/on-exit-leak-free-2.1.2.tgz", @@ -1942,22 +2154,12 @@ } }, "node_modules/pathe": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/pathe/-/pathe-1.1.2.tgz", - "integrity": "sha512-whLdWMYL2TwI08hn8/ZqAbrVemu0LNaNNJZX73O6qaIdCTfXutsLhMkjdENX0qhsQ9uIimo4/aQOmXkoon2nDQ==", + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/pathe/-/pathe-2.0.3.tgz", + "integrity": "sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w==", "dev": true, "license": "MIT" }, - "node_modules/pathval": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/pathval/-/pathval-2.0.1.tgz", - "integrity": "sha512-//nshmD55c46FuFw26xV/xFAaB5HF9Xdap7HJBBnrKdAd6/GxDBaNA1870O79+9ueg61cZLSVc+OaFlfmObYVQ==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 14.16" - } - }, "node_modules/picocolors": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", @@ -1965,6 +2167,19 @@ "dev": true, "license": "ISC" }, + "node_modules/picomatch": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.4.tgz", + "integrity": "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, "node_modules/pino": { "version": "9.14.0", "resolved": "https://registry.npmjs.org/pino/-/pino-9.14.0.tgz", @@ -2036,9 +2251,9 @@ "license": "MIT" }, "node_modules/postcss": { - "version": "8.5.14", - "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.14.tgz", - "integrity": "sha512-SoSL4+OSEtR99LHFZQiJLkT59C5B1amGO1NzTwj7TT1qCUgUO6hxOvzkOYxD+vMrXBM3XJIKzokoERdqQq/Zmg==", + "version": "8.5.15", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.15.tgz", + "integrity": "sha512-FfR8sjd4em2T6fb3I2MwAJU7HWVMr9zba+enmQeeWFfCbm+UOC/0X4DS8XtpUTMwWMGbjKYP7xjfNekzyGmB3A==", "dev": true, "funding": [ { @@ -2056,7 +2271,7 @@ ], "license": "MIT", "dependencies": { - "nanoid": "^3.3.11", + "nanoid": "^3.3.12", "picocolors": "^1.1.1", "source-map-js": "^1.2.1" }, @@ -2149,57 +2364,39 @@ "integrity": "sha512-q1b3N5QkRUWUl7iyylaaj3kOpIT0N2i9MqIEQXP73GVsN9cw3fdx8X63cEmWhJGi2PPCF23Ijp7ktmd39rawIA==", "license": "MIT" }, - "node_modules/rollup": { - "version": "4.60.3", - "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.60.3.tgz", - "integrity": "sha512-pAQK9HalE84QSm4Po3EmWIZPd3FnjkShVkiMlz1iligWYkWQ7wHYd1PF/T7QZ5TVSD6uSTon5gBVMSM4JfBV+A==", + "node_modules/rolldown": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/rolldown/-/rolldown-1.0.3.tgz", + "integrity": "sha512-i00lAJ2ks1BYr7rjNjKC7BcqAS7nVfiT3QX1SI5aY+AFHblCmaUf9OE9dbdzDvW6dJxbi2ZCZiy9v3CcwOiX3g==", "dev": true, "license": "MIT", "dependencies": { - "@types/estree": "1.0.8" + "@oxc-project/types": "=0.133.0", + "@rolldown/pluginutils": "^1.0.0" }, "bin": { - "rollup": "dist/bin/rollup" + "rolldown": "bin/cli.mjs" }, "engines": { - "node": ">=18.0.0", - "npm": ">=8.0.0" + "node": "^20.19.0 || >=22.12.0" }, "optionalDependencies": { - "@rollup/rollup-android-arm-eabi": "4.60.3", - "@rollup/rollup-android-arm64": "4.60.3", - "@rollup/rollup-darwin-arm64": "4.60.3", - "@rollup/rollup-darwin-x64": "4.60.3", - "@rollup/rollup-freebsd-arm64": "4.60.3", - "@rollup/rollup-freebsd-x64": "4.60.3", - "@rollup/rollup-linux-arm-gnueabihf": "4.60.3", - "@rollup/rollup-linux-arm-musleabihf": "4.60.3", - "@rollup/rollup-linux-arm64-gnu": "4.60.3", - "@rollup/rollup-linux-arm64-musl": "4.60.3", - "@rollup/rollup-linux-loong64-gnu": "4.60.3", - "@rollup/rollup-linux-loong64-musl": "4.60.3", - "@rollup/rollup-linux-ppc64-gnu": "4.60.3", - "@rollup/rollup-linux-ppc64-musl": "4.60.3", - "@rollup/rollup-linux-riscv64-gnu": "4.60.3", - "@rollup/rollup-linux-riscv64-musl": "4.60.3", - "@rollup/rollup-linux-s390x-gnu": "4.60.3", - "@rollup/rollup-linux-x64-gnu": "4.60.3", - "@rollup/rollup-linux-x64-musl": "4.60.3", - "@rollup/rollup-openbsd-x64": "4.60.3", - "@rollup/rollup-openharmony-arm64": "4.60.3", - "@rollup/rollup-win32-arm64-msvc": "4.60.3", - "@rollup/rollup-win32-ia32-msvc": "4.60.3", - "@rollup/rollup-win32-x64-gnu": "4.60.3", - "@rollup/rollup-win32-x64-msvc": "4.60.3", - "fsevents": "~2.3.2" - } - }, - "node_modules/rollup/node_modules/@types/estree": { - "version": "1.0.8", - "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.8.tgz", - "integrity": "sha512-dWHzHa2WqEXI/O1E9OjrocMTKJl2mSrEolh1Iomrv6U+JuNwaHXsXx9bLu5gG7BUWFIN0skIQJQ/L1rIex4X6w==", - "dev": true, - "license": "MIT" + "@rolldown/binding-android-arm64": "1.0.3", + "@rolldown/binding-darwin-arm64": "1.0.3", + "@rolldown/binding-darwin-x64": "1.0.3", + "@rolldown/binding-freebsd-x64": "1.0.3", + "@rolldown/binding-linux-arm-gnueabihf": "1.0.3", + "@rolldown/binding-linux-arm64-gnu": "1.0.3", + "@rolldown/binding-linux-arm64-musl": "1.0.3", + "@rolldown/binding-linux-ppc64-gnu": "1.0.3", + "@rolldown/binding-linux-s390x-gnu": "1.0.3", + "@rolldown/binding-linux-x64-gnu": "1.0.3", + "@rolldown/binding-linux-x64-musl": "1.0.3", + "@rolldown/binding-openharmony-arm64": "1.0.3", + "@rolldown/binding-wasm32-wasi": "1.0.3", + "@rolldown/binding-win32-arm64-msvc": "1.0.3", + "@rolldown/binding-win32-x64-msvc": "1.0.3" + } }, "node_modules/safe-regex2": { "version": "5.1.1", @@ -2324,9 +2521,9 @@ } }, "node_modules/std-env": { - "version": "3.10.0", - "resolved": "https://registry.npmjs.org/std-env/-/std-env-3.10.0.tgz", - "integrity": "sha512-5GS12FdOZNliM5mAOxFRg7Ir0pWz8MdpYm6AY6VPkGpbA7ZzmbzNcBJQ0GPvvyWgcY7QAhCgf9Uy89I03faLkg==", + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/std-env/-/std-env-4.1.0.tgz", + "integrity": "sha512-Rq7ybcX2RuC55r9oaPVEW7/xu3tj8u4GeBYHBWCychFtzMIr86A7e3PPEBPT37sHStKX3+TiX/Fr/ACmJLVlLQ==", "dev": true, "license": "MIT" }, @@ -2359,36 +2556,36 @@ "license": "MIT" }, "node_modules/tinyexec": { - "version": "0.3.2", - "resolved": "https://registry.npmjs.org/tinyexec/-/tinyexec-0.3.2.tgz", - "integrity": "sha512-KQQR9yN7R5+OSwaK0XQoj22pwHoTlgYqmUscPYoknOoWCWfj/5/ABTMRi69FrKU5ffPVh5QcFikpWJI/P1ocHA==", - "dev": true, - "license": "MIT" - }, - "node_modules/tinypool": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/tinypool/-/tinypool-1.1.1.tgz", - "integrity": "sha512-Zba82s87IFq9A9XmjiX5uZA/ARWDrB03OHlq+Vw1fSdt0I+4/Kutwy8BP4Y/y/aORMo61FQ0vIb5j44vSo5Pkg==", + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/tinyexec/-/tinyexec-1.2.4.tgz", + "integrity": "sha512-SHf/r48b7vOrjve9PxJo3MN5v5yuyjHvdUcrQffT3WXMUfnGmHDVbC4k3sHJaJTgZCwpUplIaAo5ANtMyp3YHg==", "dev": true, "license": "MIT", "engines": { - "node": "^18.0.0 || >=20.0.0" + "node": ">=18" } }, - "node_modules/tinyrainbow": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/tinyrainbow/-/tinyrainbow-1.2.0.tgz", - "integrity": "sha512-weEDEq7Z5eTHPDh4xjX789+fHfF+P8boiFB+0vbWzpbnbsEr/GRaohi/uMKxg8RZMXnl1ItAi/IUHWMsjDV7kQ==", + "node_modules/tinyglobby": { + "version": "0.2.17", + "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.17.tgz", + "integrity": "sha512-wXR/dYpcqKmfWpEdZjiKJOwCNFndD0DMnrW/cYjVGttEkBfVgcLFHoNrlj47mjOVic9yyNu65alsgF4NQyTa2g==", "dev": true, "license": "MIT", + "dependencies": { + "fdir": "^6.5.0", + "picomatch": "^4.0.4" + }, "engines": { - "node": ">=14.0.0" + "node": ">=12.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/SuperchupuDev" } }, - "node_modules/tinyspy": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/tinyspy/-/tinyspy-3.0.2.tgz", - "integrity": "sha512-n1cw8k1k0x4pgA2+9XrOkFydTerNcJ1zWCO5Nn9scWHTD+5tp8dghT2x1uduQePZTZgd3Tupf+x9BxJjeJi77Q==", + "node_modules/tinyrainbow": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/tinyrainbow/-/tinyrainbow-3.1.0.tgz", + "integrity": "sha512-Bf+ILmBgretUrdJxzXM0SgXLZ3XfiaUuOj/IKQHuTXip+05Xn+uyEYdVg0kYDipTBcLrCVyUzAPz7QmArb0mmw==", "dev": true, "license": "MIT", "engines": { @@ -2413,6 +2610,14 @@ "node": ">=0.6" } }, + "node_modules/tslib": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", + "dev": true, + "license": "0BSD", + "optional": true + }, "node_modules/tsx": { "version": "4.21.0", "resolved": "https://registry.npmjs.org/tsx/-/tsx-4.21.0.tgz", @@ -2464,21 +2669,23 @@ "license": "MIT" }, "node_modules/vite": { - "version": "5.4.21", - "resolved": "https://registry.npmjs.org/vite/-/vite-5.4.21.tgz", - "integrity": "sha512-o5a9xKjbtuhY6Bi5S3+HvbRERmouabWbyUcpXXUA1u+GNUKoROi9byOJ8M0nHbHYHkYICiMlqxkg1KkYmm25Sw==", + "version": "8.0.16", + "resolved": "https://registry.npmjs.org/vite/-/vite-8.0.16.tgz", + "integrity": "sha512-h9bXPmJichP5fLmVQo3PyaGSDE2n3aPuomeAlVRm0JLmt4rY6zmPKd59HYI4LNW8oTK7tlTsuC7l/m7awx9Jcw==", "dev": true, "license": "MIT", "dependencies": { - "esbuild": "^0.21.3", - "postcss": "^8.4.43", - "rollup": "^4.20.0" + "lightningcss": "^1.32.0", + "picomatch": "^4.0.4", + "postcss": "^8.5.15", + "rolldown": "1.0.3", + "tinyglobby": "^0.2.17" }, "bin": { "vite": "bin/vite.js" }, "engines": { - "node": "^18.0.0 || >=20.0.0" + "node": "^20.19.0 || >=22.12.0" }, "funding": { "url": "https://github.com/vitejs/vite?sponsor=1" @@ -2487,23 +2694,33 @@ "fsevents": "~2.3.3" }, "peerDependencies": { - "@types/node": "^18.0.0 || >=20.0.0", - "less": "*", - "lightningcss": "^1.21.0", - "sass": "*", - "sass-embedded": "*", - "stylus": "*", - "sugarss": "*", - "terser": "^5.4.0" + "@types/node": "^20.19.0 || >=22.12.0", + "@vitejs/devtools": "^0.1.18", + "esbuild": "^0.27.0 || ^0.28.0", + "jiti": ">=1.21.0", + "less": "^4.0.0", + "sass": "^1.70.0", + "sass-embedded": "^1.70.0", + "stylus": ">=0.54.8", + "sugarss": "^5.0.0", + "terser": "^5.16.0", + "tsx": "^4.8.1", + "yaml": "^2.4.2" }, "peerDependenciesMeta": { "@types/node": { "optional": true }, - "less": { + "@vitejs/devtools": { + "optional": true + }, + "esbuild": { + "optional": true + }, + "jiti": { "optional": true }, - "lightningcss": { + "less": { "optional": true }, "sass": { @@ -2520,515 +2737,81 @@ }, "terser": { "optional": true + }, + "tsx": { + "optional": true + }, + "yaml": { + "optional": true } } }, - "node_modules/vite-node": { - "version": "2.1.9", - "resolved": "https://registry.npmjs.org/vite-node/-/vite-node-2.1.9.tgz", - "integrity": "sha512-AM9aQ/IPrW/6ENLQg3AGY4K1N2TGZdR5e4gu/MmmR2xR3Ll1+dib+nook92g4TV3PXVyeyxdWwtaCAiUL0hMxA==", + "node_modules/vitest": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/vitest/-/vitest-4.1.0.tgz", + "integrity": "sha512-YbDrMF9jM2Lqc++2530UourxZHmkKLxrs4+mYhEwqWS97WJ7wOYEkcr+QfRgJ3PW9wz3odRijLZjHEaRLTNbqw==", "dev": true, "license": "MIT", "dependencies": { - "cac": "^6.7.14", - "debug": "^4.3.7", - "es-module-lexer": "^1.5.4", - "pathe": "^1.1.2", - "vite": "^5.0.0" + "@vitest/expect": "4.1.0", + "@vitest/mocker": "4.1.0", + "@vitest/pretty-format": "4.1.0", + "@vitest/runner": "4.1.0", + "@vitest/snapshot": "4.1.0", + "@vitest/spy": "4.1.0", + "@vitest/utils": "4.1.0", + "es-module-lexer": "^2.0.0", + "expect-type": "^1.3.0", + "magic-string": "^0.30.21", + "obug": "^2.1.1", + "pathe": "^2.0.3", + "picomatch": "^4.0.3", + "std-env": "^4.0.0-rc.1", + "tinybench": "^2.9.0", + "tinyexec": "^1.0.2", + "tinyglobby": "^0.2.15", + "tinyrainbow": "^3.0.3", + "vite": "^6.0.0 || ^7.0.0 || ^8.0.0-0", + "why-is-node-running": "^2.3.0" }, "bin": { - "vite-node": "vite-node.mjs" + "vitest": "vitest.mjs" }, "engines": { - "node": "^18.0.0 || >=20.0.0" - }, - "funding": { - "url": "https://opencollective.com/vitest" - } - }, - "node_modules/vite/node_modules/@esbuild/aix-ppc64": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.21.5.tgz", - "integrity": "sha512-1SDgH6ZSPTlggy1yI6+Dbkiz8xzpHJEVAlF/AM1tHPLsf5STom9rwtjE4hKAF20FfXXNTFqEYXyJNWh1GiZedQ==", - "cpu": [ - "ppc64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "aix" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/vite/node_modules/@esbuild/android-arm": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.21.5.tgz", - "integrity": "sha512-vCPvzSjpPHEi1siZdlvAlsPxXl7WbOVUBBAowWug4rJHb68Ox8KualB+1ocNvT5fjv6wpkX6o/iEpbDrf68zcg==", - "cpu": [ - "arm" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/vite/node_modules/@esbuild/android-arm64": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.21.5.tgz", - "integrity": "sha512-c0uX9VAUBQ7dTDCjq+wdyGLowMdtR/GoC2U5IYk/7D1H1JYC0qseD7+11iMP2mRLN9RcCMRcjC4YMclCzGwS/A==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/vite/node_modules/@esbuild/android-x64": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.21.5.tgz", - "integrity": "sha512-D7aPRUUNHRBwHxzxRvp856rjUHRFW1SdQATKXH2hqA0kAZb1hKmi02OpYRacl0TxIGz/ZmXWlbZgjwWYaCakTA==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/vite/node_modules/@esbuild/darwin-arm64": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.21.5.tgz", - "integrity": "sha512-DwqXqZyuk5AiWWf3UfLiRDJ5EDd49zg6O9wclZ7kUMv2WRFr4HKjXp/5t8JZ11QbQfUS6/cRCKGwYhtNAY88kQ==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/vite/node_modules/@esbuild/darwin-x64": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.21.5.tgz", - "integrity": "sha512-se/JjF8NlmKVG4kNIuyWMV/22ZaerB+qaSi5MdrXtd6R08kvs2qCN4C09miupktDitvh8jRFflwGFBQcxZRjbw==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/vite/node_modules/@esbuild/freebsd-arm64": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.21.5.tgz", - "integrity": "sha512-5JcRxxRDUJLX8JXp/wcBCy3pENnCgBR9bN6JsY4OmhfUtIHe3ZW0mawA7+RDAcMLrMIZaf03NlQiX9DGyB8h4g==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "freebsd" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/vite/node_modules/@esbuild/freebsd-x64": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.21.5.tgz", - "integrity": "sha512-J95kNBj1zkbMXtHVH29bBriQygMXqoVQOQYA+ISs0/2l3T9/kj42ow2mpqerRBxDJnmkUDCaQT/dfNXWX/ZZCQ==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "freebsd" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/vite/node_modules/@esbuild/linux-arm": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.21.5.tgz", - "integrity": "sha512-bPb5AHZtbeNGjCKVZ9UGqGwo8EUu4cLq68E95A53KlxAPRmUyYv2D6F0uUI65XisGOL1hBP5mTronbgo+0bFcA==", - "cpu": [ - "arm" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/vite/node_modules/@esbuild/linux-arm64": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.21.5.tgz", - "integrity": "sha512-ibKvmyYzKsBeX8d8I7MH/TMfWDXBF3db4qM6sy+7re0YXya+K1cem3on9XgdT2EQGMu4hQyZhan7TeQ8XkGp4Q==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/vite/node_modules/@esbuild/linux-ia32": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.21.5.tgz", - "integrity": "sha512-YvjXDqLRqPDl2dvRODYmmhz4rPeVKYvppfGYKSNGdyZkA01046pLWyRKKI3ax8fbJoK5QbxblURkwK/MWY18Tg==", - "cpu": [ - "ia32" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/vite/node_modules/@esbuild/linux-loong64": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.21.5.tgz", - "integrity": "sha512-uHf1BmMG8qEvzdrzAqg2SIG/02+4/DHB6a9Kbya0XDvwDEKCoC8ZRWI5JJvNdUjtciBGFQ5PuBlpEOXQj+JQSg==", - "cpu": [ - "loong64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/vite/node_modules/@esbuild/linux-mips64el": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.21.5.tgz", - "integrity": "sha512-IajOmO+KJK23bj52dFSNCMsz1QP1DqM6cwLUv3W1QwyxkyIWecfafnI555fvSGqEKwjMXVLokcV5ygHW5b3Jbg==", - "cpu": [ - "mips64el" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/vite/node_modules/@esbuild/linux-ppc64": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.21.5.tgz", - "integrity": "sha512-1hHV/Z4OEfMwpLO8rp7CvlhBDnjsC3CttJXIhBi+5Aj5r+MBvy4egg7wCbe//hSsT+RvDAG7s81tAvpL2XAE4w==", - "cpu": [ - "ppc64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/vite/node_modules/@esbuild/linux-riscv64": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.21.5.tgz", - "integrity": "sha512-2HdXDMd9GMgTGrPWnJzP2ALSokE/0O5HhTUvWIbD3YdjME8JwvSCnNGBnTThKGEB91OZhzrJ4qIIxk/SBmyDDA==", - "cpu": [ - "riscv64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/vite/node_modules/@esbuild/linux-s390x": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.21.5.tgz", - "integrity": "sha512-zus5sxzqBJD3eXxwvjN1yQkRepANgxE9lgOW2qLnmr8ikMTphkjgXu1HR01K4FJg8h1kEEDAqDcZQtbrRnB41A==", - "cpu": [ - "s390x" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/vite/node_modules/@esbuild/linux-x64": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.21.5.tgz", - "integrity": "sha512-1rYdTpyv03iycF1+BhzrzQJCdOuAOtaqHTWJZCWvijKD2N5Xu0TtVC8/+1faWqcP9iBCWOmjmhoH94dH82BxPQ==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/vite/node_modules/@esbuild/netbsd-x64": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.21.5.tgz", - "integrity": "sha512-Woi2MXzXjMULccIwMnLciyZH4nCIMpWQAs049KEeMvOcNADVxo0UBIQPfSmxB3CWKedngg7sWZdLvLczpe0tLg==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "netbsd" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/vite/node_modules/@esbuild/openbsd-x64": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.21.5.tgz", - "integrity": "sha512-HLNNw99xsvx12lFBUwoT8EVCsSvRNDVxNpjZ7bPn947b8gJPzeHWyNVhFsaerc0n3TsbOINvRP2byTZ5LKezow==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "openbsd" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/vite/node_modules/@esbuild/sunos-x64": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.21.5.tgz", - "integrity": "sha512-6+gjmFpfy0BHU5Tpptkuh8+uw3mnrvgs+dSPQXQOv3ekbordwnzTVEb4qnIvQcYXq6gzkyTnoZ9dZG+D4garKg==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "sunos" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/vite/node_modules/@esbuild/win32-arm64": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.21.5.tgz", - "integrity": "sha512-Z0gOTd75VvXqyq7nsl93zwahcTROgqvuAcYDUr+vOv8uHhNSKROyU961kgtCD1e95IqPKSQKH7tBTslnS3tA8A==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/vite/node_modules/@esbuild/win32-ia32": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.21.5.tgz", - "integrity": "sha512-SWXFF1CL2RVNMaVs+BBClwtfZSvDgtL//G/smwAc5oVK/UPu2Gu9tIaRgFmYFFKrmg3SyAjSrElf0TiJ1v8fYA==", - "cpu": [ - "ia32" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/vite/node_modules/@esbuild/win32-x64": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.21.5.tgz", - "integrity": "sha512-tQd/1efJuzPC6rCFwEvLtci/xNFcTZknmXs98FYDfGE4wP9ClFV98nyKrzJKVPMhdDnjzLhdUyMX4PsQAPjwIw==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=12" - } - }, - "node_modules/vite/node_modules/esbuild": { - "version": "0.21.5", - "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.21.5.tgz", - "integrity": "sha512-mg3OPMV4hXywwpoDxu3Qda5xCKQi+vCTZq8S9J/EpkhB2HzKXq4SNFZE3+NK93JYxc8VMSep+lOUSC/RVKaBqw==", - "dev": true, - "hasInstallScript": true, - "license": "MIT", - "bin": { - "esbuild": "bin/esbuild" - }, - "engines": { - "node": ">=12" - }, - "optionalDependencies": { - "@esbuild/aix-ppc64": "0.21.5", - "@esbuild/android-arm": "0.21.5", - "@esbuild/android-arm64": "0.21.5", - "@esbuild/android-x64": "0.21.5", - "@esbuild/darwin-arm64": "0.21.5", - "@esbuild/darwin-x64": "0.21.5", - "@esbuild/freebsd-arm64": "0.21.5", - "@esbuild/freebsd-x64": "0.21.5", - "@esbuild/linux-arm": "0.21.5", - "@esbuild/linux-arm64": "0.21.5", - "@esbuild/linux-ia32": "0.21.5", - "@esbuild/linux-loong64": "0.21.5", - "@esbuild/linux-mips64el": "0.21.5", - "@esbuild/linux-ppc64": "0.21.5", - "@esbuild/linux-riscv64": "0.21.5", - "@esbuild/linux-s390x": "0.21.5", - "@esbuild/linux-x64": "0.21.5", - "@esbuild/netbsd-x64": "0.21.5", - "@esbuild/openbsd-x64": "0.21.5", - "@esbuild/sunos-x64": "0.21.5", - "@esbuild/win32-arm64": "0.21.5", - "@esbuild/win32-ia32": "0.21.5", - "@esbuild/win32-x64": "0.21.5" - } - }, - "node_modules/vitest": { - "version": "2.1.9", - "resolved": "https://registry.npmjs.org/vitest/-/vitest-2.1.9.tgz", - "integrity": "sha512-MSmPM9REYqDGBI8439mA4mWhV5sKmDlBKWIYbA3lRb2PTHACE0mgKwA8yQ2xq9vxDTuk4iPrECBAEW2aoFXY0Q==", - "dev": true, - "license": "MIT", - "dependencies": { - "@vitest/expect": "2.1.9", - "@vitest/mocker": "2.1.9", - "@vitest/pretty-format": "^2.1.9", - "@vitest/runner": "2.1.9", - "@vitest/snapshot": "2.1.9", - "@vitest/spy": "2.1.9", - "@vitest/utils": "2.1.9", - "chai": "^5.1.2", - "debug": "^4.3.7", - "expect-type": "^1.1.0", - "magic-string": "^0.30.12", - "pathe": "^1.1.2", - "std-env": "^3.8.0", - "tinybench": "^2.9.0", - "tinyexec": "^0.3.1", - "tinypool": "^1.0.1", - "tinyrainbow": "^1.2.0", - "vite": "^5.0.0", - "vite-node": "2.1.9", - "why-is-node-running": "^2.3.0" - }, - "bin": { - "vitest": "vitest.mjs" - }, - "engines": { - "node": "^18.0.0 || >=20.0.0" + "node": "^20.0.0 || ^22.0.0 || >=24.0.0" }, "funding": { "url": "https://opencollective.com/vitest" }, "peerDependencies": { "@edge-runtime/vm": "*", - "@types/node": "^18.0.0 || >=20.0.0", - "@vitest/browser": "2.1.9", - "@vitest/ui": "2.1.9", + "@opentelemetry/api": "^1.9.0", + "@types/node": "^20.0.0 || ^22.0.0 || >=24.0.0", + "@vitest/browser-playwright": "4.1.0", + "@vitest/browser-preview": "4.1.0", + "@vitest/browser-webdriverio": "4.1.0", + "@vitest/ui": "4.1.0", "happy-dom": "*", - "jsdom": "*" + "jsdom": "*", + "vite": "^6.0.0 || ^7.0.0 || ^8.0.0-0" }, "peerDependenciesMeta": { "@edge-runtime/vm": { "optional": true }, + "@opentelemetry/api": { + "optional": true + }, "@types/node": { "optional": true }, - "@vitest/browser": { + "@vitest/browser-playwright": { + "optional": true + }, + "@vitest/browser-preview": { + "optional": true + }, + "@vitest/browser-webdriverio": { "optional": true }, "@vitest/ui": { @@ -3039,6 +2822,9 @@ }, "jsdom": { "optional": true + }, + "vite": { + "optional": false } } }, diff --git a/web/backend/package.json b/web/backend/package.json index 88b60f04..549435f2 100644 --- a/web/backend/package.json +++ b/web/backend/package.json @@ -22,7 +22,7 @@ "@types/node": "^22.5.0", "tsx": "^4.19.0", "typescript": "^5.5.4", - "vitest": "^2.0.5" + "vitest": "^4.1.0" }, "engines": { "node": ">=20" diff --git a/web/frontend/src/App.tsx b/web/frontend/src/App.tsx index 2021799f..4e9e277b 100644 --- a/web/frontend/src/App.tsx +++ b/web/frontend/src/App.tsx @@ -21,6 +21,7 @@ import { SchedulesPage } from "./pages/Schedules"; import { SearchPage } from "./pages/Search"; import { SharingPage } from "./pages/Sharing"; import { StoragePage } from "./pages/Storage"; +import { StreamsPage } from "./pages/Streams"; import { WorkspacesPage } from "./pages/Workspaces"; import { UIStateProvider, useUIState } from "./state"; import { ThemeProvider } from "./theme"; @@ -38,6 +39,8 @@ function Router() { return ; case "storage": return ; + case "stream": + return ; case "jobs": return ; case "branches": diff --git a/web/frontend/src/layout/Sidebar.tsx b/web/frontend/src/layout/Sidebar.tsx index 0e6f4646..720a6ef6 100644 --- a/web/frontend/src/layout/Sidebar.tsx +++ b/web/frontend/src/layout/Sidebar.tsx @@ -16,6 +16,7 @@ import { Network, PackageSearch, PlayCircle, + Radio, Search, Sparkles, Terminal, @@ -47,6 +48,7 @@ const SECTIONS: Array<{ { id: "configs", label: "Configs", icon: Braces }, { id: "components", label: "Components", icon: PackageSearch }, { id: "storage", label: "Storage", icon: Database }, + { id: "stream", label: "Data Streams", icon: Radio }, { id: "jobs", label: "Jobs", icon: PlayCircle }, { id: "search", label: "Search", icon: Search }, ], diff --git a/web/frontend/src/pages/Streams.tsx b/web/frontend/src/pages/Streams.tsx new file mode 100644 index 00000000..81c03ce9 --- /dev/null +++ b/web/frontend/src/pages/Streams.tsx @@ -0,0 +1,510 @@ +import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; +import { Copy, Eye, EyeOff, Plus, Radio, Trash2 } from "lucide-react"; +import { useState } from "react"; +import { api } from "../api/client"; +import { Drawer } from "../components/Drawer"; +import { Empty, ErrorBox, Loading, PageTitle } from "../components/Empty"; +import { JsonView } from "../components/JsonView"; +import { DataTable } from "../components/Table"; +import { useUIState } from "../state"; +import type { DataStreamDetail, DataStreamSource } from "../types"; + +/** + * Data Streams (OpenTelemetry / OTLP). Mirrors `kbagent stream *`: + * list sources, create an OTLP/HTTP source (auto-provisioning the + * logs/metrics/traces sinks), inspect a source's endpoints + destination, + * and delete one. The OTLP ingest URL embeds a secret that the backend + * masks unless `reveal=true` -- the detail drawer exposes a reveal toggle. + */ +interface StreamListResp { + alias: string; + branch_id: string; + sources: DataStreamSource[]; +} + +/** + * The stream control-plane API types its branch ref as a string. The UI's + * global `branchId` is a numeric Storage branch ID; numeric IDs are valid refs + * (see `test_branch_override` in `tests/test_stream_service.py`, which drives + * `branch_id="1234"`), so we stringify here to match the API contract + * explicitly rather than lean on JSON/query coercion. `null` (default branch) + * maps to `undefined`, letting the backend fall back to its `"default"` ref. + */ +function branchRef(branchId: number | null): string | undefined { + return branchId != null ? String(branchId) : undefined; +} + +export function StreamsPage() { + const { project, branchId } = useUIState(); + const qc = useQueryClient(); + const [selected, setSelected] = useState(null); + const [showCreate, setShowCreate] = useState(false); + + const q = useQuery({ + queryKey: ["streams", project, branchId], + queryFn: () => + api.get(`/stream/${encodeURIComponent(project ?? "")}/list`, { + query: { branch: branchRef(branchId) }, + }), + enabled: !!project, + }); + + const deleteMu = useMutation({ + // `stream delete` is exposed as POST /delete (not HTTP DELETE) so the + // dry-run flag can ride in the body alongside the source id. + mutationFn: (sourceId: string) => + api.post(`/stream/${encodeURIComponent(project ?? "")}/delete`, { + source_id: sourceId, + branch_id: branchRef(branchId), + }), + onSuccess: () => { + qc.invalidateQueries({ queryKey: ["streams"] }); + setSelected(null); + }, + }); + + return ( +

+ bucket via auto-provisioned logs/metrics/traces sinks.`} + actions={ + + } + /> + {deleteMu.error ? ( + + ) : null} + {!project ? ( + + ) : q.isLoading ? ( + + ) : q.error ? ( + + ) : ( + s.source_id} + onRowClick={(s) => setSelected(s)} + emptyMessage="No Data Streams sources yet. Create one to get an OTLP ingest endpoint." + columns={[ + { + header: "Name", + cell: (s) => {s.name}, + }, + { + header: "Source ID", + cell: (s) => {s.source_id}, + }, + { + header: "Type", + cell: (s) => {s.type || "?"}, + }, + { + header: "Endpoint", + cell: (s) => + s.base_endpoint ? ( + + {s.base_endpoint} + + ) : ( + - + ), + }, + { + header: "", + align: "right", + cell: (s) => ( + + ), + }, + ]} + /> + )} + {showCreate && project ? ( + setShowCreate(false)} + onCreated={(sourceId) => { + setShowCreate(false); + qc.invalidateQueries({ queryKey: ["streams"] }); + const created = q.data?.sources.find((s) => s.source_id === sourceId); + // Open the detail drawer for the new source. If the list hasn't + // refetched yet we synthesize a minimal row -- the drawer fetches + // its own full detail by source_id regardless. + setSelected( + created ?? { + source_id: sourceId, + name: sourceId, + type: "otlp", + description: "", + base_endpoint: "", + }, + ); + }} + /> + ) : null} + {selected && project ? ( + setSelected(null)} + onDelete={(sourceId) => { + if (confirm(`Delete Data Stream source '${sourceId}'?`)) { + deleteMu.mutate(sourceId); + } + }} + /> + ) : null} +
+ ); +} + +function CreateSourceDrawer({ + project, + branchId, + onClose, + onCreated, +}: { + project: string; + branchId: number | null; + onClose: () => void; + onCreated: (sourceId: string) => void; +}) { + const [name, setName] = useState(""); + const [sourceType, setSourceType] = useState<"otlp" | "http">("otlp"); + const [provisionSinks, setProvisionSinks] = useState(true); + const [ifNotExists, setIfNotExists] = useState(false); + const [error, setError] = useState(null); + + const createMu = useMutation({ + mutationFn: () => + api.post(`/stream/${encodeURIComponent(project)}/create-source`, { + name, + source_type: sourceType, + branch_id: branchRef(branchId), + provision_sinks: provisionSinks, + if_not_exists: ifNotExists, + }), + onSuccess: (detail) => onCreated(detail.source_id), + onError: (err) => setError((err as Error).message), + }); + + return ( + { + setError(null); + createMu.mutate(); + }} + > + + {createMu.isPending ? "creating..." : "Create"} + + } + > +
+ + +
+ Type +
+ {(["otlp", "http"] as const).map((t) => ( + + ))} +
+ + {sourceType === "otlp" + ? "OpenTelemetry Protocol -- logs, metrics, and traces over http/protobuf." + : "Generic HTTP ingest source."} + +
+ + {sourceType === "otlp" ? ( + + ) : null} + + + + {error ? : null} +
+ + ); +} + +function SourceDetailDrawer({ + project, + branchId, + source, + onClose, + onDelete, +}: { + project: string; + branchId: number | null; + source: DataStreamSource; + onClose: () => void; + onDelete: (sourceId: string) => void; +}) { + const [reveal, setReveal] = useState(false); + const [tab, setTab] = useState<"overview" | "raw">("overview"); + + const q = useQuery({ + queryKey: ["stream-detail", project, source.source_id, branchId, reveal], + queryFn: () => + api.get(`/stream/${encodeURIComponent(project)}/detail`, { + query: { + source_id: source.source_id, + branch: branchRef(branchId), + reveal, + }, + }), + }); + + const detail = q.data; + const signals = detail ? Object.entries(detail.signal_endpoints ?? {}) : []; + const tables = detail ? Object.entries(detail.destination?.tables ?? {}) : []; + + return ( + + + + + } + > + {q.isLoading ? : null} + {q.error ? : null} + {detail ? ( + <> +
+ + +
+ + {tab === "overview" ? ( +
+
+

Source

+
+ + + + + + +
+
+ +
+
+

Ingest endpoint

+ {detail.secret_revealed ? ( + secret revealed + ) : ( + secret masked + )} +
+ + {!detail.secret_revealed ? ( +
+ The OTLP URL embeds a write secret (masked as ***). + Use "Reveal secret" to copy the full URL into your OTLP + exporter -- treat it like a credential. +
+ ) : null} + {signals.length > 0 ? ( +
+
+ Per-signal endpoints +
+ {signals.map(([signal, url]) => ( +
+ {signal} + +
+ ))} +
+ ) : null} +
+ +
+

Destination

+ {detail.destination?.bucket || + detail.destination?.buckets?.length || + tables.length > 0 ? ( +
+ {detail.destination?.bucket ? ( + + ) : detail.destination?.buckets?.length ? ( + + ) : null} + {tables.map(([signal, tableId]) => ( +
+ {signal} + {tableId} +
+ ))} +
+ ) : ( +
+ No sinks yet -- this source has no destination tables. For + OTLP, create it with sink provisioning enabled. +
+ )} +
+ + {detail.import_conditions ? ( +
+

Import conditions

+ +
+ ) : null} +
+ ) : ( + + )} + + ) : null} + + ); +} + +/** Endpoint URL with an inline copy button. Long URLs wrap (break-all). */ +function EndpointRow({ url }: { url: string }) { + const [copied, setCopied] = useState(false); + const copy = async () => { + try { + await navigator.clipboard.writeText(url); + setCopied(true); + window.setTimeout(() => setCopied(false), 1200); + } catch { + /* Clipboard API blocked by browser permissions -- silent. */ + } + }; + if (!url) { + return -; + } + return ( +
+ {url} + +
+ ); +} + +function KV({ label, value, mono = false }: { label: string; value: string; mono?: boolean }) { + return ( +
+
{label}
+
+ {value || "—"} +
+
+ ); +} diff --git a/web/frontend/src/state.tsx b/web/frontend/src/state.tsx index e8e24321..16aa212b 100644 --- a/web/frontend/src/state.tsx +++ b/web/frontend/src/state.tsx @@ -10,6 +10,7 @@ export type PageId = | "projects" | "configs" | "storage" + | "stream" | "jobs" | "branches" | "workspaces" diff --git a/web/frontend/src/types.ts b/web/frontend/src/types.ts index 9410e32d..b6021dc1 100644 --- a/web/frontend/src/types.ts +++ b/web/frontend/src/types.ts @@ -177,3 +177,46 @@ export interface Component { component_type: string; description?: string; } + +/** + * Data Streams (OTLP) source -- list view. The secret embedded in the OTLP + * ingest URL is never part of the list payload; only the secret-free + * `base_endpoint` is surfaced. Mirrors `StreamService._summarise_sources`. + */ +export interface DataStreamSource { + source_id: string; + name: string; + type: string; + description: string; + base_endpoint: string; +} + +/** + * Full `stream detail` picture for one source. `endpoint` / `signal_endpoints` + * are masked (the secret replaced with `***`) unless the request opted in with + * `reveal=true`, in which case `secret_revealed` is true. Mirrors + * `StreamService._assemble_detail`. + */ +export interface DataStreamDetail { + alias?: string; + status?: string; // "created" | "skipped" on create-source + branch_id: string; + source_id: string; + name: string; + type: string; + description: string; + endpoint: string; + base_endpoint: string; + signal_endpoints: Record; + protocol: string; + secret_revealed: boolean; + destination: { + bucket: string; + buckets: string[]; + tables: Record; + }; + import_conditions: Record | null; + // Raw passthrough -- surfaced only via the detail drawer's "Raw JSON" tab. + sinks: Array>; + source: Record; +}