From c7bc1c2f73fe1cd79d6a8f5a57d9596197a3bfa5 Mon Sep 17 00:00:00 2001 From: hudsonaikins-crown Date: Fri, 29 May 2026 13:43:53 -0400 Subject: [PATCH] Add cost-aware agent skill standards --- .github/workflows/ci.yml | 66 ++++++ .woodpecker.yml | 93 --------- AGENTS.md | 1 - README.md | 8 +- docs/DOCS_INDEX.md | 2 + docs/cost-intelligence-system-design.md | 75 +++++++ docs/cost-model-standards.md | 56 +++++ docs/deployment/WOODPECKER_HOSTINGER_SETUP.md | 17 +- internal/config/parser_test.go | 71 +++++++ internal/config/validator.go | 10 + pkg/types/cost.go | 21 ++ pkg/types/structures.go | 13 +- provider_catalog/ai_saas_defaults.yml | 85 ++++++++ schemas/cost-source.schema.json | 36 ++++ scripts/judge_cost_standards.go | 177 ++++++++++++++++ skills/profitctl-cost-aware/SKILL.md | 74 +++++++ .../profitctl-cost-aware/agents/openai.yaml | 7 + .../templates/cloud-run-ai-saas.yml | 176 ++++++++++++++++ .../templates/cloudflare-workers-ai-saas.yml | 165 +++++++++++++++ .../references/templates/vercel-ai-saas.yml | 176 ++++++++++++++++ .../scripts/run_profitctl_scenarios.py | 192 ++++++++++++++++++ 21 files changed, 1418 insertions(+), 103 deletions(-) create mode 100644 docs/cost-intelligence-system-design.md create mode 100644 docs/cost-model-standards.md create mode 100644 provider_catalog/ai_saas_defaults.yml create mode 100644 schemas/cost-source.schema.json create mode 100644 scripts/judge_cost_standards.go create mode 100644 skills/profitctl-cost-aware/SKILL.md create mode 100644 skills/profitctl-cost-aware/agents/openai.yaml create mode 100644 skills/profitctl-cost-aware/references/templates/cloud-run-ai-saas.yml create mode 100644 skills/profitctl-cost-aware/references/templates/cloudflare-workers-ai-saas.yml create mode 100644 skills/profitctl-cost-aware/references/templates/vercel-ai-saas.yml create mode 100755 skills/profitctl-cost-aware/scripts/run_profitctl_scenarios.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 73b41cb..65c896b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -78,3 +78,69 @@ jobs: bash scripts/install.sh "${INSTALL_DIR}/profitctl" --help >/dev/null "${INSTALL_DIR}/profitctl" validate -f examples/mix_profit.yml >/dev/null + + security-scan: + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - name: Check out code + uses: actions/checkout@v5 + + - name: Run scanners + run: | + set -euo pipefail + mkdir -p reports + + docker run --rm --entrypoint semgrep -v "$PWD:/src" -w /src semgrep/semgrep:1.154.0 \ + scan --config p/default --sarif --sarif-output reports/semgrep.sarif . \ + || echo "semgrep exited non-zero; preserving report if present" + [ -f reports/semgrep.sarif ] || echo '{"version":"2.1.0","runs":[]}' > reports/semgrep.sarif + + docker run --rm --entrypoint trivy -v "$PWD:/src" -w /src aquasec/trivy:0.69.3 \ + fs --scanners vuln,misconfig,secret --format sarif -o reports/trivy.sarif . \ + || echo "trivy exited non-zero; preserving report if present" + [ -f reports/trivy.sarif ] || echo '{"version":"2.1.0","runs":[]}' > reports/trivy.sarif + + docker run --rm --entrypoint gitleaks -v "$PWD:/src" -w /src ghcr.io/gitleaks/gitleaks:v8.30.0 \ + git --report-format sarif --report-path reports/gitleaks.sarif . \ + || echo "gitleaks exited non-zero; preserving report if present" + [ -f reports/gitleaks.sarif ] || echo '{"version":"2.1.0","runs":[]}' > reports/gitleaks.sarif + + docker run --rm --entrypoint osv-scanner -v "$PWD:/src" -w /src ghcr.io/google/osv-scanner:v2.3.3 \ + scan --recursive --format json --output reports/osv.json . \ + || echo "osv-scanner exited non-zero; preserving report if present" + [ -f reports/osv.json ] || echo '{"results":[]}' > reports/osv.json + + - name: Upload scan findings + env: + APPSEC_API_URL: ${{ secrets.APPSEC_API_URL }} + APPSEC_API_TOKEN: ${{ secrets.APPSEC_API_TOKEN }} + APPSEC_STRICT_API: ${{ vars.APPSEC_STRICT_API || 'false' }} + CI_PIPELINE_ID: ${{ github.run_id }}-${{ github.run_attempt }} + PR_NUMBER: ${{ github.event.pull_request.number || '' }} + run: | + set -euo pipefail + upload() { + tool="$1" + format="$2" + report="$3" + if [ -n "${PR_NUMBER}" ]; then + sh scripts/ci/upload_scan.sh "$APPSEC_API_URL" "$GITHUB_REPOSITORY" "$GITHUB_SHA" "$tool" "$format" "$PR_NUMBER" "$report" + else + sh scripts/ci/upload_scan.sh "$APPSEC_API_URL" "$GITHUB_REPOSITORY" "$GITHUB_SHA" "$tool" "$format" "$report" + fi + } + + upload semgrep sarif reports/semgrep.sarif + upload trivy sarif reports/trivy.sarif + upload gitleaks sarif reports/gitleaks.sarif + upload osv json reports/osv.json + + - name: Evaluate policy gate + if: github.event_name == 'pull_request' + env: + APPSEC_API_URL: ${{ secrets.APPSEC_API_URL }} + APPSEC_API_TOKEN: ${{ secrets.APPSEC_API_TOKEN }} + APPSEC_STRICT_API: ${{ vars.APPSEC_STRICT_API || 'false' }} + run: | + sh scripts/ci/evaluate_gate.sh "$APPSEC_API_URL" "$GITHUB_REPOSITORY" "${{ github.event.pull_request.number }}" diff --git a/.woodpecker.yml b/.woodpecker.yml index 6cae3b0..b5d1b1c 100644 --- a/.woodpecker.yml +++ b/.woodpecker.yml @@ -58,96 +58,3 @@ steps: bash scripts/release/install-tool.sh cosign "$${TOOL_DIR}" export PATH="$${TOOL_DIR}:$${PATH}" doppler run --project profitctl --config prd_ci_woodpecker -- bash scripts/release/smoke-published-release.sh "$${TAG}" - ---- -labels: - pool: shared-kvm - -when: - - event: pull_request - branch: [main] - -steps: - - name: verify-go-quality - image: golang:1.24 - commands: - - go version - - test -z "$(gofmt -l .)" - - go mod tidy - - git diff --exit-code go.mod go.sum - - go vet ./... - - go test ./... - - - name: semgrep - image: semgrep/semgrep:1.154.0 - commands: - - mkdir -p reports - - semgrep scan --config p/default --sarif --sarif-output reports/semgrep.sarif . || true - - '[ -f reports/semgrep.sarif ] || echo ''{"version":"2.1.0","runs":[]}'' > reports/semgrep.sarif' - - - name: trivy-fs - image: aquasec/trivy:0.69.3 - commands: - - mkdir -p reports - - trivy fs --scanners vuln,misconfig,secret --format sarif -o reports/trivy.sarif . || echo '{"version":"2.1.0","runs":[]}' > reports/trivy.sarif - - - name: gitleaks - image: ghcr.io/gitleaks/gitleaks:v8.30.0 - commands: - - mkdir -p reports - - gitleaks git --report-format sarif --report-path reports/gitleaks.sarif . || true - - '[ -f reports/gitleaks.sarif ] || echo ''{"version":"2.1.0","runs":[]}'' > reports/gitleaks.sarif' - - - name: osv - image: ghcr.io/google/osv-scanner:v2.3.3 - commands: - - mkdir -p reports - - osv-scanner scan --recursive --format json --output reports/osv.json . || true - - '[ -f reports/osv.json ] || echo ''{"results":[]}'' > reports/osv.json' - - - name: upload-findings - image: curlimages/curl:8.11.0 - environment: - APPSEC_API_URL: - from_secret: APPSEC_API_URL - APPSEC_API_TOKEN: - from_secret: APPSEC_API_TOKEN - APPSEC_STRICT_API: - from_secret: APPSEC_STRICT_API - commands: - - sh scripts/ci/upload_scan.sh "$APPSEC_API_URL" "$CI_REPO" "$CI_COMMIT_SHA" semgrep sarif "$CI_COMMIT_PULL_REQUEST" reports/semgrep.sarif - - sh scripts/ci/upload_scan.sh "$APPSEC_API_URL" "$CI_REPO" "$CI_COMMIT_SHA" trivy sarif "$CI_COMMIT_PULL_REQUEST" reports/trivy.sarif - - sh scripts/ci/upload_scan.sh "$APPSEC_API_URL" "$CI_REPO" "$CI_COMMIT_SHA" gitleaks sarif "$CI_COMMIT_PULL_REQUEST" reports/gitleaks.sarif - - sh scripts/ci/upload_scan.sh "$APPSEC_API_URL" "$CI_REPO" "$CI_COMMIT_SHA" osv json "$CI_COMMIT_PULL_REQUEST" reports/osv.json - - - name: evaluate-gate - image: alpine:3.21 - environment: - APPSEC_API_URL: - from_secret: APPSEC_API_URL - APPSEC_API_TOKEN: - from_secret: APPSEC_API_TOKEN - APPSEC_STRICT_API: - from_secret: APPSEC_STRICT_API - commands: - - apk add --no-cache curl jq - - sh scripts/ci/evaluate_gate.sh "$APPSEC_API_URL" "$CI_REPO" "$CI_COMMIT_PULL_REQUEST" - ---- -labels: - pool: shared-kvm - -when: - - event: push - branch: [main] - -steps: - - name: verify-main - image: golang:1.24 - commands: - - go version - - test -z "$(gofmt -l .)" - - go mod tidy - - git diff --exit-code go.mod go.sum - - go vet ./... - - go test ./... diff --git a/AGENTS.md b/AGENTS.md index e88e857..0cc9b86 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -2,7 +2,6 @@ ## Tooling Policy -- Greptile is deprecated for this repository. - Use Codex plus native GitHub checks and review comments for local review loops. - Do not use paid review bots or usage-based review add-ons unless the user explicitly approves the spend. - Use native GitHub checks and review comments with `$check-pr` for PR readiness. diff --git a/README.md b/README.md index 8c60453..a7289f0 100644 --- a/README.md +++ b/README.md @@ -148,18 +148,22 @@ See [SECURITY.md](SECURITY.md). `ProfitCtl` can publish PR security findings into the separate `appsec-mvp` service. -- PR scans run in Woodpecker via `semgrep`, `trivy`, `gitleaks`, and `osv-scanner` +- PR and main scans run in GitHub Actions via `semgrep`, `trivy`, `gitleaks`, and `osv-scanner` - findings are uploaded to `appsec-mvp` - GitHub PRs receive: - commit status `appsec/mvp` - check run `AppSec MVP Review` - selective inline comments for high-confidence findings -Required Woodpecker secret: +Required GitHub repository secrets: - `APPSEC_API_URL` - `APPSEC_API_TOKEN` +Optional GitHub repository variable: + +- `APPSEC_STRICT_API` + Required `appsec-mvp` runtime config: - `GITHUB_APP_ID` diff --git a/docs/DOCS_INDEX.md b/docs/DOCS_INDEX.md index 6d210fd..4159863 100644 --- a/docs/DOCS_INDEX.md +++ b/docs/DOCS_INDEX.md @@ -6,6 +6,8 @@ - [Quick Start](QUICK_START.md) - [How It Works](HOW_IT_WORKS.md) - [Architecture](ARCHITECTURE.md) +- [Cost Intelligence System Design](cost-intelligence-system-design.md) +- [Cost Model Standards](cost-model-standards.md) - [Open-Core Packaging](OPEN_CORE_PACKAGING.md) - [Open-Core Roadmap](OPEN_CORE_ROADMAP.md) - [Full Economics Cost Layers](full-economics-cost-layers.md) diff --git a/docs/cost-intelligence-system-design.md b/docs/cost-intelligence-system-design.md new file mode 100644 index 0000000..68b23c6 --- /dev/null +++ b/docs/cost-intelligence-system-design.md @@ -0,0 +1,75 @@ +# Cost Intelligence System Design + +## Objective + +ProfitCtl should be the deterministic economics engine for cost-aware development decisions. Agents may gather context and propose scenarios, but ProfitCtl owns the schema, provenance contract, validation rules, and repeatable calculations. + +## Ownership Split + +| Owner | Responsibility | +| --- | --- | +| Product / ProfitCtl | Cost schema, formulas, scenario validation, provenance labels, provider catalog format, deterministic output, standards judge | +| User | Business assumptions: users, paid seats, pricing, target margin, risk tolerance, expected usage | +| Agent | Repo inspection, provider/service detection, template selection, temp scenario generation, cost-aware explanation | + +Agents must not blur facts and assumptions. Every cost or usage line should identify whether it came from a template, user input, repo detection, telemetry, invoice, or provider catalog. + +## Data Flow + +1. Agent inspects repository context: framework, hosting, database, auth, paid APIs, billing, background work, and telemetry. +2. ProfitCtl or an agent selects the closest scenario template. +3. User supplies or confirms business assumptions that materially change the answer. +4. Provider catalog entries fill planning defaults when no actuals exist. +5. Telemetry and invoices override defaults when connected. +6. ProfitCtl validates and simulates the scenario. +7. The agent returns a recommendation with assumptions, margins, p95 stress, cost per user, covenant status, and confidence. +8. The final scenario and output become a decision artifact that can later be compared against real actuals. + +## Internet Access + +V1 does not require internet access. Scenario templates and user inputs are enough for local economics checks. + +Productized catalog refresh should use controlled sources: + +- official provider pricing pages or APIs +- authenticated billing exports +- product telemetry +- runtime cost ledgers + +Do not make random web scraping part of the trust path. Scraped values are brittle, hard to cite, and hard to keep current. If web lookup is used, cache the result with source URL, capture date, and confidence. + +## System Changes + +### ProfitCtl + +- Add cost-source provenance fields to fixed and variable costs. +- Add a provider-catalog seed format for planning defaults. +- Add standards judge tooling that validates provenance and scenario quality. +- Keep core simulation math deterministic and local. + +Cost source schema lives at `schemas/cost-source.schema.json`. Seed catalog values live under `provider_catalog/` and must be treated as planning defaults until replaced by sourced provider data, telemetry, or invoices. + +### AgentOS / Condere + +- Continue using the run ledger as the source for agent/model/search costs. +- Normalize runtime cost observations into project-scoped economics signals when integrating later. +- Treat provider-specific pricing estimates as calibration inputs, not hidden model behavior. + +### Web App + +- Capture project-scoped business assumptions: users, billable seats, runs per user, tokens per run, search calls per run, pricing plan, and target margin. +- Expose those assumptions to agents and ProfitCtl as scenario inputs. +- Avoid dashboard work until scenario quality and decision usefulness are proven. + +## Confidence Model + +| Source type | Typical confidence | Meaning | +| --- | --- | --- | +| `template` | low to medium | Default planning assumption, useful for first-pass comparison | +| `repo_detected` | low to medium | Inferred from code/config; needs user or telemetry confirmation | +| `user_supplied` | medium to high | Business assumption from operator or customer | +| `provider_catalog` | medium | Provider default with source and capture date | +| `telemetry` | high | Measured usage from runtime/product systems | +| `invoice` | high | Actual billed cost or revenue export | + +High confidence should be reserved for actuals or explicit user/business inputs, not generic templates. diff --git a/docs/cost-model-standards.md b/docs/cost-model-standards.md new file mode 100644 index 0000000..688abe5 --- /dev/null +++ b/docs/cost-model-standards.md @@ -0,0 +1,56 @@ +# Cost Model Standards + +## Purpose + +These standards are the judge criteria for cost-aware agent work. A scenario can be useful while still being approximate, but the output must make assumptions, provenance, and confidence visible. + +## Required Scenario Standards + +- Every fixed and variable cost should include `source.type` and `source.confidence`. +- `source.type` must be one of `template`, `user_supplied`, `repo_detected`, `telemetry`, `invoice`, or `provider_catalog`. +- `source.confidence` must be one of `low`, `medium`, or `high`. +- Template assumptions must not use `high` confidence. +- `provider_catalog` assumptions should include `url` or `note`. +- `telemetry` and `invoice` assumptions should include `captured_at` or `note`. +- Productization and adoption costs should be modeled separately from delivery costs using `economics_layer`. +- Non-delivery costs should include an `allocation` rule when the allocation is not obvious. +- Scenarios should define covenants for margin and cost per user. +- Agent recommendations should report assumptions, margin, p95 margin, cost per user, and covenant status. + +## Accuracy Judge + +Run the local judge from the repository root: + +```bash +go run scripts/judge_cost_standards.go +go run scripts/judge_cost_standards.go path/to/scenario.yml +go run scripts/judge_cost_standards.go path/to/scenario-directory +``` + +The standards judge checks scenario quality, not provider-price truth. It should fail when: + +- the scenario does not parse or validate +- cost line provenance is missing +- source type or confidence is invalid +- a template claims high confidence +- non-delivery cost lacks allocation +- margin or cost-per-user covenant is missing + +The judge should warn when: + +- all costs are template-derived +- no actual telemetry or invoice inputs are present +- provider catalog entries lack source URLs + +## Calibration Path + +Accuracy improves as sources move from estimates to actuals: + +1. `template`: first-pass planning +2. `repo_detected`: architecture-aware planning +3. `user_supplied`: business-context planning +4. `provider_catalog`: sourced planning defaults +5. `telemetry`: measured usage +6. `invoice`: actual spend/revenue + +The product goal is not perfect prediction. It is disciplined decision quality: explain assumptions, test stress cases, and update scenarios when actuals arrive. diff --git a/docs/deployment/WOODPECKER_HOSTINGER_SETUP.md b/docs/deployment/WOODPECKER_HOSTINGER_SETUP.md index b3e6fb7..71c0d8b 100644 --- a/docs/deployment/WOODPECKER_HOSTINGER_SETUP.md +++ b/docs/deployment/WOODPECKER_HOSTINGER_SETUP.md @@ -6,10 +6,14 @@ This runbook onboards `IntelIP/ProfitCtl` into your existing Woodpecker infrastr `/.woodpecker.yml` defines: -- PR workflow (`pool=shared-kvm`): checks for `main` plus AppSec scan upload/gate evaluation -- Main workflow (`pool=shared-kvm`): push checks for `main` - Tag release workflow (`pool=shared-kvm`): semver tag release + VPS publish +PR and main verification now run in GitHub Actions: + +- `verify-go` +- `verify-install-smoke` +- `security-scan` with AppSec upload and PR gate evaluation + ## Required Secrets (Doppler: `profitctl` / `prd_ci_woodpecker`) - `GITHUB_TOKEN_RELEASE` @@ -28,13 +32,14 @@ This runbook onboards `IntelIP/ProfitCtl` into your existing Woodpecker infrastr - Events: include `tag` - Image filters: leave empty (`[]`) so command steps can consume it - Repo trust: set `IntelIP/ProfitCtl` as trusted in Woodpecker so `from_secret` works in command steps + +## Required Secrets (GitHub Actions) + - Repo secret name: `APPSEC_API_TOKEN` - Secret value: bearer token used by `appsec-mvp` ingestion and summary endpoints -- Events: include `pull_request` - Repo secret name: `APPSEC_API_URL` -- Secret value: reachable base URL for the deployed `appsec-mvp` API, for example `http://172.17.0.1:18080` -- Events: include `pull_request` -- Optional repo secret name: `APPSEC_STRICT_API` +- Secret value: reachable base URL for the deployed `appsec-mvp` API +- Optional repo variable name: `APPSEC_STRICT_API` - Rollout default: leave unset or set to `false` so AppSec transport failures do not block PRs while `/.appsec.yml` is still `report_only` - Post-rollout: set to `true` when the service is stable and you want CI to fail closed on AppSec API errors diff --git a/internal/config/parser_test.go b/internal/config/parser_test.go index 6a4942c..663f03a 100644 --- a/internal/config/parser_test.go +++ b/internal/config/parser_test.go @@ -74,6 +74,77 @@ simulation: assert.Equal(t, 100, cfg.Simulation.BaseUsers) } +func TestParseConfig_CostSourceProvenance(t *testing.T) { + configYAML := ` +simulation: + base_users: 100 + growth_factor: 1.5 + iterations: 10000 + +fixed_costs: + - name: Server + amount: 500 + period: monthly + layer: infrastructure + source: + type: provider_catalog + confidence: medium + url: https://example.com/pricing + captured_at: 2026-05-29 + note: Catalog seed used for planning. + +variable_costs: + - name: API Calls + cost_per_unit: 0.0001 + units_per_user: 10000 + distribution: normal + mean: 10000 + stddev: 2000 + layer: application + source: + type: user_supplied + confidence: high + note: Provided by product owner. +` + + tmpFile := createTempConfigFile(t, configYAML) + cfg, err := ParseConfig(tmpFile) + + assert.NoError(t, err) + assert.Equal(t, types.CostSourceProviderCatalog, cfg.FixedCosts[0].Source.Type) + assert.Equal(t, types.CostSourceConfidenceMedium, cfg.FixedCosts[0].Source.Confidence) + assert.Equal(t, "https://example.com/pricing", cfg.FixedCosts[0].Source.URL) + assert.Equal(t, types.CostSourceUserSupplied, cfg.VariableCosts[0].Source.Type) + assert.Equal(t, types.CostSourceConfidenceHigh, cfg.VariableCosts[0].Source.Confidence) +} + +func TestParseConfig_InvalidCostSourceProvenance(t *testing.T) { + configYAML := ` +simulation: + base_users: 100 + growth_factor: 1.5 + iterations: 10000 + +fixed_costs: + - name: Server + amount: 500 + period: monthly + layer: infrastructure + source: + type: scraped_guess + confidence: certain + +variable_costs: [] +` + + tmpFile := createTempConfigFile(t, configYAML) + _, err := ParseConfig(tmpFile) + + assert.Error(t, err) + assert.Contains(t, err.Error(), "Source.Type") + assert.Contains(t, err.Error(), "Source.Confidence") +} + func TestParseConfig_MinimalConfig(t *testing.T) { configYAML := ` simulation: diff --git a/internal/config/validator.go b/internal/config/validator.go index 2f1d130..5fac51a 100644 --- a/internal/config/validator.go +++ b/internal/config/validator.go @@ -67,6 +67,11 @@ func validateNestedStructs(cfg *Config, v *validator.Validate) error { return fmt.Errorf("fixed cost %q allocation validation failed: %w", fixedCost.Name, err) } } + if fixedCost.Source != nil { + if err := v.Struct(fixedCost.Source); err != nil { + return fmt.Errorf("fixed cost %q source validation failed: %w", fixedCost.Name, err) + } + } } for _, variableCost := range cfg.VariableCosts { @@ -78,6 +83,11 @@ func validateNestedStructs(cfg *Config, v *validator.Validate) error { return fmt.Errorf("variable cost %q allocation validation failed: %w", variableCost.Name, err) } } + if variableCost.Source != nil { + if err := v.Struct(variableCost.Source); err != nil { + return fmt.Errorf("variable cost %q source validation failed: %w", variableCost.Name, err) + } + } } if cfg.Pricing != nil { diff --git a/pkg/types/cost.go b/pkg/types/cost.go index 03bbdbc..10021b1 100644 --- a/pkg/types/cost.go +++ b/pkg/types/cost.go @@ -27,6 +27,27 @@ func NormalizeEconomicsLayer(layer EconomicsLayer) EconomicsLayer { return layer } +// CostSourceType records where a scenario assumption came from. +type CostSourceType string + +const ( + CostSourceTemplate CostSourceType = "template" + CostSourceUserSupplied CostSourceType = "user_supplied" + CostSourceRepoDetected CostSourceType = "repo_detected" + CostSourceTelemetry CostSourceType = "telemetry" + CostSourceInvoice CostSourceType = "invoice" + CostSourceProviderCatalog CostSourceType = "provider_catalog" +) + +// CostSourceConfidence describes how much trust to place in a scenario assumption. +type CostSourceConfidence string + +const ( + CostSourceConfidenceLow CostSourceConfidence = "low" + CostSourceConfidenceMedium CostSourceConfidence = "medium" + CostSourceConfidenceHigh CostSourceConfidence = "high" +) + // CostPeriod represents the time period for fixed costs type CostPeriod string diff --git a/pkg/types/structures.go b/pkg/types/structures.go index 89fb122..907fd40 100644 --- a/pkg/types/structures.go +++ b/pkg/types/structures.go @@ -10,6 +10,7 @@ type FixedCost struct { Layer CostLayer `yaml:"layer" validate:"required,oneof=infrastructure application service"` EconomicsLayer EconomicsLayer `yaml:"economics_layer,omitempty" validate:"omitempty,oneof=delivery productization adoption"` Allocation *EconomicsAllocation `yaml:"allocation,omitempty"` + Source *CostSource `yaml:"source,omitempty"` } // VariableCost represents a per-user variable cost with statistical distribution @@ -29,7 +30,8 @@ type VariableCost struct { Max *float64 `yaml:"max,omitempty"` // Required for uniform Rate *float64 `yaml:"rate,omitempty"` // Required for exponential - Layer CostLayer `yaml:"layer" validate:"required,oneof=infrastructure application service"` + Layer CostLayer `yaml:"layer" validate:"required,oneof=infrastructure application service"` + Source *CostSource `yaml:"source,omitempty"` } // EconomicsAllocation defines how non-delivery costs should be allocated in future reporting layers. @@ -38,6 +40,15 @@ type EconomicsAllocation struct { Divisor int `yaml:"divisor,omitempty" validate:"omitempty,min=1"` } +// CostSource captures provenance for cost and usage assumptions. +type CostSource struct { + Type CostSourceType `yaml:"type" validate:"required,oneof=template user_supplied repo_detected telemetry invoice provider_catalog"` + Confidence CostSourceConfidence `yaml:"confidence" validate:"required,oneof=low medium high"` + URL string `yaml:"url,omitempty" validate:"omitempty,url"` + CapturedAt string `yaml:"captured_at,omitempty"` + Note string `yaml:"note,omitempty"` +} + // CostLayerBreakdown separates costs by infrastructure layer type CostLayerBreakdown struct { Infrastructure float64 `json:"infrastructure"` diff --git a/provider_catalog/ai_saas_defaults.yml b/provider_catalog/ai_saas_defaults.yml new file mode 100644 index 0000000..c75564d --- /dev/null +++ b/provider_catalog/ai_saas_defaults.yml @@ -0,0 +1,85 @@ +# Seed defaults for AI SaaS planning. +# These are scenario defaults, not official provider pricing truth. +catalog_version: 2026-05-29 +status: seed_defaults_not_pricing_truth +notes: + - Replace template entries with provider_catalog, telemetry, or invoice-backed values before high-confidence decisions. + - Use official provider docs or authenticated billing exports for production calibration. + +entries: + - id: cloudflare_workers_paid_baseline + provider: cloudflare + service: workers + unit: fixed_monthly_baseline + default_amount_usd: 5 + source: + type: template + confidence: medium + captured_at: 2026-05-29 + note: Planning seed used by bundled AI SaaS templates. + + - id: cloud_run_service_baseline + provider: google_cloud + service: cloud_run + unit: fixed_monthly_baseline + default_amount_usd: 70 + source: + type: template + confidence: medium + captured_at: 2026-05-29 + note: Planning seed for low-volume AgentOS-style service. + + - id: vercel_pro_team_baseline + provider: vercel + service: hosting + unit: fixed_monthly_baseline + default_amount_usd: 20 + source: + type: template + confidence: medium + captured_at: 2026-05-29 + note: Planning seed for Vercel-hosted SaaS app. + + - id: neon_managed_postgres_baseline + provider: neon + service: postgres + unit: fixed_monthly_baseline + default_amount_usd: 30 + source: + type: template + confidence: medium + captured_at: 2026-05-29 + note: Planning seed for managed Postgres baseline. + + - id: clerk_auth_baseline + provider: clerk + service: auth + unit: fixed_monthly_baseline + default_amount_usd: 25 + source: + type: template + confidence: medium + captured_at: 2026-05-29 + note: Planning seed for auth baseline. + + - id: llm_token_spend + provider: model_provider + service: llm + unit: token + default_amount_usd: 0.000006 + source: + type: template + confidence: low + captured_at: 2026-05-29 + note: Blended token-spend planning estimate; replace with model-specific catalog or telemetry. + + - id: search_research_call + provider: search_provider + service: research + unit: call + default_amount_usd: 0.004 + source: + type: template + confidence: low + captured_at: 2026-05-29 + note: Blended search/research planning estimate; replace with provider catalog or invoice actuals. diff --git a/schemas/cost-source.schema.json b/schemas/cost-source.schema.json new file mode 100644 index 0000000..bb7001a --- /dev/null +++ b/schemas/cost-source.schema.json @@ -0,0 +1,36 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/IntelIP/ProfitCtl/schemas/cost-source.schema.json", + "title": "ProfitCtl Cost Source", + "type": "object", + "additionalProperties": false, + "required": ["type", "confidence"], + "properties": { + "type": { + "type": "string", + "enum": [ + "template", + "user_supplied", + "repo_detected", + "telemetry", + "invoice", + "provider_catalog" + ] + }, + "confidence": { + "type": "string", + "enum": ["low", "medium", "high"] + }, + "url": { + "type": "string", + "format": "uri" + }, + "captured_at": { + "type": "string", + "description": "Capture date or timestamp for the source value." + }, + "note": { + "type": "string" + } + } +} diff --git a/scripts/judge_cost_standards.go b/scripts/judge_cost_standards.go new file mode 100644 index 0000000..0baf777 --- /dev/null +++ b/scripts/judge_cost_standards.go @@ -0,0 +1,177 @@ +package main + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/IntelIP/ProfitCtl/internal/config" + "github.com/IntelIP/ProfitCtl/pkg/types" +) + +type FileReport struct { + File string `json:"file"` + Passed bool `json:"passed"` + Issues []string `json:"issues,omitempty"` + Warnings []string `json:"warnings,omitempty"` +} + +type JudgeReport struct { + Passed bool `json:"passed"` + Files []FileReport `json:"files"` +} + +func main() { + paths := os.Args[1:] + if len(paths) == 0 { + paths = []string{"skills/profitctl-cost-aware/references/templates"} + } + + files, err := collectScenarioFiles(paths) + if err != nil { + fmt.Fprintf(os.Stderr, "collect scenario files: %v\n", err) + os.Exit(2) + } + if len(files) == 0 { + fmt.Fprintln(os.Stderr, "no scenario files found") + os.Exit(2) + } + + report := JudgeReport{Passed: true} + for _, file := range files { + fileReport := judgeFile(file) + if !fileReport.Passed { + report.Passed = false + } + report.Files = append(report.Files, fileReport) + } + + encoded, err := json.MarshalIndent(report, "", " ") + if err != nil { + fmt.Fprintf(os.Stderr, "marshal report: %v\n", err) + os.Exit(2) + } + fmt.Println(string(encoded)) + if !report.Passed { + os.Exit(1) + } +} + +func collectScenarioFiles(paths []string) ([]string, error) { + var files []string + for _, raw := range paths { + path := filepath.Clean(raw) + info, err := os.Stat(path) + if err != nil { + return nil, err + } + if !info.IsDir() { + if isYAML(path) { + files = append(files, path) + } + continue + } + err = filepath.WalkDir(path, func(candidate string, entry os.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + if entry.IsDir() { + return nil + } + if isYAML(candidate) { + files = append(files, candidate) + } + return nil + }) + if err != nil { + return nil, err + } + } + return files, nil +} + +func isYAML(path string) bool { + ext := strings.ToLower(filepath.Ext(path)) + return ext == ".yml" || ext == ".yaml" +} + +func judgeFile(file string) FileReport { + report := FileReport{File: file, Passed: true} + cfg, err := config.ParseConfig(file) + if err != nil { + report.Passed = false + report.Issues = append(report.Issues, fmt.Sprintf("scenario does not parse or validate: %v", err)) + return report + } + + sourceCounts := map[types.CostSourceType]int{} + for _, cost := range cfg.FixedCosts { + judgeSource(&report, "fixed cost", cost.Name, cost.EconomicsLayer, cost.Allocation, cost.Source) + if cost.Source != nil { + sourceCounts[cost.Source.Type]++ + } + } + for _, cost := range cfg.VariableCosts { + judgeSource(&report, "variable cost", cost.Name, cost.EconomicsLayer, cost.Allocation, cost.Source) + if cost.Source != nil { + sourceCounts[cost.Source.Type]++ + } + } + + if len(cfg.Covenants) == 0 { + report.Issues = append(report.Issues, "scenario has no covenants") + } + if !hasCovenant(cfg, "margin") && !hasCovenant(cfg, "p95_margin") { + report.Issues = append(report.Issues, "scenario lacks margin or p95_margin covenant") + } + if !hasCovenant(cfg, "cost_per_user") && !hasCovenant(cfg, "p95_cost_per_user") { + report.Issues = append(report.Issues, "scenario lacks cost_per_user or p95_cost_per_user covenant") + } + + totalSources := 0 + for _, count := range sourceCounts { + totalSources += count + } + if totalSources > 0 && sourceCounts[types.CostSourceTemplate] == totalSources { + report.Warnings = append(report.Warnings, "all cost inputs are template-derived; confidence should stay low/medium until calibrated") + } + if sourceCounts[types.CostSourceTelemetry] == 0 && sourceCounts[types.CostSourceInvoice] == 0 { + report.Warnings = append(report.Warnings, "no telemetry or invoice-backed cost inputs present") + } + + if len(report.Issues) > 0 { + report.Passed = false + } + return report +} + +func judgeSource(report *FileReport, kind, name string, economicsLayer types.EconomicsLayer, allocation *types.EconomicsAllocation, source *types.CostSource) { + label := fmt.Sprintf("%s %q", kind, name) + if source == nil { + report.Issues = append(report.Issues, label+" is missing source provenance") + return + } + if source.Type == types.CostSourceTemplate && source.Confidence == types.CostSourceConfidenceHigh { + report.Issues = append(report.Issues, label+" uses high confidence for template-derived input") + } + if source.Type == types.CostSourceProviderCatalog && source.URL == "" && source.Note == "" { + report.Issues = append(report.Issues, label+" provider_catalog source needs url or note") + } + if (source.Type == types.CostSourceTelemetry || source.Type == types.CostSourceInvoice) && source.CapturedAt == "" && source.Note == "" { + report.Issues = append(report.Issues, label+" telemetry/invoice source needs captured_at or note") + } + if types.NormalizeEconomicsLayer(economicsLayer) != types.EconomicsLayerDelivery && allocation == nil { + report.Issues = append(report.Issues, label+" non-delivery cost needs allocation") + } +} + +func hasCovenant(cfg *config.Config, field string) bool { + for _, covenant := range cfg.Covenants { + if covenant.Field == field { + return true + } + } + return false +} diff --git a/skills/profitctl-cost-aware/SKILL.md b/skills/profitctl-cost-aware/SKILL.md new file mode 100644 index 0000000..f072b35 --- /dev/null +++ b/skills/profitctl-cost-aware/SKILL.md @@ -0,0 +1,74 @@ +--- +name: profitctl-cost-aware +description: Use when making architecture, infrastructure, AI model/tool, pricing, launch, or product-development decisions where cost, unit economics, recurring margin, or cost-aware tradeoffs matter. Runs ProfitCtl scenarios before recommending providers, services, or implementation approaches. +--- + +# ProfitCtl Cost-Aware + +## Purpose + +Make development and system-design advice cost-aware. Use ProfitCtl to ground architecture choices in scenario economics instead of vague provider preference. + +## Trigger This Skill + +Use this skill when the user asks about: + +- architecture/provider choices such as Cloudflare, Vercel, Cloud Run, Modal, Neon, Supabase, or managed services +- AI model, search, tool-call, agent-run, or API-cost tradeoffs +- new product, pricing, launch, or rollout assumptions +- adding paid infrastructure, billing, auth, storage, observability, or integration vendors +- whether a feature, contract, workflow, or implementation is worth its cost + +## Workflow + +1. Inspect repo context first: stack, runtime, hosting, database, auth, billing, paid APIs, expected usage, and existing docs. +2. Pick the closest template from `references/templates/` and copy it to a temp working directory. Do not edit repo-tracked scenario files unless the user asks to save one. +3. Adjust temp assumptions only when they are inferable from context or explicitly supplied by the user. If missing assumptions materially change the conclusion, ask before sounding precise. +4. Preserve `source.type`, `source.confidence`, `captured_at`, and `note` on every cost line. Do not upgrade template confidence to `high` without telemetry, invoices, or explicit user confirmation. +5. Run `profitctl validate`, then `profitctl simulate --json` or `profitctl compare`. +6. Summarize recommendation with evidence: + - assumed users, growth, ARPU, and pricing shape + - monthly fixed cost and top variable cost drivers + - gross margin, p95 margin, cost per active user, and covenant failures + - cheaper viable alternative when one exists +7. State assumptions plainly. Treat template prices as editable estimates, not current provider pricing guarantees. + +## Helper Script + +Use the helper for quick scenario runs: + +```bash +python3 /Users/hudson/.codex/skills/profitctl-cost-aware/scripts/run_profitctl_scenarios.py \ + --template cloudflare-workers-ai-saas --template cloud-run-ai-saas --compare +``` + +The script locates `profitctl` on `PATH`, or falls back to `go run .` in `/Users/hudson/Documents/GitHub/IntelIP/ProfitCtl`. + +## Defaults + +The bundled v1 templates use these guardrails: + +- gross margin must be at least `60%` +- p95 margin must be at least `40%` +- cost per active user must stay at or below `$18` + +Do not use `profitctl detect` as a required step. It needs `OPENROUTER_API_KEY`, so it is optional only. + +Before treating a scenario as decision-grade, run the standards judge: + +```bash +go run /Users/hudson/Documents/GitHub/IntelIP/ProfitCtl/scripts/judge_cost_standards.go \ + /path/to/scenario.yml +``` + +## Output Pattern + +For cost-aware recommendations, lead with: + +```text +Recommendation: [choice] because [ProfitCtl evidence]. +Assumptions: [users/growth/ARPU/top usage assumptions]. +Economics: [revenue/cost/margin/p95/cost per user/covenants]. +Tradeoff: [what gets cheaper or riskier]. +Next step: [scenario to refine or check to run]. +``` diff --git a/skills/profitctl-cost-aware/agents/openai.yaml b/skills/profitctl-cost-aware/agents/openai.yaml new file mode 100644 index 0000000..3f76562 --- /dev/null +++ b/skills/profitctl-cost-aware/agents/openai.yaml @@ -0,0 +1,7 @@ +interface: + display_name: "ProfitCtl Cost-Aware" + short_description: "Cost-aware architecture and product economics" + default_prompt: "Use $profitctl-cost-aware to compare architecture choices with ProfitCtl economics." + +policy: + allow_implicit_invocation: true diff --git a/skills/profitctl-cost-aware/references/templates/cloud-run-ai-saas.yml b/skills/profitctl-cost-aware/references/templates/cloud-run-ai-saas.yml new file mode 100644 index 0000000..9effcf4 --- /dev/null +++ b/skills/profitctl-cost-aware/references/templates/cloud-run-ai-saas.yml @@ -0,0 +1,176 @@ +# ProfitCtl template: Cloud Run + Neon + Clerk + LLM/search +project: + name: "AI SaaS - Cloud Run" + +fixed_costs: + - name: "Cloud Run service baseline" + amount: 70 + period: monthly + layer: infrastructure + economics_layer: delivery + source: + type: template + confidence: medium + captured_at: 2026-05-29 + note: Seed planning assumption from ProfitCtl AI SaaS template. + + - name: "Artifact Registry and load balancer reserve" + amount: 25 + period: monthly + layer: infrastructure + economics_layer: delivery + source: + type: template + confidence: medium + captured_at: 2026-05-29 + note: Seed reserve for Cloud Run support infrastructure. + + - name: "Neon managed Postgres baseline" + amount: 30 + period: monthly + layer: infrastructure + economics_layer: delivery + source: + type: template + confidence: medium + captured_at: 2026-05-29 + note: Seed planning assumption from ProfitCtl AI SaaS template. + + - name: "Clerk auth baseline" + amount: 25 + period: monthly + layer: service + economics_layer: delivery + source: + type: template + confidence: medium + captured_at: 2026-05-29 + note: Seed planning assumption from ProfitCtl AI SaaS template. + + - name: "Product analytics and observability" + amount: 45 + period: monthly + layer: application + economics_layer: productization + allocation: + mode: fixed_monthly + source: + type: template + confidence: medium + captured_at: 2026-05-29 + note: Productization overhead seed; replace with actual tooling spend when available. + + - name: "Operator onboarding reserve" + amount: 250 + period: monthly + layer: application + economics_layer: adoption + allocation: + mode: fixed_monthly + source: + type: template + confidence: low + captured_at: 2026-05-29 + note: Adoption/support reserve seed; user should calibrate against onboarding reality. + +variable_costs: + - name: "LLM token spend" + cost_per_unit: 0.000006 + units_per_user: 1 + distribution: normal + mean: 220000 + stddev: 70000 + layer: service + economics_layer: delivery + source: + type: template + confidence: low + captured_at: 2026-05-29 + note: Blended token-spend seed; replace with model catalog or telemetry. + + - name: "Search and research API calls" + cost_per_unit: 0.004 + units_per_user: 1 + distribution: normal + mean: 40 + stddev: 15 + layer: service + economics_layer: delivery + source: + type: template + confidence: low + captured_at: 2026-05-29 + note: Blended search/research seed; replace with provider catalog or invoice data. + + - name: "Container CPU and memory execution" + cost_per_unit: 0.0025 + units_per_user: 1 + distribution: normal + mean: 75 + stddev: 25 + layer: infrastructure + economics_layer: delivery + source: + type: template + confidence: medium + captured_at: 2026-05-29 + note: Cloud Run execution seed for architecture comparison. + + - name: "Integration sync overhead" + cost_per_unit: 0.0004 + units_per_user: 1 + distribution: normal + mean: 120 + stddev: 40 + layer: application + economics_layer: delivery + source: + type: template + confidence: low + captured_at: 2026-05-29 + note: Integration volume seed; replace with runtime telemetry. + +pricing: + mode: mix + plans: + - name: "Free" + price: 0 + share: 0.45 + limits: + users: 10 + + - name: "Starter" + price: 49 + share: 0.35 + limits: + users: 150 + + - name: "Pro" + price: 119 + share: 0.20 + limits: + users: 1500 + +covenants: + - type: threshold + field: margin + operator: gte + value: 60 + message: "Gross margin must be at least 60%" + + - type: threshold + field: p95_margin + operator: gte + value: 40 + message: "p95 margin must be at least 40%" + + - type: threshold + field: cost_per_user + operator: lte + value: 18 + message: "Cost per active user must stay below $18" + +simulation: + base_users: 100 + growth_factor: 1.20 + iterations: 10000 diff --git a/skills/profitctl-cost-aware/references/templates/cloudflare-workers-ai-saas.yml b/skills/profitctl-cost-aware/references/templates/cloudflare-workers-ai-saas.yml new file mode 100644 index 0000000..4ef80dd --- /dev/null +++ b/skills/profitctl-cost-aware/references/templates/cloudflare-workers-ai-saas.yml @@ -0,0 +1,165 @@ +# ProfitCtl template: Cloudflare Workers + Neon + Clerk + LLM/search +project: + name: "AI SaaS - Cloudflare Workers" + +fixed_costs: + - name: "Cloudflare Workers paid baseline" + amount: 5 + period: monthly + layer: infrastructure + economics_layer: delivery + source: + type: template + confidence: medium + captured_at: 2026-05-29 + note: Seed planning assumption from ProfitCtl AI SaaS template. + + - name: "Neon managed Postgres baseline" + amount: 30 + period: monthly + layer: infrastructure + economics_layer: delivery + source: + type: template + confidence: medium + captured_at: 2026-05-29 + note: Seed planning assumption from ProfitCtl AI SaaS template. + + - name: "Clerk auth baseline" + amount: 25 + period: monthly + layer: service + economics_layer: delivery + source: + type: template + confidence: medium + captured_at: 2026-05-29 + note: Seed planning assumption from ProfitCtl AI SaaS template. + + - name: "Product analytics and observability" + amount: 30 + period: monthly + layer: application + economics_layer: productization + allocation: + mode: fixed_monthly + source: + type: template + confidence: medium + captured_at: 2026-05-29 + note: Productization overhead seed; replace with actual tooling spend when available. + + - name: "Operator onboarding reserve" + amount: 250 + period: monthly + layer: application + economics_layer: adoption + allocation: + mode: fixed_monthly + source: + type: template + confidence: low + captured_at: 2026-05-29 + note: Adoption/support reserve seed; user should calibrate against onboarding reality. + +variable_costs: + - name: "LLM token spend" + cost_per_unit: 0.000006 + units_per_user: 1 + distribution: normal + mean: 220000 + stddev: 70000 + layer: service + economics_layer: delivery + source: + type: template + confidence: low + captured_at: 2026-05-29 + note: Blended token-spend seed; replace with model catalog or telemetry. + + - name: "Search and research API calls" + cost_per_unit: 0.004 + units_per_user: 1 + distribution: normal + mean: 40 + stddev: 15 + layer: service + economics_layer: delivery + source: + type: template + confidence: low + captured_at: 2026-05-29 + note: Blended search/research seed; replace with provider catalog or invoice data. + + - name: "Worker execution and queue overhead" + cost_per_unit: 0.00008 + units_per_user: 1 + distribution: normal + mean: 900 + stddev: 250 + layer: infrastructure + economics_layer: delivery + source: + type: template + confidence: medium + captured_at: 2026-05-29 + note: Worker execution seed for architecture comparison. + + - name: "Integration sync overhead" + cost_per_unit: 0.0004 + units_per_user: 1 + distribution: normal + mean: 120 + stddev: 40 + layer: application + economics_layer: delivery + source: + type: template + confidence: low + captured_at: 2026-05-29 + note: Integration volume seed; replace with runtime telemetry. + +pricing: + mode: mix + plans: + - name: "Free" + price: 0 + share: 0.45 + limits: + users: 10 + + - name: "Starter" + price: 49 + share: 0.35 + limits: + users: 150 + + - name: "Pro" + price: 119 + share: 0.20 + limits: + users: 1500 + +covenants: + - type: threshold + field: margin + operator: gte + value: 60 + message: "Gross margin must be at least 60%" + + - type: threshold + field: p95_margin + operator: gte + value: 40 + message: "p95 margin must be at least 40%" + + - type: threshold + field: cost_per_user + operator: lte + value: 18 + message: "Cost per active user must stay below $18" + +simulation: + base_users: 100 + growth_factor: 1.20 + iterations: 10000 diff --git a/skills/profitctl-cost-aware/references/templates/vercel-ai-saas.yml b/skills/profitctl-cost-aware/references/templates/vercel-ai-saas.yml new file mode 100644 index 0000000..2c0a4b0 --- /dev/null +++ b/skills/profitctl-cost-aware/references/templates/vercel-ai-saas.yml @@ -0,0 +1,176 @@ +# ProfitCtl template: Vercel + Neon + Clerk + LLM/search +project: + name: "AI SaaS - Vercel" + +fixed_costs: + - name: "Vercel Pro team baseline" + amount: 20 + period: monthly + layer: infrastructure + economics_layer: delivery + source: + type: template + confidence: medium + captured_at: 2026-05-29 + note: Seed planning assumption from ProfitCtl AI SaaS template. + + - name: "Vercel usage reserve" + amount: 35 + period: monthly + layer: infrastructure + economics_layer: delivery + source: + type: template + confidence: medium + captured_at: 2026-05-29 + note: Seed reserve for serverless usage and bandwidth. + + - name: "Neon managed Postgres baseline" + amount: 30 + period: monthly + layer: infrastructure + economics_layer: delivery + source: + type: template + confidence: medium + captured_at: 2026-05-29 + note: Seed planning assumption from ProfitCtl AI SaaS template. + + - name: "Clerk auth baseline" + amount: 25 + period: monthly + layer: service + economics_layer: delivery + source: + type: template + confidence: medium + captured_at: 2026-05-29 + note: Seed planning assumption from ProfitCtl AI SaaS template. + + - name: "Product analytics and observability" + amount: 35 + period: monthly + layer: application + economics_layer: productization + allocation: + mode: fixed_monthly + source: + type: template + confidence: medium + captured_at: 2026-05-29 + note: Productization overhead seed; replace with actual tooling spend when available. + + - name: "Operator onboarding reserve" + amount: 250 + period: monthly + layer: application + economics_layer: adoption + allocation: + mode: fixed_monthly + source: + type: template + confidence: low + captured_at: 2026-05-29 + note: Adoption/support reserve seed; user should calibrate against onboarding reality. + +variable_costs: + - name: "LLM token spend" + cost_per_unit: 0.000006 + units_per_user: 1 + distribution: normal + mean: 220000 + stddev: 70000 + layer: service + economics_layer: delivery + source: + type: template + confidence: low + captured_at: 2026-05-29 + note: Blended token-spend seed; replace with model catalog or telemetry. + + - name: "Search and research API calls" + cost_per_unit: 0.004 + units_per_user: 1 + distribution: normal + mean: 40 + stddev: 15 + layer: service + economics_layer: delivery + source: + type: template + confidence: low + captured_at: 2026-05-29 + note: Blended search/research seed; replace with provider catalog or invoice data. + + - name: "Serverless function and bandwidth overhead" + cost_per_unit: 0.0007 + units_per_user: 1 + distribution: normal + mean: 300 + stddev: 100 + layer: infrastructure + economics_layer: delivery + source: + type: template + confidence: medium + captured_at: 2026-05-29 + note: Vercel function and bandwidth seed for architecture comparison. + + - name: "Integration sync overhead" + cost_per_unit: 0.0004 + units_per_user: 1 + distribution: normal + mean: 120 + stddev: 40 + layer: application + economics_layer: delivery + source: + type: template + confidence: low + captured_at: 2026-05-29 + note: Integration volume seed; replace with runtime telemetry. + +pricing: + mode: mix + plans: + - name: "Free" + price: 0 + share: 0.45 + limits: + users: 10 + + - name: "Starter" + price: 49 + share: 0.35 + limits: + users: 150 + + - name: "Pro" + price: 119 + share: 0.20 + limits: + users: 1500 + +covenants: + - type: threshold + field: margin + operator: gte + value: 60 + message: "Gross margin must be at least 60%" + + - type: threshold + field: p95_margin + operator: gte + value: 40 + message: "p95 margin must be at least 40%" + + - type: threshold + field: cost_per_user + operator: lte + value: 18 + message: "Cost per active user must stay below $18" + +simulation: + base_users: 100 + growth_factor: 1.20 + iterations: 10000 diff --git a/skills/profitctl-cost-aware/scripts/run_profitctl_scenarios.py b/skills/profitctl-cost-aware/scripts/run_profitctl_scenarios.py new file mode 100755 index 0000000..e453ace --- /dev/null +++ b/skills/profitctl-cost-aware/scripts/run_profitctl_scenarios.py @@ -0,0 +1,192 @@ +#!/usr/bin/env python3 +"""Run bundled ProfitCtl scenarios and return compact JSON for agents.""" + +from __future__ import annotations + +import argparse +import json +import os +import shutil +import subprocess +import sys +import tempfile +from pathlib import Path + + +DEFAULT_PROFITCTL_REPO = Path("/Users/hudson/Documents/GitHub/IntelIP/ProfitCtl") +SKILL_DIR = Path(__file__).resolve().parents[1] +TEMPLATE_DIR = SKILL_DIR / "references" / "templates" + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--template", + action="append", + dest="templates", + help="Template name without .yml, or path to a scenario YAML file. Repeatable.", + ) + parser.add_argument( + "--compare", + action="store_true", + help="Run profitctl compare when at least two scenarios are selected.", + ) + parser.add_argument( + "--profitctl-repo", + default=os.environ.get("PROFITCTL_REPO", str(DEFAULT_PROFITCTL_REPO)), + help="ProfitCtl repo path for go-run fallback.", + ) + return parser.parse_args() + + +def resolve_profitctl(repo: Path) -> tuple[list[str], Path | None]: + binary = shutil.which("profitctl") + if binary: + return [binary], None + if (repo / "go.mod").exists() and (repo / "main.go").exists(): + return ["go", "run", "."], repo + raise SystemExit( + json.dumps( + { + "status": "error", + "reason": "profitctl not found on PATH and ProfitCtl repo fallback is unavailable", + "profitctl_repo": str(repo), + }, + indent=2, + ) + ) + + +def resolve_template(raw: str) -> Path: + candidate = Path(raw).expanduser() + if candidate.exists(): + return candidate.resolve() + + for suffix in ("", ".yml", ".yaml"): + bundled = TEMPLATE_DIR / f"{raw}{suffix}" + if bundled.exists(): + return bundled.resolve() + + available = sorted(path.stem for path in TEMPLATE_DIR.glob("*.yml")) + raise SystemExit( + json.dumps( + { + "status": "error", + "reason": f"template not found: {raw}", + "available_templates": available, + }, + indent=2, + ) + ) + + +def run_command(command: list[str], cwd: Path | None) -> subprocess.CompletedProcess[str]: + return subprocess.run( + command, + cwd=str(cwd) if cwd else None, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + + +def compact_simulation(data: dict[str, object]) -> dict[str, object]: + scenario = data.get("scenario", {}) or {} + costs = data.get("costs", {}) or {} + fixed = costs.get("fixed", {}) if isinstance(costs, dict) else {} + margin = data.get("margin", {}) or {} + stress = data.get("stress_test", {}) or {} + p95 = stress.get("p95", {}) if isinstance(stress, dict) else {} + revenue = data.get("revenue", {}) or {} + covenants = data.get("covenants", {}) or {} + + return { + "users": scenario.get("users"), + "growth_factor": scenario.get("growth_factor"), + "revenue": revenue.get("total"), + "fixed_monthly_cost": fixed.get("monthly"), + "total_cost": costs.get("total") if isinstance(costs, dict) else None, + "gross_margin": margin.get("gross") if isinstance(margin, dict) else None, + "p95_margin": p95.get("margin") if isinstance(p95, dict) else None, + "cost_per_user": margin.get("cost_per_user") if isinstance(margin, dict) else None, + "p95_cost_per_user": p95.get("cost_per_user") if isinstance(p95, dict) else None, + "covenants_passed": covenants.get("passed") if isinstance(covenants, dict) else None, + "violations": covenants.get("violations", []) if isinstance(covenants, dict) else [], + } + + +def scenario_name(path: Path) -> str: + return path.stem + + +def main() -> int: + args = parse_args() + selected = args.templates or [ + "cloudflare-workers-ai-saas", + "cloud-run-ai-saas", + "vercel-ai-saas", + ] + repo = Path(args.profitctl_repo).expanduser().resolve() + profitctl_cmd, cwd = resolve_profitctl(repo) + + output: dict[str, object] = { + "status": "ok", + "profitctl_command": " ".join(profitctl_cmd), + "scenarios": [], + } + + with tempfile.TemporaryDirectory(prefix="profitctl-skill-") as tmp_raw: + tmp = Path(tmp_raw) + scenario_paths: list[Path] = [] + + for raw in selected: + source = resolve_template(raw) + destination = tmp / source.name + shutil.copy2(source, destination) + scenario_paths.append(destination) + + validate = run_command(profitctl_cmd + ["validate", "-f", str(destination)], cwd) + simulate = run_command(profitctl_cmd + ["simulate", "-f", str(destination), "--json"], cwd) + + scenario_result: dict[str, object] = { + "name": scenario_name(destination), + "source_template": str(source), + "validate_exit_code": validate.returncode, + "simulate_exit_code": simulate.returncode, + "validate_stdout": validate.stdout.strip(), + } + + if simulate.stdout.strip(): + try: + parsed = json.loads(simulate.stdout) + scenario_result["summary"] = compact_simulation(parsed) + except json.JSONDecodeError: + scenario_result["simulate_stdout"] = simulate.stdout.strip() + + if validate.stderr.strip(): + scenario_result["validate_stderr"] = validate.stderr.strip() + if simulate.stderr.strip(): + scenario_result["simulate_stderr"] = simulate.stderr.strip() + + cast_scenarios = output["scenarios"] + assert isinstance(cast_scenarios, list) + cast_scenarios.append(scenario_result) + + if args.compare and len(scenario_paths) >= 2: + compare = run_command( + profitctl_cmd + ["compare", *[str(path) for path in scenario_paths]], + cwd, + ) + output["compare"] = { + "exit_code": compare.returncode, + "stdout": compare.stdout.strip(), + "stderr": compare.stderr.strip(), + } + + print(json.dumps(output, indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + sys.exit(main())